1use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::sync::atomic::{AtomicBool, Ordering};
52use std::sync::{Mutex, MutexGuard};
53use std::time::Duration;
54
55use anyhow::{Context, Result, bail};
56use jiff::Timestamp;
57use serde::{Deserialize, Serialize};
58use tokio::sync::Notify;
59
60use crate::ask::{self, Questions};
61use crate::clean;
62use crate::conduct::Conductor;
63use crate::config::{Config, MergeMode};
64use crate::graph::Runner;
65use crate::land;
66use crate::notices::{self, Link, Notice};
67use crate::queue::{Queue, Task, TaskStatus};
68use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
69use crate::triage;
70
71pub const SCHEMA: u32 = 1;
73
74pub const HEARTBEAT: Duration = Duration::from_secs(5);
78
79pub const STALE_SECS: i64 = 30;
88
89pub const POLL: Duration = Duration::from_secs(5);
91
92pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
96
97pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
119#[serde(default)]
120pub struct Current {
121 pub task: String,
123 pub run: String,
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct Status {
135 pub schema: u32,
137 pub pid: u32,
139 pub started_at: Timestamp,
141 pub updated_at: Timestamp,
143 pub idle: bool,
145 pub current: Vec<Current>,
151 pub completed: usize,
153 pub polls: u64,
155}
156
157impl Status {
158 #[must_use]
160 pub fn new() -> Self {
161 let now = Timestamp::now();
162 Self {
163 schema: SCHEMA,
164 pid: std::process::id(),
165 started_at: now,
166 updated_at: now,
167 idle: true,
168 current: Vec::new(),
169 completed: 0,
170 polls: 0,
171 }
172 }
173}
174
175impl Default for Status {
176 fn default() -> Self {
177 Self::new()
178 }
179}
180
181#[derive(Debug, Clone)]
183pub struct Opts {
184 pub repo: PathBuf,
186 pub config: Option<PathBuf>,
188 pub poll: Duration,
190 pub max_attempts: usize,
192 pub once: bool,
194 pub merge: Option<String>,
196 pub worktrees_root: Option<PathBuf>,
205}
206
207impl Default for Opts {
208 fn default() -> Self {
209 Self {
210 repo: PathBuf::from("."),
211 config: None,
212 poll: POLL,
213 max_attempts: 2,
214 once: false,
215 merge: None,
216 worktrees_root: None,
217 }
218 }
219}
220
221fn max_concurrent(n: usize) -> usize {
226 n.max(1)
227}
228
229#[must_use]
231pub fn status_path() -> PathBuf {
232 crate::run::home().join("daemon.json")
233}
234
235pub fn write_status(status: &Status) -> Result<()> {
237 write_status_to(&status_path(), status)
238}
239
240pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
245 if let Some(parent) = path.parent() {
246 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
247 }
248 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
249 let tmp = path.with_extension("json.tmp");
250 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
251 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
252 Ok(())
253}
254
255pub fn clear_status() {
258 clear_status_at(&status_path());
259}
260
261fn clear_status_at(path: &Path) {
265 let _ = std::fs::remove_file(path);
266}
267
268#[derive(Debug, Clone, Default)]
281pub struct Stop {
282 stopped: Arc<AtomicBool>,
286 busy: Arc<std::sync::atomic::AtomicUsize>,
292 wake: Arc<Notify>,
296 pause: crate::graph::Pause,
299}
300
301impl Stop {
302 #[must_use]
304 pub fn new() -> Self {
305 Self::default()
306 }
307
308 pub fn stop(&self) {
311 self.stopped.store(true, Ordering::SeqCst);
312 self.wake.notify_one();
316 }
317
318 #[must_use]
320 pub fn stopped(&self) -> bool {
321 self.stopped.load(Ordering::SeqCst)
322 }
323
324 #[must_use]
332 pub fn finishing(&self) -> bool {
333 self.stopped() && self.busy_now()
334 }
335
336 pub fn park(&self, cause: crate::run::ParkCause) {
347 self.pause.park_for(cause);
348 self.stop();
349 }
350
351 #[must_use]
353 pub fn parking(&self) -> bool {
354 self.pause.parked()
355 }
356
357 #[must_use]
359 pub fn pause(&self) -> crate::graph::Pause {
360 self.pause.clone()
361 }
362
363 #[must_use]
369 pub fn busy_now(&self) -> bool {
370 self.busy.load(Ordering::SeqCst) > 0
371 }
372
373 fn enter(&self) {
375 self.busy.fetch_add(1, Ordering::SeqCst);
376 }
377
378 fn exit(&self) {
381 self.busy.fetch_sub(1, Ordering::SeqCst);
382 }
383
384 async fn idle(&self, poll: Duration) {
386 tokio::select! {
387 () = tokio::time::sleep(poll) => {}
388 () = self.wake.notified() => {}
389 }
390 }
391}
392
393#[derive(Debug, Clone, Default, Deserialize)]
400#[serde(default)]
401pub struct Reading {
402 pub schema: u32,
404 pub pid: Option<u32>,
406 pub started_at: Option<Timestamp>,
408 pub updated_at: Option<Timestamp>,
410 pub idle: bool,
412 #[serde(deserialize_with = "de_current")]
424 pub current: Vec<Current>,
425 pub completed: u64,
427 pub polls: u64,
429}
430
431fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
434where
435 D: serde::Deserializer<'de>,
436{
437 #[derive(Deserialize)]
438 #[serde(untagged)]
439 enum Shape {
440 Many(Vec<Current>),
441 One(Current),
442 }
443 Ok(
444 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
445 Shape::Many(v) => v,
446 Shape::One(c) => vec![c],
447 }),
448 )
449}
450
451impl Reading {
452 #[must_use]
455 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
456 self.updated_at
457 .map(|at| (now.as_second() - at.as_second()).max(0))
458 }
459
460 #[must_use]
464 pub fn running(&self, now: Timestamp) -> bool {
465 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
466 }
467}
468
469#[must_use]
476pub fn read_status(home: &Path) -> Option<Reading> {
477 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
478 serde_json::from_str(&body).ok()
479}
480
481#[must_use]
493pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
494 read_status(home)
495 .filter(|reading| reading.running(now))
496 .map(|reading| reading.current)
497 .unwrap_or_default()
498}
499
500#[must_use]
502pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
503 current_work(home, now).iter().any(|c| c.run == run)
504}
505
506#[must_use]
516pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
517 current_work(home, now)
518 .iter()
519 .any(|c| crate::run::short_of(&c.run) == short)
520}
521
522#[must_use]
524pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
525 current_work(home, now).iter().any(|c| c.task == task)
526}
527
528pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
572 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
573}
574
575fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
579where
580 F: Fn(u32) -> bool,
581{
582 let this_process = std::process::id();
583 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
584 .into_iter()
585 .flatten()
586 .flatten()
587 .map(|e| e.path())
588 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
589 .filter(|p| {
590 match std::fs::read_to_string(p)
591 .ok()
592 .and_then(|body| body.trim().parse::<u32>().ok())
593 {
594 Some(pid) if pid == this_process => false,
598 Some(pid) => !pid_alive(pid),
599 None => p
600 .metadata()
601 .and_then(|m| m.modified())
602 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
603 .is_ok_and(|age| age >= older_than),
604 }
605 })
606 .filter(|p| std::fs::remove_file(p).is_ok())
607 .filter_map(|p| {
608 p.file_stem()
609 .and_then(|s| s.to_str())
610 .map(std::borrow::ToOwned::to_owned)
611 })
612 .collect();
613 swept.sort_unstable();
614 swept
615}
616
617fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
622 task.status == TaskStatus::Running
623 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
624 && !is_working_on_task(home, &task.id, now)
625}
626
627fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
630 queue
631 .list()
632 .into_iter()
633 .filter(|t| is_stalled(t, home, now))
634 .collect()
635}
636
637fn queued_tasks(queue: &Queue) -> Vec<Task> {
643 queue
644 .list()
645 .into_iter()
646 .filter(|t| t.status == TaskStatus::Queued)
647 .collect()
648}
649
650fn finished_tasks(queue: &Queue) -> Vec<Task> {
653 queue
654 .list()
655 .into_iter()
656 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
657 .collect()
658}
659
660fn resolve_blockers(queue: &Queue, questions: &Questions) {
677 for listed in queue.list() {
678 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
679 continue;
680 }
681 let Ok(_claim) = queue.claim(&listed.id) else {
682 continue;
683 };
684 let Ok(mut task) = queue.get(&listed.id) else {
685 continue;
686 };
687 if task.status != TaskStatus::Blocked {
688 continue;
689 }
690 let deleted = queue.apply_deleted_blockers(&mut task, questions);
694 if !deleted.is_empty() {
695 record(queue, &mut task);
696 for id in &deleted {
697 queue.note_dependency_deleted(&task, id);
698 }
699 if task.status != TaskStatus::Blocked {
700 continue;
701 }
702 }
703 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
704 if !missing.is_empty() {
705 let language = language_of(&task, Path::new("."));
706 let mut cause = missing.clone();
707 cause.sort();
708 cause.dedup();
709 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
710 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
711 &task.blocked_by,
712 &missing,
713 &language,
714 )));
715 record(queue, &mut task);
716 continue;
717 }
718 if let Some(q) = task.blocked_by.iter().find_map(|id| {
721 let crate::queue::Dependency::Question(full) =
722 crate::queue::dependency(queue, questions, id)
723 else {
724 return None;
725 };
726 questions.get(&full).ok().filter(|q| {
727 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
728 })
729 }) {
730 let language = language_of(&task, Path::new("."));
731 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
732 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
733 record(queue, &mut task);
734 continue;
735 }
736 let mut changed = false;
737 for id in task.blocked_by.clone() {
738 let resolved = crate::queue::dependency(queue, questions, &id);
739 if let crate::queue::Dependency::Task(full) = &resolved {
740 if queue
741 .get(full)
742 .is_ok_and(|dep| dep.status == TaskStatus::Done)
743 {
744 task.unblock(&id);
745 changed = true;
746 }
747 continue;
748 }
749 if let crate::queue::Dependency::Question(full) = resolved
750 && let Ok(q) = questions.get(&full)
751 && q.status == ask::QuestionStatus::Answered
752 {
753 let answer = match &q.answer {
754 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
755 None => String::new(),
756 };
757 task.record_answer(q.summary.clone(), answer);
758 task.unblock(&id);
759 changed = true;
760 }
761 }
762 if changed {
763 record(queue, &mut task);
764 }
765 }
766}
767
768fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
770 if crate::lang::is_japanese(language) {
771 format!(
772 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
773 q.short(),
774 q.summary
775 )
776 } else {
777 format!(
778 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
779 q.short(),
780 q.summary
781 )
782 }
783}
784
785#[derive(Debug, Clone, Copy, PartialEq, Eq)]
788pub(crate) enum ActionStanding {
789 NoAction,
791 Pending,
793 Applied,
796 Stale,
798 Busy,
800}
801
802impl ActionStanding {
803 pub(crate) fn daemon_owns(self) -> bool {
805 matches!(self, Self::Pending | Self::Applied | Self::Stale)
806 }
807}
808
809pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
813 if q.chosen_action().is_none() {
814 ActionStanding::NoAction
815 } else if task.action_applied(&q.id) {
816 ActionStanding::Applied
817 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
818 ActionStanding::Stale
819 } else if matches!(
820 task.status,
821 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
822 ) {
823 ActionStanding::Busy
824 } else {
825 ActionStanding::Pending
826 }
827}
828
829#[derive(Debug, Clone, PartialEq, Eq)]
831enum ActionDecision {
832 Skip,
834 Resume(String),
836 Requeue,
838 Done,
840 Stale,
843 Refuse(String),
846}
847
848fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
856where
857 F: FnOnce(&str) -> Result<RunState>,
858{
859 let Some(action) = q.chosen_action() else {
860 return ActionDecision::Skip;
861 };
862 if task.action_applied(&q.id)
863 || matches!(
864 task.status,
865 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
866 )
867 {
868 return ActionDecision::Skip;
869 }
870 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
873 return ActionDecision::Stale;
874 }
875 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
876 let stopping = q.node == land::APPROVAL_NODE
877 && q.resolution()
878 .as_deref()
879 .is_some_and(crate::merge_decision::stops);
880 let decision_at = if repair || stopping {
884 q.answered_at.unwrap_or(q.asked_at)
885 } else {
886 q.asked_at
887 };
888 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
889 return if repair || stopping {
890 ActionDecision::Refuse(format!(
891 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
892 if repair { "repair" } else { "stop" },
893 q.id,
894 q.run,
895 task.id
896 ))
897 } else {
898 ActionDecision::Stale
899 };
900 }
901 match action {
902 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
903 ask::ChoiceAction::Done => ActionDecision::Done,
904 ask::ChoiceAction::Resume { run } => {
905 if task.runs.last() != Some(run) {
906 return ActionDecision::Refuse((p.resume_not_latest)(
907 q.short(),
908 ask::short_id(run),
909 ));
910 }
911 match load(run) {
912 Ok(s) if stopping => {
913 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
914 || !s
915 .land_approval
916 .as_ref()
917 .is_some_and(|binding| binding.question == q.id)
918 {
919 ActionDecision::Stale
920 } else if s.liveness(false) == Liveness::Live {
921 ActionDecision::Skip
922 } else if s.status == RunStatus::Landing && !s.released() {
923 ActionDecision::Resume(run.clone())
924 } else {
925 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
926 }
927 }
928 Ok(s)
929 if q.node == land::APPROVAL_NODE
930 && q.resolution().as_deref() == Some(land::REPAIR) =>
931 {
932 if s.merge_repairs.iter().any(|r| r.question == q.id)
933 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
934 {
935 ActionDecision::Stale
938 } else if s.liveness(false) == Liveness::Live {
939 ActionDecision::Skip
940 } else if s.status != RunStatus::Landing
941 || s.released()
942 || s.reviews.is_empty()
943 || !implementers_match(task, &s)
944 || !s
945 .land_approval
946 .as_ref()
947 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
948 {
949 ActionDecision::Refuse(format!(
950 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
951 q.id, run, run
952 ))
953 } else {
954 ActionDecision::Resume(run.clone())
955 }
956 }
957 Ok(s) if s.status.resumable() && !s.released() && !exhausted_review_budget(&s) => {
958 ActionDecision::Resume(run.clone())
959 }
960 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
961 q.short(),
962 ask::short_id(run),
963 )),
964 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
965 q.short(),
966 ask::short_id(run),
967 &format!("{e:#}"),
968 )),
969 }
970 }
971 }
972}
973
974struct Phrases {
988 graph_stopped: fn(&str, &str) -> String,
990 quorum_lost: &'static str,
991 quota_took_out: &'static str,
993 run_ended: &'static str,
995 waiting_for_answer: &'static str,
997 recovered_running: &'static str,
998 no_run_to_recover: &'static str,
999 could_not_start: &'static str,
1000 resume_not_latest: fn(&str, &str) -> String,
1001 resume_cannot_progress: fn(&str, &str) -> String,
1002 resume_unreadable: fn(&str, &str, &str) -> String,
1003 handover_refused: fn(&crate::handover::Refused) -> String,
1006 handover_hint: &'static str,
1008 reviewers_never_answered: fn(usize, usize) -> String,
1011}
1012
1013const PHRASES_EN: Phrases = Phrases {
1014 graph_stopped: |status, detail| {
1015 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1016 },
1017 quorum_lost: "the judging panel lost its quorum",
1018 quota_took_out: "; quota took out ",
1019 run_ended: "run ended ",
1020 waiting_for_answer: " - waiting for operator answer to question ",
1021 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1022 no_run_to_recover: "task was `running` with no live daemon and no readable \
1023 run to recover; held for a human to check what happened",
1024 could_not_start: "could not start the run: ",
1025 resume_not_latest: |q, run| {
1026 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1027 },
1028 resume_cannot_progress: |q, run| {
1029 format!("question {q} asked to resume run {run}, which cannot make progress")
1030 },
1031 resume_unreadable: |q, run, e| {
1032 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1033 },
1034 handover_refused: |r| r.to_string(),
1035 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1036 reviewers_never_answered: |missing, rounds| {
1037 format!(
1038 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1039 refusing to call it clean"
1040 )
1041 },
1042};
1043
1044const PHRASES_JA: Phrases = Phrases {
1045 graph_stopped: |status, detail| {
1046 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1047 },
1048 quorum_lost: "審査パネルが定足数を失いました",
1049 quota_took_out: "。クォータで脱落: ",
1050 run_ended: "run 終了: ",
1051 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1052 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1053 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1054 何が起きたか人が確認するため保留にしました",
1055 could_not_start: "run を開始できませんでした: ",
1056 resume_not_latest: |q, run| {
1057 format!(
1058 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1059 )
1060 },
1061 resume_cannot_progress: |q, run| {
1062 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1063 },
1064 resume_unreadable: |q, run, e| {
1065 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1066 },
1067 handover_refused: |r| {
1068 use crate::handover::Refused;
1069 match r {
1070 Refused::Foreign { branch, path, why } => format!(
1071 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1072 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1073 から、やり直してください(詳細: {why})"
1074 ),
1075 Refused::Unsafe { branch, path, why } => format!(
1076 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1077 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1078 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1079 ),
1080 Refused::ReleaseFailed { branch, path, run } => format!(
1081 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1082 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1083 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1084 ),
1085 }
1086 },
1087 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1088 reviewers_never_answered: |missing, rounds| {
1089 format!(
1090 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1091 クリーンとは認めません"
1092 )
1093 },
1094};
1095
1096fn phrases(language: &str) -> &'static Phrases {
1097 if crate::lang::is_japanese(language) {
1098 &PHRASES_JA
1099 } else {
1100 &PHRASES_EN
1101 }
1102}
1103
1104fn language_of(task: &Task, fallback: &Path) -> String {
1108 crate::lang::of_repo(&repo_for(task, fallback))
1109}
1110
1111pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1115 if q.node == crate::conduct::NODE {
1116 return tasks.iter().find(|t| t.id == q.run);
1117 }
1118 tasks.iter().find(|t| t.runs.contains(&q.run))
1119}
1120
1121pub(crate) fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1130 let tasks = queue.list();
1131 for q in questions.list() {
1132 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1134 continue;
1135 }
1136 if q.chosen_action().is_none() {
1137 continue;
1138 }
1139 let Some(listed) = task_of_question(&tasks, &q) else {
1140 if q.node == land::APPROVAL_NODE
1141 && q.resolution().as_deref().is_some_and(|choice| {
1142 choice == land::REPAIR || crate::merge_decision::stops(choice)
1143 })
1144 {
1145 notices::raise_in(
1146 home,
1147 Notice::warn(
1148 &format!("repair-owner:{}", q.id),
1149 format!(
1150 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1151 q.run, q.run
1152 ),
1153 ),
1154 );
1155 }
1156 continue;
1157 };
1158 if listed.action_applied(&q.id) {
1159 continue;
1160 }
1161 let Ok(_claim) = queue.claim(&listed.id) else {
1162 continue;
1163 };
1164 let Ok(mut task) = queue.get(&listed.id) else {
1165 continue;
1166 };
1167 let language = language_of(&task, Path::new("."));
1168 let decision = decide_action(&task, &q, phrases(&language), |id| {
1169 RunState::load_under(id, home)
1170 });
1171 let ran = matches!(
1172 decision,
1173 ActionDecision::Resume(_) | ActionDecision::Requeue | ActionDecision::Done
1174 );
1175 if ran && asker_may_still_read(questions, &q, home) {
1176 continue;
1185 }
1186 let done = matches!(decision, ActionDecision::Done);
1187 match decision {
1188 ActionDecision::Skip => continue,
1189 ActionDecision::Resume(run) => {
1190 task.release_operator("choice", Some(&q.id));
1191 task.resume_override = Some(crate::queue::OperatorResume {
1192 question_id: q.id.clone(),
1193 at: Timestamp::now(),
1194 conductor_rehold: None,
1195 forced: true,
1196 pinned_run: Some(run),
1197 });
1198 }
1199 ActionDecision::Stale => {}
1200 ActionDecision::Requeue => {
1201 task.release_operator("choice-requeue", Some(&q.id));
1202 task.requeue();
1203 }
1204 ActionDecision::Done => task.succeed(),
1205 ActionDecision::Refuse(why) => {
1206 if task.operator_held() {
1207 task.hold_reason = Some(match task.hold_reason.take() {
1210 Some(reason) => format!("{reason}\n\n{why}"),
1211 None => why,
1212 });
1213 } else {
1214 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1215 task.hold_machine(Some(why));
1216 }
1217 notices::raise(
1218 Notice::warn(
1219 &format!("action:{}", q.id),
1220 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1221 )
1222 .link(Link::Task {
1223 id: task.id.clone(),
1224 }),
1225 );
1226 }
1227 }
1228 task.mark_action_applied(&q.id);
1229 record(queue, &mut task);
1232 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1233 supersede_prior_runs(&task, home);
1234 }
1235 let _ = questions.update(&q.id, |r| {
1238 r.answer_delivered = true;
1239 Ok(())
1240 });
1241 }
1242}
1243
1244fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1248 let now = Timestamp::now();
1249 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1250 return true;
1251 }
1252 q.node != crate::conduct::NODE
1253 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1254 s.seats_active()
1255 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1256 })
1257}
1258
1259fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1270 let tasks = queue.list();
1271 let by_id: std::collections::BTreeMap<&str, &Task> =
1272 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1273 let referenced: Vec<&str> = tasks
1274 .iter()
1275 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1276 .collect();
1277
1278 for mut question in questions.list() {
1279 if !question.status.open() || question.node != crate::conduct::NODE {
1280 continue;
1281 }
1282 if referenced
1287 .iter()
1288 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1289 {
1290 continue;
1291 }
1292 let Some(task) = by_id.get(question.run.as_str()) else {
1293 continue;
1294 };
1295 question.abandon(format!(
1296 "task {} no longer waits for this answer",
1297 task.short()
1298 ));
1299 if let Err(e) = questions.put(&mut question) {
1300 tracing::warn!(
1301 "could not retire question {} for task {}: {e:#}",
1302 question.short(),
1303 task.short()
1304 );
1305 }
1306 }
1307}
1308
1309#[derive(Debug, Clone, Copy)]
1316pub struct Verdict {
1317 pub status: RunStatus,
1319 pub left_pr: bool,
1321 pub quota_hit: bool,
1323 pub parked: bool,
1325 pub no_viable_candidates: bool,
1332}
1333
1334pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1391 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1392}
1393
1394fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1396 if verdict.parked {
1402 task.park(detail);
1403 return;
1404 }
1405 match verdict.status {
1406 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1407 RunStatus::AlreadyInBase => task.already_landed(detail),
1408 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1409 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1410 task.stall(detail)
1411 }
1412 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1413 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1414 RunStatus::Blocked => task.fail(detail, max_attempts),
1415 RunStatus::VerifiedNoop => task.handed_off(detail),
1416 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1417 }
1418}
1419
1420pub fn supersede_prior_runs(task: &Task, home: &Path) {
1469 let now = Timestamp::now();
1470 let last_run_succeeded = task
1477 .runs
1478 .last()
1479 .and_then(|id| RunState::load_under(id, home).ok())
1480 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1481 for id in task.superseded_attempts(last_run_succeeded) {
1482 let mut state = match RunState::load_under(id, home) {
1483 Ok(s) => s,
1484 Err(e) => {
1485 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1486 continue;
1487 }
1488 };
1489 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1490 continue;
1491 }
1492 let daemon_claims = is_working_on(home, id, now);
1493 if state.liveness(daemon_claims) == Liveness::Live {
1494 continue;
1495 }
1496 state.status = RunStatus::Superseded;
1497 if let Err(e) = state.save_under(home) {
1498 tracing::warn!("could not mark run {id} superseded: {e:#}");
1499 }
1500 }
1501}
1502
1503fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1525 for task in queue.list() {
1526 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1527 continue;
1528 }
1529 supersede_prior_runs(&task, home);
1530 }
1531}
1532
1533fn settle_and_diagnose(
1540 task: &mut Task,
1541 verdict: Verdict,
1542 detail: &str,
1543 max_attempts: usize,
1544 state: &RunState,
1545) {
1546 let p = phrases(&state.config.graph.language);
1547 if let Some(d) = &state.merge_decision {
1550 use crate::merge_decision as md;
1551 if task.runs.last().is_some_and(|id| id != &state.id) {
1552 return;
1553 }
1554 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1555 task.succeed();
1556 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1557 if state.liveness(false) != Liveness::Dead {
1558 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1559 return;
1560 }
1561 let replacement = md::Replacement {
1562 question: d.question.clone(),
1563 run: state.id.clone(),
1564 pr: d.target.pr.clone(),
1565 agent: agent.to_owned(),
1566 context: format!(
1567 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1568 d.target.pr,
1569 state.id,
1570 serde_json::to_string(&state.reviews).unwrap_or_default()
1571 ),
1572 };
1573 task.requeue();
1574 task.pr_recovery = None;
1575 task.review_of = None;
1576 task.replacement = Some(replacement);
1577 } else {
1578 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1579 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1580 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1581 }
1582 task.mark_action_applied(&d.question);
1583 if let Some(pass) = state.passes.last() {
1584 task.record_pass(&state.id, pass.index, task.attempts, None);
1585 }
1586 return;
1587 }
1588 if let Some(replacement) = &task.replacement
1589 && !state.parked
1590 && (state.viable().is_empty()
1591 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1592 {
1593 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1594 return;
1595 }
1596 let attempts_before = task.attempts;
1597 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1603 task.notice_cause(
1604 &format!("resource:{}:{}", block.kind, block.node),
1605 None,
1606 None,
1607 );
1608 task.last_error = Some(detail.to_owned());
1609 task.diagnostic = None;
1610 task.attempts = task.attempts.saturating_sub(1);
1611 task.hold_machine(Some(format!(
1612 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1613 no agent was asked and no attempt was spent",
1614 crate::cache::RESOURCE_HOLD,
1615 block.kind,
1616 block.node
1617 )));
1618 if let Some(pass) = state.passes.last() {
1619 task.record_pass(&state.id, pass.index, attempts_before, None);
1620 }
1621 return;
1622 }
1623 let unresolved = task.hold_notice_cause.clone();
1624 settle_in(task, verdict, detail, max_attempts, p);
1625 if state.prep_failure.is_some() {
1626 task.hold_notice_cause = unresolved;
1629 task.startup_notice_cause("startup-candidates");
1630 }
1631 if let Some(pass) = state.passes.last() {
1634 let parked = match &pass.end {
1635 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
1636 _ => None,
1637 };
1638 task.record_pass(&state.id, pass.index, attempts_before, parked);
1639 }
1640 if task.status == TaskStatus::Held {
1641 if state.prep_failure.is_none() {
1642 task.notice_cause(
1643 label(verdict.status),
1644 Some(&state.id),
1645 state.passes.last().map(|p| p.index),
1646 );
1647 }
1648 task.diagnostic = diagnostic(state);
1649 note_open_question(task, &state.id, p);
1650 }
1651}
1652
1653fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
1668 let Some(home) = crate::run::try_home() else {
1669 return;
1670 };
1671 let open = Questions::at(home.join("questions")).open_for(run);
1672 let Some(q) = open.first() else {
1673 return;
1674 };
1675 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
1676 decisions.sort_unstable();
1677 task.notice_cause(
1678 &format!("questions:{}", decisions.join(",")),
1679 Some(run),
1680 None,
1681 );
1682 let base = task.hold_reason.clone().unwrap_or_default();
1683 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
1684}
1685
1686fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
1697 match last_run {
1698 Some(state) => {
1699 let verdict = Verdict {
1700 status: state.status,
1701 left_pr: state.pr.is_some(),
1702 quota_hit: !state.quota.is_empty(),
1703 parked: state.parked,
1704 no_viable_candidates: state.viable().is_empty(),
1705 };
1706 let detail = format!(
1707 "{}{}",
1708 phrases(&state.config.graph.language).recovered_running,
1709 describe(&state)
1710 );
1711 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
1712 }
1713 None => {
1714 let why = phrases(language).no_run_to_recover;
1717 task.last_error = Some(why.to_owned());
1718 task.notice_cause("missing-run", None, None);
1721 task.hold_machine(Some(why.to_owned()));
1722 }
1723 }
1724}
1725
1726fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
1748 let mut reclaimed = Vec::new();
1749 for listed in queue.list() {
1750 if listed.status != TaskStatus::Running {
1751 continue;
1752 }
1753 let Ok(_claim) = queue.claim(&listed.id) else {
1754 continue;
1755 };
1756 let Ok(mut task) = queue.get(&listed.id) else {
1760 continue;
1761 };
1762 if task.status != TaskStatus::Running {
1763 continue;
1764 }
1765 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
1766 if let Some(state) = &last_run
1776 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
1777 {
1778 tracing::warn!("abandon questions for {}: {e:#}", state.id);
1779 }
1780 let language = if last_run.is_none() {
1781 language_of(&task, Path::new("."))
1782 } else {
1783 String::new()
1784 };
1785 reclaim(&mut task, last_run, max_attempts, &language);
1786 if task.status == TaskStatus::Done {
1787 supersede_prior_runs(&task, &crate::run::home());
1788 }
1789 record(queue, &mut task);
1790 reclaimed.push(task.id.clone());
1791 }
1792 reclaimed
1793}
1794
1795fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
1827 reclaim_abandoned_runs_with(
1828 home,
1829 now,
1830 crate::proc::pid_status,
1831 crate::proc::process_started_at,
1832 )
1833}
1834
1835fn reclaim_abandoned_runs_with<F, G>(
1841 home: &Path,
1842 now: Timestamp,
1843 query: F,
1844 identity: G,
1845) -> Vec<String>
1846where
1847 F: Fn(u32) -> Option<bool>,
1848 G: Fn(u32) -> Option<String>,
1849{
1850 let mut abandoned = Vec::new();
1851 for entry in std::fs::read_dir(home.join("runs"))
1852 .into_iter()
1853 .flatten()
1854 .flatten()
1855 {
1856 let id = entry.file_name().to_string_lossy().into_owned();
1857 if !crate::run::is_run_id(&id) {
1858 continue;
1859 }
1860 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
1868 continue;
1869 };
1870 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
1871 continue;
1872 };
1873 if state.status.done() || !state.active_all_overrun(now) {
1874 continue;
1875 }
1876 let daemon_claims = is_working_on(home, &id, now);
1888 if state.liveness_with(daemon_claims, &query, &identity) != crate::run::Liveness::Dead {
1889 continue;
1890 }
1891 state.abandon("daemon");
1892 if let Err(e) = state.save_under(home) {
1893 tracing::warn!("could not persist abandoned run {id}: {e:#}");
1894 continue;
1895 }
1896 if let Some(notice) = notices::run_ended(&state) {
1899 notices::raise_in(home, notice);
1900 }
1901 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
1909 tracing::warn!("abandon questions for {id}: {e:#}");
1910 }
1911 abandoned.push(id);
1912 }
1913 abandoned
1914}
1915
1916pub async fn serve(opts: Opts) -> Result<()> {
1922 serve_until(opts, Stop::new()).await
1923}
1924
1925pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
1942 let signal = {
1943 let stop = stop.clone();
1944 tokio::spawn(async move {
1945 if tokio::signal::ctrl_c().await.is_ok() {
1946 stop.stop();
1947 tracing::info!("shutdown requested; a run in flight will be finished first");
1948 }
1949 })
1950 };
1951
1952 let worktrees_root = opts
1953 .worktrees_root
1954 .clone()
1955 .unwrap_or_else(crate::run::default_worktree_root);
1956 let outcome = drive(
1957 &opts,
1958 &Queue::open(),
1959 &status_path(),
1960 &crate::run::home(),
1961 &worktrees_root,
1962 &stop,
1963 )
1964 .await;
1965
1966 signal.abort();
1967 outcome
1968}
1969
1970async fn drive(
1983 opts: &Opts,
1984 queue: &Queue,
1985 status_file: &Path,
1986 home: &Path,
1987 worktrees_root: &Path,
1988 stop: &Stop,
1989) -> Result<()> {
1990 let status = Arc::new(Mutex::new(Status::new()));
1998 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
1999 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2000
2001 let daemon_cfg = prepare(&opts.repo, opts)
2007 .map(|c| c.daemon)
2008 .unwrap_or_default();
2009 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2010
2011 let waiter = tokio::spawn(crate::waiter::run(
2016 crate::waiter::Waiter::new(
2017 crate::ask::Questions::at(home.join("questions")),
2018 home.to_path_buf(),
2019 prepare(&opts.repo, opts).ok(),
2020 ),
2021 stop.clone(),
2022 ));
2023
2024 let deputies = tokio::spawn(crate::deputy::run(
2028 crate::deputy::Deputies::new(
2029 crate::ask::Questions::at(home.join("questions")),
2030 home.to_path_buf(),
2031 prepare(&opts.repo, opts).ok(),
2032 opts.repo.clone(),
2033 daemon_cfg.max_deputies,
2034 {
2035 let stop = stop.clone();
2036 Arc::new(move || stop.parking())
2037 },
2038 ),
2039 stop.clone(),
2040 ));
2041
2042 let fetcher = tokio::spawn(fetch_loop(opts.repo.clone(), opts.clone(), stop.clone()));
2045
2046 let release_watch = tokio::spawn(crate::release_watch::run(
2049 crate::release_watch::Watcher::new(
2050 Box::new(crate::release_watch::GhForge),
2051 home.to_path_buf(),
2052 ),
2053 {
2054 let (repo, opts) = (opts.repo.clone(), opts.clone());
2055 move || release_watch_settings(&repo, &opts)
2056 },
2057 stop.clone(),
2058 ));
2059
2060 tracing::info!(
2061 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2062 queue.root().display(),
2063 opts.poll.as_secs(),
2064 opts.max_attempts,
2065 concurrency,
2066 if daemon_cfg.pause_for_interrupts {
2067 ", interrupts enabled"
2068 } else {
2069 ""
2070 }
2071 );
2072
2073 janitor(&opts.repo, opts, home, worktrees_root).await;
2076 resweep_superseded_attempts(queue, home);
2077
2078 let outcome = poll(
2079 opts,
2080 queue,
2081 &status,
2082 home,
2083 worktrees_root,
2084 stop,
2085 DispatchLimits {
2086 max_concurrent: concurrency,
2087 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2088 },
2089 )
2090 .await;
2091
2092 beat.abort();
2093 waiter.abort();
2094 deputies.abort();
2095 fetcher.abort();
2096 release_watch.abort();
2097 clear_status_at(status_file);
2098 outcome
2099}
2100
2101fn release_watch_settings(repo: &Path, opts: &Opts) -> (Vec<PathBuf>, u64) {
2104 let Ok(cfg) = prepare(repo, opts) else {
2105 return (Vec::new(), 0);
2106 };
2107 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2108 .into_iter()
2109 .map(|r| r.path)
2110 .collect();
2111 let mut known: Vec<PathBuf> = Queue::open().list().into_iter().map(|t| t.repo).collect();
2114 let runs = crate::run::home().join("runs");
2115 for id in crate::run::list_ids_in(&runs) {
2116 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2117 .ok()
2118 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2119 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2120 known.extend(repo);
2121 }
2122 for k in known {
2123 let k = k.canonicalize().unwrap_or(k);
2124 if k.join(".git").exists() && !paths.contains(&k) {
2125 paths.push(k);
2126 }
2127 }
2128 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2129 if !paths.contains(&own) {
2130 paths.push(own);
2131 }
2132 (paths, cfg.daemon.release_stall_minutes)
2133}
2134
2135const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2137
2138async fn fetch_loop(repo: PathBuf, opts: Opts, stop: Stop) {
2142 while !stop.stopped() {
2143 let (interval, roots) = match prepare(&repo, &opts) {
2144 Ok(c) => (c.repos.fetch_interval, c.repos.roots),
2145 Err(_) => (0, Vec::new()),
2146 };
2147 if interval > 0 && !roots.is_empty() {
2148 let r = crate::clean::fetch_origins(&roots, FETCH_TIMEOUT, || stop.stopped()).await;
2149 tracing::debug!("fetch origins: {r:?}");
2150 }
2151 let wait = if interval > 0 { interval } else { 60 };
2153 let mut slept = 0;
2154 while slept < wait && !stop.stopped() {
2155 tokio::time::sleep(Duration::from_secs(1)).await;
2156 slept += 1;
2157 }
2158 }
2159}
2160
2161async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2167 loop {
2168 tokio::time::sleep(HEARTBEAT).await;
2169 let snapshot = {
2170 let mut guard = lock(&status);
2171 guard.updated_at = Timestamp::now();
2172 guard.clone()
2173 };
2174 if let Err(e) = write_status_to(&path, &snapshot) {
2175 tracing::warn!("could not refresh the daemon status file: {e:#}");
2178 }
2179 }
2180}
2181
2182#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2185enum LandResume {
2186 NotLanding,
2189 StillWaiting,
2194 Ready,
2198}
2199
2200fn land_resume_state(task: &Task) -> LandResume {
2204 let Some(run_id) = task.runs.last() else {
2205 return LandResume::NotLanding;
2206 };
2207 let Ok(state) = RunState::load(run_id) else {
2208 return LandResume::NotLanding;
2209 };
2210 if !state.parked {
2211 return LandResume::NotLanding;
2212 }
2213 if state.status == RunStatus::Landing
2214 && state.land_approval.as_ref().is_some_and(|binding| {
2215 ask::Questions::open()
2216 .get(&binding.question)
2217 .is_ok_and(|q| {
2218 q.run == state.id
2219 && q.resolution()
2220 .as_deref()
2221 .is_some_and(crate::merge_decision::stops)
2222 })
2223 })
2224 {
2225 return LandResume::Ready;
2226 }
2227 if !implementers_match(task, &state) {
2228 return LandResume::NotLanding;
2229 }
2230 let node = if state.status == RunStatus::Landing {
2234 land::APPROVAL_NODE
2235 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2236 crate::github_text::ASK_NODE
2237 } else {
2238 return LandResume::NotLanding;
2239 };
2240 let store = ask::Questions::open();
2241 let waiting = store
2242 .list()
2243 .into_iter()
2244 .filter(|q| &q.run == run_id && q.node == node)
2245 .max_by(|a, b| a.id.cmp(&b.id));
2246 let Some(mut q) = waiting else {
2247 return LandResume::Ready;
2248 };
2249 if !q.status.open() {
2250 if q.resolution().as_deref() == Some(land::REPAIR) {
2253 return if state.liveness(false) == Liveness::Live {
2254 LandResume::StillWaiting
2255 } else {
2256 LandResume::NotLanding
2257 };
2258 }
2259 return LandResume::Ready;
2260 }
2261 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2268 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2269 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2270 q.abandon(format!(
2271 "no answer within {}s of asking",
2272 timeout.as_secs().max(1)
2273 ));
2274 if store.put(&mut q).is_ok() {
2277 return LandResume::Ready;
2278 }
2279 }
2280 LandResume::StillWaiting
2281}
2282
2283const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2291
2292const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2304
2305struct InFlightGuard<'a> {
2318 status: &'a Arc<Mutex<Status>>,
2319 stop: &'a Stop,
2320 task_id: &'a str,
2321}
2322
2323impl Drop for InFlightGuard<'_> {
2324 fn drop(&mut self) {
2325 lock(self.status).current.retain(|c| c.task != self.task_id);
2326 self.stop.exit();
2327 }
2328}
2329
2330#[derive(Debug, Clone, PartialEq, Eq)]
2352enum Interrupt {
2353 Idle,
2355 Parking {
2367 parked: Vec<String>,
2368 interrupt_task: String,
2369 },
2370 Running {
2378 parked: Vec<String>,
2379 interrupt_task: String,
2380 },
2381 Resuming { parked: Vec<String> },
2388}
2389
2390fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
2411 match state {
2412 Interrupt::Idle => {
2413 if in_flight.len() != 1 {
2425 return Interrupt::Idle;
2426 }
2427 match runnable.iter().find(|t| t.interrupt) {
2428 Some(t) => Interrupt::Parking {
2429 parked: in_flight.to_vec(),
2430 interrupt_task: t.id.clone(),
2431 },
2432 None => Interrupt::Idle,
2433 }
2434 }
2435 Interrupt::Parking {
2436 parked,
2437 interrupt_task,
2438 } => {
2439 if in_flight.iter().any(|id| parked.contains(id)) {
2440 Interrupt::Parking {
2442 parked,
2443 interrupt_task,
2444 }
2445 } else if in_flight.contains(&interrupt_task) {
2446 Interrupt::Running {
2447 parked,
2448 interrupt_task,
2449 }
2450 } else if runnable.iter().any(|t| t.id == interrupt_task) {
2451 Interrupt::Parking {
2455 parked,
2456 interrupt_task,
2457 }
2458 } else {
2459 Interrupt::Resuming { parked }
2464 }
2465 }
2466 Interrupt::Running {
2467 parked,
2468 interrupt_task,
2469 } => {
2470 if in_flight.contains(&interrupt_task) {
2471 Interrupt::Running {
2472 parked,
2473 interrupt_task,
2474 }
2475 } else {
2476 Interrupt::Resuming { parked }
2482 }
2483 }
2484 Interrupt::Resuming { parked } => {
2485 if in_flight.iter().any(|id| parked.contains(id)) {
2486 Interrupt::Idle
2492 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
2493 Interrupt::Resuming { parked }
2494 } else {
2495 Interrupt::Idle
2498 }
2499 }
2500 }
2501}
2502
2503fn advance_interrupt_tick(
2509 enabled: bool,
2510 state: Interrupt,
2511 in_flight: &[String],
2512 runnable: &[Task],
2513) -> Interrupt {
2514 if !enabled {
2515 return Interrupt::Idle;
2516 }
2517 advance_interrupt(state, in_flight, runnable)
2518}
2519
2520fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
2525 match state {
2526 Interrupt::Idle => candidates,
2527 Interrupt::Parking {
2528 parked,
2529 interrupt_task,
2530 } => {
2531 if in_flight.iter().any(|id| parked.contains(id)) {
2532 Vec::new()
2533 } else {
2534 candidates
2535 .into_iter()
2536 .filter(|t| &t.id == interrupt_task)
2537 .collect()
2538 }
2539 }
2540 Interrupt::Running { .. } => Vec::new(),
2541 Interrupt::Resuming { parked } => candidates
2549 .into_iter()
2550 .find(|t| parked.contains(&t.id))
2551 .into_iter()
2552 .collect(),
2553 }
2554}
2555
2556struct DispatchLimits {
2560 max_concurrent: usize,
2563 pause_for_interrupts: bool,
2565}
2566
2567#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2576enum PermitKind {
2577 None,
2582 Urgent,
2589 Ordinary,
2592}
2593
2594fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
2597 if priority {
2598 PermitKind::None
2599 } else if urgent {
2600 PermitKind::Urgent
2601 } else {
2602 PermitKind::Ordinary
2603 }
2604}
2605
2606struct DispatchClaim {
2607 task: Task,
2608 claim: crate::queue::Claim,
2609 permit: Option<tokio::sync::OwnedSemaphorePermit>,
2610}
2611
2612fn claim_dispatch(
2615 queue: &Queue,
2616 candidate: &Task,
2617 cooling_down: bool,
2618 ordinary: &Arc<tokio::sync::Semaphore>,
2619 urgent: &Arc<tokio::sync::Semaphore>,
2620) -> Option<DispatchClaim> {
2621 let Ok(claim) = queue.claim(&candidate.id) else {
2622 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
2623 return None;
2624 };
2625 let task = match queue.get(&candidate.id) {
2626 Ok(t) if t.status.runnable() => t,
2627 Ok(_) => return None,
2628 Err(e) => {
2629 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
2630 return None;
2631 }
2632 };
2633 let resume = land_resume_state(&task);
2634 if resume == LandResume::StillWaiting {
2635 return None;
2636 }
2637 let priority = resume == LandResume::Ready;
2638 if !priority && cooling_down {
2639 return None;
2640 }
2641 let permit = match permit_kind(priority, task.urgent) {
2644 PermitKind::None => None,
2645 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
2646 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
2647 };
2648 Some(DispatchClaim {
2649 task,
2650 claim,
2651 permit,
2652 })
2653}
2654
2655async fn poll(
2674 opts: &Opts,
2675 queue: &Queue,
2676 status: &Arc<Mutex<Status>>,
2677 home: &Path,
2678 worktrees_root: &Path,
2679 stop: &Stop,
2680 limits: DispatchLimits,
2681) -> Result<()> {
2682 let DispatchLimits {
2683 max_concurrent,
2684 pause_for_interrupts,
2685 } = limits;
2686 let mut attempted: Vec<String> = Vec::new();
2691 let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
2692 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(1));
2700 let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
2706 let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
2707 let mut conductor = Conductor::new();
2708 let mut cache_last_checked: Option<Timestamp> = None;
2711 let mut interrupt = Interrupt::Idle;
2713 let mut interrupt_pauses: std::collections::HashMap<String, crate::graph::Pause> =
2719 std::collections::HashMap::new();
2720
2721 while !stop.stopped() {
2722 lock(status).polls += 1;
2723
2724 while let Some(result) = inflight.try_join_next() {
2729 if let Err(e) = result {
2730 tracing::error!("a spawned attempt did not finish cleanly: {e}");
2731 notices::raise(Notice::error(
2732 "loop:attempt",
2733 "A queued attempt ended abnormally; check the task it was running.",
2734 ));
2735 }
2736 }
2737
2738 let swept = sweep_stale_claims(queue, STALE_CLAIM);
2739 if !swept.is_empty() {
2740 tracing::warn!(
2741 "swept {} stale claim(s) left behind by an earlier daemon: {}",
2742 swept.len(),
2743 swept.join(", ")
2744 );
2745 }
2746 let now = Timestamp::now();
2751
2752 if !stop.busy_now() {
2757 maybe_prune_cache_between_runs(
2758 &opts.repo,
2759 opts,
2760 home,
2761 stop,
2762 &mut cache_last_checked,
2763 now,
2764 )
2765 .await;
2766 }
2767
2768 let stalled = stalled_tasks(queue, home, now);
2769 let stalled_ids: std::collections::BTreeSet<_> =
2770 stalled.iter().map(|task| task.id.clone()).collect();
2771 let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
2772 if !reclaimed.is_empty() {
2773 tracing::warn!(
2774 "reclaimed {} task(s) left `running` by a daemon that never \
2775 recorded the outcome: {}",
2776 reclaimed.len(),
2777 reclaimed.join(", ")
2778 );
2779 }
2780 let abandoned_runs = reclaim_abandoned_runs(home, now);
2781 if !abandoned_runs.is_empty() {
2782 tracing::warn!(
2783 "failed {} run(s) left behind by a killed process, past every \
2784 active seat's own timeout: {}",
2785 abandoned_runs.len(),
2786 abandoned_runs.join(", ")
2787 );
2788 }
2789
2790 let questions = Questions::at(home.join("questions"));
2795
2796 resolve_blockers(queue, &questions);
2799 apply_choice_actions(queue, &questions, home);
2800 triage::apply_answers(queue, &questions);
2801 reconcile_task_questions(queue, &questions);
2802 {
2806 let talks = crate::talk::Talks::at(home.join("talks"));
2807 crate::chat_notify::sweep(queue, &talks, home);
2808 crate::chat_watch::sweep(home, &talks);
2809 crate::chat_notify::start_turns(&talks, stop.parking(), &mut inflight);
2810 }
2811
2812 let finished: Vec<Task> = finished_tasks(queue)
2818 .into_iter()
2819 .filter(|task| !stalled_ids.contains(&task.id))
2820 .filter(|task| !awaiting_resume_with(task, |id| RunState::load_under(id, home)))
2824 .collect();
2825 let queued = queued_tasks(queue);
2826 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
2830 && conductor.worth_a_look(queue, &stalled, &finished)
2831 {
2832 match prepare(&opts.repo, opts) {
2833 Ok(cfg) => {
2834 conductor
2835 .maybe_run(
2836 &cfg,
2837 &opts.repo,
2838 queue,
2839 &questions,
2840 home,
2841 &queued,
2842 &stalled,
2843 &finished,
2844 opts.max_attempts,
2845 )
2846 .await;
2847 }
2848 Err(e) => {
2849 tracing::warn!("conductor: no config: {e:#}");
2850 notices::raise(Notice::warn(
2851 "loop:no-config",
2852 "The loop could not read this repository's config, so held tasks are not being triaged.",
2853 ));
2854 }
2855 }
2856 }
2857
2858 let candidates: Vec<Task> = runnable(queue)
2859 .into_iter()
2860 .filter(|t| !opts.once || !attempted.contains(&t.id))
2861 .collect();
2862
2863 let in_flight: Vec<String> = lock(status)
2868 .current
2869 .iter()
2870 .map(|c| c.task.clone())
2871 .collect();
2872 interrupt_pauses.retain(|id, _| in_flight.contains(id));
2873
2874 interrupt =
2875 advance_interrupt_tick(pause_for_interrupts, interrupt, &in_flight, &candidates);
2876 if let Interrupt::Parking {
2877 parked,
2878 interrupt_task,
2879 } = &interrupt
2880 {
2881 let reason = format!(
2882 "task {} asked to run first",
2883 crate::run::short_of(interrupt_task)
2884 );
2885 for id in parked {
2886 if let Some(pause) = interrupt_pauses.get(id) {
2887 pause.park_because(reason.clone());
2888 }
2889 }
2890 }
2891 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
2892
2893 let cooling_down =
2894 lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
2895
2896 let mut started_any = false;
2897 for candidate in candidates {
2898 if stop.stopped() {
2899 break;
2900 }
2901
2902 let Some(DispatchClaim {
2903 mut task,
2904 claim,
2905 permit,
2906 }) = claim_dispatch(queue, &candidate, cooling_down, &sem, &urgent_sem)
2907 else {
2908 continue;
2909 };
2910
2911 let task_id = task.id.clone();
2912 attempted.push(task_id.clone());
2913 lock(status).idle = false;
2914 stop.enter();
2917 started_any = true;
2918
2919 let run_pause = crate::graph::Pause::new();
2923 interrupt_pauses.insert(task_id.clone(), run_pause.clone());
2924
2925 let opts = opts.clone();
2926 let queue = queue.clone();
2927 let status = Arc::clone(status);
2928 let stop = stop.clone();
2929 let quota_cooldown_until = Arc::clone("a_cooldown_until);
2930 inflight.spawn(async move {
2931 let _claim = claim;
2935 let _permit = permit;
2936 let _inflight = InFlightGuard {
2938 status: &status,
2939 stop: &stop,
2940 task_id: &task_id,
2941 };
2942 let quota = attempt(&opts, &queue, &status, &stop, run_pause, &mut task).await;
2943 lock(&status).completed += 1;
2944 let now = Timestamp::now();
2950 if let Some(until) = cooldown_until("a, now) {
2951 let wait = until.as_second() - now.as_second();
2952 *lock("a_cooldown_until) = Some(until);
2953 let hint = quota
2954 .iter()
2955 .find(|q| q.reset.is_some())
2956 .and_then(|q| q.reset.as_deref());
2957 match hint {
2958 Some(h) => tracing::warn!(
2959 "quota hit; waiting {wait}s before taking another ordinary task \
2960 (CLI reported reset: {h})"
2961 ),
2962 None => tracing::warn!(
2963 "quota hit; waiting {wait}s before taking another ordinary task \
2964 (no reset hint reported)"
2965 ),
2966 }
2967 }
2968 });
2969 }
2970
2971 if started_any {
2972 continue;
2973 }
2974
2975 if stop.busy_now() {
2976 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
2981 continue;
2982 }
2983
2984 lock(status).idle = true;
2986 if opts.once {
2987 janitor(&opts.repo, opts, home, worktrees_root).await;
2991 resweep_superseded_attempts(queue, home);
2992 triage_held(queue, home, opts).await;
2993 break;
2994 }
2995 stop.idle(opts.poll).await;
2996 if stop.stopped() {
2997 continue;
2998 }
2999 janitor(&opts.repo, opts, home, worktrees_root).await;
3005 resweep_superseded_attempts(queue, home);
3006 triage_held(queue, home, opts).await;
3007 }
3008
3009 while let Some(result) = inflight.join_next().await {
3014 if let Err(e) = result {
3015 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3016 notices::raise(Notice::error(
3017 "loop:attempt",
3018 "A queued attempt ended abnormally; check the task it was running.",
3019 ));
3020 }
3021 }
3022 Ok(())
3023}
3024
3025async fn attempt(
3031 opts: &Opts,
3032 queue: &Queue,
3033 status: &Arc<Mutex<Status>>,
3034 stop: &Stop,
3035 interrupt_pause: crate::graph::Pause,
3036 task: &mut Task,
3037) -> Vec<QuotaLoss> {
3038 let repo = repo_for(task, &opts.repo);
3039 tracing::info!(
3040 "task {} — {} (repo {})",
3041 task.short(),
3042 task.title,
3043 repo.display()
3044 );
3045
3046 if let Some(previous) = task.runs.last().and_then(|id| RunState::load(id).ok())
3050 && previous
3051 .merge_decision
3052 .as_ref()
3053 .is_some_and(|decision| !decision.closed)
3054 && previous.status != RunStatus::Landing
3055 && !crate::merge_decision::resumable_hold(&previous)
3056 {
3057 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3058 record(queue, task);
3059 return Vec::new();
3060 }
3061 let stop_owner = task
3062 .runs
3063 .last()
3064 .and_then(|id| RunState::load(id).ok())
3065 .filter(|state| {
3066 crate::merge_decision::resumable_hold(state)
3067 || state.status == RunStatus::Landing
3068 && state.land_approval.as_ref().is_some_and(|binding| {
3069 Questions::open().get(&binding.question).is_ok_and(|q| {
3070 q.run == state.id
3071 && q.resolution()
3072 .as_deref()
3073 .is_some_and(crate::merge_decision::stops)
3074 })
3075 })
3076 });
3077 if let Some(owner) = stop_owner {
3078 let mut runner = match Runner::resume(&owner.id) {
3079 Ok(runner) => runner,
3080 Err(e) => {
3081 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
3082 record(queue, task);
3083 return Vec::new();
3084 }
3085 };
3086 runner.on_pause(stop.pause());
3087 runner.watch_interrupt(interrupt_pause);
3088 let quota_before = runner.state.quota.clone();
3089 task.start(owner.id.clone());
3090 record(queue, task);
3091 lock(status).current.push(Current {
3092 task: task.id.clone(),
3093 run: owner.id,
3094 });
3095 runner.state.next_pass_task_entry = true;
3096 let result = runner.execute().await;
3097 return finish_attempt(
3098 opts.max_attempts,
3099 queue,
3100 task,
3101 &runner.state,
3102 "a_before,
3103 result,
3104 );
3105 }
3106
3107 let explicit = match &task.overrides {
3111 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
3112 Some(_) => opts.config.as_deref(),
3113 None => opts.config.as_deref(),
3114 };
3115 if let Err(e) = Config::discover_fetched(&repo, explicit).await {
3116 let reason = format!("[config] {e:#}");
3117 record_config_failure(task, &e);
3118 task.last_error = Some(reason.clone());
3119 task.hold_machine(Some(reason.clone()));
3120 record(queue, task);
3121 tracing::warn!("holding {}: {reason}", task.short());
3122 return Vec::new();
3123 }
3124 let mut config = match prepare_for(&repo, opts, task) {
3125 Ok(c) => c,
3126 Err(e) => {
3127 task.attempts += 1;
3131 task.fail_startup(
3132 "startup-config",
3133 format!("config: {e:#}"),
3134 opts.max_attempts,
3135 );
3136 record(queue, task);
3137 return Vec::new();
3138 }
3139 };
3140 apply_task_implementers(&mut config, task);
3141 if let Some(replacement) = &task.replacement {
3142 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
3143 old.liveness(false) == Liveness::Dead
3144 && old.merge_decision.as_ref().is_some_and(|d| {
3145 d.closed
3146 && d.question == replacement.question
3147 && d.replacement_run.as_ref().is_none_or(|run| {
3148 task.runs.last() == Some(run)
3149 && RunState::load(run).is_ok_and(|current| {
3150 current.replaces.as_ref().is_some_and(|pin| {
3151 pin.question == replacement.question
3152 && pin.agent == replacement.agent
3153 })
3154 })
3155 })
3156 })
3157 });
3158 if !safe {
3159 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
3160 record(queue, task);
3161 return Vec::new();
3162 }
3163 match config.agent(&replacement.agent) {
3164 Ok(agent) if crate::agent::installed(agent) => {}
3165 _ => {
3166 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
3167 record(queue, task);
3168 return Vec::new();
3169 }
3170 }
3171 config.graph.implementers = 1;
3172 config.graph.land_approval = true;
3173 config.merge.mode = MergeMode::Pr;
3174 config.graph.selected_implementer = Some(replacement.agent.clone());
3175 }
3176 let p = phrases(&config.graph.language);
3177 let start_failed = p.could_not_start;
3178
3179 if let Some(reason) = disk_gate(&repo, &config) {
3187 task.notice_cause("disk-space", None, None);
3188 task.last_error = Some(reason.clone());
3189 task.hold_machine(Some(reason.clone()));
3190 record(queue, task);
3191 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
3192 notices::raise(
3195 Notice::warn(
3196 &format!("disk:{}", repo.display()),
3197 "A task was held for want of free disk space; free some, then release it from the queue.",
3198 )
3199 .link(Link::Task {
3200 id: task.id.clone(),
3201 }),
3202 );
3203 return Vec::new();
3204 }
3205
3206 let unfinished = (!task.fresh_start)
3226 .then(|| unfinished_run(&task.runs, task.short()))
3227 .flatten()
3228 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
3229 let previous = task.runs.last().and_then(|id| RunState::load(id).ok());
3235 let review_branch = take_review_branch(task, previous.as_ref());
3236 let branch_exists = match &review_branch {
3237 Some(branch) => crate::git::branch_exists(&repo, branch)
3238 .await
3239 .unwrap_or(false),
3240 None => false,
3241 };
3242 let proposed = match (&review_branch, &unfinished, &task.review_of) {
3246 (None, None, Some(branch)) => Starter::Review(branch.clone()),
3247 _ => choose_starter(
3248 review_branch.as_deref(),
3249 branch_exists,
3250 unfinished.as_deref(),
3251 ),
3252 };
3253 let retry_selected = task.replacement.is_some()
3254 && task.resumption.is_some()
3255 && !task.fresh_start
3256 && previous.as_ref().is_some_and(|run| {
3257 run.replaces.is_some()
3258 && ((run.status == RunStatus::Failed && run.viable().is_empty())
3259 || (run.status == RunStatus::Blocked && run.tally.is_none()))
3260 });
3261 let stop_pin = task
3262 .resume_override
3263 .as_ref()
3264 .filter(|pin| {
3265 Questions::open().get(&pin.question_id).is_ok_and(|q| {
3266 q.node == land::APPROVAL_NODE
3267 && q.resolution()
3268 .as_deref()
3269 .is_some_and(crate::merge_decision::stops)
3270 })
3271 })
3272 .and_then(|pin| pin.pinned_run.clone());
3273 let starter = if let Some(run) = stop_pin {
3274 Starter::Resume(run)
3277 } else if retry_selected {
3278 Starter::Resume(previous.as_ref().unwrap().id.clone())
3279 } else if let Some(selected) = &task.pr_recovery {
3280 if selected.check_task(&task.id, &task.instruction).is_ok() {
3281 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
3282 } else {
3283 Starter::Review(selected.branch.clone())
3284 }
3285 } else {
3286 starter_with_implementers(task, previous.as_ref(), proposed)
3287 };
3288 if let Some(pin) = &task.resume_override
3291 && let Some(run) = &pin.pinned_run
3292 && Questions::open().get(&pin.question_id).map_or(true, |q| {
3293 q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR)
3294 })
3295 && starter != Starter::Resume(run.clone())
3296 {
3297 let run = run.clone();
3298 task.notice_cause("repair-owner", Some(&run), None);
3299 task.hold_machine(Some(format!(
3300 "repair owner cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
3301 task.id
3302 )));
3303 record(queue, task);
3304 return Vec::new();
3305 }
3306 if task.replacement.is_some()
3307 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
3308 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
3309 {
3310 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
3311 record(queue, task);
3312 return Vec::new();
3313 }
3314 let attachments = match task_attachments(queue, task) {
3319 Ok(a) => a,
3320 Err(e) => {
3321 task.attempts += 1;
3322 task.fail_startup_target(
3323 "startup-attachments",
3324 e.name.clone(),
3325 format!("{start_failed}{e:#}"),
3326 opts.max_attempts,
3327 );
3328 record(queue, task);
3329 return Vec::new();
3330 }
3331 };
3332 let started = match &starter {
3333 Starter::Review(_) if task.pr_recovery.is_some() => {
3334 let selected = task.pr_recovery.clone().expect("selected recovery");
3335 match selected.check_task(&task.id, &task.instruction) {
3336 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
3337 .await
3338 .map(|mut r| {
3339 r.state.attachments = attachments.clone();
3340 r
3341 }),
3342 Err(e) => Err(e),
3343 }
3344 }
3345 Starter::Review(branch) => {
3346 tracing::info!(
3347 "task {} reopens `{branch}` as a review-only pass",
3348 task.short()
3349 );
3350 let takeover = crate::handover::Takeover {
3353 earlier: task.earlier_attempts().to_vec(),
3354 home: crate::run::home(),
3355 choice: take_divergence_answer(branch, &config.merge.remote, task),
3356 task_instruction: Some(instruction_for(task)),
3357 };
3358 Runner::review_taking_over(
3359 &repo,
3360 branch,
3361 config,
3362 Some(takeover),
3363 crate::run::Origin::queue(&task.id),
3364 )
3365 .await
3366 }
3367 Starter::Resume(id) => {
3368 tracing::info!("resuming run {id} rather than competing again");
3369 Runner::resume(id).and_then(|mut r| {
3370 if retry_selected {
3371 for candidate in &mut r.state.candidates {
3374 candidate.failed = None;
3375 candidate.empty = false;
3376 candidate.commits = 0;
3377 }
3378 r.state.status = RunStatus::Implementing;
3379 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
3380 }
3381 if r.state.prep_failure.is_some() {
3382 let approval = task
3383 .resumption
3384 .as_ref()
3385 .context("preparation retry requires an explicit task release")?;
3386 r.retry_preparation(approval.at)?;
3387 }
3388 if let Some(instruction) =
3389 prepare_instruction(&starter, Some(&r.state.instruction), task)
3390 {
3391 r.state.instruction = instruction;
3392 }
3393 r.state.attachments = attachments.clone();
3394 if task.replacement.is_some() {
3395 r.state.config.merge.mode = MergeMode::Pr;
3396 r.state.config.graph.land_approval = true;
3397 }
3398 if task.replacement.is_none() && let Some(mode) = task
3401 .overrides
3402 .as_ref()
3403 .and_then(|o| o.merge.as_deref())
3404 .and_then(|m| merge_mode(m).ok())
3405 {
3406 r.state.config.merge.mode = mode;
3407 }
3408 Ok(r)
3409 })
3410 }
3411 Starter::Start => {
3412 if let Some(branch) = &review_branch {
3413 tracing::warn!(
3414 "conductor chose review for task {} but branch `{branch}` no longer \
3415 exists; requeuing as a fresh competition instead",
3416 task.short()
3417 );
3418 }
3419 let instruction = prepare_instruction(&starter, None, task)
3420 .unwrap_or_else(|| task.instruction.clone());
3421 Runner::start_naming(
3422 &repo,
3423 instruction,
3424 &task.title,
3425 config,
3426 crate::run::Origin::queue(&task.id),
3427 )
3428 .await
3429 .map(|mut r| {
3430 r.state.attachments = attachments.clone();
3431 r
3432 })
3433 }
3434 };
3435 let mut runner = match started {
3436 Ok(r) => r,
3437 Err(e) if task.pr_recovery.is_some() => {
3441 task.startup_notice_cause("startup-public-head");
3442 task.attempts += 1;
3443 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
3444 task.link_run(&partial.run);
3445 task.record_pass(&partial.run, 0, task.attempts, None);
3446 }
3447 task.last_error = Some(format!("{start_failed}{e:#}"));
3448 task.hold_unrequested_retry(Some(format!(
3449 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
3450 )));
3451 record(queue, task);
3452 return Vec::new();
3453 }
3454 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
3455 let detail = match e.downcast_ref::<crate::handover::Refused>() {
3456 Some(r) => (p.handover_refused)(r),
3457 None => format!("{e:#}"),
3458 };
3459 task.startup_notice_cause("startup-handover");
3460 let reason = format!("{start_failed}{detail}");
3461 task.last_error = Some(reason.clone());
3462 let branch = match &starter {
3465 Starter::Review(branch) => Some(branch.clone()),
3466 _ => None,
3467 };
3468 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
3472 record(queue, task);
3473 tracing::warn!(
3474 "holding {} for a branch it cannot take over: {e:#}",
3475 task.short()
3476 );
3477 return Vec::new();
3478 }
3479 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
3484 let d = e
3485 .downcast_ref::<crate::reconcile::Diverged>()
3486 .expect("checked by the guard");
3487 let mut q = ask::Question::new(
3488 task.id.clone(),
3489 crate::reconcile::NODE.to_owned(),
3490 crate::reconcile::SEAT.to_owned(),
3491 d.summary(),
3492 d.detail(),
3493 d.choices(),
3494 );
3495 match Questions::open().put(&mut q) {
3496 Ok(()) => {
3497 task.last_error = Some(format!("{e:#}"));
3498 task.review_branch = Some(d.branch.clone());
3501 task.review_instruction = Some(task.instruction.clone());
3502 task.block(vec![q.id.clone()], Some(d.summary()));
3503 }
3504 Err(put) => {
3505 tracing::warn!("could not file the divergence question: {put:#}");
3506 task.attempts += 1;
3507 task.fail_startup(
3508 "startup-question-store",
3509 format!("{start_failed}{e:#}"),
3510 opts.max_attempts,
3511 );
3512 }
3513 }
3514 record(queue, task);
3515 return Vec::new();
3516 }
3517 Err(e) => {
3518 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
3524 && let Starter::Review(branch) = &starter
3525 {
3526 task.review_branch = Some(branch.clone());
3527 task.review_instruction = Some(task.instruction.clone());
3528 task.last_error = Some(format!("{start_failed}{e:#}"));
3529 task.status = crate::queue::TaskStatus::Failed;
3530 record(queue, task);
3531 return Vec::new();
3532 }
3533 task.attempts += 1;
3534 task.last_error = Some(format!("{start_failed}{e:#}"));
3535 task.startup_notice_cause("startup-source");
3536 task.hold_unrequested_retry(Some(format!(
3537 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
3538 )));
3539 record(queue, task);
3540 return Vec::new();
3541 }
3542 };
3543 if let Some(replacement) = &task.replacement {
3544 if matches!(starter, Starter::Start) {
3545 runner.state.instruction.push_str(&format!(
3546 "\n\n# Replacement review context\n{}",
3547 replacement.context
3548 ));
3549 }
3550 runner.state.replaces = Some(replacement.clone());
3551 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
3552 if let Ok(mut old) = RunState::load(&replacement.run)
3553 && let Some(d) = &mut old.merge_decision
3554 {
3555 d.replacement_run = Some(runner.state.id.clone());
3556 old.event(
3557 "replacement",
3558 format!(
3559 "replacement run {} owns the next attempt; old PR remains closed",
3560 runner.state.id
3561 ),
3562 );
3563 if let Err(e) = old.save() {
3564 task.hold_manual(Some(format!(
3565 "Could not link replacement history: {e}; no agents dispatched"
3566 )));
3567 record(queue, task);
3568 return Vec::new();
3569 }
3570 }
3571 }
3572 runner.state.task_instruction = Some(task.instruction.clone());
3573 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
3576 if runner.state.origin_chat.is_none() {
3577 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
3578 }
3579 runner.on_pause(stop.pause());
3581 runner.watch_interrupt(interrupt_pause);
3585
3586 let run = runner.state.id.clone();
3589 task.start(run.clone());
3590 record(queue, task);
3591 lock(status).current.push(Current {
3592 task: task.id.clone(),
3593 run,
3594 });
3595
3596 let quota_before = runner.state.quota.clone();
3599 runner.state.next_pass_task_entry = true;
3600 let result = runner.execute().await;
3601 finish_attempt(
3602 opts.max_attempts,
3603 queue,
3604 task,
3605 &runner.state,
3606 "a_before,
3607 result,
3608 )
3609}
3610
3611pub fn finish_attempt(
3616 max_attempts: usize,
3617 queue: &Queue,
3618 task: &mut Task,
3619 state: &RunState,
3620 quota_before: &[QuotaLoss],
3621 result: Result<()>,
3622) -> Vec<QuotaLoss> {
3623 let detail = match result {
3624 Ok(()) => describe(state),
3625 Err(e) => format!("{e:#}"),
3626 };
3627 let fresh = losses_this_attempt(quota_before, &state.quota);
3628 let verdict = Verdict {
3629 status: state.status,
3630 left_pr: state.pr.is_some(),
3633 quota_hit: !fresh.is_empty(),
3639 parked: state.parked,
3643 no_viable_candidates: state.viable().is_empty(),
3646 };
3647 if let Some(replacement) = &state.replaces
3648 && let Some(pr) = &state.pr
3649 && let Ok(mut old) = RunState::load(&replacement.run)
3650 && let Some(d) = &mut old.merge_decision
3651 {
3652 d.replacement_pr = Some(pr.url.clone());
3653 let _ = old.save();
3654 }
3655 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
3656 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
3657 task.hold_unrequested_retry(Some(format!(
3658 "Candidate setup failed: {detail}. Select a complete implementation with `magi task recover-pr`, or correct the referenced source before explicitly releasing the task."
3659 )));
3660 }
3661 if task.status == TaskStatus::Done {
3662 supersede_prior_runs(task, &crate::run::home());
3663 }
3664 record(queue, task);
3665 tracing::info!(
3666 "task {} is {} after run {} ({})",
3667 task.short(),
3668 task.status.as_str(),
3669 state.short(),
3670 label(state.status)
3671 );
3672 fresh
3673}
3674
3675pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
3679 let waits_on_owner = withheld_text_wait(task).is_some();
3684 if task.status.runnable()
3685 && !waits_on_owner
3686 && !(task.fresh_start && task.replacement.is_some())
3687 {
3688 let why = task.last_error.clone().map_or_else(
3689 || "the run did not finish".to_owned(),
3690 |e| format!("the run did not finish: {e}"),
3691 );
3692 task.hold_unrequested_retry(Some(format!(
3693 "{why}. It was started by hand, so it is not retried \
3694 automatically; `magi task release` retries it."
3695 )));
3696 record(queue, task);
3697 }
3698}
3699
3700#[must_use]
3704pub fn loop_would_resume(run: &str) -> bool {
3705 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
3706}
3707
3708#[must_use]
3718pub fn foreign_loop(
3719 reading: Option<&Reading>,
3720 now: Timestamp,
3721 own_pid: u32,
3722) -> Option<Option<u32>> {
3723 let reading = reading.filter(|r| r.running(now))?;
3724 match reading.pid {
3725 Some(pid) if pid == own_pid => None,
3726 pid => Some(pid),
3727 }
3728}
3729
3730pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
3745 let status = Arc::new(Mutex::new(Status::new()));
3746 let stop = Stop::new();
3747 attempt(
3748 opts,
3749 queue,
3750 &status,
3751 &stop,
3752 crate::graph::Pause::new(),
3753 task,
3754 )
3755 .await;
3756 let mut resumes = 0;
3765 while resumes < MAX_WITHHELD_RESUMES {
3766 let Some(timeout) = withheld_text_wait(task) else {
3767 break;
3768 };
3769 eprintln!(
3770 "waiting for the owner's answer on a withheld pull request text \
3771 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
3772 );
3773 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
3776 let ready = loop {
3777 match land_resume_state(task) {
3778 LandResume::StillWaiting if std::time::Instant::now() < limit => {
3779 tokio::time::sleep(WITHHELD_POLL).await;
3780 }
3781 LandResume::Ready => break true,
3782 _ => break false,
3783 }
3784 };
3785 if !ready {
3786 break;
3787 }
3788 resumes += 1;
3789 attempt(
3790 opts,
3791 queue,
3792 &status,
3793 &stop,
3794 crate::graph::Pause::new(),
3795 task,
3796 )
3797 .await;
3798 }
3799 hold_if_runnable(queue, task);
3800}
3801
3802const MAX_WITHHELD_RESUMES: u32 = 3;
3805
3806const WITHHELD_POLL: Duration = Duration::from_secs(15);
3808
3809fn withheld_text_wait(task: &Task) -> Option<u64> {
3812 let id = task.runs.last()?;
3813 let s = RunState::load(id).ok()?;
3814 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
3815 .then_some(s.config.graph.answer_timeout)
3816}
3817
3818fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
3828 after
3829 .iter()
3830 .filter(|q| !before.contains(q))
3831 .cloned()
3832 .collect()
3833}
3834
3835fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
3838 if quota.is_empty() {
3839 return None;
3840 }
3841 let with_hint = quota.iter().find(|q| q.reset.is_some());
3842 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
3843 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
3844 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
3845 Some(
3846 now.checked_add(jiff::SignedDuration::from_secs(secs))
3847 .unwrap_or(Timestamp::MAX),
3848 )
3849}
3850
3851fn implementers_match(task: &Task, state: &RunState) -> bool {
3853 task.replacement.as_ref().map_or_else(
3854 || {
3855 task.implementers()
3856 .is_none_or(|n| n == state.config.graph.implementers)
3857 },
3858 |replacement| {
3859 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
3860 && state.config.graph.implementers == 1
3861 },
3862 )
3863}
3864
3865fn starter_with_implementers(
3867 task: &Task,
3868 previous: Option<&RunState>,
3869 proposed: Starter,
3870) -> Starter {
3871 if previous.is_some_and(|s| !implementers_match(task, s)) {
3872 task.review_of
3873 .as_ref()
3874 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
3875 } else {
3876 proposed
3877 }
3878}
3879
3880fn apply_task_implementers(config: &mut Config, task: &Task) {
3882 if let Some(n) = task.implementers() {
3883 config.graph.implementers = n;
3884 }
3885}
3886
3887fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
3892 let Some(o) = &task.overrides else {
3893 return prepare(repo, opts);
3894 };
3895 let own = Opts {
3899 config: o.config.clone(),
3900 merge: None,
3901 ..opts.clone()
3902 };
3903 let mut config = prepare(repo, if o.inherit_daemon_options { opts } else { &own })?;
3904 o.apply(&mut config);
3905 Ok(config)
3906}
3907
3908fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
3910 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
3911 if let Some(mode) = &opts.merge {
3912 config.merge.mode = merge_mode(mode)?;
3913 }
3914 Ok(config)
3915}
3916
3917async fn maybe_prune_cache_between_runs(
3948 repo: &Path,
3949 opts: &Opts,
3950 home: &Path,
3951 stop: &Stop,
3952 last_checked: &mut Option<Timestamp>,
3953 now: Timestamp,
3954) {
3955 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
3956 return;
3957 }
3958 *last_checked = Some(now);
3959 let cfg = match prepare(repo, opts) {
3960 Ok(cfg) => cfg,
3961 Err(e) => {
3962 tracing::warn!("cache check: no config: {e:#}");
3963 return;
3964 }
3965 };
3966 match clean::prune_cache_if_over_limit(&cfg, home) {
3967 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
3968 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
3969 pruned.files,
3970 pruned.freed
3971 ),
3972 Ok(_) => {}
3973 Err(e) => {
3974 tracing::warn!("housekeep: prune cache: {e:#}");
3975 notices::raise_in(
3976 home,
3977 Notice::warn(
3978 "housekeep:cache",
3979 "Pruning the shared build cache failed; disk usage may keep growing.",
3980 ),
3981 );
3982 }
3983 }
3984}
3985
3986fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
3990 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
3991}
3992
3993async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
4012 let cfg = match prepare(repo, opts) {
4013 Ok(cfg) => cfg,
4014 Err(e) => {
4015 tracing::warn!("housekeep: no config: {e:#}");
4016 return;
4017 }
4018 };
4019 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
4028 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
4029 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
4034 let mut extra = Vec::new();
4035 if out.unreadable > 0 {
4036 extra.push(format!("{} unreadable", out.unreadable));
4037 }
4038 if out.orphaned_worktrees > 0 {
4039 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
4040 }
4041 let detail = if extra.is_empty() {
4042 String::new()
4043 } else {
4044 format!(" ({})", extra.join(", "))
4045 };
4046 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
4047 }
4048 if out.external_merges_recorded > 0 {
4049 tracing::info!(
4050 "housekeep: recorded {} run(s) as merged externally",
4051 out.external_merges_recorded
4052 );
4053 }
4054 if out.stale_pr_states_repaired > 0 {
4055 tracing::info!(
4056 "housekeep: rewrote {} run record(s) whose pull request had already settled",
4057 out.stale_pr_states_repaired
4058 );
4059 }
4060 if out.cache_files > 0 {
4061 tracing::info!(
4062 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
4063 out.cache_files,
4064 out.cache_freed
4065 );
4066 }
4067 if out.questions_abandoned > 0 {
4068 tracing::info!(
4069 "housekeep: abandoned {} question(s) left open by a finished run",
4070 out.questions_abandoned
4071 );
4072 }
4073}
4074
4075async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
4084 let questions = Questions::at(home.join("questions"));
4085 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
4086 if report.is_empty() {
4087 return;
4088 }
4089 if !report.quarantined.is_empty() {
4090 tracing::info!(
4091 "triage: held {} blocked task(s) whose blocked-on task or \
4092 question no longer exists: {}",
4093 report.quarantined.len(),
4094 report.quarantined.join(", ")
4095 );
4096 }
4097 if !report.resumed.is_empty() {
4098 tracing::info!(
4099 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
4100 report.resumed.len(),
4101 report.resumed.join(", ")
4102 );
4103 }
4104 if !report.asked.is_empty() {
4105 tracing::info!(
4106 "triage: asked about {} held task(s): {}",
4107 report.asked.len(),
4108 report.asked.join(", ")
4109 );
4110 }
4111 if !report.answered.is_empty() {
4112 tracing::info!(
4113 "triage: applied {} operator answer(s): {}",
4114 report.answered.len(),
4115 report.answered.join(", ")
4116 );
4117 }
4118}
4119
4120fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
4127 disk_gate_with(repo, config, crate::disk::free_bytes)
4128}
4129
4130fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
4134 repo: &Path,
4135 config: &Config,
4136 free_bytes: F,
4137) -> Option<String> {
4138 let min = config.disk.min_free_bytes;
4139 if min == 0 {
4140 return None;
4141 }
4142 match free_bytes(repo) {
4143 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
4144 Err(e) => Some(crate::disk::unmeasured_in(repo, &e, &config.graph.language)),
4145 }
4146}
4147
4148const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
4155
4156const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
4160
4161fn quota_wait(
4170 reset_at: Option<Timestamp>,
4171 now: Timestamp,
4172 fallback: Duration,
4173 cap: Duration,
4174) -> Duration {
4175 match reset_at {
4176 Some(at) if at > now => {
4177 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
4178 Duration::from_secs(secs).min(cap)
4179 }
4180 _ => fallback,
4181 }
4182}
4183
4184fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
4198 parse_reset_hint_zoned(text, now)
4199 .or_else(|| parse_reset_hint_dated(text))
4200 .or_else(|| parse_reset_hint_relative(text, recorded))
4201}
4202
4203fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
4207 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
4208 let mut rest = rest.trim();
4209 if rest.is_empty() {
4210 return None;
4211 }
4212 let mut total: i64 = 0;
4213 let mut matched = false;
4214 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
4215 if let Some((digits, tail)) = rest.split_once(unit)
4216 && !digits.is_empty()
4217 && digits.bytes().all(|b| b.is_ascii_digit())
4218 {
4219 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
4220 rest = tail;
4221 matched = true;
4222 }
4223 }
4224 if !rest.is_empty() || !matched {
4225 return None;
4226 }
4227 recorded
4228 .checked_add(jiff::SignedDuration::from_secs(total))
4229 .ok()
4230}
4231
4232fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
4236 let clock = clock.trim().to_lowercase();
4237 let (digits, pm) = clock
4238 .strip_suffix("am")
4239 .map(|d| (d, false))
4240 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
4241 let (h, m) = digits.trim().split_once(':')?;
4242 let mut hour: i8 = h.trim().parse().ok()?;
4243 let minute: i8 = m.trim().parse().ok()?;
4244 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
4245 return None;
4246 }
4247 if pm && hour != 12 {
4248 hour += 12;
4249 } else if !pm && hour == 12 {
4250 hour = 0;
4251 }
4252 Some((hour, minute))
4253}
4254
4255fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
4260 let open = text.find('(')?;
4261 let close = text.rfind(')')?;
4262 if close <= open {
4263 return None;
4264 }
4265 let zone = text[open + 1..close].trim();
4266 let (hour, minute) = parse_12h_clock(&text[..open])?;
4267 let tz = jiff::tz::TimeZone::get(zone).ok()?;
4268 let candidate = now
4269 .to_zoned(tz)
4270 .with()
4271 .hour(hour)
4272 .minute(minute)
4273 .second(0)
4274 .millisecond(0)
4275 .microsecond(0)
4276 .nanosecond(0)
4277 .build()
4278 .ok()?;
4279 let mut at = candidate.timestamp();
4280 if at <= now {
4281 at += jiff::SignedDuration::from_hours(24);
4282 }
4283 Some(at)
4284}
4285
4286fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
4295 let words: Vec<&str> = text.split_whitespace().collect();
4296 if words.len() < 5 {
4297 return None;
4298 }
4299 (0..=words.len() - 5)
4300 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
4301}
4302
4303fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
4309 if trailing.is_some_and(|next| next.starts_with('(')) {
4310 return None;
4311 }
4312 let month = month_number(window[0])?;
4313 let day_token = window[1].strip_suffix(',')?.to_lowercase();
4314 let day_digits = ["st", "nd", "rd", "th"]
4315 .iter()
4316 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
4317 let day: i8 = day_digits.parse().ok()?;
4318 let year_token = window[2];
4319 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
4320 return None;
4321 }
4322 let year: i16 = year_token.parse().ok()?;
4323 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
4327 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
4328 let date = jiff::civil::Date::new(year, month, day).ok()?;
4329 let candidate = date
4330 .at(hour, minute, 0, 0)
4331 .to_zoned(jiff::tz::TimeZone::UTC)
4332 .ok()?;
4333 Some(candidate.timestamp())
4334}
4335
4336fn month_number(name: &str) -> Option<i8> {
4339 const NAMES: [&str; 12] = [
4340 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
4341 ];
4342 let lower = name.to_lowercase();
4343 NAMES
4344 .iter()
4345 .position(|n| *n == lower.as_str())
4346 .map(|i| i as i8 + 1)
4347}
4348
4349fn exhausted_review_budget(state: &RunState) -> bool {
4361 state.status == RunStatus::Blocked
4364 && !state.pr_publication_pending
4365 && state.reviews.len() >= state.config.graph.review_rounds
4366}
4367
4368fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
4405 unfinished_run_with(runs, short, RunState::load)
4406}
4407
4408fn unfinished_run_with<F>(runs: &[String], short: &str, load: F) -> Option<String>
4411where
4412 F: FnOnce(&str) -> Result<RunState>,
4413{
4414 let id = runs.last()?;
4415 match load(id) {
4416 Ok(s)
4421 if s.status.resumable()
4422 && !s.released()
4423 && !exhausted_review_budget(&s)
4424 && s.liveness(false) != crate::run::Liveness::Live =>
4425 {
4426 Some(id.clone())
4427 }
4428 Ok(_) => None,
4429 Err(e) => {
4430 tracing::warn!("could not read run {id} for task {short}: {e:#}");
4431 None
4432 }
4433 }
4434}
4435
4436fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
4444where
4445 F: FnOnce(&str) -> Result<RunState>,
4446{
4447 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
4449 || task.fresh_start
4450 || task.review_branch.is_some()
4451 {
4452 return false;
4453 }
4454 let Some(id) = task.runs.last() else {
4455 return false;
4456 };
4457 load(id).is_ok_and(|s| {
4458 s.parked && s.status.resumable() && !s.released() && !exhausted_review_budget(&s)
4459 })
4460}
4461
4462#[derive(Debug, Clone, PartialEq, Eq)]
4465enum Starter {
4466 Review(String),
4469 Resume(String),
4471 Start,
4473}
4474
4475fn selected_starter(
4477 selected: &crate::pr_recovery::Selection,
4478 previous: Option<&RunState>,
4479 unfinished: Option<&str>,
4480) -> Starter {
4481 if let Some(id) = unfinished
4482 && previous.is_some_and(|r| {
4483 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
4484 })
4485 {
4486 Starter::Resume(id.to_owned())
4487 } else {
4488 Starter::Review(selected.branch.clone())
4489 }
4490}
4491
4492fn take_divergence_answer(
4496 branch: &str,
4497 remote: &str,
4498 task: &mut Task,
4499) -> Option<crate::reconcile::Choice> {
4500 let summary = crate::reconcile::summary_for(branch, remote);
4501 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
4502 (a.question == summary)
4503 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
4504 .flatten()
4505 .map(|c| (i, c))
4506 })?;
4507 task.answers.remove(idx);
4508 Some(choice)
4509}
4510
4511fn choose_starter(
4523 review_branch: Option<&str>,
4524 branch_exists: bool,
4525 unfinished: Option<&str>,
4526) -> Starter {
4527 match review_branch {
4528 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
4529 Some(_) => Starter::Start,
4530 None => match unfinished {
4531 Some(id) => Starter::Resume(id.to_owned()),
4532 None => Starter::Start,
4533 },
4534 }
4535}
4536
4537fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
4539 let branch = task.review_branch.take();
4540 let instruction = task.review_instruction.take();
4541 branch.filter(|branch| match &instruction {
4542 Some(authored) => *authored == task.instruction,
4543 None => previous.map_or_else(
4546 || task.review_of.as_ref() == Some(branch),
4547 |run| compatible_run_instruction(run, task),
4548 ),
4549 })
4550}
4551
4552fn may_resume_run(run: &RunState, task: &Task) -> bool {
4555 task.resume_override
4556 .as_ref()
4557 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
4558 || compatible_run_instruction(run, task)
4559}
4560
4561pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
4563 if let Some(authored) = &run.task_instruction {
4564 return compatible_instruction(authored, task);
4565 }
4566 if compatible_instruction(&run.instruction, task) {
4567 return true;
4568 }
4569 if run.reviewed_commits.is_some() {
4572 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
4573 return strip_answers_block(authored, task).trim() == task.instruction.trim();
4574 }
4575 if let Some(branch) = &task.review_of {
4576 let canonical = format!(
4577 "Review the branch `{branch}` (review-only: no implementation, no judging)."
4578 );
4579 let generated = format!(
4580 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
4581 );
4582 return task.instruction == canonical && run.instruction.starts_with(&generated);
4583 }
4584 }
4585 false
4586}
4587
4588pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
4590 strip_answers_block(old, task) == task.instruction
4591}
4592
4593fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
4596 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
4597 return fallback.to_path_buf();
4598 }
4599 task.repo.clone()
4600}
4601
4602const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
4606
4607fn answers_block(task: &Task, count: usize) -> String {
4609 let mut s = ANSWERS_HEADER.to_owned();
4610 for a in &task.answers[..count] {
4611 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
4612 }
4613 s
4614}
4615
4616fn resumption_block(r: &crate::queue::Resumption) -> String {
4619 format!(
4620 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
4621 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
4622 r.generation, r.at, r.instruction, r.answers_through,
4623 )
4624}
4625
4626fn append_answers(base: &str, task: &Task) -> String {
4628 let mut s = base.to_owned();
4629 if !task.answers.is_empty() {
4630 s.push_str(&answers_block(task, task.answers.len()));
4631 }
4632 if let Some(r) = &task.resumption {
4633 s.push_str(&resumption_block(r));
4634 }
4635 s
4636}
4637
4638fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
4642 let mut instruction = instruction;
4643 for approval in task
4644 .resumption
4645 .iter()
4646 .chain(task.resumption_history.iter().rev())
4647 {
4648 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
4649 instruction = base;
4650 break;
4651 }
4652 }
4653 for count in (1..=task.answers.len()).rev() {
4654 let block = answers_block(task, count);
4655 if let Some(base) = instruction.strip_suffix(&block) {
4656 return base;
4657 }
4658 }
4659 instruction
4660}
4661
4662fn instruction_for(task: &Task) -> String {
4670 append_answers(&task.instruction, task)
4671}
4672
4673fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
4685 append_answers(strip_answers_block(old_instruction, task), task)
4686}
4687
4688fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
4690 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
4691 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
4692 if let Some(cause) = task.hold_notice_cause.as_mut() {
4693 cause.target = failure.map(|f| f.target.clone());
4694 }
4695}
4696
4697#[derive(Debug, thiserror::Error)]
4698#[error("attachment `{name}` is recorded on the task but {path} is missing")]
4699struct MissingAttachment {
4700 name: String,
4701 path: PathBuf,
4702}
4703
4704fn task_attachments(
4706 queue: &Queue,
4707 task: &Task,
4708) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
4709 let paths = queue.attachment_paths(task);
4710 for (name, path) in task.attachments.iter().zip(&paths) {
4711 if !path.is_file() {
4712 return Err(MissingAttachment {
4713 name: name.clone(),
4714 path: path.clone(),
4715 });
4716 }
4717 }
4718 Ok(paths)
4719}
4720
4721fn prepare_instruction(
4732 starter: &Starter,
4733 old_instruction: Option<&str>,
4734 task: &Task,
4735) -> Option<String> {
4736 match starter {
4737 Starter::Start => Some(instruction_for(task)),
4738 Starter::Resume(_) => Some(resumed_instruction(
4739 old_instruction.expect("a resumed run always has a prior instruction"),
4740 task,
4741 )),
4742 Starter::Review(_) => None,
4743 }
4744}
4745
4746fn record(queue: &Queue, task: &mut Task) {
4750 if let Err(e) = queue.put(task) {
4751 tracing::error!("could not record task {}: {e:#}", task.short());
4752 let notice = Notice::error(
4753 "loop:record",
4754 "The loop could not save a task's state; check the disk.",
4755 )
4756 .link(Link::Task {
4757 id: task.id.clone(),
4758 });
4759 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
4761 Some(home) => notices::raise_in(home, notice),
4762 None => notices::raise(notice),
4763 }
4764 }
4765}
4766
4767fn runnable(queue: &Queue) -> Vec<Task> {
4773 let mut tasks: Vec<Task> = queue
4774 .list()
4775 .into_iter()
4776 .filter(|t| t.status.runnable())
4777 .collect();
4778 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)));
4779 tasks
4780}
4781
4782fn describe(state: &RunState) -> String {
4796 let p = phrases(&state.config.graph.language);
4797 let mut detail = if state.status == RunStatus::Stalled {
4798 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
4799 seats.sort_unstable();
4800 seats.dedup();
4801 if seats.is_empty() {
4802 p.quorum_lost.to_owned()
4803 } else {
4804 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
4805 }
4806 } else {
4807 format!("{}{}", p.run_ended, state.status.display_label())
4808 };
4809 if let Some(last) = state.events.last() {
4810 let unanswered = state
4814 .reviews
4815 .last()
4816 .filter(|r| {
4817 state.status == RunStatus::Blocked
4818 && last.node == "review"
4819 && r.incomplete()
4820 && r.blocking == 0
4821 && r.round == state.config.graph.review_rounds
4822 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
4823 })
4824 .map(|r| (r.expected - r.answered, r.round));
4825 match unanswered {
4826 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
4827 detail.push_str(&format!(
4828 " ({}: {})",
4829 last.node,
4830 (p.reviewers_never_answered)(missing, rounds)
4831 ));
4832 }
4833 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
4834 }
4835 }
4836 detail.push_str(&format!(" [run {}]", state.id));
4837 detail
4838}
4839
4840const DIAGNOSTIC_MAX: usize = 4_000;
4846
4847const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
4852
4853fn diagnostic(state: &RunState) -> Option<String> {
4867 let mut parts: Vec<String> = Vec::new();
4868
4869 for o in state.gate.iter().filter(|o| !o.ok()) {
4871 parts.push(format!(
4872 "gate `{}` failed ({:?}):\n{}",
4873 o.command,
4874 o.code,
4875 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
4876 ));
4877 }
4878
4879 if let Some(last) = state
4882 .events
4883 .iter()
4884 .rev()
4885 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
4886 {
4887 parts.push(last.message.clone());
4888 }
4889
4890 if state.viable().is_empty() {
4897 for c in &state.candidates {
4898 if let Some(evidence) = &c.verified_noop {
4899 parts.push(format!(
4900 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
4901 c.label
4902 ));
4903 } else if !c.summary.trim().is_empty() {
4904 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
4905 } else if let Some(why) = &c.failed {
4906 parts.push(format!("candidate {}: {why}", c.label));
4907 }
4908 }
4909 }
4910
4911 if parts.is_empty() {
4912 return None;
4913 }
4914 Some(crate::run::tail(
4919 &parts.join("\n\n"),
4920 DIAGNOSTIC_MAX.saturating_sub(100),
4921 ))
4922}
4923
4924fn label(status: RunStatus) -> &'static str {
4932 status.as_str()
4933}
4934
4935pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
4937 match mode {
4938 "none" => Ok(MergeMode::None),
4939 "local" => Ok(MergeMode::Local),
4940 "pr" => Ok(MergeMode::Pr),
4941 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
4942 }
4943}
4944
4945fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
4950 mutex
4951 .lock()
4952 .unwrap_or_else(std::sync::PoisonError::into_inner)
4953}
4954
4955#[cfg(test)]
4956mod tests {
4957 use super::*;
4958 use crate::queue::{Source, TaskStatus};
4959 use crate::run::{Candidate, CommandOutcome};
4960 use pretty_assertions::assert_eq;
4961
4962 fn task() -> Task {
4963 Task::new(
4964 "add retries".to_owned(),
4965 "add retries".to_owned(),
4966 PathBuf::from("/repo"),
4967 Source::Human,
4968 )
4969 }
4970
4971 #[test]
4972 fn selected_public_head_overrides_historical_resume_and_pin() {
4973 let mut t = task();
4974 let mut old = run_state(RunStatus::Blocked);
4975 let selection = crate::pr_recovery::Selection {
4976 url: "url".into(),
4977 branch: format!("magi/task-{}", t.id),
4978 head: "a".repeat(40),
4979 instruction: t.instruction.clone(),
4980 approved_at: Timestamp::now(),
4981 };
4982 t.resume_override = Some(crate::queue::OperatorResume {
4983 pinned_run: Some(old.id.clone()),
4984 question_id: "choice".into(),
4985 at: Timestamp::now(),
4986 conductor_rehold: None,
4987 forced: false,
4988 });
4989 assert_eq!(
4990 selected_starter(&selection, Some(&old), Some(&old.id)),
4991 Starter::Review(selection.branch.clone())
4992 );
4993 old.pr_recovery = Some(selection.clone());
4994 assert_eq!(
4995 selected_starter(&selection, Some(&old), Some(&old.id)),
4996 Starter::Review(selection.branch.clone())
4997 );
4998 old.reviewed_commits = Some(vec!["implementation".into()]);
4999 assert_eq!(
5000 selected_starter(&selection, Some(&old), Some(&old.id)),
5001 Starter::Resume(old.id.clone())
5002 );
5003 let mut changed = selection.clone();
5004 changed.head = "b".repeat(40);
5005 assert_eq!(
5006 selected_starter(&changed, Some(&old), Some(&old.id)),
5007 Starter::Review(changed.branch)
5008 );
5009 }
5010
5011 #[test]
5012 fn candidate_transplant_failure_is_one_actionable_hold() {
5013 let dir = tempfile::tempdir().unwrap();
5014 let q = Queue::at(dir.path().join("queue"));
5015 let mut t = task();
5016 let mut state = run_state(RunStatus::Blocked);
5017 state.event("prep", "cannot apply referenced commit: conflict");
5018 state.prep_failure = Some("conflict".into());
5019 t.start(state.id.clone());
5020 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
5021 assert_eq!(t.attempts, 1);
5022 assert_eq!(t.status, TaskStatus::Held);
5023 assert!(t.operator_held());
5024 assert!(!t.status.runnable());
5025 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
5026 let first = t.chat_notice_key().unwrap();
5027 t.chat_notices.push(first.clone());
5028 t.release_operator("test", None);
5029 state.id = "another-unprepared-run".into();
5030 t.start(state.id.clone());
5031 finish_attempt(
5032 3,
5033 &q,
5034 &mut t,
5035 &state,
5036 &[],
5037 Err(anyhow::anyhow!("still conflicts")),
5038 );
5039 assert_eq!(
5040 t.chat_notice_key().as_ref(),
5041 Some(&first),
5042 "no work happened"
5043 );
5044 t.release_operator("test", None);
5045 state.prep_failure = None;
5046 state.status = RunStatus::Failed;
5047 state.begin_pass();
5048 t.start(state.id.clone());
5049 finish_attempt(
5050 1,
5051 &q,
5052 &mut t,
5053 &state,
5054 &[],
5055 Err(anyhow::anyhow!("new gate failure")),
5056 );
5057 let failure = t.chat_notice_key().unwrap();
5058 assert_ne!(failure, first, "approved recovery reached a real failure");
5059 t.release_operator("test", None);
5060 state.begin_pass();
5061 t.start(state.id.clone());
5062 finish_attempt(
5063 1,
5064 &q,
5065 &mut t,
5066 &state,
5067 &[],
5068 Err(anyhow::anyhow!("new failure on resume")),
5069 );
5070 assert_ne!(
5071 t.chat_notice_key().as_ref(),
5072 Some(&failure),
5073 "a new settled pass"
5074 );
5075 }
5076
5077 #[test]
5078 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
5079 for source in ["cli", "web"] {
5080 let mut t = task();
5081 let mut run = run_state(RunStatus::Blocked);
5082 run.instruction = "Run A original criteria".into();
5083 run.task_instruction = Some(run.instruction.clone());
5084 t.start(run.id.clone());
5085 t.release_operator("choice", Some("resume-question"));
5086 t.resume_override = Some(crate::queue::OperatorResume {
5087 question_id: "resume-question".into(),
5088 at: Timestamp::now(),
5089 conductor_rehold: None,
5090 forced: true,
5091 pinned_run: Some(run.id.clone()),
5092 });
5093 t.hold_machine(Some("not enough free space to start a run".into()));
5094 let mut unchanged = t.clone();
5095 unchanged
5096 .edit("title changed".into(), unchanged.instruction.clone())
5097 .unwrap();
5098 unchanged.release_operator(source, None);
5099 assert_eq!(
5100 unchanged
5101 .resume_override
5102 .as_ref()
5103 .unwrap()
5104 .pinned_run
5105 .as_deref(),
5106 Some(run.id.as_str())
5107 );
5108 assert!(!compatible_run_instruction(&run, &unchanged));
5109 assert!(may_resume_run(&run, &unchanged));
5110
5111 t.edit(
5112 "approved recovery".into(),
5113 "Reuse measured PR head B; fix Clippy; review".into(),
5114 )
5115 .unwrap();
5116 assert!(t.resume_override.is_none());
5117 t.release_operator(source, None);
5118 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
5119 assert!(reloaded.resume_override.is_none());
5120 assert!(!may_resume_run(&run, &reloaded));
5121 let unfinished =
5122 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
5123 .filter(|_| may_resume_run(&run, &reloaded));
5124 assert_eq!(
5125 choose_starter(None, false, unfinished.as_deref()),
5126 Starter::Start
5127 );
5128 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
5129 assert_eq!(
5130 reloaded.resumption_history[0].question_id.as_deref(),
5131 Some("resume-question")
5132 );
5133
5134 unchanged.instruction = "Reuse measured PR head B".into();
5137 unchanged.hold_machine(Some("disk hold".into()));
5138 unchanged.release_operator(source, None);
5139 assert!(unchanged.resume_override.is_none());
5140 assert!(!may_resume_run(&run, &unchanged));
5141 }
5142 }
5143
5144 #[test]
5145 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
5146 let mut t = task();
5147 let mut run = run_state(RunStatus::Blocked);
5148 t.review_of = Some("approved-head".into());
5149 t.instruction =
5150 "Review the branch `approved-head` (review-only: no implementation, no judging)."
5151 .into();
5152 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
5153 run.reviewed_commits = Some(vec!["Commit log".into()]);
5154 t.start(run.id.clone());
5155 t.hold_machine(Some("interrupted review".into()));
5156 t.release_operator("cli", None);
5157 assert!(
5158 compatible_run_instruction(&run, &t),
5159 "legacy explicit review remains resumable"
5160 );
5161 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
5162 .filter(|_| compatible_run_instruction(&run, &t));
5163 assert_eq!(
5164 choose_starter(None, false, unfinished.as_deref()),
5165 Starter::Resume(run.id.clone())
5166 );
5167 run.task_instruction = Some(t.instruction.clone());
5168 assert!(compatible_run_instruction(&run, &t));
5169 let reloaded: RunState =
5170 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
5171 assert!(compatible_run_instruction(&reloaded, &t));
5172 t.edit(
5173 "new approved head".into(),
5174 "Reuse measured PR head B".into(),
5175 )
5176 .unwrap();
5177 assert!(!compatible_run_instruction(&run, &t));
5178
5179 t.review_of = None;
5181 t.instruction = "Fix Clippy on approved head".into();
5182 run.task_instruction = None;
5183 run.instruction = format!(
5184 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
5185 t.instruction
5186 );
5187 assert!(compatible_run_instruction(&run, &t));
5188 t.instruction.push('\n');
5189 assert!(
5190 compatible_run_instruction(&run, &t),
5191 "legacy generated review trimmed the original criteria"
5192 );
5193 t.hold_for_handover(Some("approved-head".into()), "refused".into());
5194 let mut unchanged = t.clone();
5195 assert_eq!(
5196 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
5197 Some("approved-head")
5198 );
5199 t.edit("new plan".into(), "Reuse measured PR head B".into())
5200 .unwrap();
5201 assert!(take_review_branch(&mut t, Some(&run)).is_none());
5202 assert!(!compatible_run_instruction(&run, &t));
5203 t.review_branch = Some("approved-head".into());
5204 t.review_instruction = None;
5205 assert!(
5206 take_review_branch(&mut t, Some(&run)).is_none(),
5207 "legacy retained candidate cannot replace a revised approved head"
5208 );
5209 }
5210
5211 #[test]
5212 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
5213 crate::run::pin_test_home();
5214 let dir = tempfile::tempdir().unwrap();
5215 let queue = Queue::at(dir.path().join("queue"));
5216 for resume in [true, false] {
5217 let state = run_state(RunStatus::Blocked);
5218 let mut t = task();
5219 t.start(state.id.clone());
5220 let action = if resume {
5221 ask::ChoiceAction::Resume {
5222 run: state.id.clone(),
5223 }
5224 } else {
5225 ask::ChoiceAction::Requeue
5226 };
5227 let mut q = action_question(&state.id, action);
5228 q.status = ask::QuestionStatus::Open;
5229 q.answer = None;
5230 q.answered_at = None;
5231 t.fail("run stopped with a question", 3);
5232 hold_if_runnable(&queue, &mut t);
5233 assert!(t.operator_decided_at.is_none());
5234 assert!(t.operator_held());
5235 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
5236 let expected = if resume {
5237 ActionDecision::Resume(state.id.clone())
5238 } else {
5239 ActionDecision::Requeue
5240 };
5241 assert_eq!(
5242 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5243 expected
5244 );
5245 t.hold_manual(Some("later explicit hold".into()));
5246 assert_eq!(
5247 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5248 ActionDecision::Stale
5249 );
5250 }
5251 }
5252
5253 #[test]
5254 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
5255 let mut t = task();
5256 t.answers.push(crate::queue::AnsweredQuestion {
5257 question: "Inspect the remote?".into(),
5258 answer: "Inspect then wait".into(),
5259 });
5260 let old = instruction_for(&t);
5261 assert!(compatible_instruction(&old, &t));
5262 t.instruction = "Reuse approved PR head and review".into();
5263 t.release_operator("cli", None);
5264 assert!(!compatible_instruction(&old, &t));
5265 let approved = instruction_for(&t);
5266 assert!(approved.contains("waiting premises are superseded"));
5267 assert!(compatible_instruction(&approved, &t));
5268 assert_eq!(resumed_instruction(&approved, &t), approved);
5269 t.release_operator("web", None);
5270 assert!(compatible_instruction(&approved, &t));
5271 assert_eq!(t.resumption_history.len(), 1);
5272 t.start("run".into());
5273 assert!(instruction_for(&t).contains("Explicit operator resumption"));
5274 }
5275
5276 #[test]
5280 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
5281 let mut state = RunState::new(
5282 PathBuf::from("/repo"),
5283 "main".to_owned(),
5284 "abc1234".to_owned(),
5285 "task".to_owned(),
5286 Config::default(),
5287 );
5288 state.status = RunStatus::Blocked;
5289 state.resource_block = Some(crate::run::ResourceBlock {
5290 node: "graft".to_owned(),
5291 kind: "shared build cache".to_owned(),
5292 detail: "timed out".to_owned(),
5293 at: jiff::Timestamp::now(),
5294 });
5295 let mut t = task();
5296 t.start("20260902-000000-aaaa".to_owned());
5297 assert_eq!(t.attempts, 1);
5298 let verdict = Verdict {
5299 status: RunStatus::Blocked,
5300 left_pr: false,
5301 quota_hit: false,
5302 parked: false,
5303 no_viable_candidates: false,
5304 };
5305 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
5306 assert_eq!(t.status, TaskStatus::Held);
5307 assert_eq!(t.attempts, 0, "the attempt is refunded");
5308 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
5309 assert!(
5310 !t.operator_held(),
5311 "a machine hold, released without a human"
5312 );
5313
5314 let mut p = task();
5316 p.start("20260902-000000-bbbb".to_owned());
5317 settle_and_diagnose(
5318 &mut p,
5319 Verdict {
5320 parked: true,
5321 ..verdict
5322 },
5323 "parked",
5324 2,
5325 &state,
5326 );
5327 assert_eq!(p.status, TaskStatus::Parked);
5328 }
5329
5330 #[test]
5331 fn record_failure_notice_links_the_task() {
5332 let dir = tempfile::tempdir().unwrap();
5333 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
5336 let queue = Queue::at(dir.path().join("queue"));
5337 let mut t = interrupt_task("rec-fail-1");
5338 record(&queue, &mut t);
5339
5340 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
5341 assert_eq!(found.len(), 1);
5342 assert_eq!(found[0].key, "loop:record");
5343 assert_eq!(
5344 found[0].link,
5345 Some(Link::Task {
5346 id: "rec-fail-1".to_owned()
5347 })
5348 );
5349 }
5350
5351 fn interrupt_task(id: &str) -> Task {
5354 let mut t = task();
5355 t.id = id.to_owned();
5356 t.interrupt = true;
5357 t
5358 }
5359
5360 fn task_with_id(id: &str) -> Task {
5362 let mut t = task();
5363 t.id = id.to_owned();
5364 t
5365 }
5366
5367 fn urgent_task(id: &str) -> Task {
5369 let mut t = task();
5370 t.id = id.to_owned();
5371 t.urgent = true;
5372 t
5373 }
5374
5375 #[test]
5376 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
5377 let dir = tempfile::tempdir().unwrap();
5378 let queue = Queue::at(dir.path().join("queue"));
5379 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5380 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5381 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
5382 let mut first = task();
5383 queue.put(&mut first).unwrap();
5384 let stale_plain = queue.get(&first.id).unwrap();
5385 first.set_urgent(true).unwrap();
5386 queue.put(&mut first).unwrap();
5387 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
5388 let mut second = urgent_task("second");
5389 queue.put(&mut second).unwrap();
5390 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
5391 drop(queue.claim(&second.id).unwrap());
5392 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
5393 drop(dispatched);
5394 let stale_urgent = queue.get(&first.id).unwrap();
5395 first.set_urgent(false).unwrap();
5396 queue.put(&mut first).unwrap();
5397 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
5398 assert_eq!(urgent.available_permits(), 1);
5399 drop(queue.claim(&first.id).unwrap());
5400 drop(occupied);
5401 }
5402
5403 #[test]
5404 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
5405 let dir = tempfile::tempdir().unwrap();
5406 let queue = Queue::at(dir.path().join("queue"));
5407 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5408 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5409 let mut t = task();
5410 queue.put(&mut t).unwrap();
5411 for enabled in [true, false] {
5412 let stale = queue.get(&t.id).unwrap();
5413 t.set_urgent(enabled).unwrap();
5414 queue.put(&mut t).unwrap();
5415 assert_ne!(stale.urgent, enabled);
5416 let DispatchClaim {
5417 task: fresh,
5418 claim,
5419 permit,
5420 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
5421 assert_eq!(fresh.urgent, enabled);
5422 assert_eq!(ordinary.available_permits(), usize::from(enabled));
5423 assert_eq!(urgent.available_permits(), usize::from(!enabled));
5424 assert!(queue.claim(&t.id).is_err());
5425 assert_eq!(fresh.attempts, stale.attempts);
5426 drop((claim, permit));
5427 assert_eq!(ordinary.available_permits(), 1);
5428 assert_eq!(urgent.available_permits(), 1);
5429 }
5430 t.set_urgent(true).unwrap();
5431 queue.put(&mut t).unwrap();
5432 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
5433 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5434 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
5435 drop(queue.claim(&t.id).unwrap());
5436 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
5437 drop(occupied);
5438 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
5439 drop(queue.claim(&t.id).unwrap());
5440 let held = queue.claim(&t.id).unwrap();
5441 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5442 drop(held);
5443 t.status = TaskStatus::Held;
5444 queue.put(&mut t).unwrap();
5445 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5446 drop(queue.claim(&t.id).unwrap());
5447 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
5448 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5449 drop(queue.claim(&t.id).unwrap());
5450 assert_eq!(urgent.available_permits(), 1);
5451 }
5452
5453 #[test]
5458 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
5459 assert_eq!(permit_kind(true, false), PermitKind::None);
5460 assert_eq!(permit_kind(true, true), PermitKind::None);
5461 }
5462
5463 #[test]
5468 fn permit_kind_separates_urgent_from_ordinary() {
5469 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
5470 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
5471 }
5472
5473 #[test]
5480 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
5481 let cfg = Config::default();
5482 let repo = Path::new("/any/repo/path");
5483
5484 let reason =
5485 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
5486 assert!(reason.contains("1024"), "{reason}");
5487 assert!(
5488 reason.contains(&cfg.disk.min_free_bytes.to_string()),
5489 "{reason}"
5490 );
5491
5492 assert_eq!(
5493 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
5494 None,
5495 "exactly at the floor is open"
5496 );
5497 assert_eq!(
5498 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
5499 None,
5500 "comfortably above the floor is open"
5501 );
5502 }
5503
5504 #[test]
5505 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
5506 let mut cfg = Config::default();
5507 cfg.disk.min_free_bytes = 0;
5508 let repo = Path::new("/any/repo/path");
5509 assert_eq!(
5510 disk_gate_with(repo, &cfg, |_| Ok(0)),
5511 None,
5512 "a zero floor never measures at all"
5513 );
5514 }
5515
5516 #[test]
5517 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
5518 let cfg = Config::default();
5519 let repo = Path::new("/any/repo/path");
5520 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
5521 .expect("a measurement failure must close the gate, not open it");
5522 assert!(reason.contains("could not measure"), "{reason}");
5523 }
5524
5525 #[test]
5526 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
5527 let ordinary = task();
5528 let next = advance_interrupt(
5529 Interrupt::Idle,
5530 std::slice::from_ref(&ordinary.id),
5531 std::slice::from_ref(&ordinary),
5532 );
5533 assert_eq!(next, Interrupt::Idle);
5534 }
5535
5536 #[test]
5537 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
5538 let marked = interrupt_task("marked");
5541 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5542 assert_eq!(next, Interrupt::Idle);
5543 }
5544
5545 #[test]
5546 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
5547 let marked = interrupt_task("marked");
5548 let next = advance_interrupt(
5549 Interrupt::Idle,
5550 &["running".to_owned()],
5551 std::slice::from_ref(&marked),
5552 );
5553 assert_eq!(
5554 next,
5555 Interrupt::Parking {
5556 parked: vec!["running".to_owned()],
5557 interrupt_task: "marked".to_owned(),
5558 }
5559 );
5560 }
5561
5562 #[test]
5571 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
5572 let marked = interrupt_task("marked");
5573
5574 let two = advance_interrupt(
5575 Interrupt::Idle,
5576 &["a".to_owned(), "b".to_owned()],
5577 std::slice::from_ref(&marked),
5578 );
5579 assert_eq!(two, Interrupt::Idle);
5580
5581 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5582 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
5583 }
5584
5585 #[test]
5586 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
5587 let state = Interrupt::Parking {
5588 parked: vec!["running".to_owned()],
5589 interrupt_task: "marked".to_owned(),
5590 };
5591 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
5593 assert_eq!(still_going, state);
5594
5595 let stopped_but_not_yet_dispatched =
5599 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
5600 assert_eq!(stopped_but_not_yet_dispatched, state);
5601
5602 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
5604 assert_eq!(
5605 dispatched,
5606 Interrupt::Running {
5607 parked: vec!["running".to_owned()],
5608 interrupt_task: "marked".to_owned(),
5609 }
5610 );
5611 }
5612
5613 #[test]
5614 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
5615 let state = Interrupt::Running {
5616 parked: vec!["running".to_owned()],
5617 interrupt_task: "marked".to_owned(),
5618 };
5619 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
5620 assert_eq!(still_running, state);
5621
5622 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
5629 assert_eq!(
5630 ended,
5631 Interrupt::Resuming {
5632 parked: vec!["running".to_owned()]
5633 }
5634 );
5635 }
5636
5637 #[test]
5638 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
5639 let state = Interrupt::Resuming {
5640 parked: vec!["running".to_owned()],
5641 };
5642 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
5643 assert_eq!(still_waiting, state);
5644
5645 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
5646 assert_eq!(dispatched, Interrupt::Idle);
5647 }
5648
5649 #[test]
5655 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
5656 {
5657 let state = Interrupt::Parking {
5658 parked: vec!["running".to_owned()],
5659 interrupt_task: "marked".to_owned(),
5660 };
5661 let next = advance_interrupt(state, &[], &[]);
5664 assert_eq!(
5665 next,
5666 Interrupt::Resuming {
5667 parked: vec!["running".to_owned()]
5668 },
5669 "abandoning the interrupt must not abandon the resume it owes"
5670 );
5671 }
5672
5673 #[test]
5676 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
5677 let state = Interrupt::Resuming {
5678 parked: vec!["running".to_owned()],
5679 };
5680 let next = advance_interrupt(state, &[], &[]);
5681 assert_eq!(
5682 next,
5683 Interrupt::Idle,
5684 "nothing is left to wait for; the loop must not stay wedged"
5685 );
5686 }
5687
5688 #[test]
5689 fn disabled_by_config_the_sequence_can_never_leave_idle() {
5690 let marked = interrupt_task("marked");
5691 let next = advance_interrupt_tick(
5692 false,
5693 Interrupt::Idle,
5694 &["running".to_owned()],
5695 std::slice::from_ref(&marked),
5696 );
5697 assert_eq!(
5698 next,
5699 Interrupt::Idle,
5700 "an unmarked, unconfigured daemon must behave exactly as before"
5701 );
5702 }
5703
5704 #[test]
5705 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
5706 let state = Interrupt::Parking {
5707 parked: vec!["running".to_owned()],
5708 interrupt_task: "marked".to_owned(),
5709 };
5710 let candidates = vec![interrupt_task("marked"), task()];
5711 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5712 assert!(
5713 allowed.is_empty(),
5714 "nothing may dispatch - not even the interrupt task itself - \
5715 until the parked run has actually stopped"
5716 );
5717 }
5718
5719 #[test]
5733 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
5734 for state in [
5735 Interrupt::Parking {
5736 parked: vec!["running".to_owned()],
5737 interrupt_task: "marked".to_owned(),
5738 },
5739 Interrupt::Running {
5740 parked: vec!["running".to_owned()],
5741 interrupt_task: "marked".to_owned(),
5742 },
5743 Interrupt::Resuming {
5744 parked: vec!["running".to_owned()],
5745 },
5746 ] {
5747 let candidates = vec![
5748 interrupt_task("marked"),
5749 urgent_task("hot"),
5750 task_with_id("ordinary"),
5751 ];
5752 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5753 assert!(
5754 !allowed.iter().any(|t| t.id == "hot"),
5755 "an urgent candidate must wait out the same gate as anything \
5756 else while the run it would run alongside has not actually \
5757 left flight, for state {state:?}: {allowed:?}"
5758 );
5759 }
5760 }
5761
5762 #[test]
5768 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
5769 let state = Interrupt::Resuming {
5770 parked: vec!["hot".to_owned()],
5771 };
5772 let candidates = vec![urgent_task("hot"), task()];
5773 let allowed = interrupt_gate(&state, &[], candidates);
5774 assert_eq!(
5775 allowed.iter().filter(|t| t.id == "hot").count(),
5776 1,
5777 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
5778 );
5779 }
5780
5781 #[test]
5782 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
5783 let state = Interrupt::Parking {
5784 parked: vec!["running".to_owned()],
5785 interrupt_task: "marked".to_owned(),
5786 };
5787 let other = task();
5788 let candidates = vec![interrupt_task("marked"), other.clone()];
5789 let allowed = interrupt_gate(&state, &[], candidates);
5790 assert_eq!(allowed.len(), 1);
5791 assert_eq!(allowed[0].id, "marked");
5792 }
5793
5794 #[test]
5795 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
5796 let state = Interrupt::Running {
5797 parked: vec!["running".to_owned()],
5798 interrupt_task: "marked".to_owned(),
5799 };
5800 let candidates = vec![task(), task()];
5801 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
5802 assert!(allowed.is_empty());
5803 }
5804
5805 #[test]
5812 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
5813 let state = Interrupt::Resuming {
5814 parked: vec!["a".to_owned(), "c".to_owned()],
5815 };
5816 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
5817 let allowed = interrupt_gate(&state, &[], candidates);
5818 assert_eq!(
5819 allowed.len(),
5820 1,
5821 "at most one candidate may be offered while resuming: {allowed:?}"
5822 );
5823 assert_eq!(allowed[0].id, "a");
5824 }
5825
5826 #[test]
5827 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
5828 let state = Interrupt::Resuming {
5829 parked: vec!["a".to_owned()],
5830 };
5831 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
5832 assert!(allowed.is_empty());
5833 }
5834
5835 #[test]
5841 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
5842 let running = task(); let marked = interrupt_task("marked");
5844
5845 let mut state = Interrupt::Idle;
5846 let in_flight = vec![running.id.clone()];
5848 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
5849 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
5850 assert!(gated.is_empty(), "still waiting on `running` to park");
5851
5852 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
5854 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
5855 assert_eq!(
5856 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5857 vec!["marked"],
5858 "only the interrupt task may be offered to the dispatcher now"
5859 );
5860
5861 state = advance_interrupt_tick(
5863 true,
5864 state,
5865 &["marked".to_owned()],
5866 std::slice::from_ref(&running),
5867 );
5868 let gated = interrupt_gate(
5869 &state,
5870 &["marked".to_owned()],
5871 vec![marked.clone(), running.clone()],
5872 );
5873 assert!(
5874 gated.is_empty(),
5875 "the parked run must not be offered back while the interrupt \
5876 task is still running"
5877 );
5878
5879 let other = task_with_id("other");
5883 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
5884 assert_eq!(
5885 state,
5886 Interrupt::Resuming {
5887 parked: vec![running.id.clone()]
5888 }
5889 );
5890 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
5891 assert_eq!(
5892 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5893 vec![running.id.as_str()],
5894 "exactly the parked run resumes - not the unrelated task, even \
5895 though it was offered first"
5896 );
5897
5898 state = advance_interrupt_tick(
5902 true,
5903 state,
5904 std::slice::from_ref(&running.id),
5905 std::slice::from_ref(&other),
5906 );
5907 assert_eq!(state, Interrupt::Idle);
5908 let gated = interrupt_gate(
5909 &state,
5910 std::slice::from_ref(&running.id),
5911 vec![other.clone()],
5912 );
5913 assert_eq!(
5914 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5915 vec![other.id.as_str()],
5916 "ordinary dispatch is unrestricted again"
5917 );
5918 }
5919
5920 #[test]
5921 fn every_run_status_settles_the_task_it_came_from() {
5922 let table = [
5924 (RunStatus::Merged, TaskStatus::Done, 1),
5925 (RunStatus::Ready, TaskStatus::Done, 1),
5926 (RunStatus::Stalled, TaskStatus::Failed, 0),
5927 (RunStatus::Blocked, TaskStatus::Failed, 1),
5928 (RunStatus::Failed, TaskStatus::Failed, 1),
5929 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
5930 (RunStatus::Prep, TaskStatus::Failed, 1),
5931 (RunStatus::Implementing, TaskStatus::Failed, 1),
5932 (RunStatus::Judging, TaskStatus::Failed, 1),
5933 (RunStatus::Deliberating, TaskStatus::Failed, 1),
5934 (RunStatus::Voting, TaskStatus::Failed, 1),
5935 (RunStatus::Reviewing, TaskStatus::Failed, 1),
5936 (RunStatus::Gating, TaskStatus::Failed, 1),
5937 ];
5938 for (run, want, attempts) in table {
5939 let mut t = task();
5940 t.start("20260902-000000-aaaa".to_owned());
5941 settle(
5942 &mut t,
5943 Verdict {
5944 status: run,
5945 left_pr: false,
5946 parked: false,
5947 quota_hit: matches!(run, RunStatus::Stalled),
5948 no_viable_candidates: false,
5949 },
5950 "why",
5951 2,
5952 );
5953 assert_eq!(t.status, want, "task status after {}", label(run));
5954 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
5955 }
5956 }
5957
5958 #[test]
5959 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
5960 let mut stalled = task();
5961 stalled.start("20260902-000000-aaaa".to_owned());
5962 settle(
5963 &mut stalled,
5964 Verdict {
5965 status: RunStatus::Stalled,
5966 left_pr: false,
5967 parked: false,
5968 quota_hit: true,
5969 no_viable_candidates: false,
5970 },
5971 "quota",
5972 1,
5973 );
5974 assert_eq!(stalled.attempts, 0);
5975 assert!(
5976 stalled.status.runnable(),
5977 "a machine problem must leave the task in line"
5978 );
5979
5980 let mut blocked = task();
5981 blocked.start("20260902-000000-aaaa".to_owned());
5982 settle(
5983 &mut blocked,
5984 Verdict {
5985 status: RunStatus::Blocked,
5986 left_pr: false,
5987 parked: false,
5988 quota_hit: false,
5989 no_viable_candidates: false,
5990 },
5991 "findings open",
5992 1,
5993 );
5994 assert_eq!(blocked.attempts, 1);
5995 assert_eq!(
5996 blocked.status,
5997 TaskStatus::Held,
5998 "the last attempt hands the task to a human"
5999 );
6000 }
6001
6002 #[test]
6003 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
6004 let mut delivered = task();
6007 delivered.start("20260903-080619-01c2".to_owned());
6008 settle(
6009 &mut delivered,
6010 Verdict {
6011 status: RunStatus::Blocked,
6012 left_pr: true,
6013 parked: false,
6014 quota_hit: false,
6015 no_viable_candidates: false,
6016 },
6017 "no check status",
6018 4,
6019 );
6020 assert_eq!(
6021 delivered.status,
6022 TaskStatus::Held,
6023 "a pull request waiting on CI or a person is not a retryable failure"
6024 );
6025 assert!(
6026 !delivered.status.runnable(),
6027 "the loop must not pick this task up again"
6028 );
6029 assert_eq!(
6030 delivered.last_error.as_deref(),
6031 Some("no check status"),
6032 "the operator needs to be told what the gate was waiting for"
6033 );
6034
6035 let mut empty_handed = task();
6038 empty_handed.start("20260903-080619-01c2".to_owned());
6039 settle(
6040 &mut empty_handed,
6041 Verdict {
6042 status: RunStatus::Blocked,
6043 left_pr: false,
6044 parked: false,
6045 quota_hit: false,
6046 no_viable_candidates: false,
6047 },
6048 "findings open",
6049 4,
6050 );
6051 assert_eq!(empty_handed.status, TaskStatus::Failed);
6052 assert!(empty_handed.status.runnable());
6053 }
6054
6055 #[test]
6056 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
6057 let mut noop = task();
6063 noop.start("20260912-131304-391f".to_owned());
6064 settle(
6065 &mut noop,
6066 Verdict {
6067 status: RunStatus::VerifiedNoop,
6068 left_pr: false,
6069 parked: false,
6070 quota_hit: false,
6071 no_viable_candidates: true,
6072 },
6073 "candidate A: already fixed by b32cfc4, on main",
6074 4,
6075 );
6076 assert_eq!(
6077 noop.status,
6078 TaskStatus::Held,
6079 "an unverified claim is a request for a human, not a failure"
6080 );
6081 assert!(
6082 !noop.status.runnable(),
6083 "the loop must not requeue this on the same unverified claim"
6084 );
6085 assert_eq!(noop.attempts, 1);
6090 }
6091
6092 #[test]
6093 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
6094 let mut parked = task();
6099 parked.start("20260903-183634-2d98".to_owned());
6100 settle(
6101 &mut parked,
6102 Verdict {
6103 status: RunStatus::Implementing,
6104 left_pr: false,
6105 quota_hit: false,
6106 parked: true,
6107 no_viable_candidates: false,
6108 },
6109 "parked after `implementing`",
6110 2,
6111 );
6112 assert_eq!(parked.attempts, 0, "a park is refunded");
6113 assert!(
6114 parked.status.runnable(),
6115 "and the task stays in line so the next loop resumes its run"
6116 );
6117 assert_eq!(parked.status, TaskStatus::Parked);
6118 assert_eq!(
6119 parked.park_reason.as_deref(),
6120 Some("parked after `implementing`"),
6121 "the card says where it stopped"
6122 );
6123 assert_eq!(parked.last_error, None, "nothing failed");
6124
6125 let mut broken = task();
6129 broken.start("20260903-183634-2d98".to_owned());
6130 settle(
6131 &mut broken,
6132 Verdict {
6133 status: RunStatus::Implementing,
6134 left_pr: false,
6135 quota_hit: false,
6136 parked: false,
6137 no_viable_candidates: false,
6138 },
6139 "returned mid-flight",
6140 2,
6141 );
6142 assert_eq!(broken.attempts, 1);
6143 }
6144
6145 #[test]
6146 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
6147 let mut flaky = task();
6152 flaky.start("20260903-123023-e633".to_owned());
6153 settle(
6154 &mut flaky,
6155 Verdict {
6156 status: RunStatus::Stalled,
6157 left_pr: false,
6158 parked: false,
6159 quota_hit: false,
6160 no_viable_candidates: false,
6161 },
6162 "verdict rests on 1 of 3 judges",
6163 2,
6164 );
6165 assert_eq!(
6166 flaky.attempts, 1,
6167 "flakiness spends an attempt, so `max_attempts` still bounds it"
6168 );
6169 assert!(flaky.status.runnable(), "and it is still worth retrying");
6170
6171 let mut limited = task();
6173 limited.start("20260903-123023-e633".to_owned());
6174 settle(
6175 &mut limited,
6176 Verdict {
6177 status: RunStatus::Stalled,
6178 left_pr: false,
6179 parked: false,
6180 quota_hit: true,
6181 no_viable_candidates: false,
6182 },
6183 "judge-2, judge-3 out of quota",
6184 2,
6185 );
6186 assert_eq!(limited.attempts, 0, "a quota window is refunded");
6187 assert!(limited.status.runnable());
6188
6189 let mut worn = task();
6192 for _ in 0..2 {
6193 worn.release();
6194 }
6195 worn.start("20260903-123023-e633".to_owned());
6196 worn.attempts = 2;
6197 settle(
6198 &mut worn,
6199 Verdict {
6200 status: RunStatus::Stalled,
6201 left_pr: false,
6202 parked: false,
6203 quota_hit: false,
6204 no_viable_candidates: false,
6205 },
6206 "no quorum again",
6207 2,
6208 );
6209 assert_eq!(worn.status, TaskStatus::Held);
6210 assert!(!worn.status.runnable());
6211 }
6212
6213 #[test]
6214 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
6215 let mut wiped_out = task();
6222 wiped_out.start("20260907-025000-a1b2".to_owned());
6223 settle(
6224 &mut wiped_out,
6225 Verdict {
6226 status: RunStatus::Failed,
6227 left_pr: false,
6228 parked: false,
6229 quota_hit: true,
6230 no_viable_candidates: true,
6231 },
6232 "no candidate produced a change; nothing to judge",
6233 2,
6234 );
6235 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
6236 assert!(
6237 wiped_out.status.runnable(),
6238 "a machine problem must leave the task in line"
6239 );
6240
6241 let mut partial_progress = task();
6247 partial_progress.start("20260907-025500-c3d4".to_owned());
6248 settle(
6249 &mut partial_progress,
6250 Verdict {
6251 status: RunStatus::Failed,
6252 left_pr: false,
6253 parked: false,
6254 quota_hit: true,
6255 no_viable_candidates: false,
6256 },
6257 "gate failed on the winning candidate",
6258 2,
6259 );
6260 assert_eq!(
6261 partial_progress.attempts, 1,
6262 "a candidate that actually produced a change spends the attempt \
6263 even though some other seat hit its quota"
6264 );
6265 assert!(partial_progress.status.runnable());
6266 }
6267
6268 #[test]
6269 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
6270 let mut t = task();
6276 t.start("20260907-025000-a1b2".to_owned());
6277 let mut state = run_state(RunStatus::Failed);
6278 state.quota.push(QuotaLoss {
6279 seat: "cand-a".to_owned(),
6280 node: "implement".to_owned(),
6281 at: Timestamp::now(),
6282 reset: None,
6283 });
6284 assert!(
6285 state.viable().is_empty(),
6286 "no candidate was added, so nothing is viable"
6287 );
6288 reclaim(&mut t, Some(state), 2, "en");
6289 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
6290 assert!(t.status.runnable());
6291 }
6292
6293 #[test]
6294 fn a_held_task_is_never_offered_to_the_loop() {
6295 let dir = tempfile::tempdir().unwrap();
6296 let queue = Queue::at(dir.path().to_path_buf());
6297 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
6298 let mut t = task();
6299 t.id = format!("2026090{n}-000000-000{n}");
6300 t.priority = priority;
6301 queue.put(&mut t).unwrap();
6302 }
6303 let mut held = task();
6304 held.id = "20260909-000000-9999".to_owned();
6305 held.priority = 99;
6306 held.hold_machine(None);
6307 queue.put(&mut held).unwrap();
6308
6309 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
6310 assert_eq!(order.len(), 3);
6311 assert!(!order.contains(&held.id));
6312 assert_eq!(
6313 order.first().cloned(),
6314 queue.next_runnable().map(|t| t.id),
6315 "the loop's first candidate is exactly what the queue offers"
6316 );
6317 assert_eq!(
6318 order,
6319 vec![
6320 "20260902-000000-0002".to_owned(),
6321 "20260903-000000-0003".to_owned(),
6322 "20260901-000000-0001".to_owned(),
6323 ],
6324 "priority first, then oldest, so nothing starves"
6325 );
6326 }
6327
6328 #[test]
6329 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
6330 let dir = tempfile::tempdir().unwrap();
6331 let queue = Queue::at(dir.path().to_path_buf());
6332 let mut old = task();
6333 old.id = "20260101-000000-old0".to_owned();
6334 queue.put(&mut old).unwrap();
6335 let mut fresh = task();
6336 fresh.id = "20260101-000000-new0".to_owned();
6337 queue.put(&mut fresh).unwrap();
6338
6339 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
6343 std::thread::sleep(Duration::from_millis(60));
6344 let live = queue.claim(&fresh.id).unwrap();
6345
6346 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6347 assert_eq!(swept, vec![old.id.clone()]);
6348 assert!(
6349 queue.claim(&old.id).is_ok(),
6350 "an unparseable lock older than the threshold is swept"
6351 );
6352 assert!(
6353 queue.claim(&fresh.id).is_err(),
6354 "a live pid protects its lock regardless of age"
6355 );
6356 drop(live);
6357 }
6358
6359 #[test]
6360 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
6361 let dir = tempfile::tempdir().unwrap();
6371 let queue = Queue::at(dir.path().to_path_buf());
6372 let mut t = task();
6373 t.id = "20260101-000000-live".to_owned();
6374 queue.put(&mut t).unwrap();
6375
6376 let claim = queue.claim(&t.id).unwrap();
6377 std::thread::sleep(Duration::from_millis(60));
6378
6379 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6380 assert!(
6381 swept.is_empty(),
6382 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
6383 );
6384 assert!(
6385 queue.claim(&t.id).is_err(),
6386 "the lock still protects its task"
6387 );
6388 drop(claim);
6389 }
6390
6391 fn injected_dead_pid() -> u32 {
6394 std::process::id().checked_add(1).unwrap_or(1)
6395 }
6396
6397 #[test]
6398 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
6399 let dir = tempfile::tempdir().unwrap();
6400 let queue = Queue::at(dir.path().to_path_buf());
6401 let mut t = task();
6402 t.id = "20260101-000000-dead".to_owned();
6403 queue.put(&mut t).unwrap();
6404 let dead_pid = injected_dead_pid();
6405
6406 std::fs::write(
6411 dir.path().join(format!("{}.lock", t.id)),
6412 dead_pid.to_string(),
6413 )
6414 .unwrap();
6415
6416 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6417 pid != dead_pid
6418 });
6419 assert_eq!(
6420 swept,
6421 vec![t.id.clone()],
6422 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
6423 );
6424 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
6425 }
6426
6427 #[test]
6428 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
6429 let dir = tempfile::tempdir().unwrap();
6430 let queue = Queue::at(dir.path().to_path_buf());
6431 let mut t = task();
6432 t.id = "20260101-000000-late".to_owned();
6433 queue.put(&mut t).unwrap();
6434 let dead_pid = injected_dead_pid();
6435
6436 assert!(
6439 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
6440 "nothing has claimed the task yet"
6441 );
6442
6443 std::fs::write(
6446 dir.path().join(format!("{}.lock", t.id)),
6447 dead_pid.to_string(),
6448 )
6449 .unwrap();
6450
6451 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6455 pid != dead_pid
6456 });
6457 assert_eq!(swept, vec![t.id.clone()]);
6458 }
6459
6460 #[test]
6461 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
6462 crate::run::pin_test_home();
6467 let dir = tempfile::tempdir().unwrap();
6468 let queue = Queue::at(dir.path().to_path_buf());
6469 let mut t = task();
6470 t.id = "20260101-000000-crsh".to_owned();
6471 t.status = TaskStatus::Running;
6472 t.attempts = 1;
6473 t.runs.push("20260904-000000-4043".to_owned());
6477 queue.put(&mut t).unwrap();
6478 let dead_pid = injected_dead_pid();
6479
6480 std::fs::write(
6483 dir.path().join(format!("{}.lock", t.id)),
6484 dead_pid.to_string(),
6485 )
6486 .unwrap();
6487
6488 assert!(reclaim_orphaned_running(&queue, 2).is_empty());
6494 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
6495
6496 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6497 pid != dead_pid
6498 });
6499 assert_eq!(swept, vec![t.id.clone()]);
6500
6501 let reclaimed = reclaim_orphaned_running(&queue, 2);
6502 assert_eq!(reclaimed, vec![t.id.clone()]);
6503 let after = queue.get(&t.id).unwrap();
6504 assert_eq!(
6505 after.status,
6506 TaskStatus::Held,
6507 "no run.json to recover from, so a human is asked"
6508 );
6509 assert_eq!(
6510 after.runs,
6511 vec!["20260904-000000-4043".to_owned()],
6512 "the crashed run's id is kept as evidence, not discarded"
6513 );
6514 }
6515
6516 #[test]
6517 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
6518 let dir = tempfile::tempdir().unwrap();
6519 let queue = Queue::at(dir.path().to_path_buf());
6520 let mut t = task();
6521 t.id = "20260101-000000-unknown".to_owned();
6522 queue.put(&mut t).unwrap();
6523 let dead_pid = injected_dead_pid();
6524 std::fs::write(
6525 dir.path().join(format!("{}.lock", t.id)),
6526 dead_pid.to_string(),
6527 )
6528 .unwrap();
6529
6530 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
6531 assert!(swept.is_empty(), "an unknown pid must keep its lock");
6532 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
6533 }
6534
6535 fn run_state_in(status: RunStatus, language: &str) -> RunState {
6536 let mut s = run_state(status);
6537 s.config.graph.language = language.to_owned();
6538 s
6539 }
6540
6541 fn unstarted_verdict(status: RunStatus) -> Verdict {
6542 Verdict {
6543 status,
6544 left_pr: false,
6545 quota_hit: false,
6546 parked: false,
6547 no_viable_candidates: false,
6548 }
6549 }
6550
6551 #[test]
6552 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
6553 let mut t = task();
6554 t.attempts = 1;
6555 settle_in(
6556 &mut t,
6557 unstarted_verdict(RunStatus::AlreadyInBase),
6558 "already in main",
6559 1,
6560 phrases("en"),
6561 );
6562 assert_eq!(t.status, TaskStatus::Done);
6563 assert_eq!(t.attempts, 0);
6564 }
6565
6566 #[test]
6567 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
6568 let reason = |language: &str| {
6569 let mut t = task();
6570 settle_in(
6571 &mut t,
6572 unstarted_verdict(RunStatus::Judging),
6573 "boom",
6574 1,
6575 phrases(language),
6576 );
6577 t.last_error.or(t.hold_reason).unwrap_or_default()
6578 };
6579 assert!(
6580 reason("en").starts_with("the graph stopped at `"),
6581 "{}",
6582 reason("en")
6583 );
6584 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
6585 assert!(reason("日本語").contains("boom"));
6586 assert_eq!(reason("fr"), reason("en"));
6587 }
6588
6589 #[test]
6590 fn describe_follows_the_run_language_and_keeps_the_run_id() {
6591 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
6592 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
6593 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
6594 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
6595 assert!(ja.contains("[run "), "{ja}");
6596 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
6597 assert!(ended.starts_with("run 終了: "), "{ended}");
6598 let mut de = run_state_in(RunStatus::Failed, "de");
6599 let mut en = run_state_in(RunStatus::Failed, "en");
6600 de.id = "same".to_owned();
6601 en.id = "same".to_owned();
6602 assert_eq!(describe(&de), describe(&en));
6603 }
6604
6605 #[test]
6606 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
6607 use crate::handover::Refused;
6608 let cases = [
6609 Refused::Foreign {
6610 branch: "b".into(),
6611 path: "/w/x".into(),
6612 why: "made by hand".into(),
6613 },
6614 Refused::Unsafe {
6615 branch: "b".into(),
6616 path: "/w/x".into(),
6617 why: "its worktree has uncommitted changes (a.rs)".into(),
6618 },
6619 Refused::ReleaseFailed {
6620 branch: "b".into(),
6621 path: "/w/x".into(),
6622 run: "ab12".into(),
6623 },
6624 ];
6625 for r in &cases {
6626 let en = (phrases("en").handover_refused)(r);
6627 assert_eq!(en, r.to_string());
6628 let ja = (phrases("ja").handover_refused)(r);
6629 assert!(
6630 !ja.contains("is checked out") && !ja.contains("try again"),
6631 "{ja}"
6632 );
6633 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
6634 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
6635 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
6636 }
6637 }
6638 assert!(phrases("en").handover_hint.contains("release the task"));
6639 assert!(phrases("ja").handover_hint.contains("解放"));
6640 }
6641
6642 #[test]
6643 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
6644 let build = |lang: &str| {
6645 let mut s = run_state_in(RunStatus::Blocked, lang);
6646 s.id = "same".to_owned();
6647 s.config.graph.review_rounds = 3;
6648 let mut r = review_round(3);
6649 r.expected = 3;
6650 r.answered = 1;
6651 s.reviews.push(r);
6652 s.event(
6653 "review",
6654 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
6655 );
6656 s
6657 };
6658 let en = describe(&build("en"));
6659 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
6660 let ja = describe(&build("ja"));
6661 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
6662 assert!(!ja.contains("never answered"), "{ja}");
6663 assert!(ja.contains("[run "), "{ja}");
6664
6665 let mut failed = build("ja");
6668 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
6669 command: "cargo test".to_owned(),
6670 code: Some(1),
6671 output_tail: String::new(),
6672 duration_ms: 0,
6673 resource_blocked: false,
6674 });
6675 failed.event("review", "stopped; e2e failed: cargo test");
6676 let ja = describe(&failed);
6677 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
6678 assert!(!ja.contains("席のレビュアー"), "{ja}");
6679 }
6680
6681 #[test]
6682 fn refusals_and_recovery_prose_follow_the_language() {
6683 let t = held_task_with("r1");
6684 let q = action_question(
6685 "r1",
6686 ask::ChoiceAction::Resume {
6687 run: "r1".to_owned(),
6688 },
6689 );
6690 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
6691 ActionDecision::Refuse(s) => s,
6692 other => panic!("{other:?}"),
6693 };
6694 assert!(refuse(phrases("en")).contains("could not be read: gone"));
6695 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
6696 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
6697
6698 let mut held = task();
6699 reclaim(&mut held, None, 2, "ja");
6700 assert!(held.hold_reason.unwrap().contains("保留にしました"));
6701 let mut held = task();
6702 reclaim(&mut held, None, 2, "xx");
6703 assert!(held.hold_reason.unwrap().contains("held for a human"));
6704 }
6705
6706 fn run_state(status: RunStatus) -> RunState {
6707 let mut state = RunState::new(
6708 PathBuf::from("/repo"),
6709 "main".to_owned(),
6710 "abc1234def".to_owned(),
6711 "add retries".to_owned(),
6712 Config::default(),
6713 );
6714 state.status = status;
6715 state
6716 }
6717
6718 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
6719 Candidate {
6720 index: 0,
6721 label,
6722 agent: "claude".to_owned(),
6723 branch: format!("magi/x/{label}"),
6724 worktree: PathBuf::from("/repo"),
6725 summary: summary.to_owned(),
6726 stat: String::new(),
6727 files: 0,
6728 commits: usize::from(!empty),
6729 empty,
6730 failed: failed.map(str::to_owned),
6731 verified_noop: None,
6732 duration_ms: 0,
6733 folded: false,
6734 }
6735 }
6736
6737 #[test]
6738 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
6739 let mut state = run_state(RunStatus::Blocked);
6740 state.gate = vec![
6741 CommandOutcome {
6742 command: "cargo make check".to_owned(),
6743 code: Some(0),
6744 output_tail: "ok".to_owned(),
6745 duration_ms: 0,
6746 resource_blocked: false,
6747 },
6748 CommandOutcome {
6749 command: "cargo test".to_owned(),
6750 code: Some(101),
6751 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
6752 duration_ms: 0,
6753 resource_blocked: false,
6754 },
6755 ];
6756 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
6757 assert!(d.contains("cargo test"), "{d}");
6758 assert!(
6759 !d.contains("cargo make check"),
6760 "a passing check is not a diagnostic: {d}"
6761 );
6762 assert!(d.contains("assertion failed"), "{d}");
6763 }
6764
6765 #[test]
6766 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
6767 let mut state = run_state(RunStatus::Blocked);
6768 state.event(
6769 "land",
6770 "stopped: the fixer produced no commit while 2 check(s) were failing \
6771 (build, lint); stopping instead of looping on an unchanged tree",
6772 );
6773 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
6774 assert!(d.contains("build"), "{d}");
6775 assert!(d.contains("lint"), "{d}");
6776 assert!(d.contains("fixer produced no commit"), "{d}");
6777 }
6778
6779 #[test]
6780 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
6781 let state = run_state(RunStatus::VerifiedNoop);
6786 let d = describe(&state);
6787 assert!(
6788 d.contains("agent-verified no-op"),
6789 "expected the display label, not the wire spelling: {d}"
6790 );
6791 assert!(!d.contains("verified_noop"), "{d}");
6792 }
6793
6794 #[test]
6795 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
6796 let mut state = run_state(RunStatus::Failed);
6802 state.candidates = vec![candidate(
6803 'A',
6804 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
6805 true,
6806 None,
6807 )];
6808 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
6809 assert!(d.contains("candidate A"), "{d}");
6810 assert!(d.contains("tagged v1.2.3"), "{d}");
6811 }
6812
6813 #[test]
6814 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
6815 let mut state = run_state(RunStatus::Failed);
6816 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
6817 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
6818 assert!(d.contains("candidate A"), "{d}");
6819 assert!(d.contains("agent timed out"), "{d}");
6820 }
6821
6822 #[test]
6823 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
6824 let mut state = run_state(RunStatus::Failed);
6827 state.candidates = vec![candidate('A', "did the work", false, None)];
6828 assert!(diagnostic(&state).is_none());
6829 }
6830
6831 #[test]
6832 fn diagnostic_is_bounded_however_much_a_run_printed() {
6833 let mut state = run_state(RunStatus::Blocked);
6834 state.gate = vec![
6835 CommandOutcome {
6836 command: "cargo test".to_owned(),
6837 code: Some(101),
6838 output_tail: "x".repeat(50_000),
6839 duration_ms: 0,
6840 resource_blocked: false,
6841 },
6842 CommandOutcome {
6843 command: "cargo clippy".to_owned(),
6844 code: Some(1),
6845 output_tail: "y".repeat(50_000),
6846 duration_ms: 0,
6847 resource_blocked: false,
6848 },
6849 ];
6850 state.candidates = vec![
6851 candidate('A', &"z".repeat(50_000), true, None),
6852 candidate('B', &"w".repeat(50_000), true, None),
6853 ];
6854 let d = diagnostic(&state).expect("plenty here to diagnose");
6855 assert!(
6856 d.len() <= DIAGNOSTIC_MAX,
6857 "diagnostic grew to {} bytes, unbounded",
6858 d.len()
6859 );
6860 }
6861
6862 #[test]
6863 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
6864 let mut state = run_state(RunStatus::Blocked);
6865 state.gate = vec![CommandOutcome {
6866 command: "cargo test".to_owned(),
6867 code: Some(101),
6868 output_tail: "assertion failed".to_owned(),
6869 duration_ms: 0,
6870 resource_blocked: false,
6871 }];
6872 let verdict = Verdict {
6873 status: RunStatus::Blocked,
6874 left_pr: false,
6875 quota_hit: false,
6876 parked: false,
6877 no_viable_candidates: false,
6878 };
6879
6880 let mut t = task();
6883 t.start("run-1".to_owned());
6884 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
6885 assert_eq!(t.status, TaskStatus::Failed);
6886 assert!(t.diagnostic.is_none());
6887
6888 t.start("run-2".to_owned());
6891 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
6892 assert_eq!(t.status, TaskStatus::Held);
6893 let d = t.diagnostic.expect("a held task must carry its diagnostic");
6894 assert!(d.contains("cargo test"), "{d}");
6895 }
6896
6897 #[test]
6898 fn a_held_task_names_the_open_question_it_is_waiting_on() {
6899 crate::run::pin_test_home();
6904 let home = crate::run::home();
6905 let state = run_state(RunStatus::VerifiedNoop);
6906 let mut q = ask::Question::new(
6907 state.id.clone(),
6908 "implement".to_owned(),
6909 "impl-A".to_owned(),
6910 "is this really a no-op?".to_owned(),
6911 String::new(),
6912 Vec::new(),
6913 );
6914 Questions::at(home.join("questions")).put(&mut q).unwrap();
6915
6916 let verdict = Verdict {
6917 status: RunStatus::VerifiedNoop,
6918 left_pr: false,
6919 quota_hit: false,
6920 parked: false,
6921 no_viable_candidates: false,
6922 };
6923 let mut t = task();
6924 t.start(state.id.clone());
6925 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
6926
6927 assert_eq!(t.status, TaskStatus::Held);
6928 let reason = t.hold_reason.expect("a held task must record why");
6929 assert!(
6930 reason.starts_with("run ended agent-verified no-op"),
6931 "the original settle reason must survive unchanged: {reason}"
6932 );
6933 assert!(
6934 reason.contains(q.short()),
6935 "the open question's id must be named so the notice is actionable: {reason}"
6936 );
6937 }
6938
6939 #[test]
6940 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
6941 crate::run::pin_test_home();
6942 let state = run_state(RunStatus::VerifiedNoop);
6943
6944 let verdict = Verdict {
6945 status: RunStatus::VerifiedNoop,
6946 left_pr: false,
6947 quota_hit: false,
6948 parked: false,
6949 no_viable_candidates: false,
6950 };
6951 let mut t = task();
6952 t.start(state.id.clone());
6953 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
6954
6955 assert_eq!(t.status, TaskStatus::Held);
6956 assert_eq!(
6957 t.hold_reason.as_deref(),
6958 Some("run ended agent-verified no-op"),
6959 "nothing to append when the question was already answered or never asked"
6960 );
6961 }
6962
6963 #[test]
6964 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
6965 crate::run::pin_test_home();
6966 let mut first = run_state(RunStatus::Blocked);
6967 first.id = "20260101-000000-sup1".to_owned();
6968 first.save().unwrap();
6969 let mut second = run_state(RunStatus::Merged);
6970 second.id = "20260101-000000-sup2".to_owned();
6971 second.save().unwrap();
6972
6973 let mut t = task();
6974 t.runs = vec![first.id.clone(), second.id.clone()];
6975 t.status = TaskStatus::Done;
6976
6977 supersede_prior_runs(&t, &crate::run::home());
6978
6979 assert_eq!(
6980 RunState::load(&first.id).unwrap().status,
6981 RunStatus::Superseded,
6982 "the first attempt's Blocked no longer needs anyone's attention"
6983 );
6984 assert_eq!(
6985 RunState::load(&second.id).unwrap().status,
6986 RunStatus::Merged,
6987 "the run that actually succeeded is left exactly as it was"
6988 );
6989 }
6990
6991 #[test]
6992 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
6993 crate::run::pin_test_home();
6999 let mut first = run_state(RunStatus::Blocked);
7000 first.id = "20260101-000000-sup9".to_owned();
7001 first.driver_pid = Some(std::process::id());
7004 first.driver_started_at = Some(
7005 crate::proc::process_started_at(std::process::id())
7006 .expect("this test process's own start time must be queryable"),
7007 );
7008 first.save().unwrap();
7009 let mut second = run_state(RunStatus::Merged);
7010 second.id = "20260101-000000-supa".to_owned();
7011 second.save().unwrap();
7012
7013 let mut t = task();
7014 t.runs = vec![first.id.clone(), second.id.clone()];
7015 t.status = TaskStatus::Done;
7016
7017 supersede_prior_runs(&t, &crate::run::home());
7018
7019 assert_eq!(
7020 RunState::load(&first.id).unwrap().status,
7021 RunStatus::Blocked,
7022 "a live driver_pid means something is still actually working this run, \
7023 even though no daemon claims it - rewriting under it would just be \
7024 undone the next time that process saves"
7025 );
7026 }
7027
7028 #[test]
7029 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
7030 let dir = tempfile::tempdir().unwrap();
7036 let home = dir.path().to_path_buf();
7037 let queue = Queue::at(dir.path().join("queue"));
7038
7039 let mut first = run_state(RunStatus::Blocked);
7040 first.id = "20260101-000000-supd".to_owned();
7041 first.driver_pid = Some(std::process::id());
7042 first.driver_started_at = Some(
7043 crate::proc::process_started_at(std::process::id())
7044 .expect("this test process's own start time must be queryable"),
7045 );
7046 first.save_under(&home).unwrap();
7047 let mut second = run_state(RunStatus::Merged);
7048 second.id = "20260101-000000-supe".to_owned();
7049 second.save_under(&home).unwrap();
7050
7051 let mut t = task();
7052 t.runs = vec![first.id.clone(), second.id.clone()];
7053 t.status = TaskStatus::Done;
7054 queue.put(&mut t).unwrap();
7055
7056 resweep_superseded_attempts(&queue, &home);
7057 assert_eq!(
7058 RunState::load_under(&first.id, &home).unwrap().status,
7059 RunStatus::Blocked,
7060 "still live on the first pass, so still untouched"
7061 );
7062
7063 let mut stale = RunState::load_under(&first.id, &home).unwrap();
7069 stale.driver_started_at = Some("1".to_owned());
7070 stale.save_under(&home).unwrap();
7071
7072 resweep_superseded_attempts(&queue, &home);
7073 assert_eq!(
7074 RunState::load_under(&first.id, &home).unwrap().status,
7075 RunStatus::Superseded,
7076 "the second pass catches up what the first one correctly skipped"
7077 );
7078 }
7079
7080 #[test]
7081 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
7082 crate::run::pin_test_home();
7083 let mut first = run_state(RunStatus::Blocked);
7084 first.id = "20260101-000000-sup3".to_owned();
7085 first.save().unwrap();
7086 let mut second = run_state(RunStatus::Blocked);
7087 second.id = "20260101-000000-sup4".to_owned();
7088 second.save().unwrap();
7089
7090 let mut t = task();
7091 t.runs = vec![first.id.clone(), second.id.clone()];
7092 t.status = TaskStatus::Failed;
7096
7097 supersede_prior_runs(&t, &crate::run::home());
7098
7099 assert_eq!(
7100 RunState::load(&first.id).unwrap().status,
7101 RunStatus::Blocked
7102 );
7103 assert_eq!(
7104 RunState::load(&second.id).unwrap().status,
7105 RunStatus::Blocked
7106 );
7107 }
7108
7109 #[test]
7110 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
7111 crate::run::pin_test_home();
7115 let mut first = run_state(RunStatus::Blocked);
7116 first.id = "20260101-000000-sup5".to_owned();
7117 first.save().unwrap();
7118
7119 let mut t = task();
7120 t.runs = vec![first.id.clone()];
7121 t.status = TaskStatus::Done;
7122
7123 supersede_prior_runs(&t, &crate::run::home());
7124
7125 assert_eq!(
7126 RunState::load(&first.id).unwrap().status,
7127 RunStatus::Blocked,
7128 "a single-attempt task has no earlier run to supersede"
7129 );
7130 }
7131
7132 #[test]
7133 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
7134 crate::run::pin_test_home();
7141 let mut first = run_state(RunStatus::Blocked);
7142 first.id = "20260101-000000-supb".to_owned();
7143 first.save().unwrap();
7144 let mut second = run_state(RunStatus::Failed);
7145 second.id = "20260101-000000-supc".to_owned();
7146 second.save().unwrap();
7147
7148 let mut t = task();
7149 t.runs = vec![first.id.clone(), second.id.clone()];
7150 t.status = TaskStatus::Done;
7151
7152 supersede_prior_runs(&t, &crate::run::home());
7153
7154 assert_eq!(
7155 RunState::load(&first.id).unwrap().status,
7156 RunStatus::Blocked,
7157 "the task's last attempt never landed, so there is nothing here \
7158 actually superseding it"
7159 );
7160 }
7161
7162 #[test]
7163 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
7164 crate::run::pin_test_home();
7168 let mut failed = run_state(RunStatus::Failed);
7169 failed.id = "20260101-000000-sup6".to_owned();
7170 failed.save().unwrap();
7171 let mut noop = run_state(RunStatus::VerifiedNoop);
7172 noop.id = "20260101-000000-sup7".to_owned();
7173 noop.save().unwrap();
7174 let mut winner = run_state(RunStatus::Ready);
7175 winner.id = "20260101-000000-sup8".to_owned();
7176 winner.save().unwrap();
7177
7178 let mut t = task();
7179 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
7180 t.status = TaskStatus::Done;
7181
7182 supersede_prior_runs(&t, &crate::run::home());
7183
7184 assert_eq!(
7185 RunState::load(&failed.id).unwrap().status,
7186 RunStatus::Failed
7187 );
7188 assert_eq!(
7189 RunState::load(&noop.id).unwrap().status,
7190 RunStatus::VerifiedNoop
7191 );
7192 }
7193
7194 fn approval_question(run: &str) -> ask::Question {
7195 ask::Question::new(
7196 run.to_owned(),
7197 land::APPROVAL_NODE.to_owned(),
7198 "land".to_owned(),
7199 "merge?".to_owned(),
7200 String::new(),
7201 vec!["merge".to_owned(), "hold".to_owned()],
7202 )
7203 }
7204
7205 #[test]
7206 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
7207 crate::run::pin_test_home();
7208 let mut state = run_state(RunStatus::Landing);
7209 state.id = "20260101-000000-fre1".to_owned();
7210 state.parked = true;
7211 state.save().unwrap();
7212 ask::Questions::open()
7213 .put(&mut approval_question(&state.id))
7214 .unwrap();
7215
7216 let mut t = task();
7217 t.runs.push(state.id.clone());
7218 assert_eq!(
7219 land_resume_state(&t),
7220 LandResume::StillWaiting,
7221 "nobody has answered and the timeout has not passed"
7222 );
7223 }
7224
7225 #[test]
7226 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
7227 crate::run::pin_test_home();
7228 let mut state = run_state(RunStatus::Gating);
7229 state.id = "20260101-000000-gt01".to_owned();
7230 state.parked = true;
7231 state.config.graph.answer_timeout = 60;
7232 state.github_text = Some(crate::run::GithubTextAsk {
7233 rewrite: None,
7234 fingerprint: "f".into(),
7235 title: true,
7236 body: false,
7237 categories: vec!["title-language".into()],
7238 question: None,
7239 asks: 1,
7240 resolved: false,
7241 chosen_title: None,
7242 choice: None,
7243 });
7244 state.save().unwrap();
7245 let store = ask::Questions::open();
7246 let mut q = ask::Question::new(
7247 state.id.clone(),
7248 crate::github_text::ASK_NODE.to_owned(),
7249 crate::github_text::ASK_SEAT.to_owned(),
7250 "s".into(),
7251 String::new(),
7252 vec!["use fallback".into()],
7253 );
7254 store.put(&mut q).unwrap();
7255 let mut t = task();
7256 t.runs.push(state.id.clone());
7257 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
7258 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7259 store.put(&mut q).unwrap();
7260 assert_eq!(land_resume_state(&t), LandResume::Ready);
7261 assert!(!store.get(&q.id).unwrap().status.open());
7262 state.github_text.as_mut().unwrap().resolved = true;
7264 state.save().unwrap();
7265 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
7266 }
7267
7268 #[test]
7269 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
7270 crate::run::pin_test_home();
7275 let mut state = run_state(RunStatus::Landing);
7276 state.id = "20260101-000000-exp1".to_owned();
7277 state.parked = true;
7278 state.config.graph.answer_timeout = 60;
7279 state.save().unwrap();
7280
7281 let store = ask::Questions::open();
7282 let mut q = approval_question(&state.id);
7283 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7284 store.put(&mut q).unwrap();
7285
7286 let mut t = task();
7287 t.runs.push(state.id.clone());
7288 assert_eq!(
7289 land_resume_state(&t),
7290 LandResume::Ready,
7291 "an expired question must not be waited on forever"
7292 );
7293
7294 let after = store.get(&q.id).unwrap();
7295 assert!(
7296 !after.status.open(),
7297 "the question is abandoned, not silently ignored"
7298 );
7299 assert!(
7300 after.resolution().is_none(),
7301 "an abandoned question is not read as a decision"
7302 );
7303 }
7304
7305 #[test]
7306 fn reclaim_settles_a_running_task_against_its_last_run() {
7307 let mut t = task();
7308 t.start("20260904-000000-4043".to_owned());
7309 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
7310 assert_eq!(
7311 t.status,
7312 TaskStatus::Done,
7313 "a run that actually finished must not stay `running` forever"
7314 );
7315 }
7316
7317 #[test]
7318 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
7319 let mut t = task();
7320 t.start("20260904-000000-4043".to_owned());
7321 t.last_error = Some("earlier trouble".to_owned());
7322 let mut state = run_state(RunStatus::Judging);
7323 state.parked = true;
7324 reclaim(&mut t, Some(state), 1, "en");
7325 assert_eq!(t.status, TaskStatus::Parked);
7326 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
7327 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7328 assert!(t.park_reason.is_some());
7329 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
7330 assert!(!t.fresh_start);
7331 assert!(t.status.runnable());
7332 }
7333
7334 #[test]
7335 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
7336 let mut t = task();
7337 t.start("20260904-000000-4043".to_owned());
7338 t.last_error = Some("earlier trouble".to_owned());
7339 settle(
7340 &mut t,
7341 Verdict {
7342 status: RunStatus::Judging,
7343 left_pr: false,
7344 quota_hit: false,
7345 parked: true,
7346 no_viable_candidates: false,
7347 },
7348 "parked after `judging`",
7349 1,
7350 );
7351 assert_eq!(t.status, TaskStatus::Parked);
7352 assert_eq!(t.attempts, 0);
7353 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7354 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
7355 assert_eq!(t.schema, crate::queue::SCHEMA);
7356 assert!(
7357 crate::notices::task_held(&t).is_none(),
7358 "a park is not news"
7359 );
7360 t.start("20260904-000000-4043".to_owned());
7361 assert_eq!(t.park_reason, None);
7362 }
7363
7364 #[test]
7365 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
7366 let mut t = task();
7370 t.start("20260904-000000-4043".to_owned());
7371 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
7372 assert_eq!(t.status, TaskStatus::Failed);
7373 assert!(t.status.runnable());
7374 }
7375
7376 #[test]
7377 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
7378 let mut t = task();
7379 t.start("20260904-000000-4043".to_owned());
7380 assert_eq!(t.attempts, 1);
7381 let mut run = run_state(RunStatus::Implementing);
7382 run.id = "20260904-000000-4043".to_owned();
7383 run.begin_pass();
7384 let cause = crate::run::ParkCause::Upgrade {
7385 from: "0.121.0".to_owned(),
7386 to: Some("0.121.1".to_owned()),
7387 };
7388 run.end_pass_parked(cause.clone());
7389 run.parked = true;
7390 let verdict = Verdict {
7391 status: run.status,
7392 left_pr: false,
7393 quota_hit: false,
7394 parked: true,
7395 no_viable_candidates: false,
7396 };
7397 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
7398 assert_eq!(t.attempts, 0);
7399 assert_eq!(t.passes.len(), 1);
7400 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7401 assert_eq!(t.passes[0].parked, Some(cause));
7402 assert_eq!(t.passes[0].pass_index, 0);
7403
7404 t.start("20260904-000000-4043".to_owned());
7407 run.begin_pass();
7408 run.status = RunStatus::Failed;
7409 let verdict = Verdict {
7410 status: RunStatus::Failed,
7411 left_pr: false,
7412 quota_hit: false,
7413 parked: false,
7414 no_viable_candidates: false,
7415 };
7416 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
7417 assert_eq!(t.passes.len(), 2);
7418 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7419 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
7420 assert_eq!(t.passes[1].pass_index, 1);
7421 assert_eq!(t.attempts, 1);
7422 }
7423
7424 #[test]
7425 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
7426 let mut t = task();
7427 t.start("20260904-000000-4043".to_owned());
7428 reclaim(&mut t, None, 2, "en");
7429 assert_eq!(t.status, TaskStatus::Held);
7430 assert!(
7431 t.last_error
7432 .as_deref()
7433 .is_some_and(|e| e.contains("running")),
7434 "the operator needs to know why this task was held"
7435 );
7436 }
7437
7438 #[test]
7439 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
7440 let dir = tempfile::tempdir().unwrap();
7441 let queue = Queue::at(dir.path().to_path_buf());
7442
7443 let mut orphaned = task();
7445 orphaned.id = "20260904-000000-orph".to_owned();
7446 orphaned.status = TaskStatus::Running;
7447 orphaned.attempts = 1;
7448 queue.put(&mut orphaned).unwrap();
7449
7450 let mut alive = task();
7451 alive.id = "20260904-000000-live".to_owned();
7452 alive.status = TaskStatus::Running;
7453 alive.attempts = 1;
7454 queue.put(&mut alive).unwrap();
7455 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
7456
7457 let mut queued = task();
7458 queued.id = "20260904-000000-wait".to_owned();
7459 queue.put(&mut queued).unwrap();
7460
7461 let reclaimed = reclaim_orphaned_running(&queue, 2);
7462 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
7463
7464 assert_eq!(
7465 queue.get(&orphaned.id).unwrap().status,
7466 TaskStatus::Held,
7467 "nothing was driving it and there was no run to recover"
7468 );
7469 assert_eq!(
7470 queue.get(&alive.id).unwrap().status,
7471 TaskStatus::Running,
7472 "a live claim must protect the task it belongs to"
7473 );
7474 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
7475 }
7476
7477 fn read_run_under(home: &Path, id: &str) -> RunState {
7483 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
7484 serde_json::from_str(&body).unwrap()
7485 }
7486
7487 #[test]
7488 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
7489 let dir = tempfile::tempdir().unwrap();
7490 let home = dir.path().to_path_buf();
7491 let now = Timestamp::now();
7492 let overrun_seat = || crate::run::ActiveSeat {
7493 node: "implement".to_owned(),
7494 started_at: now - jiff::SignedDuration::new(21_000, 0),
7495 timeout_secs: 3_600,
7496 attempt: 0,
7497 task: None,
7498 command: None,
7499 index: None,
7500 total: None,
7501 };
7502
7503 let mut dead = run_state(RunStatus::Implementing);
7504 dead.id = "20260101-000000-dead".to_owned();
7505 dead.active.insert("impl-A".to_owned(), overrun_seat());
7506 dead.driver_pid = Some(4242);
7509 dead.save_under(&home).unwrap();
7510
7511 let mut alive = run_state(RunStatus::Implementing);
7514 alive.id = "20260101-000000-aliv".to_owned();
7515 alive.active.insert("impl-A".to_owned(), overrun_seat());
7516 alive.save_under(&home).unwrap();
7517 let mut status = Status::new();
7518 status.current = vec![Current {
7519 task: "20260101-000000-task".to_owned(),
7520 run: alive.id.clone(),
7521 }];
7522 write_status_to(&home.join("daemon.json"), &status).unwrap();
7523
7524 let questions = Questions::at(home.join("questions"));
7528 let mut q = ask::Question::new(
7529 dead.id.clone(),
7530 "implement".to_owned(),
7531 "impl-A".to_owned(),
7532 "Which storage backend?".to_owned(),
7533 String::new(),
7534 vec!["SQLite".to_owned(), "Redis".to_owned()],
7535 );
7536 questions.put(&mut q).unwrap();
7537
7538 let abandoned = reclaim_abandoned_runs_with(
7539 &home,
7540 now,
7541 |pid| if pid == 4242 { Some(false) } else { None },
7542 |_| panic!("a query answering Dead outright needs no identity corroboration"),
7543 );
7544 assert_eq!(abandoned, vec![dead.id.clone()]);
7545
7546 let reloaded = read_run_under(&home, &dead.id);
7547 assert_eq!(reloaded.status, RunStatus::Failed);
7548 assert!(reloaded.active.is_empty());
7549 assert!(
7550 !questions.get(&q.id).unwrap().status.open(),
7551 "the failed run's own open question must be settled in the same pass"
7552 );
7553
7554 let still_alive = read_run_under(&home, &alive.id);
7555 assert_eq!(
7556 still_alive.status,
7557 RunStatus::Implementing,
7558 "a live daemon's claim protects it"
7559 );
7560 assert!(!still_alive.active.is_empty());
7561 }
7562
7563 #[test]
7573 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
7574 let dir = tempfile::tempdir().unwrap();
7575 let home = dir.path().to_path_buf();
7576 let now = Timestamp::now();
7577
7578 let mut manual = run_state(RunStatus::Reviewing);
7579 manual.id = "20260101-000000-manl".to_owned();
7580 manual.active.insert(
7581 "review-1".to_owned(),
7582 crate::run::ActiveSeat {
7583 node: "review".to_owned(),
7584 started_at: now - jiff::SignedDuration::new(21_000, 0),
7585 timeout_secs: 3_600,
7586 attempt: 0,
7587 task: None,
7588 command: None,
7589 index: None,
7590 total: None,
7591 },
7592 );
7593 manual.driver_pid = Some(4242);
7597 manual.driver_started_at = Some("1790000000".to_owned());
7598 manual.save_under(&home).unwrap();
7599
7600 let abandoned = reclaim_abandoned_runs_with(
7601 &home,
7602 now,
7603 |pid| if pid == 4242 { Some(true) } else { None },
7604 |pid| {
7605 if pid == 4242 {
7606 Some("1790000000".to_owned())
7607 } else {
7608 None
7609 }
7610 },
7611 );
7612 assert!(
7613 abandoned.is_empty(),
7614 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
7615 );
7616
7617 let reloaded = read_run_under(&home, &manual.id);
7618 assert_eq!(reloaded.status, RunStatus::Reviewing);
7619 assert!(!reloaded.active.is_empty());
7620 }
7621
7622 #[test]
7623 fn an_already_claimed_task_is_skipped_rather_than_failed() {
7624 let dir = tempfile::tempdir().unwrap();
7625 let queue = Queue::at(dir.path().to_path_buf());
7626 let mut only = task();
7627 queue.put(&mut only).unwrap();
7628
7629 let _elsewhere = queue.claim(&only.id).unwrap();
7630 let candidates = runnable(&queue);
7631 assert_eq!(candidates.len(), 1, "the task is still runnable");
7632 assert!(
7633 queue.claim(&candidates[0].id).is_err(),
7634 "the loop cannot take a claim somebody else holds"
7635 );
7636
7637 let after = queue.get(&only.id).unwrap();
7638 assert_eq!(after.status, TaskStatus::Queued);
7639 assert_eq!(
7640 after.attempts, 0,
7641 "losing the race is not an attempt at the task"
7642 );
7643 assert_eq!(after.last_error, None);
7644 }
7645
7646 #[test]
7647 fn the_status_file_round_trips_and_its_heartbeat_advances() {
7648 let dir = tempfile::tempdir().unwrap();
7649 let path = dir.path().join("daemon.json");
7650
7651 let mut status = Status::new();
7652 status.idle = false;
7653 status.completed = 7;
7654 status.current = vec![Current {
7655 task: "20260902-000000-t111".to_owned(),
7656 run: "20260902-000001-r111".to_owned(),
7657 }];
7658 write_status_to(&path, &status).unwrap();
7659 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7660 assert_eq!(first.schema, SCHEMA);
7661 assert_eq!(first.pid, std::process::id());
7662 assert!(!first.idle);
7663 assert_eq!(first.completed, 7);
7664 assert_eq!(first.current, status.current);
7665 assert!(
7666 !path.with_extension("json.tmp").exists(),
7667 "the temp file is renamed, not left behind"
7668 );
7669
7670 std::thread::sleep(Duration::from_millis(5));
7671 status.updated_at = Timestamp::now();
7672 status.polls = 3;
7673 write_status_to(&path, &status).unwrap();
7674 let second: Status =
7675 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7676 assert!(
7677 second.updated_at > first.updated_at,
7678 "a reader can only detect staleness if the heartbeat moves"
7679 );
7680 assert_eq!(
7681 second.started_at, first.started_at,
7682 "the start time is not a heartbeat"
7683 );
7684 assert_eq!(second.polls, 3);
7685 }
7686
7687 #[test]
7688 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
7689 let dir = tempfile::tempdir().unwrap();
7690
7691 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
7692
7693 let mut status = Status::new();
7694 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
7695 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7696 let stale = read_status(dir.path()).unwrap();
7697 assert!(
7698 !stale.running(Timestamp::now()),
7699 "a minute without a heartbeat is a dead daemon, not a busy one"
7700 );
7701 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
7702
7703 status.updated_at = Timestamp::now();
7704 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7705 let fresh = read_status(dir.path()).unwrap();
7706 assert!(fresh.running(Timestamp::now()));
7707 }
7708
7709 #[test]
7710 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
7711 let dir = tempfile::tempdir().unwrap();
7712 let now = Timestamp::now();
7713 let mine = "20260903-080619-01c2";
7714
7715 assert!(
7716 !is_working_on(dir.path(), mine, now),
7717 "no status file means nobody is working on anything"
7718 );
7719
7720 let mut status = Status::new();
7721 status.current = vec![Current {
7722 task: "20260903-080340-0167".to_owned(),
7723 run: mine.to_owned(),
7724 }];
7725 status.updated_at = now;
7726 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7727 assert!(is_working_on(dir.path(), mine, now));
7728 assert!(
7729 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
7730 "a daemon busy with one run is not working on another"
7731 );
7732
7733 status.updated_at = now - jiff::SignedDuration::from_secs(600);
7736 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7737 assert!(
7738 !is_working_on(dir.path(), mine, now),
7739 "a stale heartbeat is a dead daemon, so its run is a leftover"
7740 );
7741 }
7742
7743 #[test]
7744 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
7745 let dir = tempfile::tempdir().unwrap();
7746 let now = Timestamp::now();
7747
7748 assert!(
7749 !is_working_on_short(dir.path(), "01c2", now),
7750 "no status file means nobody is working on anything"
7751 );
7752
7753 let mut status = Status::new();
7754 status.current = vec![Current {
7755 task: "20260903-080340-0167".to_owned(),
7756 run: "20260903-080619-01c2".to_owned(),
7757 }];
7758 status.updated_at = now;
7759 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7760 assert!(
7761 is_working_on_short(dir.path(), "01c2", now),
7762 "the run's short id is the last block of its full id"
7763 );
7764 assert!(
7765 !is_working_on_short(dir.path(), "3cbf", now),
7766 "a daemon busy with one worktree bay is not working on another"
7767 );
7768 }
7769
7770 #[test]
7771 fn a_newer_status_file_still_yields_a_reading() {
7772 let dir = tempfile::tempdir().unwrap();
7773 std::fs::write(
7776 dir.path().join("daemon.json"),
7777 serde_json::json!({
7778 "schema": 2,
7779 "updated_at": Timestamp::now().to_string(),
7780 "idle": true,
7781 "surprise": { "nested": [1, 2, 3] },
7782 })
7783 .to_string(),
7784 )
7785 .unwrap();
7786
7787 let reading = read_status(dir.path()).expect("a forward-compatible read");
7788 assert!(reading.running(Timestamp::now()));
7789 assert!(reading.idle);
7790 assert!(reading.current.is_empty());
7791 }
7792
7793 #[test]
7794 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
7795 let dir = tempfile::tempdir().unwrap();
7801 std::fs::write(
7802 dir.path().join("daemon.json"),
7803 serde_json::json!({
7804 "schema": 1,
7805 "pid": 4242,
7806 "updated_at": Timestamp::now().to_string(),
7807 "idle": false,
7808 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
7809 "completed": 3,
7810 "polls": 9,
7811 })
7812 .to_string(),
7813 )
7814 .unwrap();
7815
7816 let reading = read_status(dir.path()).expect("an older shape must still parse");
7817 assert!(reading.running(Timestamp::now()));
7818 assert_eq!(
7819 reading.current,
7820 vec![Current {
7821 task: "20260902-140501-aaaa".to_owned(),
7822 run: "20260902-140502-bbbb".to_owned(),
7823 }]
7824 );
7825 }
7826
7827 #[test]
7828 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
7829 let dir = tempfile::tempdir().unwrap();
7830 std::fs::write(
7831 dir.path().join("daemon.json"),
7832 serde_json::json!({
7833 "schema": 1,
7834 "updated_at": Timestamp::now().to_string(),
7835 "idle": true,
7836 "current": null,
7837 })
7838 .to_string(),
7839 )
7840 .unwrap();
7841 let with_null = read_status(dir.path()).expect("null must still parse");
7842 assert!(with_null.current.is_empty());
7843
7844 std::fs::write(
7845 dir.path().join("daemon.json"),
7846 serde_json::json!({
7847 "schema": 1,
7848 "updated_at": Timestamp::now().to_string(),
7849 "idle": true,
7850 })
7851 .to_string(),
7852 )
7853 .unwrap();
7854 let absent = read_status(dir.path()).expect("a missing field must still parse");
7855 assert!(absent.current.is_empty());
7856 }
7857
7858 #[test]
7859 fn a_task_without_a_repository_runs_in_the_daemons_default() {
7860 let fallback = Path::new("/default");
7861 let mut blank = task();
7862 blank.repo = PathBuf::new();
7863 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
7864 let mut dot = task();
7865 dot.repo = PathBuf::from(".");
7866 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
7867 assert_eq!(
7868 repo_for(&task(), fallback),
7869 PathBuf::from("/repo"),
7870 "a task that names a repository keeps it"
7871 );
7872 }
7873
7874 #[test]
7875 fn task_implementers_edit_inherits_daemon_config_and_merge() {
7876 let dir = tempfile::tempdir().unwrap();
7877 let path = dir.path().join("explicit.toml");
7878 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
7879 let opts = Opts {
7880 config: Some(path),
7881 merge: Some("none".into()),
7882 ..Default::default()
7883 };
7884 let mut t = task();
7885 t.set_implementers(
7886 2,
7887 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7888 .unwrap(),
7889 )
7890 .unwrap();
7891 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
7892 assert_eq!(cfg.graph.implementers, 2);
7893 assert_eq!(cfg.graph.judges, 1);
7894 assert_eq!(cfg.merge.mode, MergeMode::None);
7895 t.set_implementers(1, &cfg).unwrap();
7896 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
7897 apply_task_implementers(&mut cfg, &t);
7898 assert_eq!(cfg.graph.implementers, 1);
7899 assert_eq!(cfg.graph.judges, 1);
7900 assert_eq!(cfg.merge.mode, MergeMode::None);
7901 }
7902
7903 #[test]
7904 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
7905 let dir = tempfile::tempdir().unwrap();
7906 let repo = dir.path().join("repo");
7907 std::fs::create_dir(&repo).unwrap();
7908 std::fs::write(
7909 repo.join("magi.toml"),
7910 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
7911 )
7912 .unwrap();
7913 let daemon_config = dir.path().join("daemon.toml");
7914 std::fs::write(
7915 &daemon_config,
7916 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
7917 )
7918 .unwrap();
7919 let opts = Opts {
7920 config: Some(daemon_config),
7921 merge: Some("pr".into()),
7922 ..Default::default()
7923 };
7924 let q = Queue::at(dir.path().join("queue"));
7925 let validation: Config =
7926 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7927 .unwrap();
7928 for record in ["{}", r#"{"candidates":2}"#] {
7930 let overrides = serde_json::from_str(record).unwrap();
7931 let filing = crate::direct::Filing {
7932 instruction: "work".into(),
7933 title: "direct".into(),
7934 repo: repo.clone(),
7935 source: crate::queue::Source::Human,
7936 solo: false,
7937 overrides,
7938 review_of: None,
7939 };
7940 let filed =
7941 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
7942 let crate::direct::Filed::Standalone {
7943 mut task,
7944 claim: _claim,
7945 } = filed
7946 else {
7947 panic!("local filing")
7948 };
7949 let before = prepare_for(&repo, &opts, &task).unwrap();
7950 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
7951 assert_eq!(
7952 before.merge.mode,
7953 MergeMode::None,
7954 "daemon --merge must not leak"
7955 );
7956 for n in [2, 1, 2] {
7957 task.set_implementers(n, &validation).unwrap();
7958 q.put(&mut task).unwrap();
7959 task = q.get(&task.id).unwrap();
7960 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
7961 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
7962 apply_task_implementers(&mut cfg, &task);
7963 assert_eq!(cfg.graph.implementers, usize::from(n));
7964 assert_eq!(cfg.graph.judges, 2);
7965 assert_eq!(cfg.merge.mode, MergeMode::None);
7966 }
7967 }
7968 }
7969
7970 #[test]
7971 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
7972 let cfg: Config =
7973 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7974 .unwrap();
7975 let mut t = task();
7976 t.review_of = Some("review-target".into());
7977 t.hold_manual(Some("retry".into()));
7978 let mut previous = RunState::new(
7979 PathBuf::from("/repo"),
7980 "main".into(),
7981 "abc1234".into(),
7982 "review".into(),
7983 cfg.clone(),
7984 );
7985 previous.config.graph.implementers = 2;
7986 for count in [1, 3] {
7987 t.set_implementers(count, &cfg).unwrap();
7988 t.release();
7989 assert_eq!(t.review_of.as_deref(), Some("review-target"));
7990 for proposed in [
7991 Starter::Resume(previous.id.clone()),
7992 Starter::Review("recovery-target".into()),
7993 ] {
7994 assert_eq!(
7995 starter_with_implementers(&t, Some(&previous), proposed),
7996 Starter::Review("review-target".into())
7997 );
7998 }
7999 let mut configured = cfg.clone();
8000 apply_task_implementers(&mut configured, &t);
8001 assert_eq!(configured.graph.implementers, usize::from(count));
8002 assert_eq!(previous.config.graph.implementers, 2);
8003 }
8004 }
8005
8006 #[test]
8007 fn task_implementers_dispatch_and_resume_agree() {
8008 let mut t = task();
8009 let cfg: crate::config::Config =
8010 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8011 .unwrap();
8012 let mut run = RunState::new(
8013 PathBuf::from("/repo"),
8014 "main".into(),
8015 "abc1234".into(),
8016 "work".into(),
8017 cfg.clone(),
8018 );
8019 run.config.graph.implementers = 2;
8020 for n in [2, 1, 3, 2] {
8021 t.set_implementers(n, &cfg).unwrap();
8022 let mut dispatched = cfg.clone();
8023 apply_task_implementers(&mut dispatched, &t);
8024 assert_eq!(dispatched.graph.implementers, usize::from(n));
8025 assert_eq!(implementers_match(&t, &run), n == 2);
8026 for proposed in [
8027 Starter::Resume(run.id.clone()),
8028 Starter::Review("old-branch".into()),
8029 ] {
8030 let expected = if n == 2 {
8031 proposed.clone()
8032 } else {
8033 Starter::Start
8034 };
8035 assert_eq!(
8036 starter_with_implementers(&t, Some(&run), proposed),
8037 expected
8038 );
8039 }
8040 assert_eq!(
8041 run.config.graph.implementers, 2,
8042 "history must not be rewritten"
8043 );
8044 }
8045 t.solo = false;
8046 t.overrides = None;
8047 assert!(implementers_match(&t, &run));
8048 }
8049
8050 #[test]
8051 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
8052 let mut solo_cfg = Config::default();
8058 solo_cfg.graph.implementers = 3;
8059 let mut solo_task = task();
8060 solo_task.solo = true;
8061 apply_task_implementers(&mut solo_cfg, &solo_task);
8062 assert_eq!(solo_cfg.graph.implementers, 1);
8063
8064 let mut plain_cfg = Config::default();
8065 plain_cfg.graph.implementers = 3;
8066 let plain_task = task();
8067 assert!(!plain_task.solo);
8068 apply_task_implementers(&mut plain_cfg, &plain_task);
8069 assert_eq!(
8070 plain_cfg.graph.implementers, 3,
8071 "a task that did not ask to run alone keeps the config's candidates"
8072 );
8073 }
8074
8075 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
8076 QuotaLoss {
8077 seat: seat.into(),
8078 node: "judge".into(),
8079 at: at.parse().unwrap(),
8080 reset: reset.map(str::to_string),
8081 }
8082 }
8083
8084 #[test]
8085 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
8086 let old: Vec<QuotaLoss> = (1..=4)
8087 .map(|i| {
8088 loss(
8089 &format!("judge-{i}"),
8090 "2026-09-23T05:23:00Z",
8091 Some("2:40pm (Asia/Tokyo)"),
8092 )
8093 })
8094 .collect();
8095 let fresh = losses_this_attempt(&old, &old);
8096 assert!(fresh.is_empty());
8097 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
8098 }
8100
8101 #[test]
8102 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
8103 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
8104 let now = Timestamp::now();
8105 let mut after = old.clone();
8106 after.push(loss("judge-2", &now.to_string(), None));
8107 let fresh = losses_this_attempt(&old, &after);
8108 assert_eq!(fresh, vec![after[1].clone()]);
8109 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
8110 assert_eq!(
8111 until,
8112 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
8113 );
8114 }
8115
8116 #[test]
8117 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
8118 let before = vec![
8121 loss("judge-1", "2026-09-23T05:23:00Z", None),
8122 loss("judge-2", "2026-09-23T05:24:00Z", None),
8123 ];
8124 let after = vec![
8125 loss("judge-2", "2026-09-23T05:24:00Z", None),
8126 loss("judge-1", "2026-09-24T01:00:00Z", None),
8127 ];
8128 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
8129 }
8130
8131 #[test]
8132 fn merge_overrides_are_parsed_or_refused() {
8133 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
8134 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
8135 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
8136 assert!(merge_mode("squash").is_err());
8137 }
8138
8139 #[test]
8140 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
8141 let now = Timestamp::now();
8142 let fallback = Duration::from_secs(300);
8143 let cap = Duration::from_secs(1800);
8144
8145 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
8147
8148 let soon = now + jiff::SignedDuration::from_secs(600);
8150 assert_eq!(
8151 quota_wait(Some(soon), now, fallback, cap),
8152 Duration::from_secs(600)
8153 );
8154
8155 let past = now - jiff::SignedDuration::from_secs(60);
8158 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
8159
8160 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
8163 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
8164 }
8165
8166 #[test]
8167 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
8168 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8169
8170 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
8171 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
8172
8173 let already_past =
8177 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
8178 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
8179
8180 assert!(
8181 parse_reset_hint("session limit reached", now, now).is_none(),
8182 "free text with no recognised shape is not guessed at"
8183 );
8184 assert!(
8185 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
8186 "an unresolvable zone name is not guessed at either"
8187 );
8188 }
8189
8190 #[test]
8191 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
8192 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8193
8194 let at = parse_reset_hint(
8195 "You've hit your usage limit. Visit \
8196 https://chatgpt.com/codex/settings/usage to purchase more \
8197 credits or try again at Sep 19th, 2026 5:10 PM.",
8198 now,
8199 now,
8200 )
8201 .expect("the codex reset wording is a recognised shape");
8202 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
8203
8204 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
8209 .expect("an explicit year needs no rollover");
8210 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
8211
8212 assert!(
8213 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
8214 "a two-digit year is not the documented shape and is not guessed at"
8215 );
8216 assert!(
8217 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
8218 "a four-letter month name is not the documented three-letter abbreviation"
8219 );
8220 assert!(
8221 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
8222 "an explicit zone on the dated shape is a format nobody has \
8223 documented, and is refused rather than guessed at as UTC"
8224 );
8225 }
8226
8227 #[test]
8228 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
8229 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
8230 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
8231
8232 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
8233 assert_eq!(at.as_second() - recorded.as_second(), 3769);
8234
8235 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
8236 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
8237
8238 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
8239 assert!(
8240 parse_reset_hint(bad, now, recorded).is_none(),
8241 "{bad:?} must not be guessed at"
8242 );
8243 }
8244 }
8245
8246 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
8250 let config = dir.join("magi.toml");
8251 std::fs::write(
8252 &config,
8253 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
8254 )
8255 .unwrap();
8256 let opts = Opts {
8257 poll: Duration::from_secs(30),
8258 config: Some(config),
8259 repo: dir.join("repo"),
8263 ..Opts::default()
8264 };
8265 let home = dir.join("home");
8274 let worktrees = dir.join("wt");
8275 (
8276 opts,
8277 Queue::at(dir.join("queue")),
8278 home.join("daemon.json"),
8279 home,
8280 worktrees,
8281 )
8282 }
8283
8284 #[test]
8285 fn a_stop_is_idempotent_and_once_set_stays_set() {
8286 let stop = Stop::new();
8287 assert!(!stop.stopped());
8288
8289 stop.stop();
8290 assert!(stop.stopped());
8291 stop.stop();
8292 assert!(stop.stopped(), "a second stop is not a toggle");
8293
8294 let shared = stop.clone();
8295 assert!(
8296 shared.stopped(),
8297 "a clone is the same stop; that is how the loop and its caller share one"
8298 );
8299 }
8300
8301 #[test]
8302 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
8303 let stop = Stop::new();
8304 stop.enter();
8305 assert!(
8306 !stop.finishing(),
8307 "a busy loop nobody has asked to stop is just running"
8308 );
8309
8310 stop.stop();
8311 assert!(
8312 stop.finishing(),
8313 "a stop asked for mid-run has not landed until the run is settled"
8314 );
8315
8316 stop.exit();
8317 assert!(
8318 !stop.finishing(),
8319 "once the run is settled the stop has landed and there is nothing to finish"
8320 );
8321 }
8322
8323 #[test]
8324 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
8325 let stop = Stop::new();
8326 stop.enter();
8327 stop.enter();
8328 stop.stop();
8329 assert!(stop.finishing(), "two runs still in flight");
8330
8331 stop.exit();
8332 assert!(
8333 stop.finishing(),
8334 "one run finished, but a sibling is still working"
8335 );
8336
8337 stop.exit();
8338 assert!(
8339 !stop.finishing(),
8340 "the last run out is what actually lands the stop"
8341 );
8342 }
8343
8344 #[tokio::test]
8345 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
8346 let dir = tempfile::tempdir().unwrap();
8347 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8348 let stop = Stop::new();
8349 stop.stop();
8350
8351 let began = std::time::Instant::now();
8352 tokio::time::timeout(
8353 Duration::from_secs(2),
8354 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8355 )
8356 .await
8357 .expect("a stopped loop must return, not sit out its poll interval")
8358 .expect("the loop's own setup and teardown must not fail");
8359 assert!(
8360 began.elapsed() < opts.poll,
8361 "returned only after {:?}, which is a poll interval, not a stop",
8362 began.elapsed()
8363 );
8364 }
8365
8366 #[tokio::test]
8367 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
8368 let dir = tempfile::tempdir().unwrap();
8369 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8370 let stop = Stop::new();
8371
8372 let asker = {
8375 let stop = stop.clone();
8376 tokio::spawn(async move {
8377 tokio::time::sleep(Duration::from_millis(20)).await;
8378 stop.stop();
8379 })
8380 };
8381
8382 let began = std::time::Instant::now();
8383 tokio::time::timeout(
8384 Duration::from_secs(2),
8385 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8386 )
8387 .await
8388 .expect("a stop asked for while idle must wake the wait")
8389 .expect("the loop's own setup and teardown must not fail");
8390 asker.await.unwrap();
8391 assert!(
8392 began.elapsed() < opts.poll,
8393 "returned only after {:?}, so the stop waited on the sleep",
8394 began.elapsed()
8395 );
8396 }
8397
8398 #[tokio::test]
8399 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
8400 let dir = tempfile::tempdir().unwrap();
8401 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8402 let stop = Stop::new();
8403 stop.stop();
8404
8405 tokio::time::timeout(
8406 Duration::from_secs(2),
8407 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8408 )
8409 .await
8410 .expect("a stopped loop must return")
8411 .expect("the loop's own setup and teardown must not fail");
8412
8413 assert!(
8414 home.is_dir(),
8415 "the loop did publish a status file, so its removal is the teardown and not an absence"
8416 );
8417 assert!(
8418 !status_file.exists(),
8419 "a stopped loop clears its status file"
8420 );
8421 assert!(
8422 read_status(&home).is_none(),
8423 "a reader must see no daemon at all, not a heartbeat that merely stopped"
8424 );
8425 }
8426
8427 #[tokio::test]
8428 async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
8429 let dir = tempfile::tempdir().unwrap();
8430 let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8431 opts.once = true;
8432
8433 let mut settled = RunState::new(
8434 dir.path().join("repo"),
8435 "main".to_owned(),
8436 "abc1234".to_owned(),
8437 "fixture".to_owned(),
8438 Config::default(),
8439 );
8440 settled.status = RunStatus::Ready;
8441 let run_dir = home.join("runs").join(&settled.id);
8442 std::fs::create_dir_all(&run_dir).unwrap();
8443 std::fs::write(
8444 run_dir.join("run.json"),
8445 serde_json::to_string_pretty(&settled).unwrap(),
8446 )
8447 .unwrap();
8448 let questions = Questions::at(home.join("questions"));
8449 let mut question = ask::Question::new(
8450 settled.id.clone(),
8451 "review".to_owned(),
8452 "reviewer-1".to_owned(),
8453 "Continue?".to_owned(),
8454 String::new(),
8455 Vec::new(),
8456 );
8457 questions.put(&mut question).unwrap();
8458
8459 drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
8460 .await
8461 .unwrap();
8462
8463 assert_eq!(
8464 questions.get(&question.id).unwrap().status,
8465 ask::QuestionStatus::Abandoned,
8466 "an empty --once drain still performs startup question cleanup"
8467 );
8468 }
8469
8470 #[test]
8471 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
8472 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8473
8474 assert!(
8475 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
8476 "never checked before: due at once"
8477 );
8478
8479 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
8480 assert!(
8481 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
8482 "well inside the interval: not due yet"
8483 );
8484
8485 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
8486 assert!(
8487 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
8488 "exactly at the edge: not yet due, same convention as `clean::due`"
8489 );
8490
8491 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8492 assert!(
8493 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
8494 "past the interval: due again"
8495 );
8496 }
8497
8498 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
8503 let config = dir.join("magi.toml");
8504 std::fs::write(
8510 &config,
8511 format!(
8512 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
8513 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
8514 cache_dir.display()
8515 ),
8516 )
8517 .unwrap();
8518 Opts {
8519 config: Some(config),
8520 repo: dir.join("repo"),
8521 ..Opts::default()
8522 }
8523 }
8524
8525 #[tokio::test]
8526 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
8527 let dir = tempfile::tempdir().unwrap();
8528 let home = dir.path().join("home");
8529 let cache_dir = dir.path().join("cache");
8530 std::fs::create_dir_all(&cache_dir).unwrap();
8531 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8532 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8533
8534 let running = Stop::new();
8537 let mut last_checked = None;
8538 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8539 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
8540 .await;
8541 assert_eq!(
8542 crate::disk::dir_size(&cache_dir),
8543 0,
8544 "over the cap on the first check ever: pruned at once, no idle queue required"
8545 );
8546 assert_eq!(last_checked, Some(t0));
8547
8548 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
8550 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
8551 maybe_prune_cache_between_runs(
8552 &opts.repo,
8553 &opts,
8554 &home,
8555 &running,
8556 &mut last_checked,
8557 too_soon,
8558 )
8559 .await;
8560 assert_eq!(
8561 crate::disk::dir_size(&cache_dir),
8562 10,
8563 "too soon since the last check: left alone rather than rescanned every call"
8564 );
8565 assert_eq!(
8566 last_checked,
8567 Some(t0),
8568 "an idle check does not reset the clock"
8569 );
8570
8571 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8573 maybe_prune_cache_between_runs(
8574 &opts.repo,
8575 &opts,
8576 &home,
8577 &running,
8578 &mut last_checked,
8579 due_again,
8580 )
8581 .await;
8582 assert_eq!(
8583 crate::disk::dir_size(&cache_dir),
8584 0,
8585 "due again: pruned back under the cap"
8586 );
8587 }
8588
8589 #[tokio::test]
8597 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
8598 let dir = tempfile::tempdir().unwrap();
8599 let home = dir.path().join("home");
8600 let cache_dir = dir.path().join("cache");
8601 std::fs::create_dir_all(&cache_dir).unwrap();
8602 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8603 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8604
8605 let stop = Stop::new();
8606 stop.stop();
8607 assert!(
8608 !stop.finishing(),
8609 "no run is in flight at a between-runs boundary, so nothing else \
8610 would tell the operator this stop had not taken effect yet"
8611 );
8612
8613 let mut last_checked = None;
8614 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8615 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
8616 .await;
8617 assert_eq!(
8618 crate::disk::dir_size(&cache_dir),
8619 10,
8620 "over its cap, and due for the first check ever, but a stop outranks \
8621 it: the cap is a standing policy the next start measures again"
8622 );
8623 assert_eq!(
8624 last_checked, None,
8625 "a check that never happened must not claim the interval"
8626 );
8627 }
8628
8629 #[tokio::test]
8640 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
8641 let dir = tempfile::tempdir().unwrap();
8642 let cache_dir = dir.path().join("cache");
8643 std::fs::create_dir_all(&cache_dir).unwrap();
8644 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
8645
8646 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
8647 opts.poll = Duration::from_millis(20);
8648 opts.max_attempts = 1_000;
8649
8650 let queue = Queue::at(dir.path().join("queue"));
8651 for i in 0..128 {
8652 let mut task = Task::new(
8653 format!("queued work {i}"),
8654 format!("queued work {i}"),
8655 opts.repo.clone(),
8656 Source::Human,
8657 );
8658 queue.put(&mut task).unwrap();
8659 }
8660
8661 let home = dir.path().join("home");
8662 let worktrees = dir.path().join("wt");
8663 let status_file = home.join("daemon.json");
8664 let stop = Stop::new();
8665 let stopper = {
8666 let stop = stop.clone();
8667 let queue = Queue::at(dir.path().join("queue"));
8668 tokio::spawn(async move {
8669 for _ in 0..2_000 {
8673 if queue.list().iter().filter(|t| t.attempts == 1).count() >= 2 {
8674 break;
8675 }
8676 tokio::time::sleep(Duration::from_millis(5)).await;
8677 }
8678 stop.stop();
8679 })
8680 };
8681
8682 tokio::time::timeout(
8683 Duration::from_secs(20),
8684 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8685 )
8686 .await
8687 .expect("the loop must not hang on a queue that keeps producing failing work")
8688 .expect("the loop's own setup and teardown must not fail");
8689 stopper.await.unwrap();
8690
8691 let after = queue.list();
8692 let completed = after.iter().filter(|t| t.attempts == 1).count();
8693 assert!(
8694 completed >= 2,
8695 "the harness must have processed multiple tasks to exercise a busy queue \
8696 (got {completed} completed task(s))"
8697 );
8698 assert!(
8699 after.iter().any(|t| t.status.runnable()),
8700 "unprocessed tasks must remain: the external stop, rather than an \
8701 exhausted backlog, must end the test"
8702 );
8703 assert!(
8704 after
8705 .iter()
8706 .filter(|t| t.attempts > 0)
8707 .all(|t| { t.attempts == 1 && t.status == TaskStatus::Held }),
8708 "a busy backlog must not retry a task's unchanged startup failure"
8709 );
8710
8711 assert_eq!(
8712 crate::disk::dir_size(&cache_dir),
8713 0,
8714 "an oversized cache must not be left to grow unboundedly just because the \
8715 queue kept the loop busy the whole time"
8716 );
8717 }
8718
8719 #[test]
8720 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
8721 let dir = tempfile::tempdir().unwrap();
8722 let queue = Queue::at(dir.path().join("queue"));
8723 let questions = Questions::at(dir.path().join("questions"));
8724 let mut task = task();
8725 queue.put(&mut task).unwrap();
8726
8727 let mut task_question = ask::Question::new(
8728 task.id.clone(),
8729 crate::conduct::NODE.to_owned(),
8730 "conduct".to_owned(),
8731 "Which backend?".to_owned(),
8732 String::new(),
8733 Vec::new(),
8734 );
8735 questions.put(&mut task_question).unwrap();
8736 task.block(vec![task_question.id.clone()], None);
8737 queue.put(&mut task).unwrap();
8738
8739 let mut run_question = ask::Question::new(
8740 "20260101-000000-run1".to_owned(),
8741 "review".to_owned(),
8742 "reviewer-1".to_owned(),
8743 "Run question".to_owned(),
8744 String::new(),
8745 Vec::new(),
8746 );
8747 questions.put(&mut run_question).unwrap();
8748
8749 let mut coincidental = ask::Question::new(
8754 task.id.clone(),
8755 "review".to_owned(),
8756 "reviewer-1".to_owned(),
8757 "Unrelated review question".to_owned(),
8758 String::new(),
8759 Vec::new(),
8760 );
8761 questions.put(&mut coincidental).unwrap();
8762
8763 reconcile_task_questions(&queue, &questions);
8764 assert!(questions.get(&task_question.id).unwrap().status.open());
8765 assert!(questions.get(&run_question.id).unwrap().status.open());
8766 assert!(questions.get(&coincidental.id).unwrap().status.open());
8767
8768 task.release();
8769 queue.put(&mut task).unwrap();
8770 reconcile_task_questions(&queue, &questions);
8771 assert_eq!(
8772 questions.get(&task_question.id).unwrap().status,
8773 ask::QuestionStatus::Abandoned
8774 );
8775 assert!(
8776 questions.get(&run_question.id).unwrap().status.open(),
8777 "run questions remain the run janitor's responsibility"
8778 );
8779 assert!(
8780 questions.get(&coincidental.id).unwrap().status.open(),
8781 "a non-conductor question must not be abandoned just because its \
8782 run id coincides with a task id"
8783 );
8784 }
8785
8786 #[test]
8787 fn a_freshly_started_running_task_is_never_stalled() {
8788 let dir = tempfile::tempdir().unwrap();
8789 let mut t = task();
8790 t.start("run-1".to_owned());
8791 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
8794 }
8795
8796 #[test]
8797 fn a_long_running_task_with_no_live_daemon_is_stalled() {
8798 let dir = tempfile::tempdir().unwrap();
8799 let mut t = task();
8800 t.start("run-1".to_owned());
8801 t.updated_at = Timestamp::now()
8802 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8803 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
8804 assert_eq!(
8805 stalled_tasks(
8806 &Queue::at(dir.path().join("q")),
8807 dir.path(),
8808 Timestamp::now()
8809 )
8810 .len(),
8811 0,
8812 "the task was never written to this queue"
8813 );
8814 }
8815
8816 #[test]
8817 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
8818 let dir = tempfile::tempdir().unwrap();
8819 let mut t = task();
8820 t.id = "20260903-080340-0167".to_owned();
8821 t.start("20260903-080619-01c2".to_owned());
8822 t.updated_at = Timestamp::now()
8823 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8824
8825 let mut status = Status::new();
8826 status.current = vec![Current {
8827 task: t.id.clone(),
8828 run: "20260903-080619-01c2".to_owned(),
8829 }];
8830 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8831
8832 assert!(
8833 !is_stalled(&t, dir.path(), Timestamp::now()),
8834 "a live daemon's own heartbeat rules out stalled, however long the task has run"
8835 );
8836 }
8837
8838 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
8842 let path = queue.path_of(id);
8843 let body = std::fs::read_to_string(&path).unwrap();
8844 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
8845 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
8846 v["updated_at"] = serde_json::Value::String(old.to_string());
8847 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
8848 }
8849
8850 #[test]
8851 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
8852 let dir = tempfile::tempdir().unwrap();
8865 let queue = Queue::at(dir.path().join("queue"));
8866 let home = dir.path().join("home");
8867
8868 let mut t = task();
8869 t.id = "20260101-000001-lock".to_owned();
8870 t.start("run-1".to_owned());
8871 queue.put(&mut t).unwrap();
8872 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
8873 std::fs::write(
8874 dir.path().join("queue").join(format!("{}.lock", t.id)),
8875 "not a pid",
8876 )
8877 .unwrap();
8878
8879 let now = Timestamp::now();
8880 assert!(
8881 reclaim_orphaned_running(&queue, 2).is_empty(),
8882 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
8883 and reclaim must leave the task alone"
8884 );
8885 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8886
8887 let stalled = stalled_tasks(&queue, &home, now);
8888 assert_eq!(
8889 stalled.len(),
8890 1,
8891 "reclaim's inability to claim it yet must not hide it from the conductor"
8892 );
8893 assert_eq!(stalled[0].id, t.id);
8894 }
8895
8896 #[test]
8897 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
8898 let dir = tempfile::tempdir().unwrap();
8899 crate::run::set_home(dir.path().join("run-home"));
8900 let queue = Queue::at(dir.path().join("queue"));
8901 let home = dir.path().join("home");
8902 let questions = Questions::at(dir.path().join("questions"));
8903
8904 let mut t = task();
8905 t.id = "20260101-000003-dead".to_owned();
8906 t.start("missing-run".to_owned());
8907 queue.put(&mut t).unwrap();
8908 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
8909
8910 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
8913 assert_eq!(
8914 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
8915 [&t.id]
8916 );
8917 assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
8918 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
8919
8920 crate::conduct::apply(
8923 &queue,
8924 &questions,
8925 &crate::conduct::Verdict {
8926 decisions: vec![crate::conduct::Decision {
8927 id: t.id.clone(),
8928 recovery: Some(crate::conduct::Recovery::Requeue),
8929 ..crate::conduct::Decision::default()
8930 }],
8931 },
8932 )
8933 .unwrap();
8934 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
8935 }
8936
8937 #[test]
8938 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
8939 let dir = tempfile::tempdir().unwrap();
8940 let queue = Queue::at(dir.path().join("queue"));
8941 let home = dir.path().join("home");
8942
8943 let mut fresh = task();
8944 fresh.id = "20260101-000001-aaaa".to_owned();
8945 fresh.start("run-1".to_owned());
8946 queue.put(&mut fresh).unwrap();
8947
8948 let mut old = task();
8949 old.id = "20260101-000002-bbbb".to_owned();
8950 old.start("run-2".to_owned());
8951 queue.put(&mut old).unwrap();
8952 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
8953
8954 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
8955 assert_eq!(stalled.len(), 1);
8956 assert_eq!(stalled[0].id, old.id);
8957 }
8958
8959 #[test]
8960 fn queued_and_finished_task_views_partition_by_status() {
8961 let dir = tempfile::tempdir().unwrap();
8962 let queue = Queue::at(dir.path().join("queue"));
8963
8964 let mut queued = task();
8965 queued.id = "20260101-000001-aaaa".to_owned();
8966 queue.put(&mut queued).unwrap();
8967
8968 let mut failed = task();
8969 failed.id = "20260101-000002-bbbb".to_owned();
8970 failed.start("run-1".to_owned());
8971 failed.fail("gate red", 5);
8972 queue.put(&mut failed).unwrap();
8973
8974 let mut held = task();
8975 held.id = "20260101-000003-cccc".to_owned();
8976 held.hold_machine(None);
8977 queue.put(&mut held).unwrap();
8978
8979 let mut running = task();
8980 running.id = "20260101-000004-dddd".to_owned();
8981 running.start("run-2".to_owned());
8982 queue.put(&mut running).unwrap();
8983
8984 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
8985 assert_eq!(queued_ids, [queued.id.clone()]);
8986
8987 let mut finished_ids: Vec<String> =
8988 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
8989 finished_ids.sort_unstable();
8990 let mut want = vec![failed.id.clone(), held.id.clone()];
8991 want.sort_unstable();
8992 assert_eq!(finished_ids, want);
8993 }
8994
8995 #[test]
8996 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
8997 for short in [false, true] {
8998 let dir = tempfile::tempdir().unwrap();
8999 let queue = Queue::at(dir.path().join("queue"));
9000 let questions = ask::Questions::at(dir.path().join("questions"));
9001 let mut running = task();
9002 running.id = "20261013-192213-feb2".into();
9003 running.start("existing-run".into());
9004 queue.put(&mut running).unwrap();
9005 let mut middle = task();
9006 middle.id = "20261013-193611-ca5b".into();
9007 middle.block(
9008 vec![if short {
9009 "feb2".into()
9010 } else {
9011 running.id.clone()
9012 }],
9013 None,
9014 );
9015 queue.put(&mut middle).unwrap();
9016 let mut waiting = task();
9017 waiting.id = "20261013-202849-ee37".into();
9018 waiting.block(
9019 vec![if short {
9020 "ca5b".into()
9021 } else {
9022 middle.id.clone()
9023 }],
9024 Some("wait for implementation".into()),
9025 );
9026 queue.put(&mut waiting).unwrap();
9027 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
9028 let inv = crate::blockers::Inventory::load(&queue, &questions);
9029 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
9030 assert!(inv.stuck().is_empty());
9031 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9032 assert!(report.quarantined.is_empty());
9033 assert!(report.asked.is_empty());
9034 resolve_blockers(&queue, &questions);
9035 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9036 assert!(questions.list().is_empty());
9037 running.succeed();
9040 queue.put(&mut running).unwrap();
9041 resolve_blockers(&queue, &questions);
9042 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
9043 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9044 middle = queue.get(&middle.id).unwrap();
9045 middle.succeed();
9046 queue.put(&mut middle).unwrap();
9047 resolve_blockers(&queue, &questions);
9048 let after = queue.get(&waiting.id).unwrap();
9049 assert_eq!(after.status, TaskStatus::Queued);
9050 assert!(after.blocked_by.is_empty());
9051 assert_eq!(after.instruction, waiting.instruction);
9052 }
9053 }
9054
9055 #[test]
9056 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
9057 for future_schema in [false, true] {
9058 let name = "20261013-193611-ca5b";
9059 let mut future = task();
9060 future.id = name.into();
9061 future.schema = crate::queue::SCHEMA + 1;
9062 let contents = if future_schema {
9063 serde_json::to_string(&future).unwrap()
9064 } else {
9065 "corrupt".into()
9066 };
9067 let dir = tempfile::tempdir().unwrap();
9068 let queue = Queue::at(dir.path().join("queue"));
9069 let questions = ask::Questions::at(dir.path().join("questions"));
9070 let mut waiting = task();
9071 waiting.id = "20261013-202849-ee37".into();
9072 waiting.block(vec!["ca5b".into()], None);
9073 queue.put(&mut waiting).unwrap();
9074 std::fs::write(queue.path_of(name), contents).unwrap();
9075 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
9076 assert!(
9077 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
9078 );
9079 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9080 assert!(report.asked.is_empty());
9081 resolve_blockers(&queue, &questions);
9082 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9083 assert!(questions.list().is_empty());
9084 }
9085 let dir = tempfile::tempdir().unwrap();
9086 let queue = Queue::at(dir.path().join("queue"));
9087 let questions = ask::Questions::at(dir.path().join("questions"));
9088 let mut waiting = task();
9089 waiting.id = "20261013-202849-ee37".into();
9090 waiting.block(vec!["ca5b".into()], None);
9091 queue.put(&mut waiting).unwrap();
9092 let mut done = task();
9093 done.id = "20261013-193611-ca5b".into();
9094 done.succeed();
9095 queue.put(&mut done).unwrap();
9096 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
9099 assert!(queue.get("ca5b").is_err());
9100 assert_eq!(
9101 crate::queue::dependency(&queue, &questions, "ca5b"),
9102 crate::queue::Dependency::Ambiguous
9103 );
9104 resolve_blockers(&queue, &questions);
9105 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9106 assert!(report.asked.is_empty());
9107 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9108 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
9109 }
9110
9111 #[test]
9112 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
9113 for held in [false, true] {
9114 for question in [false, true] {
9115 for reference in ["ca5b", "20261013-193611-ca5b"] {
9116 let dir = tempfile::tempdir().unwrap();
9117 let queue = Queue::at(dir.path().join("queue"));
9118 let questions = ask::Questions::at(dir.path().join("questions"));
9119 let mut prerequisite = task();
9120 prerequisite.id = "20261013-193611-ca5b".into();
9121 prerequisite.start("existing-run".into());
9122 queue.put(&mut prerequisite).unwrap();
9123 let mut waiting = task();
9124 waiting.id = "20261013-202849-ee37".into();
9125 if held {
9126 waiting.hold_machine(Some("waiting for separate consent".into()));
9127 }
9128 queue.put(&mut waiting).unwrap();
9129 let reason = "Deleting saved data requires separate owner consent";
9130 crate::conduct::apply(
9131 &queue,
9132 &questions,
9133 &crate::conduct::Verdict {
9134 decisions: vec![crate::conduct::Decision {
9135 id: waiting.id.clone(),
9136 blocked_by: vec![reference.into()],
9137 question: question.then(|| "May the saved data be deleted?".into()),
9138 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
9139 reason: Some(reason.into()),
9140 ..crate::conduct::Decision::default()
9141 }],
9142 },
9143 )
9144 .unwrap();
9145 let before = queue.get(&waiting.id).unwrap();
9146 if question {
9147 assert_eq!(before.status, TaskStatus::Blocked);
9148 let q = questions.get(&before.blocked_by[0]).unwrap();
9149 assert_eq!(q.summary, "May the saved data be deleted?");
9150 assert!(q.status.open());
9151 } else {
9152 assert_eq!(before.status, TaskStatus::Held);
9153 let recorded_reason = before.hold_reason.as_deref().unwrap();
9154 assert!(recorded_reason.starts_with(reason));
9155 if held {
9156 assert!(recorded_reason.contains("waiting for separate consent"));
9157 }
9158 }
9159 prerequisite.succeed();
9160 queue.put(&mut prerequisite).unwrap();
9161 resolve_blockers(&queue, &questions);
9162 let after = queue.get(&waiting.id).unwrap();
9163 assert_eq!(after.status, before.status);
9164 assert_eq!(after.blocked_by, before.blocked_by);
9165 assert_eq!(after.hold_reason, before.hold_reason);
9166 assert_eq!(after.instruction, waiting.instruction);
9167 if question {
9168 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
9169 }
9170 assert!(
9171 queue.next_runnable().is_none(),
9172 "dependency completion is not consent"
9173 );
9174 }
9175 }
9176 }
9177 }
9178
9179 #[test]
9180 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
9181 let dir = tempfile::tempdir().unwrap();
9182 let queue = Queue::at(dir.path().join("queue"));
9183 let questions = ask::Questions::at(dir.path().join("questions"));
9184
9185 let mut dep = task();
9186 dep.id = "20260101-000001-dep0".to_owned();
9187 dep.succeed();
9188 queue.put(&mut dep).unwrap();
9189
9190 let mut still_going = task();
9191 still_going.id = "20260101-000002-dep1".to_owned();
9192 queue.put(&mut still_going).unwrap();
9193
9194 let mut blocked = task();
9195 blocked.id = "20260101-000003-main".to_owned();
9196 blocked.block(
9197 vec![dep.id.clone(), still_going.id.clone()],
9198 Some("waits on both".to_owned()),
9199 );
9200 queue.put(&mut blocked).unwrap();
9201
9202 resolve_blockers(&queue, &questions);
9203
9204 let after = queue.get(&blocked.id).unwrap();
9205 assert_eq!(
9206 after.status,
9207 TaskStatus::Blocked,
9208 "one dependency is still outstanding"
9209 );
9210 assert_eq!(after.blocked_by, [still_going.id.clone()]);
9211 }
9212
9213 #[test]
9214 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
9215 let dir = tempfile::tempdir().unwrap();
9216 let queue = Queue::at(dir.path().join("queue"));
9217 let questions = ask::Questions::at(dir.path().join("questions"));
9218
9219 let mut q = crate::ask::Question::new(
9220 "20260101-000001-main".to_owned(),
9221 crate::conduct::NODE.to_owned(),
9222 "conduct".to_owned(),
9223 "Which backend?".to_owned(),
9224 String::new(),
9225 Vec::new(),
9226 );
9227 questions.put(&mut q).unwrap();
9228 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
9229 .unwrap();
9230 questions.put(&mut q).unwrap();
9231
9232 let mut blocked = task();
9233 blocked.id = "20260101-000001-main".to_owned();
9234 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
9235 queue.put(&mut blocked).unwrap();
9236
9237 resolve_blockers(&queue, &questions);
9238
9239 let after = queue.get(&blocked.id).unwrap();
9240 assert_eq!(
9241 after.status,
9242 TaskStatus::Queued,
9243 "the only blocker resolved"
9244 );
9245 assert_eq!(after.answers.len(), 1);
9246 assert_eq!(after.answers[0].question, "Which backend?");
9247 assert_eq!(after.answers[0].answer, "SQLite");
9248
9249 let instruction = instruction_for(&after);
9251 assert!(instruction.contains("Which backend?"));
9252 assert!(instruction.contains("SQLite"));
9253 }
9254
9255 #[test]
9256 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
9257 let dir = tempfile::tempdir().unwrap();
9258 let queue = Queue::at(dir.path().join("queue"));
9259 let questions = ask::Questions::at(dir.path().join("questions"));
9260
9261 let mut q = crate::ask::Question::new(
9262 "20260101-000001-main".to_owned(),
9263 crate::conduct::NODE.to_owned(),
9264 "conduct".to_owned(),
9265 "Is the setup done?".to_owned(),
9266 String::new(),
9267 Vec::new(),
9268 );
9269 q.abandon("no answer within 60s of asking");
9270 questions.put(&mut q).unwrap();
9271
9272 let mut blocked = task();
9273 blocked.id = "20260101-000001-main".to_owned();
9274 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
9275 queue.put(&mut blocked).unwrap();
9276
9277 resolve_blockers(&queue, &questions);
9278
9279 let after = queue.get(&blocked.id).unwrap();
9280 assert_eq!(
9281 after.status,
9282 TaskStatus::Held,
9283 "never left blocked on nothing"
9284 );
9285 assert!(!after.operator_held(), "a machine hold, for triage");
9286 assert!(
9287 after
9288 .hold_reason
9289 .as_deref()
9290 .unwrap_or_default()
9291 .contains("went unanswered")
9292 );
9293 }
9294
9295 #[test]
9296 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
9297 let dir = tempfile::tempdir().unwrap();
9303 let queue = Queue::at(dir.path().join("queue"));
9304 let questions = ask::Questions::at(dir.path().join("questions"));
9305
9306 let mut q = crate::ask::Question::new(
9307 "20260101-000001-main".to_owned(),
9308 crate::conduct::NODE.to_owned(),
9309 "conduct".to_owned(),
9310 "How should this be handled?".to_owned(),
9311 String::new(),
9312 Vec::new(),
9313 );
9314 questions.put(&mut q).unwrap();
9315 q.answer(crate::ask::Answer::Text(
9316 "leave it held, a human will look at it later".to_owned(),
9317 ))
9318 .unwrap();
9319 questions.put(&mut q).unwrap();
9320
9321 let mut held = task();
9322 held.id = "20260101-000001-main".to_owned();
9323 held.hold_machine(Some("out of attempts".to_owned()));
9324 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
9325 queue.put(&mut held).unwrap();
9326
9327 resolve_blockers(&queue, &questions);
9328
9329 let after = queue.get(&held.id).unwrap();
9330 assert_eq!(after.status, TaskStatus::Held);
9331 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
9332 assert_eq!(
9333 after.answers[0].answer,
9334 "leave it held, a human will look at it later"
9335 );
9336 }
9337
9338 #[test]
9339 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
9340 let dir = tempfile::tempdir().unwrap();
9341 let queue = Queue::at(dir.path().join("queue"));
9342 let questions = ask::Questions::at(dir.path().join("questions"));
9343 let mut dep = task();
9344 dep.id = "20260101-000001-gone".to_owned();
9345 queue.put(&mut dep).unwrap();
9346 let mut blocked = task();
9347 blocked.id = "20260101-000003-main".to_owned();
9348 blocked.block(vec![dep.id.clone()], None);
9349 queue.put(&mut blocked).unwrap();
9350
9351 let claim = queue.claim(&blocked.id).unwrap();
9353 queue.remove(&dep.id, false, &questions).unwrap();
9354 drop(claim);
9355 resolve_blockers(&queue, &questions);
9356
9357 let after = queue.get(&blocked.id).unwrap();
9358 assert_eq!(after.status, TaskStatus::Queued);
9359 assert!(after.blocked_by.is_empty());
9360 }
9361
9362 #[test]
9363 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
9364 let dir = tempfile::tempdir().unwrap();
9370 let queue = Queue::at(dir.path().join("queue"));
9371 let questions = ask::Questions::at(dir.path().join("questions"));
9372
9373 let mut still_going = task();
9374 still_going.id = "20260101-000002-dep1".to_owned();
9375 queue.put(&mut still_going).unwrap();
9376
9377 let mut blocked = task();
9378 blocked.id = "20260101-000003-main".to_owned();
9379 blocked.block(
9380 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
9381 Some("waits on both".to_owned()),
9382 );
9383 queue.put(&mut blocked).unwrap();
9384
9385 resolve_blockers(&queue, &questions);
9386
9387 let after = queue.get(&blocked.id).unwrap();
9388 assert_eq!(
9389 after.status,
9390 TaskStatus::Held,
9391 "a missing dependency must not leave the task blocked forever"
9392 );
9393 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
9394 assert!(after.blocked_by.is_empty());
9395 let reason = after.hold_reason.as_deref().unwrap_or_default();
9396 assert!(
9397 reason.contains("20260101-000001-gone"),
9398 "the missing id must be named so an operator can tell what happened: {reason}"
9399 );
9400 assert!(
9401 reason.contains(&still_going.id),
9402 "the still-valid dependency must not silently vanish from the record: {reason}"
9403 );
9404 }
9405
9406 #[test]
9407 fn instruction_for_is_unchanged_without_any_answers() {
9408 let t = task();
9409 assert_eq!(instruction_for(&t), t.instruction);
9410 }
9411
9412 #[test]
9413 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
9414 let dir = tempfile::tempdir().unwrap();
9415 let q = Queue::at(dir.path().join("queue"));
9416 let src = dir.path().join("shot.png");
9417 std::fs::write(&src, "x").unwrap();
9418 let mut t = task();
9419 q.attach(&mut t, &[src]).unwrap();
9420 let paths = task_attachments(&q, &t).unwrap();
9421 assert_eq!(paths.len(), 1);
9422 assert!(paths[0].is_absolute() && paths[0].is_file());
9423 std::fs::remove_file(&paths[0]).unwrap();
9424 let err = task_attachments(&q, &t).unwrap_err().to_string();
9425 assert!(err.contains("shot.png"), "{err}");
9426 }
9427
9428 #[tokio::test]
9429 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
9430 let dir = tempfile::tempdir().unwrap();
9431 let queue_path = dir.path().join("queue");
9432 let queue = Queue::at(queue_path.clone());
9433 let mut t = task();
9434 t.source = Source::Agent {
9435 run: "originating-chat".into(),
9436 node: crate::queue::CHAT_NODE.into(),
9437 };
9438 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
9439 stage: "config-fetch",
9440 target: "origin/main".into(),
9441 });
9442 record_config_failure(&mut t, &fetch);
9443 t.hold_machine(Some(format!("{fetch:#}")));
9444 let fetch_key = t.chat_notice_due().unwrap();
9445 t.chat_notices.push(fetch_key.clone());
9446 queue.put(&mut t).unwrap();
9447
9448 let queue = Queue::at(queue_path);
9449 let mut t = queue.get(&t.id).unwrap();
9450 let a = dir.path().join("a.toml");
9451 let b = dir.path().join("b.toml");
9452 for (path, text, new_cause) in [
9453 (&a, "[graph", true),
9454 (&a, "[verify", false),
9455 (&a, "[graph]\nimplementers = 'invalid-type'", true),
9456 (&b, "[graph]\nimplementers = 'invalid-type'", true),
9457 ] {
9458 t.release_operator("test", None);
9459 std::fs::write(path, text).unwrap();
9460 let error = Config::discover_fetched(dir.path(), Some(path))
9461 .await
9462 .unwrap_err();
9463 record_config_failure(&mut t, &error);
9464 t.hold_machine(Some(format!("{error:#}")));
9465 let due = t.chat_notice_due();
9466 assert_eq!(due.is_some(), new_cause, "{error:#}");
9467 if let Some(key) = due {
9468 assert_ne!(key, fetch_key);
9469 t.chat_notices.push(key);
9470 }
9471 }
9472 t.succeed();
9473 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
9474 }
9475
9476 #[test]
9477 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
9478 let dir = tempfile::tempdir().unwrap();
9479 let queue_path = dir.path().join("queue");
9480 let queue = Queue::at(queue_path.clone());
9481 let mut t = task();
9482 t.source = crate::queue::Source::Agent {
9483 run: "originating-chat".into(),
9484 node: crate::queue::CHAT_NODE.into(),
9485 };
9486 let a = dir.path().join("a.png");
9487 let b = dir.path().join("b.png");
9488 std::fs::write(&a, "a").unwrap();
9489 std::fs::write(&b, "b").unwrap();
9490 queue.attach(&mut t, &[a, b]).unwrap();
9491 let paths = task_attachments(&queue, &t).unwrap();
9492 for path in &paths {
9493 std::fs::remove_file(path).unwrap();
9494 }
9495 let missing = task_attachments(&queue, &t).unwrap_err();
9496 t.attempts += 1;
9497 let reason = missing.to_string();
9498 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9499 let first = t.chat_notice_due().expect("first attachment notice");
9500 t.chat_notices.push(first.clone());
9501 queue.put(&mut t).unwrap();
9502
9503 let queue = Queue::at(queue_path.clone());
9504 let mut t = queue.get(&t.id).unwrap();
9505 assert!(t.chat_notice_due().is_none(), "restart does not replay");
9506 t.release_operator("test", None);
9507 t.attempts += 1;
9508 let missing = task_attachments(&queue, &t).unwrap_err();
9509 t.fail_startup_target(
9510 "startup-attachments",
9511 missing.name,
9512 "same missing attachment with regenerated wording",
9513 1,
9514 );
9515 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
9516
9517 std::fs::write(&paths[0], "restored").unwrap();
9518 t.release_operator("test", None);
9519 t.attempts += 1;
9520 let missing = task_attachments(&queue, &t).unwrap_err();
9521 let reason = missing.to_string();
9522 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9523 let second = t
9524 .chat_notice_due()
9525 .expect("different attachment must notify");
9526 assert_ne!(first, second);
9527 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
9528 }
9529
9530 #[test]
9531 fn resumed_instruction_is_unchanged_without_any_answers() {
9532 let t = task();
9533 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
9534 }
9535
9536 #[test]
9537 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
9538 let mut t = task();
9539 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9540 let old = t.instruction.clone();
9544
9545 let refreshed = resumed_instruction(&old, &t);
9546 assert!(refreshed.starts_with(&old), "the original text is kept");
9547 assert!(refreshed.contains("Which backend?"));
9548 assert!(refreshed.contains("SQLite"));
9549 }
9550
9551 #[test]
9552 fn resumed_instruction_keeps_an_original_answers_heading() {
9553 let mut t = task();
9554 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
9555 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9556
9557 let refreshed = resumed_instruction(&t.instruction, &t);
9558
9559 assert!(
9560 refreshed.starts_with(&t.instruction),
9561 "an answers heading in the original instruction is not the appended block"
9562 );
9563 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
9564 assert!(refreshed.contains("Which backend?"));
9565 assert!(refreshed.contains("SQLite"));
9566
9567 let repeated = resumed_instruction(&refreshed, &t);
9568 assert_eq!(
9569 repeated, refreshed,
9570 "only the final appended block is refreshed"
9571 );
9572 }
9573
9574 #[test]
9575 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
9576 let mut t = task();
9577 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9578
9579 let once = resumed_instruction(&t.instruction, &t);
9583 let twice = resumed_instruction(&once, &t);
9584 assert_eq!(once, twice);
9585 assert_eq!(once.matches("Which backend?").count(), 1);
9586
9587 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
9589 let refreshed = resumed_instruction(&once, &t);
9590 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
9591 assert!(refreshed.contains("Which backend?"));
9592 assert!(refreshed.contains("Which cache?"));
9593 }
9594
9595 #[test]
9596 fn prepare_instruction_covers_all_three_starters() {
9597 let mut t = task();
9598 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9599
9600 assert_eq!(
9603 prepare_instruction(&Starter::Start, None, &t),
9604 Some(instruction_for(&t))
9605 );
9606
9607 let old = t.instruction.clone();
9610 assert_eq!(
9611 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
9612 Some(resumed_instruction(&old, &t))
9613 );
9614
9615 assert_eq!(
9619 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
9620 None
9621 );
9622 }
9623
9624 #[test]
9625 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
9626 assert_eq!(
9627 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
9628 Starter::Review("magi/eba2/A".to_owned())
9629 );
9630 }
9631
9632 #[test]
9633 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
9634 assert_eq!(
9635 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
9636 Starter::Start,
9637 "a vanished review branch must not fall back to resuming the old run either"
9638 );
9639 }
9640
9641 #[test]
9642 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
9643 let mut t = task();
9644 t.start("old-run".to_owned());
9645 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
9646 t.release();
9647 let branch = t.review_branch.take();
9648 assert_eq!(
9649 choose_starter(branch.as_deref(), true, Some("old-run")),
9650 Starter::Review("magi/eba2/A".to_owned()),
9651 "a review wins over resuming the old run"
9652 );
9653 }
9654
9655 #[test]
9656 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
9657 assert_eq!(
9658 choose_starter(None, false, Some("some-run")),
9659 Starter::Resume("some-run".to_owned())
9660 );
9661 assert_eq!(choose_starter(None, false, None), Starter::Start);
9662 }
9663
9664 #[test]
9665 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
9666 let mut released = task();
9667 released.start("stalled-run".to_owned());
9668 released.requeue();
9669 let unfinished = (!released.fresh_start)
9670 .then(|| Some("stalled-run".to_owned()))
9671 .flatten();
9672 assert_eq!(
9673 choose_starter(None, false, unfinished.as_deref()),
9674 Starter::Start,
9675 "release keeps run history but must not resume it"
9676 );
9677 assert_eq!(released.runs, ["stalled-run"]);
9678 }
9679
9680 #[test]
9681 fn an_ordinary_release_keeps_a_resumable_run_available() {
9682 let mut released = task();
9683 released.start("stalled-run".to_owned());
9684 released.release();
9685 let unfinished = (!released.fresh_start)
9686 .then(|| Some("stalled-run".to_owned()))
9687 .flatten();
9688 assert_eq!(
9689 choose_starter(None, false, unfinished.as_deref()),
9690 Starter::Resume("stalled-run".to_owned()),
9691 "manual release must preserve the normal resume path"
9692 );
9693 }
9694
9695 #[test]
9696 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
9697 let mut state = run_state(RunStatus::Blocked);
9698 state.config.graph.review_rounds = 3;
9699 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
9700 assert!(exhausted_review_budget(&state));
9701
9702 state.reviews.pop();
9704 assert!(!exhausted_review_budget(&state));
9705
9706 let mut stalled = run_state(RunStatus::Stalled);
9709 stalled.config.graph.review_rounds = 1;
9710 stalled.reviews = vec![review_round(1)];
9711 assert!(!exhausted_review_budget(&stalled));
9712 }
9713
9714 fn review_round(round: usize) -> crate::run::ReviewRound {
9715 crate::run::ReviewRound {
9716 round,
9717 head: "deadbeef".to_owned(),
9718 verified_head: None,
9719 verified_at: None,
9720 reviews: Vec::new(),
9721 e2e: Vec::new(),
9722 verify_retried: false,
9723 e2e_deferred: false,
9724 e2e_defer_reason: None,
9725 fix: None,
9726 blocking: 0,
9727 answered: 1,
9728 expected: 1,
9729 clean: false,
9730 progressed: true,
9731 vote_split: false,
9732 reconsideration: Vec::new(),
9733 verdict: None,
9734 }
9735 }
9736
9737 #[test]
9738 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
9739 let mut run = RunState::new(
9740 PathBuf::from("/repo"),
9741 "main".to_owned(),
9742 "abc1234def".to_owned(),
9743 "add retries".to_owned(),
9744 Config::default(),
9745 );
9746 run.status = RunStatus::Judging;
9747 run.parked = true;
9748 let mut task = Task::new(
9749 "add retries".to_owned(),
9750 "add retries".to_owned(),
9751 PathBuf::from("/repo"),
9752 crate::queue::Source::Human,
9753 );
9754 task.status = TaskStatus::Failed;
9755 task.runs = vec![run.id.clone()];
9756 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
9757 assert!(with(&task, &run), "parked after judging is the case");
9758 let mut parked_task = task.clone();
9759 parked_task.status = TaskStatus::Parked;
9760 assert!(with(&parked_task, &run), "the Parked status is the case");
9761
9762 let mut not_parked = run.clone();
9763 not_parked.parked = false;
9764 not_parked.status = RunStatus::Stalled;
9765 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
9766
9767 let mut fresh = task.clone();
9768 fresh.fresh_start = true;
9769 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
9770
9771 let mut review = task.clone();
9772 review.review_branch = Some("magi/x/A".to_owned());
9773 assert!(!with(&review, &run), "review is ranked before resume");
9774
9775 let mut held = task.clone();
9776 held.status = TaskStatus::Held;
9777 assert!(!with(&held, &run), "a hold stays visible to the conductor");
9778
9779 let mut released = run.clone();
9780 released.released_to = Some("20260901-000000-new1".to_owned());
9781 assert!(!with(&task, &released), "nothing left to resume into");
9782
9783 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
9784 "unreadable"
9785 )));
9786 }
9787
9788 #[test]
9789 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
9790 let mut released = RunState::new(
9791 PathBuf::from("/repo"),
9792 "main".to_owned(),
9793 "abc1234def".to_owned(),
9794 "add retries".to_owned(),
9795 Config::default(),
9796 );
9797 released.status = RunStatus::Blocked;
9798 assert_eq!(
9799 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9800 Some(released.id.clone())
9801 );
9802 released.released_to = Some("20260901-000000-new1".to_owned());
9803 assert_eq!(
9804 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9805 None,
9806 "there is nothing left to resume it into"
9807 );
9808 }
9809
9810 #[test]
9811 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
9812 let mut exhausted = RunState::new(
9821 PathBuf::from("/repo"),
9822 "main".to_owned(),
9823 "abc1234def".to_owned(),
9824 "add retries".to_owned(),
9825 Config::default(),
9826 );
9827 exhausted.status = RunStatus::Blocked;
9828 exhausted.config.graph.review_rounds = 1;
9829 exhausted.reviews = vec![review_round(1)];
9830
9831 assert_eq!(
9832 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
9833 None,
9834 "an exhausted `Blocked` run must not be offered as resumable"
9835 );
9836
9837 let mut has_budget_left = RunState::new(
9840 PathBuf::from("/repo"),
9841 "main".to_owned(),
9842 "abc1234def".to_owned(),
9843 "add retries".to_owned(),
9844 Config::default(),
9845 );
9846 has_budget_left.status = RunStatus::Blocked;
9847 has_budget_left.config.graph.review_rounds = 3;
9848 has_budget_left.reviews = vec![review_round(1)];
9849
9850 assert_eq!(
9851 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
9852 Ok(has_budget_left.clone())
9853 }),
9854 Some(has_budget_left.id.clone())
9855 );
9856 }
9857
9858 #[test]
9859 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
9860 let mut run = RunState::new(
9861 PathBuf::from("/repo"),
9862 "main".to_owned(),
9863 "abc1234def".to_owned(),
9864 "add retries".to_owned(),
9865 Config::default(),
9866 );
9867 run.status = RunStatus::Blocked;
9868 run.config.graph.review_rounds = 1;
9869 run.reviews = vec![review_round(1)];
9870 run.pr_publication_pending = true;
9871 assert_eq!(
9872 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
9873 Some(run.id.clone()),
9874 "a spent review budget must not turn a failed publication into a re-competition"
9875 );
9876 }
9877
9878 #[test]
9879 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
9880 let mut older_stalled = RunState::new(
9888 PathBuf::from("/repo"),
9889 "main".to_owned(),
9890 "abc1234def".to_owned(),
9891 "add retries".to_owned(),
9892 Config::default(),
9893 );
9894 older_stalled.status = RunStatus::Stalled;
9895
9896 let mut newest_exhausted = RunState::new(
9897 PathBuf::from("/repo"),
9898 "main".to_owned(),
9899 "abc1234def".to_owned(),
9900 "add retries".to_owned(),
9901 Config::default(),
9902 );
9903 newest_exhausted.status = RunStatus::Blocked;
9904 newest_exhausted.config.graph.review_rounds = 1;
9905 newest_exhausted.reviews = vec![review_round(1)];
9906
9907 assert_eq!(
9908 unfinished_run_with(
9909 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
9910 "t",
9911 |_| Ok(newest_exhausted.clone())
9912 ),
9913 None,
9914 "the newest run is exhausted, so nothing here is worth resuming - \
9915 least of all the older, already-superseded run"
9916 );
9917 }
9918
9919 #[test]
9920 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
9921 assert_eq!(
9922 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
9923 Err(anyhow::anyhow!("fixture is absent"))
9924 }),
9925 None
9926 );
9927 }
9928
9929 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
9930 let mut q = ask::Question::new(
9931 run.to_owned(),
9932 "implement".to_owned(),
9933 "impl-A".to_owned(),
9934 "continue?".to_owned(),
9935 String::new(),
9936 vec!["resume で続行する".to_owned(), "other".to_owned()],
9937 );
9938 q.actions.insert("resume で続行する".to_owned(), action);
9939 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
9940 .unwrap();
9941 q
9942 }
9943
9944 fn held_task_with(run: &str) -> Task {
9945 let mut t = task();
9946 t.runs = vec![run.to_owned()];
9947 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
9948 t
9949 }
9950
9951 fn resume_action(run: &str) -> ask::ChoiceAction {
9952 ask::ChoiceAction::Resume { run: run.into() }
9953 }
9954
9955 #[test]
9956 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
9957 for parked in [false, true] {
9958 let mut t = task();
9959 t.start("A".into());
9960 t.start("B".into());
9961 assert_eq!(t.runs, vec!["A", "B"]);
9962 t.start("A".into());
9963 let mut resumed = run_state(RunStatus::Blocked);
9964 resumed.id = "A".into();
9965 resumed.parked = parked;
9966 resumed.driver_exited = true;
9967 t.hold_machine(Some("resumed run stopped".into()));
9968 let q = action_question("A", resume_action("A"));
9969 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
9970 assert_eq!(
9971 decide_action(&t, &q, &PHRASES_EN, |id| {
9972 assert_eq!(id, "A");
9973 Ok(resumed.clone())
9974 }),
9975 ActionDecision::Resume("A".into())
9976 );
9977 assert_eq!(
9978 action_standing(&t, &action_question("B", resume_action("B"))),
9979 ActionStanding::Stale
9980 );
9981 t.release_operator("cli", None);
9982 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
9983 assert_eq!(id, "A");
9984 Ok(resumed.clone())
9985 });
9986 assert_eq!(
9987 choose_starter(None, false, unfinished.as_deref()),
9988 Starter::Resume("A".into())
9989 );
9990 assert_eq!(t.runs, vec!["B", "A"]);
9991 }
9992 }
9993
9994 #[test]
9995 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
9996 let mut state = crate::land::tests::repair_state(true, true);
9997 state.status = RunStatus::Landing;
9998 state.parked = true;
9999 state.driver_exited = true;
10000 let mut q = ask::Question::new(
10001 state.id.clone(),
10002 land::APPROVAL_NODE.into(),
10003 "land".into(),
10004 "Merge?".into(),
10005 String::new(),
10006 vec![land::APPROVE.into(), land::HOLD.into()],
10007 );
10008 q.id = state.id.clone();
10009 state.land_approval = Some(crate::run::LandApproval {
10010 question: q.id.clone(),
10011 head: "current-head".into(),
10012 });
10013 assert!(land::upgrade_repair_choice(&mut q, &state));
10014 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10015 let mut task = held_task_with(&state.id);
10016 task.status = TaskStatus::Parked;
10017 assert_eq!(
10018 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10019 ActionDecision::Resume(state.id.clone())
10020 );
10021 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10022 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
10023 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
10024 assert_eq!(
10025 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10026 ActionDecision::Resume(state.id.clone()),
10027 "a repair answer newer than the manual hold resumes the existing owner"
10028 );
10029 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10030 for answered_at in [q.answered_at, None] {
10031 let mut older = q.clone();
10032 older.answered_at = answered_at;
10033 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
10034 panic!("a later manual hold must prevent dispatch")
10035 });
10036 let ActionDecision::Refuse(reason) = refused else {
10037 panic!("expected an actionable refusal, got {refused:?}");
10038 };
10039 assert!(reason.contains("later operator decision"));
10040 assert!(reason.contains(&format!("magi task release {}", task.id)));
10041 }
10042 task.operator_decided_at = None;
10043 let mut superseded = state.clone();
10044 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
10045 assert_eq!(
10046 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
10047 ActionDecision::Stale
10048 );
10049 for case in 1..4 {
10050 let mut changed = state.clone();
10051 match case {
10052 1 => changed.status = RunStatus::Blocked,
10053 2 => changed.released_to = Some("new-owner".into()),
10054 _ => changed.land_approval.as_mut().unwrap().head.clear(),
10055 }
10056 assert!(matches!(
10057 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
10058 ActionDecision::Refuse(_)
10059 ));
10060 }
10061 assert!(matches!(
10062 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
10063 ActionDecision::Refuse(_)
10064 ));
10065 task.runs.push("replacement".into());
10066 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10067 assert_eq!(
10068 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
10069 "old owner must not load"
10070 )),
10071 ActionDecision::Stale
10072 );
10073 }
10074
10075 #[test]
10076 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
10077 let mut state = crate::land::tests::repair_state(true, true);
10078 state.status = RunStatus::Landing;
10079 state.parked = true;
10080 state.driver_exited = true;
10081 let mut q = ask::Question::new(
10082 state.id.clone(),
10083 land::APPROVAL_NODE.into(),
10084 "land".into(),
10085 "Merge?".into(),
10086 String::new(),
10087 vec![land::APPROVE.into(), land::HOLD.into()],
10088 );
10089 q.id = state.id.clone();
10090 state.land_approval = Some(crate::run::LandApproval {
10091 question: q.id.clone(),
10092 head: "current-head".into(),
10093 });
10094 state.save().unwrap();
10095 land::upgrade_repair_choice(&mut q, &state);
10096 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10097 Questions::open().put(&mut q).unwrap();
10098 let tmp = tempfile::tempdir().unwrap();
10099 let queue = Queue::at(tmp.path().join("queue"));
10100 let mut task = held_task_with(&state.id);
10101 task.status = TaskStatus::Parked;
10102 queue.put(&mut task).unwrap();
10103 let empty = Arc::new(tokio::sync::Semaphore::new(0));
10104 let free = Arc::new(tokio::sync::Semaphore::new(1));
10105 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
10106 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
10107 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
10108 assert!(claim.permit.is_some());
10109 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
10110 drop(claim);
10111 Questions::open()
10112 .update(&q.id, |q| {
10113 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
10114 Ok(())
10115 })
10116 .unwrap();
10117 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
10118 assert!(claim.permit.is_none());
10119 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
10120 }
10121
10122 #[test]
10123 fn decide_action_resumes_only_the_latest_resumable_run() {
10124 let t = held_task_with("r1");
10125 let q = action_question("r1", resume_action("r1"));
10126 let load = |s: RunState| move |_: &str| Ok(s);
10127 assert_eq!(
10128 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
10129 ActionDecision::Resume("r1".into())
10130 );
10131 let q_other = action_question("r1", resume_action("r0"));
10133 assert!(matches!(
10134 decide_action(
10135 &t,
10136 &q_other,
10137 &PHRASES_EN,
10138 load(run_state(RunStatus::Blocked))
10139 ),
10140 ActionDecision::Refuse(_)
10141 ));
10142 assert!(matches!(
10144 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
10145 ActionDecision::Refuse(_)
10146 ));
10147 let mut released = run_state(RunStatus::Blocked);
10149 released.released_to = Some("elsewhere".into());
10150 assert!(matches!(
10151 decide_action(&t, &q, &PHRASES_EN, load(released)),
10152 ActionDecision::Refuse(_)
10153 ));
10154 assert!(matches!(
10156 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
10157 ActionDecision::Refuse(_)
10158 ));
10159 }
10160
10161 #[test]
10162 fn decide_action_ignores_a_question_about_an_earlier_run() {
10163 let mut t = held_task_with("r1");
10164 t.runs.push("r2".to_owned());
10165 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10166 assert_eq!(
10167 decide_action(
10168 &t,
10169 &action_question("r1", ask::ChoiceAction::Done),
10170 &PHRASES_EN,
10171 never
10172 ),
10173 ActionDecision::Stale
10174 );
10175 }
10176
10177 #[test]
10178 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
10179 let mut t = held_task_with("r1");
10180 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10181 assert_eq!(
10182 decide_action(
10183 &t,
10184 &action_question("r1", ask::ChoiceAction::Requeue),
10185 &PHRASES_EN,
10186 never
10187 ),
10188 ActionDecision::Requeue
10189 );
10190 let done_q = action_question("r1", ask::ChoiceAction::Done);
10191 assert_eq!(
10192 decide_action(&t, &done_q, &PHRASES_EN, never),
10193 ActionDecision::Done
10194 );
10195 t.mark_action_applied(&done_q.id);
10196 assert_eq!(
10197 decide_action(&t, &done_q, &PHRASES_EN, never),
10198 ActionDecision::Skip
10199 );
10200
10201 let mut plain = action_question("r1", ask::ChoiceAction::Done);
10203 plain.actions.clear();
10204 assert_eq!(
10205 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
10206 ActionDecision::Skip
10207 );
10208 let mut running = held_task_with("r1");
10210 running.status = TaskStatus::Running;
10211 assert_eq!(
10212 decide_action(
10213 &running,
10214 &action_question("r1", ask::ChoiceAction::Done),
10215 &PHRASES_EN,
10216 never
10217 ),
10218 ActionDecision::Skip
10219 );
10220 }
10221
10222 #[test]
10223 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
10224 crate::run::pin_test_home();
10225 for legacy in [false, true] {
10226 let dir = tempfile::tempdir().unwrap();
10227 let queue = Queue::at(dir.path().join("queue"));
10228 let questions = Questions::at(dir.path().join("questions"));
10229 let run = "20260101-000000-hold";
10230 let mut task = held_task_with(run);
10231 task.status = TaskStatus::Parked;
10232 let mut q = ask::Question::new(
10233 run.into(),
10234 land::APPROVAL_NODE.into(),
10235 "land".into(),
10236 "Repair?".into(),
10237 String::new(),
10238 vec![land::REPAIR.into()],
10239 );
10240 q.actions.insert(land::REPAIR.into(), resume_action(run));
10241 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10242 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10244 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
10245 questions.put(&mut q).unwrap();
10246 task.hold_manual(Some("Keep this PR paused".into()));
10247 if legacy {
10248 task.hold_source = None;
10249 }
10250 queue.put(&mut task).unwrap();
10251
10252 apply_choice_actions(&queue, &questions, dir.path());
10253 let after = queue.get(&task.id).unwrap();
10254 assert!(after.operator_held());
10255 assert_eq!(after.hold_source, task.hold_source);
10256 assert_eq!(after.operator_decided_at, task.operator_decided_at);
10257 assert_eq!(after.runs, task.runs);
10258 assert_eq!(after.instruction, task.instruction);
10259 assert!(after.resume_override.is_none());
10260 assert!(after.action_applied(&q.id));
10261 let reason = after.hold_reason.as_deref().unwrap();
10262 assert!(reason.starts_with("Keep this PR paused"));
10263 assert!(reason.contains("later operator decision"));
10264 assert!(reason.contains(&format!("magi task release {}", task.id)));
10265
10266 for recovery in [
10267 crate::conduct::Recovery::Review,
10268 crate::conduct::Recovery::Requeue,
10269 ] {
10270 crate::conduct::apply(
10271 &queue,
10272 &questions,
10273 &crate::conduct::Verdict {
10274 decisions: vec![crate::conduct::Decision {
10275 id: task.id.clone(),
10276 recovery: Some(recovery),
10277 ..Default::default()
10278 }],
10279 },
10280 )
10281 .unwrap();
10282 }
10283 apply_choice_actions(&queue, &questions, dir.path());
10286 let still_held = queue.get(&task.id).unwrap();
10287 assert!(still_held.operator_held());
10288 assert_eq!(still_held.hold_reason, after.hold_reason);
10289 assert_eq!(still_held.runs, task.runs);
10290 assert!(still_held.review_branch.is_none());
10291 assert!(!still_held.fresh_start);
10292 assert!(queue.next_runnable().is_none());
10293 }
10294 }
10295
10296 #[test]
10297 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
10298 let dir = tempfile::tempdir().unwrap();
10299 let queue = Queue::at(dir.path().join("queue"));
10300 let questions = Questions::at(dir.path().join("questions"));
10301 let home = dir.path().join("home");
10302 let mut state = run_state(RunStatus::Blocked);
10303 state.id = "20260101-000000-act1".to_owned();
10304 state.save_under(&home).unwrap();
10305
10306 let mut t = held_task_with(&state.id);
10307 queue.put(&mut t).unwrap();
10308 let mut q = action_question(&state.id, resume_action(&state.id));
10309 questions.put(&mut q).unwrap();
10310
10311 apply_choice_actions(&queue, &questions, &home);
10312 let after = queue.get(&t.id).unwrap();
10313 assert_eq!(after.status, TaskStatus::Queued);
10314 assert!(!after.fresh_start);
10315 assert!(after.action_applied(&q.id));
10316 let pin = after.resume_override.clone().unwrap();
10317 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
10318 assert!(pin.forced);
10319
10320 let mut again = queue.get(&t.id).unwrap();
10322 again.hold_machine(Some("later".into()));
10323 queue.put(&mut again).unwrap();
10324 apply_choice_actions(&queue, &questions, &home);
10325 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10326 }
10327
10328 #[test]
10329 fn a_delivered_answer_is_still_acted_on_exactly_once() {
10330 let dir = tempfile::tempdir().unwrap();
10331 let queue = Queue::at(dir.path().join("queue"));
10332 let questions = Questions::at(dir.path().join("questions"));
10333 let home = dir.path().join("home");
10334 let mut state = run_state(RunStatus::Blocked);
10335 state.id = "20260101-000000-act2".to_owned();
10336 state.save_under(&home).unwrap();
10337
10338 let mut t = held_task_with(&state.id);
10339 queue.put(&mut t).unwrap();
10340 let mut q = action_question(&state.id, resume_action(&state.id));
10341 q.answer_delivered = true;
10342 questions.put(&mut q).unwrap();
10343
10344 apply_choice_actions(&queue, &questions, &home);
10347 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10348 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
10349
10350 let mut again = queue.get(&t.id).unwrap();
10352 again.hold_machine(Some("later".into()));
10353 queue.put(&mut again).unwrap();
10354 apply_choice_actions(&queue, &questions, &home);
10355 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10356 }
10357
10358 #[test]
10359 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
10360 let dir = tempfile::tempdir().unwrap();
10361 let queue = Queue::at(dir.path().join("queue"));
10362 let questions = Questions::at(dir.path().join("questions"));
10363 let home = dir.path().join("home");
10364 let mut state = run_state(RunStatus::Blocked);
10365 state.id = "20260101-000000-act3".to_owned();
10366 state.save_under(&home).unwrap();
10367 let mut t = held_task_with(&state.id);
10368 queue.put(&mut t).unwrap();
10369 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
10370 questions.put(&mut q).unwrap();
10371
10372 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
10373 apply_choice_actions(&queue, &questions, &home);
10374 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10375
10376 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
10377 apply_choice_actions(&queue, &questions, &home);
10378 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10379 }
10380
10381 #[test]
10382 fn action_standing_tells_the_reasons_apart() {
10383 let mut t = held_task_with("r1");
10384 let q = action_question("r1", ask::ChoiceAction::Requeue);
10385 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
10386 let mut plain = q.clone();
10387 plain.actions.clear();
10388 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
10389
10390 t.mark_action_applied(&q.id);
10392 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
10393
10394 let mut t = held_task_with("r1");
10396 t.start("r2".to_owned());
10397 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
10398 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
10400 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10401 t.status = TaskStatus::Blocked;
10402 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10403
10404 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
10406 c.node = crate::conduct::NODE.to_owned();
10407 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
10408 assert!(!ActionStanding::Busy.daemon_owns());
10409 }
10410
10411 #[test]
10412 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
10413 let dir = tempfile::tempdir().unwrap();
10414 let queue = Queue::at(dir.path().join("queue"));
10415 let questions = Questions::at(dir.path().join("questions"));
10416 let home = dir.path().join("home");
10417 let mut state = run_state(RunStatus::Blocked);
10418 state.id = "20260101-000000-act4".to_owned();
10419 state.save_under(&home).unwrap();
10420 let mut t = held_task_with(&state.id);
10421 t.status = TaskStatus::Running;
10422 queue.put(&mut t).unwrap();
10423 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
10424 q.cwd = Some(dir.path().display().to_string());
10425 questions.put(&mut q).unwrap();
10426
10427 let running = queue.get(&t.id).unwrap();
10430 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
10431 assert!(matches!(
10432 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
10433 crate::waiter::Action::Deliver(_)
10434 ));
10435 apply_choice_actions(&queue, &questions, &home);
10437 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10438 let mut back = queue.get(&t.id).unwrap();
10439 back.hold_machine(Some("later".into()));
10440 queue.put(&mut back).unwrap();
10441 apply_choice_actions(&queue, &questions, &home);
10442 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
10443 let mut again = queue.get(&t.id).unwrap();
10444 again.hold_machine(Some("again".into()));
10445 queue.put(&mut again).unwrap();
10446 apply_choice_actions(&queue, &questions, &home);
10447 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10448 }
10449}