1use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::sync::atomic::{AtomicBool, Ordering};
52use std::sync::{Mutex, MutexGuard};
53use std::time::Duration;
54
55use anyhow::{Context, Result, bail};
56use jiff::Timestamp;
57use serde::{Deserialize, Serialize};
58use tokio::sync::Notify;
59
60use crate::ask::{self, Questions};
61use crate::clean;
62use crate::conduct::Conductor;
63use crate::config::{Config, MergeMode};
64use crate::graph::Runner;
65use crate::land;
66use crate::notices::{self, Link, Notice};
67use crate::queue::{Queue, Task, TaskStatus};
68use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
69use crate::triage;
70
71pub const SCHEMA: u32 = 1;
73
74pub const HEARTBEAT: Duration = Duration::from_secs(5);
78
79pub const STALE_SECS: i64 = 30;
88
89pub const POLL: Duration = Duration::from_secs(5);
91
92pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
96
97pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
119#[serde(default)]
120pub struct Current {
121 pub task: String,
123 pub run: String,
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct Status {
135 pub schema: u32,
137 pub pid: u32,
139 pub started_at: Timestamp,
141 pub updated_at: Timestamp,
143 pub idle: bool,
145 pub current: Vec<Current>,
151 pub completed: usize,
153 pub polls: u64,
155}
156
157impl Status {
158 #[must_use]
160 pub fn new() -> Self {
161 let now = Timestamp::now();
162 Self {
163 schema: SCHEMA,
164 pid: std::process::id(),
165 started_at: now,
166 updated_at: now,
167 idle: true,
168 current: Vec::new(),
169 completed: 0,
170 polls: 0,
171 }
172 }
173}
174
175impl Default for Status {
176 fn default() -> Self {
177 Self::new()
178 }
179}
180
181#[derive(Debug, Clone)]
183pub struct Opts {
184 pub repo: PathBuf,
186 pub config: Option<PathBuf>,
188 pub poll: Duration,
190 pub max_attempts: usize,
192 pub once: bool,
194 pub merge: Option<String>,
196 pub worktrees_root: Option<PathBuf>,
205}
206
207impl Default for Opts {
208 fn default() -> Self {
209 Self {
210 repo: PathBuf::from("."),
211 config: None,
212 poll: POLL,
213 max_attempts: 2,
214 once: false,
215 merge: None,
216 worktrees_root: None,
217 }
218 }
219}
220
221fn max_concurrent(n: usize) -> usize {
226 n.max(1)
227}
228
229#[must_use]
231pub fn status_path() -> PathBuf {
232 crate::run::home().join("daemon.json")
233}
234
235pub fn write_status(status: &Status) -> Result<()> {
237 write_status_to(&status_path(), status)
238}
239
240pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
245 if let Some(parent) = path.parent() {
246 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
247 }
248 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
249 let tmp = path.with_extension("json.tmp");
250 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
251 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
252 Ok(())
253}
254
255pub fn clear_status() {
258 clear_status_at(&status_path());
259}
260
261fn clear_status_at(path: &Path) {
265 let _ = std::fs::remove_file(path);
266}
267
268#[derive(Debug, Clone, Default)]
281pub struct Stop {
282 stopped: Arc<AtomicBool>,
286 busy: Arc<std::sync::atomic::AtomicUsize>,
292 wake: Arc<Notify>,
296 pause: crate::graph::Pause,
299}
300
301impl Stop {
302 #[must_use]
304 pub fn new() -> Self {
305 Self::default()
306 }
307
308 pub fn stop(&self) {
311 self.stopped.store(true, Ordering::SeqCst);
312 self.wake.notify_one();
316 }
317
318 #[must_use]
320 pub fn stopped(&self) -> bool {
321 self.stopped.load(Ordering::SeqCst)
322 }
323
324 #[must_use]
332 pub fn finishing(&self) -> bool {
333 self.stopped() && self.busy_now()
334 }
335
336 pub fn park(&self, cause: crate::run::ParkCause) {
347 self.pause.park_for(cause);
348 self.stop();
349 }
350
351 #[must_use]
353 pub fn parking(&self) -> bool {
354 self.pause.parked()
355 }
356
357 #[must_use]
359 pub fn pause(&self) -> crate::graph::Pause {
360 self.pause.clone()
361 }
362
363 #[must_use]
369 pub fn busy_now(&self) -> bool {
370 self.busy.load(Ordering::SeqCst) > 0
371 }
372
373 fn enter(&self) {
375 self.busy.fetch_add(1, Ordering::SeqCst);
376 }
377
378 fn exit(&self) {
381 self.busy.fetch_sub(1, Ordering::SeqCst);
382 }
383
384 async fn idle(&self, poll: Duration) {
386 tokio::select! {
387 () = tokio::time::sleep(poll) => {}
388 () = self.wake.notified() => {}
389 }
390 }
391}
392
393#[derive(Debug, Clone, Default, Deserialize)]
400#[serde(default)]
401pub struct Reading {
402 pub schema: u32,
404 pub pid: Option<u32>,
406 pub started_at: Option<Timestamp>,
408 pub updated_at: Option<Timestamp>,
410 pub idle: bool,
412 #[serde(deserialize_with = "de_current")]
424 pub current: Vec<Current>,
425 pub completed: u64,
427 pub polls: u64,
429}
430
431fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
434where
435 D: serde::Deserializer<'de>,
436{
437 #[derive(Deserialize)]
438 #[serde(untagged)]
439 enum Shape {
440 Many(Vec<Current>),
441 One(Current),
442 }
443 Ok(
444 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
445 Shape::Many(v) => v,
446 Shape::One(c) => vec![c],
447 }),
448 )
449}
450
451impl Reading {
452 #[must_use]
455 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
456 self.updated_at
457 .map(|at| (now.as_second() - at.as_second()).max(0))
458 }
459
460 #[must_use]
464 pub fn running(&self, now: Timestamp) -> bool {
465 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
466 }
467}
468
469#[must_use]
476pub fn read_status(home: &Path) -> Option<Reading> {
477 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
478 serde_json::from_str(&body).ok()
479}
480
481#[must_use]
493pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
494 read_status(home)
495 .filter(|reading| reading.running(now))
496 .map(|reading| reading.current)
497 .unwrap_or_default()
498}
499
500#[must_use]
502pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
503 current_work(home, now).iter().any(|c| c.run == run)
504}
505
506#[must_use]
516pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
517 current_work(home, now)
518 .iter()
519 .any(|c| crate::run::short_of(&c.run) == short)
520}
521
522#[must_use]
524pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
525 current_work(home, now).iter().any(|c| c.task == task)
526}
527
528pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
572 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
573}
574
575fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
579where
580 F: Fn(u32) -> bool,
581{
582 let this_process = std::process::id();
583 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
584 .into_iter()
585 .flatten()
586 .flatten()
587 .map(|e| e.path())
588 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
589 .filter(|p| {
590 match std::fs::read_to_string(p)
591 .ok()
592 .and_then(|body| body.trim().parse::<u32>().ok())
593 {
594 Some(pid) if pid == this_process => false,
598 Some(pid) => !pid_alive(pid),
599 None => p
600 .metadata()
601 .and_then(|m| m.modified())
602 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
603 .is_ok_and(|age| age >= older_than),
604 }
605 })
606 .filter(|p| std::fs::remove_file(p).is_ok())
607 .filter_map(|p| {
608 p.file_stem()
609 .and_then(|s| s.to_str())
610 .map(std::borrow::ToOwned::to_owned)
611 })
612 .collect();
613 swept.sort_unstable();
614 swept
615}
616
617fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
622 task.status == TaskStatus::Running
623 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
624 && !is_working_on_task(home, &task.id, now)
625}
626
627fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
630 queue
631 .list()
632 .into_iter()
633 .filter(|t| is_stalled(t, home, now))
634 .collect()
635}
636
637fn queued_tasks(queue: &Queue) -> Vec<Task> {
643 queue
644 .list()
645 .into_iter()
646 .filter(|t| t.status == TaskStatus::Queued)
647 .collect()
648}
649
650fn finished_tasks(queue: &Queue) -> Vec<Task> {
653 queue
654 .list()
655 .into_iter()
656 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
657 .collect()
658}
659
660fn resolve_blockers(queue: &Queue, questions: &Questions) {
677 for listed in queue.list() {
678 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
679 continue;
680 }
681 let Ok(_claim) = queue.claim(&listed.id) else {
682 continue;
683 };
684 let Ok(mut task) = queue.get(&listed.id) else {
685 continue;
686 };
687 if task.status != TaskStatus::Blocked {
688 continue;
689 }
690 let deleted = queue.apply_deleted_blockers(&mut task, questions);
694 if !deleted.is_empty() {
695 record(queue, &mut task);
696 for id in &deleted {
697 queue.note_dependency_deleted(&task, id);
698 }
699 if task.status != TaskStatus::Blocked {
700 continue;
701 }
702 }
703 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
704 if !missing.is_empty() {
705 let language = language_of(&task, Path::new("."));
706 let mut cause = missing.clone();
707 cause.sort();
708 cause.dedup();
709 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
710 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
711 &task.blocked_by,
712 &missing,
713 &language,
714 )));
715 record(queue, &mut task);
716 continue;
717 }
718 if let Some(q) = task.blocked_by.iter().find_map(|id| {
721 let crate::queue::Dependency::Question(full) =
722 crate::queue::dependency(queue, questions, id)
723 else {
724 return None;
725 };
726 questions.get(&full).ok().filter(|q| {
727 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
728 })
729 }) {
730 let language = language_of(&task, Path::new("."));
731 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
732 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
733 record(queue, &mut task);
734 continue;
735 }
736 let mut changed = false;
737 for id in task.blocked_by.clone() {
738 let resolved = crate::queue::dependency(queue, questions, &id);
739 if let crate::queue::Dependency::Task(full) = &resolved {
740 if queue
741 .get(full)
742 .is_ok_and(|dep| dep.status == TaskStatus::Done)
743 {
744 task.unblock(&id);
745 changed = true;
746 }
747 continue;
748 }
749 if let crate::queue::Dependency::Question(full) = resolved
750 && let Ok(q) = questions.get(&full)
751 && q.status == ask::QuestionStatus::Answered
752 {
753 let answer = match &q.answer {
754 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
755 None => String::new(),
756 };
757 task.record_answer(q.summary.clone(), answer);
758 task.unblock(&id);
759 changed = true;
760 }
761 }
762 if changed {
763 record(queue, &mut task);
764 }
765 }
766}
767
768fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
770 if crate::lang::is_japanese(language) {
771 format!(
772 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
773 q.short(),
774 q.summary
775 )
776 } else {
777 format!(
778 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
779 q.short(),
780 q.summary
781 )
782 }
783}
784
785#[derive(Debug, Clone, Copy, PartialEq, Eq)]
788pub(crate) enum ActionStanding {
789 NoAction,
791 Pending,
793 Applied,
796 Stale,
798 Busy,
800}
801
802impl ActionStanding {
803 pub(crate) fn daemon_owns(self) -> bool {
805 matches!(self, Self::Pending | Self::Applied | Self::Stale)
806 }
807}
808
809pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
813 if q.chosen_action().is_none() {
814 ActionStanding::NoAction
815 } else if task.action_applied(&q.id) {
816 ActionStanding::Applied
817 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
818 ActionStanding::Stale
819 } else if matches!(
820 task.status,
821 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
822 ) {
823 ActionStanding::Busy
824 } else {
825 ActionStanding::Pending
826 }
827}
828
829#[derive(Debug, Clone, PartialEq, Eq)]
831enum ActionDecision {
832 Skip,
834 Resume(String),
836 Requeue,
838 Done,
840 Stale,
843 Refuse(String),
846}
847
848fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
856where
857 F: FnOnce(&str) -> Result<RunState>,
858{
859 let Some(action) = q.chosen_action() else {
860 return ActionDecision::Skip;
861 };
862 if task.action_applied(&q.id)
863 || matches!(
864 task.status,
865 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
866 )
867 {
868 return ActionDecision::Skip;
869 }
870 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
873 return ActionDecision::Stale;
874 }
875 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
876 let stopping = q.node == land::APPROVAL_NODE
877 && q.resolution()
878 .as_deref()
879 .is_some_and(crate::merge_decision::stops);
880 let decision_at = if repair || stopping {
884 q.answered_at.unwrap_or(q.asked_at)
885 } else {
886 q.asked_at
887 };
888 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
889 return if repair || stopping {
890 ActionDecision::Refuse(format!(
891 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
892 if repair { "repair" } else { "stop" },
893 q.id,
894 q.run,
895 task.id
896 ))
897 } else {
898 ActionDecision::Stale
899 };
900 }
901 match action {
902 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
903 ask::ChoiceAction::Done => ActionDecision::Done,
904 ask::ChoiceAction::Resume { run } => {
905 if task.runs.last() != Some(run) {
906 return ActionDecision::Refuse((p.resume_not_latest)(
907 q.short(),
908 ask::short_id(run),
909 ));
910 }
911 match load(run) {
912 Ok(s) if stopping => {
913 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
914 || !s
915 .land_approval
916 .as_ref()
917 .is_some_and(|binding| binding.question == q.id)
918 {
919 ActionDecision::Stale
920 } else if s.liveness(false) == Liveness::Live {
921 ActionDecision::Skip
922 } else if s.status == RunStatus::Landing && !s.released() {
923 ActionDecision::Resume(run.clone())
924 } else {
925 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
926 }
927 }
928 Ok(s)
929 if q.node == land::APPROVAL_NODE
930 && q.resolution().as_deref() == Some(land::REPAIR) =>
931 {
932 if s.merge_repairs.iter().any(|r| r.question == q.id)
933 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
934 {
935 ActionDecision::Stale
938 } else if s.liveness(false) == Liveness::Live {
939 ActionDecision::Skip
940 } else if s.status != RunStatus::Landing
941 || s.released()
942 || s.reviews.is_empty()
943 || !implementers_match(task, &s)
944 || !s
945 .land_approval
946 .as_ref()
947 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
948 {
949 ActionDecision::Refuse(format!(
950 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
951 q.id, run, run
952 ))
953 } else {
954 ActionDecision::Resume(run.clone())
955 }
956 }
957 Ok(s) if s.status.resumable() && !s.released() && !exhausted_review_budget(&s) => {
958 ActionDecision::Resume(run.clone())
959 }
960 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
961 q.short(),
962 ask::short_id(run),
963 )),
964 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
965 q.short(),
966 ask::short_id(run),
967 &format!("{e:#}"),
968 )),
969 }
970 }
971 }
972}
973
974struct Phrases {
988 graph_stopped: fn(&str, &str) -> String,
990 quorum_lost: &'static str,
991 quota_took_out: &'static str,
993 run_ended: &'static str,
995 waiting_for_answer: &'static str,
997 recovered_running: &'static str,
998 no_run_to_recover: &'static str,
999 could_not_start: &'static str,
1000 resume_not_latest: fn(&str, &str) -> String,
1001 resume_cannot_progress: fn(&str, &str) -> String,
1002 resume_unreadable: fn(&str, &str, &str) -> String,
1003 handover_refused: fn(&crate::handover::Refused) -> String,
1006 handover_hint: &'static str,
1008 reviewers_never_answered: fn(usize, usize) -> String,
1011}
1012
1013const PHRASES_EN: Phrases = Phrases {
1014 graph_stopped: |status, detail| {
1015 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1016 },
1017 quorum_lost: "the judging panel lost its quorum",
1018 quota_took_out: "; quota took out ",
1019 run_ended: "run ended ",
1020 waiting_for_answer: " - waiting for operator answer to question ",
1021 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1022 no_run_to_recover: "task was `running` with no live daemon and no readable \
1023 run to recover; held for a human to check what happened",
1024 could_not_start: "could not start the run: ",
1025 resume_not_latest: |q, run| {
1026 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1027 },
1028 resume_cannot_progress: |q, run| {
1029 format!("question {q} asked to resume run {run}, which cannot make progress")
1030 },
1031 resume_unreadable: |q, run, e| {
1032 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1033 },
1034 handover_refused: |r| r.to_string(),
1035 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1036 reviewers_never_answered: |missing, rounds| {
1037 format!(
1038 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1039 refusing to call it clean"
1040 )
1041 },
1042};
1043
1044const PHRASES_JA: Phrases = Phrases {
1045 graph_stopped: |status, detail| {
1046 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1047 },
1048 quorum_lost: "審査パネルが定足数を失いました",
1049 quota_took_out: "。クォータで脱落: ",
1050 run_ended: "run 終了: ",
1051 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1052 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1053 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1054 何が起きたか人が確認するため保留にしました",
1055 could_not_start: "run を開始できませんでした: ",
1056 resume_not_latest: |q, run| {
1057 format!(
1058 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1059 )
1060 },
1061 resume_cannot_progress: |q, run| {
1062 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1063 },
1064 resume_unreadable: |q, run, e| {
1065 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1066 },
1067 handover_refused: |r| {
1068 use crate::handover::Refused;
1069 match r {
1070 Refused::Foreign { branch, path, why } => format!(
1071 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1072 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1073 から、やり直してください(詳細: {why})"
1074 ),
1075 Refused::Unsafe { branch, path, why } => format!(
1076 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1077 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1078 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1079 ),
1080 Refused::ReleaseFailed { branch, path, run } => format!(
1081 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1082 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1083 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1084 ),
1085 }
1086 },
1087 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1088 reviewers_never_answered: |missing, rounds| {
1089 format!(
1090 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1091 クリーンとは認めません"
1092 )
1093 },
1094};
1095
1096fn phrases(language: &str) -> &'static Phrases {
1097 if crate::lang::is_japanese(language) {
1098 &PHRASES_JA
1099 } else {
1100 &PHRASES_EN
1101 }
1102}
1103
1104fn language_of(task: &Task, fallback: &Path) -> String {
1108 crate::lang::of_repo(&repo_for(task, fallback))
1109}
1110
1111pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1115 if q.node == crate::conduct::NODE {
1116 return tasks.iter().find(|t| t.id == q.run);
1117 }
1118 tasks.iter().find(|t| t.runs.contains(&q.run))
1119}
1120
1121pub(crate) fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1130 let tasks = queue.list();
1131 for q in questions.list() {
1132 if q.chosen_action().is_none() {
1133 continue;
1134 }
1135 let Some(listed) = task_of_question(&tasks, &q) else {
1136 if q.node == land::APPROVAL_NODE
1137 && q.resolution().as_deref().is_some_and(|choice| {
1138 choice == land::REPAIR || crate::merge_decision::stops(choice)
1139 })
1140 {
1141 notices::raise_in(
1142 home,
1143 Notice::warn(
1144 &format!("repair-owner:{}", q.id),
1145 format!(
1146 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1147 q.run, q.run
1148 ),
1149 ),
1150 );
1151 }
1152 continue;
1153 };
1154 if listed.action_applied(&q.id) {
1155 continue;
1156 }
1157 let Ok(_claim) = queue.claim(&listed.id) else {
1158 continue;
1159 };
1160 let Ok(mut task) = queue.get(&listed.id) else {
1161 continue;
1162 };
1163 let language = language_of(&task, Path::new("."));
1164 let decision = decide_action(&task, &q, phrases(&language), |id| {
1165 RunState::load_under(id, home)
1166 });
1167 let ran = matches!(
1168 decision,
1169 ActionDecision::Resume(_) | ActionDecision::Requeue | ActionDecision::Done
1170 );
1171 if ran && asker_may_still_read(questions, &q, home) {
1172 continue;
1181 }
1182 let done = matches!(decision, ActionDecision::Done);
1183 match decision {
1184 ActionDecision::Skip => continue,
1185 ActionDecision::Resume(run) => {
1186 task.release_operator("choice", Some(&q.id));
1187 task.resume_override = Some(crate::queue::OperatorResume {
1188 question_id: q.id.clone(),
1189 at: Timestamp::now(),
1190 conductor_rehold: None,
1191 forced: true,
1192 pinned_run: Some(run),
1193 });
1194 }
1195 ActionDecision::Stale => {}
1196 ActionDecision::Requeue => {
1197 task.release_operator("choice-requeue", Some(&q.id));
1198 task.requeue();
1199 }
1200 ActionDecision::Done => task.succeed(),
1201 ActionDecision::Refuse(why) => {
1202 if task.operator_held() {
1203 task.hold_reason = Some(match task.hold_reason.take() {
1206 Some(reason) => format!("{reason}\n\n{why}"),
1207 None => why,
1208 });
1209 } else {
1210 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1211 task.hold_machine(Some(why));
1212 }
1213 notices::raise(
1214 Notice::warn(
1215 &format!("action:{}", q.id),
1216 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1217 )
1218 .link(Link::Task {
1219 id: task.id.clone(),
1220 }),
1221 );
1222 }
1223 }
1224 task.mark_action_applied(&q.id);
1225 record(queue, &mut task);
1228 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1229 supersede_prior_runs(&task, home);
1230 }
1231 let _ = questions.update(&q.id, |r| {
1234 r.answer_delivered = true;
1235 Ok(())
1236 });
1237 }
1238}
1239
1240fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1244 let now = Timestamp::now();
1245 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1246 return true;
1247 }
1248 q.node != crate::conduct::NODE
1249 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1250 s.seats_active()
1251 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1252 })
1253}
1254
1255fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1266 let tasks = queue.list();
1267 let by_id: std::collections::BTreeMap<&str, &Task> =
1268 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1269 let referenced: Vec<&str> = tasks
1270 .iter()
1271 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1272 .collect();
1273
1274 for mut question in questions.list() {
1275 if !question.status.open() || question.node != crate::conduct::NODE {
1276 continue;
1277 }
1278 if referenced
1283 .iter()
1284 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1285 {
1286 continue;
1287 }
1288 let Some(task) = by_id.get(question.run.as_str()) else {
1289 continue;
1290 };
1291 question.abandon(format!(
1292 "task {} no longer waits for this answer",
1293 task.short()
1294 ));
1295 if let Err(e) = questions.put(&mut question) {
1296 tracing::warn!(
1297 "could not retire question {} for task {}: {e:#}",
1298 question.short(),
1299 task.short()
1300 );
1301 }
1302 }
1303}
1304
1305#[derive(Debug, Clone, Copy)]
1312pub struct Verdict {
1313 pub status: RunStatus,
1315 pub left_pr: bool,
1317 pub quota_hit: bool,
1319 pub parked: bool,
1321 pub no_viable_candidates: bool,
1328}
1329
1330pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1387 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1388}
1389
1390fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1392 if verdict.parked {
1398 task.park(detail);
1399 return;
1400 }
1401 match verdict.status {
1402 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1403 RunStatus::AlreadyInBase => task.already_landed(detail),
1404 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1405 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1406 task.stall(detail)
1407 }
1408 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1409 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1410 RunStatus::Blocked => task.fail(detail, max_attempts),
1411 RunStatus::VerifiedNoop => task.handed_off(detail),
1412 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1413 }
1414}
1415
1416pub fn supersede_prior_runs(task: &Task, home: &Path) {
1465 let now = Timestamp::now();
1466 let last_run_succeeded = task
1473 .runs
1474 .last()
1475 .and_then(|id| RunState::load_under(id, home).ok())
1476 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1477 for id in task.superseded_attempts(last_run_succeeded) {
1478 let mut state = match RunState::load_under(id, home) {
1479 Ok(s) => s,
1480 Err(e) => {
1481 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1482 continue;
1483 }
1484 };
1485 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1486 continue;
1487 }
1488 let daemon_claims = is_working_on(home, id, now);
1489 if state.liveness(daemon_claims) == Liveness::Live {
1490 continue;
1491 }
1492 state.status = RunStatus::Superseded;
1493 if let Err(e) = state.save_under(home) {
1494 tracing::warn!("could not mark run {id} superseded: {e:#}");
1495 }
1496 }
1497}
1498
1499fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1521 for task in queue.list() {
1522 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1523 continue;
1524 }
1525 supersede_prior_runs(&task, home);
1526 }
1527}
1528
1529fn settle_and_diagnose(
1536 task: &mut Task,
1537 verdict: Verdict,
1538 detail: &str,
1539 max_attempts: usize,
1540 state: &RunState,
1541) {
1542 let p = phrases(&state.config.graph.language);
1543 if let Some(d) = &state.merge_decision {
1546 use crate::merge_decision as md;
1547 if task.runs.last().is_some_and(|id| id != &state.id) {
1548 return;
1549 }
1550 if d.choice == md::WITHDRAW && d.closed {
1551 task.succeed();
1552 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1553 if state.liveness(false) != Liveness::Dead {
1554 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1555 return;
1556 }
1557 let replacement = md::Replacement {
1558 question: d.question.clone(),
1559 run: state.id.clone(),
1560 pr: d.target.pr.clone(),
1561 agent: agent.to_owned(),
1562 context: format!(
1563 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1564 d.target.pr,
1565 state.id,
1566 serde_json::to_string(&state.reviews).unwrap_or_default()
1567 ),
1568 };
1569 task.requeue();
1570 task.pr_recovery = None;
1571 task.review_of = None;
1572 task.replacement = Some(replacement);
1573 } else {
1574 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1575 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1576 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1577 }
1578 task.mark_action_applied(&d.question);
1579 if let Some(pass) = state.passes.last() {
1580 task.record_pass(&state.id, pass.index, task.attempts, None);
1581 }
1582 return;
1583 }
1584 if let Some(replacement) = &task.replacement
1585 && !state.parked
1586 && (state.viable().is_empty()
1587 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1588 {
1589 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1590 return;
1591 }
1592 let attempts_before = task.attempts;
1593 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1599 task.notice_cause(
1600 &format!("resource:{}:{}", block.kind, block.node),
1601 None,
1602 None,
1603 );
1604 task.last_error = Some(detail.to_owned());
1605 task.diagnostic = None;
1606 task.attempts = task.attempts.saturating_sub(1);
1607 task.hold_machine(Some(format!(
1608 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1609 no agent was asked and no attempt was spent",
1610 crate::cache::RESOURCE_HOLD,
1611 block.kind,
1612 block.node
1613 )));
1614 if let Some(pass) = state.passes.last() {
1615 task.record_pass(&state.id, pass.index, attempts_before, None);
1616 }
1617 return;
1618 }
1619 let unresolved = task.hold_notice_cause.clone();
1620 settle_in(task, verdict, detail, max_attempts, p);
1621 if state.prep_failure.is_some() {
1622 task.hold_notice_cause = unresolved;
1625 task.startup_notice_cause("startup-candidates");
1626 }
1627 if let Some(pass) = state.passes.last() {
1630 let parked = match &pass.end {
1631 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
1632 _ => None,
1633 };
1634 task.record_pass(&state.id, pass.index, attempts_before, parked);
1635 }
1636 if task.status == TaskStatus::Held {
1637 if state.prep_failure.is_none() {
1638 task.notice_cause(
1639 label(verdict.status),
1640 Some(&state.id),
1641 state.passes.last().map(|p| p.index),
1642 );
1643 }
1644 task.diagnostic = diagnostic(state);
1645 note_open_question(task, &state.id, p);
1646 }
1647}
1648
1649fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
1664 let Some(home) = crate::run::try_home() else {
1665 return;
1666 };
1667 let open = Questions::at(home.join("questions")).open_for(run);
1668 let Some(q) = open.first() else {
1669 return;
1670 };
1671 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
1672 decisions.sort_unstable();
1673 task.notice_cause(
1674 &format!("questions:{}", decisions.join(",")),
1675 Some(run),
1676 None,
1677 );
1678 let base = task.hold_reason.clone().unwrap_or_default();
1679 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
1680}
1681
1682fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
1693 match last_run {
1694 Some(state) => {
1695 let verdict = Verdict {
1696 status: state.status,
1697 left_pr: state.pr.is_some(),
1698 quota_hit: !state.quota.is_empty(),
1699 parked: state.parked,
1700 no_viable_candidates: state.viable().is_empty(),
1701 };
1702 let detail = format!(
1703 "{}{}",
1704 phrases(&state.config.graph.language).recovered_running,
1705 describe(&state)
1706 );
1707 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
1708 }
1709 None => {
1710 let why = phrases(language).no_run_to_recover;
1713 task.last_error = Some(why.to_owned());
1714 task.notice_cause("missing-run", None, None);
1717 task.hold_machine(Some(why.to_owned()));
1718 }
1719 }
1720}
1721
1722fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
1744 let mut reclaimed = Vec::new();
1745 for listed in queue.list() {
1746 if listed.status != TaskStatus::Running {
1747 continue;
1748 }
1749 let Ok(_claim) = queue.claim(&listed.id) else {
1750 continue;
1751 };
1752 let Ok(mut task) = queue.get(&listed.id) else {
1756 continue;
1757 };
1758 if task.status != TaskStatus::Running {
1759 continue;
1760 }
1761 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
1762 if let Some(state) = &last_run
1772 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
1773 {
1774 tracing::warn!("abandon questions for {}: {e:#}", state.id);
1775 }
1776 let language = if last_run.is_none() {
1777 language_of(&task, Path::new("."))
1778 } else {
1779 String::new()
1780 };
1781 reclaim(&mut task, last_run, max_attempts, &language);
1782 if task.status == TaskStatus::Done {
1783 supersede_prior_runs(&task, &crate::run::home());
1784 }
1785 record(queue, &mut task);
1786 reclaimed.push(task.id.clone());
1787 }
1788 reclaimed
1789}
1790
1791fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
1823 reclaim_abandoned_runs_with(
1824 home,
1825 now,
1826 crate::proc::pid_status,
1827 crate::proc::process_started_at,
1828 )
1829}
1830
1831fn reclaim_abandoned_runs_with<F, G>(
1837 home: &Path,
1838 now: Timestamp,
1839 query: F,
1840 identity: G,
1841) -> Vec<String>
1842where
1843 F: Fn(u32) -> Option<bool>,
1844 G: Fn(u32) -> Option<String>,
1845{
1846 let mut abandoned = Vec::new();
1847 for entry in std::fs::read_dir(home.join("runs"))
1848 .into_iter()
1849 .flatten()
1850 .flatten()
1851 {
1852 let id = entry.file_name().to_string_lossy().into_owned();
1853 if !crate::run::is_run_id(&id) {
1854 continue;
1855 }
1856 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
1864 continue;
1865 };
1866 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
1867 continue;
1868 };
1869 if state.status.done() || !state.active_all_overrun(now) {
1870 continue;
1871 }
1872 let daemon_claims = is_working_on(home, &id, now);
1884 if state.liveness_with(daemon_claims, &query, &identity) != crate::run::Liveness::Dead {
1885 continue;
1886 }
1887 state.abandon("daemon");
1888 if let Err(e) = state.save_under(home) {
1889 tracing::warn!("could not persist abandoned run {id}: {e:#}");
1890 continue;
1891 }
1892 if let Some(notice) = notices::run_ended(&state) {
1895 notices::raise_in(home, notice);
1896 }
1897 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
1905 tracing::warn!("abandon questions for {id}: {e:#}");
1906 }
1907 abandoned.push(id);
1908 }
1909 abandoned
1910}
1911
1912pub async fn serve(opts: Opts) -> Result<()> {
1918 serve_until(opts, Stop::new()).await
1919}
1920
1921pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
1938 let signal = {
1939 let stop = stop.clone();
1940 tokio::spawn(async move {
1941 if tokio::signal::ctrl_c().await.is_ok() {
1942 stop.stop();
1943 tracing::info!("shutdown requested; a run in flight will be finished first");
1944 }
1945 })
1946 };
1947
1948 let worktrees_root = opts
1949 .worktrees_root
1950 .clone()
1951 .unwrap_or_else(crate::run::default_worktree_root);
1952 let outcome = drive(
1953 &opts,
1954 &Queue::open(),
1955 &status_path(),
1956 &crate::run::home(),
1957 &worktrees_root,
1958 &stop,
1959 )
1960 .await;
1961
1962 signal.abort();
1963 outcome
1964}
1965
1966async fn drive(
1979 opts: &Opts,
1980 queue: &Queue,
1981 status_file: &Path,
1982 home: &Path,
1983 worktrees_root: &Path,
1984 stop: &Stop,
1985) -> Result<()> {
1986 let status = Arc::new(Mutex::new(Status::new()));
1994 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
1995 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
1996
1997 let daemon_cfg = prepare(&opts.repo, opts)
2003 .map(|c| c.daemon)
2004 .unwrap_or_default();
2005 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2006
2007 let waiter = tokio::spawn(crate::waiter::run(
2012 crate::waiter::Waiter::new(
2013 crate::ask::Questions::at(home.join("questions")),
2014 home.to_path_buf(),
2015 prepare(&opts.repo, opts).ok(),
2016 ),
2017 stop.clone(),
2018 ));
2019
2020 let deputies = tokio::spawn(crate::deputy::run(
2024 crate::deputy::Deputies::new(
2025 crate::ask::Questions::at(home.join("questions")),
2026 home.to_path_buf(),
2027 prepare(&opts.repo, opts).ok(),
2028 opts.repo.clone(),
2029 daemon_cfg.max_deputies,
2030 {
2031 let stop = stop.clone();
2032 Arc::new(move || stop.parking())
2033 },
2034 ),
2035 stop.clone(),
2036 ));
2037
2038 let fetcher = tokio::spawn(fetch_loop(opts.repo.clone(), opts.clone(), stop.clone()));
2041
2042 let release_watch = tokio::spawn(crate::release_watch::run(
2045 crate::release_watch::Watcher::new(
2046 Box::new(crate::release_watch::GhForge),
2047 home.to_path_buf(),
2048 ),
2049 {
2050 let (repo, opts) = (opts.repo.clone(), opts.clone());
2051 move || release_watch_settings(&repo, &opts)
2052 },
2053 stop.clone(),
2054 ));
2055
2056 tracing::info!(
2057 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2058 queue.root().display(),
2059 opts.poll.as_secs(),
2060 opts.max_attempts,
2061 concurrency,
2062 if daemon_cfg.pause_for_interrupts {
2063 ", interrupts enabled"
2064 } else {
2065 ""
2066 }
2067 );
2068
2069 janitor(&opts.repo, opts, home, worktrees_root).await;
2072 resweep_superseded_attempts(queue, home);
2073
2074 let outcome = poll(
2075 opts,
2076 queue,
2077 &status,
2078 home,
2079 worktrees_root,
2080 stop,
2081 DispatchLimits {
2082 max_concurrent: concurrency,
2083 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2084 },
2085 )
2086 .await;
2087
2088 beat.abort();
2089 waiter.abort();
2090 deputies.abort();
2091 fetcher.abort();
2092 release_watch.abort();
2093 clear_status_at(status_file);
2094 outcome
2095}
2096
2097fn release_watch_settings(repo: &Path, opts: &Opts) -> (Vec<PathBuf>, u64) {
2100 let Ok(cfg) = prepare(repo, opts) else {
2101 return (Vec::new(), 0);
2102 };
2103 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2104 .into_iter()
2105 .map(|r| r.path)
2106 .collect();
2107 let mut known: Vec<PathBuf> = Queue::open().list().into_iter().map(|t| t.repo).collect();
2110 let runs = crate::run::home().join("runs");
2111 for id in crate::run::list_ids_in(&runs) {
2112 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2113 .ok()
2114 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2115 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2116 known.extend(repo);
2117 }
2118 for k in known {
2119 let k = k.canonicalize().unwrap_or(k);
2120 if k.join(".git").exists() && !paths.contains(&k) {
2121 paths.push(k);
2122 }
2123 }
2124 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2125 if !paths.contains(&own) {
2126 paths.push(own);
2127 }
2128 (paths, cfg.daemon.release_stall_minutes)
2129}
2130
2131const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2133
2134async fn fetch_loop(repo: PathBuf, opts: Opts, stop: Stop) {
2138 while !stop.stopped() {
2139 let (interval, roots) = match prepare(&repo, &opts) {
2140 Ok(c) => (c.repos.fetch_interval, c.repos.roots),
2141 Err(_) => (0, Vec::new()),
2142 };
2143 if interval > 0 && !roots.is_empty() {
2144 let r = crate::clean::fetch_origins(&roots, FETCH_TIMEOUT, || stop.stopped()).await;
2145 tracing::debug!("fetch origins: {r:?}");
2146 }
2147 let wait = if interval > 0 { interval } else { 60 };
2149 let mut slept = 0;
2150 while slept < wait && !stop.stopped() {
2151 tokio::time::sleep(Duration::from_secs(1)).await;
2152 slept += 1;
2153 }
2154 }
2155}
2156
2157async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2163 loop {
2164 tokio::time::sleep(HEARTBEAT).await;
2165 let snapshot = {
2166 let mut guard = lock(&status);
2167 guard.updated_at = Timestamp::now();
2168 guard.clone()
2169 };
2170 if let Err(e) = write_status_to(&path, &snapshot) {
2171 tracing::warn!("could not refresh the daemon status file: {e:#}");
2174 }
2175 }
2176}
2177
2178#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2181enum LandResume {
2182 NotLanding,
2185 StillWaiting,
2190 Ready,
2194}
2195
2196fn land_resume_state(task: &Task) -> LandResume {
2200 let Some(run_id) = task.runs.last() else {
2201 return LandResume::NotLanding;
2202 };
2203 let Ok(state) = RunState::load(run_id) else {
2204 return LandResume::NotLanding;
2205 };
2206 if !state.parked {
2207 return LandResume::NotLanding;
2208 }
2209 if state.status == RunStatus::Landing
2210 && state.land_approval.as_ref().is_some_and(|binding| {
2211 ask::Questions::open()
2212 .get(&binding.question)
2213 .is_ok_and(|q| {
2214 q.run == state.id
2215 && q.resolution()
2216 .as_deref()
2217 .is_some_and(crate::merge_decision::stops)
2218 })
2219 })
2220 {
2221 return LandResume::Ready;
2222 }
2223 if !implementers_match(task, &state) {
2224 return LandResume::NotLanding;
2225 }
2226 let node = if state.status == RunStatus::Landing {
2230 land::APPROVAL_NODE
2231 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2232 crate::github_text::ASK_NODE
2233 } else {
2234 return LandResume::NotLanding;
2235 };
2236 let store = ask::Questions::open();
2237 let waiting = store
2238 .list()
2239 .into_iter()
2240 .filter(|q| &q.run == run_id && q.node == node)
2241 .max_by(|a, b| a.id.cmp(&b.id));
2242 let Some(mut q) = waiting else {
2243 return LandResume::Ready;
2244 };
2245 if !q.status.open() {
2246 if q.resolution().as_deref() == Some(land::REPAIR) {
2249 return if state.liveness(false) == Liveness::Live {
2250 LandResume::StillWaiting
2251 } else {
2252 LandResume::NotLanding
2253 };
2254 }
2255 return LandResume::Ready;
2256 }
2257 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2264 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2265 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2266 q.abandon(format!(
2267 "no answer within {}s of asking",
2268 timeout.as_secs().max(1)
2269 ));
2270 if store.put(&mut q).is_ok() {
2273 return LandResume::Ready;
2274 }
2275 }
2276 LandResume::StillWaiting
2277}
2278
2279const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2287
2288const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2300
2301struct InFlightGuard<'a> {
2314 status: &'a Arc<Mutex<Status>>,
2315 stop: &'a Stop,
2316 task_id: &'a str,
2317}
2318
2319impl Drop for InFlightGuard<'_> {
2320 fn drop(&mut self) {
2321 lock(self.status).current.retain(|c| c.task != self.task_id);
2322 self.stop.exit();
2323 }
2324}
2325
2326#[derive(Debug, Clone, PartialEq, Eq)]
2348enum Interrupt {
2349 Idle,
2351 Parking {
2363 parked: Vec<String>,
2364 interrupt_task: String,
2365 },
2366 Running {
2374 parked: Vec<String>,
2375 interrupt_task: String,
2376 },
2377 Resuming { parked: Vec<String> },
2384}
2385
2386fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
2407 match state {
2408 Interrupt::Idle => {
2409 if in_flight.len() != 1 {
2421 return Interrupt::Idle;
2422 }
2423 match runnable.iter().find(|t| t.interrupt) {
2424 Some(t) => Interrupt::Parking {
2425 parked: in_flight.to_vec(),
2426 interrupt_task: t.id.clone(),
2427 },
2428 None => Interrupt::Idle,
2429 }
2430 }
2431 Interrupt::Parking {
2432 parked,
2433 interrupt_task,
2434 } => {
2435 if in_flight.iter().any(|id| parked.contains(id)) {
2436 Interrupt::Parking {
2438 parked,
2439 interrupt_task,
2440 }
2441 } else if in_flight.contains(&interrupt_task) {
2442 Interrupt::Running {
2443 parked,
2444 interrupt_task,
2445 }
2446 } else if runnable.iter().any(|t| t.id == interrupt_task) {
2447 Interrupt::Parking {
2451 parked,
2452 interrupt_task,
2453 }
2454 } else {
2455 Interrupt::Resuming { parked }
2460 }
2461 }
2462 Interrupt::Running {
2463 parked,
2464 interrupt_task,
2465 } => {
2466 if in_flight.contains(&interrupt_task) {
2467 Interrupt::Running {
2468 parked,
2469 interrupt_task,
2470 }
2471 } else {
2472 Interrupt::Resuming { parked }
2478 }
2479 }
2480 Interrupt::Resuming { parked } => {
2481 if in_flight.iter().any(|id| parked.contains(id)) {
2482 Interrupt::Idle
2488 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
2489 Interrupt::Resuming { parked }
2490 } else {
2491 Interrupt::Idle
2494 }
2495 }
2496 }
2497}
2498
2499fn advance_interrupt_tick(
2505 enabled: bool,
2506 state: Interrupt,
2507 in_flight: &[String],
2508 runnable: &[Task],
2509) -> Interrupt {
2510 if !enabled {
2511 return Interrupt::Idle;
2512 }
2513 advance_interrupt(state, in_flight, runnable)
2514}
2515
2516fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
2521 match state {
2522 Interrupt::Idle => candidates,
2523 Interrupt::Parking {
2524 parked,
2525 interrupt_task,
2526 } => {
2527 if in_flight.iter().any(|id| parked.contains(id)) {
2528 Vec::new()
2529 } else {
2530 candidates
2531 .into_iter()
2532 .filter(|t| &t.id == interrupt_task)
2533 .collect()
2534 }
2535 }
2536 Interrupt::Running { .. } => Vec::new(),
2537 Interrupt::Resuming { parked } => candidates
2545 .into_iter()
2546 .find(|t| parked.contains(&t.id))
2547 .into_iter()
2548 .collect(),
2549 }
2550}
2551
2552struct DispatchLimits {
2556 max_concurrent: usize,
2559 pause_for_interrupts: bool,
2561}
2562
2563#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2572enum PermitKind {
2573 None,
2578 Urgent,
2585 Ordinary,
2588}
2589
2590fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
2593 if priority {
2594 PermitKind::None
2595 } else if urgent {
2596 PermitKind::Urgent
2597 } else {
2598 PermitKind::Ordinary
2599 }
2600}
2601
2602struct DispatchClaim {
2603 task: Task,
2604 claim: crate::queue::Claim,
2605 permit: Option<tokio::sync::OwnedSemaphorePermit>,
2606}
2607
2608fn claim_dispatch(
2611 queue: &Queue,
2612 candidate: &Task,
2613 cooling_down: bool,
2614 ordinary: &Arc<tokio::sync::Semaphore>,
2615 urgent: &Arc<tokio::sync::Semaphore>,
2616) -> Option<DispatchClaim> {
2617 let Ok(claim) = queue.claim(&candidate.id) else {
2618 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
2619 return None;
2620 };
2621 let task = match queue.get(&candidate.id) {
2622 Ok(t) if t.status.runnable() => t,
2623 Ok(_) => return None,
2624 Err(e) => {
2625 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
2626 return None;
2627 }
2628 };
2629 let resume = land_resume_state(&task);
2630 if resume == LandResume::StillWaiting {
2631 return None;
2632 }
2633 let priority = resume == LandResume::Ready;
2634 if !priority && cooling_down {
2635 return None;
2636 }
2637 let permit = match permit_kind(priority, task.urgent) {
2640 PermitKind::None => None,
2641 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
2642 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
2643 };
2644 Some(DispatchClaim {
2645 task,
2646 claim,
2647 permit,
2648 })
2649}
2650
2651async fn poll(
2670 opts: &Opts,
2671 queue: &Queue,
2672 status: &Arc<Mutex<Status>>,
2673 home: &Path,
2674 worktrees_root: &Path,
2675 stop: &Stop,
2676 limits: DispatchLimits,
2677) -> Result<()> {
2678 let DispatchLimits {
2679 max_concurrent,
2680 pause_for_interrupts,
2681 } = limits;
2682 let mut attempted: Vec<String> = Vec::new();
2687 let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
2688 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(1));
2696 let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
2702 let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
2703 let mut conductor = Conductor::new();
2704 let mut cache_last_checked: Option<Timestamp> = None;
2707 let mut interrupt = Interrupt::Idle;
2709 let mut interrupt_pauses: std::collections::HashMap<String, crate::graph::Pause> =
2715 std::collections::HashMap::new();
2716
2717 while !stop.stopped() {
2718 lock(status).polls += 1;
2719
2720 while let Some(result) = inflight.try_join_next() {
2725 if let Err(e) = result {
2726 tracing::error!("a spawned attempt did not finish cleanly: {e}");
2727 notices::raise(Notice::error(
2728 "loop:attempt",
2729 "A queued attempt ended abnormally; check the task it was running.",
2730 ));
2731 }
2732 }
2733
2734 let swept = sweep_stale_claims(queue, STALE_CLAIM);
2735 if !swept.is_empty() {
2736 tracing::warn!(
2737 "swept {} stale claim(s) left behind by an earlier daemon: {}",
2738 swept.len(),
2739 swept.join(", ")
2740 );
2741 }
2742 let now = Timestamp::now();
2747
2748 if !stop.busy_now() {
2753 maybe_prune_cache_between_runs(
2754 &opts.repo,
2755 opts,
2756 home,
2757 stop,
2758 &mut cache_last_checked,
2759 now,
2760 )
2761 .await;
2762 }
2763
2764 let stalled = stalled_tasks(queue, home, now);
2765 let stalled_ids: std::collections::BTreeSet<_> =
2766 stalled.iter().map(|task| task.id.clone()).collect();
2767 let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
2768 if !reclaimed.is_empty() {
2769 tracing::warn!(
2770 "reclaimed {} task(s) left `running` by a daemon that never \
2771 recorded the outcome: {}",
2772 reclaimed.len(),
2773 reclaimed.join(", ")
2774 );
2775 }
2776 let abandoned_runs = reclaim_abandoned_runs(home, now);
2777 if !abandoned_runs.is_empty() {
2778 tracing::warn!(
2779 "failed {} run(s) left behind by a killed process, past every \
2780 active seat's own timeout: {}",
2781 abandoned_runs.len(),
2782 abandoned_runs.join(", ")
2783 );
2784 }
2785
2786 let questions = Questions::at(home.join("questions"));
2791
2792 resolve_blockers(queue, &questions);
2795 apply_choice_actions(queue, &questions, home);
2796 triage::apply_answers(queue, &questions);
2797 reconcile_task_questions(queue, &questions);
2798 {
2802 let talks = crate::talk::Talks::at(home.join("talks"));
2803 crate::chat_notify::sweep(queue, &talks, home);
2804 crate::chat_watch::sweep(home, &talks);
2805 crate::chat_notify::start_turns(&talks, stop.parking(), &mut inflight);
2806 }
2807
2808 let finished: Vec<Task> = finished_tasks(queue)
2814 .into_iter()
2815 .filter(|task| !stalled_ids.contains(&task.id))
2816 .filter(|task| !awaiting_resume_with(task, |id| RunState::load_under(id, home)))
2820 .collect();
2821 let queued = queued_tasks(queue);
2822 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
2826 && conductor.worth_a_look(queue, &stalled, &finished)
2827 {
2828 match prepare(&opts.repo, opts) {
2829 Ok(cfg) => {
2830 conductor
2831 .maybe_run(
2832 &cfg,
2833 &opts.repo,
2834 queue,
2835 &questions,
2836 home,
2837 &queued,
2838 &stalled,
2839 &finished,
2840 opts.max_attempts,
2841 )
2842 .await;
2843 }
2844 Err(e) => {
2845 tracing::warn!("conductor: no config: {e:#}");
2846 notices::raise(Notice::warn(
2847 "loop:no-config",
2848 "The loop could not read this repository's config, so held tasks are not being triaged.",
2849 ));
2850 }
2851 }
2852 }
2853
2854 let candidates: Vec<Task> = runnable(queue)
2855 .into_iter()
2856 .filter(|t| !opts.once || !attempted.contains(&t.id))
2857 .collect();
2858
2859 let in_flight: Vec<String> = lock(status)
2864 .current
2865 .iter()
2866 .map(|c| c.task.clone())
2867 .collect();
2868 interrupt_pauses.retain(|id, _| in_flight.contains(id));
2869
2870 interrupt =
2871 advance_interrupt_tick(pause_for_interrupts, interrupt, &in_flight, &candidates);
2872 if let Interrupt::Parking {
2873 parked,
2874 interrupt_task,
2875 } = &interrupt
2876 {
2877 let reason = format!(
2878 "task {} asked to run first",
2879 crate::run::short_of(interrupt_task)
2880 );
2881 for id in parked {
2882 if let Some(pause) = interrupt_pauses.get(id) {
2883 pause.park_because(reason.clone());
2884 }
2885 }
2886 }
2887 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
2888
2889 let cooling_down =
2890 lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
2891
2892 let mut started_any = false;
2893 for candidate in candidates {
2894 if stop.stopped() {
2895 break;
2896 }
2897
2898 let Some(DispatchClaim {
2899 mut task,
2900 claim,
2901 permit,
2902 }) = claim_dispatch(queue, &candidate, cooling_down, &sem, &urgent_sem)
2903 else {
2904 continue;
2905 };
2906
2907 let task_id = task.id.clone();
2908 attempted.push(task_id.clone());
2909 lock(status).idle = false;
2910 stop.enter();
2913 started_any = true;
2914
2915 let run_pause = crate::graph::Pause::new();
2919 interrupt_pauses.insert(task_id.clone(), run_pause.clone());
2920
2921 let opts = opts.clone();
2922 let queue = queue.clone();
2923 let status = Arc::clone(status);
2924 let stop = stop.clone();
2925 let quota_cooldown_until = Arc::clone("a_cooldown_until);
2926 inflight.spawn(async move {
2927 let _claim = claim;
2931 let _permit = permit;
2932 let _inflight = InFlightGuard {
2934 status: &status,
2935 stop: &stop,
2936 task_id: &task_id,
2937 };
2938 let quota = attempt(&opts, &queue, &status, &stop, run_pause, &mut task).await;
2939 lock(&status).completed += 1;
2940 let now = Timestamp::now();
2946 if let Some(until) = cooldown_until("a, now) {
2947 let wait = until.as_second() - now.as_second();
2948 *lock("a_cooldown_until) = Some(until);
2949 let hint = quota
2950 .iter()
2951 .find(|q| q.reset.is_some())
2952 .and_then(|q| q.reset.as_deref());
2953 match hint {
2954 Some(h) => tracing::warn!(
2955 "quota hit; waiting {wait}s before taking another ordinary task \
2956 (CLI reported reset: {h})"
2957 ),
2958 None => tracing::warn!(
2959 "quota hit; waiting {wait}s before taking another ordinary task \
2960 (no reset hint reported)"
2961 ),
2962 }
2963 }
2964 });
2965 }
2966
2967 if started_any {
2968 continue;
2969 }
2970
2971 if stop.busy_now() {
2972 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
2977 continue;
2978 }
2979
2980 lock(status).idle = true;
2982 if opts.once {
2983 janitor(&opts.repo, opts, home, worktrees_root).await;
2987 resweep_superseded_attempts(queue, home);
2988 triage_held(queue, home, opts).await;
2989 break;
2990 }
2991 stop.idle(opts.poll).await;
2992 if stop.stopped() {
2993 continue;
2994 }
2995 janitor(&opts.repo, opts, home, worktrees_root).await;
3001 resweep_superseded_attempts(queue, home);
3002 triage_held(queue, home, opts).await;
3003 }
3004
3005 while let Some(result) = inflight.join_next().await {
3010 if let Err(e) = result {
3011 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3012 notices::raise(Notice::error(
3013 "loop:attempt",
3014 "A queued attempt ended abnormally; check the task it was running.",
3015 ));
3016 }
3017 }
3018 Ok(())
3019}
3020
3021async fn attempt(
3027 opts: &Opts,
3028 queue: &Queue,
3029 status: &Arc<Mutex<Status>>,
3030 stop: &Stop,
3031 interrupt_pause: crate::graph::Pause,
3032 task: &mut Task,
3033) -> Vec<QuotaLoss> {
3034 let repo = repo_for(task, &opts.repo);
3035 tracing::info!(
3036 "task {} — {} (repo {})",
3037 task.short(),
3038 task.title,
3039 repo.display()
3040 );
3041
3042 if let Some(previous) = task.runs.last().and_then(|id| RunState::load(id).ok())
3046 && previous
3047 .merge_decision
3048 .as_ref()
3049 .is_some_and(|decision| !decision.closed)
3050 && previous.status != RunStatus::Landing
3051 && !crate::merge_decision::resumable_hold(&previous)
3052 {
3053 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3054 record(queue, task);
3055 return Vec::new();
3056 }
3057 let stop_owner = task
3058 .runs
3059 .last()
3060 .and_then(|id| RunState::load(id).ok())
3061 .filter(|state| {
3062 crate::merge_decision::resumable_hold(state)
3063 || state.status == RunStatus::Landing
3064 && state.land_approval.as_ref().is_some_and(|binding| {
3065 Questions::open().get(&binding.question).is_ok_and(|q| {
3066 q.run == state.id
3067 && q.resolution()
3068 .as_deref()
3069 .is_some_and(crate::merge_decision::stops)
3070 })
3071 })
3072 });
3073 if let Some(owner) = stop_owner {
3074 let mut runner = match Runner::resume(&owner.id) {
3075 Ok(runner) => runner,
3076 Err(e) => {
3077 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
3078 record(queue, task);
3079 return Vec::new();
3080 }
3081 };
3082 runner.on_pause(stop.pause());
3083 runner.watch_interrupt(interrupt_pause);
3084 let quota_before = runner.state.quota.clone();
3085 task.start(owner.id.clone());
3086 record(queue, task);
3087 lock(status).current.push(Current {
3088 task: task.id.clone(),
3089 run: owner.id,
3090 });
3091 runner.state.next_pass_task_entry = true;
3092 let result = runner.execute().await;
3093 return finish_attempt(
3094 opts.max_attempts,
3095 queue,
3096 task,
3097 &runner.state,
3098 "a_before,
3099 result,
3100 );
3101 }
3102
3103 let explicit = match &task.overrides {
3107 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
3108 Some(_) => opts.config.as_deref(),
3109 None => opts.config.as_deref(),
3110 };
3111 if let Err(e) = Config::discover_fetched(&repo, explicit).await {
3112 let reason = format!("[config] {e:#}");
3113 record_config_failure(task, &e);
3114 task.last_error = Some(reason.clone());
3115 task.hold_machine(Some(reason.clone()));
3116 record(queue, task);
3117 tracing::warn!("holding {}: {reason}", task.short());
3118 return Vec::new();
3119 }
3120 let mut config = match prepare_for(&repo, opts, task) {
3121 Ok(c) => c,
3122 Err(e) => {
3123 task.attempts += 1;
3127 task.fail_startup(
3128 "startup-config",
3129 format!("config: {e:#}"),
3130 opts.max_attempts,
3131 );
3132 record(queue, task);
3133 return Vec::new();
3134 }
3135 };
3136 apply_task_implementers(&mut config, task);
3137 if let Some(replacement) = &task.replacement {
3138 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
3139 old.liveness(false) == Liveness::Dead
3140 && old.merge_decision.as_ref().is_some_and(|d| {
3141 d.closed
3142 && d.question == replacement.question
3143 && d.replacement_run.as_ref().is_none_or(|run| {
3144 task.runs.last() == Some(run)
3145 && RunState::load(run).is_ok_and(|current| {
3146 current.replaces.as_ref().is_some_and(|pin| {
3147 pin.question == replacement.question
3148 && pin.agent == replacement.agent
3149 })
3150 })
3151 })
3152 })
3153 });
3154 if !safe {
3155 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
3156 record(queue, task);
3157 return Vec::new();
3158 }
3159 match config.agent(&replacement.agent) {
3160 Ok(agent) if crate::agent::installed(agent) => {}
3161 _ => {
3162 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
3163 record(queue, task);
3164 return Vec::new();
3165 }
3166 }
3167 config.graph.implementers = 1;
3168 config.graph.land_approval = true;
3169 config.merge.mode = MergeMode::Pr;
3170 config.graph.selected_implementer = Some(replacement.agent.clone());
3171 }
3172 let p = phrases(&config.graph.language);
3173 let start_failed = p.could_not_start;
3174
3175 if let Some(reason) = disk_gate(&repo, &config) {
3183 task.notice_cause("disk-space", None, None);
3184 task.last_error = Some(reason.clone());
3185 task.hold_machine(Some(reason.clone()));
3186 record(queue, task);
3187 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
3188 notices::raise(
3191 Notice::warn(
3192 &format!("disk:{}", repo.display()),
3193 "A task was held for want of free disk space; free some, then release it from the queue.",
3194 )
3195 .link(Link::Task {
3196 id: task.id.clone(),
3197 }),
3198 );
3199 return Vec::new();
3200 }
3201
3202 let unfinished = (!task.fresh_start)
3222 .then(|| unfinished_run(&task.runs, task.short()))
3223 .flatten()
3224 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
3225 let previous = task.runs.last().and_then(|id| RunState::load(id).ok());
3231 let review_branch = take_review_branch(task, previous.as_ref());
3232 let branch_exists = match &review_branch {
3233 Some(branch) => crate::git::branch_exists(&repo, branch)
3234 .await
3235 .unwrap_or(false),
3236 None => false,
3237 };
3238 let proposed = match (&review_branch, &unfinished, &task.review_of) {
3242 (None, None, Some(branch)) => Starter::Review(branch.clone()),
3243 _ => choose_starter(
3244 review_branch.as_deref(),
3245 branch_exists,
3246 unfinished.as_deref(),
3247 ),
3248 };
3249 let retry_selected = task.replacement.is_some()
3250 && task.resumption.is_some()
3251 && !task.fresh_start
3252 && previous.as_ref().is_some_and(|run| {
3253 run.replaces.is_some()
3254 && ((run.status == RunStatus::Failed && run.viable().is_empty())
3255 || (run.status == RunStatus::Blocked && run.tally.is_none()))
3256 });
3257 let stop_pin = task
3258 .resume_override
3259 .as_ref()
3260 .filter(|pin| {
3261 Questions::open().get(&pin.question_id).is_ok_and(|q| {
3262 q.node == land::APPROVAL_NODE
3263 && q.resolution()
3264 .as_deref()
3265 .is_some_and(crate::merge_decision::stops)
3266 })
3267 })
3268 .and_then(|pin| pin.pinned_run.clone());
3269 let starter = if let Some(run) = stop_pin {
3270 Starter::Resume(run)
3273 } else if retry_selected {
3274 Starter::Resume(previous.as_ref().unwrap().id.clone())
3275 } else if let Some(selected) = &task.pr_recovery {
3276 if selected.check_task(&task.id, &task.instruction).is_ok() {
3277 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
3278 } else {
3279 Starter::Review(selected.branch.clone())
3280 }
3281 } else {
3282 starter_with_implementers(task, previous.as_ref(), proposed)
3283 };
3284 if let Some(pin) = &task.resume_override
3287 && let Some(run) = &pin.pinned_run
3288 && Questions::open().get(&pin.question_id).map_or(true, |q| {
3289 q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR)
3290 })
3291 && starter != Starter::Resume(run.clone())
3292 {
3293 let run = run.clone();
3294 task.notice_cause("repair-owner", Some(&run), None);
3295 task.hold_machine(Some(format!(
3296 "repair owner cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
3297 task.id
3298 )));
3299 record(queue, task);
3300 return Vec::new();
3301 }
3302 if task.replacement.is_some()
3303 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
3304 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
3305 {
3306 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
3307 record(queue, task);
3308 return Vec::new();
3309 }
3310 let attachments = match task_attachments(queue, task) {
3315 Ok(a) => a,
3316 Err(e) => {
3317 task.attempts += 1;
3318 task.fail_startup_target(
3319 "startup-attachments",
3320 e.name.clone(),
3321 format!("{start_failed}{e:#}"),
3322 opts.max_attempts,
3323 );
3324 record(queue, task);
3325 return Vec::new();
3326 }
3327 };
3328 let started = match &starter {
3329 Starter::Review(_) if task.pr_recovery.is_some() => {
3330 let selected = task.pr_recovery.clone().expect("selected recovery");
3331 match selected.check_task(&task.id, &task.instruction) {
3332 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
3333 .await
3334 .map(|mut r| {
3335 r.state.attachments = attachments.clone();
3336 r
3337 }),
3338 Err(e) => Err(e),
3339 }
3340 }
3341 Starter::Review(branch) => {
3342 tracing::info!(
3343 "task {} reopens `{branch}` as a review-only pass",
3344 task.short()
3345 );
3346 let takeover = crate::handover::Takeover {
3349 earlier: task.earlier_attempts().to_vec(),
3350 home: crate::run::home(),
3351 choice: take_divergence_answer(branch, &config.merge.remote, task),
3352 task_instruction: Some(instruction_for(task)),
3353 };
3354 Runner::review_taking_over(
3355 &repo,
3356 branch,
3357 config,
3358 Some(takeover),
3359 crate::run::Origin::queue(&task.id),
3360 )
3361 .await
3362 }
3363 Starter::Resume(id) => {
3364 tracing::info!("resuming run {id} rather than competing again");
3365 Runner::resume(id).and_then(|mut r| {
3366 if retry_selected {
3367 for candidate in &mut r.state.candidates {
3370 candidate.failed = None;
3371 candidate.empty = false;
3372 candidate.commits = 0;
3373 }
3374 r.state.status = RunStatus::Implementing;
3375 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
3376 }
3377 if r.state.prep_failure.is_some() {
3378 let approval = task
3379 .resumption
3380 .as_ref()
3381 .context("preparation retry requires an explicit task release")?;
3382 r.retry_preparation(approval.at)?;
3383 }
3384 if let Some(instruction) =
3385 prepare_instruction(&starter, Some(&r.state.instruction), task)
3386 {
3387 r.state.instruction = instruction;
3388 }
3389 r.state.attachments = attachments.clone();
3390 if task.replacement.is_some() {
3391 r.state.config.merge.mode = MergeMode::Pr;
3392 r.state.config.graph.land_approval = true;
3393 }
3394 if task.replacement.is_none() && let Some(mode) = task
3397 .overrides
3398 .as_ref()
3399 .and_then(|o| o.merge.as_deref())
3400 .and_then(|m| merge_mode(m).ok())
3401 {
3402 r.state.config.merge.mode = mode;
3403 }
3404 Ok(r)
3405 })
3406 }
3407 Starter::Start => {
3408 if let Some(branch) = &review_branch {
3409 tracing::warn!(
3410 "conductor chose review for task {} but branch `{branch}` no longer \
3411 exists; requeuing as a fresh competition instead",
3412 task.short()
3413 );
3414 }
3415 let instruction = prepare_instruction(&starter, None, task)
3416 .unwrap_or_else(|| task.instruction.clone());
3417 Runner::start_naming(
3418 &repo,
3419 instruction,
3420 &task.title,
3421 config,
3422 crate::run::Origin::queue(&task.id),
3423 )
3424 .await
3425 .map(|mut r| {
3426 r.state.attachments = attachments.clone();
3427 r
3428 })
3429 }
3430 };
3431 let mut runner = match started {
3432 Ok(r) => r,
3433 Err(e) if task.pr_recovery.is_some() => {
3437 task.startup_notice_cause("startup-public-head");
3438 task.attempts += 1;
3439 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
3440 task.link_run(&partial.run);
3441 task.record_pass(&partial.run, 0, task.attempts, None);
3442 }
3443 task.last_error = Some(format!("{start_failed}{e:#}"));
3444 task.hold_unrequested_retry(Some(format!(
3445 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
3446 )));
3447 record(queue, task);
3448 return Vec::new();
3449 }
3450 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
3451 let detail = match e.downcast_ref::<crate::handover::Refused>() {
3452 Some(r) => (p.handover_refused)(r),
3453 None => format!("{e:#}"),
3454 };
3455 task.startup_notice_cause("startup-handover");
3456 let reason = format!("{start_failed}{detail}");
3457 task.last_error = Some(reason.clone());
3458 let branch = match &starter {
3461 Starter::Review(branch) => Some(branch.clone()),
3462 _ => None,
3463 };
3464 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
3468 record(queue, task);
3469 tracing::warn!(
3470 "holding {} for a branch it cannot take over: {e:#}",
3471 task.short()
3472 );
3473 return Vec::new();
3474 }
3475 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
3480 let d = e
3481 .downcast_ref::<crate::reconcile::Diverged>()
3482 .expect("checked by the guard");
3483 let mut q = ask::Question::new(
3484 task.id.clone(),
3485 crate::reconcile::NODE.to_owned(),
3486 crate::reconcile::SEAT.to_owned(),
3487 d.summary(),
3488 d.detail(),
3489 d.choices(),
3490 );
3491 match Questions::open().put(&mut q) {
3492 Ok(()) => {
3493 task.last_error = Some(format!("{e:#}"));
3494 task.review_branch = Some(d.branch.clone());
3497 task.review_instruction = Some(task.instruction.clone());
3498 task.block(vec![q.id.clone()], Some(d.summary()));
3499 }
3500 Err(put) => {
3501 tracing::warn!("could not file the divergence question: {put:#}");
3502 task.attempts += 1;
3503 task.fail_startup(
3504 "startup-question-store",
3505 format!("{start_failed}{e:#}"),
3506 opts.max_attempts,
3507 );
3508 }
3509 }
3510 record(queue, task);
3511 return Vec::new();
3512 }
3513 Err(e) => {
3514 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
3520 && let Starter::Review(branch) = &starter
3521 {
3522 task.review_branch = Some(branch.clone());
3523 task.review_instruction = Some(task.instruction.clone());
3524 task.last_error = Some(format!("{start_failed}{e:#}"));
3525 task.status = crate::queue::TaskStatus::Failed;
3526 record(queue, task);
3527 return Vec::new();
3528 }
3529 task.attempts += 1;
3530 task.last_error = Some(format!("{start_failed}{e:#}"));
3531 task.startup_notice_cause("startup-source");
3532 task.hold_unrequested_retry(Some(format!(
3533 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
3534 )));
3535 record(queue, task);
3536 return Vec::new();
3537 }
3538 };
3539 if let Some(replacement) = &task.replacement {
3540 if matches!(starter, Starter::Start) {
3541 runner.state.instruction.push_str(&format!(
3542 "\n\n# Replacement review context\n{}",
3543 replacement.context
3544 ));
3545 }
3546 runner.state.replaces = Some(replacement.clone());
3547 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
3548 if let Ok(mut old) = RunState::load(&replacement.run)
3549 && let Some(d) = &mut old.merge_decision
3550 {
3551 d.replacement_run = Some(runner.state.id.clone());
3552 old.event(
3553 "replacement",
3554 format!(
3555 "replacement run {} owns the next attempt; old PR remains closed",
3556 runner.state.id
3557 ),
3558 );
3559 if let Err(e) = old.save() {
3560 task.hold_manual(Some(format!(
3561 "Could not link replacement history: {e}; no agents dispatched"
3562 )));
3563 record(queue, task);
3564 return Vec::new();
3565 }
3566 }
3567 }
3568 runner.state.task_instruction = Some(task.instruction.clone());
3569 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
3572 if runner.state.origin_chat.is_none() {
3573 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
3574 }
3575 runner.on_pause(stop.pause());
3577 runner.watch_interrupt(interrupt_pause);
3581
3582 let run = runner.state.id.clone();
3585 task.start(run.clone());
3586 record(queue, task);
3587 lock(status).current.push(Current {
3588 task: task.id.clone(),
3589 run,
3590 });
3591
3592 let quota_before = runner.state.quota.clone();
3595 runner.state.next_pass_task_entry = true;
3596 let result = runner.execute().await;
3597 finish_attempt(
3598 opts.max_attempts,
3599 queue,
3600 task,
3601 &runner.state,
3602 "a_before,
3603 result,
3604 )
3605}
3606
3607pub fn finish_attempt(
3612 max_attempts: usize,
3613 queue: &Queue,
3614 task: &mut Task,
3615 state: &RunState,
3616 quota_before: &[QuotaLoss],
3617 result: Result<()>,
3618) -> Vec<QuotaLoss> {
3619 let detail = match result {
3620 Ok(()) => describe(state),
3621 Err(e) => format!("{e:#}"),
3622 };
3623 let fresh = losses_this_attempt(quota_before, &state.quota);
3624 let verdict = Verdict {
3625 status: state.status,
3626 left_pr: state.pr.is_some(),
3629 quota_hit: !fresh.is_empty(),
3635 parked: state.parked,
3639 no_viable_candidates: state.viable().is_empty(),
3642 };
3643 if let Some(replacement) = &state.replaces
3644 && let Some(pr) = &state.pr
3645 && let Ok(mut old) = RunState::load(&replacement.run)
3646 && let Some(d) = &mut old.merge_decision
3647 {
3648 d.replacement_pr = Some(pr.url.clone());
3649 let _ = old.save();
3650 }
3651 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
3652 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
3653 task.hold_unrequested_retry(Some(format!(
3654 "Candidate setup failed: {detail}. Select a complete implementation with `magi task recover-pr`, or correct the referenced source before explicitly releasing the task."
3655 )));
3656 }
3657 if task.status == TaskStatus::Done {
3658 supersede_prior_runs(task, &crate::run::home());
3659 }
3660 record(queue, task);
3661 tracing::info!(
3662 "task {} is {} after run {} ({})",
3663 task.short(),
3664 task.status.as_str(),
3665 state.short(),
3666 label(state.status)
3667 );
3668 fresh
3669}
3670
3671pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
3675 let waits_on_owner = withheld_text_wait(task).is_some();
3680 if task.status.runnable()
3681 && !waits_on_owner
3682 && !(task.fresh_start && task.replacement.is_some())
3683 {
3684 let why = task.last_error.clone().map_or_else(
3685 || "the run did not finish".to_owned(),
3686 |e| format!("the run did not finish: {e}"),
3687 );
3688 task.hold_unrequested_retry(Some(format!(
3689 "{why}. It was started by hand, so it is not retried \
3690 automatically; `magi task release` retries it."
3691 )));
3692 record(queue, task);
3693 }
3694}
3695
3696#[must_use]
3700pub fn loop_would_resume(run: &str) -> bool {
3701 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
3702}
3703
3704#[must_use]
3714pub fn foreign_loop(
3715 reading: Option<&Reading>,
3716 now: Timestamp,
3717 own_pid: u32,
3718) -> Option<Option<u32>> {
3719 let reading = reading.filter(|r| r.running(now))?;
3720 match reading.pid {
3721 Some(pid) if pid == own_pid => None,
3722 pid => Some(pid),
3723 }
3724}
3725
3726pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
3741 let status = Arc::new(Mutex::new(Status::new()));
3742 let stop = Stop::new();
3743 attempt(
3744 opts,
3745 queue,
3746 &status,
3747 &stop,
3748 crate::graph::Pause::new(),
3749 task,
3750 )
3751 .await;
3752 let mut resumes = 0;
3761 while resumes < MAX_WITHHELD_RESUMES {
3762 let Some(timeout) = withheld_text_wait(task) else {
3763 break;
3764 };
3765 eprintln!(
3766 "waiting for the owner's answer on a withheld pull request text \
3767 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
3768 );
3769 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
3772 let ready = loop {
3773 match land_resume_state(task) {
3774 LandResume::StillWaiting if std::time::Instant::now() < limit => {
3775 tokio::time::sleep(WITHHELD_POLL).await;
3776 }
3777 LandResume::Ready => break true,
3778 _ => break false,
3779 }
3780 };
3781 if !ready {
3782 break;
3783 }
3784 resumes += 1;
3785 attempt(
3786 opts,
3787 queue,
3788 &status,
3789 &stop,
3790 crate::graph::Pause::new(),
3791 task,
3792 )
3793 .await;
3794 }
3795 hold_if_runnable(queue, task);
3796}
3797
3798const MAX_WITHHELD_RESUMES: u32 = 3;
3801
3802const WITHHELD_POLL: Duration = Duration::from_secs(15);
3804
3805fn withheld_text_wait(task: &Task) -> Option<u64> {
3808 let id = task.runs.last()?;
3809 let s = RunState::load(id).ok()?;
3810 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
3811 .then_some(s.config.graph.answer_timeout)
3812}
3813
3814fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
3824 after
3825 .iter()
3826 .filter(|q| !before.contains(q))
3827 .cloned()
3828 .collect()
3829}
3830
3831fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
3834 if quota.is_empty() {
3835 return None;
3836 }
3837 let with_hint = quota.iter().find(|q| q.reset.is_some());
3838 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
3839 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
3840 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
3841 Some(
3842 now.checked_add(jiff::SignedDuration::from_secs(secs))
3843 .unwrap_or(Timestamp::MAX),
3844 )
3845}
3846
3847fn implementers_match(task: &Task, state: &RunState) -> bool {
3849 task.replacement.as_ref().map_or_else(
3850 || {
3851 task.implementers()
3852 .is_none_or(|n| n == state.config.graph.implementers)
3853 },
3854 |replacement| {
3855 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
3856 && state.config.graph.implementers == 1
3857 },
3858 )
3859}
3860
3861fn starter_with_implementers(
3863 task: &Task,
3864 previous: Option<&RunState>,
3865 proposed: Starter,
3866) -> Starter {
3867 if previous.is_some_and(|s| !implementers_match(task, s)) {
3868 task.review_of
3869 .as_ref()
3870 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
3871 } else {
3872 proposed
3873 }
3874}
3875
3876fn apply_task_implementers(config: &mut Config, task: &Task) {
3878 if let Some(n) = task.implementers() {
3879 config.graph.implementers = n;
3880 }
3881}
3882
3883fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
3888 let Some(o) = &task.overrides else {
3889 return prepare(repo, opts);
3890 };
3891 let own = Opts {
3895 config: o.config.clone(),
3896 merge: None,
3897 ..opts.clone()
3898 };
3899 let mut config = prepare(repo, if o.inherit_daemon_options { opts } else { &own })?;
3900 o.apply(&mut config);
3901 Ok(config)
3902}
3903
3904fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
3906 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
3907 if let Some(mode) = &opts.merge {
3908 config.merge.mode = merge_mode(mode)?;
3909 }
3910 Ok(config)
3911}
3912
3913async fn maybe_prune_cache_between_runs(
3944 repo: &Path,
3945 opts: &Opts,
3946 home: &Path,
3947 stop: &Stop,
3948 last_checked: &mut Option<Timestamp>,
3949 now: Timestamp,
3950) {
3951 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
3952 return;
3953 }
3954 *last_checked = Some(now);
3955 let cfg = match prepare(repo, opts) {
3956 Ok(cfg) => cfg,
3957 Err(e) => {
3958 tracing::warn!("cache check: no config: {e:#}");
3959 return;
3960 }
3961 };
3962 match clean::prune_cache_if_over_limit(&cfg, home) {
3963 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
3964 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
3965 pruned.files,
3966 pruned.freed
3967 ),
3968 Ok(_) => {}
3969 Err(e) => {
3970 tracing::warn!("housekeep: prune cache: {e:#}");
3971 notices::raise_in(
3972 home,
3973 Notice::warn(
3974 "housekeep:cache",
3975 "Pruning the shared build cache failed; disk usage may keep growing.",
3976 ),
3977 );
3978 }
3979 }
3980}
3981
3982fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
3986 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
3987}
3988
3989async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
4008 let cfg = match prepare(repo, opts) {
4009 Ok(cfg) => cfg,
4010 Err(e) => {
4011 tracing::warn!("housekeep: no config: {e:#}");
4012 return;
4013 }
4014 };
4015 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
4024 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
4025 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
4030 let mut extra = Vec::new();
4031 if out.unreadable > 0 {
4032 extra.push(format!("{} unreadable", out.unreadable));
4033 }
4034 if out.orphaned_worktrees > 0 {
4035 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
4036 }
4037 let detail = if extra.is_empty() {
4038 String::new()
4039 } else {
4040 format!(" ({})", extra.join(", "))
4041 };
4042 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
4043 }
4044 if out.external_merges_recorded > 0 {
4045 tracing::info!(
4046 "housekeep: recorded {} run(s) as merged externally",
4047 out.external_merges_recorded
4048 );
4049 }
4050 if out.stale_pr_states_repaired > 0 {
4051 tracing::info!(
4052 "housekeep: rewrote {} run record(s) whose pull request had already settled",
4053 out.stale_pr_states_repaired
4054 );
4055 }
4056 if out.cache_files > 0 {
4057 tracing::info!(
4058 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
4059 out.cache_files,
4060 out.cache_freed
4061 );
4062 }
4063 if out.questions_abandoned > 0 {
4064 tracing::info!(
4065 "housekeep: abandoned {} question(s) left open by a finished run",
4066 out.questions_abandoned
4067 );
4068 }
4069}
4070
4071async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
4080 let questions = Questions::at(home.join("questions"));
4081 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
4082 if report.is_empty() {
4083 return;
4084 }
4085 if !report.quarantined.is_empty() {
4086 tracing::info!(
4087 "triage: held {} blocked task(s) whose blocked-on task or \
4088 question no longer exists: {}",
4089 report.quarantined.len(),
4090 report.quarantined.join(", ")
4091 );
4092 }
4093 if !report.resumed.is_empty() {
4094 tracing::info!(
4095 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
4096 report.resumed.len(),
4097 report.resumed.join(", ")
4098 );
4099 }
4100 if !report.asked.is_empty() {
4101 tracing::info!(
4102 "triage: asked about {} held task(s): {}",
4103 report.asked.len(),
4104 report.asked.join(", ")
4105 );
4106 }
4107 if !report.answered.is_empty() {
4108 tracing::info!(
4109 "triage: applied {} operator answer(s): {}",
4110 report.answered.len(),
4111 report.answered.join(", ")
4112 );
4113 }
4114}
4115
4116fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
4123 disk_gate_with(repo, config, crate::disk::free_bytes)
4124}
4125
4126fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
4130 repo: &Path,
4131 config: &Config,
4132 free_bytes: F,
4133) -> Option<String> {
4134 let min = config.disk.min_free_bytes;
4135 if min == 0 {
4136 return None;
4137 }
4138 match free_bytes(repo) {
4139 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
4140 Err(e) => Some(crate::disk::unmeasured_in(repo, &e, &config.graph.language)),
4141 }
4142}
4143
4144const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
4151
4152const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
4156
4157fn quota_wait(
4166 reset_at: Option<Timestamp>,
4167 now: Timestamp,
4168 fallback: Duration,
4169 cap: Duration,
4170) -> Duration {
4171 match reset_at {
4172 Some(at) if at > now => {
4173 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
4174 Duration::from_secs(secs).min(cap)
4175 }
4176 _ => fallback,
4177 }
4178}
4179
4180fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
4194 parse_reset_hint_zoned(text, now)
4195 .or_else(|| parse_reset_hint_dated(text))
4196 .or_else(|| parse_reset_hint_relative(text, recorded))
4197}
4198
4199fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
4203 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
4204 let mut rest = rest.trim();
4205 if rest.is_empty() {
4206 return None;
4207 }
4208 let mut total: i64 = 0;
4209 let mut matched = false;
4210 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
4211 if let Some((digits, tail)) = rest.split_once(unit)
4212 && !digits.is_empty()
4213 && digits.bytes().all(|b| b.is_ascii_digit())
4214 {
4215 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
4216 rest = tail;
4217 matched = true;
4218 }
4219 }
4220 if !rest.is_empty() || !matched {
4221 return None;
4222 }
4223 recorded
4224 .checked_add(jiff::SignedDuration::from_secs(total))
4225 .ok()
4226}
4227
4228fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
4232 let clock = clock.trim().to_lowercase();
4233 let (digits, pm) = clock
4234 .strip_suffix("am")
4235 .map(|d| (d, false))
4236 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
4237 let (h, m) = digits.trim().split_once(':')?;
4238 let mut hour: i8 = h.trim().parse().ok()?;
4239 let minute: i8 = m.trim().parse().ok()?;
4240 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
4241 return None;
4242 }
4243 if pm && hour != 12 {
4244 hour += 12;
4245 } else if !pm && hour == 12 {
4246 hour = 0;
4247 }
4248 Some((hour, minute))
4249}
4250
4251fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
4256 let open = text.find('(')?;
4257 let close = text.rfind(')')?;
4258 if close <= open {
4259 return None;
4260 }
4261 let zone = text[open + 1..close].trim();
4262 let (hour, minute) = parse_12h_clock(&text[..open])?;
4263 let tz = jiff::tz::TimeZone::get(zone).ok()?;
4264 let candidate = now
4265 .to_zoned(tz)
4266 .with()
4267 .hour(hour)
4268 .minute(minute)
4269 .second(0)
4270 .millisecond(0)
4271 .microsecond(0)
4272 .nanosecond(0)
4273 .build()
4274 .ok()?;
4275 let mut at = candidate.timestamp();
4276 if at <= now {
4277 at += jiff::SignedDuration::from_hours(24);
4278 }
4279 Some(at)
4280}
4281
4282fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
4291 let words: Vec<&str> = text.split_whitespace().collect();
4292 if words.len() < 5 {
4293 return None;
4294 }
4295 (0..=words.len() - 5)
4296 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
4297}
4298
4299fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
4305 if trailing.is_some_and(|next| next.starts_with('(')) {
4306 return None;
4307 }
4308 let month = month_number(window[0])?;
4309 let day_token = window[1].strip_suffix(',')?.to_lowercase();
4310 let day_digits = ["st", "nd", "rd", "th"]
4311 .iter()
4312 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
4313 let day: i8 = day_digits.parse().ok()?;
4314 let year_token = window[2];
4315 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
4316 return None;
4317 }
4318 let year: i16 = year_token.parse().ok()?;
4319 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
4323 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
4324 let date = jiff::civil::Date::new(year, month, day).ok()?;
4325 let candidate = date
4326 .at(hour, minute, 0, 0)
4327 .to_zoned(jiff::tz::TimeZone::UTC)
4328 .ok()?;
4329 Some(candidate.timestamp())
4330}
4331
4332fn month_number(name: &str) -> Option<i8> {
4335 const NAMES: [&str; 12] = [
4336 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
4337 ];
4338 let lower = name.to_lowercase();
4339 NAMES
4340 .iter()
4341 .position(|n| *n == lower.as_str())
4342 .map(|i| i as i8 + 1)
4343}
4344
4345fn exhausted_review_budget(state: &RunState) -> bool {
4357 state.status == RunStatus::Blocked
4360 && !state.pr_publication_pending
4361 && state.reviews.len() >= state.config.graph.review_rounds
4362}
4363
4364fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
4401 unfinished_run_with(runs, short, RunState::load)
4402}
4403
4404fn unfinished_run_with<F>(runs: &[String], short: &str, load: F) -> Option<String>
4407where
4408 F: FnOnce(&str) -> Result<RunState>,
4409{
4410 let id = runs.last()?;
4411 match load(id) {
4412 Ok(s)
4417 if s.status.resumable()
4418 && !s.released()
4419 && !exhausted_review_budget(&s)
4420 && s.liveness(false) != crate::run::Liveness::Live =>
4421 {
4422 Some(id.clone())
4423 }
4424 Ok(_) => None,
4425 Err(e) => {
4426 tracing::warn!("could not read run {id} for task {short}: {e:#}");
4427 None
4428 }
4429 }
4430}
4431
4432fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
4440where
4441 F: FnOnce(&str) -> Result<RunState>,
4442{
4443 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
4445 || task.fresh_start
4446 || task.review_branch.is_some()
4447 {
4448 return false;
4449 }
4450 let Some(id) = task.runs.last() else {
4451 return false;
4452 };
4453 load(id).is_ok_and(|s| {
4454 s.parked && s.status.resumable() && !s.released() && !exhausted_review_budget(&s)
4455 })
4456}
4457
4458#[derive(Debug, Clone, PartialEq, Eq)]
4461enum Starter {
4462 Review(String),
4465 Resume(String),
4467 Start,
4469}
4470
4471fn selected_starter(
4473 selected: &crate::pr_recovery::Selection,
4474 previous: Option<&RunState>,
4475 unfinished: Option<&str>,
4476) -> Starter {
4477 if let Some(id) = unfinished
4478 && previous.is_some_and(|r| {
4479 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
4480 })
4481 {
4482 Starter::Resume(id.to_owned())
4483 } else {
4484 Starter::Review(selected.branch.clone())
4485 }
4486}
4487
4488fn take_divergence_answer(
4492 branch: &str,
4493 remote: &str,
4494 task: &mut Task,
4495) -> Option<crate::reconcile::Choice> {
4496 let summary = crate::reconcile::summary_for(branch, remote);
4497 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
4498 (a.question == summary)
4499 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
4500 .flatten()
4501 .map(|c| (i, c))
4502 })?;
4503 task.answers.remove(idx);
4504 Some(choice)
4505}
4506
4507fn choose_starter(
4519 review_branch: Option<&str>,
4520 branch_exists: bool,
4521 unfinished: Option<&str>,
4522) -> Starter {
4523 match review_branch {
4524 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
4525 Some(_) => Starter::Start,
4526 None => match unfinished {
4527 Some(id) => Starter::Resume(id.to_owned()),
4528 None => Starter::Start,
4529 },
4530 }
4531}
4532
4533fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
4535 let branch = task.review_branch.take();
4536 let instruction = task.review_instruction.take();
4537 branch.filter(|branch| match &instruction {
4538 Some(authored) => *authored == task.instruction,
4539 None => previous.map_or_else(
4542 || task.review_of.as_ref() == Some(branch),
4543 |run| compatible_run_instruction(run, task),
4544 ),
4545 })
4546}
4547
4548fn may_resume_run(run: &RunState, task: &Task) -> bool {
4551 task.resume_override
4552 .as_ref()
4553 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
4554 || compatible_run_instruction(run, task)
4555}
4556
4557pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
4559 if let Some(authored) = &run.task_instruction {
4560 return compatible_instruction(authored, task);
4561 }
4562 if compatible_instruction(&run.instruction, task) {
4563 return true;
4564 }
4565 if run.reviewed_commits.is_some() {
4568 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
4569 return strip_answers_block(authored, task).trim() == task.instruction.trim();
4570 }
4571 if let Some(branch) = &task.review_of {
4572 let canonical = format!(
4573 "Review the branch `{branch}` (review-only: no implementation, no judging)."
4574 );
4575 let generated = format!(
4576 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
4577 );
4578 return task.instruction == canonical && run.instruction.starts_with(&generated);
4579 }
4580 }
4581 false
4582}
4583
4584pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
4586 strip_answers_block(old, task) == task.instruction
4587}
4588
4589fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
4592 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
4593 return fallback.to_path_buf();
4594 }
4595 task.repo.clone()
4596}
4597
4598const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
4602
4603fn answers_block(task: &Task, count: usize) -> String {
4605 let mut s = ANSWERS_HEADER.to_owned();
4606 for a in &task.answers[..count] {
4607 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
4608 }
4609 s
4610}
4611
4612fn resumption_block(r: &crate::queue::Resumption) -> String {
4615 format!(
4616 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
4617 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
4618 r.generation, r.at, r.instruction, r.answers_through,
4619 )
4620}
4621
4622fn append_answers(base: &str, task: &Task) -> String {
4624 let mut s = base.to_owned();
4625 if !task.answers.is_empty() {
4626 s.push_str(&answers_block(task, task.answers.len()));
4627 }
4628 if let Some(r) = &task.resumption {
4629 s.push_str(&resumption_block(r));
4630 }
4631 s
4632}
4633
4634fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
4638 let mut instruction = instruction;
4639 for approval in task
4640 .resumption
4641 .iter()
4642 .chain(task.resumption_history.iter().rev())
4643 {
4644 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
4645 instruction = base;
4646 break;
4647 }
4648 }
4649 for count in (1..=task.answers.len()).rev() {
4650 let block = answers_block(task, count);
4651 if let Some(base) = instruction.strip_suffix(&block) {
4652 return base;
4653 }
4654 }
4655 instruction
4656}
4657
4658fn instruction_for(task: &Task) -> String {
4666 append_answers(&task.instruction, task)
4667}
4668
4669fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
4681 append_answers(strip_answers_block(old_instruction, task), task)
4682}
4683
4684fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
4686 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
4687 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
4688 if let Some(cause) = task.hold_notice_cause.as_mut() {
4689 cause.target = failure.map(|f| f.target.clone());
4690 }
4691}
4692
4693#[derive(Debug, thiserror::Error)]
4694#[error("attachment `{name}` is recorded on the task but {path} is missing")]
4695struct MissingAttachment {
4696 name: String,
4697 path: PathBuf,
4698}
4699
4700fn task_attachments(
4702 queue: &Queue,
4703 task: &Task,
4704) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
4705 let paths = queue.attachment_paths(task);
4706 for (name, path) in task.attachments.iter().zip(&paths) {
4707 if !path.is_file() {
4708 return Err(MissingAttachment {
4709 name: name.clone(),
4710 path: path.clone(),
4711 });
4712 }
4713 }
4714 Ok(paths)
4715}
4716
4717fn prepare_instruction(
4728 starter: &Starter,
4729 old_instruction: Option<&str>,
4730 task: &Task,
4731) -> Option<String> {
4732 match starter {
4733 Starter::Start => Some(instruction_for(task)),
4734 Starter::Resume(_) => Some(resumed_instruction(
4735 old_instruction.expect("a resumed run always has a prior instruction"),
4736 task,
4737 )),
4738 Starter::Review(_) => None,
4739 }
4740}
4741
4742fn record(queue: &Queue, task: &mut Task) {
4746 if let Err(e) = queue.put(task) {
4747 tracing::error!("could not record task {}: {e:#}", task.short());
4748 let notice = Notice::error(
4749 "loop:record",
4750 "The loop could not save a task's state; check the disk.",
4751 )
4752 .link(Link::Task {
4753 id: task.id.clone(),
4754 });
4755 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
4757 Some(home) => notices::raise_in(home, notice),
4758 None => notices::raise(notice),
4759 }
4760 }
4761}
4762
4763fn runnable(queue: &Queue) -> Vec<Task> {
4769 let mut tasks: Vec<Task> = queue
4770 .list()
4771 .into_iter()
4772 .filter(|t| t.status.runnable())
4773 .collect();
4774 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)));
4775 tasks
4776}
4777
4778fn describe(state: &RunState) -> String {
4792 let p = phrases(&state.config.graph.language);
4793 let mut detail = if state.status == RunStatus::Stalled {
4794 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
4795 seats.sort_unstable();
4796 seats.dedup();
4797 if seats.is_empty() {
4798 p.quorum_lost.to_owned()
4799 } else {
4800 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
4801 }
4802 } else {
4803 format!("{}{}", p.run_ended, state.status.display_label())
4804 };
4805 if let Some(last) = state.events.last() {
4806 let unanswered = state
4810 .reviews
4811 .last()
4812 .filter(|r| {
4813 state.status == RunStatus::Blocked
4814 && last.node == "review"
4815 && r.incomplete()
4816 && r.blocking == 0
4817 && r.round == state.config.graph.review_rounds
4818 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
4819 })
4820 .map(|r| (r.expected - r.answered, r.round));
4821 match unanswered {
4822 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
4823 detail.push_str(&format!(
4824 " ({}: {})",
4825 last.node,
4826 (p.reviewers_never_answered)(missing, rounds)
4827 ));
4828 }
4829 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
4830 }
4831 }
4832 detail.push_str(&format!(" [run {}]", state.id));
4833 detail
4834}
4835
4836const DIAGNOSTIC_MAX: usize = 4_000;
4842
4843const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
4848
4849fn diagnostic(state: &RunState) -> Option<String> {
4863 let mut parts: Vec<String> = Vec::new();
4864
4865 for o in state.gate.iter().filter(|o| !o.ok()) {
4867 parts.push(format!(
4868 "gate `{}` failed ({:?}):\n{}",
4869 o.command,
4870 o.code,
4871 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
4872 ));
4873 }
4874
4875 if let Some(last) = state
4878 .events
4879 .iter()
4880 .rev()
4881 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
4882 {
4883 parts.push(last.message.clone());
4884 }
4885
4886 if state.viable().is_empty() {
4893 for c in &state.candidates {
4894 if let Some(evidence) = &c.verified_noop {
4895 parts.push(format!(
4896 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
4897 c.label
4898 ));
4899 } else if !c.summary.trim().is_empty() {
4900 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
4901 } else if let Some(why) = &c.failed {
4902 parts.push(format!("candidate {}: {why}", c.label));
4903 }
4904 }
4905 }
4906
4907 if parts.is_empty() {
4908 return None;
4909 }
4910 Some(crate::run::tail(
4915 &parts.join("\n\n"),
4916 DIAGNOSTIC_MAX.saturating_sub(100),
4917 ))
4918}
4919
4920fn label(status: RunStatus) -> &'static str {
4928 status.as_str()
4929}
4930
4931pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
4933 match mode {
4934 "none" => Ok(MergeMode::None),
4935 "local" => Ok(MergeMode::Local),
4936 "pr" => Ok(MergeMode::Pr),
4937 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
4938 }
4939}
4940
4941fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
4946 mutex
4947 .lock()
4948 .unwrap_or_else(std::sync::PoisonError::into_inner)
4949}
4950
4951#[cfg(test)]
4952mod tests {
4953 use super::*;
4954 use crate::queue::{Source, TaskStatus};
4955 use crate::run::{Candidate, CommandOutcome};
4956 use pretty_assertions::assert_eq;
4957
4958 fn task() -> Task {
4959 Task::new(
4960 "add retries".to_owned(),
4961 "add retries".to_owned(),
4962 PathBuf::from("/repo"),
4963 Source::Human,
4964 )
4965 }
4966
4967 #[test]
4968 fn selected_public_head_overrides_historical_resume_and_pin() {
4969 let mut t = task();
4970 let mut old = run_state(RunStatus::Blocked);
4971 let selection = crate::pr_recovery::Selection {
4972 url: "url".into(),
4973 branch: format!("magi/task-{}", t.id),
4974 head: "a".repeat(40),
4975 instruction: t.instruction.clone(),
4976 approved_at: Timestamp::now(),
4977 };
4978 t.resume_override = Some(crate::queue::OperatorResume {
4979 pinned_run: Some(old.id.clone()),
4980 question_id: "choice".into(),
4981 at: Timestamp::now(),
4982 conductor_rehold: None,
4983 forced: false,
4984 });
4985 assert_eq!(
4986 selected_starter(&selection, Some(&old), Some(&old.id)),
4987 Starter::Review(selection.branch.clone())
4988 );
4989 old.pr_recovery = Some(selection.clone());
4990 assert_eq!(
4991 selected_starter(&selection, Some(&old), Some(&old.id)),
4992 Starter::Review(selection.branch.clone())
4993 );
4994 old.reviewed_commits = Some(vec!["implementation".into()]);
4995 assert_eq!(
4996 selected_starter(&selection, Some(&old), Some(&old.id)),
4997 Starter::Resume(old.id.clone())
4998 );
4999 let mut changed = selection.clone();
5000 changed.head = "b".repeat(40);
5001 assert_eq!(
5002 selected_starter(&changed, Some(&old), Some(&old.id)),
5003 Starter::Review(changed.branch)
5004 );
5005 }
5006
5007 #[test]
5008 fn candidate_transplant_failure_is_one_actionable_hold() {
5009 let dir = tempfile::tempdir().unwrap();
5010 let q = Queue::at(dir.path().join("queue"));
5011 let mut t = task();
5012 let mut state = run_state(RunStatus::Blocked);
5013 state.event("prep", "cannot apply referenced commit: conflict");
5014 state.prep_failure = Some("conflict".into());
5015 t.start(state.id.clone());
5016 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
5017 assert_eq!(t.attempts, 1);
5018 assert_eq!(t.status, TaskStatus::Held);
5019 assert!(t.operator_held());
5020 assert!(!t.status.runnable());
5021 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
5022 let first = t.chat_notice_key().unwrap();
5023 t.chat_notices.push(first.clone());
5024 t.release_operator("test", None);
5025 state.id = "another-unprepared-run".into();
5026 t.start(state.id.clone());
5027 finish_attempt(
5028 3,
5029 &q,
5030 &mut t,
5031 &state,
5032 &[],
5033 Err(anyhow::anyhow!("still conflicts")),
5034 );
5035 assert_eq!(
5036 t.chat_notice_key().as_ref(),
5037 Some(&first),
5038 "no work happened"
5039 );
5040 t.release_operator("test", None);
5041 state.prep_failure = None;
5042 state.status = RunStatus::Failed;
5043 state.begin_pass();
5044 t.start(state.id.clone());
5045 finish_attempt(
5046 1,
5047 &q,
5048 &mut t,
5049 &state,
5050 &[],
5051 Err(anyhow::anyhow!("new gate failure")),
5052 );
5053 let failure = t.chat_notice_key().unwrap();
5054 assert_ne!(failure, first, "approved recovery reached a real failure");
5055 t.release_operator("test", None);
5056 state.begin_pass();
5057 t.start(state.id.clone());
5058 finish_attempt(
5059 1,
5060 &q,
5061 &mut t,
5062 &state,
5063 &[],
5064 Err(anyhow::anyhow!("new failure on resume")),
5065 );
5066 assert_ne!(
5067 t.chat_notice_key().as_ref(),
5068 Some(&failure),
5069 "a new settled pass"
5070 );
5071 }
5072
5073 #[test]
5074 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
5075 for source in ["cli", "web"] {
5076 let mut t = task();
5077 let mut run = run_state(RunStatus::Blocked);
5078 run.instruction = "Run A original criteria".into();
5079 run.task_instruction = Some(run.instruction.clone());
5080 t.start(run.id.clone());
5081 t.release_operator("choice", Some("resume-question"));
5082 t.resume_override = Some(crate::queue::OperatorResume {
5083 question_id: "resume-question".into(),
5084 at: Timestamp::now(),
5085 conductor_rehold: None,
5086 forced: true,
5087 pinned_run: Some(run.id.clone()),
5088 });
5089 t.hold_machine(Some("not enough free space to start a run".into()));
5090 let mut unchanged = t.clone();
5091 unchanged
5092 .edit("title changed".into(), unchanged.instruction.clone())
5093 .unwrap();
5094 unchanged.release_operator(source, None);
5095 assert_eq!(
5096 unchanged
5097 .resume_override
5098 .as_ref()
5099 .unwrap()
5100 .pinned_run
5101 .as_deref(),
5102 Some(run.id.as_str())
5103 );
5104 assert!(!compatible_run_instruction(&run, &unchanged));
5105 assert!(may_resume_run(&run, &unchanged));
5106
5107 t.edit(
5108 "approved recovery".into(),
5109 "Reuse measured PR head B; fix Clippy; review".into(),
5110 )
5111 .unwrap();
5112 assert!(t.resume_override.is_none());
5113 t.release_operator(source, None);
5114 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
5115 assert!(reloaded.resume_override.is_none());
5116 assert!(!may_resume_run(&run, &reloaded));
5117 let unfinished =
5118 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
5119 .filter(|_| may_resume_run(&run, &reloaded));
5120 assert_eq!(
5121 choose_starter(None, false, unfinished.as_deref()),
5122 Starter::Start
5123 );
5124 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
5125 assert_eq!(
5126 reloaded.resumption_history[0].question_id.as_deref(),
5127 Some("resume-question")
5128 );
5129
5130 unchanged.instruction = "Reuse measured PR head B".into();
5133 unchanged.hold_machine(Some("disk hold".into()));
5134 unchanged.release_operator(source, None);
5135 assert!(unchanged.resume_override.is_none());
5136 assert!(!may_resume_run(&run, &unchanged));
5137 }
5138 }
5139
5140 #[test]
5141 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
5142 let mut t = task();
5143 let mut run = run_state(RunStatus::Blocked);
5144 t.review_of = Some("approved-head".into());
5145 t.instruction =
5146 "Review the branch `approved-head` (review-only: no implementation, no judging)."
5147 .into();
5148 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
5149 run.reviewed_commits = Some(vec!["Commit log".into()]);
5150 t.start(run.id.clone());
5151 t.hold_machine(Some("interrupted review".into()));
5152 t.release_operator("cli", None);
5153 assert!(
5154 compatible_run_instruction(&run, &t),
5155 "legacy explicit review remains resumable"
5156 );
5157 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
5158 .filter(|_| compatible_run_instruction(&run, &t));
5159 assert_eq!(
5160 choose_starter(None, false, unfinished.as_deref()),
5161 Starter::Resume(run.id.clone())
5162 );
5163 run.task_instruction = Some(t.instruction.clone());
5164 assert!(compatible_run_instruction(&run, &t));
5165 let reloaded: RunState =
5166 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
5167 assert!(compatible_run_instruction(&reloaded, &t));
5168 t.edit(
5169 "new approved head".into(),
5170 "Reuse measured PR head B".into(),
5171 )
5172 .unwrap();
5173 assert!(!compatible_run_instruction(&run, &t));
5174
5175 t.review_of = None;
5177 t.instruction = "Fix Clippy on approved head".into();
5178 run.task_instruction = None;
5179 run.instruction = format!(
5180 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
5181 t.instruction
5182 );
5183 assert!(compatible_run_instruction(&run, &t));
5184 t.instruction.push('\n');
5185 assert!(
5186 compatible_run_instruction(&run, &t),
5187 "legacy generated review trimmed the original criteria"
5188 );
5189 t.hold_for_handover(Some("approved-head".into()), "refused".into());
5190 let mut unchanged = t.clone();
5191 assert_eq!(
5192 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
5193 Some("approved-head")
5194 );
5195 t.edit("new plan".into(), "Reuse measured PR head B".into())
5196 .unwrap();
5197 assert!(take_review_branch(&mut t, Some(&run)).is_none());
5198 assert!(!compatible_run_instruction(&run, &t));
5199 t.review_branch = Some("approved-head".into());
5200 t.review_instruction = None;
5201 assert!(
5202 take_review_branch(&mut t, Some(&run)).is_none(),
5203 "legacy retained candidate cannot replace a revised approved head"
5204 );
5205 }
5206
5207 #[test]
5208 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
5209 crate::run::pin_test_home();
5210 let dir = tempfile::tempdir().unwrap();
5211 let queue = Queue::at(dir.path().join("queue"));
5212 for resume in [true, false] {
5213 let state = run_state(RunStatus::Blocked);
5214 let mut t = task();
5215 t.start(state.id.clone());
5216 let action = if resume {
5217 ask::ChoiceAction::Resume {
5218 run: state.id.clone(),
5219 }
5220 } else {
5221 ask::ChoiceAction::Requeue
5222 };
5223 let mut q = action_question(&state.id, action);
5224 q.status = ask::QuestionStatus::Open;
5225 q.answer = None;
5226 q.answered_at = None;
5227 t.fail("run stopped with a question", 3);
5228 hold_if_runnable(&queue, &mut t);
5229 assert!(t.operator_decided_at.is_none());
5230 assert!(t.operator_held());
5231 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
5232 let expected = if resume {
5233 ActionDecision::Resume(state.id.clone())
5234 } else {
5235 ActionDecision::Requeue
5236 };
5237 assert_eq!(
5238 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5239 expected
5240 );
5241 t.hold_manual(Some("later explicit hold".into()));
5242 assert_eq!(
5243 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5244 ActionDecision::Stale
5245 );
5246 }
5247 }
5248
5249 #[test]
5250 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
5251 let mut t = task();
5252 t.answers.push(crate::queue::AnsweredQuestion {
5253 question: "Inspect the remote?".into(),
5254 answer: "Inspect then wait".into(),
5255 });
5256 let old = instruction_for(&t);
5257 assert!(compatible_instruction(&old, &t));
5258 t.instruction = "Reuse approved PR head and review".into();
5259 t.release_operator("cli", None);
5260 assert!(!compatible_instruction(&old, &t));
5261 let approved = instruction_for(&t);
5262 assert!(approved.contains("waiting premises are superseded"));
5263 assert!(compatible_instruction(&approved, &t));
5264 assert_eq!(resumed_instruction(&approved, &t), approved);
5265 t.release_operator("web", None);
5266 assert!(compatible_instruction(&approved, &t));
5267 assert_eq!(t.resumption_history.len(), 1);
5268 t.start("run".into());
5269 assert!(instruction_for(&t).contains("Explicit operator resumption"));
5270 }
5271
5272 #[test]
5276 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
5277 let mut state = RunState::new(
5278 PathBuf::from("/repo"),
5279 "main".to_owned(),
5280 "abc1234".to_owned(),
5281 "task".to_owned(),
5282 Config::default(),
5283 );
5284 state.status = RunStatus::Blocked;
5285 state.resource_block = Some(crate::run::ResourceBlock {
5286 node: "graft".to_owned(),
5287 kind: "shared build cache".to_owned(),
5288 detail: "timed out".to_owned(),
5289 at: jiff::Timestamp::now(),
5290 });
5291 let mut t = task();
5292 t.start("20260902-000000-aaaa".to_owned());
5293 assert_eq!(t.attempts, 1);
5294 let verdict = Verdict {
5295 status: RunStatus::Blocked,
5296 left_pr: false,
5297 quota_hit: false,
5298 parked: false,
5299 no_viable_candidates: false,
5300 };
5301 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
5302 assert_eq!(t.status, TaskStatus::Held);
5303 assert_eq!(t.attempts, 0, "the attempt is refunded");
5304 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
5305 assert!(
5306 !t.operator_held(),
5307 "a machine hold, released without a human"
5308 );
5309
5310 let mut p = task();
5312 p.start("20260902-000000-bbbb".to_owned());
5313 settle_and_diagnose(
5314 &mut p,
5315 Verdict {
5316 parked: true,
5317 ..verdict
5318 },
5319 "parked",
5320 2,
5321 &state,
5322 );
5323 assert_eq!(p.status, TaskStatus::Parked);
5324 }
5325
5326 #[test]
5327 fn record_failure_notice_links_the_task() {
5328 let dir = tempfile::tempdir().unwrap();
5329 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
5332 let queue = Queue::at(dir.path().join("queue"));
5333 let mut t = interrupt_task("rec-fail-1");
5334 record(&queue, &mut t);
5335
5336 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
5337 assert_eq!(found.len(), 1);
5338 assert_eq!(found[0].key, "loop:record");
5339 assert_eq!(
5340 found[0].link,
5341 Some(Link::Task {
5342 id: "rec-fail-1".to_owned()
5343 })
5344 );
5345 }
5346
5347 fn interrupt_task(id: &str) -> Task {
5350 let mut t = task();
5351 t.id = id.to_owned();
5352 t.interrupt = true;
5353 t
5354 }
5355
5356 fn task_with_id(id: &str) -> Task {
5358 let mut t = task();
5359 t.id = id.to_owned();
5360 t
5361 }
5362
5363 fn urgent_task(id: &str) -> Task {
5365 let mut t = task();
5366 t.id = id.to_owned();
5367 t.urgent = true;
5368 t
5369 }
5370
5371 #[test]
5372 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
5373 let dir = tempfile::tempdir().unwrap();
5374 let queue = Queue::at(dir.path().join("queue"));
5375 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5376 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5377 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
5378 let mut first = task();
5379 queue.put(&mut first).unwrap();
5380 let stale_plain = queue.get(&first.id).unwrap();
5381 first.set_urgent(true).unwrap();
5382 queue.put(&mut first).unwrap();
5383 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
5384 let mut second = urgent_task("second");
5385 queue.put(&mut second).unwrap();
5386 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
5387 drop(queue.claim(&second.id).unwrap());
5388 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
5389 drop(dispatched);
5390 let stale_urgent = queue.get(&first.id).unwrap();
5391 first.set_urgent(false).unwrap();
5392 queue.put(&mut first).unwrap();
5393 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
5394 assert_eq!(urgent.available_permits(), 1);
5395 drop(queue.claim(&first.id).unwrap());
5396 drop(occupied);
5397 }
5398
5399 #[test]
5400 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
5401 let dir = tempfile::tempdir().unwrap();
5402 let queue = Queue::at(dir.path().join("queue"));
5403 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5404 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5405 let mut t = task();
5406 queue.put(&mut t).unwrap();
5407 for enabled in [true, false] {
5408 let stale = queue.get(&t.id).unwrap();
5409 t.set_urgent(enabled).unwrap();
5410 queue.put(&mut t).unwrap();
5411 assert_ne!(stale.urgent, enabled);
5412 let DispatchClaim {
5413 task: fresh,
5414 claim,
5415 permit,
5416 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
5417 assert_eq!(fresh.urgent, enabled);
5418 assert_eq!(ordinary.available_permits(), usize::from(enabled));
5419 assert_eq!(urgent.available_permits(), usize::from(!enabled));
5420 assert!(queue.claim(&t.id).is_err());
5421 assert_eq!(fresh.attempts, stale.attempts);
5422 drop((claim, permit));
5423 assert_eq!(ordinary.available_permits(), 1);
5424 assert_eq!(urgent.available_permits(), 1);
5425 }
5426 t.set_urgent(true).unwrap();
5427 queue.put(&mut t).unwrap();
5428 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
5429 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5430 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
5431 drop(queue.claim(&t.id).unwrap());
5432 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
5433 drop(occupied);
5434 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
5435 drop(queue.claim(&t.id).unwrap());
5436 let held = queue.claim(&t.id).unwrap();
5437 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5438 drop(held);
5439 t.status = TaskStatus::Held;
5440 queue.put(&mut t).unwrap();
5441 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5442 drop(queue.claim(&t.id).unwrap());
5443 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
5444 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5445 drop(queue.claim(&t.id).unwrap());
5446 assert_eq!(urgent.available_permits(), 1);
5447 }
5448
5449 #[test]
5454 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
5455 assert_eq!(permit_kind(true, false), PermitKind::None);
5456 assert_eq!(permit_kind(true, true), PermitKind::None);
5457 }
5458
5459 #[test]
5464 fn permit_kind_separates_urgent_from_ordinary() {
5465 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
5466 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
5467 }
5468
5469 #[test]
5476 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
5477 let cfg = Config::default();
5478 let repo = Path::new("/any/repo/path");
5479
5480 let reason =
5481 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
5482 assert!(reason.contains("1024"), "{reason}");
5483 assert!(
5484 reason.contains(&cfg.disk.min_free_bytes.to_string()),
5485 "{reason}"
5486 );
5487
5488 assert_eq!(
5489 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
5490 None,
5491 "exactly at the floor is open"
5492 );
5493 assert_eq!(
5494 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
5495 None,
5496 "comfortably above the floor is open"
5497 );
5498 }
5499
5500 #[test]
5501 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
5502 let mut cfg = Config::default();
5503 cfg.disk.min_free_bytes = 0;
5504 let repo = Path::new("/any/repo/path");
5505 assert_eq!(
5506 disk_gate_with(repo, &cfg, |_| Ok(0)),
5507 None,
5508 "a zero floor never measures at all"
5509 );
5510 }
5511
5512 #[test]
5513 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
5514 let cfg = Config::default();
5515 let repo = Path::new("/any/repo/path");
5516 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
5517 .expect("a measurement failure must close the gate, not open it");
5518 assert!(reason.contains("could not measure"), "{reason}");
5519 }
5520
5521 #[test]
5522 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
5523 let ordinary = task();
5524 let next = advance_interrupt(
5525 Interrupt::Idle,
5526 std::slice::from_ref(&ordinary.id),
5527 std::slice::from_ref(&ordinary),
5528 );
5529 assert_eq!(next, Interrupt::Idle);
5530 }
5531
5532 #[test]
5533 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
5534 let marked = interrupt_task("marked");
5537 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5538 assert_eq!(next, Interrupt::Idle);
5539 }
5540
5541 #[test]
5542 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
5543 let marked = interrupt_task("marked");
5544 let next = advance_interrupt(
5545 Interrupt::Idle,
5546 &["running".to_owned()],
5547 std::slice::from_ref(&marked),
5548 );
5549 assert_eq!(
5550 next,
5551 Interrupt::Parking {
5552 parked: vec!["running".to_owned()],
5553 interrupt_task: "marked".to_owned(),
5554 }
5555 );
5556 }
5557
5558 #[test]
5567 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
5568 let marked = interrupt_task("marked");
5569
5570 let two = advance_interrupt(
5571 Interrupt::Idle,
5572 &["a".to_owned(), "b".to_owned()],
5573 std::slice::from_ref(&marked),
5574 );
5575 assert_eq!(two, Interrupt::Idle);
5576
5577 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5578 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
5579 }
5580
5581 #[test]
5582 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
5583 let state = Interrupt::Parking {
5584 parked: vec!["running".to_owned()],
5585 interrupt_task: "marked".to_owned(),
5586 };
5587 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
5589 assert_eq!(still_going, state);
5590
5591 let stopped_but_not_yet_dispatched =
5595 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
5596 assert_eq!(stopped_but_not_yet_dispatched, state);
5597
5598 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
5600 assert_eq!(
5601 dispatched,
5602 Interrupt::Running {
5603 parked: vec!["running".to_owned()],
5604 interrupt_task: "marked".to_owned(),
5605 }
5606 );
5607 }
5608
5609 #[test]
5610 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
5611 let state = Interrupt::Running {
5612 parked: vec!["running".to_owned()],
5613 interrupt_task: "marked".to_owned(),
5614 };
5615 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
5616 assert_eq!(still_running, state);
5617
5618 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
5625 assert_eq!(
5626 ended,
5627 Interrupt::Resuming {
5628 parked: vec!["running".to_owned()]
5629 }
5630 );
5631 }
5632
5633 #[test]
5634 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
5635 let state = Interrupt::Resuming {
5636 parked: vec!["running".to_owned()],
5637 };
5638 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
5639 assert_eq!(still_waiting, state);
5640
5641 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
5642 assert_eq!(dispatched, Interrupt::Idle);
5643 }
5644
5645 #[test]
5651 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
5652 {
5653 let state = Interrupt::Parking {
5654 parked: vec!["running".to_owned()],
5655 interrupt_task: "marked".to_owned(),
5656 };
5657 let next = advance_interrupt(state, &[], &[]);
5660 assert_eq!(
5661 next,
5662 Interrupt::Resuming {
5663 parked: vec!["running".to_owned()]
5664 },
5665 "abandoning the interrupt must not abandon the resume it owes"
5666 );
5667 }
5668
5669 #[test]
5672 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
5673 let state = Interrupt::Resuming {
5674 parked: vec!["running".to_owned()],
5675 };
5676 let next = advance_interrupt(state, &[], &[]);
5677 assert_eq!(
5678 next,
5679 Interrupt::Idle,
5680 "nothing is left to wait for; the loop must not stay wedged"
5681 );
5682 }
5683
5684 #[test]
5685 fn disabled_by_config_the_sequence_can_never_leave_idle() {
5686 let marked = interrupt_task("marked");
5687 let next = advance_interrupt_tick(
5688 false,
5689 Interrupt::Idle,
5690 &["running".to_owned()],
5691 std::slice::from_ref(&marked),
5692 );
5693 assert_eq!(
5694 next,
5695 Interrupt::Idle,
5696 "an unmarked, unconfigured daemon must behave exactly as before"
5697 );
5698 }
5699
5700 #[test]
5701 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
5702 let state = Interrupt::Parking {
5703 parked: vec!["running".to_owned()],
5704 interrupt_task: "marked".to_owned(),
5705 };
5706 let candidates = vec![interrupt_task("marked"), task()];
5707 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5708 assert!(
5709 allowed.is_empty(),
5710 "nothing may dispatch - not even the interrupt task itself - \
5711 until the parked run has actually stopped"
5712 );
5713 }
5714
5715 #[test]
5729 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
5730 for state in [
5731 Interrupt::Parking {
5732 parked: vec!["running".to_owned()],
5733 interrupt_task: "marked".to_owned(),
5734 },
5735 Interrupt::Running {
5736 parked: vec!["running".to_owned()],
5737 interrupt_task: "marked".to_owned(),
5738 },
5739 Interrupt::Resuming {
5740 parked: vec!["running".to_owned()],
5741 },
5742 ] {
5743 let candidates = vec![
5744 interrupt_task("marked"),
5745 urgent_task("hot"),
5746 task_with_id("ordinary"),
5747 ];
5748 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5749 assert!(
5750 !allowed.iter().any(|t| t.id == "hot"),
5751 "an urgent candidate must wait out the same gate as anything \
5752 else while the run it would run alongside has not actually \
5753 left flight, for state {state:?}: {allowed:?}"
5754 );
5755 }
5756 }
5757
5758 #[test]
5764 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
5765 let state = Interrupt::Resuming {
5766 parked: vec!["hot".to_owned()],
5767 };
5768 let candidates = vec![urgent_task("hot"), task()];
5769 let allowed = interrupt_gate(&state, &[], candidates);
5770 assert_eq!(
5771 allowed.iter().filter(|t| t.id == "hot").count(),
5772 1,
5773 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
5774 );
5775 }
5776
5777 #[test]
5778 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
5779 let state = Interrupt::Parking {
5780 parked: vec!["running".to_owned()],
5781 interrupt_task: "marked".to_owned(),
5782 };
5783 let other = task();
5784 let candidates = vec![interrupt_task("marked"), other.clone()];
5785 let allowed = interrupt_gate(&state, &[], candidates);
5786 assert_eq!(allowed.len(), 1);
5787 assert_eq!(allowed[0].id, "marked");
5788 }
5789
5790 #[test]
5791 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
5792 let state = Interrupt::Running {
5793 parked: vec!["running".to_owned()],
5794 interrupt_task: "marked".to_owned(),
5795 };
5796 let candidates = vec![task(), task()];
5797 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
5798 assert!(allowed.is_empty());
5799 }
5800
5801 #[test]
5808 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
5809 let state = Interrupt::Resuming {
5810 parked: vec!["a".to_owned(), "c".to_owned()],
5811 };
5812 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
5813 let allowed = interrupt_gate(&state, &[], candidates);
5814 assert_eq!(
5815 allowed.len(),
5816 1,
5817 "at most one candidate may be offered while resuming: {allowed:?}"
5818 );
5819 assert_eq!(allowed[0].id, "a");
5820 }
5821
5822 #[test]
5823 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
5824 let state = Interrupt::Resuming {
5825 parked: vec!["a".to_owned()],
5826 };
5827 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
5828 assert!(allowed.is_empty());
5829 }
5830
5831 #[test]
5837 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
5838 let running = task(); let marked = interrupt_task("marked");
5840
5841 let mut state = Interrupt::Idle;
5842 let in_flight = vec![running.id.clone()];
5844 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
5845 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
5846 assert!(gated.is_empty(), "still waiting on `running` to park");
5847
5848 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
5850 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
5851 assert_eq!(
5852 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5853 vec!["marked"],
5854 "only the interrupt task may be offered to the dispatcher now"
5855 );
5856
5857 state = advance_interrupt_tick(
5859 true,
5860 state,
5861 &["marked".to_owned()],
5862 std::slice::from_ref(&running),
5863 );
5864 let gated = interrupt_gate(
5865 &state,
5866 &["marked".to_owned()],
5867 vec![marked.clone(), running.clone()],
5868 );
5869 assert!(
5870 gated.is_empty(),
5871 "the parked run must not be offered back while the interrupt \
5872 task is still running"
5873 );
5874
5875 let other = task_with_id("other");
5879 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
5880 assert_eq!(
5881 state,
5882 Interrupt::Resuming {
5883 parked: vec![running.id.clone()]
5884 }
5885 );
5886 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
5887 assert_eq!(
5888 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5889 vec![running.id.as_str()],
5890 "exactly the parked run resumes - not the unrelated task, even \
5891 though it was offered first"
5892 );
5893
5894 state = advance_interrupt_tick(
5898 true,
5899 state,
5900 std::slice::from_ref(&running.id),
5901 std::slice::from_ref(&other),
5902 );
5903 assert_eq!(state, Interrupt::Idle);
5904 let gated = interrupt_gate(
5905 &state,
5906 std::slice::from_ref(&running.id),
5907 vec![other.clone()],
5908 );
5909 assert_eq!(
5910 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5911 vec![other.id.as_str()],
5912 "ordinary dispatch is unrestricted again"
5913 );
5914 }
5915
5916 #[test]
5917 fn every_run_status_settles_the_task_it_came_from() {
5918 let table = [
5920 (RunStatus::Merged, TaskStatus::Done, 1),
5921 (RunStatus::Ready, TaskStatus::Done, 1),
5922 (RunStatus::Stalled, TaskStatus::Failed, 0),
5923 (RunStatus::Blocked, TaskStatus::Failed, 1),
5924 (RunStatus::Failed, TaskStatus::Failed, 1),
5925 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
5926 (RunStatus::Prep, TaskStatus::Failed, 1),
5927 (RunStatus::Implementing, TaskStatus::Failed, 1),
5928 (RunStatus::Judging, TaskStatus::Failed, 1),
5929 (RunStatus::Deliberating, TaskStatus::Failed, 1),
5930 (RunStatus::Voting, TaskStatus::Failed, 1),
5931 (RunStatus::Reviewing, TaskStatus::Failed, 1),
5932 (RunStatus::Gating, TaskStatus::Failed, 1),
5933 ];
5934 for (run, want, attempts) in table {
5935 let mut t = task();
5936 t.start("20260902-000000-aaaa".to_owned());
5937 settle(
5938 &mut t,
5939 Verdict {
5940 status: run,
5941 left_pr: false,
5942 parked: false,
5943 quota_hit: matches!(run, RunStatus::Stalled),
5944 no_viable_candidates: false,
5945 },
5946 "why",
5947 2,
5948 );
5949 assert_eq!(t.status, want, "task status after {}", label(run));
5950 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
5951 }
5952 }
5953
5954 #[test]
5955 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
5956 let mut stalled = task();
5957 stalled.start("20260902-000000-aaaa".to_owned());
5958 settle(
5959 &mut stalled,
5960 Verdict {
5961 status: RunStatus::Stalled,
5962 left_pr: false,
5963 parked: false,
5964 quota_hit: true,
5965 no_viable_candidates: false,
5966 },
5967 "quota",
5968 1,
5969 );
5970 assert_eq!(stalled.attempts, 0);
5971 assert!(
5972 stalled.status.runnable(),
5973 "a machine problem must leave the task in line"
5974 );
5975
5976 let mut blocked = task();
5977 blocked.start("20260902-000000-aaaa".to_owned());
5978 settle(
5979 &mut blocked,
5980 Verdict {
5981 status: RunStatus::Blocked,
5982 left_pr: false,
5983 parked: false,
5984 quota_hit: false,
5985 no_viable_candidates: false,
5986 },
5987 "findings open",
5988 1,
5989 );
5990 assert_eq!(blocked.attempts, 1);
5991 assert_eq!(
5992 blocked.status,
5993 TaskStatus::Held,
5994 "the last attempt hands the task to a human"
5995 );
5996 }
5997
5998 #[test]
5999 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
6000 let mut delivered = task();
6003 delivered.start("20260903-080619-01c2".to_owned());
6004 settle(
6005 &mut delivered,
6006 Verdict {
6007 status: RunStatus::Blocked,
6008 left_pr: true,
6009 parked: false,
6010 quota_hit: false,
6011 no_viable_candidates: false,
6012 },
6013 "no check status",
6014 4,
6015 );
6016 assert_eq!(
6017 delivered.status,
6018 TaskStatus::Held,
6019 "a pull request waiting on CI or a person is not a retryable failure"
6020 );
6021 assert!(
6022 !delivered.status.runnable(),
6023 "the loop must not pick this task up again"
6024 );
6025 assert_eq!(
6026 delivered.last_error.as_deref(),
6027 Some("no check status"),
6028 "the operator needs to be told what the gate was waiting for"
6029 );
6030
6031 let mut empty_handed = task();
6034 empty_handed.start("20260903-080619-01c2".to_owned());
6035 settle(
6036 &mut empty_handed,
6037 Verdict {
6038 status: RunStatus::Blocked,
6039 left_pr: false,
6040 parked: false,
6041 quota_hit: false,
6042 no_viable_candidates: false,
6043 },
6044 "findings open",
6045 4,
6046 );
6047 assert_eq!(empty_handed.status, TaskStatus::Failed);
6048 assert!(empty_handed.status.runnable());
6049 }
6050
6051 #[test]
6052 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
6053 let mut noop = task();
6059 noop.start("20260912-131304-391f".to_owned());
6060 settle(
6061 &mut noop,
6062 Verdict {
6063 status: RunStatus::VerifiedNoop,
6064 left_pr: false,
6065 parked: false,
6066 quota_hit: false,
6067 no_viable_candidates: true,
6068 },
6069 "candidate A: already fixed by b32cfc4, on main",
6070 4,
6071 );
6072 assert_eq!(
6073 noop.status,
6074 TaskStatus::Held,
6075 "an unverified claim is a request for a human, not a failure"
6076 );
6077 assert!(
6078 !noop.status.runnable(),
6079 "the loop must not requeue this on the same unverified claim"
6080 );
6081 assert_eq!(noop.attempts, 1);
6086 }
6087
6088 #[test]
6089 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
6090 let mut parked = task();
6095 parked.start("20260903-183634-2d98".to_owned());
6096 settle(
6097 &mut parked,
6098 Verdict {
6099 status: RunStatus::Implementing,
6100 left_pr: false,
6101 quota_hit: false,
6102 parked: true,
6103 no_viable_candidates: false,
6104 },
6105 "parked after `implementing`",
6106 2,
6107 );
6108 assert_eq!(parked.attempts, 0, "a park is refunded");
6109 assert!(
6110 parked.status.runnable(),
6111 "and the task stays in line so the next loop resumes its run"
6112 );
6113 assert_eq!(parked.status, TaskStatus::Parked);
6114 assert_eq!(
6115 parked.park_reason.as_deref(),
6116 Some("parked after `implementing`"),
6117 "the card says where it stopped"
6118 );
6119 assert_eq!(parked.last_error, None, "nothing failed");
6120
6121 let mut broken = task();
6125 broken.start("20260903-183634-2d98".to_owned());
6126 settle(
6127 &mut broken,
6128 Verdict {
6129 status: RunStatus::Implementing,
6130 left_pr: false,
6131 quota_hit: false,
6132 parked: false,
6133 no_viable_candidates: false,
6134 },
6135 "returned mid-flight",
6136 2,
6137 );
6138 assert_eq!(broken.attempts, 1);
6139 }
6140
6141 #[test]
6142 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
6143 let mut flaky = task();
6148 flaky.start("20260903-123023-e633".to_owned());
6149 settle(
6150 &mut flaky,
6151 Verdict {
6152 status: RunStatus::Stalled,
6153 left_pr: false,
6154 parked: false,
6155 quota_hit: false,
6156 no_viable_candidates: false,
6157 },
6158 "verdict rests on 1 of 3 judges",
6159 2,
6160 );
6161 assert_eq!(
6162 flaky.attempts, 1,
6163 "flakiness spends an attempt, so `max_attempts` still bounds it"
6164 );
6165 assert!(flaky.status.runnable(), "and it is still worth retrying");
6166
6167 let mut limited = task();
6169 limited.start("20260903-123023-e633".to_owned());
6170 settle(
6171 &mut limited,
6172 Verdict {
6173 status: RunStatus::Stalled,
6174 left_pr: false,
6175 parked: false,
6176 quota_hit: true,
6177 no_viable_candidates: false,
6178 },
6179 "judge-2, judge-3 out of quota",
6180 2,
6181 );
6182 assert_eq!(limited.attempts, 0, "a quota window is refunded");
6183 assert!(limited.status.runnable());
6184
6185 let mut worn = task();
6188 for _ in 0..2 {
6189 worn.release();
6190 }
6191 worn.start("20260903-123023-e633".to_owned());
6192 worn.attempts = 2;
6193 settle(
6194 &mut worn,
6195 Verdict {
6196 status: RunStatus::Stalled,
6197 left_pr: false,
6198 parked: false,
6199 quota_hit: false,
6200 no_viable_candidates: false,
6201 },
6202 "no quorum again",
6203 2,
6204 );
6205 assert_eq!(worn.status, TaskStatus::Held);
6206 assert!(!worn.status.runnable());
6207 }
6208
6209 #[test]
6210 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
6211 let mut wiped_out = task();
6218 wiped_out.start("20260907-025000-a1b2".to_owned());
6219 settle(
6220 &mut wiped_out,
6221 Verdict {
6222 status: RunStatus::Failed,
6223 left_pr: false,
6224 parked: false,
6225 quota_hit: true,
6226 no_viable_candidates: true,
6227 },
6228 "no candidate produced a change; nothing to judge",
6229 2,
6230 );
6231 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
6232 assert!(
6233 wiped_out.status.runnable(),
6234 "a machine problem must leave the task in line"
6235 );
6236
6237 let mut partial_progress = task();
6243 partial_progress.start("20260907-025500-c3d4".to_owned());
6244 settle(
6245 &mut partial_progress,
6246 Verdict {
6247 status: RunStatus::Failed,
6248 left_pr: false,
6249 parked: false,
6250 quota_hit: true,
6251 no_viable_candidates: false,
6252 },
6253 "gate failed on the winning candidate",
6254 2,
6255 );
6256 assert_eq!(
6257 partial_progress.attempts, 1,
6258 "a candidate that actually produced a change spends the attempt \
6259 even though some other seat hit its quota"
6260 );
6261 assert!(partial_progress.status.runnable());
6262 }
6263
6264 #[test]
6265 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
6266 let mut t = task();
6272 t.start("20260907-025000-a1b2".to_owned());
6273 let mut state = run_state(RunStatus::Failed);
6274 state.quota.push(QuotaLoss {
6275 seat: "cand-a".to_owned(),
6276 node: "implement".to_owned(),
6277 at: Timestamp::now(),
6278 reset: None,
6279 });
6280 assert!(
6281 state.viable().is_empty(),
6282 "no candidate was added, so nothing is viable"
6283 );
6284 reclaim(&mut t, Some(state), 2, "en");
6285 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
6286 assert!(t.status.runnable());
6287 }
6288
6289 #[test]
6290 fn a_held_task_is_never_offered_to_the_loop() {
6291 let dir = tempfile::tempdir().unwrap();
6292 let queue = Queue::at(dir.path().to_path_buf());
6293 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
6294 let mut t = task();
6295 t.id = format!("2026090{n}-000000-000{n}");
6296 t.priority = priority;
6297 queue.put(&mut t).unwrap();
6298 }
6299 let mut held = task();
6300 held.id = "20260909-000000-9999".to_owned();
6301 held.priority = 99;
6302 held.hold_machine(None);
6303 queue.put(&mut held).unwrap();
6304
6305 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
6306 assert_eq!(order.len(), 3);
6307 assert!(!order.contains(&held.id));
6308 assert_eq!(
6309 order.first().cloned(),
6310 queue.next_runnable().map(|t| t.id),
6311 "the loop's first candidate is exactly what the queue offers"
6312 );
6313 assert_eq!(
6314 order,
6315 vec![
6316 "20260902-000000-0002".to_owned(),
6317 "20260903-000000-0003".to_owned(),
6318 "20260901-000000-0001".to_owned(),
6319 ],
6320 "priority first, then oldest, so nothing starves"
6321 );
6322 }
6323
6324 #[test]
6325 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
6326 let dir = tempfile::tempdir().unwrap();
6327 let queue = Queue::at(dir.path().to_path_buf());
6328 let mut old = task();
6329 old.id = "20260101-000000-old0".to_owned();
6330 queue.put(&mut old).unwrap();
6331 let mut fresh = task();
6332 fresh.id = "20260101-000000-new0".to_owned();
6333 queue.put(&mut fresh).unwrap();
6334
6335 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
6339 std::thread::sleep(Duration::from_millis(60));
6340 let live = queue.claim(&fresh.id).unwrap();
6341
6342 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6343 assert_eq!(swept, vec![old.id.clone()]);
6344 assert!(
6345 queue.claim(&old.id).is_ok(),
6346 "an unparseable lock older than the threshold is swept"
6347 );
6348 assert!(
6349 queue.claim(&fresh.id).is_err(),
6350 "a live pid protects its lock regardless of age"
6351 );
6352 drop(live);
6353 }
6354
6355 #[test]
6356 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
6357 let dir = tempfile::tempdir().unwrap();
6367 let queue = Queue::at(dir.path().to_path_buf());
6368 let mut t = task();
6369 t.id = "20260101-000000-live".to_owned();
6370 queue.put(&mut t).unwrap();
6371
6372 let claim = queue.claim(&t.id).unwrap();
6373 std::thread::sleep(Duration::from_millis(60));
6374
6375 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6376 assert!(
6377 swept.is_empty(),
6378 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
6379 );
6380 assert!(
6381 queue.claim(&t.id).is_err(),
6382 "the lock still protects its task"
6383 );
6384 drop(claim);
6385 }
6386
6387 fn injected_dead_pid() -> u32 {
6390 std::process::id().checked_add(1).unwrap_or(1)
6391 }
6392
6393 #[test]
6394 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
6395 let dir = tempfile::tempdir().unwrap();
6396 let queue = Queue::at(dir.path().to_path_buf());
6397 let mut t = task();
6398 t.id = "20260101-000000-dead".to_owned();
6399 queue.put(&mut t).unwrap();
6400 let dead_pid = injected_dead_pid();
6401
6402 std::fs::write(
6407 dir.path().join(format!("{}.lock", t.id)),
6408 dead_pid.to_string(),
6409 )
6410 .unwrap();
6411
6412 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6413 pid != dead_pid
6414 });
6415 assert_eq!(
6416 swept,
6417 vec![t.id.clone()],
6418 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
6419 );
6420 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
6421 }
6422
6423 #[test]
6424 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
6425 let dir = tempfile::tempdir().unwrap();
6426 let queue = Queue::at(dir.path().to_path_buf());
6427 let mut t = task();
6428 t.id = "20260101-000000-late".to_owned();
6429 queue.put(&mut t).unwrap();
6430 let dead_pid = injected_dead_pid();
6431
6432 assert!(
6435 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
6436 "nothing has claimed the task yet"
6437 );
6438
6439 std::fs::write(
6442 dir.path().join(format!("{}.lock", t.id)),
6443 dead_pid.to_string(),
6444 )
6445 .unwrap();
6446
6447 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6451 pid != dead_pid
6452 });
6453 assert_eq!(swept, vec![t.id.clone()]);
6454 }
6455
6456 #[test]
6457 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
6458 crate::run::pin_test_home();
6463 let dir = tempfile::tempdir().unwrap();
6464 let queue = Queue::at(dir.path().to_path_buf());
6465 let mut t = task();
6466 t.id = "20260101-000000-crsh".to_owned();
6467 t.status = TaskStatus::Running;
6468 t.attempts = 1;
6469 t.runs.push("20260904-000000-4043".to_owned());
6473 queue.put(&mut t).unwrap();
6474 let dead_pid = injected_dead_pid();
6475
6476 std::fs::write(
6479 dir.path().join(format!("{}.lock", t.id)),
6480 dead_pid.to_string(),
6481 )
6482 .unwrap();
6483
6484 assert!(reclaim_orphaned_running(&queue, 2).is_empty());
6490 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
6491
6492 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6493 pid != dead_pid
6494 });
6495 assert_eq!(swept, vec![t.id.clone()]);
6496
6497 let reclaimed = reclaim_orphaned_running(&queue, 2);
6498 assert_eq!(reclaimed, vec![t.id.clone()]);
6499 let after = queue.get(&t.id).unwrap();
6500 assert_eq!(
6501 after.status,
6502 TaskStatus::Held,
6503 "no run.json to recover from, so a human is asked"
6504 );
6505 assert_eq!(
6506 after.runs,
6507 vec!["20260904-000000-4043".to_owned()],
6508 "the crashed run's id is kept as evidence, not discarded"
6509 );
6510 }
6511
6512 #[test]
6513 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
6514 let dir = tempfile::tempdir().unwrap();
6515 let queue = Queue::at(dir.path().to_path_buf());
6516 let mut t = task();
6517 t.id = "20260101-000000-unknown".to_owned();
6518 queue.put(&mut t).unwrap();
6519 let dead_pid = injected_dead_pid();
6520 std::fs::write(
6521 dir.path().join(format!("{}.lock", t.id)),
6522 dead_pid.to_string(),
6523 )
6524 .unwrap();
6525
6526 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
6527 assert!(swept.is_empty(), "an unknown pid must keep its lock");
6528 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
6529 }
6530
6531 fn run_state_in(status: RunStatus, language: &str) -> RunState {
6532 let mut s = run_state(status);
6533 s.config.graph.language = language.to_owned();
6534 s
6535 }
6536
6537 fn unstarted_verdict(status: RunStatus) -> Verdict {
6538 Verdict {
6539 status,
6540 left_pr: false,
6541 quota_hit: false,
6542 parked: false,
6543 no_viable_candidates: false,
6544 }
6545 }
6546
6547 #[test]
6548 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
6549 let mut t = task();
6550 t.attempts = 1;
6551 settle_in(
6552 &mut t,
6553 unstarted_verdict(RunStatus::AlreadyInBase),
6554 "already in main",
6555 1,
6556 phrases("en"),
6557 );
6558 assert_eq!(t.status, TaskStatus::Done);
6559 assert_eq!(t.attempts, 0);
6560 }
6561
6562 #[test]
6563 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
6564 let reason = |language: &str| {
6565 let mut t = task();
6566 settle_in(
6567 &mut t,
6568 unstarted_verdict(RunStatus::Judging),
6569 "boom",
6570 1,
6571 phrases(language),
6572 );
6573 t.last_error.or(t.hold_reason).unwrap_or_default()
6574 };
6575 assert!(
6576 reason("en").starts_with("the graph stopped at `"),
6577 "{}",
6578 reason("en")
6579 );
6580 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
6581 assert!(reason("日本語").contains("boom"));
6582 assert_eq!(reason("fr"), reason("en"));
6583 }
6584
6585 #[test]
6586 fn describe_follows_the_run_language_and_keeps_the_run_id() {
6587 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
6588 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
6589 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
6590 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
6591 assert!(ja.contains("[run "), "{ja}");
6592 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
6593 assert!(ended.starts_with("run 終了: "), "{ended}");
6594 let mut de = run_state_in(RunStatus::Failed, "de");
6595 let mut en = run_state_in(RunStatus::Failed, "en");
6596 de.id = "same".to_owned();
6597 en.id = "same".to_owned();
6598 assert_eq!(describe(&de), describe(&en));
6599 }
6600
6601 #[test]
6602 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
6603 use crate::handover::Refused;
6604 let cases = [
6605 Refused::Foreign {
6606 branch: "b".into(),
6607 path: "/w/x".into(),
6608 why: "made by hand".into(),
6609 },
6610 Refused::Unsafe {
6611 branch: "b".into(),
6612 path: "/w/x".into(),
6613 why: "its worktree has uncommitted changes (a.rs)".into(),
6614 },
6615 Refused::ReleaseFailed {
6616 branch: "b".into(),
6617 path: "/w/x".into(),
6618 run: "ab12".into(),
6619 },
6620 ];
6621 for r in &cases {
6622 let en = (phrases("en").handover_refused)(r);
6623 assert_eq!(en, r.to_string());
6624 let ja = (phrases("ja").handover_refused)(r);
6625 assert!(
6626 !ja.contains("is checked out") && !ja.contains("try again"),
6627 "{ja}"
6628 );
6629 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
6630 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
6631 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
6632 }
6633 }
6634 assert!(phrases("en").handover_hint.contains("release the task"));
6635 assert!(phrases("ja").handover_hint.contains("解放"));
6636 }
6637
6638 #[test]
6639 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
6640 let build = |lang: &str| {
6641 let mut s = run_state_in(RunStatus::Blocked, lang);
6642 s.id = "same".to_owned();
6643 s.config.graph.review_rounds = 3;
6644 let mut r = review_round(3);
6645 r.expected = 3;
6646 r.answered = 1;
6647 s.reviews.push(r);
6648 s.event(
6649 "review",
6650 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
6651 );
6652 s
6653 };
6654 let en = describe(&build("en"));
6655 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
6656 let ja = describe(&build("ja"));
6657 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
6658 assert!(!ja.contains("never answered"), "{ja}");
6659 assert!(ja.contains("[run "), "{ja}");
6660
6661 let mut failed = build("ja");
6664 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
6665 command: "cargo test".to_owned(),
6666 code: Some(1),
6667 output_tail: String::new(),
6668 duration_ms: 0,
6669 resource_blocked: false,
6670 });
6671 failed.event("review", "stopped; e2e failed: cargo test");
6672 let ja = describe(&failed);
6673 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
6674 assert!(!ja.contains("席のレビュアー"), "{ja}");
6675 }
6676
6677 #[test]
6678 fn refusals_and_recovery_prose_follow_the_language() {
6679 let t = held_task_with("r1");
6680 let q = action_question(
6681 "r1",
6682 ask::ChoiceAction::Resume {
6683 run: "r1".to_owned(),
6684 },
6685 );
6686 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
6687 ActionDecision::Refuse(s) => s,
6688 other => panic!("{other:?}"),
6689 };
6690 assert!(refuse(phrases("en")).contains("could not be read: gone"));
6691 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
6692 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
6693
6694 let mut held = task();
6695 reclaim(&mut held, None, 2, "ja");
6696 assert!(held.hold_reason.unwrap().contains("保留にしました"));
6697 let mut held = task();
6698 reclaim(&mut held, None, 2, "xx");
6699 assert!(held.hold_reason.unwrap().contains("held for a human"));
6700 }
6701
6702 fn run_state(status: RunStatus) -> RunState {
6703 let mut state = RunState::new(
6704 PathBuf::from("/repo"),
6705 "main".to_owned(),
6706 "abc1234def".to_owned(),
6707 "add retries".to_owned(),
6708 Config::default(),
6709 );
6710 state.status = status;
6711 state
6712 }
6713
6714 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
6715 Candidate {
6716 index: 0,
6717 label,
6718 agent: "claude".to_owned(),
6719 branch: format!("magi/x/{label}"),
6720 worktree: PathBuf::from("/repo"),
6721 summary: summary.to_owned(),
6722 stat: String::new(),
6723 files: 0,
6724 commits: usize::from(!empty),
6725 empty,
6726 failed: failed.map(str::to_owned),
6727 verified_noop: None,
6728 duration_ms: 0,
6729 folded: false,
6730 }
6731 }
6732
6733 #[test]
6734 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
6735 let mut state = run_state(RunStatus::Blocked);
6736 state.gate = vec![
6737 CommandOutcome {
6738 command: "cargo make check".to_owned(),
6739 code: Some(0),
6740 output_tail: "ok".to_owned(),
6741 duration_ms: 0,
6742 resource_blocked: false,
6743 },
6744 CommandOutcome {
6745 command: "cargo test".to_owned(),
6746 code: Some(101),
6747 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
6748 duration_ms: 0,
6749 resource_blocked: false,
6750 },
6751 ];
6752 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
6753 assert!(d.contains("cargo test"), "{d}");
6754 assert!(
6755 !d.contains("cargo make check"),
6756 "a passing check is not a diagnostic: {d}"
6757 );
6758 assert!(d.contains("assertion failed"), "{d}");
6759 }
6760
6761 #[test]
6762 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
6763 let mut state = run_state(RunStatus::Blocked);
6764 state.event(
6765 "land",
6766 "stopped: the fixer produced no commit while 2 check(s) were failing \
6767 (build, lint); stopping instead of looping on an unchanged tree",
6768 );
6769 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
6770 assert!(d.contains("build"), "{d}");
6771 assert!(d.contains("lint"), "{d}");
6772 assert!(d.contains("fixer produced no commit"), "{d}");
6773 }
6774
6775 #[test]
6776 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
6777 let state = run_state(RunStatus::VerifiedNoop);
6782 let d = describe(&state);
6783 assert!(
6784 d.contains("agent-verified no-op"),
6785 "expected the display label, not the wire spelling: {d}"
6786 );
6787 assert!(!d.contains("verified_noop"), "{d}");
6788 }
6789
6790 #[test]
6791 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
6792 let mut state = run_state(RunStatus::Failed);
6798 state.candidates = vec![candidate(
6799 'A',
6800 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
6801 true,
6802 None,
6803 )];
6804 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
6805 assert!(d.contains("candidate A"), "{d}");
6806 assert!(d.contains("tagged v1.2.3"), "{d}");
6807 }
6808
6809 #[test]
6810 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
6811 let mut state = run_state(RunStatus::Failed);
6812 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
6813 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
6814 assert!(d.contains("candidate A"), "{d}");
6815 assert!(d.contains("agent timed out"), "{d}");
6816 }
6817
6818 #[test]
6819 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
6820 let mut state = run_state(RunStatus::Failed);
6823 state.candidates = vec![candidate('A', "did the work", false, None)];
6824 assert!(diagnostic(&state).is_none());
6825 }
6826
6827 #[test]
6828 fn diagnostic_is_bounded_however_much_a_run_printed() {
6829 let mut state = run_state(RunStatus::Blocked);
6830 state.gate = vec![
6831 CommandOutcome {
6832 command: "cargo test".to_owned(),
6833 code: Some(101),
6834 output_tail: "x".repeat(50_000),
6835 duration_ms: 0,
6836 resource_blocked: false,
6837 },
6838 CommandOutcome {
6839 command: "cargo clippy".to_owned(),
6840 code: Some(1),
6841 output_tail: "y".repeat(50_000),
6842 duration_ms: 0,
6843 resource_blocked: false,
6844 },
6845 ];
6846 state.candidates = vec![
6847 candidate('A', &"z".repeat(50_000), true, None),
6848 candidate('B', &"w".repeat(50_000), true, None),
6849 ];
6850 let d = diagnostic(&state).expect("plenty here to diagnose");
6851 assert!(
6852 d.len() <= DIAGNOSTIC_MAX,
6853 "diagnostic grew to {} bytes, unbounded",
6854 d.len()
6855 );
6856 }
6857
6858 #[test]
6859 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
6860 let mut state = run_state(RunStatus::Blocked);
6861 state.gate = vec![CommandOutcome {
6862 command: "cargo test".to_owned(),
6863 code: Some(101),
6864 output_tail: "assertion failed".to_owned(),
6865 duration_ms: 0,
6866 resource_blocked: false,
6867 }];
6868 let verdict = Verdict {
6869 status: RunStatus::Blocked,
6870 left_pr: false,
6871 quota_hit: false,
6872 parked: false,
6873 no_viable_candidates: false,
6874 };
6875
6876 let mut t = task();
6879 t.start("run-1".to_owned());
6880 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
6881 assert_eq!(t.status, TaskStatus::Failed);
6882 assert!(t.diagnostic.is_none());
6883
6884 t.start("run-2".to_owned());
6887 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
6888 assert_eq!(t.status, TaskStatus::Held);
6889 let d = t.diagnostic.expect("a held task must carry its diagnostic");
6890 assert!(d.contains("cargo test"), "{d}");
6891 }
6892
6893 #[test]
6894 fn a_held_task_names_the_open_question_it_is_waiting_on() {
6895 crate::run::pin_test_home();
6900 let home = crate::run::home();
6901 let state = run_state(RunStatus::VerifiedNoop);
6902 let mut q = ask::Question::new(
6903 state.id.clone(),
6904 "implement".to_owned(),
6905 "impl-A".to_owned(),
6906 "is this really a no-op?".to_owned(),
6907 String::new(),
6908 Vec::new(),
6909 );
6910 Questions::at(home.join("questions")).put(&mut q).unwrap();
6911
6912 let verdict = Verdict {
6913 status: RunStatus::VerifiedNoop,
6914 left_pr: false,
6915 quota_hit: false,
6916 parked: false,
6917 no_viable_candidates: false,
6918 };
6919 let mut t = task();
6920 t.start(state.id.clone());
6921 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
6922
6923 assert_eq!(t.status, TaskStatus::Held);
6924 let reason = t.hold_reason.expect("a held task must record why");
6925 assert!(
6926 reason.starts_with("run ended agent-verified no-op"),
6927 "the original settle reason must survive unchanged: {reason}"
6928 );
6929 assert!(
6930 reason.contains(q.short()),
6931 "the open question's id must be named so the notice is actionable: {reason}"
6932 );
6933 }
6934
6935 #[test]
6936 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
6937 crate::run::pin_test_home();
6938 let state = run_state(RunStatus::VerifiedNoop);
6939
6940 let verdict = Verdict {
6941 status: RunStatus::VerifiedNoop,
6942 left_pr: false,
6943 quota_hit: false,
6944 parked: false,
6945 no_viable_candidates: false,
6946 };
6947 let mut t = task();
6948 t.start(state.id.clone());
6949 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
6950
6951 assert_eq!(t.status, TaskStatus::Held);
6952 assert_eq!(
6953 t.hold_reason.as_deref(),
6954 Some("run ended agent-verified no-op"),
6955 "nothing to append when the question was already answered or never asked"
6956 );
6957 }
6958
6959 #[test]
6960 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
6961 crate::run::pin_test_home();
6962 let mut first = run_state(RunStatus::Blocked);
6963 first.id = "20260101-000000-sup1".to_owned();
6964 first.save().unwrap();
6965 let mut second = run_state(RunStatus::Merged);
6966 second.id = "20260101-000000-sup2".to_owned();
6967 second.save().unwrap();
6968
6969 let mut t = task();
6970 t.runs = vec![first.id.clone(), second.id.clone()];
6971 t.status = TaskStatus::Done;
6972
6973 supersede_prior_runs(&t, &crate::run::home());
6974
6975 assert_eq!(
6976 RunState::load(&first.id).unwrap().status,
6977 RunStatus::Superseded,
6978 "the first attempt's Blocked no longer needs anyone's attention"
6979 );
6980 assert_eq!(
6981 RunState::load(&second.id).unwrap().status,
6982 RunStatus::Merged,
6983 "the run that actually succeeded is left exactly as it was"
6984 );
6985 }
6986
6987 #[test]
6988 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
6989 crate::run::pin_test_home();
6995 let mut first = run_state(RunStatus::Blocked);
6996 first.id = "20260101-000000-sup9".to_owned();
6997 first.driver_pid = Some(std::process::id());
7000 first.driver_started_at = Some(
7001 crate::proc::process_started_at(std::process::id())
7002 .expect("this test process's own start time must be queryable"),
7003 );
7004 first.save().unwrap();
7005 let mut second = run_state(RunStatus::Merged);
7006 second.id = "20260101-000000-supa".to_owned();
7007 second.save().unwrap();
7008
7009 let mut t = task();
7010 t.runs = vec![first.id.clone(), second.id.clone()];
7011 t.status = TaskStatus::Done;
7012
7013 supersede_prior_runs(&t, &crate::run::home());
7014
7015 assert_eq!(
7016 RunState::load(&first.id).unwrap().status,
7017 RunStatus::Blocked,
7018 "a live driver_pid means something is still actually working this run, \
7019 even though no daemon claims it - rewriting under it would just be \
7020 undone the next time that process saves"
7021 );
7022 }
7023
7024 #[test]
7025 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
7026 let dir = tempfile::tempdir().unwrap();
7032 let home = dir.path().to_path_buf();
7033 let queue = Queue::at(dir.path().join("queue"));
7034
7035 let mut first = run_state(RunStatus::Blocked);
7036 first.id = "20260101-000000-supd".to_owned();
7037 first.driver_pid = Some(std::process::id());
7038 first.driver_started_at = Some(
7039 crate::proc::process_started_at(std::process::id())
7040 .expect("this test process's own start time must be queryable"),
7041 );
7042 first.save_under(&home).unwrap();
7043 let mut second = run_state(RunStatus::Merged);
7044 second.id = "20260101-000000-supe".to_owned();
7045 second.save_under(&home).unwrap();
7046
7047 let mut t = task();
7048 t.runs = vec![first.id.clone(), second.id.clone()];
7049 t.status = TaskStatus::Done;
7050 queue.put(&mut t).unwrap();
7051
7052 resweep_superseded_attempts(&queue, &home);
7053 assert_eq!(
7054 RunState::load_under(&first.id, &home).unwrap().status,
7055 RunStatus::Blocked,
7056 "still live on the first pass, so still untouched"
7057 );
7058
7059 let mut stale = RunState::load_under(&first.id, &home).unwrap();
7065 stale.driver_started_at = Some("1".to_owned());
7066 stale.save_under(&home).unwrap();
7067
7068 resweep_superseded_attempts(&queue, &home);
7069 assert_eq!(
7070 RunState::load_under(&first.id, &home).unwrap().status,
7071 RunStatus::Superseded,
7072 "the second pass catches up what the first one correctly skipped"
7073 );
7074 }
7075
7076 #[test]
7077 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
7078 crate::run::pin_test_home();
7079 let mut first = run_state(RunStatus::Blocked);
7080 first.id = "20260101-000000-sup3".to_owned();
7081 first.save().unwrap();
7082 let mut second = run_state(RunStatus::Blocked);
7083 second.id = "20260101-000000-sup4".to_owned();
7084 second.save().unwrap();
7085
7086 let mut t = task();
7087 t.runs = vec![first.id.clone(), second.id.clone()];
7088 t.status = TaskStatus::Failed;
7092
7093 supersede_prior_runs(&t, &crate::run::home());
7094
7095 assert_eq!(
7096 RunState::load(&first.id).unwrap().status,
7097 RunStatus::Blocked
7098 );
7099 assert_eq!(
7100 RunState::load(&second.id).unwrap().status,
7101 RunStatus::Blocked
7102 );
7103 }
7104
7105 #[test]
7106 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
7107 crate::run::pin_test_home();
7111 let mut first = run_state(RunStatus::Blocked);
7112 first.id = "20260101-000000-sup5".to_owned();
7113 first.save().unwrap();
7114
7115 let mut t = task();
7116 t.runs = vec![first.id.clone()];
7117 t.status = TaskStatus::Done;
7118
7119 supersede_prior_runs(&t, &crate::run::home());
7120
7121 assert_eq!(
7122 RunState::load(&first.id).unwrap().status,
7123 RunStatus::Blocked,
7124 "a single-attempt task has no earlier run to supersede"
7125 );
7126 }
7127
7128 #[test]
7129 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
7130 crate::run::pin_test_home();
7137 let mut first = run_state(RunStatus::Blocked);
7138 first.id = "20260101-000000-supb".to_owned();
7139 first.save().unwrap();
7140 let mut second = run_state(RunStatus::Failed);
7141 second.id = "20260101-000000-supc".to_owned();
7142 second.save().unwrap();
7143
7144 let mut t = task();
7145 t.runs = vec![first.id.clone(), second.id.clone()];
7146 t.status = TaskStatus::Done;
7147
7148 supersede_prior_runs(&t, &crate::run::home());
7149
7150 assert_eq!(
7151 RunState::load(&first.id).unwrap().status,
7152 RunStatus::Blocked,
7153 "the task's last attempt never landed, so there is nothing here \
7154 actually superseding it"
7155 );
7156 }
7157
7158 #[test]
7159 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
7160 crate::run::pin_test_home();
7164 let mut failed = run_state(RunStatus::Failed);
7165 failed.id = "20260101-000000-sup6".to_owned();
7166 failed.save().unwrap();
7167 let mut noop = run_state(RunStatus::VerifiedNoop);
7168 noop.id = "20260101-000000-sup7".to_owned();
7169 noop.save().unwrap();
7170 let mut winner = run_state(RunStatus::Ready);
7171 winner.id = "20260101-000000-sup8".to_owned();
7172 winner.save().unwrap();
7173
7174 let mut t = task();
7175 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
7176 t.status = TaskStatus::Done;
7177
7178 supersede_prior_runs(&t, &crate::run::home());
7179
7180 assert_eq!(
7181 RunState::load(&failed.id).unwrap().status,
7182 RunStatus::Failed
7183 );
7184 assert_eq!(
7185 RunState::load(&noop.id).unwrap().status,
7186 RunStatus::VerifiedNoop
7187 );
7188 }
7189
7190 fn approval_question(run: &str) -> ask::Question {
7191 ask::Question::new(
7192 run.to_owned(),
7193 land::APPROVAL_NODE.to_owned(),
7194 "land".to_owned(),
7195 "merge?".to_owned(),
7196 String::new(),
7197 vec!["merge".to_owned(), "hold".to_owned()],
7198 )
7199 }
7200
7201 #[test]
7202 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
7203 crate::run::pin_test_home();
7204 let mut state = run_state(RunStatus::Landing);
7205 state.id = "20260101-000000-fre1".to_owned();
7206 state.parked = true;
7207 state.save().unwrap();
7208 ask::Questions::open()
7209 .put(&mut approval_question(&state.id))
7210 .unwrap();
7211
7212 let mut t = task();
7213 t.runs.push(state.id.clone());
7214 assert_eq!(
7215 land_resume_state(&t),
7216 LandResume::StillWaiting,
7217 "nobody has answered and the timeout has not passed"
7218 );
7219 }
7220
7221 #[test]
7222 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
7223 crate::run::pin_test_home();
7224 let mut state = run_state(RunStatus::Gating);
7225 state.id = "20260101-000000-gt01".to_owned();
7226 state.parked = true;
7227 state.config.graph.answer_timeout = 60;
7228 state.github_text = Some(crate::run::GithubTextAsk {
7229 rewrite: None,
7230 fingerprint: "f".into(),
7231 title: true,
7232 body: false,
7233 categories: vec!["title-language".into()],
7234 question: None,
7235 asks: 1,
7236 resolved: false,
7237 chosen_title: None,
7238 choice: None,
7239 });
7240 state.save().unwrap();
7241 let store = ask::Questions::open();
7242 let mut q = ask::Question::new(
7243 state.id.clone(),
7244 crate::github_text::ASK_NODE.to_owned(),
7245 crate::github_text::ASK_SEAT.to_owned(),
7246 "s".into(),
7247 String::new(),
7248 vec!["use fallback".into()],
7249 );
7250 store.put(&mut q).unwrap();
7251 let mut t = task();
7252 t.runs.push(state.id.clone());
7253 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
7254 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7255 store.put(&mut q).unwrap();
7256 assert_eq!(land_resume_state(&t), LandResume::Ready);
7257 assert!(!store.get(&q.id).unwrap().status.open());
7258 state.github_text.as_mut().unwrap().resolved = true;
7260 state.save().unwrap();
7261 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
7262 }
7263
7264 #[test]
7265 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
7266 crate::run::pin_test_home();
7271 let mut state = run_state(RunStatus::Landing);
7272 state.id = "20260101-000000-exp1".to_owned();
7273 state.parked = true;
7274 state.config.graph.answer_timeout = 60;
7275 state.save().unwrap();
7276
7277 let store = ask::Questions::open();
7278 let mut q = approval_question(&state.id);
7279 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7280 store.put(&mut q).unwrap();
7281
7282 let mut t = task();
7283 t.runs.push(state.id.clone());
7284 assert_eq!(
7285 land_resume_state(&t),
7286 LandResume::Ready,
7287 "an expired question must not be waited on forever"
7288 );
7289
7290 let after = store.get(&q.id).unwrap();
7291 assert!(
7292 !after.status.open(),
7293 "the question is abandoned, not silently ignored"
7294 );
7295 assert!(
7296 after.resolution().is_none(),
7297 "an abandoned question is not read as a decision"
7298 );
7299 }
7300
7301 #[test]
7302 fn reclaim_settles_a_running_task_against_its_last_run() {
7303 let mut t = task();
7304 t.start("20260904-000000-4043".to_owned());
7305 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
7306 assert_eq!(
7307 t.status,
7308 TaskStatus::Done,
7309 "a run that actually finished must not stay `running` forever"
7310 );
7311 }
7312
7313 #[test]
7314 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
7315 let mut t = task();
7316 t.start("20260904-000000-4043".to_owned());
7317 t.last_error = Some("earlier trouble".to_owned());
7318 let mut state = run_state(RunStatus::Judging);
7319 state.parked = true;
7320 reclaim(&mut t, Some(state), 1, "en");
7321 assert_eq!(t.status, TaskStatus::Parked);
7322 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
7323 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7324 assert!(t.park_reason.is_some());
7325 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
7326 assert!(!t.fresh_start);
7327 assert!(t.status.runnable());
7328 }
7329
7330 #[test]
7331 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
7332 let mut t = task();
7333 t.start("20260904-000000-4043".to_owned());
7334 t.last_error = Some("earlier trouble".to_owned());
7335 settle(
7336 &mut t,
7337 Verdict {
7338 status: RunStatus::Judging,
7339 left_pr: false,
7340 quota_hit: false,
7341 parked: true,
7342 no_viable_candidates: false,
7343 },
7344 "parked after `judging`",
7345 1,
7346 );
7347 assert_eq!(t.status, TaskStatus::Parked);
7348 assert_eq!(t.attempts, 0);
7349 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7350 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
7351 assert_eq!(t.schema, crate::queue::SCHEMA);
7352 assert!(
7353 crate::notices::task_held(&t).is_none(),
7354 "a park is not news"
7355 );
7356 t.start("20260904-000000-4043".to_owned());
7357 assert_eq!(t.park_reason, None);
7358 }
7359
7360 #[test]
7361 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
7362 let mut t = task();
7366 t.start("20260904-000000-4043".to_owned());
7367 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
7368 assert_eq!(t.status, TaskStatus::Failed);
7369 assert!(t.status.runnable());
7370 }
7371
7372 #[test]
7373 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
7374 let mut t = task();
7375 t.start("20260904-000000-4043".to_owned());
7376 assert_eq!(t.attempts, 1);
7377 let mut run = run_state(RunStatus::Implementing);
7378 run.id = "20260904-000000-4043".to_owned();
7379 run.begin_pass();
7380 let cause = crate::run::ParkCause::Upgrade {
7381 from: "0.121.0".to_owned(),
7382 to: Some("0.121.1".to_owned()),
7383 };
7384 run.end_pass_parked(cause.clone());
7385 run.parked = true;
7386 let verdict = Verdict {
7387 status: run.status,
7388 left_pr: false,
7389 quota_hit: false,
7390 parked: true,
7391 no_viable_candidates: false,
7392 };
7393 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
7394 assert_eq!(t.attempts, 0);
7395 assert_eq!(t.passes.len(), 1);
7396 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7397 assert_eq!(t.passes[0].parked, Some(cause));
7398 assert_eq!(t.passes[0].pass_index, 0);
7399
7400 t.start("20260904-000000-4043".to_owned());
7403 run.begin_pass();
7404 run.status = RunStatus::Failed;
7405 let verdict = Verdict {
7406 status: RunStatus::Failed,
7407 left_pr: false,
7408 quota_hit: false,
7409 parked: false,
7410 no_viable_candidates: false,
7411 };
7412 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
7413 assert_eq!(t.passes.len(), 2);
7414 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7415 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
7416 assert_eq!(t.passes[1].pass_index, 1);
7417 assert_eq!(t.attempts, 1);
7418 }
7419
7420 #[test]
7421 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
7422 let mut t = task();
7423 t.start("20260904-000000-4043".to_owned());
7424 reclaim(&mut t, None, 2, "en");
7425 assert_eq!(t.status, TaskStatus::Held);
7426 assert!(
7427 t.last_error
7428 .as_deref()
7429 .is_some_and(|e| e.contains("running")),
7430 "the operator needs to know why this task was held"
7431 );
7432 }
7433
7434 #[test]
7435 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
7436 let dir = tempfile::tempdir().unwrap();
7437 let queue = Queue::at(dir.path().to_path_buf());
7438
7439 let mut orphaned = task();
7441 orphaned.id = "20260904-000000-orph".to_owned();
7442 orphaned.status = TaskStatus::Running;
7443 orphaned.attempts = 1;
7444 queue.put(&mut orphaned).unwrap();
7445
7446 let mut alive = task();
7447 alive.id = "20260904-000000-live".to_owned();
7448 alive.status = TaskStatus::Running;
7449 alive.attempts = 1;
7450 queue.put(&mut alive).unwrap();
7451 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
7452
7453 let mut queued = task();
7454 queued.id = "20260904-000000-wait".to_owned();
7455 queue.put(&mut queued).unwrap();
7456
7457 let reclaimed = reclaim_orphaned_running(&queue, 2);
7458 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
7459
7460 assert_eq!(
7461 queue.get(&orphaned.id).unwrap().status,
7462 TaskStatus::Held,
7463 "nothing was driving it and there was no run to recover"
7464 );
7465 assert_eq!(
7466 queue.get(&alive.id).unwrap().status,
7467 TaskStatus::Running,
7468 "a live claim must protect the task it belongs to"
7469 );
7470 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
7471 }
7472
7473 fn read_run_under(home: &Path, id: &str) -> RunState {
7479 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
7480 serde_json::from_str(&body).unwrap()
7481 }
7482
7483 #[test]
7484 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
7485 let dir = tempfile::tempdir().unwrap();
7486 let home = dir.path().to_path_buf();
7487 let now = Timestamp::now();
7488 let overrun_seat = || crate::run::ActiveSeat {
7489 node: "implement".to_owned(),
7490 started_at: now - jiff::SignedDuration::new(21_000, 0),
7491 timeout_secs: 3_600,
7492 attempt: 0,
7493 task: None,
7494 command: None,
7495 index: None,
7496 total: None,
7497 };
7498
7499 let mut dead = run_state(RunStatus::Implementing);
7500 dead.id = "20260101-000000-dead".to_owned();
7501 dead.active.insert("impl-A".to_owned(), overrun_seat());
7502 dead.driver_pid = Some(4242);
7505 dead.save_under(&home).unwrap();
7506
7507 let mut alive = run_state(RunStatus::Implementing);
7510 alive.id = "20260101-000000-aliv".to_owned();
7511 alive.active.insert("impl-A".to_owned(), overrun_seat());
7512 alive.save_under(&home).unwrap();
7513 let mut status = Status::new();
7514 status.current = vec![Current {
7515 task: "20260101-000000-task".to_owned(),
7516 run: alive.id.clone(),
7517 }];
7518 write_status_to(&home.join("daemon.json"), &status).unwrap();
7519
7520 let questions = Questions::at(home.join("questions"));
7524 let mut q = ask::Question::new(
7525 dead.id.clone(),
7526 "implement".to_owned(),
7527 "impl-A".to_owned(),
7528 "Which storage backend?".to_owned(),
7529 String::new(),
7530 vec!["SQLite".to_owned(), "Redis".to_owned()],
7531 );
7532 questions.put(&mut q).unwrap();
7533
7534 let abandoned = reclaim_abandoned_runs_with(
7535 &home,
7536 now,
7537 |pid| if pid == 4242 { Some(false) } else { None },
7538 |_| panic!("a query answering Dead outright needs no identity corroboration"),
7539 );
7540 assert_eq!(abandoned, vec![dead.id.clone()]);
7541
7542 let reloaded = read_run_under(&home, &dead.id);
7543 assert_eq!(reloaded.status, RunStatus::Failed);
7544 assert!(reloaded.active.is_empty());
7545 assert!(
7546 !questions.get(&q.id).unwrap().status.open(),
7547 "the failed run's own open question must be settled in the same pass"
7548 );
7549
7550 let still_alive = read_run_under(&home, &alive.id);
7551 assert_eq!(
7552 still_alive.status,
7553 RunStatus::Implementing,
7554 "a live daemon's claim protects it"
7555 );
7556 assert!(!still_alive.active.is_empty());
7557 }
7558
7559 #[test]
7569 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
7570 let dir = tempfile::tempdir().unwrap();
7571 let home = dir.path().to_path_buf();
7572 let now = Timestamp::now();
7573
7574 let mut manual = run_state(RunStatus::Reviewing);
7575 manual.id = "20260101-000000-manl".to_owned();
7576 manual.active.insert(
7577 "review-1".to_owned(),
7578 crate::run::ActiveSeat {
7579 node: "review".to_owned(),
7580 started_at: now - jiff::SignedDuration::new(21_000, 0),
7581 timeout_secs: 3_600,
7582 attempt: 0,
7583 task: None,
7584 command: None,
7585 index: None,
7586 total: None,
7587 },
7588 );
7589 manual.driver_pid = Some(4242);
7593 manual.driver_started_at = Some("1790000000".to_owned());
7594 manual.save_under(&home).unwrap();
7595
7596 let abandoned = reclaim_abandoned_runs_with(
7597 &home,
7598 now,
7599 |pid| if pid == 4242 { Some(true) } else { None },
7600 |pid| {
7601 if pid == 4242 {
7602 Some("1790000000".to_owned())
7603 } else {
7604 None
7605 }
7606 },
7607 );
7608 assert!(
7609 abandoned.is_empty(),
7610 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
7611 );
7612
7613 let reloaded = read_run_under(&home, &manual.id);
7614 assert_eq!(reloaded.status, RunStatus::Reviewing);
7615 assert!(!reloaded.active.is_empty());
7616 }
7617
7618 #[test]
7619 fn an_already_claimed_task_is_skipped_rather_than_failed() {
7620 let dir = tempfile::tempdir().unwrap();
7621 let queue = Queue::at(dir.path().to_path_buf());
7622 let mut only = task();
7623 queue.put(&mut only).unwrap();
7624
7625 let _elsewhere = queue.claim(&only.id).unwrap();
7626 let candidates = runnable(&queue);
7627 assert_eq!(candidates.len(), 1, "the task is still runnable");
7628 assert!(
7629 queue.claim(&candidates[0].id).is_err(),
7630 "the loop cannot take a claim somebody else holds"
7631 );
7632
7633 let after = queue.get(&only.id).unwrap();
7634 assert_eq!(after.status, TaskStatus::Queued);
7635 assert_eq!(
7636 after.attempts, 0,
7637 "losing the race is not an attempt at the task"
7638 );
7639 assert_eq!(after.last_error, None);
7640 }
7641
7642 #[test]
7643 fn the_status_file_round_trips_and_its_heartbeat_advances() {
7644 let dir = tempfile::tempdir().unwrap();
7645 let path = dir.path().join("daemon.json");
7646
7647 let mut status = Status::new();
7648 status.idle = false;
7649 status.completed = 7;
7650 status.current = vec![Current {
7651 task: "20260902-000000-t111".to_owned(),
7652 run: "20260902-000001-r111".to_owned(),
7653 }];
7654 write_status_to(&path, &status).unwrap();
7655 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7656 assert_eq!(first.schema, SCHEMA);
7657 assert_eq!(first.pid, std::process::id());
7658 assert!(!first.idle);
7659 assert_eq!(first.completed, 7);
7660 assert_eq!(first.current, status.current);
7661 assert!(
7662 !path.with_extension("json.tmp").exists(),
7663 "the temp file is renamed, not left behind"
7664 );
7665
7666 std::thread::sleep(Duration::from_millis(5));
7667 status.updated_at = Timestamp::now();
7668 status.polls = 3;
7669 write_status_to(&path, &status).unwrap();
7670 let second: Status =
7671 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7672 assert!(
7673 second.updated_at > first.updated_at,
7674 "a reader can only detect staleness if the heartbeat moves"
7675 );
7676 assert_eq!(
7677 second.started_at, first.started_at,
7678 "the start time is not a heartbeat"
7679 );
7680 assert_eq!(second.polls, 3);
7681 }
7682
7683 #[test]
7684 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
7685 let dir = tempfile::tempdir().unwrap();
7686
7687 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
7688
7689 let mut status = Status::new();
7690 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
7691 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7692 let stale = read_status(dir.path()).unwrap();
7693 assert!(
7694 !stale.running(Timestamp::now()),
7695 "a minute without a heartbeat is a dead daemon, not a busy one"
7696 );
7697 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
7698
7699 status.updated_at = Timestamp::now();
7700 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7701 let fresh = read_status(dir.path()).unwrap();
7702 assert!(fresh.running(Timestamp::now()));
7703 }
7704
7705 #[test]
7706 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
7707 let dir = tempfile::tempdir().unwrap();
7708 let now = Timestamp::now();
7709 let mine = "20260903-080619-01c2";
7710
7711 assert!(
7712 !is_working_on(dir.path(), mine, now),
7713 "no status file means nobody is working on anything"
7714 );
7715
7716 let mut status = Status::new();
7717 status.current = vec![Current {
7718 task: "20260903-080340-0167".to_owned(),
7719 run: mine.to_owned(),
7720 }];
7721 status.updated_at = now;
7722 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7723 assert!(is_working_on(dir.path(), mine, now));
7724 assert!(
7725 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
7726 "a daemon busy with one run is not working on another"
7727 );
7728
7729 status.updated_at = now - jiff::SignedDuration::from_secs(600);
7732 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7733 assert!(
7734 !is_working_on(dir.path(), mine, now),
7735 "a stale heartbeat is a dead daemon, so its run is a leftover"
7736 );
7737 }
7738
7739 #[test]
7740 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
7741 let dir = tempfile::tempdir().unwrap();
7742 let now = Timestamp::now();
7743
7744 assert!(
7745 !is_working_on_short(dir.path(), "01c2", now),
7746 "no status file means nobody is working on anything"
7747 );
7748
7749 let mut status = Status::new();
7750 status.current = vec![Current {
7751 task: "20260903-080340-0167".to_owned(),
7752 run: "20260903-080619-01c2".to_owned(),
7753 }];
7754 status.updated_at = now;
7755 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7756 assert!(
7757 is_working_on_short(dir.path(), "01c2", now),
7758 "the run's short id is the last block of its full id"
7759 );
7760 assert!(
7761 !is_working_on_short(dir.path(), "3cbf", now),
7762 "a daemon busy with one worktree bay is not working on another"
7763 );
7764 }
7765
7766 #[test]
7767 fn a_newer_status_file_still_yields_a_reading() {
7768 let dir = tempfile::tempdir().unwrap();
7769 std::fs::write(
7772 dir.path().join("daemon.json"),
7773 serde_json::json!({
7774 "schema": 2,
7775 "updated_at": Timestamp::now().to_string(),
7776 "idle": true,
7777 "surprise": { "nested": [1, 2, 3] },
7778 })
7779 .to_string(),
7780 )
7781 .unwrap();
7782
7783 let reading = read_status(dir.path()).expect("a forward-compatible read");
7784 assert!(reading.running(Timestamp::now()));
7785 assert!(reading.idle);
7786 assert!(reading.current.is_empty());
7787 }
7788
7789 #[test]
7790 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
7791 let dir = tempfile::tempdir().unwrap();
7797 std::fs::write(
7798 dir.path().join("daemon.json"),
7799 serde_json::json!({
7800 "schema": 1,
7801 "pid": 4242,
7802 "updated_at": Timestamp::now().to_string(),
7803 "idle": false,
7804 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
7805 "completed": 3,
7806 "polls": 9,
7807 })
7808 .to_string(),
7809 )
7810 .unwrap();
7811
7812 let reading = read_status(dir.path()).expect("an older shape must still parse");
7813 assert!(reading.running(Timestamp::now()));
7814 assert_eq!(
7815 reading.current,
7816 vec![Current {
7817 task: "20260902-140501-aaaa".to_owned(),
7818 run: "20260902-140502-bbbb".to_owned(),
7819 }]
7820 );
7821 }
7822
7823 #[test]
7824 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
7825 let dir = tempfile::tempdir().unwrap();
7826 std::fs::write(
7827 dir.path().join("daemon.json"),
7828 serde_json::json!({
7829 "schema": 1,
7830 "updated_at": Timestamp::now().to_string(),
7831 "idle": true,
7832 "current": null,
7833 })
7834 .to_string(),
7835 )
7836 .unwrap();
7837 let with_null = read_status(dir.path()).expect("null must still parse");
7838 assert!(with_null.current.is_empty());
7839
7840 std::fs::write(
7841 dir.path().join("daemon.json"),
7842 serde_json::json!({
7843 "schema": 1,
7844 "updated_at": Timestamp::now().to_string(),
7845 "idle": true,
7846 })
7847 .to_string(),
7848 )
7849 .unwrap();
7850 let absent = read_status(dir.path()).expect("a missing field must still parse");
7851 assert!(absent.current.is_empty());
7852 }
7853
7854 #[test]
7855 fn a_task_without_a_repository_runs_in_the_daemons_default() {
7856 let fallback = Path::new("/default");
7857 let mut blank = task();
7858 blank.repo = PathBuf::new();
7859 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
7860 let mut dot = task();
7861 dot.repo = PathBuf::from(".");
7862 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
7863 assert_eq!(
7864 repo_for(&task(), fallback),
7865 PathBuf::from("/repo"),
7866 "a task that names a repository keeps it"
7867 );
7868 }
7869
7870 #[test]
7871 fn task_implementers_edit_inherits_daemon_config_and_merge() {
7872 let dir = tempfile::tempdir().unwrap();
7873 let path = dir.path().join("explicit.toml");
7874 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
7875 let opts = Opts {
7876 config: Some(path),
7877 merge: Some("none".into()),
7878 ..Default::default()
7879 };
7880 let mut t = task();
7881 t.set_implementers(
7882 2,
7883 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7884 .unwrap(),
7885 )
7886 .unwrap();
7887 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
7888 assert_eq!(cfg.graph.implementers, 2);
7889 assert_eq!(cfg.graph.judges, 1);
7890 assert_eq!(cfg.merge.mode, MergeMode::None);
7891 t.set_implementers(1, &cfg).unwrap();
7892 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
7893 apply_task_implementers(&mut cfg, &t);
7894 assert_eq!(cfg.graph.implementers, 1);
7895 assert_eq!(cfg.graph.judges, 1);
7896 assert_eq!(cfg.merge.mode, MergeMode::None);
7897 }
7898
7899 #[test]
7900 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
7901 let dir = tempfile::tempdir().unwrap();
7902 let repo = dir.path().join("repo");
7903 std::fs::create_dir(&repo).unwrap();
7904 std::fs::write(
7905 repo.join("magi.toml"),
7906 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
7907 )
7908 .unwrap();
7909 let daemon_config = dir.path().join("daemon.toml");
7910 std::fs::write(
7911 &daemon_config,
7912 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
7913 )
7914 .unwrap();
7915 let opts = Opts {
7916 config: Some(daemon_config),
7917 merge: Some("pr".into()),
7918 ..Default::default()
7919 };
7920 let q = Queue::at(dir.path().join("queue"));
7921 let validation: Config =
7922 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7923 .unwrap();
7924 for record in ["{}", r#"{"candidates":2}"#] {
7926 let overrides = serde_json::from_str(record).unwrap();
7927 let filing = crate::direct::Filing {
7928 instruction: "work".into(),
7929 title: "direct".into(),
7930 repo: repo.clone(),
7931 source: crate::queue::Source::Human,
7932 solo: false,
7933 overrides,
7934 review_of: None,
7935 };
7936 let filed =
7937 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
7938 let crate::direct::Filed::Standalone {
7939 mut task,
7940 claim: _claim,
7941 } = filed
7942 else {
7943 panic!("local filing")
7944 };
7945 let before = prepare_for(&repo, &opts, &task).unwrap();
7946 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
7947 assert_eq!(
7948 before.merge.mode,
7949 MergeMode::None,
7950 "daemon --merge must not leak"
7951 );
7952 for n in [2, 1, 2] {
7953 task.set_implementers(n, &validation).unwrap();
7954 q.put(&mut task).unwrap();
7955 task = q.get(&task.id).unwrap();
7956 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
7957 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
7958 apply_task_implementers(&mut cfg, &task);
7959 assert_eq!(cfg.graph.implementers, usize::from(n));
7960 assert_eq!(cfg.graph.judges, 2);
7961 assert_eq!(cfg.merge.mode, MergeMode::None);
7962 }
7963 }
7964 }
7965
7966 #[test]
7967 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
7968 let cfg: Config =
7969 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7970 .unwrap();
7971 let mut t = task();
7972 t.review_of = Some("review-target".into());
7973 t.hold_manual(Some("retry".into()));
7974 let mut previous = RunState::new(
7975 PathBuf::from("/repo"),
7976 "main".into(),
7977 "abc1234".into(),
7978 "review".into(),
7979 cfg.clone(),
7980 );
7981 previous.config.graph.implementers = 2;
7982 for count in [1, 3] {
7983 t.set_implementers(count, &cfg).unwrap();
7984 t.release();
7985 assert_eq!(t.review_of.as_deref(), Some("review-target"));
7986 for proposed in [
7987 Starter::Resume(previous.id.clone()),
7988 Starter::Review("recovery-target".into()),
7989 ] {
7990 assert_eq!(
7991 starter_with_implementers(&t, Some(&previous), proposed),
7992 Starter::Review("review-target".into())
7993 );
7994 }
7995 let mut configured = cfg.clone();
7996 apply_task_implementers(&mut configured, &t);
7997 assert_eq!(configured.graph.implementers, usize::from(count));
7998 assert_eq!(previous.config.graph.implementers, 2);
7999 }
8000 }
8001
8002 #[test]
8003 fn task_implementers_dispatch_and_resume_agree() {
8004 let mut t = task();
8005 let cfg: crate::config::Config =
8006 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8007 .unwrap();
8008 let mut run = RunState::new(
8009 PathBuf::from("/repo"),
8010 "main".into(),
8011 "abc1234".into(),
8012 "work".into(),
8013 cfg.clone(),
8014 );
8015 run.config.graph.implementers = 2;
8016 for n in [2, 1, 3, 2] {
8017 t.set_implementers(n, &cfg).unwrap();
8018 let mut dispatched = cfg.clone();
8019 apply_task_implementers(&mut dispatched, &t);
8020 assert_eq!(dispatched.graph.implementers, usize::from(n));
8021 assert_eq!(implementers_match(&t, &run), n == 2);
8022 for proposed in [
8023 Starter::Resume(run.id.clone()),
8024 Starter::Review("old-branch".into()),
8025 ] {
8026 let expected = if n == 2 {
8027 proposed.clone()
8028 } else {
8029 Starter::Start
8030 };
8031 assert_eq!(
8032 starter_with_implementers(&t, Some(&run), proposed),
8033 expected
8034 );
8035 }
8036 assert_eq!(
8037 run.config.graph.implementers, 2,
8038 "history must not be rewritten"
8039 );
8040 }
8041 t.solo = false;
8042 t.overrides = None;
8043 assert!(implementers_match(&t, &run));
8044 }
8045
8046 #[test]
8047 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
8048 let mut solo_cfg = Config::default();
8054 solo_cfg.graph.implementers = 3;
8055 let mut solo_task = task();
8056 solo_task.solo = true;
8057 apply_task_implementers(&mut solo_cfg, &solo_task);
8058 assert_eq!(solo_cfg.graph.implementers, 1);
8059
8060 let mut plain_cfg = Config::default();
8061 plain_cfg.graph.implementers = 3;
8062 let plain_task = task();
8063 assert!(!plain_task.solo);
8064 apply_task_implementers(&mut plain_cfg, &plain_task);
8065 assert_eq!(
8066 plain_cfg.graph.implementers, 3,
8067 "a task that did not ask to run alone keeps the config's candidates"
8068 );
8069 }
8070
8071 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
8072 QuotaLoss {
8073 seat: seat.into(),
8074 node: "judge".into(),
8075 at: at.parse().unwrap(),
8076 reset: reset.map(str::to_string),
8077 }
8078 }
8079
8080 #[test]
8081 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
8082 let old: Vec<QuotaLoss> = (1..=4)
8083 .map(|i| {
8084 loss(
8085 &format!("judge-{i}"),
8086 "2026-09-23T05:23:00Z",
8087 Some("2:40pm (Asia/Tokyo)"),
8088 )
8089 })
8090 .collect();
8091 let fresh = losses_this_attempt(&old, &old);
8092 assert!(fresh.is_empty());
8093 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
8094 }
8096
8097 #[test]
8098 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
8099 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
8100 let now = Timestamp::now();
8101 let mut after = old.clone();
8102 after.push(loss("judge-2", &now.to_string(), None));
8103 let fresh = losses_this_attempt(&old, &after);
8104 assert_eq!(fresh, vec![after[1].clone()]);
8105 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
8106 assert_eq!(
8107 until,
8108 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
8109 );
8110 }
8111
8112 #[test]
8113 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
8114 let before = vec![
8117 loss("judge-1", "2026-09-23T05:23:00Z", None),
8118 loss("judge-2", "2026-09-23T05:24:00Z", None),
8119 ];
8120 let after = vec![
8121 loss("judge-2", "2026-09-23T05:24:00Z", None),
8122 loss("judge-1", "2026-09-24T01:00:00Z", None),
8123 ];
8124 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
8125 }
8126
8127 #[test]
8128 fn merge_overrides_are_parsed_or_refused() {
8129 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
8130 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
8131 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
8132 assert!(merge_mode("squash").is_err());
8133 }
8134
8135 #[test]
8136 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
8137 let now = Timestamp::now();
8138 let fallback = Duration::from_secs(300);
8139 let cap = Duration::from_secs(1800);
8140
8141 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
8143
8144 let soon = now + jiff::SignedDuration::from_secs(600);
8146 assert_eq!(
8147 quota_wait(Some(soon), now, fallback, cap),
8148 Duration::from_secs(600)
8149 );
8150
8151 let past = now - jiff::SignedDuration::from_secs(60);
8154 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
8155
8156 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
8159 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
8160 }
8161
8162 #[test]
8163 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
8164 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8165
8166 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
8167 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
8168
8169 let already_past =
8173 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
8174 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
8175
8176 assert!(
8177 parse_reset_hint("session limit reached", now, now).is_none(),
8178 "free text with no recognised shape is not guessed at"
8179 );
8180 assert!(
8181 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
8182 "an unresolvable zone name is not guessed at either"
8183 );
8184 }
8185
8186 #[test]
8187 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
8188 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8189
8190 let at = parse_reset_hint(
8191 "You've hit your usage limit. Visit \
8192 https://chatgpt.com/codex/settings/usage to purchase more \
8193 credits or try again at Sep 19th, 2026 5:10 PM.",
8194 now,
8195 now,
8196 )
8197 .expect("the codex reset wording is a recognised shape");
8198 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
8199
8200 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
8205 .expect("an explicit year needs no rollover");
8206 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
8207
8208 assert!(
8209 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
8210 "a two-digit year is not the documented shape and is not guessed at"
8211 );
8212 assert!(
8213 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
8214 "a four-letter month name is not the documented three-letter abbreviation"
8215 );
8216 assert!(
8217 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
8218 "an explicit zone on the dated shape is a format nobody has \
8219 documented, and is refused rather than guessed at as UTC"
8220 );
8221 }
8222
8223 #[test]
8224 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
8225 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
8226 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
8227
8228 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
8229 assert_eq!(at.as_second() - recorded.as_second(), 3769);
8230
8231 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
8232 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
8233
8234 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
8235 assert!(
8236 parse_reset_hint(bad, now, recorded).is_none(),
8237 "{bad:?} must not be guessed at"
8238 );
8239 }
8240 }
8241
8242 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
8246 let config = dir.join("magi.toml");
8247 std::fs::write(
8248 &config,
8249 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
8250 )
8251 .unwrap();
8252 let opts = Opts {
8253 poll: Duration::from_secs(30),
8254 config: Some(config),
8255 repo: dir.join("repo"),
8259 ..Opts::default()
8260 };
8261 let home = dir.join("home");
8270 let worktrees = dir.join("wt");
8271 (
8272 opts,
8273 Queue::at(dir.join("queue")),
8274 home.join("daemon.json"),
8275 home,
8276 worktrees,
8277 )
8278 }
8279
8280 #[test]
8281 fn a_stop_is_idempotent_and_once_set_stays_set() {
8282 let stop = Stop::new();
8283 assert!(!stop.stopped());
8284
8285 stop.stop();
8286 assert!(stop.stopped());
8287 stop.stop();
8288 assert!(stop.stopped(), "a second stop is not a toggle");
8289
8290 let shared = stop.clone();
8291 assert!(
8292 shared.stopped(),
8293 "a clone is the same stop; that is how the loop and its caller share one"
8294 );
8295 }
8296
8297 #[test]
8298 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
8299 let stop = Stop::new();
8300 stop.enter();
8301 assert!(
8302 !stop.finishing(),
8303 "a busy loop nobody has asked to stop is just running"
8304 );
8305
8306 stop.stop();
8307 assert!(
8308 stop.finishing(),
8309 "a stop asked for mid-run has not landed until the run is settled"
8310 );
8311
8312 stop.exit();
8313 assert!(
8314 !stop.finishing(),
8315 "once the run is settled the stop has landed and there is nothing to finish"
8316 );
8317 }
8318
8319 #[test]
8320 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
8321 let stop = Stop::new();
8322 stop.enter();
8323 stop.enter();
8324 stop.stop();
8325 assert!(stop.finishing(), "two runs still in flight");
8326
8327 stop.exit();
8328 assert!(
8329 stop.finishing(),
8330 "one run finished, but a sibling is still working"
8331 );
8332
8333 stop.exit();
8334 assert!(
8335 !stop.finishing(),
8336 "the last run out is what actually lands the stop"
8337 );
8338 }
8339
8340 #[tokio::test]
8341 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
8342 let dir = tempfile::tempdir().unwrap();
8343 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8344 let stop = Stop::new();
8345 stop.stop();
8346
8347 let began = std::time::Instant::now();
8348 tokio::time::timeout(
8349 Duration::from_secs(2),
8350 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8351 )
8352 .await
8353 .expect("a stopped loop must return, not sit out its poll interval")
8354 .expect("the loop's own setup and teardown must not fail");
8355 assert!(
8356 began.elapsed() < opts.poll,
8357 "returned only after {:?}, which is a poll interval, not a stop",
8358 began.elapsed()
8359 );
8360 }
8361
8362 #[tokio::test]
8363 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
8364 let dir = tempfile::tempdir().unwrap();
8365 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8366 let stop = Stop::new();
8367
8368 let asker = {
8371 let stop = stop.clone();
8372 tokio::spawn(async move {
8373 tokio::time::sleep(Duration::from_millis(20)).await;
8374 stop.stop();
8375 })
8376 };
8377
8378 let began = std::time::Instant::now();
8379 tokio::time::timeout(
8380 Duration::from_secs(2),
8381 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8382 )
8383 .await
8384 .expect("a stop asked for while idle must wake the wait")
8385 .expect("the loop's own setup and teardown must not fail");
8386 asker.await.unwrap();
8387 assert!(
8388 began.elapsed() < opts.poll,
8389 "returned only after {:?}, so the stop waited on the sleep",
8390 began.elapsed()
8391 );
8392 }
8393
8394 #[tokio::test]
8395 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
8396 let dir = tempfile::tempdir().unwrap();
8397 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8398 let stop = Stop::new();
8399 stop.stop();
8400
8401 tokio::time::timeout(
8402 Duration::from_secs(2),
8403 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8404 )
8405 .await
8406 .expect("a stopped loop must return")
8407 .expect("the loop's own setup and teardown must not fail");
8408
8409 assert!(
8410 home.is_dir(),
8411 "the loop did publish a status file, so its removal is the teardown and not an absence"
8412 );
8413 assert!(
8414 !status_file.exists(),
8415 "a stopped loop clears its status file"
8416 );
8417 assert!(
8418 read_status(&home).is_none(),
8419 "a reader must see no daemon at all, not a heartbeat that merely stopped"
8420 );
8421 }
8422
8423 #[tokio::test]
8424 async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
8425 let dir = tempfile::tempdir().unwrap();
8426 let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8427 opts.once = true;
8428
8429 let mut settled = RunState::new(
8430 dir.path().join("repo"),
8431 "main".to_owned(),
8432 "abc1234".to_owned(),
8433 "fixture".to_owned(),
8434 Config::default(),
8435 );
8436 settled.status = RunStatus::Ready;
8437 let run_dir = home.join("runs").join(&settled.id);
8438 std::fs::create_dir_all(&run_dir).unwrap();
8439 std::fs::write(
8440 run_dir.join("run.json"),
8441 serde_json::to_string_pretty(&settled).unwrap(),
8442 )
8443 .unwrap();
8444 let questions = Questions::at(home.join("questions"));
8445 let mut question = ask::Question::new(
8446 settled.id.clone(),
8447 "review".to_owned(),
8448 "reviewer-1".to_owned(),
8449 "Continue?".to_owned(),
8450 String::new(),
8451 Vec::new(),
8452 );
8453 questions.put(&mut question).unwrap();
8454
8455 drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
8456 .await
8457 .unwrap();
8458
8459 assert_eq!(
8460 questions.get(&question.id).unwrap().status,
8461 ask::QuestionStatus::Abandoned,
8462 "an empty --once drain still performs startup question cleanup"
8463 );
8464 }
8465
8466 #[test]
8467 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
8468 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8469
8470 assert!(
8471 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
8472 "never checked before: due at once"
8473 );
8474
8475 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
8476 assert!(
8477 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
8478 "well inside the interval: not due yet"
8479 );
8480
8481 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
8482 assert!(
8483 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
8484 "exactly at the edge: not yet due, same convention as `clean::due`"
8485 );
8486
8487 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8488 assert!(
8489 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
8490 "past the interval: due again"
8491 );
8492 }
8493
8494 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
8499 let config = dir.join("magi.toml");
8500 std::fs::write(
8506 &config,
8507 format!(
8508 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
8509 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
8510 cache_dir.display()
8511 ),
8512 )
8513 .unwrap();
8514 Opts {
8515 config: Some(config),
8516 repo: dir.join("repo"),
8517 ..Opts::default()
8518 }
8519 }
8520
8521 #[tokio::test]
8522 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
8523 let dir = tempfile::tempdir().unwrap();
8524 let home = dir.path().join("home");
8525 let cache_dir = dir.path().join("cache");
8526 std::fs::create_dir_all(&cache_dir).unwrap();
8527 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8528 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8529
8530 let running = Stop::new();
8533 let mut last_checked = None;
8534 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8535 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
8536 .await;
8537 assert_eq!(
8538 crate::disk::dir_size(&cache_dir),
8539 0,
8540 "over the cap on the first check ever: pruned at once, no idle queue required"
8541 );
8542 assert_eq!(last_checked, Some(t0));
8543
8544 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
8546 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
8547 maybe_prune_cache_between_runs(
8548 &opts.repo,
8549 &opts,
8550 &home,
8551 &running,
8552 &mut last_checked,
8553 too_soon,
8554 )
8555 .await;
8556 assert_eq!(
8557 crate::disk::dir_size(&cache_dir),
8558 10,
8559 "too soon since the last check: left alone rather than rescanned every call"
8560 );
8561 assert_eq!(
8562 last_checked,
8563 Some(t0),
8564 "an idle check does not reset the clock"
8565 );
8566
8567 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8569 maybe_prune_cache_between_runs(
8570 &opts.repo,
8571 &opts,
8572 &home,
8573 &running,
8574 &mut last_checked,
8575 due_again,
8576 )
8577 .await;
8578 assert_eq!(
8579 crate::disk::dir_size(&cache_dir),
8580 0,
8581 "due again: pruned back under the cap"
8582 );
8583 }
8584
8585 #[tokio::test]
8593 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
8594 let dir = tempfile::tempdir().unwrap();
8595 let home = dir.path().join("home");
8596 let cache_dir = dir.path().join("cache");
8597 std::fs::create_dir_all(&cache_dir).unwrap();
8598 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8599 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8600
8601 let stop = Stop::new();
8602 stop.stop();
8603 assert!(
8604 !stop.finishing(),
8605 "no run is in flight at a between-runs boundary, so nothing else \
8606 would tell the operator this stop had not taken effect yet"
8607 );
8608
8609 let mut last_checked = None;
8610 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8611 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
8612 .await;
8613 assert_eq!(
8614 crate::disk::dir_size(&cache_dir),
8615 10,
8616 "over its cap, and due for the first check ever, but a stop outranks \
8617 it: the cap is a standing policy the next start measures again"
8618 );
8619 assert_eq!(
8620 last_checked, None,
8621 "a check that never happened must not claim the interval"
8622 );
8623 }
8624
8625 #[tokio::test]
8636 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
8637 let dir = tempfile::tempdir().unwrap();
8638 let cache_dir = dir.path().join("cache");
8639 std::fs::create_dir_all(&cache_dir).unwrap();
8640 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
8641
8642 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
8643 opts.poll = Duration::from_millis(20);
8644 opts.max_attempts = 1_000;
8645
8646 let queue = Queue::at(dir.path().join("queue"));
8647 for i in 0..128 {
8648 let mut task = Task::new(
8649 format!("queued work {i}"),
8650 format!("queued work {i}"),
8651 opts.repo.clone(),
8652 Source::Human,
8653 );
8654 queue.put(&mut task).unwrap();
8655 }
8656
8657 let home = dir.path().join("home");
8658 let worktrees = dir.path().join("wt");
8659 let status_file = home.join("daemon.json");
8660 let stop = Stop::new();
8661 let stopper = {
8662 let stop = stop.clone();
8663 let queue = Queue::at(dir.path().join("queue"));
8664 tokio::spawn(async move {
8665 for _ in 0..2_000 {
8669 if queue.list().iter().filter(|t| t.attempts == 1).count() >= 2 {
8670 break;
8671 }
8672 tokio::time::sleep(Duration::from_millis(5)).await;
8673 }
8674 stop.stop();
8675 })
8676 };
8677
8678 tokio::time::timeout(
8679 Duration::from_secs(20),
8680 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8681 )
8682 .await
8683 .expect("the loop must not hang on a queue that keeps producing failing work")
8684 .expect("the loop's own setup and teardown must not fail");
8685 stopper.await.unwrap();
8686
8687 let after = queue.list();
8688 let completed = after.iter().filter(|t| t.attempts == 1).count();
8689 assert!(
8690 completed >= 2,
8691 "the harness must have processed multiple tasks to exercise a busy queue \
8692 (got {completed} completed task(s))"
8693 );
8694 assert!(
8695 after.iter().any(|t| t.status.runnable()),
8696 "unprocessed tasks must remain: the external stop, rather than an \
8697 exhausted backlog, must end the test"
8698 );
8699 assert!(
8700 after
8701 .iter()
8702 .filter(|t| t.attempts > 0)
8703 .all(|t| { t.attempts == 1 && t.status == TaskStatus::Held }),
8704 "a busy backlog must not retry a task's unchanged startup failure"
8705 );
8706
8707 assert_eq!(
8708 crate::disk::dir_size(&cache_dir),
8709 0,
8710 "an oversized cache must not be left to grow unboundedly just because the \
8711 queue kept the loop busy the whole time"
8712 );
8713 }
8714
8715 #[test]
8716 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
8717 let dir = tempfile::tempdir().unwrap();
8718 let queue = Queue::at(dir.path().join("queue"));
8719 let questions = Questions::at(dir.path().join("questions"));
8720 let mut task = task();
8721 queue.put(&mut task).unwrap();
8722
8723 let mut task_question = ask::Question::new(
8724 task.id.clone(),
8725 crate::conduct::NODE.to_owned(),
8726 "conduct".to_owned(),
8727 "Which backend?".to_owned(),
8728 String::new(),
8729 Vec::new(),
8730 );
8731 questions.put(&mut task_question).unwrap();
8732 task.block(vec![task_question.id.clone()], None);
8733 queue.put(&mut task).unwrap();
8734
8735 let mut run_question = ask::Question::new(
8736 "20260101-000000-run1".to_owned(),
8737 "review".to_owned(),
8738 "reviewer-1".to_owned(),
8739 "Run question".to_owned(),
8740 String::new(),
8741 Vec::new(),
8742 );
8743 questions.put(&mut run_question).unwrap();
8744
8745 let mut coincidental = ask::Question::new(
8750 task.id.clone(),
8751 "review".to_owned(),
8752 "reviewer-1".to_owned(),
8753 "Unrelated review question".to_owned(),
8754 String::new(),
8755 Vec::new(),
8756 );
8757 questions.put(&mut coincidental).unwrap();
8758
8759 reconcile_task_questions(&queue, &questions);
8760 assert!(questions.get(&task_question.id).unwrap().status.open());
8761 assert!(questions.get(&run_question.id).unwrap().status.open());
8762 assert!(questions.get(&coincidental.id).unwrap().status.open());
8763
8764 task.release();
8765 queue.put(&mut task).unwrap();
8766 reconcile_task_questions(&queue, &questions);
8767 assert_eq!(
8768 questions.get(&task_question.id).unwrap().status,
8769 ask::QuestionStatus::Abandoned
8770 );
8771 assert!(
8772 questions.get(&run_question.id).unwrap().status.open(),
8773 "run questions remain the run janitor's responsibility"
8774 );
8775 assert!(
8776 questions.get(&coincidental.id).unwrap().status.open(),
8777 "a non-conductor question must not be abandoned just because its \
8778 run id coincides with a task id"
8779 );
8780 }
8781
8782 #[test]
8783 fn a_freshly_started_running_task_is_never_stalled() {
8784 let dir = tempfile::tempdir().unwrap();
8785 let mut t = task();
8786 t.start("run-1".to_owned());
8787 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
8790 }
8791
8792 #[test]
8793 fn a_long_running_task_with_no_live_daemon_is_stalled() {
8794 let dir = tempfile::tempdir().unwrap();
8795 let mut t = task();
8796 t.start("run-1".to_owned());
8797 t.updated_at = Timestamp::now()
8798 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8799 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
8800 assert_eq!(
8801 stalled_tasks(
8802 &Queue::at(dir.path().join("q")),
8803 dir.path(),
8804 Timestamp::now()
8805 )
8806 .len(),
8807 0,
8808 "the task was never written to this queue"
8809 );
8810 }
8811
8812 #[test]
8813 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
8814 let dir = tempfile::tempdir().unwrap();
8815 let mut t = task();
8816 t.id = "20260903-080340-0167".to_owned();
8817 t.start("20260903-080619-01c2".to_owned());
8818 t.updated_at = Timestamp::now()
8819 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8820
8821 let mut status = Status::new();
8822 status.current = vec![Current {
8823 task: t.id.clone(),
8824 run: "20260903-080619-01c2".to_owned(),
8825 }];
8826 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8827
8828 assert!(
8829 !is_stalled(&t, dir.path(), Timestamp::now()),
8830 "a live daemon's own heartbeat rules out stalled, however long the task has run"
8831 );
8832 }
8833
8834 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
8838 let path = queue.path_of(id);
8839 let body = std::fs::read_to_string(&path).unwrap();
8840 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
8841 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
8842 v["updated_at"] = serde_json::Value::String(old.to_string());
8843 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
8844 }
8845
8846 #[test]
8847 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
8848 let dir = tempfile::tempdir().unwrap();
8861 let queue = Queue::at(dir.path().join("queue"));
8862 let home = dir.path().join("home");
8863
8864 let mut t = task();
8865 t.id = "20260101-000001-lock".to_owned();
8866 t.start("run-1".to_owned());
8867 queue.put(&mut t).unwrap();
8868 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
8869 std::fs::write(
8870 dir.path().join("queue").join(format!("{}.lock", t.id)),
8871 "not a pid",
8872 )
8873 .unwrap();
8874
8875 let now = Timestamp::now();
8876 assert!(
8877 reclaim_orphaned_running(&queue, 2).is_empty(),
8878 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
8879 and reclaim must leave the task alone"
8880 );
8881 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8882
8883 let stalled = stalled_tasks(&queue, &home, now);
8884 assert_eq!(
8885 stalled.len(),
8886 1,
8887 "reclaim's inability to claim it yet must not hide it from the conductor"
8888 );
8889 assert_eq!(stalled[0].id, t.id);
8890 }
8891
8892 #[test]
8893 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
8894 let dir = tempfile::tempdir().unwrap();
8895 crate::run::set_home(dir.path().join("run-home"));
8896 let queue = Queue::at(dir.path().join("queue"));
8897 let home = dir.path().join("home");
8898 let questions = Questions::at(dir.path().join("questions"));
8899
8900 let mut t = task();
8901 t.id = "20260101-000003-dead".to_owned();
8902 t.start("missing-run".to_owned());
8903 queue.put(&mut t).unwrap();
8904 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
8905
8906 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
8909 assert_eq!(
8910 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
8911 [&t.id]
8912 );
8913 assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
8914 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
8915
8916 crate::conduct::apply(
8919 &queue,
8920 &questions,
8921 &crate::conduct::Verdict {
8922 decisions: vec![crate::conduct::Decision {
8923 id: t.id.clone(),
8924 recovery: Some(crate::conduct::Recovery::Requeue),
8925 ..crate::conduct::Decision::default()
8926 }],
8927 },
8928 )
8929 .unwrap();
8930 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
8931 }
8932
8933 #[test]
8934 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
8935 let dir = tempfile::tempdir().unwrap();
8936 let queue = Queue::at(dir.path().join("queue"));
8937 let home = dir.path().join("home");
8938
8939 let mut fresh = task();
8940 fresh.id = "20260101-000001-aaaa".to_owned();
8941 fresh.start("run-1".to_owned());
8942 queue.put(&mut fresh).unwrap();
8943
8944 let mut old = task();
8945 old.id = "20260101-000002-bbbb".to_owned();
8946 old.start("run-2".to_owned());
8947 queue.put(&mut old).unwrap();
8948 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
8949
8950 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
8951 assert_eq!(stalled.len(), 1);
8952 assert_eq!(stalled[0].id, old.id);
8953 }
8954
8955 #[test]
8956 fn queued_and_finished_task_views_partition_by_status() {
8957 let dir = tempfile::tempdir().unwrap();
8958 let queue = Queue::at(dir.path().join("queue"));
8959
8960 let mut queued = task();
8961 queued.id = "20260101-000001-aaaa".to_owned();
8962 queue.put(&mut queued).unwrap();
8963
8964 let mut failed = task();
8965 failed.id = "20260101-000002-bbbb".to_owned();
8966 failed.start("run-1".to_owned());
8967 failed.fail("gate red", 5);
8968 queue.put(&mut failed).unwrap();
8969
8970 let mut held = task();
8971 held.id = "20260101-000003-cccc".to_owned();
8972 held.hold_machine(None);
8973 queue.put(&mut held).unwrap();
8974
8975 let mut running = task();
8976 running.id = "20260101-000004-dddd".to_owned();
8977 running.start("run-2".to_owned());
8978 queue.put(&mut running).unwrap();
8979
8980 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
8981 assert_eq!(queued_ids, [queued.id.clone()]);
8982
8983 let mut finished_ids: Vec<String> =
8984 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
8985 finished_ids.sort_unstable();
8986 let mut want = vec![failed.id.clone(), held.id.clone()];
8987 want.sort_unstable();
8988 assert_eq!(finished_ids, want);
8989 }
8990
8991 #[test]
8992 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
8993 for short in [false, true] {
8994 let dir = tempfile::tempdir().unwrap();
8995 let queue = Queue::at(dir.path().join("queue"));
8996 let questions = ask::Questions::at(dir.path().join("questions"));
8997 let mut running = task();
8998 running.id = "20261013-192213-feb2".into();
8999 running.start("existing-run".into());
9000 queue.put(&mut running).unwrap();
9001 let mut middle = task();
9002 middle.id = "20261013-193611-ca5b".into();
9003 middle.block(
9004 vec![if short {
9005 "feb2".into()
9006 } else {
9007 running.id.clone()
9008 }],
9009 None,
9010 );
9011 queue.put(&mut middle).unwrap();
9012 let mut waiting = task();
9013 waiting.id = "20261013-202849-ee37".into();
9014 waiting.block(
9015 vec![if short {
9016 "ca5b".into()
9017 } else {
9018 middle.id.clone()
9019 }],
9020 Some("wait for implementation".into()),
9021 );
9022 queue.put(&mut waiting).unwrap();
9023 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
9024 let inv = crate::blockers::Inventory::load(&queue, &questions);
9025 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
9026 assert!(inv.stuck().is_empty());
9027 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9028 assert!(report.quarantined.is_empty());
9029 assert!(report.asked.is_empty());
9030 resolve_blockers(&queue, &questions);
9031 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9032 assert!(questions.list().is_empty());
9033 running.succeed();
9036 queue.put(&mut running).unwrap();
9037 resolve_blockers(&queue, &questions);
9038 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
9039 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9040 middle = queue.get(&middle.id).unwrap();
9041 middle.succeed();
9042 queue.put(&mut middle).unwrap();
9043 resolve_blockers(&queue, &questions);
9044 let after = queue.get(&waiting.id).unwrap();
9045 assert_eq!(after.status, TaskStatus::Queued);
9046 assert!(after.blocked_by.is_empty());
9047 assert_eq!(after.instruction, waiting.instruction);
9048 }
9049 }
9050
9051 #[test]
9052 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
9053 for future_schema in [false, true] {
9054 let name = "20261013-193611-ca5b";
9055 let mut future = task();
9056 future.id = name.into();
9057 future.schema = crate::queue::SCHEMA + 1;
9058 let contents = if future_schema {
9059 serde_json::to_string(&future).unwrap()
9060 } else {
9061 "corrupt".into()
9062 };
9063 let dir = tempfile::tempdir().unwrap();
9064 let queue = Queue::at(dir.path().join("queue"));
9065 let questions = ask::Questions::at(dir.path().join("questions"));
9066 let mut waiting = task();
9067 waiting.id = "20261013-202849-ee37".into();
9068 waiting.block(vec!["ca5b".into()], None);
9069 queue.put(&mut waiting).unwrap();
9070 std::fs::write(queue.path_of(name), contents).unwrap();
9071 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
9072 assert!(
9073 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
9074 );
9075 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9076 assert!(report.asked.is_empty());
9077 resolve_blockers(&queue, &questions);
9078 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9079 assert!(questions.list().is_empty());
9080 }
9081 let dir = tempfile::tempdir().unwrap();
9082 let queue = Queue::at(dir.path().join("queue"));
9083 let questions = ask::Questions::at(dir.path().join("questions"));
9084 let mut waiting = task();
9085 waiting.id = "20261013-202849-ee37".into();
9086 waiting.block(vec!["ca5b".into()], None);
9087 queue.put(&mut waiting).unwrap();
9088 let mut done = task();
9089 done.id = "20261013-193611-ca5b".into();
9090 done.succeed();
9091 queue.put(&mut done).unwrap();
9092 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
9095 assert!(queue.get("ca5b").is_err());
9096 assert_eq!(
9097 crate::queue::dependency(&queue, &questions, "ca5b"),
9098 crate::queue::Dependency::Ambiguous
9099 );
9100 resolve_blockers(&queue, &questions);
9101 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9102 assert!(report.asked.is_empty());
9103 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9104 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
9105 }
9106
9107 #[test]
9108 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
9109 for held in [false, true] {
9110 for question in [false, true] {
9111 for reference in ["ca5b", "20261013-193611-ca5b"] {
9112 let dir = tempfile::tempdir().unwrap();
9113 let queue = Queue::at(dir.path().join("queue"));
9114 let questions = ask::Questions::at(dir.path().join("questions"));
9115 let mut prerequisite = task();
9116 prerequisite.id = "20261013-193611-ca5b".into();
9117 prerequisite.start("existing-run".into());
9118 queue.put(&mut prerequisite).unwrap();
9119 let mut waiting = task();
9120 waiting.id = "20261013-202849-ee37".into();
9121 if held {
9122 waiting.hold_machine(Some("waiting for separate consent".into()));
9123 }
9124 queue.put(&mut waiting).unwrap();
9125 let reason = "Deleting saved data requires separate owner consent";
9126 crate::conduct::apply(
9127 &queue,
9128 &questions,
9129 &crate::conduct::Verdict {
9130 decisions: vec![crate::conduct::Decision {
9131 id: waiting.id.clone(),
9132 blocked_by: vec![reference.into()],
9133 question: question.then(|| "May the saved data be deleted?".into()),
9134 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
9135 reason: Some(reason.into()),
9136 ..crate::conduct::Decision::default()
9137 }],
9138 },
9139 )
9140 .unwrap();
9141 let before = queue.get(&waiting.id).unwrap();
9142 if question {
9143 assert_eq!(before.status, TaskStatus::Blocked);
9144 let q = questions.get(&before.blocked_by[0]).unwrap();
9145 assert_eq!(q.summary, "May the saved data be deleted?");
9146 assert!(q.status.open());
9147 } else {
9148 assert_eq!(before.status, TaskStatus::Held);
9149 let recorded_reason = before.hold_reason.as_deref().unwrap();
9150 assert!(recorded_reason.starts_with(reason));
9151 if held {
9152 assert!(recorded_reason.contains("waiting for separate consent"));
9153 }
9154 }
9155 prerequisite.succeed();
9156 queue.put(&mut prerequisite).unwrap();
9157 resolve_blockers(&queue, &questions);
9158 let after = queue.get(&waiting.id).unwrap();
9159 assert_eq!(after.status, before.status);
9160 assert_eq!(after.blocked_by, before.blocked_by);
9161 assert_eq!(after.hold_reason, before.hold_reason);
9162 assert_eq!(after.instruction, waiting.instruction);
9163 if question {
9164 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
9165 }
9166 assert!(
9167 queue.next_runnable().is_none(),
9168 "dependency completion is not consent"
9169 );
9170 }
9171 }
9172 }
9173 }
9174
9175 #[test]
9176 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
9177 let dir = tempfile::tempdir().unwrap();
9178 let queue = Queue::at(dir.path().join("queue"));
9179 let questions = ask::Questions::at(dir.path().join("questions"));
9180
9181 let mut dep = task();
9182 dep.id = "20260101-000001-dep0".to_owned();
9183 dep.succeed();
9184 queue.put(&mut dep).unwrap();
9185
9186 let mut still_going = task();
9187 still_going.id = "20260101-000002-dep1".to_owned();
9188 queue.put(&mut still_going).unwrap();
9189
9190 let mut blocked = task();
9191 blocked.id = "20260101-000003-main".to_owned();
9192 blocked.block(
9193 vec![dep.id.clone(), still_going.id.clone()],
9194 Some("waits on both".to_owned()),
9195 );
9196 queue.put(&mut blocked).unwrap();
9197
9198 resolve_blockers(&queue, &questions);
9199
9200 let after = queue.get(&blocked.id).unwrap();
9201 assert_eq!(
9202 after.status,
9203 TaskStatus::Blocked,
9204 "one dependency is still outstanding"
9205 );
9206 assert_eq!(after.blocked_by, [still_going.id.clone()]);
9207 }
9208
9209 #[test]
9210 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
9211 let dir = tempfile::tempdir().unwrap();
9212 let queue = Queue::at(dir.path().join("queue"));
9213 let questions = ask::Questions::at(dir.path().join("questions"));
9214
9215 let mut q = crate::ask::Question::new(
9216 "20260101-000001-main".to_owned(),
9217 crate::conduct::NODE.to_owned(),
9218 "conduct".to_owned(),
9219 "Which backend?".to_owned(),
9220 String::new(),
9221 Vec::new(),
9222 );
9223 questions.put(&mut q).unwrap();
9224 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
9225 .unwrap();
9226 questions.put(&mut q).unwrap();
9227
9228 let mut blocked = task();
9229 blocked.id = "20260101-000001-main".to_owned();
9230 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
9231 queue.put(&mut blocked).unwrap();
9232
9233 resolve_blockers(&queue, &questions);
9234
9235 let after = queue.get(&blocked.id).unwrap();
9236 assert_eq!(
9237 after.status,
9238 TaskStatus::Queued,
9239 "the only blocker resolved"
9240 );
9241 assert_eq!(after.answers.len(), 1);
9242 assert_eq!(after.answers[0].question, "Which backend?");
9243 assert_eq!(after.answers[0].answer, "SQLite");
9244
9245 let instruction = instruction_for(&after);
9247 assert!(instruction.contains("Which backend?"));
9248 assert!(instruction.contains("SQLite"));
9249 }
9250
9251 #[test]
9252 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
9253 let dir = tempfile::tempdir().unwrap();
9254 let queue = Queue::at(dir.path().join("queue"));
9255 let questions = ask::Questions::at(dir.path().join("questions"));
9256
9257 let mut q = crate::ask::Question::new(
9258 "20260101-000001-main".to_owned(),
9259 crate::conduct::NODE.to_owned(),
9260 "conduct".to_owned(),
9261 "Is the setup done?".to_owned(),
9262 String::new(),
9263 Vec::new(),
9264 );
9265 q.abandon("no answer within 60s of asking");
9266 questions.put(&mut q).unwrap();
9267
9268 let mut blocked = task();
9269 blocked.id = "20260101-000001-main".to_owned();
9270 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
9271 queue.put(&mut blocked).unwrap();
9272
9273 resolve_blockers(&queue, &questions);
9274
9275 let after = queue.get(&blocked.id).unwrap();
9276 assert_eq!(
9277 after.status,
9278 TaskStatus::Held,
9279 "never left blocked on nothing"
9280 );
9281 assert!(!after.operator_held(), "a machine hold, for triage");
9282 assert!(
9283 after
9284 .hold_reason
9285 .as_deref()
9286 .unwrap_or_default()
9287 .contains("went unanswered")
9288 );
9289 }
9290
9291 #[test]
9292 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
9293 let dir = tempfile::tempdir().unwrap();
9299 let queue = Queue::at(dir.path().join("queue"));
9300 let questions = ask::Questions::at(dir.path().join("questions"));
9301
9302 let mut q = crate::ask::Question::new(
9303 "20260101-000001-main".to_owned(),
9304 crate::conduct::NODE.to_owned(),
9305 "conduct".to_owned(),
9306 "How should this be handled?".to_owned(),
9307 String::new(),
9308 Vec::new(),
9309 );
9310 questions.put(&mut q).unwrap();
9311 q.answer(crate::ask::Answer::Text(
9312 "leave it held, a human will look at it later".to_owned(),
9313 ))
9314 .unwrap();
9315 questions.put(&mut q).unwrap();
9316
9317 let mut held = task();
9318 held.id = "20260101-000001-main".to_owned();
9319 held.hold_machine(Some("out of attempts".to_owned()));
9320 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
9321 queue.put(&mut held).unwrap();
9322
9323 resolve_blockers(&queue, &questions);
9324
9325 let after = queue.get(&held.id).unwrap();
9326 assert_eq!(after.status, TaskStatus::Held);
9327 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
9328 assert_eq!(
9329 after.answers[0].answer,
9330 "leave it held, a human will look at it later"
9331 );
9332 }
9333
9334 #[test]
9335 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
9336 let dir = tempfile::tempdir().unwrap();
9337 let queue = Queue::at(dir.path().join("queue"));
9338 let questions = ask::Questions::at(dir.path().join("questions"));
9339 let mut dep = task();
9340 dep.id = "20260101-000001-gone".to_owned();
9341 queue.put(&mut dep).unwrap();
9342 let mut blocked = task();
9343 blocked.id = "20260101-000003-main".to_owned();
9344 blocked.block(vec![dep.id.clone()], None);
9345 queue.put(&mut blocked).unwrap();
9346
9347 let claim = queue.claim(&blocked.id).unwrap();
9349 queue.remove(&dep.id, false, &questions).unwrap();
9350 drop(claim);
9351 resolve_blockers(&queue, &questions);
9352
9353 let after = queue.get(&blocked.id).unwrap();
9354 assert_eq!(after.status, TaskStatus::Queued);
9355 assert!(after.blocked_by.is_empty());
9356 }
9357
9358 #[test]
9359 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
9360 let dir = tempfile::tempdir().unwrap();
9366 let queue = Queue::at(dir.path().join("queue"));
9367 let questions = ask::Questions::at(dir.path().join("questions"));
9368
9369 let mut still_going = task();
9370 still_going.id = "20260101-000002-dep1".to_owned();
9371 queue.put(&mut still_going).unwrap();
9372
9373 let mut blocked = task();
9374 blocked.id = "20260101-000003-main".to_owned();
9375 blocked.block(
9376 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
9377 Some("waits on both".to_owned()),
9378 );
9379 queue.put(&mut blocked).unwrap();
9380
9381 resolve_blockers(&queue, &questions);
9382
9383 let after = queue.get(&blocked.id).unwrap();
9384 assert_eq!(
9385 after.status,
9386 TaskStatus::Held,
9387 "a missing dependency must not leave the task blocked forever"
9388 );
9389 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
9390 assert!(after.blocked_by.is_empty());
9391 let reason = after.hold_reason.as_deref().unwrap_or_default();
9392 assert!(
9393 reason.contains("20260101-000001-gone"),
9394 "the missing id must be named so an operator can tell what happened: {reason}"
9395 );
9396 assert!(
9397 reason.contains(&still_going.id),
9398 "the still-valid dependency must not silently vanish from the record: {reason}"
9399 );
9400 }
9401
9402 #[test]
9403 fn instruction_for_is_unchanged_without_any_answers() {
9404 let t = task();
9405 assert_eq!(instruction_for(&t), t.instruction);
9406 }
9407
9408 #[test]
9409 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
9410 let dir = tempfile::tempdir().unwrap();
9411 let q = Queue::at(dir.path().join("queue"));
9412 let src = dir.path().join("shot.png");
9413 std::fs::write(&src, "x").unwrap();
9414 let mut t = task();
9415 q.attach(&mut t, &[src]).unwrap();
9416 let paths = task_attachments(&q, &t).unwrap();
9417 assert_eq!(paths.len(), 1);
9418 assert!(paths[0].is_absolute() && paths[0].is_file());
9419 std::fs::remove_file(&paths[0]).unwrap();
9420 let err = task_attachments(&q, &t).unwrap_err().to_string();
9421 assert!(err.contains("shot.png"), "{err}");
9422 }
9423
9424 #[tokio::test]
9425 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
9426 let dir = tempfile::tempdir().unwrap();
9427 let queue_path = dir.path().join("queue");
9428 let queue = Queue::at(queue_path.clone());
9429 let mut t = task();
9430 t.source = Source::Agent {
9431 run: "originating-chat".into(),
9432 node: crate::queue::CHAT_NODE.into(),
9433 };
9434 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
9435 stage: "config-fetch",
9436 target: "origin/main".into(),
9437 });
9438 record_config_failure(&mut t, &fetch);
9439 t.hold_machine(Some(format!("{fetch:#}")));
9440 let fetch_key = t.chat_notice_due().unwrap();
9441 t.chat_notices.push(fetch_key.clone());
9442 queue.put(&mut t).unwrap();
9443
9444 let queue = Queue::at(queue_path);
9445 let mut t = queue.get(&t.id).unwrap();
9446 let a = dir.path().join("a.toml");
9447 let b = dir.path().join("b.toml");
9448 for (path, text, new_cause) in [
9449 (&a, "[graph", true),
9450 (&a, "[verify", false),
9451 (&a, "[graph]\nimplementers = 'invalid-type'", true),
9452 (&b, "[graph]\nimplementers = 'invalid-type'", true),
9453 ] {
9454 t.release_operator("test", None);
9455 std::fs::write(path, text).unwrap();
9456 let error = Config::discover_fetched(dir.path(), Some(path))
9457 .await
9458 .unwrap_err();
9459 record_config_failure(&mut t, &error);
9460 t.hold_machine(Some(format!("{error:#}")));
9461 let due = t.chat_notice_due();
9462 assert_eq!(due.is_some(), new_cause, "{error:#}");
9463 if let Some(key) = due {
9464 assert_ne!(key, fetch_key);
9465 t.chat_notices.push(key);
9466 }
9467 }
9468 t.succeed();
9469 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
9470 }
9471
9472 #[test]
9473 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
9474 let dir = tempfile::tempdir().unwrap();
9475 let queue_path = dir.path().join("queue");
9476 let queue = Queue::at(queue_path.clone());
9477 let mut t = task();
9478 t.source = crate::queue::Source::Agent {
9479 run: "originating-chat".into(),
9480 node: crate::queue::CHAT_NODE.into(),
9481 };
9482 let a = dir.path().join("a.png");
9483 let b = dir.path().join("b.png");
9484 std::fs::write(&a, "a").unwrap();
9485 std::fs::write(&b, "b").unwrap();
9486 queue.attach(&mut t, &[a, b]).unwrap();
9487 let paths = task_attachments(&queue, &t).unwrap();
9488 for path in &paths {
9489 std::fs::remove_file(path).unwrap();
9490 }
9491 let missing = task_attachments(&queue, &t).unwrap_err();
9492 t.attempts += 1;
9493 let reason = missing.to_string();
9494 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9495 let first = t.chat_notice_due().expect("first attachment notice");
9496 t.chat_notices.push(first.clone());
9497 queue.put(&mut t).unwrap();
9498
9499 let queue = Queue::at(queue_path.clone());
9500 let mut t = queue.get(&t.id).unwrap();
9501 assert!(t.chat_notice_due().is_none(), "restart does not replay");
9502 t.release_operator("test", None);
9503 t.attempts += 1;
9504 let missing = task_attachments(&queue, &t).unwrap_err();
9505 t.fail_startup_target(
9506 "startup-attachments",
9507 missing.name,
9508 "same missing attachment with regenerated wording",
9509 1,
9510 );
9511 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
9512
9513 std::fs::write(&paths[0], "restored").unwrap();
9514 t.release_operator("test", None);
9515 t.attempts += 1;
9516 let missing = task_attachments(&queue, &t).unwrap_err();
9517 let reason = missing.to_string();
9518 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9519 let second = t
9520 .chat_notice_due()
9521 .expect("different attachment must notify");
9522 assert_ne!(first, second);
9523 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
9524 }
9525
9526 #[test]
9527 fn resumed_instruction_is_unchanged_without_any_answers() {
9528 let t = task();
9529 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
9530 }
9531
9532 #[test]
9533 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
9534 let mut t = task();
9535 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9536 let old = t.instruction.clone();
9540
9541 let refreshed = resumed_instruction(&old, &t);
9542 assert!(refreshed.starts_with(&old), "the original text is kept");
9543 assert!(refreshed.contains("Which backend?"));
9544 assert!(refreshed.contains("SQLite"));
9545 }
9546
9547 #[test]
9548 fn resumed_instruction_keeps_an_original_answers_heading() {
9549 let mut t = task();
9550 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
9551 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9552
9553 let refreshed = resumed_instruction(&t.instruction, &t);
9554
9555 assert!(
9556 refreshed.starts_with(&t.instruction),
9557 "an answers heading in the original instruction is not the appended block"
9558 );
9559 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
9560 assert!(refreshed.contains("Which backend?"));
9561 assert!(refreshed.contains("SQLite"));
9562
9563 let repeated = resumed_instruction(&refreshed, &t);
9564 assert_eq!(
9565 repeated, refreshed,
9566 "only the final appended block is refreshed"
9567 );
9568 }
9569
9570 #[test]
9571 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
9572 let mut t = task();
9573 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9574
9575 let once = resumed_instruction(&t.instruction, &t);
9579 let twice = resumed_instruction(&once, &t);
9580 assert_eq!(once, twice);
9581 assert_eq!(once.matches("Which backend?").count(), 1);
9582
9583 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
9585 let refreshed = resumed_instruction(&once, &t);
9586 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
9587 assert!(refreshed.contains("Which backend?"));
9588 assert!(refreshed.contains("Which cache?"));
9589 }
9590
9591 #[test]
9592 fn prepare_instruction_covers_all_three_starters() {
9593 let mut t = task();
9594 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9595
9596 assert_eq!(
9599 prepare_instruction(&Starter::Start, None, &t),
9600 Some(instruction_for(&t))
9601 );
9602
9603 let old = t.instruction.clone();
9606 assert_eq!(
9607 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
9608 Some(resumed_instruction(&old, &t))
9609 );
9610
9611 assert_eq!(
9615 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
9616 None
9617 );
9618 }
9619
9620 #[test]
9621 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
9622 assert_eq!(
9623 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
9624 Starter::Review("magi/eba2/A".to_owned())
9625 );
9626 }
9627
9628 #[test]
9629 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
9630 assert_eq!(
9631 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
9632 Starter::Start,
9633 "a vanished review branch must not fall back to resuming the old run either"
9634 );
9635 }
9636
9637 #[test]
9638 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
9639 let mut t = task();
9640 t.start("old-run".to_owned());
9641 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
9642 t.release();
9643 let branch = t.review_branch.take();
9644 assert_eq!(
9645 choose_starter(branch.as_deref(), true, Some("old-run")),
9646 Starter::Review("magi/eba2/A".to_owned()),
9647 "a review wins over resuming the old run"
9648 );
9649 }
9650
9651 #[test]
9652 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
9653 assert_eq!(
9654 choose_starter(None, false, Some("some-run")),
9655 Starter::Resume("some-run".to_owned())
9656 );
9657 assert_eq!(choose_starter(None, false, None), Starter::Start);
9658 }
9659
9660 #[test]
9661 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
9662 let mut released = task();
9663 released.start("stalled-run".to_owned());
9664 released.requeue();
9665 let unfinished = (!released.fresh_start)
9666 .then(|| Some("stalled-run".to_owned()))
9667 .flatten();
9668 assert_eq!(
9669 choose_starter(None, false, unfinished.as_deref()),
9670 Starter::Start,
9671 "release keeps run history but must not resume it"
9672 );
9673 assert_eq!(released.runs, ["stalled-run"]);
9674 }
9675
9676 #[test]
9677 fn an_ordinary_release_keeps_a_resumable_run_available() {
9678 let mut released = task();
9679 released.start("stalled-run".to_owned());
9680 released.release();
9681 let unfinished = (!released.fresh_start)
9682 .then(|| Some("stalled-run".to_owned()))
9683 .flatten();
9684 assert_eq!(
9685 choose_starter(None, false, unfinished.as_deref()),
9686 Starter::Resume("stalled-run".to_owned()),
9687 "manual release must preserve the normal resume path"
9688 );
9689 }
9690
9691 #[test]
9692 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
9693 let mut state = run_state(RunStatus::Blocked);
9694 state.config.graph.review_rounds = 3;
9695 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
9696 assert!(exhausted_review_budget(&state));
9697
9698 state.reviews.pop();
9700 assert!(!exhausted_review_budget(&state));
9701
9702 let mut stalled = run_state(RunStatus::Stalled);
9705 stalled.config.graph.review_rounds = 1;
9706 stalled.reviews = vec![review_round(1)];
9707 assert!(!exhausted_review_budget(&stalled));
9708 }
9709
9710 fn review_round(round: usize) -> crate::run::ReviewRound {
9711 crate::run::ReviewRound {
9712 round,
9713 head: "deadbeef".to_owned(),
9714 verified_head: None,
9715 verified_at: None,
9716 reviews: Vec::new(),
9717 e2e: Vec::new(),
9718 verify_retried: false,
9719 e2e_deferred: false,
9720 e2e_defer_reason: None,
9721 fix: None,
9722 blocking: 0,
9723 answered: 1,
9724 expected: 1,
9725 clean: false,
9726 progressed: true,
9727 vote_split: false,
9728 reconsideration: Vec::new(),
9729 verdict: None,
9730 }
9731 }
9732
9733 #[test]
9734 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
9735 let mut run = RunState::new(
9736 PathBuf::from("/repo"),
9737 "main".to_owned(),
9738 "abc1234def".to_owned(),
9739 "add retries".to_owned(),
9740 Config::default(),
9741 );
9742 run.status = RunStatus::Judging;
9743 run.parked = true;
9744 let mut task = Task::new(
9745 "add retries".to_owned(),
9746 "add retries".to_owned(),
9747 PathBuf::from("/repo"),
9748 crate::queue::Source::Human,
9749 );
9750 task.status = TaskStatus::Failed;
9751 task.runs = vec![run.id.clone()];
9752 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
9753 assert!(with(&task, &run), "parked after judging is the case");
9754 let mut parked_task = task.clone();
9755 parked_task.status = TaskStatus::Parked;
9756 assert!(with(&parked_task, &run), "the Parked status is the case");
9757
9758 let mut not_parked = run.clone();
9759 not_parked.parked = false;
9760 not_parked.status = RunStatus::Stalled;
9761 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
9762
9763 let mut fresh = task.clone();
9764 fresh.fresh_start = true;
9765 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
9766
9767 let mut review = task.clone();
9768 review.review_branch = Some("magi/x/A".to_owned());
9769 assert!(!with(&review, &run), "review is ranked before resume");
9770
9771 let mut held = task.clone();
9772 held.status = TaskStatus::Held;
9773 assert!(!with(&held, &run), "a hold stays visible to the conductor");
9774
9775 let mut released = run.clone();
9776 released.released_to = Some("20260901-000000-new1".to_owned());
9777 assert!(!with(&task, &released), "nothing left to resume into");
9778
9779 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
9780 "unreadable"
9781 )));
9782 }
9783
9784 #[test]
9785 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
9786 let mut released = RunState::new(
9787 PathBuf::from("/repo"),
9788 "main".to_owned(),
9789 "abc1234def".to_owned(),
9790 "add retries".to_owned(),
9791 Config::default(),
9792 );
9793 released.status = RunStatus::Blocked;
9794 assert_eq!(
9795 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9796 Some(released.id.clone())
9797 );
9798 released.released_to = Some("20260901-000000-new1".to_owned());
9799 assert_eq!(
9800 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9801 None,
9802 "there is nothing left to resume it into"
9803 );
9804 }
9805
9806 #[test]
9807 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
9808 let mut exhausted = RunState::new(
9817 PathBuf::from("/repo"),
9818 "main".to_owned(),
9819 "abc1234def".to_owned(),
9820 "add retries".to_owned(),
9821 Config::default(),
9822 );
9823 exhausted.status = RunStatus::Blocked;
9824 exhausted.config.graph.review_rounds = 1;
9825 exhausted.reviews = vec![review_round(1)];
9826
9827 assert_eq!(
9828 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
9829 None,
9830 "an exhausted `Blocked` run must not be offered as resumable"
9831 );
9832
9833 let mut has_budget_left = RunState::new(
9836 PathBuf::from("/repo"),
9837 "main".to_owned(),
9838 "abc1234def".to_owned(),
9839 "add retries".to_owned(),
9840 Config::default(),
9841 );
9842 has_budget_left.status = RunStatus::Blocked;
9843 has_budget_left.config.graph.review_rounds = 3;
9844 has_budget_left.reviews = vec![review_round(1)];
9845
9846 assert_eq!(
9847 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
9848 Ok(has_budget_left.clone())
9849 }),
9850 Some(has_budget_left.id.clone())
9851 );
9852 }
9853
9854 #[test]
9855 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
9856 let mut run = RunState::new(
9857 PathBuf::from("/repo"),
9858 "main".to_owned(),
9859 "abc1234def".to_owned(),
9860 "add retries".to_owned(),
9861 Config::default(),
9862 );
9863 run.status = RunStatus::Blocked;
9864 run.config.graph.review_rounds = 1;
9865 run.reviews = vec![review_round(1)];
9866 run.pr_publication_pending = true;
9867 assert_eq!(
9868 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
9869 Some(run.id.clone()),
9870 "a spent review budget must not turn a failed publication into a re-competition"
9871 );
9872 }
9873
9874 #[test]
9875 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
9876 let mut older_stalled = RunState::new(
9884 PathBuf::from("/repo"),
9885 "main".to_owned(),
9886 "abc1234def".to_owned(),
9887 "add retries".to_owned(),
9888 Config::default(),
9889 );
9890 older_stalled.status = RunStatus::Stalled;
9891
9892 let mut newest_exhausted = RunState::new(
9893 PathBuf::from("/repo"),
9894 "main".to_owned(),
9895 "abc1234def".to_owned(),
9896 "add retries".to_owned(),
9897 Config::default(),
9898 );
9899 newest_exhausted.status = RunStatus::Blocked;
9900 newest_exhausted.config.graph.review_rounds = 1;
9901 newest_exhausted.reviews = vec![review_round(1)];
9902
9903 assert_eq!(
9904 unfinished_run_with(
9905 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
9906 "t",
9907 |_| Ok(newest_exhausted.clone())
9908 ),
9909 None,
9910 "the newest run is exhausted, so nothing here is worth resuming - \
9911 least of all the older, already-superseded run"
9912 );
9913 }
9914
9915 #[test]
9916 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
9917 assert_eq!(
9918 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
9919 Err(anyhow::anyhow!("fixture is absent"))
9920 }),
9921 None
9922 );
9923 }
9924
9925 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
9926 let mut q = ask::Question::new(
9927 run.to_owned(),
9928 "implement".to_owned(),
9929 "impl-A".to_owned(),
9930 "continue?".to_owned(),
9931 String::new(),
9932 vec!["resume で続行する".to_owned(), "other".to_owned()],
9933 );
9934 q.actions.insert("resume で続行する".to_owned(), action);
9935 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
9936 .unwrap();
9937 q
9938 }
9939
9940 fn held_task_with(run: &str) -> Task {
9941 let mut t = task();
9942 t.runs = vec![run.to_owned()];
9943 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
9944 t
9945 }
9946
9947 fn resume_action(run: &str) -> ask::ChoiceAction {
9948 ask::ChoiceAction::Resume { run: run.into() }
9949 }
9950
9951 #[test]
9952 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
9953 for parked in [false, true] {
9954 let mut t = task();
9955 t.start("A".into());
9956 t.start("B".into());
9957 assert_eq!(t.runs, vec!["A", "B"]);
9958 t.start("A".into());
9959 let mut resumed = run_state(RunStatus::Blocked);
9960 resumed.id = "A".into();
9961 resumed.parked = parked;
9962 resumed.driver_exited = true;
9963 t.hold_machine(Some("resumed run stopped".into()));
9964 let q = action_question("A", resume_action("A"));
9965 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
9966 assert_eq!(
9967 decide_action(&t, &q, &PHRASES_EN, |id| {
9968 assert_eq!(id, "A");
9969 Ok(resumed.clone())
9970 }),
9971 ActionDecision::Resume("A".into())
9972 );
9973 assert_eq!(
9974 action_standing(&t, &action_question("B", resume_action("B"))),
9975 ActionStanding::Stale
9976 );
9977 t.release_operator("cli", None);
9978 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
9979 assert_eq!(id, "A");
9980 Ok(resumed.clone())
9981 });
9982 assert_eq!(
9983 choose_starter(None, false, unfinished.as_deref()),
9984 Starter::Resume("A".into())
9985 );
9986 assert_eq!(t.runs, vec!["B", "A"]);
9987 }
9988 }
9989
9990 #[test]
9991 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
9992 let mut state = crate::land::tests::repair_state(true, true);
9993 state.status = RunStatus::Landing;
9994 state.parked = true;
9995 state.driver_exited = true;
9996 let mut q = ask::Question::new(
9997 state.id.clone(),
9998 land::APPROVAL_NODE.into(),
9999 "land".into(),
10000 "Merge?".into(),
10001 String::new(),
10002 vec![land::APPROVE.into(), land::HOLD.into()],
10003 );
10004 q.id = state.id.clone();
10005 state.land_approval = Some(crate::run::LandApproval {
10006 question: q.id.clone(),
10007 head: "current-head".into(),
10008 });
10009 assert!(land::upgrade_repair_choice(&mut q, &state));
10010 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10011 let mut task = held_task_with(&state.id);
10012 task.status = TaskStatus::Parked;
10013 assert_eq!(
10014 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10015 ActionDecision::Resume(state.id.clone())
10016 );
10017 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10018 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
10019 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
10020 assert_eq!(
10021 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10022 ActionDecision::Resume(state.id.clone()),
10023 "a repair answer newer than the manual hold resumes the existing owner"
10024 );
10025 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10026 for answered_at in [q.answered_at, None] {
10027 let mut older = q.clone();
10028 older.answered_at = answered_at;
10029 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
10030 panic!("a later manual hold must prevent dispatch")
10031 });
10032 let ActionDecision::Refuse(reason) = refused else {
10033 panic!("expected an actionable refusal, got {refused:?}");
10034 };
10035 assert!(reason.contains("later operator decision"));
10036 assert!(reason.contains(&format!("magi task release {}", task.id)));
10037 }
10038 task.operator_decided_at = None;
10039 let mut superseded = state.clone();
10040 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
10041 assert_eq!(
10042 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
10043 ActionDecision::Stale
10044 );
10045 for case in 1..4 {
10046 let mut changed = state.clone();
10047 match case {
10048 1 => changed.status = RunStatus::Blocked,
10049 2 => changed.released_to = Some("new-owner".into()),
10050 _ => changed.land_approval.as_mut().unwrap().head.clear(),
10051 }
10052 assert!(matches!(
10053 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
10054 ActionDecision::Refuse(_)
10055 ));
10056 }
10057 assert!(matches!(
10058 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
10059 ActionDecision::Refuse(_)
10060 ));
10061 task.runs.push("replacement".into());
10062 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10063 assert_eq!(
10064 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
10065 "old owner must not load"
10066 )),
10067 ActionDecision::Stale
10068 );
10069 }
10070
10071 #[test]
10072 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
10073 let mut state = crate::land::tests::repair_state(true, true);
10074 state.status = RunStatus::Landing;
10075 state.parked = true;
10076 state.driver_exited = true;
10077 let mut q = ask::Question::new(
10078 state.id.clone(),
10079 land::APPROVAL_NODE.into(),
10080 "land".into(),
10081 "Merge?".into(),
10082 String::new(),
10083 vec![land::APPROVE.into(), land::HOLD.into()],
10084 );
10085 q.id = state.id.clone();
10086 state.land_approval = Some(crate::run::LandApproval {
10087 question: q.id.clone(),
10088 head: "current-head".into(),
10089 });
10090 state.save().unwrap();
10091 land::upgrade_repair_choice(&mut q, &state);
10092 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10093 Questions::open().put(&mut q).unwrap();
10094 let tmp = tempfile::tempdir().unwrap();
10095 let queue = Queue::at(tmp.path().join("queue"));
10096 let mut task = held_task_with(&state.id);
10097 task.status = TaskStatus::Parked;
10098 queue.put(&mut task).unwrap();
10099 let empty = Arc::new(tokio::sync::Semaphore::new(0));
10100 let free = Arc::new(tokio::sync::Semaphore::new(1));
10101 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
10102 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
10103 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
10104 assert!(claim.permit.is_some());
10105 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
10106 drop(claim);
10107 Questions::open()
10108 .update(&q.id, |q| {
10109 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
10110 Ok(())
10111 })
10112 .unwrap();
10113 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
10114 assert!(claim.permit.is_none());
10115 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
10116 }
10117
10118 #[test]
10119 fn decide_action_resumes_only_the_latest_resumable_run() {
10120 let t = held_task_with("r1");
10121 let q = action_question("r1", resume_action("r1"));
10122 let load = |s: RunState| move |_: &str| Ok(s);
10123 assert_eq!(
10124 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
10125 ActionDecision::Resume("r1".into())
10126 );
10127 let q_other = action_question("r1", resume_action("r0"));
10129 assert!(matches!(
10130 decide_action(
10131 &t,
10132 &q_other,
10133 &PHRASES_EN,
10134 load(run_state(RunStatus::Blocked))
10135 ),
10136 ActionDecision::Refuse(_)
10137 ));
10138 assert!(matches!(
10140 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
10141 ActionDecision::Refuse(_)
10142 ));
10143 let mut released = run_state(RunStatus::Blocked);
10145 released.released_to = Some("elsewhere".into());
10146 assert!(matches!(
10147 decide_action(&t, &q, &PHRASES_EN, load(released)),
10148 ActionDecision::Refuse(_)
10149 ));
10150 assert!(matches!(
10152 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
10153 ActionDecision::Refuse(_)
10154 ));
10155 }
10156
10157 #[test]
10158 fn decide_action_ignores_a_question_about_an_earlier_run() {
10159 let mut t = held_task_with("r1");
10160 t.runs.push("r2".to_owned());
10161 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10162 assert_eq!(
10163 decide_action(
10164 &t,
10165 &action_question("r1", ask::ChoiceAction::Done),
10166 &PHRASES_EN,
10167 never
10168 ),
10169 ActionDecision::Stale
10170 );
10171 }
10172
10173 #[test]
10174 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
10175 let mut t = held_task_with("r1");
10176 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10177 assert_eq!(
10178 decide_action(
10179 &t,
10180 &action_question("r1", ask::ChoiceAction::Requeue),
10181 &PHRASES_EN,
10182 never
10183 ),
10184 ActionDecision::Requeue
10185 );
10186 let done_q = action_question("r1", ask::ChoiceAction::Done);
10187 assert_eq!(
10188 decide_action(&t, &done_q, &PHRASES_EN, never),
10189 ActionDecision::Done
10190 );
10191 t.mark_action_applied(&done_q.id);
10192 assert_eq!(
10193 decide_action(&t, &done_q, &PHRASES_EN, never),
10194 ActionDecision::Skip
10195 );
10196
10197 let mut plain = action_question("r1", ask::ChoiceAction::Done);
10199 plain.actions.clear();
10200 assert_eq!(
10201 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
10202 ActionDecision::Skip
10203 );
10204 let mut running = held_task_with("r1");
10206 running.status = TaskStatus::Running;
10207 assert_eq!(
10208 decide_action(
10209 &running,
10210 &action_question("r1", ask::ChoiceAction::Done),
10211 &PHRASES_EN,
10212 never
10213 ),
10214 ActionDecision::Skip
10215 );
10216 }
10217
10218 #[test]
10219 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
10220 crate::run::pin_test_home();
10221 for legacy in [false, true] {
10222 let dir = tempfile::tempdir().unwrap();
10223 let queue = Queue::at(dir.path().join("queue"));
10224 let questions = Questions::at(dir.path().join("questions"));
10225 let run = "20260101-000000-hold";
10226 let mut task = held_task_with(run);
10227 task.status = TaskStatus::Parked;
10228 let mut q = ask::Question::new(
10229 run.into(),
10230 land::APPROVAL_NODE.into(),
10231 "land".into(),
10232 "Repair?".into(),
10233 String::new(),
10234 vec![land::REPAIR.into()],
10235 );
10236 q.actions.insert(land::REPAIR.into(), resume_action(run));
10237 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10238 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10240 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
10241 questions.put(&mut q).unwrap();
10242 task.hold_manual(Some("Keep this PR paused".into()));
10243 if legacy {
10244 task.hold_source = None;
10245 }
10246 queue.put(&mut task).unwrap();
10247
10248 apply_choice_actions(&queue, &questions, dir.path());
10249 let after = queue.get(&task.id).unwrap();
10250 assert!(after.operator_held());
10251 assert_eq!(after.hold_source, task.hold_source);
10252 assert_eq!(after.operator_decided_at, task.operator_decided_at);
10253 assert_eq!(after.runs, task.runs);
10254 assert_eq!(after.instruction, task.instruction);
10255 assert!(after.resume_override.is_none());
10256 assert!(after.action_applied(&q.id));
10257 let reason = after.hold_reason.as_deref().unwrap();
10258 assert!(reason.starts_with("Keep this PR paused"));
10259 assert!(reason.contains("later operator decision"));
10260 assert!(reason.contains(&format!("magi task release {}", task.id)));
10261
10262 for recovery in [
10263 crate::conduct::Recovery::Review,
10264 crate::conduct::Recovery::Requeue,
10265 ] {
10266 crate::conduct::apply(
10267 &queue,
10268 &questions,
10269 &crate::conduct::Verdict {
10270 decisions: vec![crate::conduct::Decision {
10271 id: task.id.clone(),
10272 recovery: Some(recovery),
10273 ..Default::default()
10274 }],
10275 },
10276 )
10277 .unwrap();
10278 }
10279 apply_choice_actions(&queue, &questions, dir.path());
10282 let still_held = queue.get(&task.id).unwrap();
10283 assert!(still_held.operator_held());
10284 assert_eq!(still_held.hold_reason, after.hold_reason);
10285 assert_eq!(still_held.runs, task.runs);
10286 assert!(still_held.review_branch.is_none());
10287 assert!(!still_held.fresh_start);
10288 assert!(queue.next_runnable().is_none());
10289 }
10290 }
10291
10292 #[test]
10293 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
10294 let dir = tempfile::tempdir().unwrap();
10295 let queue = Queue::at(dir.path().join("queue"));
10296 let questions = Questions::at(dir.path().join("questions"));
10297 let home = dir.path().join("home");
10298 let mut state = run_state(RunStatus::Blocked);
10299 state.id = "20260101-000000-act1".to_owned();
10300 state.save_under(&home).unwrap();
10301
10302 let mut t = held_task_with(&state.id);
10303 queue.put(&mut t).unwrap();
10304 let mut q = action_question(&state.id, resume_action(&state.id));
10305 questions.put(&mut q).unwrap();
10306
10307 apply_choice_actions(&queue, &questions, &home);
10308 let after = queue.get(&t.id).unwrap();
10309 assert_eq!(after.status, TaskStatus::Queued);
10310 assert!(!after.fresh_start);
10311 assert!(after.action_applied(&q.id));
10312 let pin = after.resume_override.clone().unwrap();
10313 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
10314 assert!(pin.forced);
10315
10316 let mut again = queue.get(&t.id).unwrap();
10318 again.hold_machine(Some("later".into()));
10319 queue.put(&mut again).unwrap();
10320 apply_choice_actions(&queue, &questions, &home);
10321 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10322 }
10323
10324 #[test]
10325 fn a_delivered_answer_is_still_acted_on_exactly_once() {
10326 let dir = tempfile::tempdir().unwrap();
10327 let queue = Queue::at(dir.path().join("queue"));
10328 let questions = Questions::at(dir.path().join("questions"));
10329 let home = dir.path().join("home");
10330 let mut state = run_state(RunStatus::Blocked);
10331 state.id = "20260101-000000-act2".to_owned();
10332 state.save_under(&home).unwrap();
10333
10334 let mut t = held_task_with(&state.id);
10335 queue.put(&mut t).unwrap();
10336 let mut q = action_question(&state.id, resume_action(&state.id));
10337 q.answer_delivered = true;
10338 questions.put(&mut q).unwrap();
10339
10340 apply_choice_actions(&queue, &questions, &home);
10343 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10344 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
10345
10346 let mut again = queue.get(&t.id).unwrap();
10348 again.hold_machine(Some("later".into()));
10349 queue.put(&mut again).unwrap();
10350 apply_choice_actions(&queue, &questions, &home);
10351 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10352 }
10353
10354 #[test]
10355 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
10356 let dir = tempfile::tempdir().unwrap();
10357 let queue = Queue::at(dir.path().join("queue"));
10358 let questions = Questions::at(dir.path().join("questions"));
10359 let home = dir.path().join("home");
10360 let mut state = run_state(RunStatus::Blocked);
10361 state.id = "20260101-000000-act3".to_owned();
10362 state.save_under(&home).unwrap();
10363 let mut t = held_task_with(&state.id);
10364 queue.put(&mut t).unwrap();
10365 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
10366 questions.put(&mut q).unwrap();
10367
10368 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
10369 apply_choice_actions(&queue, &questions, &home);
10370 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10371
10372 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
10373 apply_choice_actions(&queue, &questions, &home);
10374 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10375 }
10376
10377 #[test]
10378 fn action_standing_tells_the_reasons_apart() {
10379 let mut t = held_task_with("r1");
10380 let q = action_question("r1", ask::ChoiceAction::Requeue);
10381 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
10382 let mut plain = q.clone();
10383 plain.actions.clear();
10384 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
10385
10386 t.mark_action_applied(&q.id);
10388 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
10389
10390 let mut t = held_task_with("r1");
10392 t.start("r2".to_owned());
10393 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
10394 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
10396 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10397 t.status = TaskStatus::Blocked;
10398 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10399
10400 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
10402 c.node = crate::conduct::NODE.to_owned();
10403 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
10404 assert!(!ActionStanding::Busy.daemon_owns());
10405 }
10406
10407 #[test]
10408 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
10409 let dir = tempfile::tempdir().unwrap();
10410 let queue = Queue::at(dir.path().join("queue"));
10411 let questions = Questions::at(dir.path().join("questions"));
10412 let home = dir.path().join("home");
10413 let mut state = run_state(RunStatus::Blocked);
10414 state.id = "20260101-000000-act4".to_owned();
10415 state.save_under(&home).unwrap();
10416 let mut t = held_task_with(&state.id);
10417 t.status = TaskStatus::Running;
10418 queue.put(&mut t).unwrap();
10419 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
10420 q.cwd = Some(dir.path().display().to_string());
10421 questions.put(&mut q).unwrap();
10422
10423 let running = queue.get(&t.id).unwrap();
10426 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
10427 assert!(matches!(
10428 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
10429 crate::waiter::Action::Deliver(_)
10430 ));
10431 apply_choice_actions(&queue, &questions, &home);
10433 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10434 let mut back = queue.get(&t.id).unwrap();
10435 back.hold_machine(Some("later".into()));
10436 queue.put(&mut back).unwrap();
10437 apply_choice_actions(&queue, &questions, &home);
10438 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
10439 let mut again = queue.get(&t.id).unwrap();
10440 again.hold_machine(Some("again".into()));
10441 queue.put(&mut again).unwrap();
10442 apply_choice_actions(&queue, &questions, &home);
10443 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10444 }
10445}