1use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::sync::atomic::{AtomicBool, Ordering};
52use std::sync::{Mutex, MutexGuard};
53use std::time::Duration;
54
55use anyhow::{Context, Result, bail};
56use jiff::Timestamp;
57use serde::{Deserialize, Serialize};
58use tokio::sync::Notify;
59
60use crate::ask::{self, Questions};
61use crate::clean;
62use crate::conduct::Conductor;
63use crate::config::{Config, MergeMode};
64use crate::graph::Runner;
65use crate::land;
66use crate::notices::{self, Link, Notice};
67use crate::queue::{Queue, Task, TaskStatus};
68use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
69use crate::triage;
70
71pub const SCHEMA: u32 = 1;
73
74pub const HEARTBEAT: Duration = Duration::from_secs(5);
78
79pub const STALE_SECS: i64 = 30;
88
89pub const POLL: Duration = Duration::from_secs(5);
91
92pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
96
97pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
119#[serde(default)]
120pub struct Current {
121 pub task: String,
123 pub run: String,
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct Status {
135 pub schema: u32,
137 pub pid: u32,
139 pub started_at: Timestamp,
141 pub updated_at: Timestamp,
143 pub idle: bool,
145 pub current: Vec<Current>,
151 pub completed: usize,
153 pub polls: u64,
155 #[serde(default)]
157 pub question_recovery: Option<crate::question_delivery::Recovery>,
158}
159
160impl Status {
161 #[must_use]
163 pub fn new() -> Self {
164 let now = Timestamp::now();
165 Self {
166 schema: SCHEMA,
167 pid: std::process::id(),
168 started_at: now,
169 updated_at: now,
170 idle: true,
171 current: Vec::new(),
172 completed: 0,
173 polls: 0,
174 question_recovery: None,
175 }
176 }
177}
178
179impl Default for Status {
180 fn default() -> Self {
181 Self::new()
182 }
183}
184
185#[derive(Debug, Clone)]
187pub struct Opts {
188 pub repo: PathBuf,
190 pub config: Option<PathBuf>,
192 pub poll: Duration,
194 pub max_attempts: usize,
196 pub once: bool,
198 pub merge: Option<String>,
200 pub worktrees_root: Option<PathBuf>,
209}
210
211impl Default for Opts {
212 fn default() -> Self {
213 Self {
214 repo: PathBuf::from("."),
215 config: None,
216 poll: POLL,
217 max_attempts: 2,
218 once: false,
219 merge: None,
220 worktrees_root: None,
221 }
222 }
223}
224
225fn max_concurrent(n: usize) -> usize {
230 n.max(1)
231}
232
233#[must_use]
235pub fn status_path() -> PathBuf {
236 crate::run::home().join("daemon.json")
237}
238
239pub fn write_status(status: &Status) -> Result<()> {
241 write_status_to(&status_path(), status)
242}
243
244pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
249 if let Some(parent) = path.parent() {
250 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
251 }
252 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
253 let tmp = path.with_extension("json.tmp");
254 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
255 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
256 Ok(())
257}
258
259pub fn clear_status() {
262 clear_status_at(&status_path());
263}
264
265fn clear_status_at(path: &Path) {
269 let _ = std::fs::remove_file(path);
270}
271
272#[derive(Debug, Clone, Default)]
285pub struct Stop {
286 stopped: Arc<AtomicBool>,
290 busy: Arc<std::sync::atomic::AtomicUsize>,
296 wake: Arc<Notify>,
300 pause: crate::graph::Pause,
303}
304
305impl Stop {
306 #[must_use]
308 pub fn new() -> Self {
309 Self::default()
310 }
311
312 pub fn stop(&self) {
315 self.stopped.store(true, Ordering::SeqCst);
316 self.wake.notify_one();
320 }
321
322 #[must_use]
324 pub fn stopped(&self) -> bool {
325 self.stopped.load(Ordering::SeqCst)
326 }
327
328 #[must_use]
336 pub fn finishing(&self) -> bool {
337 self.stopped() && self.busy_now()
338 }
339
340 pub fn park(&self, cause: crate::run::ParkCause) {
351 self.pause.park_for(cause);
352 self.stop();
353 }
354
355 #[must_use]
357 pub fn parking(&self) -> bool {
358 self.pause.parked()
359 }
360
361 #[must_use]
363 pub fn pause(&self) -> crate::graph::Pause {
364 self.pause.clone()
365 }
366
367 #[must_use]
373 pub fn busy_now(&self) -> bool {
374 self.busy.load(Ordering::SeqCst) > 0
375 }
376
377 fn enter(&self) {
379 self.busy.fetch_add(1, Ordering::SeqCst);
380 }
381
382 fn exit(&self) {
385 self.busy.fetch_sub(1, Ordering::SeqCst);
386 }
387
388 async fn idle(&self, poll: Duration) {
390 tokio::select! {
391 () = tokio::time::sleep(poll) => {}
392 () = self.wake.notified() => {}
393 }
394 }
395}
396
397#[derive(Debug, Clone, Default, Deserialize)]
404#[serde(default)]
405pub struct Reading {
406 pub schema: u32,
408 pub pid: Option<u32>,
410 pub started_at: Option<Timestamp>,
412 pub updated_at: Option<Timestamp>,
414 pub idle: bool,
416 #[serde(deserialize_with = "de_current")]
428 pub current: Vec<Current>,
429 pub completed: u64,
431 pub polls: u64,
433 #[serde(default)]
435 pub question_recovery: Option<crate::question_delivery::Recovery>,
436}
437
438fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
441where
442 D: serde::Deserializer<'de>,
443{
444 #[derive(Deserialize)]
445 #[serde(untagged)]
446 enum Shape {
447 Many(Vec<Current>),
448 One(Current),
449 }
450 Ok(
451 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
452 Shape::Many(v) => v,
453 Shape::One(c) => vec![c],
454 }),
455 )
456}
457
458impl Reading {
459 #[must_use]
462 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
463 self.updated_at
464 .map(|at| (now.as_second() - at.as_second()).max(0))
465 }
466
467 #[must_use]
471 pub fn running(&self, now: Timestamp) -> bool {
472 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
473 }
474}
475
476#[must_use]
483pub fn read_status(home: &Path) -> Option<Reading> {
484 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
485 serde_json::from_str(&body).ok()
486}
487
488#[must_use]
500pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
501 read_status(home)
502 .filter(|reading| reading.running(now))
503 .map(|reading| reading.current)
504 .unwrap_or_default()
505}
506
507#[must_use]
509pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
510 current_work(home, now).iter().any(|c| c.run == run)
511}
512
513#[must_use]
523pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
524 current_work(home, now)
525 .iter()
526 .any(|c| crate::run::short_of(&c.run) == short)
527}
528
529#[must_use]
531pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
532 current_work(home, now).iter().any(|c| c.task == task)
533}
534
535pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
579 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
580}
581
582fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
586where
587 F: Fn(u32) -> bool,
588{
589 let this_process = std::process::id();
590 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
591 .into_iter()
592 .flatten()
593 .flatten()
594 .map(|e| e.path())
595 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
596 .filter(|p| {
597 match std::fs::read_to_string(p)
598 .ok()
599 .and_then(|body| body.trim().parse::<u32>().ok())
600 {
601 Some(pid) if pid == this_process => false,
605 Some(pid) => !pid_alive(pid),
606 None => p
607 .metadata()
608 .and_then(|m| m.modified())
609 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
610 .is_ok_and(|age| age >= older_than),
611 }
612 })
613 .filter(|p| std::fs::remove_file(p).is_ok())
614 .filter_map(|p| {
615 p.file_stem()
616 .and_then(|s| s.to_str())
617 .map(std::borrow::ToOwned::to_owned)
618 })
619 .collect();
620 swept.sort_unstable();
621 swept
622}
623
624fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
629 task.status == TaskStatus::Running
630 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
631 && !is_working_on_task(home, &task.id, now)
632}
633
634fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
637 queue
638 .list()
639 .into_iter()
640 .filter(|t| is_stalled(t, home, now))
641 .collect()
642}
643
644fn queued_tasks(queue: &Queue) -> Vec<Task> {
650 queue
651 .list()
652 .into_iter()
653 .filter(|t| t.status == TaskStatus::Queued)
654 .collect()
655}
656
657fn finished_tasks(queue: &Queue) -> Vec<Task> {
660 queue
661 .list()
662 .into_iter()
663 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
664 .collect()
665}
666
667fn resolve_blockers(queue: &Queue, questions: &Questions) {
684 for listed in queue.list() {
685 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
686 continue;
687 }
688 let Ok(_claim) = queue.claim(&listed.id) else {
689 continue;
690 };
691 let Ok(mut task) = queue.get(&listed.id) else {
692 continue;
693 };
694 if task.status != TaskStatus::Blocked {
695 continue;
696 }
697 let deleted = queue.apply_deleted_blockers(&mut task, questions);
701 if !deleted.is_empty() {
702 record(queue, &mut task);
703 for id in &deleted {
704 queue.note_dependency_deleted(&task, id);
705 }
706 if task.status != TaskStatus::Blocked {
707 continue;
708 }
709 }
710 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
711 if !missing.is_empty() {
712 let language = language_of(&task, Path::new("."));
713 let mut cause = missing.clone();
714 cause.sort();
715 cause.dedup();
716 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
717 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
718 &task.blocked_by,
719 &missing,
720 &language,
721 )));
722 record(queue, &mut task);
723 continue;
724 }
725 if let Some(q) = task.blocked_by.iter().find_map(|id| {
728 let crate::queue::Dependency::Question(full) =
729 crate::queue::dependency(queue, questions, id)
730 else {
731 return None;
732 };
733 questions.get(&full).ok().filter(|q| {
734 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
735 })
736 }) {
737 let language = language_of(&task, Path::new("."));
738 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
739 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
740 record(queue, &mut task);
741 continue;
742 }
743 let mut changed = false;
744 for id in task.blocked_by.clone() {
745 let resolved = crate::queue::dependency(queue, questions, &id);
746 if let crate::queue::Dependency::Task(full) = &resolved {
747 if queue
748 .get(full)
749 .is_ok_and(|dep| dep.status == TaskStatus::Done)
750 {
751 task.unblock(&id);
752 changed = true;
753 }
754 continue;
755 }
756 if let crate::queue::Dependency::Question(full) = resolved
757 && let Ok(q) = questions.get(&full)
758 && q.status == ask::QuestionStatus::Answered
759 {
760 if q.node == crate::conduct::NODE
764 && q.chosen_action().is_some()
765 && !task.action_applied(&q.id)
766 {
767 continue;
768 }
769 let answer = match &q.answer {
770 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
771 None => String::new(),
772 };
773 task.record_answer(q.summary.clone(), answer);
774 task.unblock(&id);
775 changed = true;
776 }
777 }
778 if changed {
779 record(queue, &mut task);
780 }
781 }
782}
783
784fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
786 if crate::lang::is_japanese(language) {
787 format!(
788 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
789 q.short(),
790 q.summary
791 )
792 } else {
793 format!(
794 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
795 q.short(),
796 q.summary
797 )
798 }
799}
800
801#[derive(Debug, Clone, Copy, PartialEq, Eq)]
804pub(crate) enum ActionStanding {
805 NoAction,
807 Pending,
809 Applied,
812 Stale,
814 Busy,
816}
817
818impl ActionStanding {
819 pub(crate) fn daemon_owns(self) -> bool {
821 matches!(self, Self::Pending | Self::Applied | Self::Stale)
822 }
823}
824
825pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
829 if q.chosen_action().is_none() {
830 ActionStanding::NoAction
831 } else if task.action_applied(&q.id) {
832 ActionStanding::Applied
833 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
834 ActionStanding::Stale
835 } else if matches!(
836 task.status,
837 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
838 ) {
839 ActionStanding::Busy
840 } else {
841 ActionStanding::Pending
842 }
843}
844
845#[derive(Debug, Clone, PartialEq, Eq)]
847enum ActionDecision {
848 Skip,
850 Resume(String),
852 Requeue,
854 Hold,
856 Done,
858 Stale,
861 Refuse(String),
864}
865
866fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
874where
875 F: FnOnce(&str) -> Result<RunState>,
876{
877 let Some(action) = q.chosen_action() else {
878 return ActionDecision::Skip;
879 };
880 if task.action_applied(&q.id)
881 || matches!(
882 task.status,
883 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
884 )
885 {
886 return ActionDecision::Skip;
887 }
888 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
891 return ActionDecision::Stale;
892 }
893 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
894 let stopping = q.node == land::APPROVAL_NODE
895 && q.resolution()
896 .as_deref()
897 .is_some_and(crate::merge_decision::stops);
898 let decision_at = if repair || stopping {
902 q.answered_at.unwrap_or(q.asked_at)
903 } else {
904 q.asked_at
905 };
906 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
907 return if repair || stopping {
908 ActionDecision::Refuse(format!(
909 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
910 if repair { "repair" } else { "stop" },
911 q.id,
912 q.run,
913 task.id
914 ))
915 } else {
916 ActionDecision::Stale
917 };
918 }
919 match action {
920 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
921 ask::ChoiceAction::Hold => ActionDecision::Hold,
922 ask::ChoiceAction::Done => ActionDecision::Done,
923 ask::ChoiceAction::Resume { run } => {
924 if task.runs.last() != Some(run) {
925 return ActionDecision::Refuse((p.resume_not_latest)(
926 q.short(),
927 ask::short_id(run),
928 ));
929 }
930 match load(run) {
931 Ok(s) if stopping => {
932 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
933 || !s
934 .land_approval
935 .as_ref()
936 .is_some_and(|binding| binding.question == q.id)
937 {
938 ActionDecision::Stale
939 } else if s.liveness(false) == Liveness::Live {
940 ActionDecision::Skip
941 } else if s.status == RunStatus::Landing && !s.released() {
942 ActionDecision::Resume(run.clone())
943 } else {
944 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
945 }
946 }
947 Ok(s)
948 if q.node == land::APPROVAL_NODE
949 && q.resolution().as_deref() == Some(land::REPAIR) =>
950 {
951 if s.merge_repairs.iter().any(|r| r.question == q.id)
952 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
953 {
954 ActionDecision::Stale
957 } else if s.liveness(false) == Liveness::Live {
958 ActionDecision::Skip
959 } else if s.status != RunStatus::Landing
960 || s.released()
961 || s.reviews.is_empty()
962 || !implementers_match(task, &s)
963 || !s
964 .land_approval
965 .as_ref()
966 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
967 {
968 ActionDecision::Refuse(format!(
969 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
970 q.id, run, run
971 ))
972 } else {
973 ActionDecision::Resume(run.clone())
974 }
975 }
976 Ok(s) if s.status.resumable() && !s.released() && !exhausted_review_budget(&s) => {
977 ActionDecision::Resume(run.clone())
978 }
979 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
980 q.short(),
981 ask::short_id(run),
982 )),
983 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
984 q.short(),
985 ask::short_id(run),
986 &format!("{e:#}"),
987 )),
988 }
989 }
990 }
991}
992
993struct Phrases {
1007 graph_stopped: fn(&str, &str) -> String,
1009 quorum_lost: &'static str,
1010 quota_took_out: &'static str,
1012 run_ended: &'static str,
1014 waiting_for_answer: &'static str,
1016 recovered_running: &'static str,
1017 no_run_to_recover: &'static str,
1018 could_not_start: &'static str,
1019 resume_not_latest: fn(&str, &str) -> String,
1020 resume_cannot_progress: fn(&str, &str) -> String,
1021 resume_unreadable: fn(&str, &str, &str) -> String,
1022 handover_refused: fn(&crate::handover::Refused) -> String,
1025 handover_hint: &'static str,
1027 reviewers_never_answered: fn(usize, usize) -> String,
1030}
1031
1032const PHRASES_EN: Phrases = Phrases {
1033 graph_stopped: |status, detail| {
1034 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1035 },
1036 quorum_lost: "the judging panel lost its quorum",
1037 quota_took_out: "; quota took out ",
1038 run_ended: "run ended ",
1039 waiting_for_answer: " - waiting for operator answer to question ",
1040 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1041 no_run_to_recover: "task was `running` with no live daemon and no readable \
1042 run to recover; held for a human to check what happened",
1043 could_not_start: "could not start the run: ",
1044 resume_not_latest: |q, run| {
1045 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1046 },
1047 resume_cannot_progress: |q, run| {
1048 format!("question {q} asked to resume run {run}, which cannot make progress")
1049 },
1050 resume_unreadable: |q, run, e| {
1051 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1052 },
1053 handover_refused: |r| r.to_string(),
1054 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1055 reviewers_never_answered: |missing, rounds| {
1056 format!(
1057 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1058 refusing to call it clean"
1059 )
1060 },
1061};
1062
1063const PHRASES_JA: Phrases = Phrases {
1064 graph_stopped: |status, detail| {
1065 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1066 },
1067 quorum_lost: "審査パネルが定足数を失いました",
1068 quota_took_out: "。クォータで脱落: ",
1069 run_ended: "run 終了: ",
1070 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1071 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1072 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1073 何が起きたか人が確認するため保留にしました",
1074 could_not_start: "run を開始できませんでした: ",
1075 resume_not_latest: |q, run| {
1076 format!(
1077 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1078 )
1079 },
1080 resume_cannot_progress: |q, run| {
1081 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1082 },
1083 resume_unreadable: |q, run, e| {
1084 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1085 },
1086 handover_refused: |r| {
1087 use crate::handover::Refused;
1088 match r {
1089 Refused::Foreign { branch, path, why } => format!(
1090 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1091 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1092 から、やり直してください(詳細: {why})"
1093 ),
1094 Refused::Unsafe { branch, path, why } => format!(
1095 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1096 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1097 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1098 ),
1099 Refused::ReleaseFailed { branch, path, run } => format!(
1100 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1101 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1102 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1103 ),
1104 }
1105 },
1106 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1107 reviewers_never_answered: |missing, rounds| {
1108 format!(
1109 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1110 クリーンとは認めません"
1111 )
1112 },
1113};
1114
1115fn phrases(language: &str) -> &'static Phrases {
1116 if crate::lang::is_japanese(language) {
1117 &PHRASES_JA
1118 } else {
1119 &PHRASES_EN
1120 }
1121}
1122
1123fn language_of(task: &Task, fallback: &Path) -> String {
1127 crate::lang::of_repo(&repo_for(task, fallback))
1128}
1129
1130pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1134 if q.node == crate::conduct::NODE {
1135 return tasks.iter().find(|t| t.id == q.run);
1136 }
1137 tasks.iter().find(|t| t.runs.contains(&q.run))
1138}
1139
1140pub(crate) fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1149 let tasks = queue.list();
1150 for q in questions.list() {
1151 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1153 continue;
1154 }
1155 if q.chosen_action().is_none() {
1156 continue;
1157 }
1158 let Some(listed) = task_of_question(&tasks, &q) else {
1159 if q.node == land::APPROVAL_NODE
1160 && q.resolution().as_deref().is_some_and(|choice| {
1161 choice == land::REPAIR || crate::merge_decision::stops(choice)
1162 })
1163 {
1164 notices::raise_in(
1165 home,
1166 Notice::warn(
1167 &format!("repair-owner:{}", q.id),
1168 format!(
1169 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1170 q.run, q.run
1171 ),
1172 ),
1173 );
1174 }
1175 continue;
1176 };
1177 if listed.action_applied(&q.id) {
1178 continue;
1179 }
1180 let Ok(_claim) = queue.claim(&listed.id) else {
1181 continue;
1182 };
1183 let Ok(mut task) = queue.get(&listed.id) else {
1184 continue;
1185 };
1186 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1189 let blockers: Vec<_> = task
1190 .blocked_by
1191 .iter()
1192 .filter(|id| {
1193 crate::queue::dependency(queue, questions, id)
1194 == crate::queue::Dependency::Question(q.id.clone())
1195 })
1196 .cloned()
1197 .collect();
1198 if !blockers.is_empty() {
1199 for id in blockers {
1200 task.unblock(&id);
1201 }
1202 let answer = match &q.answer {
1203 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1204 None => String::new(),
1205 };
1206 task.record_answer(q.summary.clone(), answer);
1207 }
1208 }
1209 let language = language_of(&task, Path::new("."));
1210 let decision = decide_action(&task, &q, phrases(&language), |id| {
1211 RunState::load_under(id, home)
1212 });
1213 let ran = matches!(
1214 decision,
1215 ActionDecision::Resume(_)
1216 | ActionDecision::Requeue
1217 | ActionDecision::Hold
1218 | ActionDecision::Done
1219 );
1220 if ran && asker_may_still_read(questions, &q, home) {
1221 continue;
1230 }
1231 let done = matches!(decision, ActionDecision::Done);
1232 match decision {
1233 ActionDecision::Skip => continue,
1234 ActionDecision::Resume(run) => {
1235 task.release_operator("choice", Some(&q.id));
1236 task.resume_override = Some(crate::queue::OperatorResume {
1237 question_id: q.id.clone(),
1238 at: Timestamp::now(),
1239 conductor_rehold: None,
1240 forced: true,
1241 pinned_run: Some(run),
1242 });
1243 }
1244 ActionDecision::Stale => {}
1245 ActionDecision::Requeue => {
1246 task.release_operator("choice-requeue", Some(&q.id));
1247 task.requeue();
1248 }
1249 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1250 ActionDecision::Done => task.succeed(),
1251 ActionDecision::Refuse(why) => {
1252 if task.operator_held() {
1253 task.hold_reason = Some(match task.hold_reason.take() {
1256 Some(reason) => format!("{reason}\n\n{why}"),
1257 None => why,
1258 });
1259 } else {
1260 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1261 task.hold_machine(Some(why));
1262 }
1263 notices::raise(
1264 Notice::warn(
1265 &format!("action:{}", q.id),
1266 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1267 )
1268 .link(Link::Task {
1269 id: task.id.clone(),
1270 }),
1271 );
1272 }
1273 }
1274 task.mark_action_applied(&q.id);
1275 record(queue, &mut task);
1278 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1279 supersede_prior_runs(&task, home);
1280 }
1281 let _ = questions.update(&q.id, |r| {
1284 r.answer_delivered = true;
1285 Ok(())
1286 });
1287 }
1288}
1289
1290fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1294 let now = Timestamp::now();
1295 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1296 return true;
1297 }
1298 q.node != crate::conduct::NODE
1299 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1300 s.seats_active()
1301 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1302 })
1303}
1304
1305fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1316 let tasks = queue.list();
1317 let by_id: std::collections::BTreeMap<&str, &Task> =
1318 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1319 let referenced: Vec<&str> = tasks
1320 .iter()
1321 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1322 .collect();
1323
1324 for mut question in questions.list() {
1325 if !question.status.open() || question.node != crate::conduct::NODE {
1326 continue;
1327 }
1328 if referenced
1333 .iter()
1334 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1335 {
1336 continue;
1337 }
1338 let Some(task) = by_id.get(question.run.as_str()) else {
1339 continue;
1340 };
1341 question.abandon(format!(
1342 "task {} no longer waits for this answer",
1343 task.short()
1344 ));
1345 if let Err(e) = questions.put(&mut question) {
1346 tracing::warn!(
1347 "could not retire question {} for task {}: {e:#}",
1348 question.short(),
1349 task.short()
1350 );
1351 }
1352 }
1353}
1354
1355#[derive(Debug, Clone, Copy)]
1362pub struct Verdict {
1363 pub status: RunStatus,
1365 pub left_pr: bool,
1367 pub quota_hit: bool,
1369 pub parked: bool,
1371 pub no_viable_candidates: bool,
1378}
1379
1380pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1437 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1438}
1439
1440fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1442 if verdict.parked {
1448 task.park(detail);
1449 return;
1450 }
1451 match verdict.status {
1452 RunStatus::Merged | RunStatus::Ready
1453 if task.pr_recovery.as_ref().is_some_and(|s| {
1454 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1455 }) =>
1456 {
1457 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1458 }
1459 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1460 RunStatus::AlreadyInBase => task.already_landed(detail),
1461 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1462 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1463 task.stall(detail)
1464 }
1465 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1466 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1467 RunStatus::Blocked => task.fail(detail, max_attempts),
1468 RunStatus::VerifiedNoop => task.handed_off(detail),
1469 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1470 }
1471}
1472
1473pub fn supersede_prior_runs(task: &Task, home: &Path) {
1522 let now = Timestamp::now();
1523 let last_run_succeeded = task
1530 .runs
1531 .last()
1532 .and_then(|id| RunState::load_under(id, home).ok())
1533 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1534 for id in task.superseded_attempts(last_run_succeeded) {
1535 let mut state = match RunState::load_under(id, home) {
1536 Ok(s) => s,
1537 Err(e) => {
1538 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1539 continue;
1540 }
1541 };
1542 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1543 continue;
1544 }
1545 let daemon_claims = is_working_on(home, id, now);
1546 if state.liveness(daemon_claims) == Liveness::Live {
1547 continue;
1548 }
1549 state.status = RunStatus::Superseded;
1550 if let Err(e) = state.save_under(home) {
1551 tracing::warn!("could not mark run {id} superseded: {e:#}");
1552 }
1553 }
1554}
1555
1556fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1578 for task in queue.list() {
1579 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1580 continue;
1581 }
1582 supersede_prior_runs(&task, home);
1583 }
1584}
1585
1586fn settle_and_diagnose(
1593 task: &mut Task,
1594 verdict: Verdict,
1595 detail: &str,
1596 max_attempts: usize,
1597 state: &RunState,
1598) {
1599 let p = phrases(&state.config.graph.language);
1600 if let Some(d) = &state.merge_decision {
1603 use crate::merge_decision as md;
1604 if task.runs.last().is_some_and(|id| id != &state.id) {
1605 return;
1606 }
1607 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1608 task.succeed();
1609 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1610 if state.liveness(false) != Liveness::Dead {
1611 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1612 return;
1613 }
1614 let replacement = md::Replacement {
1615 question: d.question.clone(),
1616 run: state.id.clone(),
1617 pr: d.target.pr.clone(),
1618 agent: agent.to_owned(),
1619 context: format!(
1620 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1621 d.target.pr,
1622 state.id,
1623 serde_json::to_string(&state.reviews).unwrap_or_default()
1624 ),
1625 };
1626 task.requeue();
1627 task.pr_recovery = None;
1628 task.review_of = None;
1629 task.replacement = Some(replacement);
1630 } else {
1631 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1632 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1633 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1634 }
1635 task.mark_action_applied(&d.question);
1636 if let Some(pass) = state.passes.last() {
1637 task.record_pass(&state.id, pass.index, task.attempts, None);
1638 }
1639 return;
1640 }
1641 if let Some(replacement) = &task.replacement
1642 && !state.parked
1643 && state.resource_block.is_none()
1644 && (state.viable().is_empty()
1645 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1646 {
1647 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1648 return;
1649 }
1650 let attempts_before = task.attempts;
1651 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1657 task.notice_cause(
1658 &format!("resource:{}:{}", block.kind, block.node),
1659 None,
1660 None,
1661 );
1662 task.last_error = Some(detail.to_owned());
1663 task.diagnostic = None;
1664 task.attempts = task.attempts.saturating_sub(1);
1665 task.hold_machine(Some(format!(
1666 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1667 no agent was asked and no attempt was spent",
1668 crate::cache::RESOURCE_HOLD,
1669 block.kind,
1670 block.node
1671 )));
1672 if let Some(pass) = state.passes.last() {
1673 task.record_pass(&state.id, pass.index, attempts_before, None);
1674 }
1675 return;
1676 }
1677 let unresolved = task.hold_notice_cause.clone();
1678 if !verdict.parked
1679 && task
1680 .pr_recovery
1681 .as_ref()
1682 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
1683 && matches!(
1684 verdict.status,
1685 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
1686 )
1687 {
1688 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
1689 task.succeed();
1690 } else {
1691 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
1692 }
1693 } else if !verdict.parked
1694 && task.pr_recovery.is_some()
1695 && matches!(
1696 verdict.status,
1697 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
1698 )
1699 && !verdict.quota_hit
1700 {
1701 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
1702 } else {
1703 settle_in(task, verdict, detail, max_attempts, p);
1704 }
1705 if state.prep_failure.is_some() {
1706 task.hold_notice_cause = unresolved;
1709 task.startup_notice_cause("startup-candidates");
1710 }
1711 if let Some(pass) = state.passes.last() {
1714 let parked = match &pass.end {
1715 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
1716 _ => None,
1717 };
1718 task.record_pass(&state.id, pass.index, attempts_before, parked);
1719 }
1720 if task.status == TaskStatus::Held {
1721 if state.prep_failure.is_none() {
1722 task.notice_cause(
1723 label(verdict.status),
1724 Some(&state.id),
1725 state.passes.last().map(|p| p.index),
1726 );
1727 }
1728 task.diagnostic = diagnostic(state);
1729 note_open_question(task, &state.id, p);
1730 }
1731}
1732
1733fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
1748 let Some(home) = crate::run::try_home() else {
1749 return;
1750 };
1751 let open = Questions::at(home.join("questions")).open_for(run);
1752 let Some(q) = open.first() else {
1753 return;
1754 };
1755 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
1756 decisions.sort_unstable();
1757 task.notice_cause(
1758 &format!("questions:{}", decisions.join(",")),
1759 Some(run),
1760 None,
1761 );
1762 let base = task.hold_reason.clone().unwrap_or_default();
1763 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
1764}
1765
1766fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
1777 match last_run {
1778 Some(state) => {
1779 let verdict = Verdict {
1780 status: state.status,
1781 left_pr: state.pr.is_some(),
1782 quota_hit: !state.quota.is_empty(),
1783 parked: state.parked,
1784 no_viable_candidates: state.viable().is_empty(),
1785 };
1786 let detail = format!(
1787 "{}{}",
1788 phrases(&state.config.graph.language).recovered_running,
1789 describe(&state)
1790 );
1791 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
1792 }
1793 None => {
1794 let why = phrases(language).no_run_to_recover;
1797 task.last_error = Some(why.to_owned());
1798 task.notice_cause("missing-run", None, None);
1801 task.hold_machine(Some(why.to_owned()));
1802 }
1803 }
1804}
1805
1806fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
1828 let mut reclaimed = Vec::new();
1829 for listed in queue.list() {
1830 if listed.status != TaskStatus::Running {
1831 continue;
1832 }
1833 let Ok(_claim) = queue.claim(&listed.id) else {
1834 continue;
1835 };
1836 let Ok(mut task) = queue.get(&listed.id) else {
1840 continue;
1841 };
1842 if task.status != TaskStatus::Running {
1843 continue;
1844 }
1845 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
1846 if let Some(state) = &last_run
1856 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
1857 {
1858 tracing::warn!("abandon questions for {}: {e:#}", state.id);
1859 }
1860 let language = if last_run.is_none() {
1861 language_of(&task, Path::new("."))
1862 } else {
1863 String::new()
1864 };
1865 reclaim(&mut task, last_run, max_attempts, &language);
1866 if task.status == TaskStatus::Done {
1867 supersede_prior_runs(&task, &crate::run::home());
1868 }
1869 record(queue, &mut task);
1870 reclaimed.push(task.id.clone());
1871 }
1872 reclaimed
1873}
1874
1875fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
1907 reclaim_abandoned_runs_with(
1908 home,
1909 now,
1910 crate::proc::pid_status,
1911 crate::proc::process_started_at,
1912 )
1913}
1914
1915fn reclaim_abandoned_runs_with<F, G>(
1921 home: &Path,
1922 now: Timestamp,
1923 query: F,
1924 identity: G,
1925) -> Vec<String>
1926where
1927 F: Fn(u32) -> Option<bool>,
1928 G: Fn(u32) -> Option<String>,
1929{
1930 let mut abandoned = Vec::new();
1931 for entry in std::fs::read_dir(home.join("runs"))
1932 .into_iter()
1933 .flatten()
1934 .flatten()
1935 {
1936 let id = entry.file_name().to_string_lossy().into_owned();
1937 if !crate::run::is_run_id(&id) {
1938 continue;
1939 }
1940 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
1943 continue;
1944 };
1945 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
1953 continue;
1954 };
1955 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
1956 continue;
1957 };
1958 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
1959 continue;
1960 }
1961 let daemon_claims = is_working_on(home, &id, now);
1973 if state.liveness_under_with(home, daemon_claims, &query, &identity)
1974 != crate::run::Liveness::Dead
1975 {
1976 continue;
1977 }
1978 state.abandon("daemon");
1979 if let Err(e) = state.save_under(home) {
1980 tracing::warn!("could not persist abandoned run {id}: {e:#}");
1981 continue;
1982 }
1983 if let Some(notice) = notices::run_ended(&state) {
1986 notices::raise_in(home, notice);
1987 }
1988 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
1996 tracing::warn!("abandon questions for {id}: {e:#}");
1997 }
1998 abandoned.push(id);
1999 }
2000 abandoned
2001}
2002
2003pub async fn serve(opts: Opts) -> Result<()> {
2009 serve_until(opts, Stop::new()).await
2010}
2011
2012pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2029 let signal = {
2030 let stop = stop.clone();
2031 tokio::spawn(async move {
2032 if tokio::signal::ctrl_c().await.is_ok() {
2033 stop.stop();
2034 tracing::info!("shutdown requested; a run in flight will be finished first");
2035 }
2036 })
2037 };
2038
2039 let worktrees_root = opts
2040 .worktrees_root
2041 .clone()
2042 .unwrap_or_else(crate::run::default_worktree_root);
2043 let outcome = drive(
2044 &opts,
2045 &Queue::open(),
2046 &status_path(),
2047 &crate::run::home(),
2048 &worktrees_root,
2049 &stop,
2050 )
2051 .await;
2052
2053 signal.abort();
2054 outcome
2055}
2056
2057async fn drive(
2070 opts: &Opts,
2071 queue: &Queue,
2072 status_file: &Path,
2073 home: &Path,
2074 worktrees_root: &Path,
2075 stop: &Stop,
2076) -> Result<()> {
2077 let recovery_cfg = prepare(&opts.repo, opts).ok();
2085 let daemon_cfg = recovery_cfg
2086 .as_ref()
2087 .map(|c| c.daemon.clone())
2088 .unwrap_or_default();
2089 let reconciled = crate::driver::reconcile(home);
2090 if !reconciled.is_empty() {
2091 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2092 }
2093 let mut initial_status = Status::new();
2094 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2095 recovery_cfg.as_ref(),
2096 daemon_cfg.max_deputies,
2097 ));
2098 let status = Arc::new(Mutex::new(initial_status));
2099 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2100 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2101
2102 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2108
2109 let waiter = tokio::spawn(crate::waiter::run(
2114 crate::waiter::Waiter::new(
2115 crate::ask::Questions::at(home.join("questions")),
2116 home.to_path_buf(),
2117 recovery_cfg.clone(),
2118 ),
2119 stop.clone(),
2120 ));
2121
2122 let deputies = tokio::spawn(crate::deputy::run(
2126 crate::deputy::Deputies::new(
2127 crate::ask::Questions::at(home.join("questions")),
2128 home.to_path_buf(),
2129 recovery_cfg.clone(),
2130 opts.repo.clone(),
2131 daemon_cfg.max_deputies,
2132 {
2133 let stop = stop.clone();
2134 Arc::new(move || stop.parking())
2135 },
2136 ),
2137 stop.clone(),
2138 ));
2139
2140 let fetcher = tokio::spawn(fetch_loop(opts.repo.clone(), opts.clone(), stop.clone()));
2143
2144 let release_watch = tokio::spawn(crate::release_watch::run(
2147 crate::release_watch::Watcher::new(
2148 Box::new(crate::release_watch::GhForge),
2149 home.to_path_buf(),
2150 ),
2151 {
2152 let (repo, opts) = (opts.repo.clone(), opts.clone());
2153 move || release_watch_settings(&repo, &opts)
2154 },
2155 stop.clone(),
2156 ));
2157
2158 tracing::info!(
2159 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2160 queue.root().display(),
2161 opts.poll.as_secs(),
2162 opts.max_attempts,
2163 concurrency,
2164 if daemon_cfg.pause_for_interrupts {
2165 ", interrupts enabled"
2166 } else {
2167 ""
2168 }
2169 );
2170
2171 janitor(&opts.repo, opts, home, worktrees_root).await;
2174 resweep_superseded_attempts(queue, home);
2175
2176 let outcome = poll(
2177 opts,
2178 queue,
2179 &status,
2180 home,
2181 worktrees_root,
2182 stop,
2183 DispatchLimits {
2184 max_concurrent: concurrency,
2185 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2186 },
2187 )
2188 .await;
2189
2190 beat.abort();
2191 waiter.abort();
2192 deputies.abort();
2193 fetcher.abort();
2194 release_watch.abort();
2195 clear_status_at(status_file);
2196 outcome
2197}
2198
2199fn release_watch_settings(repo: &Path, opts: &Opts) -> (Vec<PathBuf>, u64) {
2202 let Ok(cfg) = prepare(repo, opts) else {
2203 return (Vec::new(), 0);
2204 };
2205 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2206 .into_iter()
2207 .map(|r| r.path)
2208 .collect();
2209 let mut known: Vec<PathBuf> = Queue::open().list().into_iter().map(|t| t.repo).collect();
2212 let runs = crate::run::home().join("runs");
2213 for id in crate::run::list_ids_in(&runs) {
2214 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2215 .ok()
2216 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2217 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2218 known.extend(repo);
2219 }
2220 for k in known {
2221 let k = k.canonicalize().unwrap_or(k);
2222 if k.join(".git").exists() && !paths.contains(&k) {
2223 paths.push(k);
2224 }
2225 }
2226 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2227 if !paths.contains(&own) {
2228 paths.push(own);
2229 }
2230 (paths, cfg.daemon.release_stall_minutes)
2231}
2232
2233const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2235
2236async fn fetch_loop(repo: PathBuf, opts: Opts, stop: Stop) {
2240 while !stop.stopped() {
2241 let (interval, roots) = match prepare(&repo, &opts) {
2242 Ok(c) => (c.repos.fetch_interval, c.repos.roots),
2243 Err(_) => (0, Vec::new()),
2244 };
2245 if interval > 0 && !roots.is_empty() {
2246 let r = crate::clean::fetch_origins(&roots, FETCH_TIMEOUT, || stop.stopped()).await;
2247 tracing::debug!("fetch origins: {r:?}");
2248 }
2249 let wait = if interval > 0 { interval } else { 60 };
2251 let mut slept = 0;
2252 while slept < wait && !stop.stopped() {
2253 tokio::time::sleep(Duration::from_secs(1)).await;
2254 slept += 1;
2255 }
2256 }
2257}
2258
2259async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2265 loop {
2266 tokio::time::sleep(HEARTBEAT).await;
2267 let snapshot = {
2268 let mut guard = lock(&status);
2269 guard.updated_at = Timestamp::now();
2270 guard.clone()
2271 };
2272 if let Err(e) = write_status_to(&path, &snapshot) {
2273 tracing::warn!("could not refresh the daemon status file: {e:#}");
2276 }
2277 }
2278}
2279
2280#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2283enum LandResume {
2284 NotLanding,
2287 StillWaiting,
2292 Ready,
2296}
2297
2298fn land_resume_state(task: &Task) -> LandResume {
2302 let Some(run_id) = task.runs.last() else {
2303 return LandResume::NotLanding;
2304 };
2305 let Ok(state) = RunState::load(run_id) else {
2306 return LandResume::NotLanding;
2307 };
2308 if !state.parked {
2309 return LandResume::NotLanding;
2310 }
2311 if state.status == RunStatus::Landing
2312 && state.land_approval.as_ref().is_some_and(|binding| {
2313 ask::Questions::open()
2314 .get(&binding.question)
2315 .is_ok_and(|q| {
2316 q.run == state.id
2317 && q.resolution()
2318 .as_deref()
2319 .is_some_and(crate::merge_decision::stops)
2320 })
2321 })
2322 {
2323 return LandResume::Ready;
2324 }
2325 if !implementers_match(task, &state) {
2326 return LandResume::NotLanding;
2327 }
2328 let node = if state.status == RunStatus::Landing {
2332 land::APPROVAL_NODE
2333 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2334 crate::github_text::ASK_NODE
2335 } else {
2336 return LandResume::NotLanding;
2337 };
2338 let store = ask::Questions::open();
2339 let waiting = store
2340 .list()
2341 .into_iter()
2342 .filter(|q| &q.run == run_id && q.node == node)
2343 .max_by(|a, b| a.id.cmp(&b.id));
2344 let Some(mut q) = waiting else {
2345 return LandResume::Ready;
2346 };
2347 if !q.status.open() {
2348 if q.resolution().as_deref() == Some(land::REPAIR) {
2351 return if state.liveness(false) == Liveness::Live {
2352 LandResume::StillWaiting
2353 } else {
2354 LandResume::NotLanding
2355 };
2356 }
2357 return LandResume::Ready;
2358 }
2359 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2366 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2367 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2368 q.abandon(format!(
2369 "no answer within {}s of asking",
2370 timeout.as_secs().max(1)
2371 ));
2372 if store.put(&mut q).is_ok() {
2375 return LandResume::Ready;
2376 }
2377 }
2378 LandResume::StillWaiting
2379}
2380
2381const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2389
2390const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2402
2403struct InFlightGuard<'a> {
2416 status: &'a Arc<Mutex<Status>>,
2417 stop: &'a Stop,
2418 task_id: &'a str,
2419}
2420
2421impl Drop for InFlightGuard<'_> {
2422 fn drop(&mut self) {
2423 lock(self.status).current.retain(|c| c.task != self.task_id);
2424 self.stop.exit();
2425 }
2426}
2427
2428#[derive(Debug, Clone, PartialEq, Eq)]
2450enum Interrupt {
2451 Idle,
2453 Parking {
2465 parked: Vec<String>,
2466 interrupt_task: String,
2467 },
2468 Running {
2476 parked: Vec<String>,
2477 interrupt_task: String,
2478 },
2479 Resuming { parked: Vec<String> },
2486}
2487
2488fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
2509 match state {
2510 Interrupt::Idle => {
2511 if in_flight.len() != 1 {
2523 return Interrupt::Idle;
2524 }
2525 match runnable.iter().find(|t| t.interrupt) {
2526 Some(t) => Interrupt::Parking {
2527 parked: in_flight.to_vec(),
2528 interrupt_task: t.id.clone(),
2529 },
2530 None => Interrupt::Idle,
2531 }
2532 }
2533 Interrupt::Parking {
2534 parked,
2535 interrupt_task,
2536 } => {
2537 if in_flight.iter().any(|id| parked.contains(id)) {
2538 Interrupt::Parking {
2540 parked,
2541 interrupt_task,
2542 }
2543 } else if in_flight.contains(&interrupt_task) {
2544 Interrupt::Running {
2545 parked,
2546 interrupt_task,
2547 }
2548 } else if runnable.iter().any(|t| t.id == interrupt_task) {
2549 Interrupt::Parking {
2553 parked,
2554 interrupt_task,
2555 }
2556 } else {
2557 Interrupt::Resuming { parked }
2562 }
2563 }
2564 Interrupt::Running {
2565 parked,
2566 interrupt_task,
2567 } => {
2568 if in_flight.contains(&interrupt_task) {
2569 Interrupt::Running {
2570 parked,
2571 interrupt_task,
2572 }
2573 } else {
2574 Interrupt::Resuming { parked }
2580 }
2581 }
2582 Interrupt::Resuming { parked } => {
2583 if in_flight.iter().any(|id| parked.contains(id)) {
2584 Interrupt::Idle
2590 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
2591 Interrupt::Resuming { parked }
2592 } else {
2593 Interrupt::Idle
2596 }
2597 }
2598 }
2599}
2600
2601fn advance_interrupt_tick(
2607 enabled: bool,
2608 state: Interrupt,
2609 in_flight: &[String],
2610 runnable: &[Task],
2611) -> Interrupt {
2612 if !enabled {
2613 return Interrupt::Idle;
2614 }
2615 advance_interrupt(state, in_flight, runnable)
2616}
2617
2618fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
2623 match state {
2624 Interrupt::Idle => candidates,
2625 Interrupt::Parking {
2626 parked,
2627 interrupt_task,
2628 } => {
2629 if in_flight.iter().any(|id| parked.contains(id)) {
2630 Vec::new()
2631 } else {
2632 candidates
2633 .into_iter()
2634 .filter(|t| &t.id == interrupt_task)
2635 .collect()
2636 }
2637 }
2638 Interrupt::Running { .. } => Vec::new(),
2639 Interrupt::Resuming { parked } => candidates
2647 .into_iter()
2648 .find(|t| parked.contains(&t.id))
2649 .into_iter()
2650 .collect(),
2651 }
2652}
2653
2654struct DispatchLimits {
2658 max_concurrent: usize,
2661 pause_for_interrupts: bool,
2663}
2664
2665#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2674enum PermitKind {
2675 None,
2680 Urgent,
2687 Ordinary,
2690}
2691
2692fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
2695 if priority {
2696 PermitKind::None
2697 } else if urgent {
2698 PermitKind::Urgent
2699 } else {
2700 PermitKind::Ordinary
2701 }
2702}
2703
2704struct DispatchClaim {
2705 task: Task,
2706 claim: crate::queue::Claim,
2707 permit: Option<tokio::sync::OwnedSemaphorePermit>,
2708}
2709
2710fn claim_dispatch(
2713 queue: &Queue,
2714 candidate: &Task,
2715 cooling_down: bool,
2716 ordinary: &Arc<tokio::sync::Semaphore>,
2717 urgent: &Arc<tokio::sync::Semaphore>,
2718) -> Option<DispatchClaim> {
2719 let Ok(claim) = queue.claim(&candidate.id) else {
2720 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
2721 return None;
2722 };
2723 let task = match queue.get(&candidate.id) {
2724 Ok(t) if t.status.runnable() => t,
2725 Ok(_) => return None,
2726 Err(e) => {
2727 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
2728 return None;
2729 }
2730 };
2731 let resume = land_resume_state(&task);
2732 if resume == LandResume::StillWaiting {
2733 return None;
2734 }
2735 let priority = resume == LandResume::Ready;
2736 if !priority && cooling_down {
2737 return None;
2738 }
2739 let permit = match permit_kind(priority, task.urgent) {
2742 PermitKind::None => None,
2743 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
2744 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
2745 };
2746 Some(DispatchClaim {
2747 task,
2748 claim,
2749 permit,
2750 })
2751}
2752
2753async fn poll(
2772 opts: &Opts,
2773 queue: &Queue,
2774 status: &Arc<Mutex<Status>>,
2775 home: &Path,
2776 worktrees_root: &Path,
2777 stop: &Stop,
2778 limits: DispatchLimits,
2779) -> Result<()> {
2780 let DispatchLimits {
2781 max_concurrent,
2782 pause_for_interrupts,
2783 } = limits;
2784 let mut attempted: Vec<String> = Vec::new();
2789 let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
2790 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(1));
2798 let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
2804 let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
2805 let mut conductor = Conductor::new();
2806 let mut cache_last_checked: Option<Timestamp> = None;
2809 let mut interrupt = Interrupt::Idle;
2811 let mut interrupt_pauses: std::collections::HashMap<String, crate::graph::Pause> =
2817 std::collections::HashMap::new();
2818
2819 while !stop.stopped() {
2820 lock(status).polls += 1;
2821
2822 while let Some(result) = inflight.try_join_next() {
2827 if let Err(e) = result {
2828 tracing::error!("a spawned attempt did not finish cleanly: {e}");
2829 notices::raise(Notice::error(
2830 "loop:attempt",
2831 "A queued attempt ended abnormally; check the task it was running.",
2832 ));
2833 }
2834 }
2835
2836 let swept = sweep_stale_claims(queue, STALE_CLAIM);
2837 if !swept.is_empty() {
2838 tracing::warn!(
2839 "swept {} stale claim(s) left behind by an earlier daemon: {}",
2840 swept.len(),
2841 swept.join(", ")
2842 );
2843 }
2844 let now = Timestamp::now();
2849
2850 if !stop.busy_now() {
2855 maybe_prune_cache_between_runs(
2856 &opts.repo,
2857 opts,
2858 home,
2859 stop,
2860 &mut cache_last_checked,
2861 now,
2862 )
2863 .await;
2864 }
2865
2866 let stalled = stalled_tasks(queue, home, now);
2867 let stalled_ids: std::collections::BTreeSet<_> =
2868 stalled.iter().map(|task| task.id.clone()).collect();
2869 let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
2870 if !reclaimed.is_empty() {
2871 tracing::warn!(
2872 "reclaimed {} task(s) left `running` by a daemon that never \
2873 recorded the outcome: {}",
2874 reclaimed.len(),
2875 reclaimed.join(", ")
2876 );
2877 }
2878 let abandoned_runs = reclaim_abandoned_runs(home, now);
2879 if !abandoned_runs.is_empty() {
2880 tracing::warn!(
2881 "failed {} run(s) left behind by a killed process, past every \
2882 active seat's own timeout: {}",
2883 abandoned_runs.len(),
2884 abandoned_runs.join(", ")
2885 );
2886 }
2887
2888 let questions = Questions::at(home.join("questions"));
2893
2894 resolve_blockers(queue, &questions);
2897 apply_choice_actions(queue, &questions, home);
2898 triage::apply_answers(queue, &questions);
2899 reconcile_task_questions(queue, &questions);
2900 {
2904 let talks = crate::talk::Talks::at(home.join("talks"));
2905 crate::chat_notify::sweep(queue, &talks, home);
2906 crate::chat_watch::sweep(home, &talks);
2907 crate::chat_notify::start_turns(&talks, stop.parking(), &mut inflight);
2908 }
2909
2910 let finished: Vec<Task> = finished_tasks(queue)
2916 .into_iter()
2917 .filter(|task| !stalled_ids.contains(&task.id))
2918 .filter(|task| !awaiting_resume_with(task, |id| RunState::load_under(id, home)))
2922 .collect();
2923 let queued = queued_tasks(queue);
2924 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
2928 && conductor.worth_a_look(queue, &stalled, &finished)
2929 {
2930 match prepare(&opts.repo, opts) {
2931 Ok(cfg) => {
2932 conductor
2933 .maybe_run(
2934 &cfg,
2935 &opts.repo,
2936 queue,
2937 &questions,
2938 home,
2939 &queued,
2940 &stalled,
2941 &finished,
2942 opts.max_attempts,
2943 )
2944 .await;
2945 }
2946 Err(e) => {
2947 tracing::warn!("conductor: no config: {e:#}");
2948 notices::raise(Notice::warn(
2949 "loop:no-config",
2950 "The loop could not read this repository's config, so held tasks are not being triaged.",
2951 ));
2952 }
2953 }
2954 }
2955
2956 let candidates: Vec<Task> = runnable(queue)
2957 .into_iter()
2958 .filter(|t| !opts.once || !attempted.contains(&t.id))
2959 .collect();
2960
2961 let in_flight: Vec<String> = lock(status)
2966 .current
2967 .iter()
2968 .map(|c| c.task.clone())
2969 .collect();
2970 interrupt_pauses.retain(|id, _| in_flight.contains(id));
2971
2972 interrupt =
2973 advance_interrupt_tick(pause_for_interrupts, interrupt, &in_flight, &candidates);
2974 if let Interrupt::Parking {
2975 parked,
2976 interrupt_task,
2977 } = &interrupt
2978 {
2979 let reason = format!(
2980 "task {} asked to run first",
2981 crate::run::short_of(interrupt_task)
2982 );
2983 for id in parked {
2984 if let Some(pause) = interrupt_pauses.get(id) {
2985 pause.park_because(reason.clone());
2986 }
2987 }
2988 }
2989 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
2990
2991 let cooling_down =
2992 lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
2993
2994 let mut started_any = false;
2995 for candidate in candidates {
2996 if stop.stopped() {
2997 break;
2998 }
2999
3000 let Some(DispatchClaim {
3001 mut task,
3002 claim,
3003 permit,
3004 }) = claim_dispatch(queue, &candidate, cooling_down, &sem, &urgent_sem)
3005 else {
3006 continue;
3007 };
3008
3009 let task_id = task.id.clone();
3010 attempted.push(task_id.clone());
3011 lock(status).idle = false;
3012 stop.enter();
3015 started_any = true;
3016
3017 let run_pause = crate::graph::Pause::new();
3021 interrupt_pauses.insert(task_id.clone(), run_pause.clone());
3022
3023 let opts = opts.clone();
3024 let queue = queue.clone();
3025 let status = Arc::clone(status);
3026 let stop = stop.clone();
3027 let quota_cooldown_until = Arc::clone("a_cooldown_until);
3028 inflight.spawn(async move {
3029 let _claim = claim;
3033 let _permit = permit;
3034 let _inflight = InFlightGuard {
3036 status: &status,
3037 stop: &stop,
3038 task_id: &task_id,
3039 };
3040 let quota = attempt(&opts, &queue, &status, &stop, run_pause, &mut task).await;
3041 lock(&status).completed += 1;
3042 let now = Timestamp::now();
3048 if let Some(until) = cooldown_until("a, now) {
3049 let wait = until.as_second() - now.as_second();
3050 *lock("a_cooldown_until) = Some(until);
3051 let hint = quota
3052 .iter()
3053 .find(|q| q.reset.is_some())
3054 .and_then(|q| q.reset.as_deref());
3055 match hint {
3056 Some(h) => tracing::warn!(
3057 "quota hit; waiting {wait}s before taking another ordinary task \
3058 (CLI reported reset: {h})"
3059 ),
3060 None => tracing::warn!(
3061 "quota hit; waiting {wait}s before taking another ordinary task \
3062 (no reset hint reported)"
3063 ),
3064 }
3065 }
3066 });
3067 }
3068
3069 if started_any {
3070 continue;
3071 }
3072
3073 if stop.busy_now() {
3074 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3079 continue;
3080 }
3081
3082 lock(status).idle = true;
3084 if opts.once {
3085 janitor(&opts.repo, opts, home, worktrees_root).await;
3089 resweep_superseded_attempts(queue, home);
3090 triage_held(queue, home, opts).await;
3091 break;
3092 }
3093 stop.idle(opts.poll).await;
3094 if stop.stopped() {
3095 continue;
3096 }
3097 janitor(&opts.repo, opts, home, worktrees_root).await;
3103 resweep_superseded_attempts(queue, home);
3104 triage_held(queue, home, opts).await;
3105 }
3106
3107 while let Some(result) = inflight.join_next().await {
3112 if let Err(e) = result {
3113 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3114 notices::raise(Notice::error(
3115 "loop:attempt",
3116 "A queued attempt ended abnormally; check the task it was running.",
3117 ));
3118 }
3119 }
3120 Ok(())
3121}
3122
3123async fn attempt(
3129 opts: &Opts,
3130 queue: &Queue,
3131 status: &Arc<Mutex<Status>>,
3132 stop: &Stop,
3133 interrupt_pause: crate::graph::Pause,
3134 task: &mut Task,
3135) -> Vec<QuotaLoss> {
3136 let repo = repo_for(task, &opts.repo);
3137 tracing::info!(
3138 "task {} — {} (repo {})",
3139 task.short(),
3140 task.title,
3141 repo.display()
3142 );
3143
3144 if let Some(previous) = task.runs.last().and_then(|id| RunState::load(id).ok())
3148 && previous
3149 .merge_decision
3150 .as_ref()
3151 .is_some_and(|decision| !decision.closed)
3152 && previous.status != RunStatus::Landing
3153 && !crate::merge_decision::resumable_hold(&previous)
3154 {
3155 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3156 record(queue, task);
3157 return Vec::new();
3158 }
3159 let stop_owner = task
3160 .runs
3161 .last()
3162 .and_then(|id| RunState::load(id).ok())
3163 .filter(|state| {
3164 crate::merge_decision::resumable_hold(state)
3165 || state.status == RunStatus::Landing
3166 && state.land_approval.as_ref().is_some_and(|binding| {
3167 Questions::open().get(&binding.question).is_ok_and(|q| {
3168 q.run == state.id
3169 && q.resolution()
3170 .as_deref()
3171 .is_some_and(crate::merge_decision::stops)
3172 })
3173 })
3174 });
3175 if let Some(owner) = stop_owner {
3176 let mut runner = match Runner::resume(&owner.id) {
3177 Ok(runner) => runner,
3178 Err(e) => {
3179 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
3180 record(queue, task);
3181 return Vec::new();
3182 }
3183 };
3184 runner.on_pause(stop.pause());
3185 runner.watch_interrupt(interrupt_pause);
3186 let quota_before = runner.state.quota.clone();
3187 task.start(owner.id.clone());
3188 record(queue, task);
3189 lock(status).current.push(Current {
3190 task: task.id.clone(),
3191 run: owner.id,
3192 });
3193 runner.state.next_pass_task_entry = true;
3194 let result = runner.execute().await;
3195 return finish_attempt(
3196 opts.max_attempts,
3197 queue,
3198 task,
3199 &runner.state,
3200 "a_before,
3201 result,
3202 );
3203 }
3204
3205 let unfinished = (!task.fresh_start)
3206 .then(|| unfinished_run(&task.runs, task.short()))
3207 .flatten()
3208 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
3209 let previous = task.runs.last().and_then(|id| RunState::load(id).ok());
3210 let saved_recovery = previous.as_ref().filter(|run| {
3211 task.replacement.is_none()
3212 && task.pr_recovery.as_ref().is_some_and(|selected| {
3213 selected.check_task(&task.id, &task.instruction).is_ok()
3214 && selected_starter(selected, Some(run), unfinished.as_deref())
3215 == Starter::Resume(run.id.clone())
3216 })
3217 });
3218
3219 let explicit = match &task.overrides {
3223 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
3224 Some(_) => opts.config.as_deref(),
3225 None => opts.config.as_deref(),
3226 };
3227 let (mut config, layers) = if let Some(run) = saved_recovery {
3228 let mut saved = run.clone();
3231 if let Err(e) = saved.configure_recovery_resume() {
3232 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
3233 record(queue, task);
3234 return Vec::new();
3235 }
3236 (saved.config, Vec::new())
3237 } else {
3238 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
3239 Ok(loaded) => loaded,
3240 Err(e) => {
3241 let reason = format!("[config] {e:#}");
3242 record_config_failure(task, &e);
3243 task.last_error = Some(reason.clone());
3244 task.hold_machine(Some(reason));
3245 record(queue, task);
3246 return Vec::new();
3247 }
3248 };
3249 if let Err(e) = apply_options(&mut config, opts, task) {
3250 task.hold_manual(Some(format!(
3251 "[config] {e:#}; correct the merge override before releasing the task"
3252 )));
3253 record(queue, task);
3254 return Vec::new();
3255 }
3256 if let Some(selected) = &task.pr_recovery
3257 && let Err(e) = selected.configure(&mut config)
3258 {
3259 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
3260 record(queue, task);
3261 return Vec::new();
3262 }
3263 (config, layers)
3264 };
3265 let config_evidence = crate::pr_recovery::ConfigEvidence {
3266 layers,
3267 overrides: task.overrides.clone(),
3268 daemon_merge: task
3269 .overrides
3270 .as_ref()
3271 .is_none_or(|o| o.inherit_daemon_options)
3272 .then(|| opts.merge.clone())
3273 .flatten(),
3274 };
3275 apply_task_implementers(&mut config, task);
3276 if let Some(replacement) = &task.replacement {
3277 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
3278 old.liveness(false) == Liveness::Dead
3279 && old.merge_decision.as_ref().is_some_and(|d| {
3280 d.closed
3281 && d.question == replacement.question
3282 && d.replacement_run.as_ref().is_none_or(|run| {
3283 task.runs.last() == Some(run)
3284 && RunState::load(run).is_ok_and(|current| {
3285 current.replaces.as_ref().is_some_and(|pin| {
3286 pin.question == replacement.question
3287 && pin.agent == replacement.agent
3288 })
3289 })
3290 })
3291 })
3292 });
3293 if !safe {
3294 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
3295 record(queue, task);
3296 return Vec::new();
3297 }
3298 match config.agent(&replacement.agent) {
3299 Ok(agent) if crate::agent::installed(agent) => {}
3300 _ => {
3301 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
3302 record(queue, task);
3303 return Vec::new();
3304 }
3305 }
3306 config.graph.implementers = 1;
3307 config.graph.land_approval = true;
3308 config.merge.mode = MergeMode::Pr;
3309 config.graph.selected_implementer = Some(replacement.agent.clone());
3310 }
3311 let p = phrases(&config.graph.language);
3312 let start_failed = p.could_not_start;
3313
3314 if let Some(reason) = disk_gate(&repo, &config) {
3322 task.notice_cause("disk-space", None, None);
3323 task.last_error = Some(reason.clone());
3324 task.hold_machine(Some(reason.clone()));
3325 record(queue, task);
3326 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
3327 notices::raise(
3330 Notice::warn(
3331 &format!("disk:{}", repo.display()),
3332 "A task was held for want of free disk space; free some, then release it from the queue.",
3333 )
3334 .link(Link::Task {
3335 id: task.id.clone(),
3336 }),
3337 );
3338 return Vec::new();
3339 }
3340
3341 let review_branch = take_review_branch(task, previous.as_ref());
3366 let branch_exists = match &review_branch {
3367 Some(branch) => crate::git::branch_exists(&repo, branch)
3368 .await
3369 .unwrap_or(false),
3370 None => false,
3371 };
3372 let proposed = match (&review_branch, &unfinished, &task.review_of) {
3376 (None, None, Some(branch)) => Starter::Review(branch.clone()),
3377 _ => choose_starter(
3378 review_branch.as_deref(),
3379 branch_exists,
3380 unfinished.as_deref(),
3381 ),
3382 };
3383 let retry_selected = task.replacement.is_some()
3384 && task.resumption.is_some()
3385 && !task.fresh_start
3386 && previous.as_ref().is_some_and(|run| {
3387 run.replaces.is_some()
3388 && ((run.status == RunStatus::Failed && run.viable().is_empty())
3389 || (run.status == RunStatus::Blocked && run.tally.is_none()))
3390 });
3391 let stop_pin = task
3392 .resume_override
3393 .as_ref()
3394 .filter(|pin| {
3395 Questions::open().get(&pin.question_id).is_ok_and(|q| {
3396 q.node == land::APPROVAL_NODE
3397 && q.resolution()
3398 .as_deref()
3399 .is_some_and(crate::merge_decision::stops)
3400 })
3401 })
3402 .and_then(|pin| pin.pinned_run.clone());
3403 let starter = if let Some(run) = stop_pin {
3404 Starter::Resume(run)
3407 } else if retry_selected {
3408 Starter::Resume(previous.as_ref().unwrap().id.clone())
3409 } else if let Some(selected) = &task.pr_recovery {
3410 if selected.check_task(&task.id, &task.instruction).is_ok() {
3411 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
3412 } else {
3413 Starter::Review(selected.branch.clone())
3414 }
3415 } else {
3416 starter_with_implementers(task, previous.as_ref(), proposed)
3417 };
3418 if let Some(pin) = &task.resume_override
3421 && let Some(run) = &pin.pinned_run
3422 && Questions::open().get(&pin.question_id).map_or(true, |q| {
3423 q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR)
3424 })
3425 && starter != Starter::Resume(run.clone())
3426 {
3427 let run = run.clone();
3428 task.notice_cause("repair-owner", Some(&run), None);
3429 task.hold_machine(Some(format!(
3430 "repair owner cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
3431 task.id
3432 )));
3433 record(queue, task);
3434 return Vec::new();
3435 }
3436 if task.replacement.is_some()
3437 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
3438 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
3439 {
3440 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
3441 record(queue, task);
3442 return Vec::new();
3443 }
3444 let attachments = match task_attachments(queue, task) {
3449 Ok(a) => a,
3450 Err(e) => {
3451 task.attempts += 1;
3452 task.fail_startup_target(
3453 "startup-attachments",
3454 e.name.clone(),
3455 format!("{start_failed}{e:#}"),
3456 opts.max_attempts,
3457 );
3458 record(queue, task);
3459 return Vec::new();
3460 }
3461 };
3462 let startup_pause = stop.pause();
3463 let startup_interrupt = interrupt_pause.clone();
3464 let wait = crate::repo_lock::WaitPolicy::new(
3465 Duration::from_secs(config.graph.timeout_resource_wait),
3466 move || startup_pause.parked() || startup_interrupt.parked(),
3467 );
3468 let started = wait.scope(async { match &starter {
3469 Starter::Review(_) if task.pr_recovery.is_some() => {
3470 let selected = task.pr_recovery.clone().expect("selected recovery");
3471 match selected.check_task(&task.id, &task.instruction) {
3472 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
3473 .await
3474 .map(|mut r| {
3475 r.state.attachments = attachments.clone();
3476 r
3477 }),
3478 Err(e) => Err(e),
3479 }
3480 }
3481 Starter::Review(branch) => {
3482 tracing::info!(
3483 "task {} reopens `{branch}` as a review-only pass",
3484 task.short()
3485 );
3486 let takeover = crate::handover::Takeover {
3489 earlier: task.earlier_attempts().to_vec(),
3490 home: crate::run::home(),
3491 choice: take_divergence_answer(branch, &config.merge.remote, task),
3492 task_instruction: Some(instruction_for(task)),
3493 };
3494 Runner::review_taking_over(
3495 &repo,
3496 branch,
3497 config,
3498 Some(takeover),
3499 crate::run::Origin::queue(&task.id),
3500 )
3501 .await
3502 }
3503 Starter::Resume(id) => {
3504 tracing::info!("resuming run {id} rather than competing again");
3505 Runner::resume(id).and_then(|mut r| {
3506 if retry_selected {
3507 for candidate in &mut r.state.candidates {
3510 candidate.failed = None;
3511 candidate.empty = false;
3512 candidate.commits = 0;
3513 }
3514 r.state.status = RunStatus::Implementing;
3515 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
3516 }
3517 if r.state.prep_failure.is_some() {
3518 let approval = task
3519 .resumption
3520 .as_ref()
3521 .context("preparation retry requires an explicit task release")?;
3522 r.retry_preparation(approval.at)?;
3523 }
3524 if let Some(instruction) =
3525 prepare_instruction(&starter, Some(&r.state.instruction), task)
3526 {
3527 r.state.instruction = instruction;
3528 }
3529 r.state.attachments = attachments.clone();
3530 if task.replacement.is_some() {
3531 r.state.config.merge.mode = MergeMode::Pr;
3532 r.state.config.graph.land_approval = true;
3533 }
3534 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
3538 .overrides
3539 .as_ref()
3540 .and_then(|o| o.merge.as_deref())
3541 .and_then(|m| merge_mode(m).ok())
3542 {
3543 r.state.config.merge.mode = mode;
3544 }
3545 Ok(r)
3546 })
3547 }
3548 Starter::Start => {
3549 if let Some(branch) = &review_branch {
3550 tracing::warn!(
3551 "conductor chose review for task {} but branch `{branch}` no longer \
3552 exists; requeuing as a fresh competition instead",
3553 task.short()
3554 );
3555 }
3556 let instruction = prepare_instruction(&starter, None, task)
3557 .unwrap_or_else(|| task.instruction.clone());
3558 Runner::start_naming(
3559 &repo,
3560 instruction,
3561 &task.title,
3562 config,
3563 crate::run::Origin::queue(&task.id),
3564 )
3565 .await
3566 .map(|mut r| {
3567 r.state.attachments = attachments.clone();
3568 r
3569 })
3570 }
3571 }}).await;
3572 let mut runner = match started {
3573 Ok(r) => r,
3574 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
3575 task.last_error = Some(e.to_string());
3576 if stop.parking() || interrupt_pause.parked() {
3577 let attempts = task.attempts;
3578 task.park(e.to_string());
3579 task.attempts = attempts;
3581 } else {
3582 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
3583 }
3584 record(queue, task);
3585 return Vec::new();
3586 }
3587 Err(e) if task.pr_recovery.is_some() => {
3591 task.startup_notice_cause("startup-public-head");
3592 task.attempts += 1;
3593 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
3594 task.link_run(&partial.run);
3595 task.record_pass(&partial.run, 0, task.attempts, None);
3596 }
3597 task.last_error = Some(format!("{start_failed}{e:#}"));
3598 task.hold_unrequested_retry(Some(format!(
3599 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
3600 )));
3601 record(queue, task);
3602 return Vec::new();
3603 }
3604 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
3605 let detail = match e.downcast_ref::<crate::handover::Refused>() {
3606 Some(r) => (p.handover_refused)(r),
3607 None => format!("{e:#}"),
3608 };
3609 task.startup_notice_cause("startup-handover");
3610 let reason = format!("{start_failed}{detail}");
3611 task.last_error = Some(reason.clone());
3612 let branch = match &starter {
3615 Starter::Review(branch) => Some(branch.clone()),
3616 _ => None,
3617 };
3618 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
3622 record(queue, task);
3623 tracing::warn!(
3624 "holding {} for a branch it cannot take over: {e:#}",
3625 task.short()
3626 );
3627 return Vec::new();
3628 }
3629 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
3634 let d = e
3635 .downcast_ref::<crate::reconcile::Diverged>()
3636 .expect("checked by the guard");
3637 let mut q = ask::Question::new(
3638 task.id.clone(),
3639 crate::reconcile::NODE.to_owned(),
3640 crate::reconcile::SEAT.to_owned(),
3641 d.summary(),
3642 d.detail(),
3643 d.choices(),
3644 );
3645 match Questions::open().put(&mut q) {
3646 Ok(()) => {
3647 task.last_error = Some(format!("{e:#}"));
3648 task.review_branch = Some(d.branch.clone());
3651 task.review_instruction = Some(task.instruction.clone());
3652 task.block(vec![q.id.clone()], Some(d.summary()));
3653 }
3654 Err(put) => {
3655 tracing::warn!("could not file the divergence question: {put:#}");
3656 task.attempts += 1;
3657 task.fail_startup(
3658 "startup-question-store",
3659 format!("{start_failed}{e:#}"),
3660 opts.max_attempts,
3661 );
3662 }
3663 }
3664 record(queue, task);
3665 return Vec::new();
3666 }
3667 Err(e) => {
3668 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
3674 && let Starter::Review(branch) = &starter
3675 {
3676 task.review_branch = Some(branch.clone());
3677 task.review_instruction = Some(task.instruction.clone());
3678 task.last_error = Some(format!("{start_failed}{e:#}"));
3679 task.status = crate::queue::TaskStatus::Failed;
3680 record(queue, task);
3681 return Vec::new();
3682 }
3683 task.attempts += 1;
3684 task.last_error = Some(format!("{start_failed}{e:#}"));
3685 task.startup_notice_cause("startup-source");
3686 task.hold_unrequested_retry(Some(format!(
3687 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
3688 )));
3689 record(queue, task);
3690 return Vec::new();
3691 }
3692 };
3693 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
3694 runner.state.recovery_config = Some(config_evidence);
3695 }
3696 if let Some(replacement) = &task.replacement {
3697 if matches!(starter, Starter::Start) {
3698 runner.state.instruction.push_str(&format!(
3699 "\n\n# Replacement review context\n{}",
3700 replacement.context
3701 ));
3702 }
3703 runner.state.replaces = Some(replacement.clone());
3704 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
3705 if let Ok(mut old) = RunState::load(&replacement.run)
3706 && let Some(d) = &mut old.merge_decision
3707 {
3708 d.replacement_run = Some(runner.state.id.clone());
3709 old.event(
3710 "replacement",
3711 format!(
3712 "replacement run {} owns the next attempt; old PR remains closed",
3713 runner.state.id
3714 ),
3715 );
3716 if let Err(e) = old.save() {
3717 task.hold_manual(Some(format!(
3718 "Could not link replacement history: {e}; no agents dispatched"
3719 )));
3720 record(queue, task);
3721 return Vec::new();
3722 }
3723 }
3724 }
3725 runner.state.task_instruction = Some(task.instruction.clone());
3726 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
3729 if runner.state.origin_chat.is_none() {
3730 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
3731 }
3732 runner.on_pause(stop.pause());
3734 runner.watch_interrupt(interrupt_pause);
3738
3739 let run = runner.state.id.clone();
3742 task.start(run.clone());
3743 record(queue, task);
3744 lock(status).current.push(Current {
3745 task: task.id.clone(),
3746 run,
3747 });
3748
3749 let quota_before = runner.state.quota.clone();
3752 runner.state.next_pass_task_entry = true;
3753 let result = runner.execute().await;
3754 finish_attempt(
3755 opts.max_attempts,
3756 queue,
3757 task,
3758 &runner.state,
3759 "a_before,
3760 result,
3761 )
3762}
3763
3764pub fn finish_attempt(
3769 max_attempts: usize,
3770 queue: &Queue,
3771 task: &mut Task,
3772 state: &RunState,
3773 quota_before: &[QuotaLoss],
3774 result: Result<()>,
3775) -> Vec<QuotaLoss> {
3776 let detail = match result {
3777 Ok(()) => describe(state),
3778 Err(e) => format!("{e:#}"),
3779 };
3780 let fresh = losses_this_attempt(quota_before, &state.quota);
3781 let verdict = Verdict {
3782 status: state.status,
3783 left_pr: state.pr.is_some(),
3786 quota_hit: !fresh.is_empty(),
3792 parked: state.parked,
3796 no_viable_candidates: state.viable().is_empty(),
3799 };
3800 if let Some(replacement) = &state.replaces
3801 && let Some(pr) = &state.pr
3802 && let Ok(mut old) = RunState::load(&replacement.run)
3803 && let Some(d) = &mut old.merge_decision
3804 {
3805 d.replacement_pr = Some(pr.url.clone());
3806 let _ = old.save();
3807 }
3808 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
3809 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
3810 task.hold_unrequested_retry(Some(format!(
3811 "Candidate setup failed: {detail}. Select a complete implementation with `magi task recover-pr`, or correct the referenced source before explicitly releasing the task."
3812 )));
3813 }
3814 if task.status == TaskStatus::Done {
3815 supersede_prior_runs(task, &crate::run::home());
3816 }
3817 record(queue, task);
3818 tracing::info!(
3819 "task {} is {} after run {} ({})",
3820 task.short(),
3821 task.status.as_str(),
3822 state.short(),
3823 label(state.status)
3824 );
3825 fresh
3826}
3827
3828pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
3832 let waits_on_owner = withheld_text_wait(task).is_some();
3837 if task.status.runnable()
3838 && !waits_on_owner
3839 && !(task.fresh_start && task.replacement.is_some())
3840 {
3841 let why = task.last_error.clone().map_or_else(
3842 || "the run did not finish".to_owned(),
3843 |e| format!("the run did not finish: {e}"),
3844 );
3845 task.hold_unrequested_retry(Some(format!(
3846 "{why}. It was started by hand, so it is not retried \
3847 automatically; `magi task release` retries it."
3848 )));
3849 record(queue, task);
3850 }
3851}
3852
3853#[must_use]
3857pub fn loop_would_resume(run: &str) -> bool {
3858 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
3859}
3860
3861#[must_use]
3871pub fn foreign_loop(
3872 reading: Option<&Reading>,
3873 now: Timestamp,
3874 own_pid: u32,
3875) -> Option<Option<u32>> {
3876 let reading = reading.filter(|r| r.running(now))?;
3877 match reading.pid {
3878 Some(pid) if pid == own_pid => None,
3879 pid => Some(pid),
3880 }
3881}
3882
3883pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
3898 let status = Arc::new(Mutex::new(Status::new()));
3899 let stop = Stop::new();
3900 attempt(
3901 opts,
3902 queue,
3903 &status,
3904 &stop,
3905 crate::graph::Pause::new(),
3906 task,
3907 )
3908 .await;
3909 let mut resumes = 0;
3918 while resumes < MAX_WITHHELD_RESUMES {
3919 let Some(timeout) = withheld_text_wait(task) else {
3920 break;
3921 };
3922 eprintln!(
3923 "waiting for the owner's answer on a withheld pull request text \
3924 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
3925 );
3926 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
3929 let ready = loop {
3930 match land_resume_state(task) {
3931 LandResume::StillWaiting if std::time::Instant::now() < limit => {
3932 tokio::time::sleep(WITHHELD_POLL).await;
3933 }
3934 LandResume::Ready => break true,
3935 _ => break false,
3936 }
3937 };
3938 if !ready {
3939 break;
3940 }
3941 resumes += 1;
3942 attempt(
3943 opts,
3944 queue,
3945 &status,
3946 &stop,
3947 crate::graph::Pause::new(),
3948 task,
3949 )
3950 .await;
3951 }
3952 hold_if_runnable(queue, task);
3953}
3954
3955const MAX_WITHHELD_RESUMES: u32 = 3;
3958
3959const WITHHELD_POLL: Duration = Duration::from_secs(15);
3961
3962fn withheld_text_wait(task: &Task) -> Option<u64> {
3965 let id = task.runs.last()?;
3966 let s = RunState::load(id).ok()?;
3967 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
3968 .then_some(s.config.graph.answer_timeout)
3969}
3970
3971fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
3981 after
3982 .iter()
3983 .filter(|q| !before.contains(q))
3984 .cloned()
3985 .collect()
3986}
3987
3988fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
3991 if quota.is_empty() {
3992 return None;
3993 }
3994 let with_hint = quota.iter().find(|q| q.reset.is_some());
3995 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
3996 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
3997 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
3998 Some(
3999 now.checked_add(jiff::SignedDuration::from_secs(secs))
4000 .unwrap_or(Timestamp::MAX),
4001 )
4002}
4003
4004fn implementers_match(task: &Task, state: &RunState) -> bool {
4006 task.replacement.as_ref().map_or_else(
4007 || {
4008 task.implementers()
4009 .is_none_or(|n| n == state.config.graph.implementers)
4010 },
4011 |replacement| {
4012 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
4013 && state.config.graph.implementers == 1
4014 },
4015 )
4016}
4017
4018fn starter_with_implementers(
4020 task: &Task,
4021 previous: Option<&RunState>,
4022 proposed: Starter,
4023) -> Starter {
4024 if previous.is_some_and(|s| !implementers_match(task, s)) {
4025 task.review_of
4026 .as_ref()
4027 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
4028 } else {
4029 proposed
4030 }
4031}
4032
4033fn apply_task_implementers(config: &mut Config, task: &Task) {
4035 if let Some(n) = task.implementers() {
4036 config.graph.implementers = n;
4037 }
4038}
4039
4040#[cfg(test)]
4045fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
4046 let explicit = match &task.overrides {
4047 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4048 _ => opts.config.as_deref(),
4049 };
4050 let (mut config, _) = Config::discover(repo, explicit)?;
4051 apply_options(&mut config, opts, task)?;
4052 Ok(config)
4053}
4054
4055fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
4056 if task
4057 .overrides
4058 .as_ref()
4059 .is_none_or(|o| o.inherit_daemon_options)
4060 && let Some(mode) = opts.merge.as_deref()
4061 {
4062 config.merge.mode = merge_mode(mode)?;
4063 }
4064 if let Some(o) = &task.overrides {
4065 o.apply(config);
4066 }
4067 Ok(())
4068}
4069
4070fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
4072 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
4073 if let Some(mode) = &opts.merge {
4074 config.merge.mode = merge_mode(mode)?;
4075 }
4076 Ok(config)
4077}
4078
4079async fn maybe_prune_cache_between_runs(
4110 repo: &Path,
4111 opts: &Opts,
4112 home: &Path,
4113 stop: &Stop,
4114 last_checked: &mut Option<Timestamp>,
4115 now: Timestamp,
4116) {
4117 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
4118 return;
4119 }
4120 *last_checked = Some(now);
4121 let cfg = match prepare(repo, opts) {
4122 Ok(cfg) => cfg,
4123 Err(e) => {
4124 tracing::warn!("cache check: no config: {e:#}");
4125 return;
4126 }
4127 };
4128 match clean::prune_cache_if_over_limit(&cfg, home) {
4129 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
4130 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
4131 pruned.files,
4132 pruned.freed
4133 ),
4134 Ok(_) => {}
4135 Err(e) => {
4136 tracing::warn!("housekeep: prune cache: {e:#}");
4137 notices::raise_in(
4138 home,
4139 Notice::warn(
4140 "housekeep:cache",
4141 "Pruning the shared build cache failed; disk usage may keep growing.",
4142 ),
4143 );
4144 }
4145 }
4146}
4147
4148fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
4152 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
4153}
4154
4155async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
4174 let cfg = match prepare(repo, opts) {
4175 Ok(cfg) => cfg,
4176 Err(e) => {
4177 tracing::warn!("housekeep: no config: {e:#}");
4178 return;
4179 }
4180 };
4181 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
4190 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
4191 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
4196 let mut extra = Vec::new();
4197 if out.unreadable > 0 {
4198 extra.push(format!("{} unreadable", out.unreadable));
4199 }
4200 if out.orphaned_worktrees > 0 {
4201 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
4202 }
4203 let detail = if extra.is_empty() {
4204 String::new()
4205 } else {
4206 format!(" ({})", extra.join(", "))
4207 };
4208 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
4209 }
4210 if out.external_merges_recorded > 0 {
4211 tracing::info!(
4212 "housekeep: recorded {} run(s) as merged externally",
4213 out.external_merges_recorded
4214 );
4215 }
4216 if out.stale_pr_states_repaired > 0 {
4217 tracing::info!(
4218 "housekeep: rewrote {} run record(s) whose pull request had already settled",
4219 out.stale_pr_states_repaired
4220 );
4221 }
4222 if out.cache_files > 0 {
4223 tracing::info!(
4224 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
4225 out.cache_files,
4226 out.cache_freed
4227 );
4228 }
4229 if out.questions_abandoned > 0 {
4230 tracing::info!(
4231 "housekeep: abandoned {} question(s) left open by a finished run",
4232 out.questions_abandoned
4233 );
4234 }
4235}
4236
4237async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
4246 let questions = Questions::at(home.join("questions"));
4247 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
4248 if report.is_empty() {
4249 return;
4250 }
4251 if !report.quarantined.is_empty() {
4252 tracing::info!(
4253 "triage: held {} blocked task(s) whose blocked-on task or \
4254 question no longer exists: {}",
4255 report.quarantined.len(),
4256 report.quarantined.join(", ")
4257 );
4258 }
4259 if !report.resumed.is_empty() {
4260 tracing::info!(
4261 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
4262 report.resumed.len(),
4263 report.resumed.join(", ")
4264 );
4265 }
4266 if !report.asked.is_empty() {
4267 tracing::info!(
4268 "triage: asked about {} held task(s): {}",
4269 report.asked.len(),
4270 report.asked.join(", ")
4271 );
4272 }
4273 if !report.answered.is_empty() {
4274 tracing::info!(
4275 "triage: applied {} operator answer(s): {}",
4276 report.answered.len(),
4277 report.answered.join(", ")
4278 );
4279 }
4280}
4281
4282fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
4289 disk_gate_with(repo, config, crate::disk::free_bytes)
4290}
4291
4292fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
4296 repo: &Path,
4297 config: &Config,
4298 free_bytes: F,
4299) -> Option<String> {
4300 let min = config.disk.min_free_bytes;
4301 if min == 0 {
4302 return None;
4303 }
4304 match free_bytes(repo) {
4305 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
4306 Err(e) => Some(crate::disk::unmeasured_in(repo, &e, &config.graph.language)),
4307 }
4308}
4309
4310const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
4317
4318const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
4322
4323fn quota_wait(
4332 reset_at: Option<Timestamp>,
4333 now: Timestamp,
4334 fallback: Duration,
4335 cap: Duration,
4336) -> Duration {
4337 match reset_at {
4338 Some(at) if at > now => {
4339 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
4340 Duration::from_secs(secs).min(cap)
4341 }
4342 _ => fallback,
4343 }
4344}
4345
4346fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
4360 parse_reset_hint_zoned(text, now)
4361 .or_else(|| parse_reset_hint_dated(text))
4362 .or_else(|| parse_reset_hint_relative(text, recorded))
4363}
4364
4365fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
4369 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
4370 let mut rest = rest.trim();
4371 if rest.is_empty() {
4372 return None;
4373 }
4374 let mut total: i64 = 0;
4375 let mut matched = false;
4376 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
4377 if let Some((digits, tail)) = rest.split_once(unit)
4378 && !digits.is_empty()
4379 && digits.bytes().all(|b| b.is_ascii_digit())
4380 {
4381 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
4382 rest = tail;
4383 matched = true;
4384 }
4385 }
4386 if !rest.is_empty() || !matched {
4387 return None;
4388 }
4389 recorded
4390 .checked_add(jiff::SignedDuration::from_secs(total))
4391 .ok()
4392}
4393
4394fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
4398 let clock = clock.trim().to_lowercase();
4399 let (digits, pm) = clock
4400 .strip_suffix("am")
4401 .map(|d| (d, false))
4402 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
4403 let (h, m) = digits.trim().split_once(':')?;
4404 let mut hour: i8 = h.trim().parse().ok()?;
4405 let minute: i8 = m.trim().parse().ok()?;
4406 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
4407 return None;
4408 }
4409 if pm && hour != 12 {
4410 hour += 12;
4411 } else if !pm && hour == 12 {
4412 hour = 0;
4413 }
4414 Some((hour, minute))
4415}
4416
4417fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
4422 let open = text.find('(')?;
4423 let close = text.rfind(')')?;
4424 if close <= open {
4425 return None;
4426 }
4427 let zone = text[open + 1..close].trim();
4428 let (hour, minute) = parse_12h_clock(&text[..open])?;
4429 let tz = jiff::tz::TimeZone::get(zone).ok()?;
4430 let candidate = now
4431 .to_zoned(tz)
4432 .with()
4433 .hour(hour)
4434 .minute(minute)
4435 .second(0)
4436 .millisecond(0)
4437 .microsecond(0)
4438 .nanosecond(0)
4439 .build()
4440 .ok()?;
4441 let mut at = candidate.timestamp();
4442 if at <= now {
4443 at += jiff::SignedDuration::from_hours(24);
4444 }
4445 Some(at)
4446}
4447
4448fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
4457 let words: Vec<&str> = text.split_whitespace().collect();
4458 if words.len() < 5 {
4459 return None;
4460 }
4461 (0..=words.len() - 5)
4462 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
4463}
4464
4465fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
4471 if trailing.is_some_and(|next| next.starts_with('(')) {
4472 return None;
4473 }
4474 let month = month_number(window[0])?;
4475 let day_token = window[1].strip_suffix(',')?.to_lowercase();
4476 let day_digits = ["st", "nd", "rd", "th"]
4477 .iter()
4478 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
4479 let day: i8 = day_digits.parse().ok()?;
4480 let year_token = window[2];
4481 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
4482 return None;
4483 }
4484 let year: i16 = year_token.parse().ok()?;
4485 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
4489 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
4490 let date = jiff::civil::Date::new(year, month, day).ok()?;
4491 let candidate = date
4492 .at(hour, minute, 0, 0)
4493 .to_zoned(jiff::tz::TimeZone::UTC)
4494 .ok()?;
4495 Some(candidate.timestamp())
4496}
4497
4498fn month_number(name: &str) -> Option<i8> {
4501 const NAMES: [&str; 12] = [
4502 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
4503 ];
4504 let lower = name.to_lowercase();
4505 NAMES
4506 .iter()
4507 .position(|n| *n == lower.as_str())
4508 .map(|i| i as i8 + 1)
4509}
4510
4511fn exhausted_review_budget(state: &RunState) -> bool {
4523 state.status == RunStatus::Blocked
4526 && !state.pr_publication_pending
4527 && state.reviews.len() >= state.config.graph.review_rounds
4528}
4529
4530fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
4567 unfinished_run_with(runs, short, RunState::load)
4568}
4569
4570fn unfinished_run_with<F>(runs: &[String], short: &str, load: F) -> Option<String>
4573where
4574 F: FnOnce(&str) -> Result<RunState>,
4575{
4576 let id = runs.last()?;
4577 match load(id) {
4578 Ok(s)
4583 if s.status.resumable()
4584 && !s.released()
4585 && !exhausted_review_budget(&s)
4586 && s.liveness(false) != crate::run::Liveness::Live =>
4587 {
4588 Some(id.clone())
4589 }
4590 Ok(_) => None,
4591 Err(e) => {
4592 tracing::warn!("could not read run {id} for task {short}: {e:#}");
4593 None
4594 }
4595 }
4596}
4597
4598fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
4606where
4607 F: FnOnce(&str) -> Result<RunState>,
4608{
4609 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
4611 || task.fresh_start
4612 || task.review_branch.is_some()
4613 {
4614 return false;
4615 }
4616 let Some(id) = task.runs.last() else {
4617 return false;
4618 };
4619 load(id).is_ok_and(|s| {
4620 s.parked && s.status.resumable() && !s.released() && !exhausted_review_budget(&s)
4621 })
4622}
4623
4624#[derive(Debug, Clone, PartialEq, Eq)]
4627enum Starter {
4628 Review(String),
4631 Resume(String),
4633 Start,
4635}
4636
4637fn selected_starter(
4639 selected: &crate::pr_recovery::Selection,
4640 previous: Option<&RunState>,
4641 unfinished: Option<&str>,
4642) -> Starter {
4643 if let Some(id) = unfinished
4644 && previous.is_some_and(|r| {
4645 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
4646 })
4647 {
4648 Starter::Resume(id.to_owned())
4649 } else {
4650 Starter::Review(selected.branch.clone())
4651 }
4652}
4653
4654fn take_divergence_answer(
4658 branch: &str,
4659 remote: &str,
4660 task: &mut Task,
4661) -> Option<crate::reconcile::Choice> {
4662 let summary = crate::reconcile::summary_for(branch, remote);
4663 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
4664 (a.question == summary)
4665 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
4666 .flatten()
4667 .map(|c| (i, c))
4668 })?;
4669 task.answers.remove(idx);
4670 Some(choice)
4671}
4672
4673fn choose_starter(
4685 review_branch: Option<&str>,
4686 branch_exists: bool,
4687 unfinished: Option<&str>,
4688) -> Starter {
4689 match review_branch {
4690 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
4691 Some(_) => Starter::Start,
4692 None => match unfinished {
4693 Some(id) => Starter::Resume(id.to_owned()),
4694 None => Starter::Start,
4695 },
4696 }
4697}
4698
4699fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
4701 let branch = task.review_branch.take();
4702 let instruction = task.review_instruction.take();
4703 branch.filter(|branch| match &instruction {
4704 Some(authored) => *authored == task.instruction,
4705 None => previous.map_or_else(
4708 || task.review_of.as_ref() == Some(branch),
4709 |run| compatible_run_instruction(run, task),
4710 ),
4711 })
4712}
4713
4714fn may_resume_run(run: &RunState, task: &Task) -> bool {
4717 task.resume_override
4718 .as_ref()
4719 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
4720 || compatible_run_instruction(run, task)
4721}
4722
4723pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
4725 if let Some(authored) = &run.task_instruction {
4726 return compatible_instruction(authored, task);
4727 }
4728 if compatible_instruction(&run.instruction, task) {
4729 return true;
4730 }
4731 if run.reviewed_commits.is_some() {
4734 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
4735 return strip_answers_block(authored, task).trim() == task.instruction.trim();
4736 }
4737 if let Some(branch) = &task.review_of {
4738 let canonical = format!(
4739 "Review the branch `{branch}` (review-only: no implementation, no judging)."
4740 );
4741 let generated = format!(
4742 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
4743 );
4744 return task.instruction == canonical && run.instruction.starts_with(&generated);
4745 }
4746 }
4747 false
4748}
4749
4750pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
4752 strip_answers_block(old, task) == task.instruction
4753}
4754
4755fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
4758 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
4759 return fallback.to_path_buf();
4760 }
4761 task.repo.clone()
4762}
4763
4764const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
4768
4769fn answers_block(task: &Task, count: usize) -> String {
4771 let mut s = ANSWERS_HEADER.to_owned();
4772 for a in &task.answers[..count] {
4773 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
4774 }
4775 s
4776}
4777
4778fn resumption_block(r: &crate::queue::Resumption) -> String {
4781 format!(
4782 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
4783 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
4784 r.generation, r.at, r.instruction, r.answers_through,
4785 )
4786}
4787
4788fn append_answers(base: &str, task: &Task) -> String {
4790 let mut s = base.to_owned();
4791 if !task.answers.is_empty() {
4792 s.push_str(&answers_block(task, task.answers.len()));
4793 }
4794 if let Some(r) = &task.resumption {
4795 s.push_str(&resumption_block(r));
4796 }
4797 s
4798}
4799
4800fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
4804 let mut instruction = instruction;
4805 for approval in task
4806 .resumption
4807 .iter()
4808 .chain(task.resumption_history.iter().rev())
4809 {
4810 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
4811 instruction = base;
4812 break;
4813 }
4814 }
4815 for count in (1..=task.answers.len()).rev() {
4816 let block = answers_block(task, count);
4817 if let Some(base) = instruction.strip_suffix(&block) {
4818 return base;
4819 }
4820 }
4821 instruction
4822}
4823
4824fn instruction_for(task: &Task) -> String {
4832 append_answers(&task.instruction, task)
4833}
4834
4835fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
4847 append_answers(strip_answers_block(old_instruction, task), task)
4848}
4849
4850fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
4852 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
4853 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
4854 if let Some(cause) = task.hold_notice_cause.as_mut() {
4855 cause.target = failure.map(|f| f.target.clone());
4856 }
4857}
4858
4859#[derive(Debug, thiserror::Error)]
4860#[error("attachment `{name}` is recorded on the task but {path} is missing")]
4861struct MissingAttachment {
4862 name: String,
4863 path: PathBuf,
4864}
4865
4866fn task_attachments(
4868 queue: &Queue,
4869 task: &Task,
4870) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
4871 let paths = queue.attachment_paths(task);
4872 for (name, path) in task.attachments.iter().zip(&paths) {
4873 if !path.is_file() {
4874 return Err(MissingAttachment {
4875 name: name.clone(),
4876 path: path.clone(),
4877 });
4878 }
4879 }
4880 Ok(paths)
4881}
4882
4883fn prepare_instruction(
4894 starter: &Starter,
4895 old_instruction: Option<&str>,
4896 task: &Task,
4897) -> Option<String> {
4898 match starter {
4899 Starter::Start => Some(instruction_for(task)),
4900 Starter::Resume(_) => Some(resumed_instruction(
4901 old_instruction.expect("a resumed run always has a prior instruction"),
4902 task,
4903 )),
4904 Starter::Review(_) => None,
4905 }
4906}
4907
4908fn record(queue: &Queue, task: &mut Task) {
4912 if let Err(e) = queue.put(task) {
4913 tracing::error!("could not record task {}: {e:#}", task.short());
4914 let notice = Notice::error(
4915 "loop:record",
4916 "The loop could not save a task's state; check the disk.",
4917 )
4918 .link(Link::Task {
4919 id: task.id.clone(),
4920 });
4921 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
4923 Some(home) => notices::raise_in(home, notice),
4924 None => notices::raise(notice),
4925 }
4926 }
4927}
4928
4929fn runnable(queue: &Queue) -> Vec<Task> {
4935 let mut tasks: Vec<Task> = queue
4936 .list()
4937 .into_iter()
4938 .filter(|t| t.status.runnable())
4939 .collect();
4940 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)));
4941 tasks
4942}
4943
4944fn describe(state: &RunState) -> String {
4958 let p = phrases(&state.config.graph.language);
4959 let mut detail = if state.status == RunStatus::Stalled {
4960 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
4961 seats.sort_unstable();
4962 seats.dedup();
4963 if seats.is_empty() {
4964 p.quorum_lost.to_owned()
4965 } else {
4966 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
4967 }
4968 } else {
4969 format!("{}{}", p.run_ended, state.status.display_label())
4970 };
4971 if let Some(last) = state.events.last() {
4972 let unanswered = state
4976 .reviews
4977 .last()
4978 .filter(|r| {
4979 state.status == RunStatus::Blocked
4980 && last.node == "review"
4981 && r.incomplete()
4982 && r.blocking == 0
4983 && r.round == state.config.graph.review_rounds
4984 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
4985 })
4986 .map(|r| (r.expected - r.answered, r.round));
4987 match unanswered {
4988 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
4989 detail.push_str(&format!(
4990 " ({}: {})",
4991 last.node,
4992 (p.reviewers_never_answered)(missing, rounds)
4993 ));
4994 }
4995 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
4996 }
4997 }
4998 detail.push_str(&format!(" [run {}]", state.id));
4999 detail
5000}
5001
5002const DIAGNOSTIC_MAX: usize = 4_000;
5008
5009const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
5014
5015fn diagnostic(state: &RunState) -> Option<String> {
5029 let mut parts: Vec<String> = Vec::new();
5030
5031 for o in state.gate.iter().filter(|o| !o.ok()) {
5033 parts.push(format!(
5034 "gate `{}` failed ({}):\n{}",
5035 o.command,
5036 notices::exit_status(o.code),
5037 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
5038 ));
5039 }
5040
5041 if let Some(last) = state
5044 .events
5045 .iter()
5046 .rev()
5047 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
5048 {
5049 parts.push(last.message.clone());
5050 }
5051
5052 if state.viable().is_empty() {
5059 for c in &state.candidates {
5060 if let Some(evidence) = &c.verified_noop {
5061 parts.push(format!(
5062 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
5063 c.label
5064 ));
5065 } else if !c.summary.trim().is_empty() {
5066 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
5067 } else if let Some(why) = &c.failed {
5068 parts.push(format!("candidate {}: {why}", c.label));
5069 }
5070 }
5071 }
5072
5073 if parts.is_empty() {
5074 return None;
5075 }
5076 Some(crate::run::tail(
5081 &parts.join("\n\n"),
5082 DIAGNOSTIC_MAX.saturating_sub(100),
5083 ))
5084}
5085
5086fn label(status: RunStatus) -> &'static str {
5094 status.as_str()
5095}
5096
5097pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
5099 match mode {
5100 "none" => Ok(MergeMode::None),
5101 "local" => Ok(MergeMode::Local),
5102 "pr" => Ok(MergeMode::Pr),
5103 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
5104 }
5105}
5106
5107fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
5112 mutex
5113 .lock()
5114 .unwrap_or_else(std::sync::PoisonError::into_inner)
5115}
5116
5117#[cfg(test)]
5118mod tests {
5119 use super::*;
5120 use crate::queue::{Source, TaskStatus};
5121 use crate::run::{Candidate, CommandOutcome};
5122 use pretty_assertions::assert_eq;
5123
5124 fn task() -> Task {
5125 Task::new(
5126 "add retries".to_owned(),
5127 "add retries".to_owned(),
5128 PathBuf::from("/repo"),
5129 Source::Human,
5130 )
5131 }
5132
5133 #[test]
5134 fn selected_public_head_overrides_historical_resume_and_pin() {
5135 let mut t = task();
5136 let mut old = run_state(RunStatus::Blocked);
5137 let selection = crate::pr_recovery::Selection {
5138 url: "url".into(),
5139 branch: format!("magi/task-{}", t.id),
5140 head: "a".repeat(40),
5141 source_head: None,
5142 disposition: Some(crate::pr_recovery::Disposition::Review),
5143 instruction: t.instruction.clone(),
5144 approved_at: Timestamp::now(),
5145 };
5146 t.resume_override = Some(crate::queue::OperatorResume {
5147 pinned_run: Some(old.id.clone()),
5148 question_id: "choice".into(),
5149 at: Timestamp::now(),
5150 conductor_rehold: None,
5151 forced: false,
5152 });
5153 assert_eq!(
5154 selected_starter(&selection, Some(&old), Some(&old.id)),
5155 Starter::Review(selection.branch.clone())
5156 );
5157 old.pr_recovery = Some(selection.clone());
5158 assert_eq!(
5159 selected_starter(&selection, Some(&old), Some(&old.id)),
5160 Starter::Review(selection.branch.clone())
5161 );
5162 old.reviewed_commits = Some(vec!["implementation".into()]);
5163 assert_eq!(
5164 selected_starter(&selection, Some(&old), Some(&old.id)),
5165 Starter::Resume(old.id.clone())
5166 );
5167 let mut changed = selection.clone();
5168 changed.head = "b".repeat(40);
5169 assert_eq!(
5170 selected_starter(&changed, Some(&old), Some(&old.id)),
5171 Starter::Review(changed.branch)
5172 );
5173 }
5174
5175 #[test]
5176 fn candidate_transplant_failure_is_one_actionable_hold() {
5177 let dir = tempfile::tempdir().unwrap();
5178 let q = Queue::at(dir.path().join("queue"));
5179 let mut t = task();
5180 let mut state = run_state(RunStatus::Blocked);
5181 state.event("prep", "cannot apply referenced commit: conflict");
5182 state.prep_failure = Some("conflict".into());
5183 t.start(state.id.clone());
5184 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
5185 assert_eq!(t.attempts, 1);
5186 assert_eq!(t.status, TaskStatus::Held);
5187 assert!(t.operator_held());
5188 assert!(!t.status.runnable());
5189 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
5190 let first = t.chat_notice_key().unwrap();
5191 t.chat_notices.push(first.clone());
5192 t.release_operator("test", None);
5193 state.id = "another-unprepared-run".into();
5194 t.start(state.id.clone());
5195 finish_attempt(
5196 3,
5197 &q,
5198 &mut t,
5199 &state,
5200 &[],
5201 Err(anyhow::anyhow!("still conflicts")),
5202 );
5203 assert_eq!(
5204 t.chat_notice_key().as_ref(),
5205 Some(&first),
5206 "no work happened"
5207 );
5208 t.release_operator("test", None);
5209 state.prep_failure = None;
5210 state.status = RunStatus::Failed;
5211 state.begin_pass();
5212 t.start(state.id.clone());
5213 finish_attempt(
5214 1,
5215 &q,
5216 &mut t,
5217 &state,
5218 &[],
5219 Err(anyhow::anyhow!("new gate failure")),
5220 );
5221 let failure = t.chat_notice_key().unwrap();
5222 assert_ne!(failure, first, "approved recovery reached a real failure");
5223 t.release_operator("test", None);
5224 state.begin_pass();
5225 t.start(state.id.clone());
5226 finish_attempt(
5227 1,
5228 &q,
5229 &mut t,
5230 &state,
5231 &[],
5232 Err(anyhow::anyhow!("new failure on resume")),
5233 );
5234 assert_ne!(
5235 t.chat_notice_key().as_ref(),
5236 Some(&failure),
5237 "a new settled pass"
5238 );
5239 }
5240
5241 #[test]
5242 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
5243 for source in ["cli", "web"] {
5244 let mut t = task();
5245 let mut run = run_state(RunStatus::Blocked);
5246 run.instruction = "Run A original criteria".into();
5247 run.task_instruction = Some(run.instruction.clone());
5248 t.start(run.id.clone());
5249 t.release_operator("choice", Some("resume-question"));
5250 t.resume_override = Some(crate::queue::OperatorResume {
5251 question_id: "resume-question".into(),
5252 at: Timestamp::now(),
5253 conductor_rehold: None,
5254 forced: true,
5255 pinned_run: Some(run.id.clone()),
5256 });
5257 t.hold_machine(Some("not enough free space to start a run".into()));
5258 let mut unchanged = t.clone();
5259 unchanged
5260 .edit("title changed".into(), unchanged.instruction.clone())
5261 .unwrap();
5262 unchanged.release_operator(source, None);
5263 assert_eq!(
5264 unchanged
5265 .resume_override
5266 .as_ref()
5267 .unwrap()
5268 .pinned_run
5269 .as_deref(),
5270 Some(run.id.as_str())
5271 );
5272 assert!(!compatible_run_instruction(&run, &unchanged));
5273 assert!(may_resume_run(&run, &unchanged));
5274
5275 t.edit(
5276 "approved recovery".into(),
5277 "Reuse measured PR head B; fix Clippy; review".into(),
5278 )
5279 .unwrap();
5280 assert!(t.resume_override.is_none());
5281 t.release_operator(source, None);
5282 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
5283 assert!(reloaded.resume_override.is_none());
5284 assert!(!may_resume_run(&run, &reloaded));
5285 let unfinished =
5286 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
5287 .filter(|_| may_resume_run(&run, &reloaded));
5288 assert_eq!(
5289 choose_starter(None, false, unfinished.as_deref()),
5290 Starter::Start
5291 );
5292 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
5293 assert_eq!(
5294 reloaded.resumption_history[0].question_id.as_deref(),
5295 Some("resume-question")
5296 );
5297
5298 unchanged.instruction = "Reuse measured PR head B".into();
5301 unchanged.hold_machine(Some("disk hold".into()));
5302 unchanged.release_operator(source, None);
5303 assert!(unchanged.resume_override.is_none());
5304 assert!(!may_resume_run(&run, &unchanged));
5305 }
5306 }
5307
5308 #[test]
5309 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
5310 let mut t = task();
5311 let mut run = run_state(RunStatus::Blocked);
5312 t.review_of = Some("approved-head".into());
5313 t.instruction =
5314 "Review the branch `approved-head` (review-only: no implementation, no judging)."
5315 .into();
5316 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
5317 run.reviewed_commits = Some(vec!["Commit log".into()]);
5318 t.start(run.id.clone());
5319 t.hold_machine(Some("interrupted review".into()));
5320 t.release_operator("cli", None);
5321 assert!(
5322 compatible_run_instruction(&run, &t),
5323 "legacy explicit review remains resumable"
5324 );
5325 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
5326 .filter(|_| compatible_run_instruction(&run, &t));
5327 assert_eq!(
5328 choose_starter(None, false, unfinished.as_deref()),
5329 Starter::Resume(run.id.clone())
5330 );
5331 run.task_instruction = Some(t.instruction.clone());
5332 assert!(compatible_run_instruction(&run, &t));
5333 let reloaded: RunState =
5334 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
5335 assert!(compatible_run_instruction(&reloaded, &t));
5336 t.edit(
5337 "new approved head".into(),
5338 "Reuse measured PR head B".into(),
5339 )
5340 .unwrap();
5341 assert!(!compatible_run_instruction(&run, &t));
5342
5343 t.review_of = None;
5345 t.instruction = "Fix Clippy on approved head".into();
5346 run.task_instruction = None;
5347 run.instruction = format!(
5348 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
5349 t.instruction
5350 );
5351 assert!(compatible_run_instruction(&run, &t));
5352 t.instruction.push('\n');
5353 assert!(
5354 compatible_run_instruction(&run, &t),
5355 "legacy generated review trimmed the original criteria"
5356 );
5357 t.hold_for_handover(Some("approved-head".into()), "refused".into());
5358 let mut unchanged = t.clone();
5359 assert_eq!(
5360 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
5361 Some("approved-head")
5362 );
5363 t.edit("new plan".into(), "Reuse measured PR head B".into())
5364 .unwrap();
5365 assert!(take_review_branch(&mut t, Some(&run)).is_none());
5366 assert!(!compatible_run_instruction(&run, &t));
5367 t.review_branch = Some("approved-head".into());
5368 t.review_instruction = None;
5369 assert!(
5370 take_review_branch(&mut t, Some(&run)).is_none(),
5371 "legacy retained candidate cannot replace a revised approved head"
5372 );
5373 }
5374
5375 #[test]
5376 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
5377 crate::run::pin_test_home();
5378 let dir = tempfile::tempdir().unwrap();
5379 let queue = Queue::at(dir.path().join("queue"));
5380 for resume in [true, false] {
5381 let state = run_state(RunStatus::Blocked);
5382 let mut t = task();
5383 t.start(state.id.clone());
5384 let action = if resume {
5385 ask::ChoiceAction::Resume {
5386 run: state.id.clone(),
5387 }
5388 } else {
5389 ask::ChoiceAction::Requeue
5390 };
5391 let mut q = action_question(&state.id, action);
5392 q.status = ask::QuestionStatus::Open;
5393 q.answer = None;
5394 q.answered_at = None;
5395 t.fail("run stopped with a question", 3);
5396 hold_if_runnable(&queue, &mut t);
5397 assert!(t.operator_decided_at.is_none());
5398 assert!(t.operator_held());
5399 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
5400 let expected = if resume {
5401 ActionDecision::Resume(state.id.clone())
5402 } else {
5403 ActionDecision::Requeue
5404 };
5405 assert_eq!(
5406 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5407 expected
5408 );
5409 t.hold_manual(Some("later explicit hold".into()));
5410 assert_eq!(
5411 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5412 ActionDecision::Stale
5413 );
5414 }
5415 }
5416
5417 #[test]
5418 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
5419 let mut t = task();
5420 t.answers.push(crate::queue::AnsweredQuestion {
5421 question: "Inspect the remote?".into(),
5422 answer: "Inspect then wait".into(),
5423 });
5424 let old = instruction_for(&t);
5425 assert!(compatible_instruction(&old, &t));
5426 t.instruction = "Reuse approved PR head and review".into();
5427 t.release_operator("cli", None);
5428 assert!(!compatible_instruction(&old, &t));
5429 let approved = instruction_for(&t);
5430 assert!(approved.contains("waiting premises are superseded"));
5431 assert!(compatible_instruction(&approved, &t));
5432 assert_eq!(resumed_instruction(&approved, &t), approved);
5433 t.release_operator("web", None);
5434 assert!(compatible_instruction(&approved, &t));
5435 assert_eq!(t.resumption_history.len(), 1);
5436 t.start("run".into());
5437 assert!(instruction_for(&t).contains("Explicit operator resumption"));
5438 }
5439
5440 #[test]
5444 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
5445 let mut state = RunState::new(
5446 PathBuf::from("/repo"),
5447 "main".to_owned(),
5448 "abc1234".to_owned(),
5449 "task".to_owned(),
5450 Config::default(),
5451 );
5452 state.status = RunStatus::Blocked;
5453 state.resource_block = Some(crate::run::ResourceBlock {
5454 node: "graft".to_owned(),
5455 kind: "shared build cache".to_owned(),
5456 detail: "timed out".to_owned(),
5457 at: jiff::Timestamp::now(),
5458 });
5459 let mut t = task();
5460 t.start("20260902-000000-aaaa".to_owned());
5461 assert_eq!(t.attempts, 1);
5462 let verdict = Verdict {
5463 status: RunStatus::Blocked,
5464 left_pr: false,
5465 quota_hit: false,
5466 parked: false,
5467 no_viable_candidates: false,
5468 };
5469 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
5470 assert_eq!(t.status, TaskStatus::Held);
5471 assert_eq!(t.attempts, 0, "the attempt is refunded");
5472 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
5473 assert!(
5474 !t.operator_held(),
5475 "a machine hold, released without a human"
5476 );
5477
5478 let mut selected = task();
5479 selected.start("20260902-000000-cccc".into());
5480 selected.replacement = Some(crate::merge_decision::Replacement {
5481 question: "fixture".into(),
5482 run: "earlier".into(),
5483 pr: "https://example.invalid/pr/1".into(),
5484 agent: "selected".into(),
5485 context: "fixture".into(),
5486 });
5487 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
5488 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
5489 assert_eq!(selected.attempts, 0);
5490 assert!(
5491 !selected.operator_held(),
5492 "a selected agent also retains a machine resource hold"
5493 );
5494 assert!(
5495 selected
5496 .hold_reason
5497 .as_deref()
5498 .unwrap()
5499 .starts_with("[resource] ")
5500 );
5501
5502 let mut p = task();
5504 p.start("20260902-000000-bbbb".to_owned());
5505 settle_and_diagnose(
5506 &mut p,
5507 Verdict {
5508 parked: true,
5509 ..verdict
5510 },
5511 "parked",
5512 2,
5513 &state,
5514 );
5515 assert_eq!(p.status, TaskStatus::Parked);
5516 }
5517
5518 #[test]
5519 fn record_failure_notice_links_the_task() {
5520 let dir = tempfile::tempdir().unwrap();
5521 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
5524 let queue = Queue::at(dir.path().join("queue"));
5525 let mut t = interrupt_task("rec-fail-1");
5526 record(&queue, &mut t);
5527
5528 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
5529 assert_eq!(found.len(), 1);
5530 assert_eq!(found[0].key, "loop:record");
5531 assert_eq!(
5532 found[0].link,
5533 Some(Link::Task {
5534 id: "rec-fail-1".to_owned()
5535 })
5536 );
5537 }
5538
5539 fn interrupt_task(id: &str) -> Task {
5542 let mut t = task();
5543 t.id = id.to_owned();
5544 t.interrupt = true;
5545 t
5546 }
5547
5548 fn task_with_id(id: &str) -> Task {
5550 let mut t = task();
5551 t.id = id.to_owned();
5552 t
5553 }
5554
5555 fn urgent_task(id: &str) -> Task {
5557 let mut t = task();
5558 t.id = id.to_owned();
5559 t.urgent = true;
5560 t
5561 }
5562
5563 #[test]
5564 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
5565 let dir = tempfile::tempdir().unwrap();
5566 let queue = Queue::at(dir.path().join("queue"));
5567 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5568 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5569 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
5570 let mut first = task();
5571 queue.put(&mut first).unwrap();
5572 let stale_plain = queue.get(&first.id).unwrap();
5573 first.set_urgent(true).unwrap();
5574 queue.put(&mut first).unwrap();
5575 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
5576 let mut second = urgent_task("second");
5577 queue.put(&mut second).unwrap();
5578 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
5579 drop(queue.claim(&second.id).unwrap());
5580 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
5581 drop(dispatched);
5582 let stale_urgent = queue.get(&first.id).unwrap();
5583 first.set_urgent(false).unwrap();
5584 queue.put(&mut first).unwrap();
5585 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
5586 assert_eq!(urgent.available_permits(), 1);
5587 drop(queue.claim(&first.id).unwrap());
5588 drop(occupied);
5589 }
5590
5591 #[test]
5592 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
5593 let dir = tempfile::tempdir().unwrap();
5594 let queue = Queue::at(dir.path().join("queue"));
5595 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5596 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5597 let mut t = task();
5598 queue.put(&mut t).unwrap();
5599 for enabled in [true, false] {
5600 let stale = queue.get(&t.id).unwrap();
5601 t.set_urgent(enabled).unwrap();
5602 queue.put(&mut t).unwrap();
5603 assert_ne!(stale.urgent, enabled);
5604 let DispatchClaim {
5605 task: fresh,
5606 claim,
5607 permit,
5608 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
5609 assert_eq!(fresh.urgent, enabled);
5610 assert_eq!(ordinary.available_permits(), usize::from(enabled));
5611 assert_eq!(urgent.available_permits(), usize::from(!enabled));
5612 assert!(queue.claim(&t.id).is_err());
5613 assert_eq!(fresh.attempts, stale.attempts);
5614 drop((claim, permit));
5615 assert_eq!(ordinary.available_permits(), 1);
5616 assert_eq!(urgent.available_permits(), 1);
5617 }
5618 t.set_urgent(true).unwrap();
5619 queue.put(&mut t).unwrap();
5620 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
5621 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5622 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
5623 drop(queue.claim(&t.id).unwrap());
5624 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
5625 drop(occupied);
5626 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
5627 drop(queue.claim(&t.id).unwrap());
5628 let held = queue.claim(&t.id).unwrap();
5629 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5630 drop(held);
5631 t.status = TaskStatus::Held;
5632 queue.put(&mut t).unwrap();
5633 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5634 drop(queue.claim(&t.id).unwrap());
5635 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
5636 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5637 drop(queue.claim(&t.id).unwrap());
5638 assert_eq!(urgent.available_permits(), 1);
5639 }
5640
5641 #[test]
5646 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
5647 assert_eq!(permit_kind(true, false), PermitKind::None);
5648 assert_eq!(permit_kind(true, true), PermitKind::None);
5649 }
5650
5651 #[test]
5656 fn permit_kind_separates_urgent_from_ordinary() {
5657 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
5658 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
5659 }
5660
5661 #[test]
5668 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
5669 let cfg = Config::default();
5670 let repo = Path::new("/any/repo/path");
5671
5672 let reason =
5673 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
5674 assert!(reason.contains("1024"), "{reason}");
5675 assert!(
5676 reason.contains(&cfg.disk.min_free_bytes.to_string()),
5677 "{reason}"
5678 );
5679
5680 assert_eq!(
5681 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
5682 None,
5683 "exactly at the floor is open"
5684 );
5685 assert_eq!(
5686 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
5687 None,
5688 "comfortably above the floor is open"
5689 );
5690 }
5691
5692 #[test]
5693 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
5694 let mut cfg = Config::default();
5695 cfg.disk.min_free_bytes = 0;
5696 let repo = Path::new("/any/repo/path");
5697 assert_eq!(
5698 disk_gate_with(repo, &cfg, |_| Ok(0)),
5699 None,
5700 "a zero floor never measures at all"
5701 );
5702 }
5703
5704 #[test]
5705 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
5706 let cfg = Config::default();
5707 let repo = Path::new("/any/repo/path");
5708 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
5709 .expect("a measurement failure must close the gate, not open it");
5710 assert!(reason.contains("could not measure"), "{reason}");
5711 }
5712
5713 #[test]
5714 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
5715 let ordinary = task();
5716 let next = advance_interrupt(
5717 Interrupt::Idle,
5718 std::slice::from_ref(&ordinary.id),
5719 std::slice::from_ref(&ordinary),
5720 );
5721 assert_eq!(next, Interrupt::Idle);
5722 }
5723
5724 #[test]
5725 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
5726 let marked = interrupt_task("marked");
5729 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5730 assert_eq!(next, Interrupt::Idle);
5731 }
5732
5733 #[test]
5734 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
5735 let marked = interrupt_task("marked");
5736 let next = advance_interrupt(
5737 Interrupt::Idle,
5738 &["running".to_owned()],
5739 std::slice::from_ref(&marked),
5740 );
5741 assert_eq!(
5742 next,
5743 Interrupt::Parking {
5744 parked: vec!["running".to_owned()],
5745 interrupt_task: "marked".to_owned(),
5746 }
5747 );
5748 }
5749
5750 #[test]
5759 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
5760 let marked = interrupt_task("marked");
5761
5762 let two = advance_interrupt(
5763 Interrupt::Idle,
5764 &["a".to_owned(), "b".to_owned()],
5765 std::slice::from_ref(&marked),
5766 );
5767 assert_eq!(two, Interrupt::Idle);
5768
5769 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5770 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
5771 }
5772
5773 #[test]
5774 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
5775 let state = Interrupt::Parking {
5776 parked: vec!["running".to_owned()],
5777 interrupt_task: "marked".to_owned(),
5778 };
5779 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
5781 assert_eq!(still_going, state);
5782
5783 let stopped_but_not_yet_dispatched =
5787 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
5788 assert_eq!(stopped_but_not_yet_dispatched, state);
5789
5790 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
5792 assert_eq!(
5793 dispatched,
5794 Interrupt::Running {
5795 parked: vec!["running".to_owned()],
5796 interrupt_task: "marked".to_owned(),
5797 }
5798 );
5799 }
5800
5801 #[test]
5802 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
5803 let state = Interrupt::Running {
5804 parked: vec!["running".to_owned()],
5805 interrupt_task: "marked".to_owned(),
5806 };
5807 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
5808 assert_eq!(still_running, state);
5809
5810 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
5817 assert_eq!(
5818 ended,
5819 Interrupt::Resuming {
5820 parked: vec!["running".to_owned()]
5821 }
5822 );
5823 }
5824
5825 #[test]
5826 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
5827 let state = Interrupt::Resuming {
5828 parked: vec!["running".to_owned()],
5829 };
5830 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
5831 assert_eq!(still_waiting, state);
5832
5833 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
5834 assert_eq!(dispatched, Interrupt::Idle);
5835 }
5836
5837 #[test]
5843 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
5844 {
5845 let state = Interrupt::Parking {
5846 parked: vec!["running".to_owned()],
5847 interrupt_task: "marked".to_owned(),
5848 };
5849 let next = advance_interrupt(state, &[], &[]);
5852 assert_eq!(
5853 next,
5854 Interrupt::Resuming {
5855 parked: vec!["running".to_owned()]
5856 },
5857 "abandoning the interrupt must not abandon the resume it owes"
5858 );
5859 }
5860
5861 #[test]
5864 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
5865 let state = Interrupt::Resuming {
5866 parked: vec!["running".to_owned()],
5867 };
5868 let next = advance_interrupt(state, &[], &[]);
5869 assert_eq!(
5870 next,
5871 Interrupt::Idle,
5872 "nothing is left to wait for; the loop must not stay wedged"
5873 );
5874 }
5875
5876 #[test]
5877 fn disabled_by_config_the_sequence_can_never_leave_idle() {
5878 let marked = interrupt_task("marked");
5879 let next = advance_interrupt_tick(
5880 false,
5881 Interrupt::Idle,
5882 &["running".to_owned()],
5883 std::slice::from_ref(&marked),
5884 );
5885 assert_eq!(
5886 next,
5887 Interrupt::Idle,
5888 "an unmarked, unconfigured daemon must behave exactly as before"
5889 );
5890 }
5891
5892 #[test]
5893 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
5894 let state = Interrupt::Parking {
5895 parked: vec!["running".to_owned()],
5896 interrupt_task: "marked".to_owned(),
5897 };
5898 let candidates = vec![interrupt_task("marked"), task()];
5899 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5900 assert!(
5901 allowed.is_empty(),
5902 "nothing may dispatch - not even the interrupt task itself - \
5903 until the parked run has actually stopped"
5904 );
5905 }
5906
5907 #[test]
5921 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
5922 for state in [
5923 Interrupt::Parking {
5924 parked: vec!["running".to_owned()],
5925 interrupt_task: "marked".to_owned(),
5926 },
5927 Interrupt::Running {
5928 parked: vec!["running".to_owned()],
5929 interrupt_task: "marked".to_owned(),
5930 },
5931 Interrupt::Resuming {
5932 parked: vec!["running".to_owned()],
5933 },
5934 ] {
5935 let candidates = vec![
5936 interrupt_task("marked"),
5937 urgent_task("hot"),
5938 task_with_id("ordinary"),
5939 ];
5940 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5941 assert!(
5942 !allowed.iter().any(|t| t.id == "hot"),
5943 "an urgent candidate must wait out the same gate as anything \
5944 else while the run it would run alongside has not actually \
5945 left flight, for state {state:?}: {allowed:?}"
5946 );
5947 }
5948 }
5949
5950 #[test]
5956 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
5957 let state = Interrupt::Resuming {
5958 parked: vec!["hot".to_owned()],
5959 };
5960 let candidates = vec![urgent_task("hot"), task()];
5961 let allowed = interrupt_gate(&state, &[], candidates);
5962 assert_eq!(
5963 allowed.iter().filter(|t| t.id == "hot").count(),
5964 1,
5965 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
5966 );
5967 }
5968
5969 #[test]
5970 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
5971 let state = Interrupt::Parking {
5972 parked: vec!["running".to_owned()],
5973 interrupt_task: "marked".to_owned(),
5974 };
5975 let other = task();
5976 let candidates = vec![interrupt_task("marked"), other.clone()];
5977 let allowed = interrupt_gate(&state, &[], candidates);
5978 assert_eq!(allowed.len(), 1);
5979 assert_eq!(allowed[0].id, "marked");
5980 }
5981
5982 #[test]
5983 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
5984 let state = Interrupt::Running {
5985 parked: vec!["running".to_owned()],
5986 interrupt_task: "marked".to_owned(),
5987 };
5988 let candidates = vec![task(), task()];
5989 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
5990 assert!(allowed.is_empty());
5991 }
5992
5993 #[test]
6000 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
6001 let state = Interrupt::Resuming {
6002 parked: vec!["a".to_owned(), "c".to_owned()],
6003 };
6004 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
6005 let allowed = interrupt_gate(&state, &[], candidates);
6006 assert_eq!(
6007 allowed.len(),
6008 1,
6009 "at most one candidate may be offered while resuming: {allowed:?}"
6010 );
6011 assert_eq!(allowed[0].id, "a");
6012 }
6013
6014 #[test]
6015 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
6016 let state = Interrupt::Resuming {
6017 parked: vec!["a".to_owned()],
6018 };
6019 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
6020 assert!(allowed.is_empty());
6021 }
6022
6023 #[test]
6029 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
6030 let running = task(); let marked = interrupt_task("marked");
6032
6033 let mut state = Interrupt::Idle;
6034 let in_flight = vec![running.id.clone()];
6036 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
6037 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
6038 assert!(gated.is_empty(), "still waiting on `running` to park");
6039
6040 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
6042 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
6043 assert_eq!(
6044 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6045 vec!["marked"],
6046 "only the interrupt task may be offered to the dispatcher now"
6047 );
6048
6049 state = advance_interrupt_tick(
6051 true,
6052 state,
6053 &["marked".to_owned()],
6054 std::slice::from_ref(&running),
6055 );
6056 let gated = interrupt_gate(
6057 &state,
6058 &["marked".to_owned()],
6059 vec![marked.clone(), running.clone()],
6060 );
6061 assert!(
6062 gated.is_empty(),
6063 "the parked run must not be offered back while the interrupt \
6064 task is still running"
6065 );
6066
6067 let other = task_with_id("other");
6071 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
6072 assert_eq!(
6073 state,
6074 Interrupt::Resuming {
6075 parked: vec![running.id.clone()]
6076 }
6077 );
6078 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
6079 assert_eq!(
6080 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6081 vec![running.id.as_str()],
6082 "exactly the parked run resumes - not the unrelated task, even \
6083 though it was offered first"
6084 );
6085
6086 state = advance_interrupt_tick(
6090 true,
6091 state,
6092 std::slice::from_ref(&running.id),
6093 std::slice::from_ref(&other),
6094 );
6095 assert_eq!(state, Interrupt::Idle);
6096 let gated = interrupt_gate(
6097 &state,
6098 std::slice::from_ref(&running.id),
6099 vec![other.clone()],
6100 );
6101 assert_eq!(
6102 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6103 vec![other.id.as_str()],
6104 "ordinary dispatch is unrestricted again"
6105 );
6106 }
6107
6108 #[test]
6109 fn every_run_status_settles_the_task_it_came_from() {
6110 let table = [
6112 (RunStatus::Merged, TaskStatus::Done, 1),
6113 (RunStatus::Ready, TaskStatus::Done, 1),
6114 (RunStatus::Stalled, TaskStatus::Failed, 0),
6115 (RunStatus::Blocked, TaskStatus::Failed, 1),
6116 (RunStatus::Failed, TaskStatus::Failed, 1),
6117 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
6118 (RunStatus::Prep, TaskStatus::Failed, 1),
6119 (RunStatus::Implementing, TaskStatus::Failed, 1),
6120 (RunStatus::Judging, TaskStatus::Failed, 1),
6121 (RunStatus::Deliberating, TaskStatus::Failed, 1),
6122 (RunStatus::Voting, TaskStatus::Failed, 1),
6123 (RunStatus::Reviewing, TaskStatus::Failed, 1),
6124 (RunStatus::Gating, TaskStatus::Failed, 1),
6125 ];
6126 for (run, want, attempts) in table {
6127 let mut t = task();
6128 t.start("20260902-000000-aaaa".to_owned());
6129 settle(
6130 &mut t,
6131 Verdict {
6132 status: run,
6133 left_pr: false,
6134 parked: false,
6135 quota_hit: matches!(run, RunStatus::Stalled),
6136 no_viable_candidates: false,
6137 },
6138 "why",
6139 2,
6140 );
6141 assert_eq!(t.status, want, "task status after {}", label(run));
6142 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
6143 }
6144 }
6145
6146 #[test]
6147 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
6148 let mut stalled = task();
6149 stalled.start("20260902-000000-aaaa".to_owned());
6150 settle(
6151 &mut stalled,
6152 Verdict {
6153 status: RunStatus::Stalled,
6154 left_pr: false,
6155 parked: false,
6156 quota_hit: true,
6157 no_viable_candidates: false,
6158 },
6159 "quota",
6160 1,
6161 );
6162 assert_eq!(stalled.attempts, 0);
6163 assert!(
6164 stalled.status.runnable(),
6165 "a machine problem must leave the task in line"
6166 );
6167
6168 let mut blocked = task();
6169 blocked.start("20260902-000000-aaaa".to_owned());
6170 settle(
6171 &mut blocked,
6172 Verdict {
6173 status: RunStatus::Blocked,
6174 left_pr: false,
6175 parked: false,
6176 quota_hit: false,
6177 no_viable_candidates: false,
6178 },
6179 "findings open",
6180 1,
6181 );
6182 assert_eq!(blocked.attempts, 1);
6183 assert_eq!(
6184 blocked.status,
6185 TaskStatus::Held,
6186 "the last attempt hands the task to a human"
6187 );
6188 }
6189
6190 #[test]
6191 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
6192 let mut delivered = task();
6195 delivered.start("20260903-080619-01c2".to_owned());
6196 settle(
6197 &mut delivered,
6198 Verdict {
6199 status: RunStatus::Blocked,
6200 left_pr: true,
6201 parked: false,
6202 quota_hit: false,
6203 no_viable_candidates: false,
6204 },
6205 "no check status",
6206 4,
6207 );
6208 assert_eq!(
6209 delivered.status,
6210 TaskStatus::Held,
6211 "a pull request waiting on CI or a person is not a retryable failure"
6212 );
6213 assert!(
6214 !delivered.status.runnable(),
6215 "the loop must not pick this task up again"
6216 );
6217 assert_eq!(
6218 delivered.last_error.as_deref(),
6219 Some("no check status"),
6220 "the operator needs to be told what the gate was waiting for"
6221 );
6222
6223 let mut empty_handed = task();
6226 empty_handed.start("20260903-080619-01c2".to_owned());
6227 settle(
6228 &mut empty_handed,
6229 Verdict {
6230 status: RunStatus::Blocked,
6231 left_pr: false,
6232 parked: false,
6233 quota_hit: false,
6234 no_viable_candidates: false,
6235 },
6236 "findings open",
6237 4,
6238 );
6239 assert_eq!(empty_handed.status, TaskStatus::Failed);
6240 assert!(empty_handed.status.runnable());
6241 }
6242
6243 #[test]
6244 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
6245 let mut noop = task();
6251 noop.start("20260912-131304-391f".to_owned());
6252 settle(
6253 &mut noop,
6254 Verdict {
6255 status: RunStatus::VerifiedNoop,
6256 left_pr: false,
6257 parked: false,
6258 quota_hit: false,
6259 no_viable_candidates: true,
6260 },
6261 "candidate A: already fixed by b32cfc4, on main",
6262 4,
6263 );
6264 assert_eq!(
6265 noop.status,
6266 TaskStatus::Held,
6267 "an unverified claim is a request for a human, not a failure"
6268 );
6269 assert!(
6270 !noop.status.runnable(),
6271 "the loop must not requeue this on the same unverified claim"
6272 );
6273 assert_eq!(noop.attempts, 1);
6278 }
6279
6280 #[test]
6281 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
6282 let mut parked = task();
6287 parked.start("20260903-183634-2d98".to_owned());
6288 settle(
6289 &mut parked,
6290 Verdict {
6291 status: RunStatus::Implementing,
6292 left_pr: false,
6293 quota_hit: false,
6294 parked: true,
6295 no_viable_candidates: false,
6296 },
6297 "parked after `implementing`",
6298 2,
6299 );
6300 assert_eq!(parked.attempts, 0, "a park is refunded");
6301 assert!(
6302 parked.status.runnable(),
6303 "and the task stays in line so the next loop resumes its run"
6304 );
6305 assert_eq!(parked.status, TaskStatus::Parked);
6306 assert_eq!(
6307 parked.park_reason.as_deref(),
6308 Some("parked after `implementing`"),
6309 "the card says where it stopped"
6310 );
6311 assert_eq!(parked.last_error, None, "nothing failed");
6312
6313 let mut broken = task();
6317 broken.start("20260903-183634-2d98".to_owned());
6318 settle(
6319 &mut broken,
6320 Verdict {
6321 status: RunStatus::Implementing,
6322 left_pr: false,
6323 quota_hit: false,
6324 parked: false,
6325 no_viable_candidates: false,
6326 },
6327 "returned mid-flight",
6328 2,
6329 );
6330 assert_eq!(broken.attempts, 1);
6331 }
6332
6333 #[test]
6334 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
6335 let mut flaky = task();
6340 flaky.start("20260903-123023-e633".to_owned());
6341 settle(
6342 &mut flaky,
6343 Verdict {
6344 status: RunStatus::Stalled,
6345 left_pr: false,
6346 parked: false,
6347 quota_hit: false,
6348 no_viable_candidates: false,
6349 },
6350 "verdict rests on 1 of 3 judges",
6351 2,
6352 );
6353 assert_eq!(
6354 flaky.attempts, 1,
6355 "flakiness spends an attempt, so `max_attempts` still bounds it"
6356 );
6357 assert!(flaky.status.runnable(), "and it is still worth retrying");
6358
6359 let mut limited = task();
6361 limited.start("20260903-123023-e633".to_owned());
6362 settle(
6363 &mut limited,
6364 Verdict {
6365 status: RunStatus::Stalled,
6366 left_pr: false,
6367 parked: false,
6368 quota_hit: true,
6369 no_viable_candidates: false,
6370 },
6371 "judge-2, judge-3 out of quota",
6372 2,
6373 );
6374 assert_eq!(limited.attempts, 0, "a quota window is refunded");
6375 assert!(limited.status.runnable());
6376
6377 let mut worn = task();
6380 for _ in 0..2 {
6381 worn.release();
6382 }
6383 worn.start("20260903-123023-e633".to_owned());
6384 worn.attempts = 2;
6385 settle(
6386 &mut worn,
6387 Verdict {
6388 status: RunStatus::Stalled,
6389 left_pr: false,
6390 parked: false,
6391 quota_hit: false,
6392 no_viable_candidates: false,
6393 },
6394 "no quorum again",
6395 2,
6396 );
6397 assert_eq!(worn.status, TaskStatus::Held);
6398 assert!(!worn.status.runnable());
6399 }
6400
6401 #[test]
6402 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
6403 let mut wiped_out = task();
6410 wiped_out.start("20260907-025000-a1b2".to_owned());
6411 settle(
6412 &mut wiped_out,
6413 Verdict {
6414 status: RunStatus::Failed,
6415 left_pr: false,
6416 parked: false,
6417 quota_hit: true,
6418 no_viable_candidates: true,
6419 },
6420 "no candidate produced a change; nothing to judge",
6421 2,
6422 );
6423 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
6424 assert!(
6425 wiped_out.status.runnable(),
6426 "a machine problem must leave the task in line"
6427 );
6428
6429 let mut partial_progress = task();
6435 partial_progress.start("20260907-025500-c3d4".to_owned());
6436 settle(
6437 &mut partial_progress,
6438 Verdict {
6439 status: RunStatus::Failed,
6440 left_pr: false,
6441 parked: false,
6442 quota_hit: true,
6443 no_viable_candidates: false,
6444 },
6445 "gate failed on the winning candidate",
6446 2,
6447 );
6448 assert_eq!(
6449 partial_progress.attempts, 1,
6450 "a candidate that actually produced a change spends the attempt \
6451 even though some other seat hit its quota"
6452 );
6453 assert!(partial_progress.status.runnable());
6454 }
6455
6456 #[test]
6457 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
6458 let mut t = task();
6464 t.start("20260907-025000-a1b2".to_owned());
6465 let mut state = run_state(RunStatus::Failed);
6466 state.quota.push(QuotaLoss {
6467 seat: "cand-a".to_owned(),
6468 node: "implement".to_owned(),
6469 at: Timestamp::now(),
6470 reset: None,
6471 });
6472 assert!(
6473 state.viable().is_empty(),
6474 "no candidate was added, so nothing is viable"
6475 );
6476 reclaim(&mut t, Some(state), 2, "en");
6477 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
6478 assert!(t.status.runnable());
6479 }
6480
6481 #[test]
6482 fn a_held_task_is_never_offered_to_the_loop() {
6483 let dir = tempfile::tempdir().unwrap();
6484 let queue = Queue::at(dir.path().to_path_buf());
6485 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
6486 let mut t = task();
6487 t.id = format!("2026090{n}-000000-000{n}");
6488 t.priority = priority;
6489 queue.put(&mut t).unwrap();
6490 }
6491 let mut held = task();
6492 held.id = "20260909-000000-9999".to_owned();
6493 held.priority = 99;
6494 held.hold_machine(None);
6495 queue.put(&mut held).unwrap();
6496
6497 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
6498 assert_eq!(order.len(), 3);
6499 assert!(!order.contains(&held.id));
6500 assert_eq!(
6501 order.first().cloned(),
6502 queue.next_runnable().map(|t| t.id),
6503 "the loop's first candidate is exactly what the queue offers"
6504 );
6505 assert_eq!(
6506 order,
6507 vec![
6508 "20260902-000000-0002".to_owned(),
6509 "20260903-000000-0003".to_owned(),
6510 "20260901-000000-0001".to_owned(),
6511 ],
6512 "priority first, then oldest, so nothing starves"
6513 );
6514 }
6515
6516 #[test]
6517 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
6518 let dir = tempfile::tempdir().unwrap();
6519 let queue = Queue::at(dir.path().to_path_buf());
6520 let mut old = task();
6521 old.id = "20260101-000000-old0".to_owned();
6522 queue.put(&mut old).unwrap();
6523 let mut fresh = task();
6524 fresh.id = "20260101-000000-new0".to_owned();
6525 queue.put(&mut fresh).unwrap();
6526
6527 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
6531 std::thread::sleep(Duration::from_millis(60));
6532 let live = queue.claim(&fresh.id).unwrap();
6533
6534 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6535 assert_eq!(swept, vec![old.id.clone()]);
6536 assert!(
6537 queue.claim(&old.id).is_ok(),
6538 "an unparseable lock older than the threshold is swept"
6539 );
6540 assert!(
6541 queue.claim(&fresh.id).is_err(),
6542 "a live pid protects its lock regardless of age"
6543 );
6544 drop(live);
6545 }
6546
6547 #[test]
6548 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
6549 let dir = tempfile::tempdir().unwrap();
6559 let queue = Queue::at(dir.path().to_path_buf());
6560 let mut t = task();
6561 t.id = "20260101-000000-live".to_owned();
6562 queue.put(&mut t).unwrap();
6563
6564 let claim = queue.claim(&t.id).unwrap();
6565 std::thread::sleep(Duration::from_millis(60));
6566
6567 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6568 assert!(
6569 swept.is_empty(),
6570 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
6571 );
6572 assert!(
6573 queue.claim(&t.id).is_err(),
6574 "the lock still protects its task"
6575 );
6576 drop(claim);
6577 }
6578
6579 fn injected_dead_pid() -> u32 {
6582 std::process::id().checked_add(1).unwrap_or(1)
6583 }
6584
6585 #[test]
6586 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
6587 let dir = tempfile::tempdir().unwrap();
6588 let queue = Queue::at(dir.path().to_path_buf());
6589 let mut t = task();
6590 t.id = "20260101-000000-dead".to_owned();
6591 queue.put(&mut t).unwrap();
6592 let dead_pid = injected_dead_pid();
6593
6594 std::fs::write(
6599 dir.path().join(format!("{}.lock", t.id)),
6600 dead_pid.to_string(),
6601 )
6602 .unwrap();
6603
6604 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6605 pid != dead_pid
6606 });
6607 assert_eq!(
6608 swept,
6609 vec![t.id.clone()],
6610 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
6611 );
6612 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
6613 }
6614
6615 #[test]
6616 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
6617 let dir = tempfile::tempdir().unwrap();
6618 let queue = Queue::at(dir.path().to_path_buf());
6619 let mut t = task();
6620 t.id = "20260101-000000-late".to_owned();
6621 queue.put(&mut t).unwrap();
6622 let dead_pid = injected_dead_pid();
6623
6624 assert!(
6627 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
6628 "nothing has claimed the task yet"
6629 );
6630
6631 std::fs::write(
6634 dir.path().join(format!("{}.lock", t.id)),
6635 dead_pid.to_string(),
6636 )
6637 .unwrap();
6638
6639 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6643 pid != dead_pid
6644 });
6645 assert_eq!(swept, vec![t.id.clone()]);
6646 }
6647
6648 #[test]
6649 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
6650 crate::run::pin_test_home();
6655 let dir = tempfile::tempdir().unwrap();
6656 let queue = Queue::at(dir.path().to_path_buf());
6657 let mut t = task();
6658 t.id = "20260101-000000-crsh".to_owned();
6659 t.status = TaskStatus::Running;
6660 t.attempts = 1;
6661 t.runs.push("20260904-000000-4043".to_owned());
6665 queue.put(&mut t).unwrap();
6666 let dead_pid = injected_dead_pid();
6667
6668 std::fs::write(
6671 dir.path().join(format!("{}.lock", t.id)),
6672 dead_pid.to_string(),
6673 )
6674 .unwrap();
6675
6676 assert!(reclaim_orphaned_running(&queue, 2).is_empty());
6682 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
6683
6684 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6685 pid != dead_pid
6686 });
6687 assert_eq!(swept, vec![t.id.clone()]);
6688
6689 let reclaimed = reclaim_orphaned_running(&queue, 2);
6690 assert_eq!(reclaimed, vec![t.id.clone()]);
6691 let after = queue.get(&t.id).unwrap();
6692 assert_eq!(
6693 after.status,
6694 TaskStatus::Held,
6695 "no run.json to recover from, so a human is asked"
6696 );
6697 assert_eq!(
6698 after.runs,
6699 vec!["20260904-000000-4043".to_owned()],
6700 "the crashed run's id is kept as evidence, not discarded"
6701 );
6702 }
6703
6704 #[test]
6705 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
6706 let dir = tempfile::tempdir().unwrap();
6707 let queue = Queue::at(dir.path().to_path_buf());
6708 let mut t = task();
6709 t.id = "20260101-000000-unknown".to_owned();
6710 queue.put(&mut t).unwrap();
6711 let dead_pid = injected_dead_pid();
6712 std::fs::write(
6713 dir.path().join(format!("{}.lock", t.id)),
6714 dead_pid.to_string(),
6715 )
6716 .unwrap();
6717
6718 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
6719 assert!(swept.is_empty(), "an unknown pid must keep its lock");
6720 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
6721 }
6722
6723 fn run_state_in(status: RunStatus, language: &str) -> RunState {
6724 let mut s = run_state(status);
6725 s.config.graph.language = language.to_owned();
6726 s
6727 }
6728
6729 fn unstarted_verdict(status: RunStatus) -> Verdict {
6730 Verdict {
6731 status,
6732 left_pr: false,
6733 quota_hit: false,
6734 parked: false,
6735 no_viable_candidates: false,
6736 }
6737 }
6738
6739 #[test]
6740 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
6741 let mut t = task();
6742 t.attempts = 1;
6743 settle_in(
6744 &mut t,
6745 unstarted_verdict(RunStatus::AlreadyInBase),
6746 "already in main",
6747 1,
6748 phrases("en"),
6749 );
6750 assert_eq!(t.status, TaskStatus::Done);
6751 assert_eq!(t.attempts, 0);
6752 }
6753
6754 #[test]
6755 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
6756 let reason = |language: &str| {
6757 let mut t = task();
6758 settle_in(
6759 &mut t,
6760 unstarted_verdict(RunStatus::Judging),
6761 "boom",
6762 1,
6763 phrases(language),
6764 );
6765 t.last_error.or(t.hold_reason).unwrap_or_default()
6766 };
6767 assert!(
6768 reason("en").starts_with("the graph stopped at `"),
6769 "{}",
6770 reason("en")
6771 );
6772 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
6773 assert!(reason("日本語").contains("boom"));
6774 assert_eq!(reason("fr"), reason("en"));
6775 }
6776
6777 #[test]
6778 fn describe_follows_the_run_language_and_keeps_the_run_id() {
6779 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
6780 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
6781 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
6782 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
6783 assert!(ja.contains("[run "), "{ja}");
6784 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
6785 assert!(ended.starts_with("run 終了: "), "{ended}");
6786 let mut de = run_state_in(RunStatus::Failed, "de");
6787 let mut en = run_state_in(RunStatus::Failed, "en");
6788 de.id = "same".to_owned();
6789 en.id = "same".to_owned();
6790 assert_eq!(describe(&de), describe(&en));
6791 }
6792
6793 #[test]
6794 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
6795 use crate::handover::Refused;
6796 let cases = [
6797 Refused::Foreign {
6798 branch: "b".into(),
6799 path: "/w/x".into(),
6800 why: "made by hand".into(),
6801 },
6802 Refused::Unsafe {
6803 branch: "b".into(),
6804 path: "/w/x".into(),
6805 why: "its worktree has uncommitted changes (a.rs)".into(),
6806 },
6807 Refused::ReleaseFailed {
6808 branch: "b".into(),
6809 path: "/w/x".into(),
6810 run: "ab12".into(),
6811 },
6812 ];
6813 for r in &cases {
6814 let en = (phrases("en").handover_refused)(r);
6815 assert_eq!(en, r.to_string());
6816 let ja = (phrases("ja").handover_refused)(r);
6817 assert!(
6818 !ja.contains("is checked out") && !ja.contains("try again"),
6819 "{ja}"
6820 );
6821 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
6822 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
6823 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
6824 }
6825 }
6826 assert!(phrases("en").handover_hint.contains("release the task"));
6827 assert!(phrases("ja").handover_hint.contains("解放"));
6828 }
6829
6830 #[test]
6831 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
6832 let build = |lang: &str| {
6833 let mut s = run_state_in(RunStatus::Blocked, lang);
6834 s.id = "same".to_owned();
6835 s.config.graph.review_rounds = 3;
6836 let mut r = review_round(3);
6837 r.expected = 3;
6838 r.answered = 1;
6839 s.reviews.push(r);
6840 s.event(
6841 "review",
6842 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
6843 );
6844 s
6845 };
6846 let en = describe(&build("en"));
6847 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
6848 let ja = describe(&build("ja"));
6849 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
6850 assert!(!ja.contains("never answered"), "{ja}");
6851 assert!(ja.contains("[run "), "{ja}");
6852
6853 let mut failed = build("ja");
6856 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
6857 command: "cargo test".to_owned(),
6858 code: Some(1),
6859 output_tail: String::new(),
6860 duration_ms: 0,
6861 resource_blocked: false,
6862 });
6863 failed.event("review", "stopped; e2e failed: cargo test");
6864 let ja = describe(&failed);
6865 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
6866 assert!(!ja.contains("席のレビュアー"), "{ja}");
6867 }
6868
6869 #[test]
6870 fn refusals_and_recovery_prose_follow_the_language() {
6871 let t = held_task_with("r1");
6872 let q = action_question(
6873 "r1",
6874 ask::ChoiceAction::Resume {
6875 run: "r1".to_owned(),
6876 },
6877 );
6878 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
6879 ActionDecision::Refuse(s) => s,
6880 other => panic!("{other:?}"),
6881 };
6882 assert!(refuse(phrases("en")).contains("could not be read: gone"));
6883 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
6884 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
6885
6886 let mut held = task();
6887 reclaim(&mut held, None, 2, "ja");
6888 assert!(held.hold_reason.unwrap().contains("保留にしました"));
6889 let mut held = task();
6890 reclaim(&mut held, None, 2, "xx");
6891 assert!(held.hold_reason.unwrap().contains("held for a human"));
6892 }
6893
6894 fn run_state(status: RunStatus) -> RunState {
6895 let mut state = RunState::new(
6896 PathBuf::from("/repo"),
6897 "main".to_owned(),
6898 "abc1234def".to_owned(),
6899 "add retries".to_owned(),
6900 Config::default(),
6901 );
6902 state.status = status;
6903 state
6904 }
6905
6906 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
6907 Candidate {
6908 index: 0,
6909 label,
6910 agent: "claude".to_owned(),
6911 branch: format!("magi/x/{label}"),
6912 worktree: PathBuf::from("/repo"),
6913 summary: summary.to_owned(),
6914 stat: String::new(),
6915 files: 0,
6916 commits: usize::from(!empty),
6917 empty,
6918 failed: failed.map(str::to_owned),
6919 verified_noop: None,
6920 duration_ms: 0,
6921 folded: false,
6922 }
6923 }
6924
6925 #[test]
6926 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
6927 let mut state = run_state(RunStatus::Blocked);
6928 state.gate = vec![
6929 CommandOutcome {
6930 command: "cargo make check".to_owned(),
6931 code: Some(0),
6932 output_tail: "ok".to_owned(),
6933 duration_ms: 0,
6934 resource_blocked: false,
6935 },
6936 CommandOutcome {
6937 command: "cargo test".to_owned(),
6938 code: Some(101),
6939 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
6940 duration_ms: 0,
6941 resource_blocked: false,
6942 },
6943 ];
6944 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
6945 assert!(d.contains("cargo test"), "{d}");
6946 assert!(
6947 !d.contains("cargo make check"),
6948 "a passing check is not a diagnostic: {d}"
6949 );
6950 assert!(d.contains("assertion failed"), "{d}");
6951 }
6952
6953 #[test]
6954 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
6955 let mut state = run_state(RunStatus::Blocked);
6956 state.event(
6957 "land",
6958 "stopped: the fixer produced no commit while 2 check(s) were failing \
6959 (build, lint); stopping instead of looping on an unchanged tree",
6960 );
6961 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
6962 assert!(d.contains("build"), "{d}");
6963 assert!(d.contains("lint"), "{d}");
6964 assert!(d.contains("fixer produced no commit"), "{d}");
6965 }
6966
6967 #[test]
6968 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
6969 let state = run_state(RunStatus::VerifiedNoop);
6974 let d = describe(&state);
6975 assert!(
6976 d.contains("agent-verified no-op"),
6977 "expected the display label, not the wire spelling: {d}"
6978 );
6979 assert!(!d.contains("verified_noop"), "{d}");
6980 }
6981
6982 #[test]
6983 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
6984 let mut state = run_state(RunStatus::Failed);
6990 state.candidates = vec![candidate(
6991 'A',
6992 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
6993 true,
6994 None,
6995 )];
6996 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
6997 assert!(d.contains("candidate A"), "{d}");
6998 assert!(d.contains("tagged v1.2.3"), "{d}");
6999 }
7000
7001 #[test]
7002 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
7003 let mut state = run_state(RunStatus::Failed);
7004 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
7005 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
7006 assert!(d.contains("candidate A"), "{d}");
7007 assert!(d.contains("agent timed out"), "{d}");
7008 }
7009
7010 #[test]
7011 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
7012 let mut state = run_state(RunStatus::Failed);
7015 state.candidates = vec![candidate('A', "did the work", false, None)];
7016 assert!(diagnostic(&state).is_none());
7017 }
7018
7019 #[test]
7020 fn diagnostic_is_bounded_however_much_a_run_printed() {
7021 let mut state = run_state(RunStatus::Blocked);
7022 state.gate = vec![
7023 CommandOutcome {
7024 command: "cargo test".to_owned(),
7025 code: Some(101),
7026 output_tail: "x".repeat(50_000),
7027 duration_ms: 0,
7028 resource_blocked: false,
7029 },
7030 CommandOutcome {
7031 command: "cargo clippy".to_owned(),
7032 code: Some(1),
7033 output_tail: "y".repeat(50_000),
7034 duration_ms: 0,
7035 resource_blocked: false,
7036 },
7037 ];
7038 state.candidates = vec![
7039 candidate('A', &"z".repeat(50_000), true, None),
7040 candidate('B', &"w".repeat(50_000), true, None),
7041 ];
7042 let d = diagnostic(&state).expect("plenty here to diagnose");
7043 assert!(
7044 d.len() <= DIAGNOSTIC_MAX,
7045 "diagnostic grew to {} bytes, unbounded",
7046 d.len()
7047 );
7048 }
7049
7050 #[test]
7051 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
7052 let mut state = run_state(RunStatus::Blocked);
7053 state.gate = vec![CommandOutcome {
7054 command: "cargo test".to_owned(),
7055 code: Some(101),
7056 output_tail: "assertion failed".to_owned(),
7057 duration_ms: 0,
7058 resource_blocked: false,
7059 }];
7060 let verdict = Verdict {
7061 status: RunStatus::Blocked,
7062 left_pr: false,
7063 quota_hit: false,
7064 parked: false,
7065 no_viable_candidates: false,
7066 };
7067
7068 let mut t = task();
7071 t.start("run-1".to_owned());
7072 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
7073 assert_eq!(t.status, TaskStatus::Failed);
7074 assert!(t.diagnostic.is_none());
7075
7076 t.start("run-2".to_owned());
7079 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
7080 assert_eq!(t.status, TaskStatus::Held);
7081 let d = t.diagnostic.expect("a held task must carry its diagnostic");
7082 assert!(d.contains("cargo test"), "{d}");
7083 }
7084
7085 #[test]
7086 fn a_held_task_names_the_open_question_it_is_waiting_on() {
7087 crate::run::pin_test_home();
7092 let home = crate::run::home();
7093 let state = run_state(RunStatus::VerifiedNoop);
7094 let mut q = ask::Question::new(
7095 state.id.clone(),
7096 "implement".to_owned(),
7097 "impl-A".to_owned(),
7098 "is this really a no-op?".to_owned(),
7099 String::new(),
7100 Vec::new(),
7101 );
7102 Questions::at(home.join("questions")).put(&mut q).unwrap();
7103
7104 let verdict = Verdict {
7105 status: RunStatus::VerifiedNoop,
7106 left_pr: false,
7107 quota_hit: false,
7108 parked: false,
7109 no_viable_candidates: false,
7110 };
7111 let mut t = task();
7112 t.start(state.id.clone());
7113 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
7114
7115 assert_eq!(t.status, TaskStatus::Held);
7116 let reason = t.hold_reason.expect("a held task must record why");
7117 assert!(
7118 reason.starts_with("run ended agent-verified no-op"),
7119 "the original settle reason must survive unchanged: {reason}"
7120 );
7121 assert!(
7122 reason.contains(q.short()),
7123 "the open question's id must be named so the notice is actionable: {reason}"
7124 );
7125 }
7126
7127 #[test]
7128 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
7129 crate::run::pin_test_home();
7130 let state = run_state(RunStatus::VerifiedNoop);
7131
7132 let verdict = Verdict {
7133 status: RunStatus::VerifiedNoop,
7134 left_pr: false,
7135 quota_hit: false,
7136 parked: false,
7137 no_viable_candidates: false,
7138 };
7139 let mut t = task();
7140 t.start(state.id.clone());
7141 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
7142
7143 assert_eq!(t.status, TaskStatus::Held);
7144 assert_eq!(
7145 t.hold_reason.as_deref(),
7146 Some("run ended agent-verified no-op"),
7147 "nothing to append when the question was already answered or never asked"
7148 );
7149 }
7150
7151 #[test]
7152 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
7153 crate::run::pin_test_home();
7154 let mut first = run_state(RunStatus::Blocked);
7155 first.id = "20260101-000000-sup1".to_owned();
7156 first.save().unwrap();
7157 let mut second = run_state(RunStatus::Merged);
7158 second.id = "20260101-000000-sup2".to_owned();
7159 second.save().unwrap();
7160
7161 let mut t = task();
7162 t.runs = vec![first.id.clone(), second.id.clone()];
7163 t.status = TaskStatus::Done;
7164
7165 supersede_prior_runs(&t, &crate::run::home());
7166
7167 assert_eq!(
7168 RunState::load(&first.id).unwrap().status,
7169 RunStatus::Superseded,
7170 "the first attempt's Blocked no longer needs anyone's attention"
7171 );
7172 assert_eq!(
7173 RunState::load(&second.id).unwrap().status,
7174 RunStatus::Merged,
7175 "the run that actually succeeded is left exactly as it was"
7176 );
7177 }
7178
7179 #[test]
7180 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
7181 crate::run::pin_test_home();
7187 let mut first = run_state(RunStatus::Blocked);
7188 first.id = "20260101-000000-sup9".to_owned();
7189 first.driver_pid = Some(std::process::id());
7192 first.driver_started_at = Some(
7193 crate::proc::process_started_at(std::process::id())
7194 .expect("this test process's own start time must be queryable"),
7195 );
7196 first.save().unwrap();
7197 let mut second = run_state(RunStatus::Merged);
7198 second.id = "20260101-000000-supa".to_owned();
7199 second.save().unwrap();
7200
7201 let mut t = task();
7202 t.runs = vec![first.id.clone(), second.id.clone()];
7203 t.status = TaskStatus::Done;
7204
7205 supersede_prior_runs(&t, &crate::run::home());
7206
7207 assert_eq!(
7208 RunState::load(&first.id).unwrap().status,
7209 RunStatus::Blocked,
7210 "a live driver_pid means something is still actually working this run, \
7211 even though no daemon claims it - rewriting under it would just be \
7212 undone the next time that process saves"
7213 );
7214 }
7215
7216 #[test]
7217 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
7218 let dir = tempfile::tempdir().unwrap();
7224 let home = dir.path().to_path_buf();
7225 let queue = Queue::at(dir.path().join("queue"));
7226
7227 let mut first = run_state(RunStatus::Blocked);
7228 first.id = "20260101-000000-supd".to_owned();
7229 first.driver_pid = Some(std::process::id());
7230 first.driver_started_at = Some(
7231 crate::proc::process_started_at(std::process::id())
7232 .expect("this test process's own start time must be queryable"),
7233 );
7234 first.save_under(&home).unwrap();
7235 let mut second = run_state(RunStatus::Merged);
7236 second.id = "20260101-000000-supe".to_owned();
7237 second.save_under(&home).unwrap();
7238
7239 let mut t = task();
7240 t.runs = vec![first.id.clone(), second.id.clone()];
7241 t.status = TaskStatus::Done;
7242 queue.put(&mut t).unwrap();
7243
7244 resweep_superseded_attempts(&queue, &home);
7245 assert_eq!(
7246 RunState::load_under(&first.id, &home).unwrap().status,
7247 RunStatus::Blocked,
7248 "still live on the first pass, so still untouched"
7249 );
7250
7251 let mut stale = RunState::load_under(&first.id, &home).unwrap();
7257 stale.driver_started_at = Some("1".to_owned());
7258 stale.save_under(&home).unwrap();
7259
7260 resweep_superseded_attempts(&queue, &home);
7261 assert_eq!(
7262 RunState::load_under(&first.id, &home).unwrap().status,
7263 RunStatus::Superseded,
7264 "the second pass catches up what the first one correctly skipped"
7265 );
7266 }
7267
7268 #[test]
7269 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
7270 crate::run::pin_test_home();
7271 let mut first = run_state(RunStatus::Blocked);
7272 first.id = "20260101-000000-sup3".to_owned();
7273 first.save().unwrap();
7274 let mut second = run_state(RunStatus::Blocked);
7275 second.id = "20260101-000000-sup4".to_owned();
7276 second.save().unwrap();
7277
7278 let mut t = task();
7279 t.runs = vec![first.id.clone(), second.id.clone()];
7280 t.status = TaskStatus::Failed;
7284
7285 supersede_prior_runs(&t, &crate::run::home());
7286
7287 assert_eq!(
7288 RunState::load(&first.id).unwrap().status,
7289 RunStatus::Blocked
7290 );
7291 assert_eq!(
7292 RunState::load(&second.id).unwrap().status,
7293 RunStatus::Blocked
7294 );
7295 }
7296
7297 #[test]
7298 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
7299 crate::run::pin_test_home();
7303 let mut first = run_state(RunStatus::Blocked);
7304 first.id = "20260101-000000-sup5".to_owned();
7305 first.save().unwrap();
7306
7307 let mut t = task();
7308 t.runs = vec![first.id.clone()];
7309 t.status = TaskStatus::Done;
7310
7311 supersede_prior_runs(&t, &crate::run::home());
7312
7313 assert_eq!(
7314 RunState::load(&first.id).unwrap().status,
7315 RunStatus::Blocked,
7316 "a single-attempt task has no earlier run to supersede"
7317 );
7318 }
7319
7320 #[test]
7321 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
7322 crate::run::pin_test_home();
7329 let mut first = run_state(RunStatus::Blocked);
7330 first.id = "20260101-000000-supb".to_owned();
7331 first.save().unwrap();
7332 let mut second = run_state(RunStatus::Failed);
7333 second.id = "20260101-000000-supc".to_owned();
7334 second.save().unwrap();
7335
7336 let mut t = task();
7337 t.runs = vec![first.id.clone(), second.id.clone()];
7338 t.status = TaskStatus::Done;
7339
7340 supersede_prior_runs(&t, &crate::run::home());
7341
7342 assert_eq!(
7343 RunState::load(&first.id).unwrap().status,
7344 RunStatus::Blocked,
7345 "the task's last attempt never landed, so there is nothing here \
7346 actually superseding it"
7347 );
7348 }
7349
7350 #[test]
7351 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
7352 crate::run::pin_test_home();
7356 let mut failed = run_state(RunStatus::Failed);
7357 failed.id = "20260101-000000-sup6".to_owned();
7358 failed.save().unwrap();
7359 let mut noop = run_state(RunStatus::VerifiedNoop);
7360 noop.id = "20260101-000000-sup7".to_owned();
7361 noop.save().unwrap();
7362 let mut winner = run_state(RunStatus::Ready);
7363 winner.id = "20260101-000000-sup8".to_owned();
7364 winner.save().unwrap();
7365
7366 let mut t = task();
7367 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
7368 t.status = TaskStatus::Done;
7369
7370 supersede_prior_runs(&t, &crate::run::home());
7371
7372 assert_eq!(
7373 RunState::load(&failed.id).unwrap().status,
7374 RunStatus::Failed
7375 );
7376 assert_eq!(
7377 RunState::load(&noop.id).unwrap().status,
7378 RunStatus::VerifiedNoop
7379 );
7380 }
7381
7382 fn approval_question(run: &str) -> ask::Question {
7383 ask::Question::new(
7384 run.to_owned(),
7385 land::APPROVAL_NODE.to_owned(),
7386 "land".to_owned(),
7387 "merge?".to_owned(),
7388 String::new(),
7389 vec!["merge".to_owned(), "hold".to_owned()],
7390 )
7391 }
7392
7393 #[test]
7394 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
7395 crate::run::pin_test_home();
7396 let mut state = run_state(RunStatus::Landing);
7397 state.id = "20260101-000000-fre1".to_owned();
7398 state.parked = true;
7399 state.save().unwrap();
7400 ask::Questions::open()
7401 .put(&mut approval_question(&state.id))
7402 .unwrap();
7403
7404 let mut t = task();
7405 t.runs.push(state.id.clone());
7406 assert_eq!(
7407 land_resume_state(&t),
7408 LandResume::StillWaiting,
7409 "nobody has answered and the timeout has not passed"
7410 );
7411 }
7412
7413 #[test]
7414 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
7415 crate::run::pin_test_home();
7416 let mut state = run_state(RunStatus::Gating);
7417 state.id = "20260101-000000-gt01".to_owned();
7418 state.parked = true;
7419 state.config.graph.answer_timeout = 60;
7420 state.github_text = Some(crate::run::GithubTextAsk {
7421 rewrite: None,
7422 fingerprint: "f".into(),
7423 title: true,
7424 body: false,
7425 categories: vec!["title-language".into()],
7426 question: None,
7427 asks: 1,
7428 resolved: false,
7429 chosen_title: None,
7430 choice: None,
7431 });
7432 state.save().unwrap();
7433 let store = ask::Questions::open();
7434 let mut q = ask::Question::new(
7435 state.id.clone(),
7436 crate::github_text::ASK_NODE.to_owned(),
7437 crate::github_text::ASK_SEAT.to_owned(),
7438 "s".into(),
7439 String::new(),
7440 vec!["use fallback".into()],
7441 );
7442 store.put(&mut q).unwrap();
7443 let mut t = task();
7444 t.runs.push(state.id.clone());
7445 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
7446 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7447 store.put(&mut q).unwrap();
7448 assert_eq!(land_resume_state(&t), LandResume::Ready);
7449 assert!(!store.get(&q.id).unwrap().status.open());
7450 state.github_text.as_mut().unwrap().resolved = true;
7452 state.save().unwrap();
7453 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
7454 }
7455
7456 #[test]
7457 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
7458 crate::run::pin_test_home();
7463 let mut state = run_state(RunStatus::Landing);
7464 state.id = "20260101-000000-exp1".to_owned();
7465 state.parked = true;
7466 state.config.graph.answer_timeout = 60;
7467 state.save().unwrap();
7468
7469 let store = ask::Questions::open();
7470 let mut q = approval_question(&state.id);
7471 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7472 store.put(&mut q).unwrap();
7473
7474 let mut t = task();
7475 t.runs.push(state.id.clone());
7476 assert_eq!(
7477 land_resume_state(&t),
7478 LandResume::Ready,
7479 "an expired question must not be waited on forever"
7480 );
7481
7482 let after = store.get(&q.id).unwrap();
7483 assert!(
7484 !after.status.open(),
7485 "the question is abandoned, not silently ignored"
7486 );
7487 assert!(
7488 after.resolution().is_none(),
7489 "an abandoned question is not read as a decision"
7490 );
7491 }
7492
7493 #[test]
7494 fn reclaim_settles_a_running_task_against_its_last_run() {
7495 let mut t = task();
7496 t.start("20260904-000000-4043".to_owned());
7497 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
7498 assert_eq!(
7499 t.status,
7500 TaskStatus::Done,
7501 "a run that actually finished must not stay `running` forever"
7502 );
7503 }
7504
7505 #[test]
7506 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
7507 let mut t = task();
7508 t.start("20260904-000000-4043".to_owned());
7509 t.last_error = Some("earlier trouble".to_owned());
7510 let mut state = run_state(RunStatus::Judging);
7511 state.parked = true;
7512 reclaim(&mut t, Some(state), 1, "en");
7513 assert_eq!(t.status, TaskStatus::Parked);
7514 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
7515 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7516 assert!(t.park_reason.is_some());
7517 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
7518 assert!(!t.fresh_start);
7519 assert!(t.status.runnable());
7520 }
7521
7522 #[test]
7523 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
7524 let mut t = task();
7525 t.start("20260904-000000-4043".to_owned());
7526 t.last_error = Some("earlier trouble".to_owned());
7527 settle(
7528 &mut t,
7529 Verdict {
7530 status: RunStatus::Judging,
7531 left_pr: false,
7532 quota_hit: false,
7533 parked: true,
7534 no_viable_candidates: false,
7535 },
7536 "parked after `judging`",
7537 1,
7538 );
7539 assert_eq!(t.status, TaskStatus::Parked);
7540 assert_eq!(t.attempts, 0);
7541 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7542 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
7543 assert_eq!(t.schema, crate::queue::SCHEMA);
7544 assert!(
7545 crate::notices::task_held(&t).is_none(),
7546 "a park is not news"
7547 );
7548 t.start("20260904-000000-4043".to_owned());
7549 assert_eq!(t.park_reason, None);
7550 }
7551
7552 #[test]
7553 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
7554 let mut t = task();
7558 t.start("20260904-000000-4043".to_owned());
7559 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
7560 assert_eq!(t.status, TaskStatus::Failed);
7561 assert!(t.status.runnable());
7562 }
7563
7564 #[test]
7565 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
7566 let mut t = task();
7567 t.start("20260904-000000-4043".to_owned());
7568 assert_eq!(t.attempts, 1);
7569 let mut run = run_state(RunStatus::Implementing);
7570 run.id = "20260904-000000-4043".to_owned();
7571 run.begin_pass();
7572 let cause = crate::run::ParkCause::Upgrade {
7573 from: "0.121.0".to_owned(),
7574 to: Some("0.121.1".to_owned()),
7575 };
7576 run.end_pass_parked(cause.clone());
7577 run.parked = true;
7578 let verdict = Verdict {
7579 status: run.status,
7580 left_pr: false,
7581 quota_hit: false,
7582 parked: true,
7583 no_viable_candidates: false,
7584 };
7585 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
7586 assert_eq!(t.attempts, 0);
7587 assert_eq!(t.passes.len(), 1);
7588 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7589 assert_eq!(t.passes[0].parked, Some(cause));
7590 assert_eq!(t.passes[0].pass_index, 0);
7591
7592 t.start("20260904-000000-4043".to_owned());
7595 run.begin_pass();
7596 run.status = RunStatus::Failed;
7597 let verdict = Verdict {
7598 status: RunStatus::Failed,
7599 left_pr: false,
7600 quota_hit: false,
7601 parked: false,
7602 no_viable_candidates: false,
7603 };
7604 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
7605 assert_eq!(t.passes.len(), 2);
7606 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7607 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
7608 assert_eq!(t.passes[1].pass_index, 1);
7609 assert_eq!(t.attempts, 1);
7610 }
7611
7612 #[test]
7613 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
7614 let mut t = task();
7615 t.start("20260904-000000-4043".to_owned());
7616 reclaim(&mut t, None, 2, "en");
7617 assert_eq!(t.status, TaskStatus::Held);
7618 assert!(
7619 t.last_error
7620 .as_deref()
7621 .is_some_and(|e| e.contains("running")),
7622 "the operator needs to know why this task was held"
7623 );
7624 }
7625
7626 #[test]
7627 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
7628 let dir = tempfile::tempdir().unwrap();
7629 let queue = Queue::at(dir.path().to_path_buf());
7630
7631 let mut orphaned = task();
7633 orphaned.id = "20260904-000000-orph".to_owned();
7634 orphaned.status = TaskStatus::Running;
7635 orphaned.attempts = 1;
7636 queue.put(&mut orphaned).unwrap();
7637
7638 let mut alive = task();
7639 alive.id = "20260904-000000-live".to_owned();
7640 alive.status = TaskStatus::Running;
7641 alive.attempts = 1;
7642 queue.put(&mut alive).unwrap();
7643 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
7644
7645 let mut queued = task();
7646 queued.id = "20260904-000000-wait".to_owned();
7647 queue.put(&mut queued).unwrap();
7648
7649 let reclaimed = reclaim_orphaned_running(&queue, 2);
7650 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
7651
7652 assert_eq!(
7653 queue.get(&orphaned.id).unwrap().status,
7654 TaskStatus::Held,
7655 "nothing was driving it and there was no run to recover"
7656 );
7657 assert_eq!(
7658 queue.get(&alive.id).unwrap().status,
7659 TaskStatus::Running,
7660 "a live claim must protect the task it belongs to"
7661 );
7662 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
7663 }
7664
7665 fn read_run_under(home: &Path, id: &str) -> RunState {
7671 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
7672 serde_json::from_str(&body).unwrap()
7673 }
7674
7675 #[test]
7676 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
7677 let dir = tempfile::tempdir().unwrap();
7678 let home = dir.path().to_path_buf();
7679 let now = Timestamp::now();
7680 let overrun_seat = || crate::run::ActiveSeat {
7681 node: "implement".to_owned(),
7682 started_at: now - jiff::SignedDuration::new(21_000, 0),
7683 timeout_secs: 3_600,
7684 attempt: 0,
7685 task: None,
7686 command: None,
7687 index: None,
7688 total: None,
7689 };
7690
7691 let mut dead = run_state(RunStatus::Implementing);
7692 dead.id = "20260101-000000-dead".to_owned();
7693 dead.active.insert("impl-A".to_owned(), overrun_seat());
7694 dead.driver_pid = Some(4242);
7697 dead.save_under(&home).unwrap();
7698
7699 let mut alive = run_state(RunStatus::Implementing);
7702 alive.id = "20260101-000000-aliv".to_owned();
7703 alive.active.insert("impl-A".to_owned(), overrun_seat());
7704 alive.save_under(&home).unwrap();
7705 let mut status = Status::new();
7706 status.current = vec![Current {
7707 task: "20260101-000000-task".to_owned(),
7708 run: alive.id.clone(),
7709 }];
7710 write_status_to(&home.join("daemon.json"), &status).unwrap();
7711
7712 let questions = Questions::at(home.join("questions"));
7716 let mut q = ask::Question::new(
7717 dead.id.clone(),
7718 "implement".to_owned(),
7719 "impl-A".to_owned(),
7720 "Which storage backend?".to_owned(),
7721 String::new(),
7722 vec!["SQLite".to_owned(), "Redis".to_owned()],
7723 );
7724 questions.put(&mut q).unwrap();
7725
7726 let abandoned = reclaim_abandoned_runs_with(
7727 &home,
7728 now,
7729 |pid| if pid == 4242 { Some(false) } else { None },
7730 |_| panic!("a query answering Dead outright needs no identity corroboration"),
7731 );
7732 assert_eq!(abandoned, vec![dead.id.clone()]);
7733
7734 let reloaded = read_run_under(&home, &dead.id);
7735 assert_eq!(reloaded.status, RunStatus::Failed);
7736 assert!(reloaded.active.is_empty());
7737 assert!(
7738 !questions.get(&q.id).unwrap().status.open(),
7739 "the failed run's own open question must be settled in the same pass"
7740 );
7741
7742 let still_alive = read_run_under(&home, &alive.id);
7743 assert_eq!(
7744 still_alive.status,
7745 RunStatus::Implementing,
7746 "a live daemon's claim protects it"
7747 );
7748 assert!(!still_alive.active.is_empty());
7749 }
7750
7751 #[test]
7761 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
7762 let dir = tempfile::tempdir().unwrap();
7763 let home = dir.path().to_path_buf();
7764 let now = Timestamp::now();
7765
7766 let mut manual = run_state(RunStatus::Reviewing);
7767 manual.id = "20260101-000000-manl".to_owned();
7768 manual.active.insert(
7769 "review-1".to_owned(),
7770 crate::run::ActiveSeat {
7771 node: "review".to_owned(),
7772 started_at: now - jiff::SignedDuration::new(21_000, 0),
7773 timeout_secs: 3_600,
7774 attempt: 0,
7775 task: None,
7776 command: None,
7777 index: None,
7778 total: None,
7779 },
7780 );
7781 manual.driver_pid = Some(4242);
7785 manual.driver_started_at = Some("1790000000".to_owned());
7786 manual.save_under(&home).unwrap();
7787
7788 let abandoned = reclaim_abandoned_runs_with(
7789 &home,
7790 now,
7791 |pid| if pid == 4242 { Some(true) } else { None },
7792 |pid| {
7793 if pid == 4242 {
7794 Some("1790000000".to_owned())
7795 } else {
7796 None
7797 }
7798 },
7799 );
7800 assert!(
7801 abandoned.is_empty(),
7802 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
7803 );
7804
7805 let reloaded = read_run_under(&home, &manual.id);
7806 assert_eq!(reloaded.status, RunStatus::Reviewing);
7807 assert!(!reloaded.active.is_empty());
7808 }
7809
7810 #[test]
7811 fn an_already_claimed_task_is_skipped_rather_than_failed() {
7812 let dir = tempfile::tempdir().unwrap();
7813 let queue = Queue::at(dir.path().to_path_buf());
7814 let mut only = task();
7815 queue.put(&mut only).unwrap();
7816
7817 let _elsewhere = queue.claim(&only.id).unwrap();
7818 let candidates = runnable(&queue);
7819 assert_eq!(candidates.len(), 1, "the task is still runnable");
7820 assert!(
7821 queue.claim(&candidates[0].id).is_err(),
7822 "the loop cannot take a claim somebody else holds"
7823 );
7824
7825 let after = queue.get(&only.id).unwrap();
7826 assert_eq!(after.status, TaskStatus::Queued);
7827 assert_eq!(
7828 after.attempts, 0,
7829 "losing the race is not an attempt at the task"
7830 );
7831 assert_eq!(after.last_error, None);
7832 }
7833
7834 #[test]
7835 fn the_status_file_round_trips_and_its_heartbeat_advances() {
7836 let dir = tempfile::tempdir().unwrap();
7837 let path = dir.path().join("daemon.json");
7838
7839 let mut status = Status::new();
7840 status.idle = false;
7841 status.completed = 7;
7842 status.current = vec![Current {
7843 task: "20260902-000000-t111".to_owned(),
7844 run: "20260902-000001-r111".to_owned(),
7845 }];
7846 write_status_to(&path, &status).unwrap();
7847 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7848 assert_eq!(first.schema, SCHEMA);
7849 assert_eq!(first.pid, std::process::id());
7850 assert!(!first.idle);
7851 assert_eq!(first.completed, 7);
7852 assert_eq!(first.current, status.current);
7853 assert!(
7854 !path.with_extension("json.tmp").exists(),
7855 "the temp file is renamed, not left behind"
7856 );
7857
7858 std::thread::sleep(Duration::from_millis(5));
7859 status.updated_at = Timestamp::now();
7860 status.polls = 3;
7861 write_status_to(&path, &status).unwrap();
7862 let second: Status =
7863 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7864 assert!(
7865 second.updated_at > first.updated_at,
7866 "a reader can only detect staleness if the heartbeat moves"
7867 );
7868 assert_eq!(
7869 second.started_at, first.started_at,
7870 "the start time is not a heartbeat"
7871 );
7872 assert_eq!(second.polls, 3);
7873 }
7874
7875 #[test]
7876 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
7877 let dir = tempfile::tempdir().unwrap();
7878
7879 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
7880
7881 let mut status = Status::new();
7882 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
7883 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7884 let stale = read_status(dir.path()).unwrap();
7885 assert!(
7886 !stale.running(Timestamp::now()),
7887 "a minute without a heartbeat is a dead daemon, not a busy one"
7888 );
7889 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
7890
7891 status.updated_at = Timestamp::now();
7892 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7893 let fresh = read_status(dir.path()).unwrap();
7894 assert!(fresh.running(Timestamp::now()));
7895 }
7896
7897 #[test]
7898 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
7899 let dir = tempfile::tempdir().unwrap();
7900 let now = Timestamp::now();
7901 let mine = "20260903-080619-01c2";
7902
7903 assert!(
7904 !is_working_on(dir.path(), mine, now),
7905 "no status file means nobody is working on anything"
7906 );
7907
7908 let mut status = Status::new();
7909 status.current = vec![Current {
7910 task: "20260903-080340-0167".to_owned(),
7911 run: mine.to_owned(),
7912 }];
7913 status.updated_at = now;
7914 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7915 assert!(is_working_on(dir.path(), mine, now));
7916 assert!(
7917 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
7918 "a daemon busy with one run is not working on another"
7919 );
7920
7921 status.updated_at = now - jiff::SignedDuration::from_secs(600);
7924 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7925 assert!(
7926 !is_working_on(dir.path(), mine, now),
7927 "a stale heartbeat is a dead daemon, so its run is a leftover"
7928 );
7929 }
7930
7931 #[test]
7932 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
7933 let dir = tempfile::tempdir().unwrap();
7934 let now = Timestamp::now();
7935
7936 assert!(
7937 !is_working_on_short(dir.path(), "01c2", now),
7938 "no status file means nobody is working on anything"
7939 );
7940
7941 let mut status = Status::new();
7942 status.current = vec![Current {
7943 task: "20260903-080340-0167".to_owned(),
7944 run: "20260903-080619-01c2".to_owned(),
7945 }];
7946 status.updated_at = now;
7947 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7948 assert!(
7949 is_working_on_short(dir.path(), "01c2", now),
7950 "the run's short id is the last block of its full id"
7951 );
7952 assert!(
7953 !is_working_on_short(dir.path(), "3cbf", now),
7954 "a daemon busy with one worktree bay is not working on another"
7955 );
7956 }
7957
7958 #[test]
7959 fn a_newer_status_file_still_yields_a_reading() {
7960 let dir = tempfile::tempdir().unwrap();
7961 std::fs::write(
7964 dir.path().join("daemon.json"),
7965 serde_json::json!({
7966 "schema": 2,
7967 "updated_at": Timestamp::now().to_string(),
7968 "idle": true,
7969 "surprise": { "nested": [1, 2, 3] },
7970 })
7971 .to_string(),
7972 )
7973 .unwrap();
7974
7975 let reading = read_status(dir.path()).expect("a forward-compatible read");
7976 assert!(reading.running(Timestamp::now()));
7977 assert!(reading.idle);
7978 assert!(reading.current.is_empty());
7979 }
7980
7981 #[test]
7982 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
7983 let dir = tempfile::tempdir().unwrap();
7989 std::fs::write(
7990 dir.path().join("daemon.json"),
7991 serde_json::json!({
7992 "schema": 1,
7993 "pid": 4242,
7994 "updated_at": Timestamp::now().to_string(),
7995 "idle": false,
7996 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
7997 "completed": 3,
7998 "polls": 9,
7999 })
8000 .to_string(),
8001 )
8002 .unwrap();
8003
8004 let reading = read_status(dir.path()).expect("an older shape must still parse");
8005 assert!(reading.running(Timestamp::now()));
8006 assert_eq!(
8007 reading.current,
8008 vec![Current {
8009 task: "20260902-140501-aaaa".to_owned(),
8010 run: "20260902-140502-bbbb".to_owned(),
8011 }]
8012 );
8013 }
8014
8015 #[test]
8016 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
8017 let dir = tempfile::tempdir().unwrap();
8018 std::fs::write(
8019 dir.path().join("daemon.json"),
8020 serde_json::json!({
8021 "schema": 1,
8022 "updated_at": Timestamp::now().to_string(),
8023 "idle": true,
8024 "current": null,
8025 })
8026 .to_string(),
8027 )
8028 .unwrap();
8029 let with_null = read_status(dir.path()).expect("null must still parse");
8030 assert!(with_null.current.is_empty());
8031
8032 std::fs::write(
8033 dir.path().join("daemon.json"),
8034 serde_json::json!({
8035 "schema": 1,
8036 "updated_at": Timestamp::now().to_string(),
8037 "idle": true,
8038 })
8039 .to_string(),
8040 )
8041 .unwrap();
8042 let absent = read_status(dir.path()).expect("a missing field must still parse");
8043 assert!(absent.current.is_empty());
8044 }
8045
8046 #[test]
8047 fn a_task_without_a_repository_runs_in_the_daemons_default() {
8048 let fallback = Path::new("/default");
8049 let mut blank = task();
8050 blank.repo = PathBuf::new();
8051 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
8052 let mut dot = task();
8053 dot.repo = PathBuf::from(".");
8054 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
8055 assert_eq!(
8056 repo_for(&task(), fallback),
8057 PathBuf::from("/repo"),
8058 "a task that names a repository keeps it"
8059 );
8060 }
8061
8062 #[test]
8063 fn task_implementers_edit_inherits_daemon_config_and_merge() {
8064 let dir = tempfile::tempdir().unwrap();
8065 let path = dir.path().join("explicit.toml");
8066 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
8067 let opts = Opts {
8068 config: Some(path),
8069 merge: Some("none".into()),
8070 ..Default::default()
8071 };
8072 let mut t = task();
8073 t.set_implementers(
8074 2,
8075 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8076 .unwrap(),
8077 )
8078 .unwrap();
8079 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
8080 assert_eq!(cfg.graph.implementers, 2);
8081 assert_eq!(cfg.graph.judges, 1);
8082 assert_eq!(cfg.merge.mode, MergeMode::None);
8083 t.set_implementers(1, &cfg).unwrap();
8084 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
8085 apply_task_implementers(&mut cfg, &t);
8086 assert_eq!(cfg.graph.implementers, 1);
8087 assert_eq!(cfg.graph.judges, 1);
8088 assert_eq!(cfg.merge.mode, MergeMode::None);
8089 }
8090
8091 #[test]
8092 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
8093 let dir = tempfile::tempdir().unwrap();
8094 let repo = dir.path().join("repo");
8095 std::fs::create_dir(&repo).unwrap();
8096 std::fs::write(
8097 repo.join("magi.toml"),
8098 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
8099 )
8100 .unwrap();
8101 let daemon_config = dir.path().join("daemon.toml");
8102 std::fs::write(
8103 &daemon_config,
8104 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
8105 )
8106 .unwrap();
8107 let opts = Opts {
8108 config: Some(daemon_config),
8109 merge: Some("pr".into()),
8110 ..Default::default()
8111 };
8112 let q = Queue::at(dir.path().join("queue"));
8113 let validation: Config =
8114 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8115 .unwrap();
8116 for record in ["{}", r#"{"candidates":2}"#] {
8118 let overrides = serde_json::from_str(record).unwrap();
8119 let filing = crate::direct::Filing {
8120 instruction: "work".into(),
8121 title: "direct".into(),
8122 repo: repo.clone(),
8123 source: crate::queue::Source::Human,
8124 solo: false,
8125 overrides,
8126 review_of: None,
8127 };
8128 let filed =
8129 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
8130 let crate::direct::Filed::Standalone {
8131 mut task,
8132 claim: _claim,
8133 } = filed
8134 else {
8135 panic!("local filing")
8136 };
8137 let before = prepare_for(&repo, &opts, &task).unwrap();
8138 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
8139 assert_eq!(
8140 before.merge.mode,
8141 MergeMode::None,
8142 "daemon --merge must not leak"
8143 );
8144 for n in [2, 1, 2] {
8145 task.set_implementers(n, &validation).unwrap();
8146 q.put(&mut task).unwrap();
8147 task = q.get(&task.id).unwrap();
8148 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
8149 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
8150 apply_task_implementers(&mut cfg, &task);
8151 assert_eq!(cfg.graph.implementers, usize::from(n));
8152 assert_eq!(cfg.graph.judges, 2);
8153 assert_eq!(cfg.merge.mode, MergeMode::None);
8154 }
8155 }
8156 }
8157
8158 #[test]
8159 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
8160 let cfg: Config =
8161 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8162 .unwrap();
8163 let mut t = task();
8164 t.review_of = Some("review-target".into());
8165 t.hold_manual(Some("retry".into()));
8166 let mut previous = RunState::new(
8167 PathBuf::from("/repo"),
8168 "main".into(),
8169 "abc1234".into(),
8170 "review".into(),
8171 cfg.clone(),
8172 );
8173 previous.config.graph.implementers = 2;
8174 for count in [1, 3] {
8175 t.set_implementers(count, &cfg).unwrap();
8176 t.release();
8177 assert_eq!(t.review_of.as_deref(), Some("review-target"));
8178 for proposed in [
8179 Starter::Resume(previous.id.clone()),
8180 Starter::Review("recovery-target".into()),
8181 ] {
8182 assert_eq!(
8183 starter_with_implementers(&t, Some(&previous), proposed),
8184 Starter::Review("review-target".into())
8185 );
8186 }
8187 let mut configured = cfg.clone();
8188 apply_task_implementers(&mut configured, &t);
8189 assert_eq!(configured.graph.implementers, usize::from(count));
8190 assert_eq!(previous.config.graph.implementers, 2);
8191 }
8192 }
8193
8194 #[test]
8195 fn task_implementers_dispatch_and_resume_agree() {
8196 let mut t = task();
8197 let cfg: crate::config::Config =
8198 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8199 .unwrap();
8200 let mut run = RunState::new(
8201 PathBuf::from("/repo"),
8202 "main".into(),
8203 "abc1234".into(),
8204 "work".into(),
8205 cfg.clone(),
8206 );
8207 run.config.graph.implementers = 2;
8208 for n in [2, 1, 3, 2] {
8209 t.set_implementers(n, &cfg).unwrap();
8210 let mut dispatched = cfg.clone();
8211 apply_task_implementers(&mut dispatched, &t);
8212 assert_eq!(dispatched.graph.implementers, usize::from(n));
8213 assert_eq!(implementers_match(&t, &run), n == 2);
8214 for proposed in [
8215 Starter::Resume(run.id.clone()),
8216 Starter::Review("old-branch".into()),
8217 ] {
8218 let expected = if n == 2 {
8219 proposed.clone()
8220 } else {
8221 Starter::Start
8222 };
8223 assert_eq!(
8224 starter_with_implementers(&t, Some(&run), proposed),
8225 expected
8226 );
8227 }
8228 assert_eq!(
8229 run.config.graph.implementers, 2,
8230 "history must not be rewritten"
8231 );
8232 }
8233 t.solo = false;
8234 t.overrides = None;
8235 assert!(implementers_match(&t, &run));
8236 }
8237
8238 #[test]
8239 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
8240 let mut solo_cfg = Config::default();
8246 solo_cfg.graph.implementers = 3;
8247 let mut solo_task = task();
8248 solo_task.solo = true;
8249 apply_task_implementers(&mut solo_cfg, &solo_task);
8250 assert_eq!(solo_cfg.graph.implementers, 1);
8251
8252 let mut plain_cfg = Config::default();
8253 plain_cfg.graph.implementers = 3;
8254 let plain_task = task();
8255 assert!(!plain_task.solo);
8256 apply_task_implementers(&mut plain_cfg, &plain_task);
8257 assert_eq!(
8258 plain_cfg.graph.implementers, 3,
8259 "a task that did not ask to run alone keeps the config's candidates"
8260 );
8261 }
8262
8263 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
8264 QuotaLoss {
8265 seat: seat.into(),
8266 node: "judge".into(),
8267 at: at.parse().unwrap(),
8268 reset: reset.map(str::to_string),
8269 }
8270 }
8271
8272 #[test]
8273 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
8274 let old: Vec<QuotaLoss> = (1..=4)
8275 .map(|i| {
8276 loss(
8277 &format!("judge-{i}"),
8278 "2026-09-23T05:23:00Z",
8279 Some("2:40pm (Asia/Tokyo)"),
8280 )
8281 })
8282 .collect();
8283 let fresh = losses_this_attempt(&old, &old);
8284 assert!(fresh.is_empty());
8285 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
8286 }
8288
8289 #[test]
8290 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
8291 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
8292 let now = Timestamp::now();
8293 let mut after = old.clone();
8294 after.push(loss("judge-2", &now.to_string(), None));
8295 let fresh = losses_this_attempt(&old, &after);
8296 assert_eq!(fresh, vec![after[1].clone()]);
8297 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
8298 assert_eq!(
8299 until,
8300 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
8301 );
8302 }
8303
8304 #[test]
8305 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
8306 let before = vec![
8309 loss("judge-1", "2026-09-23T05:23:00Z", None),
8310 loss("judge-2", "2026-09-23T05:24:00Z", None),
8311 ];
8312 let after = vec![
8313 loss("judge-2", "2026-09-23T05:24:00Z", None),
8314 loss("judge-1", "2026-09-24T01:00:00Z", None),
8315 ];
8316 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
8317 }
8318
8319 #[test]
8320 fn merge_overrides_are_parsed_or_refused() {
8321 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
8322 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
8323 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
8324 assert!(merge_mode("squash").is_err());
8325 }
8326
8327 #[test]
8328 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
8329 let now = Timestamp::now();
8330 let fallback = Duration::from_secs(300);
8331 let cap = Duration::from_secs(1800);
8332
8333 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
8335
8336 let soon = now + jiff::SignedDuration::from_secs(600);
8338 assert_eq!(
8339 quota_wait(Some(soon), now, fallback, cap),
8340 Duration::from_secs(600)
8341 );
8342
8343 let past = now - jiff::SignedDuration::from_secs(60);
8346 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
8347
8348 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
8351 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
8352 }
8353
8354 #[test]
8355 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
8356 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8357
8358 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
8359 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
8360
8361 let already_past =
8365 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
8366 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
8367
8368 assert!(
8369 parse_reset_hint("session limit reached", now, now).is_none(),
8370 "free text with no recognised shape is not guessed at"
8371 );
8372 assert!(
8373 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
8374 "an unresolvable zone name is not guessed at either"
8375 );
8376 }
8377
8378 #[test]
8379 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
8380 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8381
8382 let at = parse_reset_hint(
8383 "You've hit your usage limit. Visit \
8384 https://chatgpt.com/codex/settings/usage to purchase more \
8385 credits or try again at Sep 19th, 2026 5:10 PM.",
8386 now,
8387 now,
8388 )
8389 .expect("the codex reset wording is a recognised shape");
8390 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
8391
8392 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
8397 .expect("an explicit year needs no rollover");
8398 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
8399
8400 assert!(
8401 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
8402 "a two-digit year is not the documented shape and is not guessed at"
8403 );
8404 assert!(
8405 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
8406 "a four-letter month name is not the documented three-letter abbreviation"
8407 );
8408 assert!(
8409 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
8410 "an explicit zone on the dated shape is a format nobody has \
8411 documented, and is refused rather than guessed at as UTC"
8412 );
8413 }
8414
8415 #[test]
8416 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
8417 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
8418 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
8419
8420 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
8421 assert_eq!(at.as_second() - recorded.as_second(), 3769);
8422
8423 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
8424 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
8425
8426 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
8427 assert!(
8428 parse_reset_hint(bad, now, recorded).is_none(),
8429 "{bad:?} must not be guessed at"
8430 );
8431 }
8432 }
8433
8434 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
8438 let config = dir.join("magi.toml");
8439 std::fs::write(
8440 &config,
8441 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
8442 )
8443 .unwrap();
8444 let opts = Opts {
8445 poll: Duration::from_secs(30),
8446 config: Some(config),
8447 repo: dir.join("repo"),
8451 ..Opts::default()
8452 };
8453 let home = dir.join("home");
8462 let worktrees = dir.join("wt");
8463 (
8464 opts,
8465 Queue::at(dir.join("queue")),
8466 home.join("daemon.json"),
8467 home,
8468 worktrees,
8469 )
8470 }
8471
8472 #[test]
8473 fn a_stop_is_idempotent_and_once_set_stays_set() {
8474 let stop = Stop::new();
8475 assert!(!stop.stopped());
8476
8477 stop.stop();
8478 assert!(stop.stopped());
8479 stop.stop();
8480 assert!(stop.stopped(), "a second stop is not a toggle");
8481
8482 let shared = stop.clone();
8483 assert!(
8484 shared.stopped(),
8485 "a clone is the same stop; that is how the loop and its caller share one"
8486 );
8487 }
8488
8489 #[test]
8490 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
8491 let stop = Stop::new();
8492 stop.enter();
8493 assert!(
8494 !stop.finishing(),
8495 "a busy loop nobody has asked to stop is just running"
8496 );
8497
8498 stop.stop();
8499 assert!(
8500 stop.finishing(),
8501 "a stop asked for mid-run has not landed until the run is settled"
8502 );
8503
8504 stop.exit();
8505 assert!(
8506 !stop.finishing(),
8507 "once the run is settled the stop has landed and there is nothing to finish"
8508 );
8509 }
8510
8511 #[test]
8512 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
8513 let stop = Stop::new();
8514 stop.enter();
8515 stop.enter();
8516 stop.stop();
8517 assert!(stop.finishing(), "two runs still in flight");
8518
8519 stop.exit();
8520 assert!(
8521 stop.finishing(),
8522 "one run finished, but a sibling is still working"
8523 );
8524
8525 stop.exit();
8526 assert!(
8527 !stop.finishing(),
8528 "the last run out is what actually lands the stop"
8529 );
8530 }
8531
8532 #[tokio::test]
8533 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
8534 let dir = tempfile::tempdir().unwrap();
8535 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8536 let stop = Stop::new();
8537 stop.stop();
8538
8539 let began = std::time::Instant::now();
8540 tokio::time::timeout(
8541 Duration::from_secs(2),
8542 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8543 )
8544 .await
8545 .expect("a stopped loop must return, not sit out its poll interval")
8546 .expect("the loop's own setup and teardown must not fail");
8547 assert!(
8548 began.elapsed() < opts.poll,
8549 "returned only after {:?}, which is a poll interval, not a stop",
8550 began.elapsed()
8551 );
8552 }
8553
8554 #[tokio::test]
8555 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
8556 let dir = tempfile::tempdir().unwrap();
8557 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8558 let stop = Stop::new();
8559
8560 let asker = {
8563 let stop = stop.clone();
8564 tokio::spawn(async move {
8565 tokio::time::sleep(Duration::from_millis(20)).await;
8566 stop.stop();
8567 })
8568 };
8569
8570 let began = std::time::Instant::now();
8571 tokio::time::timeout(
8572 Duration::from_secs(2),
8573 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8574 )
8575 .await
8576 .expect("a stop asked for while idle must wake the wait")
8577 .expect("the loop's own setup and teardown must not fail");
8578 asker.await.unwrap();
8579 assert!(
8580 began.elapsed() < opts.poll,
8581 "returned only after {:?}, so the stop waited on the sleep",
8582 began.elapsed()
8583 );
8584 }
8585
8586 #[tokio::test]
8587 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
8588 let dir = tempfile::tempdir().unwrap();
8589 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8590 let stop = Stop::new();
8591 stop.stop();
8592
8593 tokio::time::timeout(
8594 Duration::from_secs(2),
8595 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8596 )
8597 .await
8598 .expect("a stopped loop must return")
8599 .expect("the loop's own setup and teardown must not fail");
8600
8601 assert!(
8602 home.is_dir(),
8603 "the loop did publish a status file, so its removal is the teardown and not an absence"
8604 );
8605 assert!(
8606 !status_file.exists(),
8607 "a stopped loop clears its status file"
8608 );
8609 assert!(
8610 read_status(&home).is_none(),
8611 "a reader must see no daemon at all, not a heartbeat that merely stopped"
8612 );
8613 }
8614
8615 #[tokio::test]
8616 async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
8617 let dir = tempfile::tempdir().unwrap();
8618 let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8619 opts.once = true;
8620
8621 let mut settled = RunState::new(
8622 dir.path().join("repo"),
8623 "main".to_owned(),
8624 "abc1234".to_owned(),
8625 "fixture".to_owned(),
8626 Config::default(),
8627 );
8628 settled.status = RunStatus::Ready;
8629 let run_dir = home.join("runs").join(&settled.id);
8630 std::fs::create_dir_all(&run_dir).unwrap();
8631 std::fs::write(
8632 run_dir.join("run.json"),
8633 serde_json::to_string_pretty(&settled).unwrap(),
8634 )
8635 .unwrap();
8636 let questions = Questions::at(home.join("questions"));
8637 let mut question = ask::Question::new(
8638 settled.id.clone(),
8639 "review".to_owned(),
8640 "reviewer-1".to_owned(),
8641 "Continue?".to_owned(),
8642 String::new(),
8643 Vec::new(),
8644 );
8645 questions.put(&mut question).unwrap();
8646
8647 drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
8648 .await
8649 .unwrap();
8650
8651 assert_eq!(
8652 questions.get(&question.id).unwrap().status,
8653 ask::QuestionStatus::Abandoned,
8654 "an empty --once drain still performs startup question cleanup"
8655 );
8656 }
8657
8658 #[test]
8659 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
8660 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8661
8662 assert!(
8663 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
8664 "never checked before: due at once"
8665 );
8666
8667 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
8668 assert!(
8669 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
8670 "well inside the interval: not due yet"
8671 );
8672
8673 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
8674 assert!(
8675 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
8676 "exactly at the edge: not yet due, same convention as `clean::due`"
8677 );
8678
8679 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8680 assert!(
8681 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
8682 "past the interval: due again"
8683 );
8684 }
8685
8686 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
8691 let config = dir.join("magi.toml");
8692 std::fs::write(
8698 &config,
8699 format!(
8700 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
8701 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
8702 cache_dir.display()
8703 ),
8704 )
8705 .unwrap();
8706 Opts {
8707 config: Some(config),
8708 repo: dir.join("repo"),
8709 ..Opts::default()
8710 }
8711 }
8712
8713 #[tokio::test]
8714 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
8715 let dir = tempfile::tempdir().unwrap();
8716 let home = dir.path().join("home");
8717 let cache_dir = dir.path().join("cache");
8718 std::fs::create_dir_all(&cache_dir).unwrap();
8719 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8720 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8721
8722 let running = Stop::new();
8725 let mut last_checked = None;
8726 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8727 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
8728 .await;
8729 assert_eq!(
8730 crate::disk::dir_size(&cache_dir),
8731 0,
8732 "over the cap on the first check ever: pruned at once, no idle queue required"
8733 );
8734 assert_eq!(last_checked, Some(t0));
8735
8736 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
8738 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
8739 maybe_prune_cache_between_runs(
8740 &opts.repo,
8741 &opts,
8742 &home,
8743 &running,
8744 &mut last_checked,
8745 too_soon,
8746 )
8747 .await;
8748 assert_eq!(
8749 crate::disk::dir_size(&cache_dir),
8750 10,
8751 "too soon since the last check: left alone rather than rescanned every call"
8752 );
8753 assert_eq!(
8754 last_checked,
8755 Some(t0),
8756 "an idle check does not reset the clock"
8757 );
8758
8759 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8761 maybe_prune_cache_between_runs(
8762 &opts.repo,
8763 &opts,
8764 &home,
8765 &running,
8766 &mut last_checked,
8767 due_again,
8768 )
8769 .await;
8770 assert_eq!(
8771 crate::disk::dir_size(&cache_dir),
8772 0,
8773 "due again: pruned back under the cap"
8774 );
8775 }
8776
8777 #[tokio::test]
8785 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
8786 let dir = tempfile::tempdir().unwrap();
8787 let home = dir.path().join("home");
8788 let cache_dir = dir.path().join("cache");
8789 std::fs::create_dir_all(&cache_dir).unwrap();
8790 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8791 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8792
8793 let stop = Stop::new();
8794 stop.stop();
8795 assert!(
8796 !stop.finishing(),
8797 "no run is in flight at a between-runs boundary, so nothing else \
8798 would tell the operator this stop had not taken effect yet"
8799 );
8800
8801 let mut last_checked = None;
8802 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8803 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
8804 .await;
8805 assert_eq!(
8806 crate::disk::dir_size(&cache_dir),
8807 10,
8808 "over its cap, and due for the first check ever, but a stop outranks \
8809 it: the cap is a standing policy the next start measures again"
8810 );
8811 assert_eq!(
8812 last_checked, None,
8813 "a check that never happened must not claim the interval"
8814 );
8815 }
8816
8817 #[tokio::test]
8828 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
8829 let dir = tempfile::tempdir().unwrap();
8830 let cache_dir = dir.path().join("cache");
8831 std::fs::create_dir_all(&cache_dir).unwrap();
8832 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
8833
8834 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
8835 opts.poll = Duration::from_millis(20);
8836 opts.max_attempts = 1_000;
8837
8838 let queue = Queue::at(dir.path().join("queue"));
8839 for i in 0..128 {
8840 let mut task = Task::new(
8841 format!("queued work {i}"),
8842 format!("queued work {i}"),
8843 opts.repo.clone(),
8844 Source::Human,
8845 );
8846 queue.put(&mut task).unwrap();
8847 }
8848
8849 let home = dir.path().join("home");
8850 let worktrees = dir.path().join("wt");
8851 let status_file = home.join("daemon.json");
8852 let stop = Stop::new();
8853 let stopper = {
8854 let stop = stop.clone();
8855 let queue = Queue::at(dir.path().join("queue"));
8856 tokio::spawn(async move {
8857 for _ in 0..2_000 {
8861 if queue.list().iter().filter(|t| t.attempts == 1).count() >= 2 {
8862 break;
8863 }
8864 tokio::time::sleep(Duration::from_millis(5)).await;
8865 }
8866 stop.stop();
8867 })
8868 };
8869
8870 tokio::time::timeout(
8871 Duration::from_secs(20),
8872 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8873 )
8874 .await
8875 .expect("the loop must not hang on a queue that keeps producing failing work")
8876 .expect("the loop's own setup and teardown must not fail");
8877 stopper.await.unwrap();
8878
8879 let after = queue.list();
8880 let completed = after.iter().filter(|t| t.attempts == 1).count();
8881 assert!(
8882 completed >= 2,
8883 "the harness must have processed multiple tasks to exercise a busy queue \
8884 (got {completed} completed task(s))"
8885 );
8886 assert!(
8887 after.iter().any(|t| t.status.runnable()),
8888 "unprocessed tasks must remain: the external stop, rather than an \
8889 exhausted backlog, must end the test"
8890 );
8891 assert!(
8892 after
8893 .iter()
8894 .filter(|t| t.attempts > 0)
8895 .all(|t| { t.attempts == 1 && t.status == TaskStatus::Held }),
8896 "a busy backlog must not retry a task's unchanged startup failure"
8897 );
8898
8899 assert_eq!(
8900 crate::disk::dir_size(&cache_dir),
8901 0,
8902 "an oversized cache must not be left to grow unboundedly just because the \
8903 queue kept the loop busy the whole time"
8904 );
8905 }
8906
8907 #[test]
8908 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
8909 let dir = tempfile::tempdir().unwrap();
8910 let queue = Queue::at(dir.path().join("queue"));
8911 let questions = Questions::at(dir.path().join("questions"));
8912 let mut task = task();
8913 queue.put(&mut task).unwrap();
8914
8915 let mut task_question = ask::Question::new(
8916 task.id.clone(),
8917 crate::conduct::NODE.to_owned(),
8918 "conduct".to_owned(),
8919 "Which backend?".to_owned(),
8920 String::new(),
8921 Vec::new(),
8922 );
8923 questions.put(&mut task_question).unwrap();
8924 task.block(vec![task_question.id.clone()], None);
8925 queue.put(&mut task).unwrap();
8926
8927 let mut run_question = ask::Question::new(
8928 "20260101-000000-run1".to_owned(),
8929 "review".to_owned(),
8930 "reviewer-1".to_owned(),
8931 "Run question".to_owned(),
8932 String::new(),
8933 Vec::new(),
8934 );
8935 questions.put(&mut run_question).unwrap();
8936
8937 let mut coincidental = ask::Question::new(
8942 task.id.clone(),
8943 "review".to_owned(),
8944 "reviewer-1".to_owned(),
8945 "Unrelated review question".to_owned(),
8946 String::new(),
8947 Vec::new(),
8948 );
8949 questions.put(&mut coincidental).unwrap();
8950
8951 reconcile_task_questions(&queue, &questions);
8952 assert!(questions.get(&task_question.id).unwrap().status.open());
8953 assert!(questions.get(&run_question.id).unwrap().status.open());
8954 assert!(questions.get(&coincidental.id).unwrap().status.open());
8955
8956 task.release();
8957 queue.put(&mut task).unwrap();
8958 reconcile_task_questions(&queue, &questions);
8959 assert_eq!(
8960 questions.get(&task_question.id).unwrap().status,
8961 ask::QuestionStatus::Abandoned
8962 );
8963 assert!(
8964 questions.get(&run_question.id).unwrap().status.open(),
8965 "run questions remain the run janitor's responsibility"
8966 );
8967 assert!(
8968 questions.get(&coincidental.id).unwrap().status.open(),
8969 "a non-conductor question must not be abandoned just because its \
8970 run id coincides with a task id"
8971 );
8972 }
8973
8974 #[test]
8975 fn a_freshly_started_running_task_is_never_stalled() {
8976 let dir = tempfile::tempdir().unwrap();
8977 let mut t = task();
8978 t.start("run-1".to_owned());
8979 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
8982 }
8983
8984 #[test]
8985 fn a_long_running_task_with_no_live_daemon_is_stalled() {
8986 let dir = tempfile::tempdir().unwrap();
8987 let mut t = task();
8988 t.start("run-1".to_owned());
8989 t.updated_at = Timestamp::now()
8990 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8991 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
8992 assert_eq!(
8993 stalled_tasks(
8994 &Queue::at(dir.path().join("q")),
8995 dir.path(),
8996 Timestamp::now()
8997 )
8998 .len(),
8999 0,
9000 "the task was never written to this queue"
9001 );
9002 }
9003
9004 #[test]
9005 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
9006 let dir = tempfile::tempdir().unwrap();
9007 let mut t = task();
9008 t.id = "20260903-080340-0167".to_owned();
9009 t.start("20260903-080619-01c2".to_owned());
9010 t.updated_at = Timestamp::now()
9011 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
9012
9013 let mut status = Status::new();
9014 status.current = vec![Current {
9015 task: t.id.clone(),
9016 run: "20260903-080619-01c2".to_owned(),
9017 }];
9018 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9019
9020 assert!(
9021 !is_stalled(&t, dir.path(), Timestamp::now()),
9022 "a live daemon's own heartbeat rules out stalled, however long the task has run"
9023 );
9024 }
9025
9026 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
9030 let path = queue.path_of(id);
9031 let body = std::fs::read_to_string(&path).unwrap();
9032 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
9033 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
9034 v["updated_at"] = serde_json::Value::String(old.to_string());
9035 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
9036 }
9037
9038 #[test]
9039 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
9040 let dir = tempfile::tempdir().unwrap();
9053 let queue = Queue::at(dir.path().join("queue"));
9054 let home = dir.path().join("home");
9055
9056 let mut t = task();
9057 t.id = "20260101-000001-lock".to_owned();
9058 t.start("run-1".to_owned());
9059 queue.put(&mut t).unwrap();
9060 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
9061 std::fs::write(
9062 dir.path().join("queue").join(format!("{}.lock", t.id)),
9063 "not a pid",
9064 )
9065 .unwrap();
9066
9067 let now = Timestamp::now();
9068 assert!(
9069 reclaim_orphaned_running(&queue, 2).is_empty(),
9070 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
9071 and reclaim must leave the task alone"
9072 );
9073 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
9074
9075 let stalled = stalled_tasks(&queue, &home, now);
9076 assert_eq!(
9077 stalled.len(),
9078 1,
9079 "reclaim's inability to claim it yet must not hide it from the conductor"
9080 );
9081 assert_eq!(stalled[0].id, t.id);
9082 }
9083
9084 #[test]
9085 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
9086 let dir = tempfile::tempdir().unwrap();
9087 crate::run::set_home(dir.path().join("run-home"));
9088 let queue = Queue::at(dir.path().join("queue"));
9089 let home = dir.path().join("home");
9090 let questions = Questions::at(dir.path().join("questions"));
9091
9092 let mut t = task();
9093 t.id = "20260101-000003-dead".to_owned();
9094 t.start("missing-run".to_owned());
9095 queue.put(&mut t).unwrap();
9096 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
9097
9098 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
9101 assert_eq!(
9102 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
9103 [&t.id]
9104 );
9105 assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
9106 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
9107
9108 crate::conduct::apply(
9111 &queue,
9112 &questions,
9113 &crate::conduct::Verdict {
9114 decisions: vec![crate::conduct::Decision {
9115 id: t.id.clone(),
9116 recovery: Some(crate::conduct::Recovery::Requeue),
9117 ..crate::conduct::Decision::default()
9118 }],
9119 },
9120 )
9121 .unwrap();
9122 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
9123 }
9124
9125 #[test]
9126 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
9127 let dir = tempfile::tempdir().unwrap();
9128 let queue = Queue::at(dir.path().join("queue"));
9129 let home = dir.path().join("home");
9130
9131 let mut fresh = task();
9132 fresh.id = "20260101-000001-aaaa".to_owned();
9133 fresh.start("run-1".to_owned());
9134 queue.put(&mut fresh).unwrap();
9135
9136 let mut old = task();
9137 old.id = "20260101-000002-bbbb".to_owned();
9138 old.start("run-2".to_owned());
9139 queue.put(&mut old).unwrap();
9140 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
9141
9142 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
9143 assert_eq!(stalled.len(), 1);
9144 assert_eq!(stalled[0].id, old.id);
9145 }
9146
9147 #[test]
9148 fn queued_and_finished_task_views_partition_by_status() {
9149 let dir = tempfile::tempdir().unwrap();
9150 let queue = Queue::at(dir.path().join("queue"));
9151
9152 let mut queued = task();
9153 queued.id = "20260101-000001-aaaa".to_owned();
9154 queue.put(&mut queued).unwrap();
9155
9156 let mut failed = task();
9157 failed.id = "20260101-000002-bbbb".to_owned();
9158 failed.start("run-1".to_owned());
9159 failed.fail("gate red", 5);
9160 queue.put(&mut failed).unwrap();
9161
9162 let mut held = task();
9163 held.id = "20260101-000003-cccc".to_owned();
9164 held.hold_machine(None);
9165 queue.put(&mut held).unwrap();
9166
9167 let mut running = task();
9168 running.id = "20260101-000004-dddd".to_owned();
9169 running.start("run-2".to_owned());
9170 queue.put(&mut running).unwrap();
9171
9172 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
9173 assert_eq!(queued_ids, [queued.id.clone()]);
9174
9175 let mut finished_ids: Vec<String> =
9176 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
9177 finished_ids.sort_unstable();
9178 let mut want = vec![failed.id.clone(), held.id.clone()];
9179 want.sort_unstable();
9180 assert_eq!(finished_ids, want);
9181 }
9182
9183 #[test]
9184 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
9185 for short in [false, true] {
9186 let dir = tempfile::tempdir().unwrap();
9187 let queue = Queue::at(dir.path().join("queue"));
9188 let questions = ask::Questions::at(dir.path().join("questions"));
9189 let mut running = task();
9190 running.id = "20261013-192213-feb2".into();
9191 running.start("existing-run".into());
9192 queue.put(&mut running).unwrap();
9193 let mut middle = task();
9194 middle.id = "20261013-193611-ca5b".into();
9195 middle.block(
9196 vec![if short {
9197 "feb2".into()
9198 } else {
9199 running.id.clone()
9200 }],
9201 None,
9202 );
9203 queue.put(&mut middle).unwrap();
9204 let mut waiting = task();
9205 waiting.id = "20261013-202849-ee37".into();
9206 waiting.block(
9207 vec![if short {
9208 "ca5b".into()
9209 } else {
9210 middle.id.clone()
9211 }],
9212 Some("wait for implementation".into()),
9213 );
9214 queue.put(&mut waiting).unwrap();
9215 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
9216 let inv = crate::blockers::Inventory::load(&queue, &questions);
9217 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
9218 assert!(inv.stuck().is_empty());
9219 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9220 assert!(report.quarantined.is_empty());
9221 assert!(report.asked.is_empty());
9222 resolve_blockers(&queue, &questions);
9223 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9224 assert!(questions.list().is_empty());
9225 running.succeed();
9228 queue.put(&mut running).unwrap();
9229 resolve_blockers(&queue, &questions);
9230 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
9231 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9232 middle = queue.get(&middle.id).unwrap();
9233 middle.succeed();
9234 queue.put(&mut middle).unwrap();
9235 resolve_blockers(&queue, &questions);
9236 let after = queue.get(&waiting.id).unwrap();
9237 assert_eq!(after.status, TaskStatus::Queued);
9238 assert!(after.blocked_by.is_empty());
9239 assert_eq!(after.instruction, waiting.instruction);
9240 }
9241 }
9242
9243 #[test]
9244 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
9245 for future_schema in [false, true] {
9246 let name = "20261013-193611-ca5b";
9247 let mut future = task();
9248 future.id = name.into();
9249 future.schema = crate::queue::SCHEMA + 1;
9250 let contents = if future_schema {
9251 serde_json::to_string(&future).unwrap()
9252 } else {
9253 "corrupt".into()
9254 };
9255 let dir = tempfile::tempdir().unwrap();
9256 let queue = Queue::at(dir.path().join("queue"));
9257 let questions = ask::Questions::at(dir.path().join("questions"));
9258 let mut waiting = task();
9259 waiting.id = "20261013-202849-ee37".into();
9260 waiting.block(vec!["ca5b".into()], None);
9261 queue.put(&mut waiting).unwrap();
9262 std::fs::write(queue.path_of(name), contents).unwrap();
9263 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
9264 assert!(
9265 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
9266 );
9267 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9268 assert!(report.asked.is_empty());
9269 resolve_blockers(&queue, &questions);
9270 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9271 assert!(questions.list().is_empty());
9272 }
9273 let dir = tempfile::tempdir().unwrap();
9274 let queue = Queue::at(dir.path().join("queue"));
9275 let questions = ask::Questions::at(dir.path().join("questions"));
9276 let mut waiting = task();
9277 waiting.id = "20261013-202849-ee37".into();
9278 waiting.block(vec!["ca5b".into()], None);
9279 queue.put(&mut waiting).unwrap();
9280 let mut done = task();
9281 done.id = "20261013-193611-ca5b".into();
9282 done.succeed();
9283 queue.put(&mut done).unwrap();
9284 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
9287 assert!(queue.get("ca5b").is_err());
9288 assert_eq!(
9289 crate::queue::dependency(&queue, &questions, "ca5b"),
9290 crate::queue::Dependency::Ambiguous
9291 );
9292 resolve_blockers(&queue, &questions);
9293 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9294 assert!(report.asked.is_empty());
9295 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9296 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
9297 }
9298
9299 #[test]
9300 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
9301 for held in [false, true] {
9302 for question in [false, true] {
9303 for reference in ["ca5b", "20261013-193611-ca5b"] {
9304 let dir = tempfile::tempdir().unwrap();
9305 let queue = Queue::at(dir.path().join("queue"));
9306 let questions = ask::Questions::at(dir.path().join("questions"));
9307 let mut prerequisite = task();
9308 prerequisite.id = "20261013-193611-ca5b".into();
9309 prerequisite.start("existing-run".into());
9310 queue.put(&mut prerequisite).unwrap();
9311 let mut waiting = task();
9312 waiting.id = "20261013-202849-ee37".into();
9313 if held {
9314 waiting.hold_machine(Some("waiting for separate consent".into()));
9315 }
9316 queue.put(&mut waiting).unwrap();
9317 let reason = "Deleting saved data requires separate owner consent";
9318 crate::conduct::apply(
9319 &queue,
9320 &questions,
9321 &crate::conduct::Verdict {
9322 decisions: vec![crate::conduct::Decision {
9323 id: waiting.id.clone(),
9324 blocked_by: vec![reference.into()],
9325 question: question.then(|| "May the saved data be deleted?".into()),
9326 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
9327 reason: Some(reason.into()),
9328 ..crate::conduct::Decision::default()
9329 }],
9330 },
9331 )
9332 .unwrap();
9333 let before = queue.get(&waiting.id).unwrap();
9334 if question {
9335 assert_eq!(before.status, TaskStatus::Blocked);
9336 let q = questions.get(&before.blocked_by[0]).unwrap();
9337 assert_eq!(q.summary, "May the saved data be deleted?");
9338 assert!(q.status.open());
9339 } else {
9340 assert_eq!(before.status, TaskStatus::Held);
9341 let recorded_reason = before.hold_reason.as_deref().unwrap();
9342 assert!(recorded_reason.starts_with(reason));
9343 if held {
9344 assert!(recorded_reason.contains("waiting for separate consent"));
9345 }
9346 }
9347 prerequisite.succeed();
9348 queue.put(&mut prerequisite).unwrap();
9349 resolve_blockers(&queue, &questions);
9350 let after = queue.get(&waiting.id).unwrap();
9351 assert_eq!(after.status, before.status);
9352 assert_eq!(after.blocked_by, before.blocked_by);
9353 assert_eq!(after.hold_reason, before.hold_reason);
9354 assert_eq!(after.instruction, waiting.instruction);
9355 if question {
9356 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
9357 }
9358 assert!(
9359 queue.next_runnable().is_none(),
9360 "dependency completion is not consent"
9361 );
9362 }
9363 }
9364 }
9365 }
9366
9367 #[test]
9368 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
9369 let dir = tempfile::tempdir().unwrap();
9370 let queue = Queue::at(dir.path().join("queue"));
9371 let questions = ask::Questions::at(dir.path().join("questions"));
9372
9373 let mut dep = task();
9374 dep.id = "20260101-000001-dep0".to_owned();
9375 dep.succeed();
9376 queue.put(&mut dep).unwrap();
9377
9378 let mut still_going = task();
9379 still_going.id = "20260101-000002-dep1".to_owned();
9380 queue.put(&mut still_going).unwrap();
9381
9382 let mut blocked = task();
9383 blocked.id = "20260101-000003-main".to_owned();
9384 blocked.block(
9385 vec![dep.id.clone(), still_going.id.clone()],
9386 Some("waits on both".to_owned()),
9387 );
9388 queue.put(&mut blocked).unwrap();
9389
9390 resolve_blockers(&queue, &questions);
9391
9392 let after = queue.get(&blocked.id).unwrap();
9393 assert_eq!(
9394 after.status,
9395 TaskStatus::Blocked,
9396 "one dependency is still outstanding"
9397 );
9398 assert_eq!(after.blocked_by, [still_going.id.clone()]);
9399 }
9400
9401 #[test]
9402 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
9403 let dir = tempfile::tempdir().unwrap();
9404 let queue = Queue::at(dir.path().join("queue"));
9405 let questions = ask::Questions::at(dir.path().join("questions"));
9406
9407 let mut q = crate::ask::Question::new(
9408 "20260101-000001-main".to_owned(),
9409 crate::conduct::NODE.to_owned(),
9410 "conduct".to_owned(),
9411 "Which backend?".to_owned(),
9412 String::new(),
9413 Vec::new(),
9414 );
9415 questions.put(&mut q).unwrap();
9416 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
9417 .unwrap();
9418 questions.put(&mut q).unwrap();
9419
9420 let mut blocked = task();
9421 blocked.id = "20260101-000001-main".to_owned();
9422 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
9423 queue.put(&mut blocked).unwrap();
9424
9425 resolve_blockers(&queue, &questions);
9426
9427 let after = queue.get(&blocked.id).unwrap();
9428 assert_eq!(
9429 after.status,
9430 TaskStatus::Queued,
9431 "the only blocker resolved"
9432 );
9433 assert_eq!(after.answers.len(), 1);
9434 assert_eq!(after.answers[0].question, "Which backend?");
9435 assert_eq!(after.answers[0].answer, "SQLite");
9436
9437 let instruction = instruction_for(&after);
9439 assert!(instruction.contains("Which backend?"));
9440 assert!(instruction.contains("SQLite"));
9441 }
9442
9443 #[test]
9444 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
9445 let dir = tempfile::tempdir().unwrap();
9446 let queue = Queue::at(dir.path().join("queue"));
9447 let questions = ask::Questions::at(dir.path().join("questions"));
9448
9449 let mut q = crate::ask::Question::new(
9450 "20260101-000001-main".to_owned(),
9451 crate::conduct::NODE.to_owned(),
9452 "conduct".to_owned(),
9453 "Is the setup done?".to_owned(),
9454 String::new(),
9455 Vec::new(),
9456 );
9457 q.abandon("no answer within 60s of asking");
9458 questions.put(&mut q).unwrap();
9459
9460 let mut blocked = task();
9461 blocked.id = "20260101-000001-main".to_owned();
9462 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
9463 queue.put(&mut blocked).unwrap();
9464
9465 resolve_blockers(&queue, &questions);
9466
9467 let after = queue.get(&blocked.id).unwrap();
9468 assert_eq!(
9469 after.status,
9470 TaskStatus::Held,
9471 "never left blocked on nothing"
9472 );
9473 assert!(!after.operator_held(), "a machine hold, for triage");
9474 assert!(
9475 after
9476 .hold_reason
9477 .as_deref()
9478 .unwrap_or_default()
9479 .contains("went unanswered")
9480 );
9481 }
9482
9483 #[test]
9484 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
9485 let dir = tempfile::tempdir().unwrap();
9491 let queue = Queue::at(dir.path().join("queue"));
9492 let questions = ask::Questions::at(dir.path().join("questions"));
9493
9494 let mut q = crate::ask::Question::new(
9495 "20260101-000001-main".to_owned(),
9496 crate::conduct::NODE.to_owned(),
9497 "conduct".to_owned(),
9498 "How should this be handled?".to_owned(),
9499 String::new(),
9500 Vec::new(),
9501 );
9502 questions.put(&mut q).unwrap();
9503 q.answer(crate::ask::Answer::Text(
9504 "leave it held, a human will look at it later".to_owned(),
9505 ))
9506 .unwrap();
9507 questions.put(&mut q).unwrap();
9508
9509 let mut held = task();
9510 held.id = "20260101-000001-main".to_owned();
9511 held.hold_machine(Some("out of attempts".to_owned()));
9512 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
9513 queue.put(&mut held).unwrap();
9514
9515 resolve_blockers(&queue, &questions);
9516
9517 let after = queue.get(&held.id).unwrap();
9518 assert_eq!(after.status, TaskStatus::Held);
9519 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
9520 assert_eq!(
9521 after.answers[0].answer,
9522 "leave it held, a human will look at it later"
9523 );
9524 }
9525
9526 #[test]
9527 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
9528 let dir = tempfile::tempdir().unwrap();
9529 let queue = Queue::at(dir.path().join("queue"));
9530 let questions = ask::Questions::at(dir.path().join("questions"));
9531 let mut dep = task();
9532 dep.id = "20260101-000001-gone".to_owned();
9533 queue.put(&mut dep).unwrap();
9534 let mut blocked = task();
9535 blocked.id = "20260101-000003-main".to_owned();
9536 blocked.block(vec![dep.id.clone()], None);
9537 queue.put(&mut blocked).unwrap();
9538
9539 let claim = queue.claim(&blocked.id).unwrap();
9541 queue.remove(&dep.id, false, &questions).unwrap();
9542 drop(claim);
9543 resolve_blockers(&queue, &questions);
9544
9545 let after = queue.get(&blocked.id).unwrap();
9546 assert_eq!(after.status, TaskStatus::Queued);
9547 assert!(after.blocked_by.is_empty());
9548 }
9549
9550 #[test]
9551 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
9552 let dir = tempfile::tempdir().unwrap();
9558 let queue = Queue::at(dir.path().join("queue"));
9559 let questions = ask::Questions::at(dir.path().join("questions"));
9560
9561 let mut still_going = task();
9562 still_going.id = "20260101-000002-dep1".to_owned();
9563 queue.put(&mut still_going).unwrap();
9564
9565 let mut blocked = task();
9566 blocked.id = "20260101-000003-main".to_owned();
9567 blocked.block(
9568 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
9569 Some("waits on both".to_owned()),
9570 );
9571 queue.put(&mut blocked).unwrap();
9572
9573 resolve_blockers(&queue, &questions);
9574
9575 let after = queue.get(&blocked.id).unwrap();
9576 assert_eq!(
9577 after.status,
9578 TaskStatus::Held,
9579 "a missing dependency must not leave the task blocked forever"
9580 );
9581 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
9582 assert!(after.blocked_by.is_empty());
9583 let reason = after.hold_reason.as_deref().unwrap_or_default();
9584 assert!(
9585 reason.contains("20260101-000001-gone"),
9586 "the missing id must be named so an operator can tell what happened: {reason}"
9587 );
9588 assert!(
9589 reason.contains(&still_going.id),
9590 "the still-valid dependency must not silently vanish from the record: {reason}"
9591 );
9592 }
9593
9594 #[test]
9595 fn instruction_for_is_unchanged_without_any_answers() {
9596 let t = task();
9597 assert_eq!(instruction_for(&t), t.instruction);
9598 }
9599
9600 #[test]
9601 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
9602 let dir = tempfile::tempdir().unwrap();
9603 let q = Queue::at(dir.path().join("queue"));
9604 let src = dir.path().join("shot.png");
9605 std::fs::write(&src, "x").unwrap();
9606 let mut t = task();
9607 q.attach(&mut t, &[src]).unwrap();
9608 let paths = task_attachments(&q, &t).unwrap();
9609 assert_eq!(paths.len(), 1);
9610 assert!(paths[0].is_absolute() && paths[0].is_file());
9611 std::fs::remove_file(&paths[0]).unwrap();
9612 let err = task_attachments(&q, &t).unwrap_err().to_string();
9613 assert!(err.contains("shot.png"), "{err}");
9614 }
9615
9616 #[tokio::test]
9617 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
9618 let dir = tempfile::tempdir().unwrap();
9619 let queue_path = dir.path().join("queue");
9620 let queue = Queue::at(queue_path.clone());
9621 let mut t = task();
9622 t.source = Source::Agent {
9623 run: "originating-chat".into(),
9624 node: crate::queue::CHAT_NODE.into(),
9625 };
9626 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
9627 stage: "config-fetch",
9628 target: "origin/main".into(),
9629 });
9630 record_config_failure(&mut t, &fetch);
9631 t.hold_machine(Some(format!("{fetch:#}")));
9632 let fetch_key = t.chat_notice_due().unwrap();
9633 t.chat_notices.push(fetch_key.clone());
9634 queue.put(&mut t).unwrap();
9635
9636 let queue = Queue::at(queue_path);
9637 let mut t = queue.get(&t.id).unwrap();
9638 let a = dir.path().join("a.toml");
9639 let b = dir.path().join("b.toml");
9640 for (path, text, new_cause) in [
9641 (&a, "[graph", true),
9642 (&a, "[verify", false),
9643 (&a, "[graph]\nimplementers = 'invalid-type'", true),
9644 (&b, "[graph]\nimplementers = 'invalid-type'", true),
9645 ] {
9646 t.release_operator("test", None);
9647 std::fs::write(path, text).unwrap();
9648 let error = Config::discover_fetched(dir.path(), Some(path))
9649 .await
9650 .unwrap_err();
9651 record_config_failure(&mut t, &error);
9652 t.hold_machine(Some(format!("{error:#}")));
9653 let due = t.chat_notice_due();
9654 assert_eq!(due.is_some(), new_cause, "{error:#}");
9655 if let Some(key) = due {
9656 assert_ne!(key, fetch_key);
9657 t.chat_notices.push(key);
9658 }
9659 }
9660 t.succeed();
9661 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
9662 }
9663
9664 #[test]
9665 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
9666 let dir = tempfile::tempdir().unwrap();
9667 let queue_path = dir.path().join("queue");
9668 let queue = Queue::at(queue_path.clone());
9669 let mut t = task();
9670 t.source = crate::queue::Source::Agent {
9671 run: "originating-chat".into(),
9672 node: crate::queue::CHAT_NODE.into(),
9673 };
9674 let a = dir.path().join("a.png");
9675 let b = dir.path().join("b.png");
9676 std::fs::write(&a, "a").unwrap();
9677 std::fs::write(&b, "b").unwrap();
9678 queue.attach(&mut t, &[a, b]).unwrap();
9679 let paths = task_attachments(&queue, &t).unwrap();
9680 for path in &paths {
9681 std::fs::remove_file(path).unwrap();
9682 }
9683 let missing = task_attachments(&queue, &t).unwrap_err();
9684 t.attempts += 1;
9685 let reason = missing.to_string();
9686 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9687 let first = t.chat_notice_due().expect("first attachment notice");
9688 t.chat_notices.push(first.clone());
9689 queue.put(&mut t).unwrap();
9690
9691 let queue = Queue::at(queue_path.clone());
9692 let mut t = queue.get(&t.id).unwrap();
9693 assert!(t.chat_notice_due().is_none(), "restart does not replay");
9694 t.release_operator("test", None);
9695 t.attempts += 1;
9696 let missing = task_attachments(&queue, &t).unwrap_err();
9697 t.fail_startup_target(
9698 "startup-attachments",
9699 missing.name,
9700 "same missing attachment with regenerated wording",
9701 1,
9702 );
9703 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
9704
9705 std::fs::write(&paths[0], "restored").unwrap();
9706 t.release_operator("test", None);
9707 t.attempts += 1;
9708 let missing = task_attachments(&queue, &t).unwrap_err();
9709 let reason = missing.to_string();
9710 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9711 let second = t
9712 .chat_notice_due()
9713 .expect("different attachment must notify");
9714 assert_ne!(first, second);
9715 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
9716 }
9717
9718 #[test]
9719 fn resumed_instruction_is_unchanged_without_any_answers() {
9720 let t = task();
9721 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
9722 }
9723
9724 #[test]
9725 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
9726 let mut t = task();
9727 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9728 let old = t.instruction.clone();
9732
9733 let refreshed = resumed_instruction(&old, &t);
9734 assert!(refreshed.starts_with(&old), "the original text is kept");
9735 assert!(refreshed.contains("Which backend?"));
9736 assert!(refreshed.contains("SQLite"));
9737 }
9738
9739 #[test]
9740 fn resumed_instruction_keeps_an_original_answers_heading() {
9741 let mut t = task();
9742 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
9743 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9744
9745 let refreshed = resumed_instruction(&t.instruction, &t);
9746
9747 assert!(
9748 refreshed.starts_with(&t.instruction),
9749 "an answers heading in the original instruction is not the appended block"
9750 );
9751 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
9752 assert!(refreshed.contains("Which backend?"));
9753 assert!(refreshed.contains("SQLite"));
9754
9755 let repeated = resumed_instruction(&refreshed, &t);
9756 assert_eq!(
9757 repeated, refreshed,
9758 "only the final appended block is refreshed"
9759 );
9760 }
9761
9762 #[test]
9763 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
9764 let mut t = task();
9765 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9766
9767 let once = resumed_instruction(&t.instruction, &t);
9771 let twice = resumed_instruction(&once, &t);
9772 assert_eq!(once, twice);
9773 assert_eq!(once.matches("Which backend?").count(), 1);
9774
9775 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
9777 let refreshed = resumed_instruction(&once, &t);
9778 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
9779 assert!(refreshed.contains("Which backend?"));
9780 assert!(refreshed.contains("Which cache?"));
9781 }
9782
9783 #[test]
9784 fn prepare_instruction_covers_all_three_starters() {
9785 let mut t = task();
9786 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9787
9788 assert_eq!(
9791 prepare_instruction(&Starter::Start, None, &t),
9792 Some(instruction_for(&t))
9793 );
9794
9795 let old = t.instruction.clone();
9798 assert_eq!(
9799 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
9800 Some(resumed_instruction(&old, &t))
9801 );
9802
9803 assert_eq!(
9807 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
9808 None
9809 );
9810 }
9811
9812 #[test]
9813 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
9814 assert_eq!(
9815 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
9816 Starter::Review("magi/eba2/A".to_owned())
9817 );
9818 }
9819
9820 #[test]
9821 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
9822 assert_eq!(
9823 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
9824 Starter::Start,
9825 "a vanished review branch must not fall back to resuming the old run either"
9826 );
9827 }
9828
9829 #[test]
9830 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
9831 let mut t = task();
9832 t.start("old-run".to_owned());
9833 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
9834 t.release();
9835 let branch = t.review_branch.take();
9836 assert_eq!(
9837 choose_starter(branch.as_deref(), true, Some("old-run")),
9838 Starter::Review("magi/eba2/A".to_owned()),
9839 "a review wins over resuming the old run"
9840 );
9841 }
9842
9843 #[test]
9844 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
9845 assert_eq!(
9846 choose_starter(None, false, Some("some-run")),
9847 Starter::Resume("some-run".to_owned())
9848 );
9849 assert_eq!(choose_starter(None, false, None), Starter::Start);
9850 }
9851
9852 #[test]
9853 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
9854 let mut released = task();
9855 released.start("stalled-run".to_owned());
9856 released.requeue();
9857 let unfinished = (!released.fresh_start)
9858 .then(|| Some("stalled-run".to_owned()))
9859 .flatten();
9860 assert_eq!(
9861 choose_starter(None, false, unfinished.as_deref()),
9862 Starter::Start,
9863 "release keeps run history but must not resume it"
9864 );
9865 assert_eq!(released.runs, ["stalled-run"]);
9866 }
9867
9868 #[test]
9869 fn an_ordinary_release_keeps_a_resumable_run_available() {
9870 let mut released = task();
9871 released.start("stalled-run".to_owned());
9872 released.release();
9873 let unfinished = (!released.fresh_start)
9874 .then(|| Some("stalled-run".to_owned()))
9875 .flatten();
9876 assert_eq!(
9877 choose_starter(None, false, unfinished.as_deref()),
9878 Starter::Resume("stalled-run".to_owned()),
9879 "manual release must preserve the normal resume path"
9880 );
9881 }
9882
9883 #[test]
9884 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
9885 let mut state = run_state(RunStatus::Blocked);
9886 state.config.graph.review_rounds = 3;
9887 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
9888 assert!(exhausted_review_budget(&state));
9889
9890 state.reviews.pop();
9892 assert!(!exhausted_review_budget(&state));
9893
9894 let mut stalled = run_state(RunStatus::Stalled);
9897 stalled.config.graph.review_rounds = 1;
9898 stalled.reviews = vec![review_round(1)];
9899 assert!(!exhausted_review_budget(&stalled));
9900 }
9901
9902 fn review_round(round: usize) -> crate::run::ReviewRound {
9903 crate::run::ReviewRound {
9904 round,
9905 head: "deadbeef".to_owned(),
9906 verified_head: None,
9907 verified_at: None,
9908 reviews: Vec::new(),
9909 e2e: Vec::new(),
9910 verify_retried: false,
9911 e2e_deferred: false,
9912 e2e_defer_reason: None,
9913 fix: None,
9914 blocking: 0,
9915 answered: 1,
9916 expected: 1,
9917 clean: false,
9918 progressed: true,
9919 vote_split: false,
9920 reconsideration: Vec::new(),
9921 verdict: None,
9922 }
9923 }
9924
9925 #[test]
9926 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
9927 let mut run = RunState::new(
9928 PathBuf::from("/repo"),
9929 "main".to_owned(),
9930 "abc1234def".to_owned(),
9931 "add retries".to_owned(),
9932 Config::default(),
9933 );
9934 run.status = RunStatus::Judging;
9935 run.parked = true;
9936 let mut task = Task::new(
9937 "add retries".to_owned(),
9938 "add retries".to_owned(),
9939 PathBuf::from("/repo"),
9940 crate::queue::Source::Human,
9941 );
9942 task.status = TaskStatus::Failed;
9943 task.runs = vec![run.id.clone()];
9944 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
9945 assert!(with(&task, &run), "parked after judging is the case");
9946 let mut parked_task = task.clone();
9947 parked_task.status = TaskStatus::Parked;
9948 assert!(with(&parked_task, &run), "the Parked status is the case");
9949
9950 let mut not_parked = run.clone();
9951 not_parked.parked = false;
9952 not_parked.status = RunStatus::Stalled;
9953 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
9954
9955 let mut fresh = task.clone();
9956 fresh.fresh_start = true;
9957 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
9958
9959 let mut review = task.clone();
9960 review.review_branch = Some("magi/x/A".to_owned());
9961 assert!(!with(&review, &run), "review is ranked before resume");
9962
9963 let mut held = task.clone();
9964 held.status = TaskStatus::Held;
9965 assert!(!with(&held, &run), "a hold stays visible to the conductor");
9966
9967 let mut released = run.clone();
9968 released.released_to = Some("20260901-000000-new1".to_owned());
9969 assert!(!with(&task, &released), "nothing left to resume into");
9970
9971 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
9972 "unreadable"
9973 )));
9974 }
9975
9976 #[test]
9977 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
9978 let mut released = RunState::new(
9979 PathBuf::from("/repo"),
9980 "main".to_owned(),
9981 "abc1234def".to_owned(),
9982 "add retries".to_owned(),
9983 Config::default(),
9984 );
9985 released.status = RunStatus::Blocked;
9986 assert_eq!(
9987 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9988 Some(released.id.clone())
9989 );
9990 released.released_to = Some("20260901-000000-new1".to_owned());
9991 assert_eq!(
9992 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9993 None,
9994 "there is nothing left to resume it into"
9995 );
9996 }
9997
9998 #[test]
9999 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
10000 let mut exhausted = RunState::new(
10009 PathBuf::from("/repo"),
10010 "main".to_owned(),
10011 "abc1234def".to_owned(),
10012 "add retries".to_owned(),
10013 Config::default(),
10014 );
10015 exhausted.status = RunStatus::Blocked;
10016 exhausted.config.graph.review_rounds = 1;
10017 exhausted.reviews = vec![review_round(1)];
10018
10019 assert_eq!(
10020 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
10021 None,
10022 "an exhausted `Blocked` run must not be offered as resumable"
10023 );
10024
10025 let mut has_budget_left = RunState::new(
10028 PathBuf::from("/repo"),
10029 "main".to_owned(),
10030 "abc1234def".to_owned(),
10031 "add retries".to_owned(),
10032 Config::default(),
10033 );
10034 has_budget_left.status = RunStatus::Blocked;
10035 has_budget_left.config.graph.review_rounds = 3;
10036 has_budget_left.reviews = vec![review_round(1)];
10037
10038 assert_eq!(
10039 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
10040 Ok(has_budget_left.clone())
10041 }),
10042 Some(has_budget_left.id.clone())
10043 );
10044 }
10045
10046 #[test]
10047 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
10048 let mut run = RunState::new(
10049 PathBuf::from("/repo"),
10050 "main".to_owned(),
10051 "abc1234def".to_owned(),
10052 "add retries".to_owned(),
10053 Config::default(),
10054 );
10055 run.status = RunStatus::Blocked;
10056 run.config.graph.review_rounds = 1;
10057 run.reviews = vec![review_round(1)];
10058 run.pr_publication_pending = true;
10059 assert_eq!(
10060 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
10061 Some(run.id.clone()),
10062 "a spent review budget must not turn a failed publication into a re-competition"
10063 );
10064 }
10065
10066 #[test]
10067 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
10068 let mut older_stalled = RunState::new(
10076 PathBuf::from("/repo"),
10077 "main".to_owned(),
10078 "abc1234def".to_owned(),
10079 "add retries".to_owned(),
10080 Config::default(),
10081 );
10082 older_stalled.status = RunStatus::Stalled;
10083
10084 let mut newest_exhausted = RunState::new(
10085 PathBuf::from("/repo"),
10086 "main".to_owned(),
10087 "abc1234def".to_owned(),
10088 "add retries".to_owned(),
10089 Config::default(),
10090 );
10091 newest_exhausted.status = RunStatus::Blocked;
10092 newest_exhausted.config.graph.review_rounds = 1;
10093 newest_exhausted.reviews = vec![review_round(1)];
10094
10095 assert_eq!(
10096 unfinished_run_with(
10097 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
10098 "t",
10099 |_| Ok(newest_exhausted.clone())
10100 ),
10101 None,
10102 "the newest run is exhausted, so nothing here is worth resuming - \
10103 least of all the older, already-superseded run"
10104 );
10105 }
10106
10107 #[test]
10108 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
10109 assert_eq!(
10110 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
10111 Err(anyhow::anyhow!("fixture is absent"))
10112 }),
10113 None
10114 );
10115 }
10116
10117 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
10118 let mut q = ask::Question::new(
10119 run.to_owned(),
10120 "implement".to_owned(),
10121 "impl-A".to_owned(),
10122 "continue?".to_owned(),
10123 String::new(),
10124 vec!["resume で続行する".to_owned(), "other".to_owned()],
10125 );
10126 q.actions.insert("resume で続行する".to_owned(), action);
10127 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
10128 .unwrap();
10129 q
10130 }
10131
10132 fn held_task_with(run: &str) -> Task {
10133 let mut t = task();
10134 t.runs = vec![run.to_owned()];
10135 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
10136 t
10137 }
10138
10139 fn resume_action(run: &str) -> ask::ChoiceAction {
10140 ask::ChoiceAction::Resume { run: run.into() }
10141 }
10142
10143 #[test]
10144 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
10145 for parked in [false, true] {
10146 let mut t = task();
10147 t.start("A".into());
10148 t.start("B".into());
10149 assert_eq!(t.runs, vec!["A", "B"]);
10150 t.start("A".into());
10151 let mut resumed = run_state(RunStatus::Blocked);
10152 resumed.id = "A".into();
10153 resumed.parked = parked;
10154 resumed.driver_exited = true;
10155 t.hold_machine(Some("resumed run stopped".into()));
10156 let q = action_question("A", resume_action("A"));
10157 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
10158 assert_eq!(
10159 decide_action(&t, &q, &PHRASES_EN, |id| {
10160 assert_eq!(id, "A");
10161 Ok(resumed.clone())
10162 }),
10163 ActionDecision::Resume("A".into())
10164 );
10165 assert_eq!(
10166 action_standing(&t, &action_question("B", resume_action("B"))),
10167 ActionStanding::Stale
10168 );
10169 t.release_operator("cli", None);
10170 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
10171 assert_eq!(id, "A");
10172 Ok(resumed.clone())
10173 });
10174 assert_eq!(
10175 choose_starter(None, false, unfinished.as_deref()),
10176 Starter::Resume("A".into())
10177 );
10178 assert_eq!(t.runs, vec!["B", "A"]);
10179 }
10180 }
10181
10182 #[test]
10183 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
10184 let mut state = crate::land::tests::repair_state(true, true);
10185 state.status = RunStatus::Landing;
10186 state.parked = true;
10187 state.driver_exited = true;
10188 let mut q = ask::Question::new(
10189 state.id.clone(),
10190 land::APPROVAL_NODE.into(),
10191 "land".into(),
10192 "Merge?".into(),
10193 String::new(),
10194 vec![land::APPROVE.into(), land::HOLD.into()],
10195 );
10196 q.id = state.id.clone();
10197 state.land_approval = Some(crate::run::LandApproval {
10198 question: q.id.clone(),
10199 head: "current-head".into(),
10200 });
10201 assert!(land::upgrade_repair_choice(&mut q, &state));
10202 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10203 let mut task = held_task_with(&state.id);
10204 task.status = TaskStatus::Parked;
10205 assert_eq!(
10206 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10207 ActionDecision::Resume(state.id.clone())
10208 );
10209 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10210 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
10211 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
10212 assert_eq!(
10213 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10214 ActionDecision::Resume(state.id.clone()),
10215 "a repair answer newer than the manual hold resumes the existing owner"
10216 );
10217 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10218 for answered_at in [q.answered_at, None] {
10219 let mut older = q.clone();
10220 older.answered_at = answered_at;
10221 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
10222 panic!("a later manual hold must prevent dispatch")
10223 });
10224 let ActionDecision::Refuse(reason) = refused else {
10225 panic!("expected an actionable refusal, got {refused:?}");
10226 };
10227 assert!(reason.contains("later operator decision"));
10228 assert!(reason.contains(&format!("magi task release {}", task.id)));
10229 }
10230 task.operator_decided_at = None;
10231 let mut superseded = state.clone();
10232 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
10233 assert_eq!(
10234 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
10235 ActionDecision::Stale
10236 );
10237 for case in 1..4 {
10238 let mut changed = state.clone();
10239 match case {
10240 1 => changed.status = RunStatus::Blocked,
10241 2 => changed.released_to = Some("new-owner".into()),
10242 _ => changed.land_approval.as_mut().unwrap().head.clear(),
10243 }
10244 assert!(matches!(
10245 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
10246 ActionDecision::Refuse(_)
10247 ));
10248 }
10249 assert!(matches!(
10250 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
10251 ActionDecision::Refuse(_)
10252 ));
10253 task.runs.push("replacement".into());
10254 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10255 assert_eq!(
10256 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
10257 "old owner must not load"
10258 )),
10259 ActionDecision::Stale
10260 );
10261 }
10262
10263 #[test]
10264 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
10265 let mut state = crate::land::tests::repair_state(true, true);
10266 state.status = RunStatus::Landing;
10267 state.parked = true;
10268 state.driver_exited = true;
10269 let mut q = ask::Question::new(
10270 state.id.clone(),
10271 land::APPROVAL_NODE.into(),
10272 "land".into(),
10273 "Merge?".into(),
10274 String::new(),
10275 vec![land::APPROVE.into(), land::HOLD.into()],
10276 );
10277 q.id = state.id.clone();
10278 state.land_approval = Some(crate::run::LandApproval {
10279 question: q.id.clone(),
10280 head: "current-head".into(),
10281 });
10282 state.save().unwrap();
10283 land::upgrade_repair_choice(&mut q, &state);
10284 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10285 Questions::open().put(&mut q).unwrap();
10286 let tmp = tempfile::tempdir().unwrap();
10287 let queue = Queue::at(tmp.path().join("queue"));
10288 let mut task = held_task_with(&state.id);
10289 task.status = TaskStatus::Parked;
10290 queue.put(&mut task).unwrap();
10291 let empty = Arc::new(tokio::sync::Semaphore::new(0));
10292 let free = Arc::new(tokio::sync::Semaphore::new(1));
10293 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
10294 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
10295 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
10296 assert!(claim.permit.is_some());
10297 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
10298 drop(claim);
10299 Questions::open()
10300 .update(&q.id, |q| {
10301 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
10302 Ok(())
10303 })
10304 .unwrap();
10305 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
10306 assert!(claim.permit.is_none());
10307 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
10308 }
10309
10310 #[test]
10311 fn decide_action_resumes_only_the_latest_resumable_run() {
10312 let t = held_task_with("r1");
10313 let q = action_question("r1", resume_action("r1"));
10314 let load = |s: RunState| move |_: &str| Ok(s);
10315 assert_eq!(
10316 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
10317 ActionDecision::Resume("r1".into())
10318 );
10319 let q_other = action_question("r1", resume_action("r0"));
10321 assert!(matches!(
10322 decide_action(
10323 &t,
10324 &q_other,
10325 &PHRASES_EN,
10326 load(run_state(RunStatus::Blocked))
10327 ),
10328 ActionDecision::Refuse(_)
10329 ));
10330 assert!(matches!(
10332 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
10333 ActionDecision::Refuse(_)
10334 ));
10335 let mut released = run_state(RunStatus::Blocked);
10337 released.released_to = Some("elsewhere".into());
10338 assert!(matches!(
10339 decide_action(&t, &q, &PHRASES_EN, load(released)),
10340 ActionDecision::Refuse(_)
10341 ));
10342 assert!(matches!(
10344 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
10345 ActionDecision::Refuse(_)
10346 ));
10347 }
10348
10349 #[test]
10350 fn decide_action_ignores_a_question_about_an_earlier_run() {
10351 let mut t = held_task_with("r1");
10352 t.runs.push("r2".to_owned());
10353 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10354 assert_eq!(
10355 decide_action(
10356 &t,
10357 &action_question("r1", ask::ChoiceAction::Done),
10358 &PHRASES_EN,
10359 never
10360 ),
10361 ActionDecision::Stale
10362 );
10363 }
10364
10365 #[test]
10366 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
10367 let mut t = held_task_with("r1");
10368 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10369 assert_eq!(
10370 decide_action(
10371 &t,
10372 &action_question("r1", ask::ChoiceAction::Requeue),
10373 &PHRASES_EN,
10374 never
10375 ),
10376 ActionDecision::Requeue
10377 );
10378 let done_q = action_question("r1", ask::ChoiceAction::Done);
10379 assert_eq!(
10380 decide_action(&t, &done_q, &PHRASES_EN, never),
10381 ActionDecision::Done
10382 );
10383 t.mark_action_applied(&done_q.id);
10384 assert_eq!(
10385 decide_action(&t, &done_q, &PHRASES_EN, never),
10386 ActionDecision::Skip
10387 );
10388
10389 let mut plain = action_question("r1", ask::ChoiceAction::Done);
10391 plain.actions.clear();
10392 assert_eq!(
10393 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
10394 ActionDecision::Skip
10395 );
10396 let mut running = held_task_with("r1");
10398 running.status = TaskStatus::Running;
10399 assert_eq!(
10400 decide_action(
10401 &running,
10402 &action_question("r1", ask::ChoiceAction::Done),
10403 &PHRASES_EN,
10404 never
10405 ),
10406 ActionDecision::Skip
10407 );
10408 }
10409
10410 #[test]
10411 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
10412 crate::run::pin_test_home();
10413 for legacy in [false, true] {
10414 let dir = tempfile::tempdir().unwrap();
10415 let queue = Queue::at(dir.path().join("queue"));
10416 let questions = Questions::at(dir.path().join("questions"));
10417 let run = "20260101-000000-hold";
10418 let mut task = held_task_with(run);
10419 task.status = TaskStatus::Parked;
10420 let mut q = ask::Question::new(
10421 run.into(),
10422 land::APPROVAL_NODE.into(),
10423 "land".into(),
10424 "Repair?".into(),
10425 String::new(),
10426 vec![land::REPAIR.into()],
10427 );
10428 q.actions.insert(land::REPAIR.into(), resume_action(run));
10429 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10430 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10432 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
10433 questions.put(&mut q).unwrap();
10434 task.hold_manual(Some("Keep this PR paused".into()));
10435 if legacy {
10436 task.hold_source = None;
10437 }
10438 queue.put(&mut task).unwrap();
10439
10440 apply_choice_actions(&queue, &questions, dir.path());
10441 let after = queue.get(&task.id).unwrap();
10442 assert!(after.operator_held());
10443 assert_eq!(after.hold_source, task.hold_source);
10444 assert_eq!(after.operator_decided_at, task.operator_decided_at);
10445 assert_eq!(after.runs, task.runs);
10446 assert_eq!(after.instruction, task.instruction);
10447 assert!(after.resume_override.is_none());
10448 assert!(after.action_applied(&q.id));
10449 let reason = after.hold_reason.as_deref().unwrap();
10450 assert!(reason.starts_with("Keep this PR paused"));
10451 assert!(reason.contains("later operator decision"));
10452 assert!(reason.contains(&format!("magi task release {}", task.id)));
10453
10454 for recovery in [
10455 crate::conduct::Recovery::Review,
10456 crate::conduct::Recovery::Requeue,
10457 ] {
10458 crate::conduct::apply(
10459 &queue,
10460 &questions,
10461 &crate::conduct::Verdict {
10462 decisions: vec![crate::conduct::Decision {
10463 id: task.id.clone(),
10464 recovery: Some(recovery),
10465 ..Default::default()
10466 }],
10467 },
10468 )
10469 .unwrap();
10470 }
10471 apply_choice_actions(&queue, &questions, dir.path());
10474 let still_held = queue.get(&task.id).unwrap();
10475 assert!(still_held.operator_held());
10476 assert_eq!(still_held.hold_reason, after.hold_reason);
10477 assert_eq!(still_held.runs, task.runs);
10478 assert!(still_held.review_branch.is_none());
10479 assert!(!still_held.fresh_start);
10480 assert!(queue.next_runnable().is_none());
10481 }
10482 }
10483
10484 #[test]
10485 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
10486 for held in [false, true] {
10487 for retry in [false, true] {
10488 let dir = tempfile::tempdir().unwrap();
10489 let queue = Queue::at(dir.path().join("queue"));
10490 let questions = Questions::at(dir.path().join("questions"));
10491 let home = dir.path().join("home");
10492 let mut t = task();
10493 if held {
10494 t.hold_machine(Some("driver state unknown".into()));
10495 }
10496 queue.put(&mut t).unwrap();
10497 let verdict = serde_json::from_value(serde_json::json!({
10498 "decisions": [{
10499 "id": t.id,
10500 "question": "Driver state unknown; retry the checks or hold?",
10501 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
10502 "actions": {
10503 "daemon: recheck and retry": {"do": "requeue"},
10504 "daemon: hold the task": {"do": "hold"}
10505 }
10506 }]
10507 }))
10508 .unwrap();
10509 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
10510 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
10511 let mut q = questions.list().pop().unwrap();
10512 let choice = if retry {
10513 "daemon: recheck and retry"
10514 } else {
10515 "daemon: hold the task"
10516 };
10517 assert_eq!(q.actions.len(), 2);
10518 q.answer(ask::Answer::Choice(choice.into())).unwrap();
10519 questions.put(&mut q).unwrap();
10520
10521 questions.beat(&q.id, ask::WaiterKind::Deputy);
10523 for _ in 0..2 {
10524 resolve_blockers(&queue, &questions);
10525 apply_choice_actions(&queue, &questions, &home);
10526 let waiting = queue.get(&t.id).unwrap();
10527 assert_eq!(waiting.status, TaskStatus::Blocked);
10528 assert_eq!(waiting.blocked_by, [q.id.clone()]);
10529 assert!(waiting.answers.is_empty());
10530 assert!(!waiting.action_applied(&q.id));
10531 assert!(runnable(&queue).is_empty());
10532 let free = Arc::new(tokio::sync::Semaphore::new(1));
10533 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
10534 }
10535 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
10536
10537 resolve_blockers(&queue, &questions);
10539 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
10540 apply_choice_actions(&queue, &questions, &home);
10541 let after = queue.get(&t.id).unwrap();
10542 assert_eq!(
10543 after.status,
10544 if retry {
10545 TaskStatus::Queued
10546 } else {
10547 TaskStatus::Held
10548 }
10549 );
10550 assert!(after.action_applied(&q.id));
10551 assert_eq!(after.operator_held(), !retry);
10552 assert_eq!(after.answers.len(), 1);
10553
10554 let mut changed = after;
10556 if retry {
10557 changed.hold_manual(Some("later hold".into()));
10558 } else {
10559 changed.release_operator("test", None);
10560 }
10561 let expected = changed.status;
10562 queue.put(&mut changed).unwrap();
10563 apply_choice_actions(&queue, &questions, &home);
10564 assert_eq!(queue.get(&t.id).unwrap().status, expected);
10565 }
10566 }
10567 }
10568
10569 #[test]
10570 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
10571 let dir = tempfile::tempdir().unwrap();
10572 let queue = Queue::at(dir.path().join("queue"));
10573 let questions = Questions::at(dir.path().join("questions"));
10574 let home = dir.path().join("home");
10575 let mut state = run_state(RunStatus::Blocked);
10576 state.id = "20260101-000000-act1".to_owned();
10577 state.save_under(&home).unwrap();
10578
10579 let mut t = held_task_with(&state.id);
10580 queue.put(&mut t).unwrap();
10581 let mut q = action_question(&state.id, resume_action(&state.id));
10582 questions.put(&mut q).unwrap();
10583
10584 apply_choice_actions(&queue, &questions, &home);
10585 let after = queue.get(&t.id).unwrap();
10586 assert_eq!(after.status, TaskStatus::Queued);
10587 assert!(!after.fresh_start);
10588 assert!(after.action_applied(&q.id));
10589 let pin = after.resume_override.clone().unwrap();
10590 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
10591 assert!(pin.forced);
10592
10593 let mut again = queue.get(&t.id).unwrap();
10595 again.hold_machine(Some("later".into()));
10596 queue.put(&mut again).unwrap();
10597 apply_choice_actions(&queue, &questions, &home);
10598 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10599 }
10600
10601 #[test]
10602 fn a_delivered_answer_is_still_acted_on_exactly_once() {
10603 let dir = tempfile::tempdir().unwrap();
10604 let queue = Queue::at(dir.path().join("queue"));
10605 let questions = Questions::at(dir.path().join("questions"));
10606 let home = dir.path().join("home");
10607 let mut state = run_state(RunStatus::Blocked);
10608 state.id = "20260101-000000-act2".to_owned();
10609 state.save_under(&home).unwrap();
10610
10611 let mut t = held_task_with(&state.id);
10612 queue.put(&mut t).unwrap();
10613 let mut q = action_question(&state.id, resume_action(&state.id));
10614 q.answer_delivered = true;
10615 questions.put(&mut q).unwrap();
10616
10617 apply_choice_actions(&queue, &questions, &home);
10620 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10621 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
10622
10623 let mut again = queue.get(&t.id).unwrap();
10625 again.hold_machine(Some("later".into()));
10626 queue.put(&mut again).unwrap();
10627 apply_choice_actions(&queue, &questions, &home);
10628 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10629 }
10630
10631 #[test]
10632 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
10633 let dir = tempfile::tempdir().unwrap();
10634 let queue = Queue::at(dir.path().join("queue"));
10635 let questions = Questions::at(dir.path().join("questions"));
10636 let home = dir.path().join("home");
10637 let mut state = run_state(RunStatus::Blocked);
10638 state.id = "20260101-000000-act3".to_owned();
10639 state.save_under(&home).unwrap();
10640 let mut t = held_task_with(&state.id);
10641 queue.put(&mut t).unwrap();
10642 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
10643 questions.put(&mut q).unwrap();
10644
10645 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
10646 apply_choice_actions(&queue, &questions, &home);
10647 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10648
10649 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
10650 apply_choice_actions(&queue, &questions, &home);
10651 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10652 }
10653
10654 #[test]
10655 fn action_standing_tells_the_reasons_apart() {
10656 let mut t = held_task_with("r1");
10657 let q = action_question("r1", ask::ChoiceAction::Requeue);
10658 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
10659 let mut plain = q.clone();
10660 plain.actions.clear();
10661 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
10662
10663 t.mark_action_applied(&q.id);
10665 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
10666
10667 let mut t = held_task_with("r1");
10669 t.start("r2".to_owned());
10670 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
10671 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
10673 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10674 t.status = TaskStatus::Blocked;
10675 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10676
10677 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
10679 c.node = crate::conduct::NODE.to_owned();
10680 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
10681 assert!(!ActionStanding::Busy.daemon_owns());
10682 }
10683
10684 #[test]
10685 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
10686 let dir = tempfile::tempdir().unwrap();
10687 let queue = Queue::at(dir.path().join("queue"));
10688 let questions = Questions::at(dir.path().join("questions"));
10689 let home = dir.path().join("home");
10690 let mut state = run_state(RunStatus::Blocked);
10691 state.id = "20260101-000000-act4".to_owned();
10692 state.save_under(&home).unwrap();
10693 let mut t = held_task_with(&state.id);
10694 t.status = TaskStatus::Running;
10695 queue.put(&mut t).unwrap();
10696 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
10697 q.cwd = Some(dir.path().display().to_string());
10698 questions.put(&mut q).unwrap();
10699
10700 let running = queue.get(&t.id).unwrap();
10703 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
10704 assert!(matches!(
10705 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
10706 crate::waiter::Action::Deliver(_)
10707 ));
10708 apply_choice_actions(&queue, &questions, &home);
10710 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10711 let mut back = queue.get(&t.id).unwrap();
10712 back.hold_machine(Some("later".into()));
10713 queue.put(&mut back).unwrap();
10714 apply_choice_actions(&queue, &questions, &home);
10715 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
10716 let mut again = queue.get(&t.id).unwrap();
10717 again.hold_machine(Some("again".into()));
10718 queue.put(&mut again).unwrap();
10719 apply_choice_actions(&queue, &questions, &home);
10720 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10721 }
10722}