1use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::sync::atomic::{AtomicBool, Ordering};
52use std::sync::{Mutex, MutexGuard};
53use std::time::Duration;
54
55use anyhow::{Context, Result, bail};
56use jiff::Timestamp;
57use serde::{Deserialize, Serialize};
58use tokio::sync::Notify;
59
60use crate::ask::{self, Questions};
61use crate::clean;
62use crate::conduct::Conductor;
63use crate::config::{Config, MergeMode};
64use crate::graph::Runner;
65use crate::land;
66use crate::notices::{self, Link, Notice};
67use crate::queue::{Queue, Task, TaskStatus};
68use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
69use crate::triage;
70
71pub const SCHEMA: u32 = 1;
73
74pub const HEARTBEAT: Duration = Duration::from_secs(5);
78
79pub const STALE_SECS: i64 = 30;
88
89pub const POLL: Duration = Duration::from_secs(5);
91
92pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
96
97pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
119#[serde(default)]
120pub struct Current {
121 pub task: String,
123 pub run: String,
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct Status {
135 pub schema: u32,
137 pub pid: u32,
139 pub started_at: Timestamp,
141 pub updated_at: Timestamp,
143 pub idle: bool,
145 pub current: Vec<Current>,
151 pub completed: usize,
153 pub polls: u64,
155 #[serde(default)]
157 pub question_recovery: Option<crate::question_delivery::Recovery>,
158}
159
160impl Status {
161 #[must_use]
163 pub fn new() -> Self {
164 let now = Timestamp::now();
165 Self {
166 schema: SCHEMA,
167 pid: std::process::id(),
168 started_at: now,
169 updated_at: now,
170 idle: true,
171 current: Vec::new(),
172 completed: 0,
173 polls: 0,
174 question_recovery: None,
175 }
176 }
177}
178
179impl Default for Status {
180 fn default() -> Self {
181 Self::new()
182 }
183}
184
185#[derive(Debug, Clone)]
187pub struct Opts {
188 pub repo: PathBuf,
190 pub config: Option<PathBuf>,
192 pub poll: Duration,
194 pub max_attempts: usize,
196 pub once: bool,
198 pub merge: Option<String>,
200 pub worktrees_root: Option<PathBuf>,
209}
210
211impl Default for Opts {
212 fn default() -> Self {
213 Self {
214 repo: PathBuf::from("."),
215 config: None,
216 poll: POLL,
217 max_attempts: 2,
218 once: false,
219 merge: None,
220 worktrees_root: None,
221 }
222 }
223}
224
225fn max_concurrent(n: usize) -> usize {
230 n.max(1)
231}
232
233#[must_use]
235pub fn status_path() -> PathBuf {
236 crate::run::home().join("daemon.json")
237}
238
239pub fn write_status(status: &Status) -> Result<()> {
241 write_status_to(&status_path(), status)
242}
243
244pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
249 if let Some(parent) = path.parent() {
250 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
251 }
252 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
253 let tmp = path.with_extension("json.tmp");
254 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
255 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
256 Ok(())
257}
258
259pub fn clear_status() {
262 clear_status_at(&status_path());
263}
264
265fn clear_status_at(path: &Path) {
269 let _ = std::fs::remove_file(path);
270}
271
272#[derive(Debug, Clone, Default)]
285pub struct Stop {
286 stopped: Arc<AtomicBool>,
290 busy: Arc<std::sync::atomic::AtomicUsize>,
296 wake: Arc<Notify>,
300 pause: crate::graph::Pause,
303}
304
305impl Stop {
306 #[must_use]
308 pub fn new() -> Self {
309 Self::default()
310 }
311
312 pub fn stop(&self) {
315 self.stopped.store(true, Ordering::SeqCst);
316 self.wake.notify_one();
320 }
321
322 #[must_use]
324 pub fn stopped(&self) -> bool {
325 self.stopped.load(Ordering::SeqCst)
326 }
327
328 #[must_use]
336 pub fn finishing(&self) -> bool {
337 self.stopped() && self.busy_now()
338 }
339
340 pub fn park(&self, cause: crate::run::ParkCause) {
351 self.pause.park_for(cause);
352 self.stop();
353 }
354
355 #[must_use]
357 pub fn parking(&self) -> bool {
358 self.pause.parked()
359 }
360
361 #[must_use]
363 pub fn pause(&self) -> crate::graph::Pause {
364 self.pause.clone()
365 }
366
367 #[must_use]
373 pub fn busy_now(&self) -> bool {
374 self.busy.load(Ordering::SeqCst) > 0
375 }
376
377 fn enter(&self) {
379 self.busy.fetch_add(1, Ordering::SeqCst);
380 }
381
382 fn exit(&self) {
385 self.busy.fetch_sub(1, Ordering::SeqCst);
386 }
387
388 async fn idle(&self, poll: Duration) {
390 tokio::select! {
391 () = tokio::time::sleep(poll) => {}
392 () = self.wake.notified() => {}
393 }
394 }
395}
396
397#[derive(Debug, Clone, Default, Deserialize)]
404#[serde(default)]
405pub struct Reading {
406 pub schema: u32,
408 pub pid: Option<u32>,
410 pub started_at: Option<Timestamp>,
412 pub updated_at: Option<Timestamp>,
414 pub idle: bool,
416 #[serde(deserialize_with = "de_current")]
428 pub current: Vec<Current>,
429 pub completed: u64,
431 pub polls: u64,
433 #[serde(default)]
435 pub question_recovery: Option<crate::question_delivery::Recovery>,
436}
437
438fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
441where
442 D: serde::Deserializer<'de>,
443{
444 #[derive(Deserialize)]
445 #[serde(untagged)]
446 enum Shape {
447 Many(Vec<Current>),
448 One(Current),
449 }
450 Ok(
451 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
452 Shape::Many(v) => v,
453 Shape::One(c) => vec![c],
454 }),
455 )
456}
457
458impl Reading {
459 #[must_use]
462 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
463 self.updated_at
464 .map(|at| (now.as_second() - at.as_second()).max(0))
465 }
466
467 #[must_use]
471 pub fn running(&self, now: Timestamp) -> bool {
472 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
473 }
474}
475
476#[must_use]
483pub fn read_status(home: &Path) -> Option<Reading> {
484 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
485 serde_json::from_str(&body).ok()
486}
487
488#[must_use]
500pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
501 read_status(home)
502 .filter(|reading| reading.running(now))
503 .map(|reading| reading.current)
504 .unwrap_or_default()
505}
506
507#[must_use]
509pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
510 current_work(home, now).iter().any(|c| c.run == run)
511}
512
513#[must_use]
523pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
524 current_work(home, now)
525 .iter()
526 .any(|c| crate::run::short_of(&c.run) == short)
527}
528
529#[must_use]
531pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
532 current_work(home, now).iter().any(|c| c.task == task)
533}
534
535pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
579 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
580}
581
582fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
586where
587 F: Fn(u32) -> bool,
588{
589 let this_process = std::process::id();
590 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
591 .into_iter()
592 .flatten()
593 .flatten()
594 .map(|e| e.path())
595 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
596 .filter(|p| {
597 match std::fs::read_to_string(p)
598 .ok()
599 .and_then(|body| body.trim().parse::<u32>().ok())
600 {
601 Some(pid) if pid == this_process => false,
605 Some(pid) => !pid_alive(pid),
606 None => p
607 .metadata()
608 .and_then(|m| m.modified())
609 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
610 .is_ok_and(|age| age >= older_than),
611 }
612 })
613 .filter(|p| std::fs::remove_file(p).is_ok())
614 .filter_map(|p| {
615 p.file_stem()
616 .and_then(|s| s.to_str())
617 .map(std::borrow::ToOwned::to_owned)
618 })
619 .collect();
620 swept.sort_unstable();
621 swept
622}
623
624fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
629 task.status == TaskStatus::Running
630 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
631 && !is_working_on_task(home, &task.id, now)
632}
633
634fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
637 queue
638 .list()
639 .into_iter()
640 .filter(|t| is_stalled(t, home, now))
641 .collect()
642}
643
644fn queued_tasks(queue: &Queue) -> Vec<Task> {
650 queue
651 .list()
652 .into_iter()
653 .filter(|t| t.status == TaskStatus::Queued)
654 .collect()
655}
656
657fn finished_tasks(queue: &Queue) -> Vec<Task> {
660 queue
661 .list()
662 .into_iter()
663 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
664 .collect()
665}
666
667fn resolve_blockers(queue: &Queue, questions: &Questions) {
684 for listed in queue.list() {
685 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
686 continue;
687 }
688 let Ok(_claim) = queue.claim(&listed.id) else {
689 continue;
690 };
691 let Ok(mut task) = queue.get(&listed.id) else {
692 continue;
693 };
694 if task.status != TaskStatus::Blocked {
695 continue;
696 }
697 let deleted = queue.apply_deleted_blockers(&mut task, questions);
701 if !deleted.is_empty() {
702 record(queue, &mut task);
703 for id in &deleted {
704 queue.note_dependency_deleted(&task, id);
705 }
706 if task.status != TaskStatus::Blocked {
707 continue;
708 }
709 }
710 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
711 if !missing.is_empty() {
712 let language = language_of(&task, Path::new("."));
713 let mut cause = missing.clone();
714 cause.sort();
715 cause.dedup();
716 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
717 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
718 &task.blocked_by,
719 &missing,
720 &language,
721 )));
722 record(queue, &mut task);
723 continue;
724 }
725 if let Some(q) = task.blocked_by.iter().find_map(|id| {
728 let crate::queue::Dependency::Question(full) =
729 crate::queue::dependency(queue, questions, id)
730 else {
731 return None;
732 };
733 questions.get(&full).ok().filter(|q| {
734 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
735 })
736 }) {
737 let language = language_of(&task, Path::new("."));
738 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
739 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
740 record(queue, &mut task);
741 continue;
742 }
743 let mut changed = false;
744 for id in task.blocked_by.clone() {
745 let resolved = crate::queue::dependency(queue, questions, &id);
746 if let crate::queue::Dependency::Task(full) = &resolved {
747 if queue
748 .get(full)
749 .is_ok_and(|dep| dep.status == TaskStatus::Done)
750 {
751 task.unblock(&id);
752 changed = true;
753 }
754 continue;
755 }
756 if let crate::queue::Dependency::Question(full) = resolved
757 && let Ok(q) = questions.get(&full)
758 && q.status == ask::QuestionStatus::Answered
759 {
760 if q.node == crate::conduct::NODE
764 && q.chosen_action().is_some()
765 && !task.action_applied(&q.id)
766 {
767 continue;
768 }
769 let answer = match &q.answer {
770 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
771 None => String::new(),
772 };
773 task.record_answer(q.summary.clone(), answer);
774 task.unblock(&id);
775 changed = true;
776 }
777 }
778 if changed {
779 record(queue, &mut task);
780 }
781 }
782}
783
784fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
786 if crate::lang::is_japanese(language) {
787 format!(
788 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
789 q.short(),
790 q.summary
791 )
792 } else {
793 format!(
794 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
795 q.short(),
796 q.summary
797 )
798 }
799}
800
801#[derive(Debug, Clone, Copy, PartialEq, Eq)]
804pub(crate) enum ActionStanding {
805 NoAction,
807 Pending,
809 Applied,
812 Stale,
814 Busy,
816}
817
818impl ActionStanding {
819 pub(crate) fn daemon_owns(self) -> bool {
821 matches!(self, Self::Pending | Self::Applied | Self::Stale)
822 }
823}
824
825pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
829 if q.chosen_action().is_none() {
830 ActionStanding::NoAction
831 } else if task.action_applied(&q.id) {
832 ActionStanding::Applied
833 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
834 ActionStanding::Stale
835 } else if matches!(
836 task.status,
837 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
838 ) {
839 ActionStanding::Busy
840 } else {
841 ActionStanding::Pending
842 }
843}
844
845#[derive(Debug, Clone, PartialEq, Eq)]
847enum ActionDecision {
848 Skip,
850 Resume(String),
852 Requeue,
854 Hold,
856 Done,
858 Stale,
861 Refuse(String),
864}
865
866fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
874where
875 F: FnOnce(&str) -> Result<RunState>,
876{
877 let Some(action) = q.chosen_action() else {
878 return ActionDecision::Skip;
879 };
880 if task.action_applied(&q.id)
881 || matches!(
882 task.status,
883 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
884 )
885 {
886 return ActionDecision::Skip;
887 }
888 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
891 return ActionDecision::Stale;
892 }
893 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
894 let stopping = q.node == land::APPROVAL_NODE
895 && q.resolution()
896 .as_deref()
897 .is_some_and(crate::merge_decision::stops);
898 let decision_at = if repair || stopping {
902 q.answered_at.unwrap_or(q.asked_at)
903 } else {
904 q.asked_at
905 };
906 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
907 return if repair || stopping {
908 ActionDecision::Refuse(format!(
909 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
910 if repair { "repair" } else { "stop" },
911 q.id,
912 q.run,
913 task.id
914 ))
915 } else {
916 ActionDecision::Stale
917 };
918 }
919 match action {
920 a if a.maintenance() => ActionDecision::Skip,
921 ask::ChoiceAction::ResumeRun { .. }
922 | ask::ChoiceAction::ReleaseTask { .. }
923 | ask::ChoiceAction::RerunChecks { .. }
924 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
925 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
926 ask::ChoiceAction::Hold => ActionDecision::Hold,
927 ask::ChoiceAction::Done => ActionDecision::Done,
928 ask::ChoiceAction::Resume { run } => {
929 if task.runs.last() != Some(run) {
930 return ActionDecision::Refuse((p.resume_not_latest)(
931 q.short(),
932 ask::short_id(run),
933 ));
934 }
935 match load(run) {
936 Ok(s) if stopping => {
937 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
938 || !s
939 .land_approval
940 .as_ref()
941 .is_some_and(|binding| binding.question == q.id)
942 {
943 ActionDecision::Stale
944 } else if s.liveness(false) == Liveness::Live {
945 ActionDecision::Skip
946 } else if s.status == RunStatus::Landing
947 && !s.released()
948 && s.retired_to.is_none()
949 {
950 ActionDecision::Resume(run.clone())
951 } else {
952 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
953 }
954 }
955 Ok(s)
956 if q.node == land::APPROVAL_NODE
957 && q.resolution().as_deref() == Some(land::REPAIR) =>
958 {
959 if s.merge_repairs.iter().any(|r| r.question == q.id)
960 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
961 {
962 ActionDecision::Stale
965 } else if s.liveness(false) == Liveness::Live {
966 ActionDecision::Skip
967 } else if s.status != RunStatus::Landing
968 || s.released()
969 || s.reviews.is_empty()
970 || !implementers_match(task, &s)
971 || !s
972 .land_approval
973 .as_ref()
974 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
975 {
976 ActionDecision::Refuse(format!(
977 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
978 q.id, run, run
979 ))
980 } else {
981 ActionDecision::Resume(run.clone())
982 }
983 }
984 Ok(s) if s.liveness(false) == Liveness::Live => {
985 ActionDecision::Refuse("run already has a live driver".into())
986 }
987 Ok(s) if crate::handover::can_resume(&s) => ActionDecision::Resume(run.clone()),
988 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
989 q.short(),
990 ask::short_id(run),
991 )),
992 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
993 q.short(),
994 ask::short_id(run),
995 &format!("{e:#}"),
996 )),
997 }
998 }
999 }
1000}
1001
1002struct Phrases {
1016 graph_stopped: fn(&str, &str) -> String,
1018 quorum_lost: &'static str,
1019 quota_took_out: &'static str,
1021 run_ended: &'static str,
1023 waiting_for_answer: &'static str,
1025 recovered_running: &'static str,
1026 no_run_to_recover: &'static str,
1027 could_not_start: &'static str,
1028 resume_not_latest: fn(&str, &str) -> String,
1029 resume_cannot_progress: fn(&str, &str) -> String,
1030 resume_unreadable: fn(&str, &str, &str) -> String,
1031 handover_refused: fn(&crate::handover::Refused) -> String,
1034 handover_hint: &'static str,
1036 reviewers_never_answered: fn(usize, usize) -> String,
1039}
1040
1041const PHRASES_EN: Phrases = Phrases {
1042 graph_stopped: |status, detail| {
1043 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1044 },
1045 quorum_lost: "the judging panel lost its quorum",
1046 quota_took_out: "; quota took out ",
1047 run_ended: "run ended ",
1048 waiting_for_answer: " - waiting for operator answer to question ",
1049 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1050 no_run_to_recover: "task was `running` with no live daemon and no readable \
1051 run to recover; held for a human to check what happened",
1052 could_not_start: "could not start the run: ",
1053 resume_not_latest: |q, run| {
1054 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1055 },
1056 resume_cannot_progress: |q, run| {
1057 format!("question {q} asked to resume run {run}, which cannot make progress")
1058 },
1059 resume_unreadable: |q, run, e| {
1060 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1061 },
1062 handover_refused: |r| r.to_string(),
1063 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1064 reviewers_never_answered: |missing, rounds| {
1065 format!(
1066 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1067 refusing to call it clean"
1068 )
1069 },
1070};
1071
1072const PHRASES_JA: Phrases = Phrases {
1073 graph_stopped: |status, detail| {
1074 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1075 },
1076 quorum_lost: "審査パネルが定足数を失いました",
1077 quota_took_out: "。クォータで脱落: ",
1078 run_ended: "run 終了: ",
1079 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1080 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1081 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1082 何が起きたか人が確認するため保留にしました",
1083 could_not_start: "run を開始できませんでした: ",
1084 resume_not_latest: |q, run| {
1085 format!(
1086 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1087 )
1088 },
1089 resume_cannot_progress: |q, run| {
1090 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1091 },
1092 resume_unreadable: |q, run, e| {
1093 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1094 },
1095 handover_refused: |r| {
1096 use crate::handover::Refused;
1097 match r {
1098 Refused::Foreign { branch, path, why } => format!(
1099 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1100 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1101 から、やり直してください(詳細: {why})"
1102 ),
1103 Refused::Unsafe { branch, path, why } => format!(
1104 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1105 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1106 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1107 ),
1108 Refused::ReleaseFailed { branch, path, run } => format!(
1109 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1110 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1111 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1112 ),
1113 }
1114 },
1115 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1116 reviewers_never_answered: |missing, rounds| {
1117 format!(
1118 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1119 クリーンとは認めません"
1120 )
1121 },
1122};
1123
1124fn phrases(language: &str) -> &'static Phrases {
1125 if crate::lang::is_japanese(language) {
1126 &PHRASES_JA
1127 } else {
1128 &PHRASES_EN
1129 }
1130}
1131
1132fn language_of(task: &Task, fallback: &Path) -> String {
1136 crate::lang::of_repo(&repo_for(task, fallback))
1137}
1138
1139pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1143 if q.node == crate::conduct::NODE {
1144 return tasks.iter().find(|t| t.id == q.run);
1145 }
1146 tasks.iter().find(|t| t.runs.contains(&q.run))
1147}
1148
1149fn repair_choice_offers(questions: &Questions) {
1152 for q in questions.list().into_iter().filter(|q| {
1153 q.status.open()
1154 && q.node == crate::conduct::NODE
1155 && q.choices
1156 .iter()
1157 .any(|c| crate::choice_effect::describe(q, c).is_none())
1158 }) {
1159 let _ = questions.update(&q.id, |current| {
1160 if current.status.open() {
1161 crate::choice_effect::executable_conductor(current);
1162 if let Some(deputy) = &mut current.deputy {
1163 deputy.brief = crate::deputy::brief(
1164 ¤t.run,
1165 ¤t.detail,
1166 ¤t.choices,
1167 ¤t.actions,
1168 );
1169 }
1170 }
1171 Ok(())
1172 });
1173 }
1174}
1175
1176pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1179 apply_maintenance_choices(queue, questions, home).await;
1180 apply_task_choice_actions(queue, questions, home);
1181}
1182
1183async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1186 for q in questions.list() {
1187 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1188 continue;
1189 };
1190 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1193 continue;
1194 }
1195 let dir = home.join("choice-actions");
1196 if std::fs::create_dir_all(&dir).is_err() {
1197 continue;
1198 }
1199 let Ok(lock) = std::fs::OpenOptions::new()
1200 .create(true)
1201 .truncate(false)
1202 .write(true)
1203 .open(dir.join(format!("{}.lock", q.id)))
1204 else {
1205 continue;
1206 };
1207 if crate::pr_head::try_lock_file(&lock).is_err() {
1208 continue;
1209 }
1210 let fresh = !home
1211 .join("choice-actions")
1212 .join(format!("{}.json", q.id))
1213 .exists();
1214 let result = async {
1215 let receipt = dir.join(format!("{}.json", q.id));
1216 if receipt.exists() {
1217 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1218 anyhow::ensure!(
1219 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1220 "previous maintenance attempt did not confirm completion; inspect its receipt"
1221 );
1222 return Ok::<_, anyhow::Error>(());
1223 }
1224 let record = |status: &str| -> anyhow::Result<()> {
1227 let tmp = receipt.with_extension("tmp");
1228 std::fs::write(
1229 &tmp,
1230 serde_json::to_vec(&serde_json::json!({
1231 "action": action, "status": status,
1232 }))?,
1233 )?;
1234 std::fs::rename(tmp, &receipt)?;
1235 Ok(())
1236 };
1237 let applied = async {
1238 match action {
1239 ask::ChoiceAction::ResumeRun { run } => {
1240 crate::resume::validate(home, run)?;
1241 record("started")?;
1242 if !crate::resume::pin_queued(queue, run, Some(q.asked_at), Some(&q.id))? {
1243 crate::resume::validate_unqueued(home, run)?;
1244 crate::resume::start(run.clone());
1245 }
1246 }
1247 ask::ChoiceAction::ReleaseTask {
1248 task,
1249 held_at,
1250 reason,
1251 } => {
1252 anyhow::ensure!(
1253 queue.resolve_id(task)? == *task,
1254 "task action requires an exact task ID"
1255 );
1256 let _claim = queue.claim(task)?;
1257 let mut task = queue.get(task)?;
1258 if task.action_applied(&q.id) {
1259 return Ok(());
1260 }
1261 if q.node == crate::conduct::NODE
1266 && q.run == task.id
1267 && task.status == TaskStatus::Blocked
1268 && task.blocked_from == Some(TaskStatus::Held)
1269 && task.blocked_by.as_slice() == [q.id.as_str()]
1270 {
1271 task.unblock(&q.id);
1272 }
1273 anyhow::ensure!(
1274 task.status == TaskStatus::Held
1275 && task.held_at == Some(*held_at)
1276 && task.hold_reason.as_deref() == reason.as_deref()
1277 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at),
1278 "task no longer has the hold this question names"
1279 );
1280 task.release_operator("choice", Some(&q.id));
1281 task.mark_action_applied(&q.id);
1282 queue.put(&mut task)?;
1283 }
1284 ask::ChoiceAction::AbandonRun { run } => {
1285 record("started")?;
1286 crate::driver::abandon_proven_dead(home, run)?;
1287 }
1288 ask::ChoiceAction::RerunChecks {
1289 repo,
1290 pr,
1291 head,
1292 runs,
1293 } => {
1294 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1295 .await?;
1296 }
1297 _ => unreachable!(),
1298 }
1299 Ok::<_, anyhow::Error>(())
1300 }
1301 .await;
1302 match applied {
1303 Ok(()) => record("applied"),
1304 Err(e) => {
1305 record(&format!("refused or failed: {e:#}"))?;
1306 Err(e)
1307 }
1308 }
1309 }
1310 .await;
1311 if home
1316 .join("choice-actions")
1317 .join(format!("{}.json", q.id))
1318 .exists()
1319 && let Some(listed) = task_of_question(&queue.list(), &q)
1320 && !listed.action_applied(&q.id)
1321 && let Ok(_claim) = queue.claim(&listed.id)
1322 && let Ok(mut origin) = queue.get(&listed.id)
1323 {
1324 if !origin.operator_held()
1325 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1326 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1327 {
1328 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1329 origin.hold_machine(Some(format!(
1330 "Maintenance choice {} settled; inspect its target before releasing this task",
1331 q.id
1332 )));
1333 } else {
1334 origin.unblock(&q.id);
1335 }
1336 }
1337 origin.mark_action_applied(&q.id);
1338 let _ = queue.put(&mut origin);
1339 }
1340 if let Err(e) = result
1341 && fresh
1342 {
1343 notices::raise_in(
1344 home,
1345 Notice::warn(
1346 &format!("action:{}", q.id),
1347 format!("Choice action {} was not completed: {e:#}", q.id),
1348 ),
1349 );
1350 }
1351 }
1352}
1353
1354pub(crate) async fn rerun_checks_once(
1361 home: &Path,
1362 request: &str,
1363 repo: &Path,
1364 pr: &str,
1365 head: &str,
1366 runs: &[String],
1367 started: impl FnOnce() -> anyhow::Result<()>,
1368) -> anyhow::Result<()> {
1369 use crate::release_watch::{GhForge, ReleaseForge};
1370 anyhow::ensure!(
1371 !runs.is_empty()
1372 && runs.len() <= 20
1373 && runs
1374 .iter()
1375 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1376 && !pr.starts_with('-'),
1377 "invalid rerun targets"
1378 );
1379 let snap = GhForge.snapshot(repo, pr).await?;
1380 anyhow::ensure!(
1381 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1382 "pull request head or lifecycle changed"
1383 );
1384 anyhow::ensure!(
1385 runs.iter().all(|r| snap
1386 .checks
1387 .iter()
1388 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1389 "named workflow is no longer failed on this pull request"
1390 );
1391 started()?;
1392 let dir = home.join("choice-actions");
1393 std::fs::create_dir_all(&dir)?;
1394 let rerun_lock = std::fs::OpenOptions::new()
1395 .create(true)
1396 .truncate(false)
1397 .write(true)
1398 .open(dir.join("reruns.lock"))?;
1399 crate::pr_head::try_lock_file(&rerun_lock)?;
1400 let ledger = dir.join("reruns.json");
1401 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1402 serde_json::from_slice(&std::fs::read(&ledger)?)?
1403 } else {
1404 Default::default()
1405 };
1406 for run in runs {
1407 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1408 if !recorded.insert(key) {
1409 continue;
1410 }
1411 let tmp = ledger.with_extension("tmp");
1412 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1413 std::fs::rename(tmp, &ledger)?;
1414 GhForge.rerun(repo, run).await?;
1415 }
1416 Ok(())
1417}
1418
1419pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1428 let tasks = queue.list();
1429 for q in questions.list() {
1430 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1432 continue;
1433 }
1434 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1435 continue;
1436 }
1437 let Some(listed) = task_of_question(&tasks, &q) else {
1438 if q.node == land::APPROVAL_NODE
1439 && q.resolution().as_deref().is_some_and(|choice| {
1440 choice == land::REPAIR || crate::merge_decision::stops(choice)
1441 })
1442 {
1443 notices::raise_in(
1444 home,
1445 Notice::warn(
1446 &format!("repair-owner:{}", q.id),
1447 format!(
1448 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1449 q.run, q.run
1450 ),
1451 ),
1452 );
1453 }
1454 continue;
1455 };
1456 if listed.action_applied(&q.id) {
1457 continue;
1458 }
1459 let Ok(_claim) = queue.claim(&listed.id) else {
1460 continue;
1461 };
1462 let Ok(mut task) = queue.get(&listed.id) else {
1463 continue;
1464 };
1465 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1468 let blockers: Vec<_> = task
1469 .blocked_by
1470 .iter()
1471 .filter(|id| {
1472 crate::queue::dependency(queue, questions, id)
1473 == crate::queue::Dependency::Question(q.id.clone())
1474 })
1475 .cloned()
1476 .collect();
1477 if !blockers.is_empty() {
1478 for id in blockers {
1479 task.unblock(&id);
1480 }
1481 let answer = match &q.answer {
1482 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1483 None => String::new(),
1484 };
1485 task.record_answer(q.summary.clone(), answer);
1486 }
1487 }
1488 let language = language_of(&task, Path::new("."));
1489 let decision = decide_action(&task, &q, phrases(&language), |id| {
1490 RunState::load_under(id, home)
1491 });
1492 let ran = matches!(
1493 decision,
1494 ActionDecision::Resume(_)
1495 | ActionDecision::Requeue
1496 | ActionDecision::Hold
1497 | ActionDecision::Done
1498 );
1499 if ran && asker_may_still_read(questions, &q, home) {
1500 continue;
1509 }
1510 let done = matches!(decision, ActionDecision::Done);
1511 match decision {
1512 ActionDecision::Skip => continue,
1513 ActionDecision::Resume(run) => {
1514 task.release_operator("choice", Some(&q.id));
1515 task.resume_override = Some(crate::queue::OperatorResume {
1516 question_id: q.id.clone(),
1517 at: Timestamp::now(),
1518 conductor_rehold: None,
1519 forced: true,
1520 pinned_run: Some(run),
1521 });
1522 }
1523 ActionDecision::Stale => {}
1524 ActionDecision::Requeue => {
1525 task.release_operator("choice-requeue", Some(&q.id));
1526 task.requeue();
1527 }
1528 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1529 ActionDecision::Done => task.succeed(),
1530 ActionDecision::Refuse(why) => {
1531 if task.operator_held() {
1532 task.hold_reason = Some(match task.hold_reason.take() {
1535 Some(reason) => format!("{reason}\n\n{why}"),
1536 None => why,
1537 });
1538 } else {
1539 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1540 task.hold_machine(Some(why));
1541 }
1542 notices::raise(
1543 Notice::warn(
1544 &format!("action:{}", q.id),
1545 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1546 )
1547 .link(Link::Task {
1548 id: task.id.clone(),
1549 }),
1550 );
1551 }
1552 }
1553 task.mark_action_applied(&q.id);
1554 record(queue, &mut task);
1557 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1558 supersede_prior_runs(&task, home);
1559 }
1560 let _ = questions.update(&q.id, |r| {
1563 r.answer_delivered = true;
1564 Ok(())
1565 });
1566 }
1567}
1568
1569fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1573 let now = Timestamp::now();
1574 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1575 return true;
1576 }
1577 q.node != crate::conduct::NODE
1578 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1579 s.seats_active()
1580 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1581 })
1582}
1583
1584fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1595 let tasks = queue.list();
1596 let by_id: std::collections::BTreeMap<&str, &Task> =
1597 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1598 let referenced: Vec<&str> = tasks
1599 .iter()
1600 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1601 .collect();
1602
1603 for mut question in questions.list() {
1604 if !question.status.open() || question.node != crate::conduct::NODE {
1605 continue;
1606 }
1607 if referenced
1612 .iter()
1613 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1614 {
1615 continue;
1616 }
1617 let Some(task) = by_id.get(question.run.as_str()) else {
1618 continue;
1619 };
1620 question.abandon(format!(
1621 "task {} no longer waits for this answer",
1622 task.short()
1623 ));
1624 if let Err(e) = questions.put(&mut question) {
1625 tracing::warn!(
1626 "could not retire question {} for task {}: {e:#}",
1627 question.short(),
1628 task.short()
1629 );
1630 }
1631 }
1632}
1633
1634#[derive(Debug, Clone, Copy)]
1641pub struct Verdict {
1642 pub status: RunStatus,
1644 pub left_pr: bool,
1646 pub quota_hit: bool,
1648 pub parked: bool,
1650 pub no_viable_candidates: bool,
1657}
1658
1659pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1716 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1717}
1718
1719fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1721 if verdict.parked {
1727 task.park(detail);
1728 return;
1729 }
1730 match verdict.status {
1731 RunStatus::Merged | RunStatus::Ready
1732 if task.pr_recovery.as_ref().is_some_and(|s| {
1733 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1734 }) =>
1735 {
1736 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1737 }
1738 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1739 RunStatus::AlreadyInBase => task.already_landed(detail),
1740 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1741 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1742 task.stall(detail)
1743 }
1744 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1745 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1746 RunStatus::Blocked => task.fail(detail, max_attempts),
1747 RunStatus::VerifiedNoop => task.handed_off(detail),
1748 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1749 }
1750}
1751
1752pub fn supersede_prior_runs(task: &Task, home: &Path) {
1801 let now = Timestamp::now();
1802 let last_run_succeeded = task
1809 .runs
1810 .last()
1811 .and_then(|id| RunState::load_under(id, home).ok())
1812 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1813 for id in task.superseded_attempts(last_run_succeeded) {
1814 let mut state = match RunState::load_under(id, home) {
1815 Ok(s) => s,
1816 Err(e) => {
1817 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1818 continue;
1819 }
1820 };
1821 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1822 continue;
1823 }
1824 let daemon_claims = is_working_on(home, id, now);
1825 if state.liveness(daemon_claims) == Liveness::Live {
1826 continue;
1827 }
1828 state.status = RunStatus::Superseded;
1829 if let Err(e) = state.save_under(home) {
1830 tracing::warn!("could not mark run {id} superseded: {e:#}");
1831 }
1832 }
1833}
1834
1835fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1857 for task in queue.list() {
1858 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1859 continue;
1860 }
1861 supersede_prior_runs(&task, home);
1862 }
1863}
1864
1865fn settle_and_diagnose(
1872 task: &mut Task,
1873 verdict: Verdict,
1874 detail: &str,
1875 max_attempts: usize,
1876 state: &RunState,
1877) {
1878 let p = phrases(&state.config.graph.language);
1879 if let Some(d) = &state.merge_decision {
1882 use crate::merge_decision as md;
1883 if task.runs.last().is_some_and(|id| id != &state.id) {
1884 return;
1885 }
1886 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1887 task.succeed();
1888 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1889 if state.liveness(false) != Liveness::Dead {
1890 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1891 return;
1892 }
1893 let replacement = md::Replacement {
1894 question: d.question.clone(),
1895 run: state.id.clone(),
1896 pr: d.target.pr.clone(),
1897 agent: agent.to_owned(),
1898 context: format!(
1899 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1900 d.target.pr,
1901 state.id,
1902 serde_json::to_string(&state.reviews).unwrap_or_default()
1903 ),
1904 };
1905 task.requeue();
1906 task.pr_recovery = None;
1907 task.review_of = None;
1908 task.replacement = Some(replacement);
1909 } else {
1910 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1911 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1912 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1913 }
1914 task.mark_action_applied(&d.question);
1915 if let Some(pass) = state.passes.last() {
1916 task.record_pass_at(
1917 &state.id,
1918 pass.index,
1919 Some(pass.started_at),
1920 task.attempts,
1921 None,
1922 );
1923 }
1924 return;
1925 }
1926 if let Some(replacement) = &task.replacement
1927 && !state.parked
1928 && state.resource_block.is_none()
1929 && (state.viable().is_empty()
1930 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1931 {
1932 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1933 return;
1934 }
1935 let attempts_before = task.attempts;
1936 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1942 task.notice_cause(
1943 &format!("resource:{}:{}", block.kind, block.node),
1944 None,
1945 None,
1946 );
1947 task.last_error = Some(detail.to_owned());
1948 task.diagnostic = None;
1949 task.attempts = task.attempts.saturating_sub(1);
1950 if block.kind == "disk pressure" {
1951 task.hold_machine(Some(format!(
1952 "{}{}",
1953 crate::disk::RUNNING_PRESSURE,
1954 block.detail
1955 )));
1956 } else {
1957 task.hold_machine(Some(format!(
1958 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1959 no agent was asked and no attempt was spent",
1960 crate::cache::RESOURCE_HOLD,
1961 block.kind,
1962 block.node
1963 )));
1964 }
1965 if let Some(pass) = state.passes.last() {
1966 task.record_pass_at(
1967 &state.id,
1968 pass.index,
1969 Some(pass.started_at),
1970 attempts_before,
1971 None,
1972 );
1973 }
1974 return;
1975 }
1976 let unresolved = task.hold_notice_cause.clone();
1977 if !verdict.parked
1978 && task
1979 .pr_recovery
1980 .as_ref()
1981 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
1982 && matches!(
1983 verdict.status,
1984 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
1985 )
1986 {
1987 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
1988 task.succeed();
1989 } else {
1990 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
1991 }
1992 } else if !verdict.parked
1993 && task.pr_recovery.is_some()
1994 && matches!(
1995 verdict.status,
1996 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
1997 )
1998 && !verdict.quota_hit
1999 {
2000 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2001 } else {
2002 settle_in(task, verdict, detail, max_attempts, p);
2003 }
2004 if !verdict.parked
2005 && !verdict.quota_hit
2006 && matches!(
2007 verdict.status,
2008 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2009 )
2010 {
2011 task.recovery_failed(state);
2012 }
2013 if state.prep_failure.is_some() {
2014 task.hold_notice_cause = unresolved;
2017 task.startup_notice_cause("startup-candidates");
2018 }
2019 if let Some(pass) = state.passes.last() {
2022 let parked = match &pass.end {
2023 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2024 _ => None,
2025 };
2026 task.record_pass_at(
2027 &state.id,
2028 pass.index,
2029 Some(pass.started_at),
2030 attempts_before,
2031 parked,
2032 );
2033 }
2034 if task.status == TaskStatus::Held {
2035 if state.prep_failure.is_none() {
2036 task.notice_cause(
2037 label(verdict.status),
2038 Some(&state.id),
2039 state.passes.last().map(|p| p.index),
2040 );
2041 }
2042 task.diagnostic = diagnostic(state);
2043 note_open_question(task, &state.id, p);
2044 }
2045}
2046
2047fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2062 let Some(home) = crate::run::try_home() else {
2063 return;
2064 };
2065 let open = Questions::at(home.join("questions")).open_for(run);
2066 let Some(q) = open.first() else {
2067 return;
2068 };
2069 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2070 decisions.sort_unstable();
2071 task.notice_cause(
2072 &format!("questions:{}", decisions.join(",")),
2073 Some(run),
2074 None,
2075 );
2076 let base = task.hold_reason.clone().unwrap_or_default();
2077 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2078}
2079
2080fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2091 match last_run {
2092 Some(state) => {
2093 let verdict = Verdict {
2094 status: state.status,
2095 left_pr: state.pr.is_some(),
2096 quota_hit: !state.quota.is_empty(),
2097 parked: state.parked,
2098 no_viable_candidates: state.viable().is_empty(),
2099 };
2100 let detail = format!(
2101 "{}{}",
2102 phrases(&state.config.graph.language).recovered_running,
2103 describe(&state)
2104 );
2105 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2106 }
2107 None => {
2108 let why = phrases(language).no_run_to_recover;
2111 task.last_error = Some(why.to_owned());
2112 task.notice_cause("missing-run", None, None);
2115 task.hold_machine(Some(why.to_owned()));
2116 }
2117 }
2118}
2119
2120fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
2142 let mut reclaimed = Vec::new();
2143 for listed in queue.list() {
2144 if listed.status != TaskStatus::Running {
2145 continue;
2146 }
2147 let Ok(_claim) = queue.claim(&listed.id) else {
2148 continue;
2149 };
2150 let Ok(mut task) = queue.get(&listed.id) else {
2154 continue;
2155 };
2156 if task.status != TaskStatus::Running {
2157 continue;
2158 }
2159 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
2160 if let Some(state) = &last_run
2170 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2171 {
2172 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2173 }
2174 let language = if last_run.is_none() {
2175 language_of(&task, Path::new("."))
2176 } else {
2177 String::new()
2178 };
2179 reclaim(&mut task, last_run, max_attempts, &language);
2180 if task.status == TaskStatus::Done {
2181 supersede_prior_runs(&task, &crate::run::home());
2182 }
2183 record(queue, &mut task);
2184 reclaimed.push(task.id.clone());
2185 }
2186 reclaimed
2187}
2188
2189fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2221 reclaim_abandoned_runs_with(
2222 home,
2223 now,
2224 crate::proc::pid_status,
2225 crate::proc::process_started_at,
2226 )
2227}
2228
2229fn reclaim_abandoned_runs_with<F, G>(
2235 home: &Path,
2236 now: Timestamp,
2237 query: F,
2238 identity: G,
2239) -> Vec<String>
2240where
2241 F: Fn(u32) -> Option<bool>,
2242 G: Fn(u32) -> Option<String>,
2243{
2244 let mut abandoned = Vec::new();
2245 for entry in std::fs::read_dir(home.join("runs"))
2246 .into_iter()
2247 .flatten()
2248 .flatten()
2249 {
2250 let id = entry.file_name().to_string_lossy().into_owned();
2251 if !crate::run::is_run_id(&id) {
2252 continue;
2253 }
2254 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2257 continue;
2258 };
2259 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2267 continue;
2268 };
2269 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2270 continue;
2271 };
2272 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2273 continue;
2274 }
2275 let daemon_claims = is_working_on(home, &id, now);
2287 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2288 != crate::run::Liveness::Dead
2289 {
2290 continue;
2291 }
2292 state.abandon("daemon");
2293 if let Err(e) = state.save_under(home) {
2294 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2295 continue;
2296 }
2297 if let Some(notice) = notices::run_ended(&state) {
2300 notices::raise_in(home, notice);
2301 }
2302 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2310 tracing::warn!("abandon questions for {id}: {e:#}");
2311 }
2312 abandoned.push(id);
2313 }
2314 abandoned
2315}
2316
2317pub async fn serve(opts: Opts) -> Result<()> {
2323 serve_until(opts, Stop::new()).await
2324}
2325
2326pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2343 let signal = {
2344 let stop = stop.clone();
2345 tokio::spawn(async move {
2346 if tokio::signal::ctrl_c().await.is_ok() {
2347 stop.stop();
2348 tracing::info!("shutdown requested; a run in flight will be finished first");
2349 }
2350 })
2351 };
2352
2353 let worktrees_root = opts
2354 .worktrees_root
2355 .clone()
2356 .unwrap_or_else(crate::run::default_worktree_root);
2357 let outcome = drive(
2358 &opts,
2359 &Queue::open(),
2360 &status_path(),
2361 &crate::run::home(),
2362 &worktrees_root,
2363 &stop,
2364 )
2365 .await;
2366
2367 signal.abort();
2368 outcome
2369}
2370
2371async fn drive(
2384 opts: &Opts,
2385 queue: &Queue,
2386 status_file: &Path,
2387 home: &Path,
2388 worktrees_root: &Path,
2389 stop: &Stop,
2390) -> Result<()> {
2391 let recovery_cfg = prepare(&opts.repo, opts).ok();
2399 let daemon_cfg = recovery_cfg
2400 .as_ref()
2401 .map(|c| c.daemon.clone())
2402 .unwrap_or_default();
2403 let reconciled = crate::driver::reconcile(home);
2404 if !reconciled.is_empty() {
2405 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2406 }
2407 let mut initial_status = Status::new();
2408 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2409 recovery_cfg.as_ref(),
2410 daemon_cfg.max_deputies,
2411 ));
2412 let status = Arc::new(Mutex::new(initial_status));
2413 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2414 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2415
2416 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2422
2423 let waiter = tokio::spawn(crate::waiter::run(
2428 crate::waiter::Waiter::new(
2429 crate::ask::Questions::at(home.join("questions")),
2430 home.to_path_buf(),
2431 recovery_cfg.clone(),
2432 ),
2433 stop.clone(),
2434 ));
2435
2436 let deputies = tokio::spawn(crate::deputy::run(
2440 crate::deputy::Deputies::new(
2441 crate::ask::Questions::at(home.join("questions")),
2442 home.to_path_buf(),
2443 recovery_cfg.clone(),
2444 opts.repo.clone(),
2445 daemon_cfg.max_deputies,
2446 {
2447 let stop = stop.clone();
2448 Arc::new(move || stop.parking())
2449 },
2450 ),
2451 stop.clone(),
2452 ));
2453
2454 let fetcher = tokio::spawn(fetch_loop(opts.repo.clone(), opts.clone(), stop.clone()));
2457
2458 let release_watch = tokio::spawn(crate::release_watch::run(
2461 crate::release_watch::Watcher::new(
2462 Box::new(crate::release_watch::GhForge),
2463 home.to_path_buf(),
2464 ),
2465 {
2466 let (repo, opts) = (opts.repo.clone(), opts.clone());
2467 move || release_watch_settings(&repo, &opts)
2468 },
2469 stop.clone(),
2470 ));
2471
2472 tracing::info!(
2473 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2474 queue.root().display(),
2475 opts.poll.as_secs(),
2476 opts.max_attempts,
2477 concurrency,
2478 if daemon_cfg.pause_for_interrupts {
2479 ", interrupts enabled"
2480 } else {
2481 ""
2482 }
2483 );
2484
2485 janitor(&opts.repo, opts, home, worktrees_root).await;
2488 resweep_superseded_attempts(queue, home);
2489
2490 let outcome = poll(
2491 opts,
2492 queue,
2493 &status,
2494 home,
2495 worktrees_root,
2496 stop,
2497 DispatchLimits {
2498 max_concurrent: concurrency,
2499 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2500 },
2501 )
2502 .await;
2503
2504 beat.abort();
2505 waiter.abort();
2506 deputies.abort();
2507 fetcher.abort();
2508 release_watch.abort();
2509 clear_status_at(status_file);
2510 outcome
2511}
2512
2513fn release_watch_settings(repo: &Path, opts: &Opts) -> (Vec<PathBuf>, u64) {
2516 let Ok(cfg) = prepare(repo, opts) else {
2517 return (Vec::new(), 0);
2518 };
2519 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2520 .into_iter()
2521 .map(|r| r.path)
2522 .collect();
2523 let mut known: Vec<PathBuf> = Queue::open().list().into_iter().map(|t| t.repo).collect();
2526 let runs = crate::run::home().join("runs");
2527 for id in crate::run::list_ids_in(&runs) {
2528 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2529 .ok()
2530 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2531 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2532 known.extend(repo);
2533 }
2534 for k in known {
2535 let k = k.canonicalize().unwrap_or(k);
2536 if k.join(".git").exists() && !paths.contains(&k) {
2537 paths.push(k);
2538 }
2539 }
2540 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2541 if !paths.contains(&own) {
2542 paths.push(own);
2543 }
2544 (paths, cfg.daemon.release_stall_minutes)
2545}
2546
2547const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2549
2550async fn fetch_loop(repo: PathBuf, opts: Opts, stop: Stop) {
2554 while !stop.stopped() {
2555 let (interval, roots) = match prepare(&repo, &opts) {
2556 Ok(c) => (c.repos.fetch_interval, c.repos.roots),
2557 Err(_) => (0, Vec::new()),
2558 };
2559 if interval > 0 && !roots.is_empty() {
2560 let r = crate::clean::fetch_origins(&roots, FETCH_TIMEOUT, || stop.stopped()).await;
2561 tracing::debug!("fetch origins: {r:?}");
2562 }
2563 let wait = if interval > 0 { interval } else { 60 };
2565 let mut slept = 0;
2566 while slept < wait && !stop.stopped() {
2567 tokio::time::sleep(Duration::from_secs(1)).await;
2568 slept += 1;
2569 }
2570 }
2571}
2572
2573async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2579 loop {
2580 tokio::time::sleep(HEARTBEAT).await;
2581 let snapshot = {
2582 let mut guard = lock(&status);
2583 guard.updated_at = Timestamp::now();
2584 guard.clone()
2585 };
2586 if let Err(e) = write_status_to(&path, &snapshot) {
2587 tracing::warn!("could not refresh the daemon status file: {e:#}");
2590 }
2591 }
2592}
2593
2594#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2597enum LandResume {
2598 NotLanding,
2601 StillWaiting,
2606 Ready,
2610}
2611
2612fn land_resume_state(task: &Task) -> LandResume {
2616 let Some(run_id) = task.runs.last() else {
2617 return LandResume::NotLanding;
2618 };
2619 let Ok(state) = RunState::load(run_id) else {
2620 return LandResume::NotLanding;
2621 };
2622 if !state.parked {
2623 return LandResume::NotLanding;
2624 }
2625 if state.status == RunStatus::Landing
2626 && state.land_approval.as_ref().is_some_and(|binding| {
2627 ask::Questions::open()
2628 .get(&binding.question)
2629 .is_ok_and(|q| {
2630 q.run == state.id
2631 && q.resolution()
2632 .as_deref()
2633 .is_some_and(crate::merge_decision::stops)
2634 })
2635 })
2636 {
2637 return LandResume::Ready;
2638 }
2639 if !implementers_match(task, &state) {
2640 return LandResume::NotLanding;
2641 }
2642 let node = if state.status == RunStatus::Landing {
2646 land::APPROVAL_NODE
2647 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2648 crate::github_text::ASK_NODE
2649 } else {
2650 return LandResume::NotLanding;
2651 };
2652 let store = ask::Questions::open();
2653 let waiting = store
2654 .list()
2655 .into_iter()
2656 .filter(|q| &q.run == run_id && q.node == node)
2657 .max_by(|a, b| a.id.cmp(&b.id));
2658 let Some(mut q) = waiting else {
2659 return LandResume::Ready;
2660 };
2661 if !q.status.open() {
2662 if q.resolution().as_deref() == Some(land::REPAIR) {
2665 return if state.liveness(false) == Liveness::Live {
2666 LandResume::StillWaiting
2667 } else {
2668 LandResume::NotLanding
2669 };
2670 }
2671 return LandResume::Ready;
2672 }
2673 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2680 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2681 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2682 q.abandon(format!(
2683 "no answer within {}s of asking",
2684 timeout.as_secs().max(1)
2685 ));
2686 if store.put(&mut q).is_ok() {
2689 return LandResume::Ready;
2690 }
2691 }
2692 LandResume::StillWaiting
2693}
2694
2695const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2703
2704const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2716
2717struct InFlightGuard<'a> {
2730 status: &'a Arc<Mutex<Status>>,
2731 stop: &'a Stop,
2732 task_id: &'a str,
2733}
2734
2735impl Drop for InFlightGuard<'_> {
2736 fn drop(&mut self) {
2737 lock(self.status).current.retain(|c| c.task != self.task_id);
2738 self.stop.exit();
2739 }
2740}
2741
2742#[derive(Debug, Clone, PartialEq, Eq)]
2764enum Interrupt {
2765 Idle,
2767 Parking {
2779 parked: Vec<String>,
2780 interrupt_task: String,
2781 },
2782 Running {
2790 parked: Vec<String>,
2791 interrupt_task: String,
2792 },
2793 Resuming { parked: Vec<String> },
2800}
2801
2802fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
2823 match state {
2824 Interrupt::Idle => {
2825 if in_flight.len() != 1 {
2837 return Interrupt::Idle;
2838 }
2839 match runnable.iter().find(|t| t.interrupt) {
2840 Some(t) => Interrupt::Parking {
2841 parked: in_flight.to_vec(),
2842 interrupt_task: t.id.clone(),
2843 },
2844 None => Interrupt::Idle,
2845 }
2846 }
2847 Interrupt::Parking {
2848 parked,
2849 interrupt_task,
2850 } => {
2851 if in_flight.iter().any(|id| parked.contains(id)) {
2852 Interrupt::Parking {
2854 parked,
2855 interrupt_task,
2856 }
2857 } else if in_flight.contains(&interrupt_task) {
2858 Interrupt::Running {
2859 parked,
2860 interrupt_task,
2861 }
2862 } else if runnable.iter().any(|t| t.id == interrupt_task) {
2863 Interrupt::Parking {
2867 parked,
2868 interrupt_task,
2869 }
2870 } else {
2871 Interrupt::Resuming { parked }
2876 }
2877 }
2878 Interrupt::Running {
2879 parked,
2880 interrupt_task,
2881 } => {
2882 if in_flight.contains(&interrupt_task) {
2883 Interrupt::Running {
2884 parked,
2885 interrupt_task,
2886 }
2887 } else {
2888 Interrupt::Resuming { parked }
2894 }
2895 }
2896 Interrupt::Resuming { parked } => {
2897 if in_flight.iter().any(|id| parked.contains(id)) {
2898 Interrupt::Idle
2904 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
2905 Interrupt::Resuming { parked }
2906 } else {
2907 Interrupt::Idle
2910 }
2911 }
2912 }
2913}
2914
2915fn advance_interrupt_tick(
2921 enabled: bool,
2922 state: Interrupt,
2923 in_flight: &[String],
2924 runnable: &[Task],
2925) -> Interrupt {
2926 if !enabled {
2927 return Interrupt::Idle;
2928 }
2929 advance_interrupt(state, in_flight, runnable)
2930}
2931
2932fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
2937 match state {
2938 Interrupt::Idle => candidates,
2939 Interrupt::Parking {
2940 parked,
2941 interrupt_task,
2942 } => {
2943 if in_flight.iter().any(|id| parked.contains(id)) {
2944 Vec::new()
2945 } else {
2946 candidates
2947 .into_iter()
2948 .filter(|t| &t.id == interrupt_task)
2949 .collect()
2950 }
2951 }
2952 Interrupt::Running { .. } => Vec::new(),
2953 Interrupt::Resuming { parked } => candidates
2961 .into_iter()
2962 .find(|t| parked.contains(&t.id))
2963 .into_iter()
2964 .collect(),
2965 }
2966}
2967
2968struct DispatchLimits {
2972 max_concurrent: usize,
2975 pause_for_interrupts: bool,
2977}
2978
2979#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2988enum PermitKind {
2989 None,
2994 Urgent,
3001 Ordinary,
3004}
3005
3006fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3009 if priority {
3010 PermitKind::None
3011 } else if urgent {
3012 PermitKind::Urgent
3013 } else {
3014 PermitKind::Ordinary
3015 }
3016}
3017
3018struct DispatchClaim {
3019 task: Task,
3020 claim: crate::queue::Claim,
3021 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3022}
3023
3024fn claim_dispatch(
3027 queue: &Queue,
3028 candidate: &Task,
3029 cooling_down: bool,
3030 ordinary: &Arc<tokio::sync::Semaphore>,
3031 urgent: &Arc<tokio::sync::Semaphore>,
3032) -> Option<DispatchClaim> {
3033 let Ok(claim) = queue.claim(&candidate.id) else {
3034 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3035 return None;
3036 };
3037 let task = match queue.get(&candidate.id) {
3038 Ok(t) if t.status.runnable() => t,
3039 Ok(_) => return None,
3040 Err(e) => {
3041 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3042 return None;
3043 }
3044 };
3045 let resume = land_resume_state(&task);
3046 if resume == LandResume::StillWaiting {
3047 return None;
3048 }
3049 let priority = resume == LandResume::Ready;
3050 if !priority && cooling_down {
3051 return None;
3052 }
3053 let permit = match permit_kind(priority, task.urgent) {
3056 PermitKind::None => None,
3057 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3058 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3059 };
3060 let stamp = Timestamp::now();
3063 let mut task = task;
3064 task.dispatched_at = Some(stamp);
3065 if let Err(e) = queue.modify(&task.id, |t| {
3066 t.dispatched_at = Some(stamp);
3067 true
3068 }) {
3069 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3070 }
3071 Some(DispatchClaim {
3072 task,
3073 claim,
3074 permit,
3075 })
3076}
3077
3078async fn poll(
3097 opts: &Opts,
3098 queue: &Queue,
3099 status: &Arc<Mutex<Status>>,
3100 home: &Path,
3101 worktrees_root: &Path,
3102 stop: &Stop,
3103 limits: DispatchLimits,
3104) -> Result<()> {
3105 let DispatchLimits {
3106 max_concurrent,
3107 pause_for_interrupts,
3108 } = limits;
3109 let mut attempted: Vec<String> = Vec::new();
3114 let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
3115 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(1));
3123 let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
3129 let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
3130 let mut conductor = Conductor::new();
3131 let mut cache_last_checked: Option<Timestamp> = None;
3134 let mut disk_last_checked: Option<Timestamp> = None;
3135 let mut interrupt = Interrupt::Idle;
3137 let mut interrupt_pauses: std::collections::HashMap<String, crate::graph::Pause> =
3143 std::collections::HashMap::new();
3144
3145 while !stop.stopped() {
3146 lock(status).polls += 1;
3147
3148 while let Some(result) = inflight.try_join_next() {
3153 if let Err(e) = result {
3154 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3155 notices::raise(Notice::error(
3156 "loop:attempt",
3157 "A queued attempt ended abnormally; check the task it was running.",
3158 ));
3159 }
3160 }
3161
3162 let swept = sweep_stale_claims(queue, STALE_CLAIM);
3163 if !swept.is_empty() {
3164 tracing::warn!(
3165 "swept {} stale claim(s) left behind by an earlier daemon: {}",
3166 swept.len(),
3167 swept.join(", ")
3168 );
3169 }
3170 let now = Timestamp::now();
3175
3176 if !stop.busy_now() {
3181 maybe_prune_cache_between_runs(
3182 &opts.repo,
3183 opts,
3184 home,
3185 stop,
3186 &mut cache_last_checked,
3187 now,
3188 )
3189 .await;
3190 }
3191
3192 if cache_check_due(disk_last_checked, now, 30) {
3193 disk_last_checked = Some(now);
3194 if let Ok(cfg) = prepare(&opts.repo, opts) {
3195 crate::retention::sweep(home, &cfg.disk, now).await;
3196 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3199 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3200 }
3201 for current in lock(status).current.clone() {
3202 if let Ok(state) = RunState::load_under(¤t.run, home)
3203 && let Some(reason) = disk_gate(&state.repo, &state.config)
3204 {
3205 if let Some(pause) = interrupt_pauses.get(¤t.task) {
3206 pause.park_because(reason);
3207 }
3208 notices::raise_in(
3209 home,
3210 Notice::warn(
3211 "disk:pressure",
3212 "Disk pressure persists after cleanup; active runs were asked to checkpoint and park. New starts and resumes wait for free space.",
3213 ),
3214 );
3215 }
3216 }
3217 }
3218 }
3219
3220 let stalled = stalled_tasks(queue, home, now);
3221 let stalled_ids: std::collections::BTreeSet<_> =
3222 stalled.iter().map(|task| task.id.clone()).collect();
3223 let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
3224 if !reclaimed.is_empty() {
3225 tracing::warn!(
3226 "reclaimed {} task(s) left `running` by a daemon that never \
3227 recorded the outcome: {}",
3228 reclaimed.len(),
3229 reclaimed.join(", ")
3230 );
3231 }
3232 let abandoned_runs = reclaim_abandoned_runs(home, now);
3233 if !abandoned_runs.is_empty() {
3234 tracing::warn!(
3235 "failed {} run(s) left behind by a killed process, past every \
3236 active seat's own timeout: {}",
3237 abandoned_runs.len(),
3238 abandoned_runs.join(", ")
3239 );
3240 }
3241
3242 let questions = Questions::at(home.join("questions"));
3247
3248 crate::task_chat::sweep(
3251 queue,
3252 &crate::talk::Talks::at(home.join("talks")),
3253 &questions,
3254 home,
3255 );
3256 repair_choice_offers(&questions);
3257 resolve_blockers(queue, &questions);
3258 apply_choice_actions(queue, &questions, home).await;
3259 triage::apply_answers(queue, &questions);
3260 reconcile_task_questions(queue, &questions);
3261 {
3265 let talks = crate::talk::Talks::at(home.join("talks"));
3266 crate::chat_notify::sweep(queue, &talks, home);
3267 crate::chat_watch::sweep(home, &talks);
3268 crate::chat_notify::start_turns(&talks, stop.parking(), &mut inflight);
3269 }
3270
3271 let finished: Vec<Task> = finished_tasks(queue)
3277 .into_iter()
3278 .filter(|task| !stalled_ids.contains(&task.id))
3279 .filter(|task| !awaiting_resume_with(task, |id| RunState::load_under(id, home)))
3283 .collect();
3284 let queued = queued_tasks(queue);
3285 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3289 && conductor.worth_a_look(queue, &stalled, &finished)
3290 {
3291 match prepare(&opts.repo, opts) {
3292 Ok(cfg) => {
3293 conductor
3294 .maybe_run(
3295 &cfg,
3296 &opts.repo,
3297 queue,
3298 &questions,
3299 home,
3300 &queued,
3301 &stalled,
3302 &finished,
3303 opts.max_attempts,
3304 )
3305 .await;
3306 }
3307 Err(e) => {
3308 tracing::warn!("conductor: no config: {e:#}");
3309 notices::raise(Notice::warn(
3310 "loop:no-config",
3311 "The loop could not read this repository's config, so held tasks are not being triaged.",
3312 ));
3313 }
3314 }
3315 }
3316
3317 let candidates: Vec<Task> = runnable(queue)
3318 .into_iter()
3319 .filter(|t| !opts.once || !attempted.contains(&t.id))
3320 .collect();
3321
3322 let in_flight: Vec<String> = lock(status)
3327 .current
3328 .iter()
3329 .map(|c| c.task.clone())
3330 .collect();
3331 interrupt_pauses.retain(|id, _| in_flight.contains(id));
3332
3333 interrupt =
3334 advance_interrupt_tick(pause_for_interrupts, interrupt, &in_flight, &candidates);
3335 if let Interrupt::Parking {
3336 parked,
3337 interrupt_task,
3338 } = &interrupt
3339 {
3340 let reason = format!(
3341 "task {} asked to run first",
3342 crate::run::short_of(interrupt_task)
3343 );
3344 for id in parked {
3345 if let Some(pause) = interrupt_pauses.get(id) {
3346 pause.park_because(reason.clone());
3347 }
3348 }
3349 }
3350 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3351
3352 let cooling_down =
3353 lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
3354
3355 let mut started_any = false;
3356 for candidate in candidates {
3357 if stop.stopped() {
3358 break;
3359 }
3360
3361 let Some(DispatchClaim {
3362 mut task,
3363 claim,
3364 permit,
3365 }) = claim_dispatch(queue, &candidate, cooling_down, &sem, &urgent_sem)
3366 else {
3367 continue;
3368 };
3369
3370 let task_id = task.id.clone();
3371 attempted.push(task_id.clone());
3372 lock(status).idle = false;
3373 stop.enter();
3376 started_any = true;
3377
3378 let run_pause = crate::graph::Pause::new();
3382 interrupt_pauses.insert(task_id.clone(), run_pause.clone());
3383
3384 let opts = opts.clone();
3385 let queue = queue.clone();
3386 let status = Arc::clone(status);
3387 let stop = stop.clone();
3388 let quota_cooldown_until = Arc::clone("a_cooldown_until);
3389 inflight.spawn(async move {
3390 let _claim = claim;
3394 let _permit = permit;
3395 let _inflight = InFlightGuard {
3397 status: &status,
3398 stop: &stop,
3399 task_id: &task_id,
3400 };
3401 let quota = attempt(&opts, &queue, &status, &stop, run_pause, &mut task).await;
3402 lock(&status).completed += 1;
3403 let now = Timestamp::now();
3409 if let Some(until) = cooldown_until("a, now) {
3410 let wait = until.as_second() - now.as_second();
3411 *lock("a_cooldown_until) = Some(until);
3412 let hint = quota
3413 .iter()
3414 .find(|q| q.reset.is_some())
3415 .and_then(|q| q.reset.as_deref());
3416 match hint {
3417 Some(h) => tracing::warn!(
3418 "quota hit; waiting {wait}s before taking another ordinary task \
3419 (CLI reported reset: {h})"
3420 ),
3421 None => tracing::warn!(
3422 "quota hit; waiting {wait}s before taking another ordinary task \
3423 (no reset hint reported)"
3424 ),
3425 }
3426 }
3427 });
3428 }
3429
3430 if started_any {
3431 continue;
3432 }
3433
3434 if stop.busy_now() {
3435 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3440 continue;
3441 }
3442
3443 lock(status).idle = true;
3445 if opts.once {
3446 janitor(&opts.repo, opts, home, worktrees_root).await;
3450 resweep_superseded_attempts(queue, home);
3451 triage_held(queue, home, opts).await;
3452 break;
3453 }
3454 stop.idle(opts.poll).await;
3455 if stop.stopped() {
3456 continue;
3457 }
3458 janitor(&opts.repo, opts, home, worktrees_root).await;
3464 resweep_superseded_attempts(queue, home);
3465 triage_held(queue, home, opts).await;
3466 }
3467
3468 while let Some(result) = inflight.join_next().await {
3473 if let Err(e) = result {
3474 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3475 notices::raise(Notice::error(
3476 "loop:attempt",
3477 "A queued attempt ended abnormally; check the task it was running.",
3478 ));
3479 }
3480 }
3481 Ok(())
3482}
3483
3484async fn public_resume_owner(
3488 home: &Path,
3489 runs: &[String],
3490 repo: &Path,
3491) -> Result<Option<RunState>> {
3492 let mut owner = None;
3493 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
3494 let _lock = crate::pr_head::owner_lock(home, id)?;
3495 let state = RunState::load_under(id, home)?;
3496 if state.released()
3497 || state.retired_to.is_some()
3498 || state.pr_taken_over_by.is_some()
3499 || !crate::handover::landing_checkpoint(&state)
3500 {
3501 continue;
3502 }
3503 anyhow::ensure!(
3504 crate::git::toplevel(&state.repo).await?.canonicalize()?
3505 == crate::git::toplevel(repo).await?.canonicalize()?,
3506 "public owner repository differs; inspect run lineage"
3507 );
3508 anyhow::ensure!(
3509 owner.is_none(),
3510 "multiple retained public owners; inspect run lineage"
3511 );
3512 anyhow::ensure!(
3513 crate::handover::can_resume(&state),
3514 "public owner cannot resume"
3515 );
3516 let claimed = is_working_on(home, id, Timestamp::now());
3517 anyhow::ensure!(
3518 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
3519 "public owner's driver is live or unknown; wait before retrying"
3520 );
3521 owner = Some(state);
3522 }
3523 Ok(owner)
3524}
3525
3526async fn attempt(
3532 opts: &Opts,
3533 queue: &Queue,
3534 status: &Arc<Mutex<Status>>,
3535 stop: &Stop,
3536 interrupt_pause: crate::graph::Pause,
3537 task: &mut Task,
3538) -> Vec<QuotaLoss> {
3539 let repo = repo_for(task, &opts.repo);
3540 tracing::info!(
3541 "task {} — {} (repo {})",
3542 task.short(),
3543 task.title,
3544 repo.display()
3545 );
3546
3547 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
3548 Ok(owner) => owner,
3549 Err(e) => {
3550 task.hold_machine(Some(format!(
3551 "Cannot verify existing public owner: {e:#}; no attempt started."
3552 )));
3553 record(queue, task);
3554 return Vec::new();
3555 }
3556 };
3557 let resume_runs = public_owner
3558 .as_ref()
3559 .map(|s| vec![s.id.clone()])
3560 .unwrap_or_else(|| task.runs.clone());
3561
3562 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
3566 && previous
3567 .merge_decision
3568 .as_ref()
3569 .is_some_and(|decision| !decision.closed)
3570 && previous.status != RunStatus::Landing
3571 && !crate::merge_decision::resumable_hold(&previous)
3572 {
3573 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3574 record(queue, task);
3575 return Vec::new();
3576 }
3577 let stop_owner = resume_runs
3578 .last()
3579 .and_then(|id| RunState::load(id).ok())
3580 .filter(|state| {
3581 crate::merge_decision::resumable_hold(state)
3582 || state.status == RunStatus::Landing
3583 && state.land_approval.as_ref().is_some_and(|binding| {
3584 Questions::open().get(&binding.question).is_ok_and(|q| {
3585 q.run == state.id
3586 && q.resolution()
3587 .as_deref()
3588 .is_some_and(crate::merge_decision::stops)
3589 })
3590 })
3591 });
3592 if let Some(owner) = stop_owner {
3593 let mut runner = match Runner::resume(&owner.id) {
3594 Ok(runner) => runner,
3595 Err(e) => {
3596 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
3597 record(queue, task);
3598 return Vec::new();
3599 }
3600 };
3601 runner.on_pause(stop.pause());
3602 runner.watch_interrupt(interrupt_pause);
3603 let quota_before = runner.state.quota.clone();
3604 task.start(owner.id.clone());
3605 record(queue, task);
3606 lock(status).current.push(Current {
3607 task: task.id.clone(),
3608 run: owner.id,
3609 });
3610 runner.state.next_pass_task_entry = true;
3611 let result = runner.execute().await;
3612 return finish_attempt(
3613 opts.max_attempts,
3614 queue,
3615 task,
3616 &runner.state,
3617 "a_before,
3618 result,
3619 );
3620 }
3621
3622 if let Some(owner) = &public_owner
3623 && (task.fresh_start || !may_resume_run(owner, task))
3624 {
3625 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
3626 record(queue, task);
3627 return Vec::new();
3628 }
3629
3630 let unfinished = (!task.fresh_start)
3631 .then(|| unfinished_run(&resume_runs, task.short()))
3632 .flatten()
3633 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
3634 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
3635 if !task.fresh_start
3636 && task.pr_recovery.is_none()
3637 && task.review_branch.is_none()
3638 && task.resume_override.is_none()
3639 && previous
3640 .as_ref()
3641 .is_some_and(|run| run.retired_to.is_some())
3642 && unfinished.is_none()
3643 {
3644 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
3645 record(queue, task);
3646 return Vec::new();
3647 }
3648
3649 let saved_recovery = previous.as_ref().filter(|run| {
3650 task.replacement.is_none()
3651 && task.pr_recovery.as_ref().is_some_and(|selected| {
3652 selected.check_task(&task.id, &task.instruction).is_ok()
3653 && selected_starter(selected, Some(run), unfinished.as_deref())
3654 == Starter::Resume(run.id.clone())
3655 })
3656 });
3657
3658 let explicit = match &task.overrides {
3662 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
3663 Some(_) => opts.config.as_deref(),
3664 None => opts.config.as_deref(),
3665 };
3666 let (mut config, layers) = if let Some(run) = saved_recovery {
3667 let mut saved = run.clone();
3670 if let Err(e) = saved.configure_recovery_resume() {
3671 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
3672 record(queue, task);
3673 return Vec::new();
3674 }
3675 (saved.config, Vec::new())
3676 } else {
3677 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
3678 Ok(loaded) => loaded,
3679 Err(e) => {
3680 let reason = format!("[config] {e:#}");
3681 record_config_failure(task, &e);
3682 task.last_error = Some(reason.clone());
3683 task.hold_machine(Some(reason));
3684 record(queue, task);
3685 return Vec::new();
3686 }
3687 };
3688 if let Err(e) = apply_options(&mut config, opts, task) {
3689 task.hold_manual(Some(format!(
3690 "[config] {e:#}; correct the merge override before releasing the task"
3691 )));
3692 record(queue, task);
3693 return Vec::new();
3694 }
3695 if let Some(selected) = &task.pr_recovery
3696 && let Err(e) = selected.configure(&mut config)
3697 {
3698 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
3699 record(queue, task);
3700 return Vec::new();
3701 }
3702 (config, layers)
3703 };
3704 let config_evidence = crate::pr_recovery::ConfigEvidence {
3705 layers,
3706 overrides: task.overrides.clone(),
3707 daemon_merge: task
3708 .overrides
3709 .as_ref()
3710 .is_none_or(|o| o.inherit_daemon_options)
3711 .then(|| opts.merge.clone())
3712 .flatten(),
3713 };
3714 apply_task_implementers(&mut config, task);
3715 if let Some(replacement) = &task.replacement {
3716 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
3717 old.liveness(false) == Liveness::Dead
3718 && old.merge_decision.as_ref().is_some_and(|d| {
3719 d.closed
3720 && d.question == replacement.question
3721 && d.replacement_run.as_ref().is_none_or(|run| {
3722 task.runs.last() == Some(run)
3723 && RunState::load(run).is_ok_and(|current| {
3724 current.replaces.as_ref().is_some_and(|pin| {
3725 pin.question == replacement.question
3726 && pin.agent == replacement.agent
3727 })
3728 })
3729 })
3730 })
3731 });
3732 if !safe {
3733 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
3734 record(queue, task);
3735 return Vec::new();
3736 }
3737 match config.agent(&replacement.agent) {
3738 Ok(agent) if crate::agent::installed(agent) => {}
3739 _ => {
3740 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
3741 record(queue, task);
3742 return Vec::new();
3743 }
3744 }
3745 config.graph.implementers = 1;
3746 config.graph.land_approval = true;
3747 config.merge.mode = MergeMode::Pr;
3748 config.graph.selected_implementer = Some(replacement.agent.clone());
3749 }
3750 let p = phrases(&config.graph.language);
3751 let start_failed = p.could_not_start;
3752 if let Ok(base) =
3757 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
3758 .await
3759 && let Ok(head) =
3760 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
3761 {
3762 let mut inputs = crate::queue::recovery_inputs(&task.sources);
3763 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
3764 let source = crate::git::rev_parse(&repo, branch)
3765 .await
3766 .unwrap_or_default();
3767 inputs.push_str(&format!(";review:{branch}:{source}"));
3768 }
3769 if let Some(reason) = task.recovery_exhausted(
3770 &head,
3771 &inputs,
3772 task.attempt_limit
3773 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
3774 ) {
3775 task.hold_unrequested_retry(Some(reason));
3776 record(queue, task);
3777 return Vec::new();
3778 }
3779 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
3780 }
3781
3782 if let Some(reason) = disk_gate(&repo, &config) {
3790 task.notice_cause("disk-space", None, None);
3791 task.last_error = Some(reason.clone());
3792 task.hold_machine(Some(reason.clone()));
3793 record(queue, task);
3794 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
3795 notices::raise(
3798 Notice::warn(
3799 &format!("disk:{}", repo.display()),
3800 "A task was held for want of free disk space; free some, then release it from the queue.",
3801 )
3802 .link(Link::Task {
3803 id: task.id.clone(),
3804 }),
3805 );
3806 return Vec::new();
3807 }
3808
3809 let review_branch = take_review_branch(task, previous.as_ref());
3834 let branch_exists = match &review_branch {
3835 Some(branch) => crate::git::branch_exists(&repo, branch)
3836 .await
3837 .unwrap_or(false),
3838 None => false,
3839 };
3840 let proposed = match (&review_branch, &unfinished, &task.review_of) {
3844 (None, None, Some(branch)) => Starter::Review(branch.clone()),
3845 _ => choose_starter(
3846 review_branch.as_deref(),
3847 branch_exists,
3848 unfinished.as_deref(),
3849 ),
3850 };
3851 let retry_selected = task.replacement.is_some()
3852 && task.resumption.is_some()
3853 && !task.fresh_start
3854 && previous.as_ref().is_some_and(|run| {
3855 run.replaces.is_some()
3856 && ((run.status == RunStatus::Failed && run.viable().is_empty())
3857 || (run.status == RunStatus::Blocked && run.tally.is_none()))
3858 });
3859 let stop_pin = task
3860 .resume_override
3861 .as_ref()
3862 .filter(|pin| {
3863 Questions::open().get(&pin.question_id).is_ok_and(|q| {
3864 q.node == land::APPROVAL_NODE
3865 && q.resolution()
3866 .as_deref()
3867 .is_some_and(crate::merge_decision::stops)
3868 })
3869 })
3870 .and_then(|pin| pin.pinned_run.clone());
3871 let starter = if let Some(owner) = &public_owner {
3872 Starter::Resume(owner.id.clone())
3873 } else if let Some(run) = stop_pin {
3874 Starter::Resume(run)
3877 } else if retry_selected {
3878 Starter::Resume(previous.as_ref().unwrap().id.clone())
3879 } else if let Some(selected) = &task.pr_recovery {
3880 if selected.check_task(&task.id, &task.instruction).is_ok() {
3881 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
3882 } else {
3883 Starter::Review(selected.branch.clone())
3884 }
3885 } else {
3886 starter_with_implementers(task, previous.as_ref(), proposed)
3887 };
3888 if let Some(pin) = &task.resume_override
3891 && let Some(run) = &pin.pinned_run
3892 && starter != Starter::Resume(run.clone())
3893 {
3894 let run = run.clone();
3895 task.notice_cause("resume-owner", Some(&run), None);
3896 task.hold_machine(Some(format!(
3897 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
3898 task.id
3899 )));
3900 record(queue, task);
3901 return Vec::new();
3902 }
3903 if task.replacement.is_some()
3904 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
3905 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
3906 {
3907 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
3908 record(queue, task);
3909 return Vec::new();
3910 }
3911 if let Some(reason) = retained_attempt_hold(
3912 task,
3913 &starter,
3914 config.disk.retained_attempt_limit,
3915 RunState::load,
3916 ) {
3917 if let Starter::Review(branch) = &starter {
3920 task.review_branch = Some(branch.clone());
3921 }
3922 task.hold_unrequested_retry(Some(reason));
3923 record(queue, task);
3924 return Vec::new();
3925 }
3926 let attachments = match task_attachments(queue, task) {
3931 Ok(a) => a,
3932 Err(e) => {
3933 task.attempts += 1;
3934 task.fail_startup_target(
3935 "startup-attachments",
3936 e.name.clone(),
3937 format!("{start_failed}{e:#}"),
3938 opts.max_attempts,
3939 );
3940 record(queue, task);
3941 return Vec::new();
3942 }
3943 };
3944 let startup_pause = stop.pause();
3945 let startup_interrupt = interrupt_pause.clone();
3946 let wait = crate::repo_lock::WaitPolicy::new(
3947 Duration::from_secs(config.graph.timeout_resource_wait),
3948 move || startup_pause.parked() || startup_interrupt.parked(),
3949 );
3950 let started = wait.scope(Box::pin(async { match &starter {
3953 Starter::Review(_) if task.pr_recovery.is_some() => {
3954 let selected = task.pr_recovery.clone().expect("selected recovery");
3955 match selected.check_task(&task.id, &task.instruction) {
3956 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
3957 .await
3958 .map(|mut r| {
3959 r.state.attachments = attachments.clone();
3960 r
3961 }),
3962 Err(e) => Err(e),
3963 }
3964 }
3965 Starter::Review(branch) => {
3966 tracing::info!(
3967 "task {} reopens `{branch}` as a review-only pass",
3968 task.short()
3969 );
3970 let takeover = crate::handover::Takeover {
3973 earlier: task.earlier_attempts().to_vec(),
3974 home: crate::run::home(),
3975 choice: take_divergence_answer(branch, &config.merge.remote, task),
3976 task_instruction: Some(instruction_for(task)),
3977 };
3978 Runner::review_taking_over(
3979 &repo,
3980 branch,
3981 config,
3982 Some(takeover),
3983 crate::run::Origin::queue(&task.id),
3984 )
3985 .await
3986 }
3987 Starter::Resume(id) => {
3988 tracing::info!("resuming run {id} rather than competing again");
3989 Runner::resume(id).and_then(|mut r| {
3990 if retry_selected {
3991 for candidate in &mut r.state.candidates {
3994 candidate.failed = None;
3995 candidate.empty = false;
3996 candidate.commits = 0;
3997 }
3998 r.state.status = RunStatus::Implementing;
3999 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4000 }
4001 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4002 r.state.sources = task.sources.clone();
4003 }
4004 if r.state.prep_failure.is_some() {
4005 let approval = task
4006 .resumption
4007 .as_ref()
4008 .context("preparation retry requires an explicit task release")?;
4009 r.retry_preparation(approval.at)?;
4010 }
4011 if let Some(instruction) =
4012 prepare_instruction(&starter, Some(&r.state.instruction), task)
4013 {
4014 r.state.instruction = instruction;
4015 }
4016 r.state.attachments = attachments.clone();
4017 if task.replacement.is_some() {
4018 r.state.config.merge.mode = MergeMode::Pr;
4019 r.state.config.graph.land_approval = true;
4020 }
4021 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4025 .overrides
4026 .as_ref()
4027 .and_then(|o| o.merge.as_deref())
4028 .and_then(|m| merge_mode(m).ok())
4029 {
4030 r.state.config.merge.mode = mode;
4031 }
4032 Ok(r)
4033 })
4034 }
4035 Starter::Start => {
4036 if let Some(branch) = &review_branch {
4037 tracing::warn!(
4038 "conductor chose review for task {} but branch `{branch}` no longer \
4039 exists; requeuing as a fresh competition instead",
4040 task.short()
4041 );
4042 }
4043 let instruction = prepare_instruction(&starter, None, task)
4044 .unwrap_or_else(|| task.instruction.clone());
4045 Runner::start_naming(
4046 &repo,
4047 instruction,
4048 &task.title,
4049 config,
4050 crate::run::Origin::queue(&task.id),
4051 )
4052 .await
4053 .map(|mut r| {
4054 r.state.attachments = attachments.clone();
4055 r.state.sources = task.sources.clone();
4056 r
4057 })
4058 }
4059 }})).await;
4060 let mut runner = match started {
4061 Ok(r) => r,
4062 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4063 task.last_error = Some(e.to_string());
4064 if stop.parking() || interrupt_pause.parked() {
4065 let attempts = task.attempts;
4066 task.park(e.to_string());
4067 task.attempts = attempts;
4069 } else {
4070 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4071 }
4072 record(queue, task);
4073 return Vec::new();
4074 }
4075 Err(e) if task.pr_recovery.is_some() => {
4079 task.startup_notice_cause("startup-public-head");
4080 task.attempts += 1;
4081 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
4082 task.link_run(&partial.run);
4083 task.record_pass(&partial.run, 0, task.attempts, None);
4084 }
4085 task.last_error = Some(format!("{start_failed}{e:#}"));
4086 task.hold_unrequested_retry(Some(format!(
4087 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4088 )));
4089 record(queue, task);
4090 return Vec::new();
4091 }
4092 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4093 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4094 Some(r) => (p.handover_refused)(r),
4095 None => format!("{e:#}"),
4096 };
4097 task.startup_notice_cause("startup-handover");
4098 let reason = format!("{start_failed}{detail}");
4099 task.last_error = Some(reason.clone());
4100 let branch = match &starter {
4103 Starter::Review(branch) => Some(branch.clone()),
4104 _ => None,
4105 };
4106 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4110 record(queue, task);
4111 tracing::warn!(
4112 "holding {} for a branch it cannot take over: {e:#}",
4113 task.short()
4114 );
4115 return Vec::new();
4116 }
4117 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4122 let d = e
4123 .downcast_ref::<crate::reconcile::Diverged>()
4124 .expect("checked by the guard");
4125 let mut q = ask::Question::new(
4126 task.id.clone(),
4127 crate::reconcile::NODE.to_owned(),
4128 crate::reconcile::SEAT.to_owned(),
4129 d.summary(),
4130 d.detail(),
4131 d.choices(),
4132 );
4133 match Questions::open().put(&mut q) {
4134 Ok(()) => {
4135 task.last_error = Some(format!("{e:#}"));
4136 task.review_branch = Some(d.branch.clone());
4139 task.review_instruction = Some(task.instruction.clone());
4140 task.block(vec![q.id.clone()], Some(d.summary()));
4141 }
4142 Err(put) => {
4143 tracing::warn!("could not file the divergence question: {put:#}");
4144 task.attempts += 1;
4145 task.fail_startup(
4146 "startup-question-store",
4147 format!("{start_failed}{e:#}"),
4148 opts.max_attempts,
4149 );
4150 }
4151 }
4152 record(queue, task);
4153 return Vec::new();
4154 }
4155 Err(e) => {
4156 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4162 && let Starter::Review(branch) = &starter
4163 {
4164 task.review_branch = Some(branch.clone());
4165 task.review_instruction = Some(task.instruction.clone());
4166 task.last_error = Some(format!("{start_failed}{e:#}"));
4167 task.status = crate::queue::TaskStatus::Failed;
4168 record(queue, task);
4169 return Vec::new();
4170 }
4171 task.attempts += 1;
4172 task.last_error = Some(format!("{start_failed}{e:#}"));
4173 task.startup_notice_cause("startup-source");
4174 task.hold_unrequested_retry(Some(format!(
4175 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4176 )));
4177 record(queue, task);
4178 return Vec::new();
4179 }
4180 };
4181 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4182 runner.state.recovery_config = Some(config_evidence);
4183 }
4184 if let Some(replacement) = &task.replacement {
4185 if matches!(starter, Starter::Start) {
4186 runner.state.instruction.push_str(&format!(
4187 "\n\n# Replacement review context\n{}",
4188 replacement.context
4189 ));
4190 }
4191 runner.state.replaces = Some(replacement.clone());
4192 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4193 if let Ok(mut old) = RunState::load(&replacement.run)
4194 && let Some(d) = &mut old.merge_decision
4195 {
4196 d.replacement_run = Some(runner.state.id.clone());
4197 old.event(
4198 "replacement",
4199 format!(
4200 "replacement run {} owns the next attempt; old PR remains closed",
4201 runner.state.id
4202 ),
4203 );
4204 if let Err(e) = old.save() {
4205 task.hold_manual(Some(format!(
4206 "Could not link replacement history: {e}; no agents dispatched"
4207 )));
4208 record(queue, task);
4209 return Vec::new();
4210 }
4211 }
4212 }
4213 runner.state.task_instruction = Some(task.instruction.clone());
4214 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4217 if runner.state.origin_chat.is_none() {
4218 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4219 }
4220 runner.on_pause(stop.pause());
4222 runner.watch_interrupt(interrupt_pause);
4226
4227 let run = runner.state.id.clone();
4230 task.start(run.clone());
4231 record(queue, task);
4232 lock(status).current.push(Current {
4233 task: task.id.clone(),
4234 run,
4235 });
4236
4237 let quota_before = runner.state.quota.clone();
4240 runner.state.next_pass_task_entry = true;
4241 let result = runner.execute().await;
4242 finish_attempt(
4243 opts.max_attempts,
4244 queue,
4245 task,
4246 &runner.state,
4247 "a_before,
4248 result,
4249 )
4250}
4251
4252pub fn finish_attempt(
4257 max_attempts: usize,
4258 queue: &Queue,
4259 task: &mut Task,
4260 state: &RunState,
4261 quota_before: &[QuotaLoss],
4262 result: Result<()>,
4263) -> Vec<QuotaLoss> {
4264 let detail = match result {
4265 Ok(()) => describe(state),
4266 Err(e) => format!("{e:#}"),
4267 };
4268 let fresh = losses_this_attempt(quota_before, &state.quota);
4269 let verdict = Verdict {
4270 status: state.status,
4271 left_pr: state.pr.is_some(),
4274 quota_hit: !fresh.is_empty(),
4280 parked: state.parked,
4284 no_viable_candidates: state.viable().is_empty(),
4287 };
4288 if let Some(replacement) = &state.replaces
4289 && let Some(pr) = &state.pr
4290 && let Ok(mut old) = RunState::load(&replacement.run)
4291 && let Some(d) = &mut old.merge_decision
4292 {
4293 d.replacement_pr = Some(pr.url.clone());
4294 let _ = old.save();
4295 }
4296 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4297 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4298 task.hold_unrequested_retry(Some(format!(
4299 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4300 task.short()
4301 )));
4302 }
4303 if task.status == TaskStatus::Done {
4304 supersede_prior_runs(task, &crate::run::home());
4305 }
4306 record(queue, task);
4307 tracing::info!(
4308 "task {} is {} after run {} ({})",
4309 task.short(),
4310 task.status.as_str(),
4311 state.short(),
4312 label(state.status)
4313 );
4314 fresh
4315}
4316
4317pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4321 let waits_on_owner = withheld_text_wait(task).is_some();
4326 if task.status.runnable()
4327 && !waits_on_owner
4328 && !(task.fresh_start && task.replacement.is_some())
4329 {
4330 let why = task.last_error.clone().map_or_else(
4331 || "the run did not finish".to_owned(),
4332 |e| format!("the run did not finish: {e}"),
4333 );
4334 task.hold_unrequested_retry(Some(format!(
4335 "{why}. It was started by hand, so it is not retried \
4336 automatically; `magi task release` retries it."
4337 )));
4338 record(queue, task);
4339 }
4340}
4341
4342#[must_use]
4346pub fn loop_would_resume(run: &str) -> bool {
4347 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
4348}
4349
4350#[must_use]
4360pub fn foreign_loop(
4361 reading: Option<&Reading>,
4362 now: Timestamp,
4363 own_pid: u32,
4364) -> Option<Option<u32>> {
4365 let reading = reading.filter(|r| r.running(now))?;
4366 match reading.pid {
4367 Some(pid) if pid == own_pid => None,
4368 pid => Some(pid),
4369 }
4370}
4371
4372pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
4387 let status = Arc::new(Mutex::new(Status::new()));
4388 let stop = Stop::new();
4389 Box::pin(attempt(
4392 opts,
4393 queue,
4394 &status,
4395 &stop,
4396 crate::graph::Pause::new(),
4397 task,
4398 ))
4399 .await;
4400 let mut resumes = 0;
4409 while resumes < MAX_WITHHELD_RESUMES {
4410 let Some(timeout) = withheld_text_wait(task) else {
4411 break;
4412 };
4413 eprintln!(
4414 "waiting for the owner's answer on a withheld pull request text \
4415 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
4416 );
4417 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
4420 let ready = loop {
4421 match land_resume_state(task) {
4422 LandResume::StillWaiting if std::time::Instant::now() < limit => {
4423 tokio::time::sleep(WITHHELD_POLL).await;
4424 }
4425 LandResume::Ready => break true,
4426 _ => break false,
4427 }
4428 };
4429 if !ready {
4430 break;
4431 }
4432 resumes += 1;
4433 Box::pin(attempt(
4434 opts,
4435 queue,
4436 &status,
4437 &stop,
4438 crate::graph::Pause::new(),
4439 task,
4440 ))
4441 .await;
4442 }
4443 hold_if_runnable(queue, task);
4444}
4445
4446const MAX_WITHHELD_RESUMES: u32 = 3;
4449
4450const WITHHELD_POLL: Duration = Duration::from_secs(15);
4452
4453fn withheld_text_wait(task: &Task) -> Option<u64> {
4456 let id = task.runs.last()?;
4457 let s = RunState::load(id).ok()?;
4458 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
4459 .then_some(s.config.graph.answer_timeout)
4460}
4461
4462fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
4472 after
4473 .iter()
4474 .filter(|q| !before.contains(q))
4475 .cloned()
4476 .collect()
4477}
4478
4479fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
4482 if quota.is_empty() {
4483 return None;
4484 }
4485 let with_hint = quota.iter().find(|q| q.reset.is_some());
4486 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
4487 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
4488 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
4489 Some(
4490 now.checked_add(jiff::SignedDuration::from_secs(secs))
4491 .unwrap_or(Timestamp::MAX),
4492 )
4493}
4494
4495fn implementers_match(task: &Task, state: &RunState) -> bool {
4497 task.replacement.as_ref().map_or_else(
4498 || {
4499 task.implementers()
4500 .is_none_or(|n| n == state.config.graph.implementers)
4501 },
4502 |replacement| {
4503 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
4504 && state.config.graph.implementers == 1
4505 },
4506 )
4507}
4508
4509fn starter_with_implementers(
4511 task: &Task,
4512 previous: Option<&RunState>,
4513 proposed: Starter,
4514) -> Starter {
4515 if previous.is_some_and(|s| !implementers_match(task, s)) {
4516 task.review_of
4517 .as_ref()
4518 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
4519 } else {
4520 proposed
4521 }
4522}
4523
4524fn apply_task_implementers(config: &mut Config, task: &Task) {
4526 if let Some(n) = task.implementers() {
4527 config.graph.implementers = n;
4528 }
4529}
4530
4531#[cfg(test)]
4536fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
4537 let explicit = match &task.overrides {
4538 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4539 _ => opts.config.as_deref(),
4540 };
4541 let (mut config, _) = Config::discover(repo, explicit)?;
4542 apply_options(&mut config, opts, task)?;
4543 Ok(config)
4544}
4545
4546fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
4547 if task
4548 .overrides
4549 .as_ref()
4550 .is_none_or(|o| o.inherit_daemon_options)
4551 && let Some(mode) = opts.merge.as_deref()
4552 {
4553 config.merge.mode = merge_mode(mode)?;
4554 }
4555 if let Some(o) = &task.overrides {
4556 o.apply(config);
4557 }
4558 Ok(())
4559}
4560
4561fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
4563 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
4564 if let Some(mode) = &opts.merge {
4565 config.merge.mode = merge_mode(mode)?;
4566 }
4567 Ok(config)
4568}
4569
4570async fn maybe_prune_cache_between_runs(
4601 repo: &Path,
4602 opts: &Opts,
4603 home: &Path,
4604 stop: &Stop,
4605 last_checked: &mut Option<Timestamp>,
4606 now: Timestamp,
4607) {
4608 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
4609 return;
4610 }
4611 *last_checked = Some(now);
4612 let cfg = match prepare(repo, opts) {
4613 Ok(cfg) => cfg,
4614 Err(e) => {
4615 tracing::warn!("cache check: no config: {e:#}");
4616 return;
4617 }
4618 };
4619 match clean::prune_cache_if_over_limit(&cfg, home) {
4620 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
4621 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
4622 pruned.files,
4623 pruned.freed
4624 ),
4625 Ok(_) => {}
4626 Err(e) => {
4627 tracing::warn!("housekeep: prune cache: {e:#}");
4628 notices::raise_in(
4629 home,
4630 Notice::warn(
4631 "housekeep:cache",
4632 "Pruning the shared build cache failed; disk usage may keep growing.",
4633 ),
4634 );
4635 }
4636 }
4637}
4638
4639fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
4643 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
4644}
4645
4646async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
4665 let cfg = match prepare(repo, opts) {
4666 Ok(cfg) => cfg,
4667 Err(e) => {
4668 tracing::warn!("housekeep: no config: {e:#}");
4669 return;
4670 }
4671 };
4672 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
4681 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
4682 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
4687 let mut extra = Vec::new();
4688 if out.unreadable > 0 {
4689 extra.push(format!("{} unreadable", out.unreadable));
4690 }
4691 if out.orphaned_worktrees > 0 {
4692 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
4693 }
4694 let detail = if extra.is_empty() {
4695 String::new()
4696 } else {
4697 format!(" ({})", extra.join(", "))
4698 };
4699 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
4700 }
4701 if out.external_merges_recorded > 0 {
4702 tracing::info!(
4703 "housekeep: recorded {} run(s) as merged externally",
4704 out.external_merges_recorded
4705 );
4706 }
4707 if out.stale_pr_states_repaired > 0 {
4708 tracing::info!(
4709 "housekeep: rewrote {} run record(s) whose pull request had already settled",
4710 out.stale_pr_states_repaired
4711 );
4712 }
4713 if out.cache_files > 0 {
4714 tracing::info!(
4715 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
4716 out.cache_files,
4717 out.cache_freed
4718 );
4719 }
4720 if out.questions_abandoned > 0 {
4721 tracing::info!(
4722 "housekeep: abandoned {} question(s) left open by a finished run",
4723 out.questions_abandoned
4724 );
4725 }
4726}
4727
4728async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
4737 let questions = Questions::at(home.join("questions"));
4738 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
4739 if report.is_empty() {
4740 return;
4741 }
4742 if !report.quarantined.is_empty() {
4743 tracing::info!(
4744 "triage: held {} blocked task(s) whose blocked-on task or \
4745 question no longer exists: {}",
4746 report.quarantined.len(),
4747 report.quarantined.join(", ")
4748 );
4749 }
4750 if !report.resumed.is_empty() {
4751 tracing::info!(
4752 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
4753 report.resumed.len(),
4754 report.resumed.join(", ")
4755 );
4756 }
4757 if !report.asked.is_empty() {
4758 tracing::info!(
4759 "triage: asked about {} held task(s): {}",
4760 report.asked.len(),
4761 report.asked.join(", ")
4762 );
4763 }
4764 if !report.answered.is_empty() {
4765 tracing::info!(
4766 "triage: applied {} operator answer(s): {}",
4767 report.answered.len(),
4768 report.answered.join(", ")
4769 );
4770 }
4771}
4772
4773pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
4780 disk_gate_with(repo, config, crate::disk::free_bytes)
4781}
4782
4783fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
4787 repo: &Path,
4788 config: &Config,
4789 free_bytes: F,
4790) -> Option<String> {
4791 let min = config.disk.min_free_bytes;
4792 if min == 0 {
4793 return None;
4794 }
4795 let mut paths = vec![repo.to_owned()];
4796 if let Some(cache) = config.cache_dir() {
4797 paths.push(cache);
4798 }
4799 if let Some(root) = &config.graph.worktree_root {
4800 paths.push(root.clone());
4801 }
4802 for path in paths {
4803 let probe = if path == repo {
4804 path.as_path()
4805 } else {
4806 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
4807 };
4808 let reason = match free_bytes(probe) {
4809 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
4810 Err(e) => Some(crate::disk::unmeasured_in(
4811 probe,
4812 &e,
4813 &config.graph.language,
4814 )),
4815 };
4816 if reason.is_some() {
4817 return reason;
4818 }
4819 }
4820 None
4821}
4822
4823const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
4830
4831const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
4835
4836fn quota_wait(
4845 reset_at: Option<Timestamp>,
4846 now: Timestamp,
4847 fallback: Duration,
4848 cap: Duration,
4849) -> Duration {
4850 match reset_at {
4851 Some(at) if at > now => {
4852 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
4853 Duration::from_secs(secs).min(cap)
4854 }
4855 _ => fallback,
4856 }
4857}
4858
4859fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
4873 parse_reset_hint_zoned(text, now)
4874 .or_else(|| parse_reset_hint_dated(text))
4875 .or_else(|| parse_reset_hint_relative(text, recorded))
4876}
4877
4878fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
4882 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
4883 let mut rest = rest.trim();
4884 if rest.is_empty() {
4885 return None;
4886 }
4887 let mut total: i64 = 0;
4888 let mut matched = false;
4889 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
4890 if let Some((digits, tail)) = rest.split_once(unit)
4891 && !digits.is_empty()
4892 && digits.bytes().all(|b| b.is_ascii_digit())
4893 {
4894 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
4895 rest = tail;
4896 matched = true;
4897 }
4898 }
4899 if !rest.is_empty() || !matched {
4900 return None;
4901 }
4902 recorded
4903 .checked_add(jiff::SignedDuration::from_secs(total))
4904 .ok()
4905}
4906
4907fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
4911 let clock = clock.trim().to_lowercase();
4912 let (digits, pm) = clock
4913 .strip_suffix("am")
4914 .map(|d| (d, false))
4915 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
4916 let (h, m) = digits.trim().split_once(':')?;
4917 let mut hour: i8 = h.trim().parse().ok()?;
4918 let minute: i8 = m.trim().parse().ok()?;
4919 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
4920 return None;
4921 }
4922 if pm && hour != 12 {
4923 hour += 12;
4924 } else if !pm && hour == 12 {
4925 hour = 0;
4926 }
4927 Some((hour, minute))
4928}
4929
4930fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
4935 let open = text.find('(')?;
4936 let close = text.rfind(')')?;
4937 if close <= open {
4938 return None;
4939 }
4940 let zone = text[open + 1..close].trim();
4941 let (hour, minute) = parse_12h_clock(&text[..open])?;
4942 let tz = jiff::tz::TimeZone::get(zone).ok()?;
4943 let candidate = now
4944 .to_zoned(tz)
4945 .with()
4946 .hour(hour)
4947 .minute(minute)
4948 .second(0)
4949 .millisecond(0)
4950 .microsecond(0)
4951 .nanosecond(0)
4952 .build()
4953 .ok()?;
4954 let mut at = candidate.timestamp();
4955 if at <= now {
4956 at += jiff::SignedDuration::from_hours(24);
4957 }
4958 Some(at)
4959}
4960
4961fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
4970 let words: Vec<&str> = text.split_whitespace().collect();
4971 if words.len() < 5 {
4972 return None;
4973 }
4974 (0..=words.len() - 5)
4975 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
4976}
4977
4978fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
4984 if trailing.is_some_and(|next| next.starts_with('(')) {
4985 return None;
4986 }
4987 let month = month_number(window[0])?;
4988 let day_token = window[1].strip_suffix(',')?.to_lowercase();
4989 let day_digits = ["st", "nd", "rd", "th"]
4990 .iter()
4991 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
4992 let day: i8 = day_digits.parse().ok()?;
4993 let year_token = window[2];
4994 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
4995 return None;
4996 }
4997 let year: i16 = year_token.parse().ok()?;
4998 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5002 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5003 let date = jiff::civil::Date::new(year, month, day).ok()?;
5004 let candidate = date
5005 .at(hour, minute, 0, 0)
5006 .to_zoned(jiff::tz::TimeZone::UTC)
5007 .ok()?;
5008 Some(candidate.timestamp())
5009}
5010
5011fn month_number(name: &str) -> Option<i8> {
5014 const NAMES: [&str; 12] = [
5015 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5016 ];
5017 let lower = name.to_lowercase();
5018 NAMES
5019 .iter()
5020 .position(|n| *n == lower.as_str())
5021 .map(|i| i as i8 + 1)
5022}
5023
5024#[cfg(test)]
5026fn exhausted_review_budget(state: &RunState) -> bool {
5027 !crate::handover::can_resume(state)
5030 && state.status == RunStatus::Blocked
5031 && !state.released()
5032 && state.retired_to.is_none()
5033}
5034
5035fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5072 unfinished_run_with(runs, short, RunState::load)
5073}
5074
5075pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5078where
5079 F: FnMut(&str) -> Result<RunState>,
5080{
5081 let mut id = runs.last()?.clone();
5082 let mut visited = std::collections::BTreeSet::new();
5083 let mut repo = None;
5084 loop {
5085 if !runs.contains(&id) || !visited.insert(id.clone()) {
5086 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5087 return None;
5088 }
5089 let state = match load(&id) {
5090 Ok(state) if state.id == id => state,
5091 Ok(_) => return None,
5092 Err(e) => {
5093 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5094 return None;
5095 }
5096 };
5097 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5098 return None;
5099 }
5100 repo = Some(state.repo.clone());
5101 if let Some(authority) = state.retired_to {
5102 id = authority;
5103 continue;
5104 }
5105 return (crate::handover::can_resume(&state)
5106 && state.liveness(false) != crate::run::Liveness::Live)
5107 .then_some(id);
5108 }
5109}
5110
5111fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5119where
5120 F: FnOnce(&str) -> Result<RunState>,
5121{
5122 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5124 || task.fresh_start
5125 || task.review_branch.is_some()
5126 {
5127 return false;
5128 }
5129 let Some(id) = task.runs.last() else {
5130 return false;
5131 };
5132 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5133}
5134
5135#[derive(Debug, Clone, PartialEq, Eq)]
5138enum Starter {
5139 Review(String),
5142 Resume(String),
5144 Start,
5146}
5147
5148fn retained_attempt_hold(
5152 task: &Task,
5153 starter: &Starter,
5154 limit: usize,
5155 mut load: impl FnMut(&str) -> Result<RunState>,
5156) -> Option<String> {
5157 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5158 return None;
5159 }
5160 let mut retained = 0;
5161 for id in task.runs.iter().collect::<std::collections::BTreeSet<_>>() {
5162 let state = match load(id) {
5163 Ok(state) if state.id == *id => state,
5164 _ => {
5165 return Some(format!(
5166 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5167 ));
5168 }
5169 };
5170 if state.retired_to.is_some() || state.released() {
5171 continue;
5172 }
5173 retained += 1;
5176 }
5177 (retained >= limit).then(|| format!(
5178 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5179 ))
5180}
5181
5182fn selected_starter(
5184 selected: &crate::pr_recovery::Selection,
5185 previous: Option<&RunState>,
5186 unfinished: Option<&str>,
5187) -> Starter {
5188 if let Some(id) = unfinished
5189 && previous.is_some_and(|r| {
5190 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5191 })
5192 {
5193 Starter::Resume(id.to_owned())
5194 } else {
5195 Starter::Review(selected.branch.clone())
5196 }
5197}
5198
5199fn take_divergence_answer(
5203 branch: &str,
5204 remote: &str,
5205 task: &mut Task,
5206) -> Option<crate::reconcile::Choice> {
5207 let summary = crate::reconcile::summary_for(branch, remote);
5208 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5209 (a.question == summary)
5210 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5211 .flatten()
5212 .map(|c| (i, c))
5213 })?;
5214 task.answers.remove(idx);
5215 Some(choice)
5216}
5217
5218fn choose_starter(
5230 review_branch: Option<&str>,
5231 branch_exists: bool,
5232 unfinished: Option<&str>,
5233) -> Starter {
5234 match review_branch {
5235 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5236 Some(_) => Starter::Start,
5237 None => match unfinished {
5238 Some(id) => Starter::Resume(id.to_owned()),
5239 None => Starter::Start,
5240 },
5241 }
5242}
5243
5244fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5246 let branch = task.review_branch.take();
5247 let instruction = task.review_instruction.take();
5248 branch.filter(|branch| match &instruction {
5249 Some(authored) => *authored == task.instruction,
5250 None => previous.map_or_else(
5253 || task.review_of.as_ref() == Some(branch),
5254 |run| compatible_run_instruction(run, task),
5255 ),
5256 })
5257}
5258
5259fn may_resume_run(run: &RunState, task: &Task) -> bool {
5262 task.resume_override
5263 .as_ref()
5264 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5265 || compatible_run_instruction(run, task)
5266}
5267
5268pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5270 if let Some(authored) = &run.task_instruction {
5271 return compatible_instruction(authored, task);
5272 }
5273 if compatible_instruction(&run.instruction, task) {
5274 return true;
5275 }
5276 if run.reviewed_commits.is_some() {
5279 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5280 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5281 }
5282 if let Some(branch) = &task.review_of {
5283 let canonical = format!(
5284 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5285 );
5286 let generated = format!(
5287 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5288 );
5289 return task.instruction == canonical && run.instruction.starts_with(&generated);
5290 }
5291 }
5292 false
5293}
5294
5295pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5297 strip_answers_block(old, task) == task.instruction
5298}
5299
5300fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5303 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5304 return fallback.to_path_buf();
5305 }
5306 task.repo.clone()
5307}
5308
5309const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
5313
5314fn answers_block(task: &Task, count: usize) -> String {
5316 let mut s = ANSWERS_HEADER.to_owned();
5317 for a in &task.answers[..count] {
5318 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
5319 }
5320 s
5321}
5322
5323fn resumption_block(r: &crate::queue::Resumption) -> String {
5326 format!(
5327 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
5328 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
5329 r.generation, r.at, r.instruction, r.answers_through,
5330 )
5331}
5332
5333fn append_answers(base: &str, task: &Task) -> String {
5335 let mut s = base.to_owned();
5336 if !task.answers.is_empty() {
5337 s.push_str(&answers_block(task, task.answers.len()));
5338 }
5339 if let Some(r) = &task.resumption {
5340 s.push_str(&resumption_block(r));
5341 }
5342 s
5343}
5344
5345fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
5349 let mut instruction = instruction;
5350 for approval in task
5351 .resumption
5352 .iter()
5353 .chain(task.resumption_history.iter().rev())
5354 {
5355 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
5356 instruction = base;
5357 break;
5358 }
5359 }
5360 for count in (1..=task.answers.len()).rev() {
5361 let block = answers_block(task, count);
5362 if let Some(base) = instruction.strip_suffix(&block) {
5363 return base;
5364 }
5365 }
5366 instruction
5367}
5368
5369fn instruction_for(task: &Task) -> String {
5377 append_answers(&task.instruction, task)
5378}
5379
5380fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
5392 append_answers(strip_answers_block(old_instruction, task), task)
5393}
5394
5395fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
5397 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
5398 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
5399 if let Some(cause) = task.hold_notice_cause.as_mut() {
5400 cause.target = failure.map(|f| f.target.clone());
5401 }
5402}
5403
5404#[derive(Debug, thiserror::Error)]
5405#[error("attachment `{name}` is recorded on the task but {path} is missing")]
5406struct MissingAttachment {
5407 name: String,
5408 path: PathBuf,
5409}
5410
5411fn task_attachments(
5413 queue: &Queue,
5414 task: &Task,
5415) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
5416 let paths = queue.attachment_paths(task);
5417 for (name, path) in task.attachments.iter().zip(&paths) {
5418 if !path.is_file() {
5419 return Err(MissingAttachment {
5420 name: name.clone(),
5421 path: path.clone(),
5422 });
5423 }
5424 }
5425 Ok(paths)
5426}
5427
5428fn prepare_instruction(
5439 starter: &Starter,
5440 old_instruction: Option<&str>,
5441 task: &Task,
5442) -> Option<String> {
5443 match starter {
5444 Starter::Start => Some(instruction_for(task)),
5445 Starter::Resume(_) => Some(resumed_instruction(
5446 old_instruction.expect("a resumed run always has a prior instruction"),
5447 task,
5448 )),
5449 Starter::Review(_) => None,
5450 }
5451}
5452
5453fn record(queue: &Queue, task: &mut Task) {
5457 if let Err(e) = queue.put(task) {
5458 tracing::error!("could not record task {}: {e:#}", task.short());
5459 let notice = Notice::error(
5460 "loop:record",
5461 "The loop could not save a task's state; check the disk.",
5462 )
5463 .link(Link::Task {
5464 id: task.id.clone(),
5465 });
5466 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
5468 Some(home) => notices::raise_in(home, notice),
5469 None => notices::raise(notice),
5470 }
5471 }
5472}
5473
5474fn runnable(queue: &Queue) -> Vec<Task> {
5480 let mut tasks: Vec<Task> = queue
5481 .list()
5482 .into_iter()
5483 .filter(|t| t.status.runnable())
5484 .collect();
5485 tasks.sort_unstable_by(crate::queue::dispatch_order);
5486 tasks
5487}
5488
5489fn describe(state: &RunState) -> String {
5503 let p = phrases(&state.config.graph.language);
5504 let mut detail = if state.status == RunStatus::Stalled {
5505 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
5506 seats.sort_unstable();
5507 seats.dedup();
5508 if seats.is_empty() {
5509 p.quorum_lost.to_owned()
5510 } else {
5511 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
5512 }
5513 } else {
5514 format!("{}{}", p.run_ended, state.status.display_label())
5515 };
5516 if let Some(last) = state.events.last() {
5517 let unanswered = state
5521 .reviews
5522 .last()
5523 .filter(|r| {
5524 state.status == RunStatus::Blocked
5525 && last.node == "review"
5526 && r.incomplete()
5527 && r.blocking == 0
5528 && r.round == state.config.graph.review_rounds
5529 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
5530 })
5531 .map(|r| (r.expected - r.answered, r.round));
5532 match unanswered {
5533 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
5534 detail.push_str(&format!(
5535 " ({}: {})",
5536 last.node,
5537 (p.reviewers_never_answered)(missing, rounds)
5538 ));
5539 }
5540 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
5541 }
5542 }
5543 detail.push_str(&format!(" [run {}]", state.id));
5544 detail
5545}
5546
5547const DIAGNOSTIC_MAX: usize = 4_000;
5553
5554const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
5559
5560fn diagnostic(state: &RunState) -> Option<String> {
5574 let mut parts: Vec<String> = Vec::new();
5575
5576 for o in state.gate.iter().filter(|o| !o.ok()) {
5578 parts.push(format!(
5579 "gate `{}` failed ({}):\n{}",
5580 o.command,
5581 notices::exit_status(o.code),
5582 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
5583 ));
5584 }
5585
5586 if let Some(last) = state
5589 .events
5590 .iter()
5591 .rev()
5592 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
5593 {
5594 parts.push(last.message.clone());
5595 }
5596
5597 if state.viable().is_empty() {
5604 for c in &state.candidates {
5605 if let Some(evidence) = &c.verified_noop {
5606 parts.push(format!(
5607 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
5608 c.label
5609 ));
5610 } else if !c.summary.trim().is_empty() {
5611 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
5612 } else if let Some(why) = &c.failed {
5613 parts.push(format!("candidate {}: {why}", c.label));
5614 }
5615 }
5616 }
5617
5618 if parts.is_empty() {
5619 return None;
5620 }
5621 Some(crate::run::tail(
5626 &parts.join("\n\n"),
5627 DIAGNOSTIC_MAX.saturating_sub(100),
5628 ))
5629}
5630
5631fn label(status: RunStatus) -> &'static str {
5639 status.as_str()
5640}
5641
5642pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
5644 match mode {
5645 "none" => Ok(MergeMode::None),
5646 "local" => Ok(MergeMode::Local),
5647 "pr" => Ok(MergeMode::Pr),
5648 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
5649 }
5650}
5651
5652fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
5657 mutex
5658 .lock()
5659 .unwrap_or_else(std::sync::PoisonError::into_inner)
5660}
5661
5662#[cfg(test)]
5663mod tests {
5664 use super::*;
5665 use crate::queue::{Source, TaskStatus};
5666 use crate::run::{Candidate, CommandOutcome};
5667 use pretty_assertions::assert_eq;
5668
5669 fn task() -> Task {
5670 Task::new(
5671 "add retries".to_owned(),
5672 "add retries".to_owned(),
5673 PathBuf::from("/repo"),
5674 Source::Human,
5675 )
5676 }
5677
5678 #[test]
5679 fn selected_public_head_overrides_historical_resume_and_pin() {
5680 let mut t = task();
5681 let mut old = run_state(RunStatus::Blocked);
5682 let selection = crate::pr_recovery::Selection {
5683 url: "url".into(),
5684 branch: format!("magi/task-{}", t.id),
5685 head: "a".repeat(40),
5686 source_head: None,
5687 source_run: None,
5688 source_proof: None,
5689 disposition: Some(crate::pr_recovery::Disposition::Review),
5690 instruction: t.instruction.clone(),
5691 approved_at: Timestamp::now(),
5692 };
5693 t.resume_override = Some(crate::queue::OperatorResume {
5694 pinned_run: Some(old.id.clone()),
5695 question_id: "choice".into(),
5696 at: Timestamp::now(),
5697 conductor_rehold: None,
5698 forced: false,
5699 });
5700 assert_eq!(
5701 selected_starter(&selection, Some(&old), Some(&old.id)),
5702 Starter::Review(selection.branch.clone())
5703 );
5704 old.pr_recovery = Some(selection.clone());
5705 assert_eq!(
5706 selected_starter(&selection, Some(&old), Some(&old.id)),
5707 Starter::Review(selection.branch.clone())
5708 );
5709 old.reviewed_commits = Some(vec!["implementation".into()]);
5710 assert_eq!(
5711 selected_starter(&selection, Some(&old), Some(&old.id)),
5712 Starter::Resume(old.id.clone())
5713 );
5714 let mut changed = selection.clone();
5715 changed.head = "b".repeat(40);
5716 assert_eq!(
5717 selected_starter(&changed, Some(&old), Some(&old.id)),
5718 Starter::Review(changed.branch)
5719 );
5720 }
5721
5722 #[test]
5723 fn candidate_transplant_failure_is_one_actionable_hold() {
5724 let dir = tempfile::tempdir().unwrap();
5725 let q = Queue::at(dir.path().join("queue"));
5726 let mut t = task();
5727 let mut state = run_state(RunStatus::Blocked);
5728 state.event("prep", "cannot apply referenced commit: conflict");
5729 state.prep_failure = Some("conflict".into());
5730 t.start(state.id.clone());
5731 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
5732 assert_eq!(t.attempts, 1);
5733 assert_eq!(t.status, TaskStatus::Held);
5734 assert!(t.operator_held());
5735 assert!(!t.status.runnable());
5736 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
5737 let first = t.chat_notice_key().unwrap();
5738 t.chat_notices.push(first.clone());
5739 t.release_operator("test", None);
5740 state.id = "another-unprepared-run".into();
5741 t.start(state.id.clone());
5742 finish_attempt(
5743 3,
5744 &q,
5745 &mut t,
5746 &state,
5747 &[],
5748 Err(anyhow::anyhow!("still conflicts")),
5749 );
5750 assert_eq!(
5751 t.chat_notice_key().as_ref(),
5752 Some(&first),
5753 "no work happened"
5754 );
5755 t.release_operator("test", None);
5756 state.prep_failure = None;
5757 state.status = RunStatus::Failed;
5758 state.begin_pass();
5759 t.start(state.id.clone());
5760 finish_attempt(
5761 1,
5762 &q,
5763 &mut t,
5764 &state,
5765 &[],
5766 Err(anyhow::anyhow!("new gate failure")),
5767 );
5768 let failure = t.chat_notice_key().unwrap();
5769 assert_ne!(failure, first, "approved recovery reached a real failure");
5770 t.release_operator("test", None);
5771 state.begin_pass();
5772 t.start(state.id.clone());
5773 finish_attempt(
5774 1,
5775 &q,
5776 &mut t,
5777 &state,
5778 &[],
5779 Err(anyhow::anyhow!("new failure on resume")),
5780 );
5781 assert_ne!(
5782 t.chat_notice_key().as_ref(),
5783 Some(&failure),
5784 "a new settled pass"
5785 );
5786 }
5787
5788 #[test]
5789 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
5790 for source in ["cli", "web"] {
5791 let mut t = task();
5792 let mut run = run_state(RunStatus::Blocked);
5793 run.instruction = "Run A original criteria".into();
5794 run.task_instruction = Some(run.instruction.clone());
5795 t.start(run.id.clone());
5796 t.release_operator("choice", Some("resume-question"));
5797 t.resume_override = Some(crate::queue::OperatorResume {
5798 question_id: "resume-question".into(),
5799 at: Timestamp::now(),
5800 conductor_rehold: None,
5801 forced: true,
5802 pinned_run: Some(run.id.clone()),
5803 });
5804 t.hold_machine(Some("not enough free space to start a run".into()));
5805 let mut unchanged = t.clone();
5806 unchanged
5807 .edit("title changed".into(), unchanged.instruction.clone())
5808 .unwrap();
5809 unchanged.release_operator(source, None);
5810 assert_eq!(
5811 unchanged
5812 .resume_override
5813 .as_ref()
5814 .unwrap()
5815 .pinned_run
5816 .as_deref(),
5817 Some(run.id.as_str())
5818 );
5819 assert!(!compatible_run_instruction(&run, &unchanged));
5820 assert!(may_resume_run(&run, &unchanged));
5821
5822 t.edit(
5823 "approved recovery".into(),
5824 "Reuse measured PR head B; fix Clippy; review".into(),
5825 )
5826 .unwrap();
5827 assert!(t.resume_override.is_none());
5828 t.release_operator(source, None);
5829 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
5830 assert!(reloaded.resume_override.is_none());
5831 assert!(!may_resume_run(&run, &reloaded));
5832 let unfinished =
5833 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
5834 .filter(|_| may_resume_run(&run, &reloaded));
5835 assert_eq!(
5836 choose_starter(None, false, unfinished.as_deref()),
5837 Starter::Start
5838 );
5839 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
5840 assert_eq!(
5841 reloaded.resumption_history[0].question_id.as_deref(),
5842 Some("resume-question")
5843 );
5844
5845 unchanged.instruction = "Reuse measured PR head B".into();
5848 unchanged.hold_machine(Some("disk hold".into()));
5849 unchanged.release_operator(source, None);
5850 assert!(unchanged.resume_override.is_none());
5851 assert!(!may_resume_run(&run, &unchanged));
5852 }
5853 }
5854
5855 #[test]
5856 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
5857 let mut t = task();
5858 let mut run = run_state(RunStatus::Blocked);
5859 t.review_of = Some("approved-head".into());
5860 t.instruction =
5861 "Review the branch `approved-head` (review-only: no implementation, no judging)."
5862 .into();
5863 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
5864 run.reviewed_commits = Some(vec!["Commit log".into()]);
5865 t.start(run.id.clone());
5866 t.hold_machine(Some("interrupted review".into()));
5867 t.release_operator("cli", None);
5868 assert!(
5869 compatible_run_instruction(&run, &t),
5870 "legacy explicit review remains resumable"
5871 );
5872 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
5873 .filter(|_| compatible_run_instruction(&run, &t));
5874 assert_eq!(
5875 choose_starter(None, false, unfinished.as_deref()),
5876 Starter::Resume(run.id.clone())
5877 );
5878 run.task_instruction = Some(t.instruction.clone());
5879 assert!(compatible_run_instruction(&run, &t));
5880 let reloaded: RunState =
5881 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
5882 assert!(compatible_run_instruction(&reloaded, &t));
5883 t.edit(
5884 "new approved head".into(),
5885 "Reuse measured PR head B".into(),
5886 )
5887 .unwrap();
5888 assert!(!compatible_run_instruction(&run, &t));
5889
5890 t.review_of = None;
5892 t.instruction = "Fix Clippy on approved head".into();
5893 run.task_instruction = None;
5894 run.instruction = format!(
5895 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
5896 t.instruction
5897 );
5898 assert!(compatible_run_instruction(&run, &t));
5899 t.instruction.push('\n');
5900 assert!(
5901 compatible_run_instruction(&run, &t),
5902 "legacy generated review trimmed the original criteria"
5903 );
5904 t.hold_for_handover(Some("approved-head".into()), "refused".into());
5905 let mut unchanged = t.clone();
5906 assert_eq!(
5907 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
5908 Some("approved-head")
5909 );
5910 t.edit("new plan".into(), "Reuse measured PR head B".into())
5911 .unwrap();
5912 assert!(take_review_branch(&mut t, Some(&run)).is_none());
5913 assert!(!compatible_run_instruction(&run, &t));
5914 t.review_branch = Some("approved-head".into());
5915 t.review_instruction = None;
5916 assert!(
5917 take_review_branch(&mut t, Some(&run)).is_none(),
5918 "legacy retained candidate cannot replace a revised approved head"
5919 );
5920 }
5921
5922 #[test]
5923 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
5924 crate::run::pin_test_home();
5925 let dir = tempfile::tempdir().unwrap();
5926 let queue = Queue::at(dir.path().join("queue"));
5927 for resume in [true, false] {
5928 let state = run_state(RunStatus::Blocked);
5929 let mut t = task();
5930 t.start(state.id.clone());
5931 let action = if resume {
5932 ask::ChoiceAction::Resume {
5933 run: state.id.clone(),
5934 }
5935 } else {
5936 ask::ChoiceAction::Requeue
5937 };
5938 let mut q = action_question(&state.id, action);
5939 q.status = ask::QuestionStatus::Open;
5940 q.answer = None;
5941 q.answered_at = None;
5942 t.fail("run stopped with a question", 3);
5943 hold_if_runnable(&queue, &mut t);
5944 assert!(t.operator_decided_at.is_none());
5945 assert!(t.operator_held());
5946 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
5947 let expected = if resume {
5948 ActionDecision::Resume(state.id.clone())
5949 } else {
5950 ActionDecision::Requeue
5951 };
5952 assert_eq!(
5953 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5954 expected
5955 );
5956 t.hold_manual(Some("later explicit hold".into()));
5957 assert_eq!(
5958 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5959 ActionDecision::Stale
5960 );
5961 }
5962 }
5963
5964 #[test]
5965 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
5966 let mut t = task();
5967 t.answers.push(crate::queue::AnsweredQuestion {
5968 question: "Inspect the remote?".into(),
5969 answer: "Inspect then wait".into(),
5970 });
5971 let old = instruction_for(&t);
5972 assert!(compatible_instruction(&old, &t));
5973 t.instruction = "Reuse approved PR head and review".into();
5974 t.release_operator("cli", None);
5975 assert!(!compatible_instruction(&old, &t));
5976 let approved = instruction_for(&t);
5977 assert!(approved.contains("waiting premises are superseded"));
5978 assert!(compatible_instruction(&approved, &t));
5979 assert_eq!(resumed_instruction(&approved, &t), approved);
5980 t.release_operator("web", None);
5981 assert!(compatible_instruction(&approved, &t));
5982 assert_eq!(t.resumption_history.len(), 1);
5983 t.start("run".into());
5984 assert!(instruction_for(&t).contains("Explicit operator resumption"));
5985 }
5986
5987 #[test]
5991 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
5992 let mut state = RunState::new(
5993 PathBuf::from("/repo"),
5994 "main".to_owned(),
5995 "abc1234".to_owned(),
5996 "task".to_owned(),
5997 Config::default(),
5998 );
5999 state.status = RunStatus::Blocked;
6000 state.resource_block = Some(crate::run::ResourceBlock {
6001 node: "graft".to_owned(),
6002 kind: "shared build cache".to_owned(),
6003 detail: "timed out".to_owned(),
6004 at: jiff::Timestamp::now(),
6005 });
6006 let mut t = task();
6007 t.start("20260902-000000-aaaa".to_owned());
6008 assert_eq!(t.attempts, 1);
6009 let verdict = Verdict {
6010 status: RunStatus::Blocked,
6011 left_pr: false,
6012 quota_hit: false,
6013 parked: false,
6014 no_viable_candidates: false,
6015 };
6016 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
6017 assert_eq!(t.status, TaskStatus::Held);
6018 assert_eq!(t.attempts, 0, "the attempt is refunded");
6019 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
6020 assert!(
6021 !t.operator_held(),
6022 "a machine hold, released without a human"
6023 );
6024
6025 let mut selected = task();
6026 selected.start("20260902-000000-cccc".into());
6027 selected.replacement = Some(crate::merge_decision::Replacement {
6028 question: "fixture".into(),
6029 run: "earlier".into(),
6030 pr: "https://example.invalid/pr/1".into(),
6031 agent: "selected".into(),
6032 context: "fixture".into(),
6033 });
6034 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
6035 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
6036 assert_eq!(selected.attempts, 0);
6037 assert!(
6038 !selected.operator_held(),
6039 "a selected agent also retains a machine resource hold"
6040 );
6041 assert!(
6042 selected
6043 .hold_reason
6044 .as_deref()
6045 .unwrap()
6046 .starts_with("[resource] ")
6047 );
6048
6049 let mut p = task();
6051 p.start("20260902-000000-bbbb".to_owned());
6052 settle_and_diagnose(
6053 &mut p,
6054 Verdict {
6055 parked: true,
6056 ..verdict
6057 },
6058 "parked",
6059 2,
6060 &state,
6061 );
6062 assert_eq!(p.status, TaskStatus::Parked);
6063 }
6064
6065 #[test]
6066 fn record_failure_notice_links_the_task() {
6067 let dir = tempfile::tempdir().unwrap();
6068 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
6071 let queue = Queue::at(dir.path().join("queue"));
6072 let mut t = interrupt_task("rec-fail-1");
6073 record(&queue, &mut t);
6074
6075 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
6076 assert_eq!(found.len(), 1);
6077 assert_eq!(found[0].key, "loop:record");
6078 assert_eq!(
6079 found[0].link,
6080 Some(Link::Task {
6081 id: "rec-fail-1".to_owned()
6082 })
6083 );
6084 }
6085
6086 fn interrupt_task(id: &str) -> Task {
6089 let mut t = task();
6090 t.id = id.to_owned();
6091 t.interrupt = true;
6092 t
6093 }
6094
6095 fn task_with_id(id: &str) -> Task {
6097 let mut t = task();
6098 t.id = id.to_owned();
6099 t
6100 }
6101
6102 fn urgent_task(id: &str) -> Task {
6104 let mut t = task();
6105 t.id = id.to_owned();
6106 t.urgent = true;
6107 t
6108 }
6109
6110 #[test]
6111 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
6112 let dir = tempfile::tempdir().unwrap();
6113 let queue = Queue::at(dir.path().join("queue"));
6114 let mut parked = task_with_id("parked");
6115 parked.start("run".into());
6116 parked.park("upgrade");
6117 parked.record_pass_at(
6118 "run",
6119 0,
6120 Some(Timestamp::now()),
6121 1,
6122 Some(crate::run::ParkCause::Upgrade {
6123 from: "old".into(),
6124 to: None,
6125 }),
6126 );
6127 let mut fresh = task_with_id("fresh");
6128 fresh.priority = 100;
6129 let mut urgent_task = urgent_task("urgent");
6130 for t in [&mut parked, &mut fresh, &mut urgent_task] {
6131 queue.put(t).unwrap();
6132 }
6133 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6134 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6135 let candidates = runnable(&queue);
6136 assert_eq!(candidates[0].id, parked.id);
6137 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
6138 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
6139 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
6140 drop(first);
6141 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
6142 let next = runnable(&queue)
6143 .into_iter()
6144 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
6145 .unwrap();
6146 assert_eq!(next.task.id, fresh.id);
6147 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
6148 drop(claimed_elsewhere);
6149 drop(next);
6150 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
6151 }
6152
6153 #[test]
6154 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
6155 let dir = tempfile::tempdir().unwrap();
6156 let queue = Queue::at(dir.path().join("queue"));
6157 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6158 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6159 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
6160 let mut first = task();
6161 queue.put(&mut first).unwrap();
6162 let stale_plain = queue.get(&first.id).unwrap();
6163 first.set_urgent(true).unwrap();
6164 queue.put(&mut first).unwrap();
6165 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
6166 let mut second = urgent_task("second");
6167 queue.put(&mut second).unwrap();
6168 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
6169 drop(queue.claim(&second.id).unwrap());
6170 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
6171 drop(dispatched);
6172 let stale_urgent = queue.get(&first.id).unwrap();
6173 first.set_urgent(false).unwrap();
6174 queue.put(&mut first).unwrap();
6175 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
6176 assert_eq!(urgent.available_permits(), 1);
6177 drop(queue.claim(&first.id).unwrap());
6178 drop(occupied);
6179 }
6180
6181 #[test]
6182 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
6183 let dir = tempfile::tempdir().unwrap();
6184 let queue = Queue::at(dir.path().join("queue"));
6185 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6186 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6187 let mut t = task();
6188 queue.put(&mut t).unwrap();
6189 for enabled in [true, false] {
6190 let stale = queue.get(&t.id).unwrap();
6191 t.set_urgent(enabled).unwrap();
6192 queue.put(&mut t).unwrap();
6193 assert_ne!(stale.urgent, enabled);
6194 let DispatchClaim {
6195 task: fresh,
6196 claim,
6197 permit,
6198 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
6199 assert_eq!(fresh.urgent, enabled);
6200 assert_eq!(ordinary.available_permits(), usize::from(enabled));
6201 assert_eq!(urgent.available_permits(), usize::from(!enabled));
6202 assert!(queue.claim(&t.id).is_err());
6203 assert_eq!(fresh.attempts, stale.attempts);
6204 drop((claim, permit));
6205 assert_eq!(ordinary.available_permits(), 1);
6206 assert_eq!(urgent.available_permits(), 1);
6207 }
6208 t.set_urgent(true).unwrap();
6209 queue.put(&mut t).unwrap();
6210 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
6211 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6212 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
6213 drop(queue.claim(&t.id).unwrap());
6214 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
6215 drop(occupied);
6216 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
6217 drop(queue.claim(&t.id).unwrap());
6218 let held = queue.claim(&t.id).unwrap();
6219 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6220 drop(held);
6221 t.status = TaskStatus::Held;
6222 queue.put(&mut t).unwrap();
6223 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6224 drop(queue.claim(&t.id).unwrap());
6225 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
6226 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6227 drop(queue.claim(&t.id).unwrap());
6228 assert_eq!(urgent.available_permits(), 1);
6229 }
6230
6231 #[test]
6236 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
6237 assert_eq!(permit_kind(true, false), PermitKind::None);
6238 assert_eq!(permit_kind(true, true), PermitKind::None);
6239 }
6240
6241 #[test]
6246 fn permit_kind_separates_urgent_from_ordinary() {
6247 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
6248 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
6249 }
6250
6251 #[test]
6258 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
6259 let cfg = Config::default();
6260 let repo = Path::new("/any/repo/path");
6261
6262 let reason =
6263 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
6264 assert!(reason.contains("1024"), "{reason}");
6265 assert!(
6266 reason.contains(&cfg.disk.min_free_bytes.to_string()),
6267 "{reason}"
6268 );
6269
6270 assert_eq!(
6271 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
6272 None,
6273 "exactly at the floor is open"
6274 );
6275 assert_eq!(
6276 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
6277 None,
6278 "comfortably above the floor is open"
6279 );
6280 }
6281
6282 #[test]
6283 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
6284 let mut cfg = Config::default();
6285 cfg.disk.min_free_bytes = 0;
6286 let repo = Path::new("/any/repo/path");
6287 assert_eq!(
6288 disk_gate_with(repo, &cfg, |_| Ok(0)),
6289 None,
6290 "a zero floor never measures at all"
6291 );
6292 }
6293
6294 #[test]
6295 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
6296 let cfg = Config::default();
6297 let repo = Path::new("/any/repo/path");
6298 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
6299 .expect("a measurement failure must close the gate, not open it");
6300 assert!(reason.contains("could not measure"), "{reason}");
6301 }
6302
6303 #[test]
6304 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
6305 let ordinary = task();
6306 let next = advance_interrupt(
6307 Interrupt::Idle,
6308 std::slice::from_ref(&ordinary.id),
6309 std::slice::from_ref(&ordinary),
6310 );
6311 assert_eq!(next, Interrupt::Idle);
6312 }
6313
6314 #[test]
6315 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
6316 let marked = interrupt_task("marked");
6319 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
6320 assert_eq!(next, Interrupt::Idle);
6321 }
6322
6323 #[test]
6324 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
6325 let marked = interrupt_task("marked");
6326 let next = advance_interrupt(
6327 Interrupt::Idle,
6328 &["running".to_owned()],
6329 std::slice::from_ref(&marked),
6330 );
6331 assert_eq!(
6332 next,
6333 Interrupt::Parking {
6334 parked: vec!["running".to_owned()],
6335 interrupt_task: "marked".to_owned(),
6336 }
6337 );
6338 }
6339
6340 #[test]
6349 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
6350 let marked = interrupt_task("marked");
6351
6352 let two = advance_interrupt(
6353 Interrupt::Idle,
6354 &["a".to_owned(), "b".to_owned()],
6355 std::slice::from_ref(&marked),
6356 );
6357 assert_eq!(two, Interrupt::Idle);
6358
6359 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
6360 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
6361 }
6362
6363 #[test]
6364 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
6365 let state = Interrupt::Parking {
6366 parked: vec!["running".to_owned()],
6367 interrupt_task: "marked".to_owned(),
6368 };
6369 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
6371 assert_eq!(still_going, state);
6372
6373 let stopped_but_not_yet_dispatched =
6377 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
6378 assert_eq!(stopped_but_not_yet_dispatched, state);
6379
6380 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
6382 assert_eq!(
6383 dispatched,
6384 Interrupt::Running {
6385 parked: vec!["running".to_owned()],
6386 interrupt_task: "marked".to_owned(),
6387 }
6388 );
6389 }
6390
6391 #[test]
6392 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
6393 let state = Interrupt::Running {
6394 parked: vec!["running".to_owned()],
6395 interrupt_task: "marked".to_owned(),
6396 };
6397 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
6398 assert_eq!(still_running, state);
6399
6400 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
6407 assert_eq!(
6408 ended,
6409 Interrupt::Resuming {
6410 parked: vec!["running".to_owned()]
6411 }
6412 );
6413 }
6414
6415 #[test]
6416 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
6417 let state = Interrupt::Resuming {
6418 parked: vec!["running".to_owned()],
6419 };
6420 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
6421 assert_eq!(still_waiting, state);
6422
6423 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
6424 assert_eq!(dispatched, Interrupt::Idle);
6425 }
6426
6427 #[test]
6433 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
6434 {
6435 let state = Interrupt::Parking {
6436 parked: vec!["running".to_owned()],
6437 interrupt_task: "marked".to_owned(),
6438 };
6439 let next = advance_interrupt(state, &[], &[]);
6442 assert_eq!(
6443 next,
6444 Interrupt::Resuming {
6445 parked: vec!["running".to_owned()]
6446 },
6447 "abandoning the interrupt must not abandon the resume it owes"
6448 );
6449 }
6450
6451 #[test]
6454 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
6455 let state = Interrupt::Resuming {
6456 parked: vec!["running".to_owned()],
6457 };
6458 let next = advance_interrupt(state, &[], &[]);
6459 assert_eq!(
6460 next,
6461 Interrupt::Idle,
6462 "nothing is left to wait for; the loop must not stay wedged"
6463 );
6464 }
6465
6466 #[test]
6467 fn disabled_by_config_the_sequence_can_never_leave_idle() {
6468 let marked = interrupt_task("marked");
6469 let next = advance_interrupt_tick(
6470 false,
6471 Interrupt::Idle,
6472 &["running".to_owned()],
6473 std::slice::from_ref(&marked),
6474 );
6475 assert_eq!(
6476 next,
6477 Interrupt::Idle,
6478 "an unmarked, unconfigured daemon must behave exactly as before"
6479 );
6480 }
6481
6482 #[test]
6483 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
6484 let state = Interrupt::Parking {
6485 parked: vec!["running".to_owned()],
6486 interrupt_task: "marked".to_owned(),
6487 };
6488 let candidates = vec![interrupt_task("marked"), task()];
6489 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
6490 assert!(
6491 allowed.is_empty(),
6492 "nothing may dispatch - not even the interrupt task itself - \
6493 until the parked run has actually stopped"
6494 );
6495 }
6496
6497 #[test]
6511 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
6512 for state in [
6513 Interrupt::Parking {
6514 parked: vec!["running".to_owned()],
6515 interrupt_task: "marked".to_owned(),
6516 },
6517 Interrupt::Running {
6518 parked: vec!["running".to_owned()],
6519 interrupt_task: "marked".to_owned(),
6520 },
6521 Interrupt::Resuming {
6522 parked: vec!["running".to_owned()],
6523 },
6524 ] {
6525 let candidates = vec![
6526 interrupt_task("marked"),
6527 urgent_task("hot"),
6528 task_with_id("ordinary"),
6529 ];
6530 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
6531 assert!(
6532 !allowed.iter().any(|t| t.id == "hot"),
6533 "an urgent candidate must wait out the same gate as anything \
6534 else while the run it would run alongside has not actually \
6535 left flight, for state {state:?}: {allowed:?}"
6536 );
6537 }
6538 }
6539
6540 #[test]
6546 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
6547 let state = Interrupt::Resuming {
6548 parked: vec!["hot".to_owned()],
6549 };
6550 let candidates = vec![urgent_task("hot"), task()];
6551 let allowed = interrupt_gate(&state, &[], candidates);
6552 assert_eq!(
6553 allowed.iter().filter(|t| t.id == "hot").count(),
6554 1,
6555 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
6556 );
6557 }
6558
6559 #[test]
6560 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
6561 let state = Interrupt::Parking {
6562 parked: vec!["running".to_owned()],
6563 interrupt_task: "marked".to_owned(),
6564 };
6565 let other = task();
6566 let candidates = vec![interrupt_task("marked"), other.clone()];
6567 let allowed = interrupt_gate(&state, &[], candidates);
6568 assert_eq!(allowed.len(), 1);
6569 assert_eq!(allowed[0].id, "marked");
6570 }
6571
6572 #[test]
6573 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
6574 let state = Interrupt::Running {
6575 parked: vec!["running".to_owned()],
6576 interrupt_task: "marked".to_owned(),
6577 };
6578 let candidates = vec![task(), task()];
6579 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
6580 assert!(allowed.is_empty());
6581 }
6582
6583 #[test]
6590 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
6591 let state = Interrupt::Resuming {
6592 parked: vec!["a".to_owned(), "c".to_owned()],
6593 };
6594 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
6595 let allowed = interrupt_gate(&state, &[], candidates);
6596 assert_eq!(
6597 allowed.len(),
6598 1,
6599 "at most one candidate may be offered while resuming: {allowed:?}"
6600 );
6601 assert_eq!(allowed[0].id, "a");
6602 }
6603
6604 #[test]
6605 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
6606 let state = Interrupt::Resuming {
6607 parked: vec!["a".to_owned()],
6608 };
6609 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
6610 assert!(allowed.is_empty());
6611 }
6612
6613 #[test]
6619 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
6620 let running = task(); let marked = interrupt_task("marked");
6622
6623 let mut state = Interrupt::Idle;
6624 let in_flight = vec![running.id.clone()];
6626 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
6627 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
6628 assert!(gated.is_empty(), "still waiting on `running` to park");
6629
6630 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
6632 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
6633 assert_eq!(
6634 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6635 vec!["marked"],
6636 "only the interrupt task may be offered to the dispatcher now"
6637 );
6638
6639 state = advance_interrupt_tick(
6641 true,
6642 state,
6643 &["marked".to_owned()],
6644 std::slice::from_ref(&running),
6645 );
6646 let gated = interrupt_gate(
6647 &state,
6648 &["marked".to_owned()],
6649 vec![marked.clone(), running.clone()],
6650 );
6651 assert!(
6652 gated.is_empty(),
6653 "the parked run must not be offered back while the interrupt \
6654 task is still running"
6655 );
6656
6657 let other = task_with_id("other");
6661 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
6662 assert_eq!(
6663 state,
6664 Interrupt::Resuming {
6665 parked: vec![running.id.clone()]
6666 }
6667 );
6668 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
6669 assert_eq!(
6670 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6671 vec![running.id.as_str()],
6672 "exactly the parked run resumes - not the unrelated task, even \
6673 though it was offered first"
6674 );
6675
6676 state = advance_interrupt_tick(
6680 true,
6681 state,
6682 std::slice::from_ref(&running.id),
6683 std::slice::from_ref(&other),
6684 );
6685 assert_eq!(state, Interrupt::Idle);
6686 let gated = interrupt_gate(
6687 &state,
6688 std::slice::from_ref(&running.id),
6689 vec![other.clone()],
6690 );
6691 assert_eq!(
6692 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6693 vec![other.id.as_str()],
6694 "ordinary dispatch is unrestricted again"
6695 );
6696 }
6697
6698 #[test]
6699 fn every_run_status_settles_the_task_it_came_from() {
6700 let table = [
6702 (RunStatus::Merged, TaskStatus::Done, 1),
6703 (RunStatus::Ready, TaskStatus::Done, 1),
6704 (RunStatus::Stalled, TaskStatus::Failed, 0),
6705 (RunStatus::Blocked, TaskStatus::Failed, 1),
6706 (RunStatus::Failed, TaskStatus::Failed, 1),
6707 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
6708 (RunStatus::Prep, TaskStatus::Failed, 1),
6709 (RunStatus::Implementing, TaskStatus::Failed, 1),
6710 (RunStatus::Judging, TaskStatus::Failed, 1),
6711 (RunStatus::Deliberating, TaskStatus::Failed, 1),
6712 (RunStatus::Voting, TaskStatus::Failed, 1),
6713 (RunStatus::Reviewing, TaskStatus::Failed, 1),
6714 (RunStatus::Gating, TaskStatus::Failed, 1),
6715 ];
6716 for (run, want, attempts) in table {
6717 let mut t = task();
6718 t.start("20260902-000000-aaaa".to_owned());
6719 settle(
6720 &mut t,
6721 Verdict {
6722 status: run,
6723 left_pr: false,
6724 parked: false,
6725 quota_hit: matches!(run, RunStatus::Stalled),
6726 no_viable_candidates: false,
6727 },
6728 "why",
6729 2,
6730 );
6731 assert_eq!(t.status, want, "task status after {}", label(run));
6732 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
6733 }
6734 }
6735
6736 #[test]
6737 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
6738 let mut stalled = task();
6739 stalled.start("20260902-000000-aaaa".to_owned());
6740 settle(
6741 &mut stalled,
6742 Verdict {
6743 status: RunStatus::Stalled,
6744 left_pr: false,
6745 parked: false,
6746 quota_hit: true,
6747 no_viable_candidates: false,
6748 },
6749 "quota",
6750 1,
6751 );
6752 assert_eq!(stalled.attempts, 0);
6753 assert!(
6754 stalled.status.runnable(),
6755 "a machine problem must leave the task in line"
6756 );
6757
6758 let mut blocked = task();
6759 blocked.start("20260902-000000-aaaa".to_owned());
6760 settle(
6761 &mut blocked,
6762 Verdict {
6763 status: RunStatus::Blocked,
6764 left_pr: false,
6765 parked: false,
6766 quota_hit: false,
6767 no_viable_candidates: false,
6768 },
6769 "findings open",
6770 1,
6771 );
6772 assert_eq!(blocked.attempts, 1);
6773 assert_eq!(
6774 blocked.status,
6775 TaskStatus::Held,
6776 "the last attempt hands the task to a human"
6777 );
6778 }
6779
6780 #[test]
6781 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
6782 let mut delivered = task();
6785 delivered.start("20260903-080619-01c2".to_owned());
6786 settle(
6787 &mut delivered,
6788 Verdict {
6789 status: RunStatus::Blocked,
6790 left_pr: true,
6791 parked: false,
6792 quota_hit: false,
6793 no_viable_candidates: false,
6794 },
6795 "no check status",
6796 4,
6797 );
6798 assert_eq!(
6799 delivered.status,
6800 TaskStatus::Held,
6801 "a pull request waiting on CI or a person is not a retryable failure"
6802 );
6803 assert!(
6804 !delivered.status.runnable(),
6805 "the loop must not pick this task up again"
6806 );
6807 assert_eq!(
6808 delivered.last_error.as_deref(),
6809 Some("no check status"),
6810 "the operator needs to be told what the gate was waiting for"
6811 );
6812
6813 let mut empty_handed = task();
6816 empty_handed.start("20260903-080619-01c2".to_owned());
6817 settle(
6818 &mut empty_handed,
6819 Verdict {
6820 status: RunStatus::Blocked,
6821 left_pr: false,
6822 parked: false,
6823 quota_hit: false,
6824 no_viable_candidates: false,
6825 },
6826 "findings open",
6827 4,
6828 );
6829 assert_eq!(empty_handed.status, TaskStatus::Failed);
6830 assert!(empty_handed.status.runnable());
6831 }
6832
6833 #[test]
6834 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
6835 let mut noop = task();
6841 noop.start("20260912-131304-391f".to_owned());
6842 settle(
6843 &mut noop,
6844 Verdict {
6845 status: RunStatus::VerifiedNoop,
6846 left_pr: false,
6847 parked: false,
6848 quota_hit: false,
6849 no_viable_candidates: true,
6850 },
6851 "candidate A: already fixed by b32cfc4, on main",
6852 4,
6853 );
6854 assert_eq!(
6855 noop.status,
6856 TaskStatus::Held,
6857 "an unverified claim is a request for a human, not a failure"
6858 );
6859 assert!(
6860 !noop.status.runnable(),
6861 "the loop must not requeue this on the same unverified claim"
6862 );
6863 assert_eq!(noop.attempts, 1);
6868 }
6869
6870 #[test]
6871 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
6872 let mut parked = task();
6877 parked.start("20260903-183634-2d98".to_owned());
6878 settle(
6879 &mut parked,
6880 Verdict {
6881 status: RunStatus::Implementing,
6882 left_pr: false,
6883 quota_hit: false,
6884 parked: true,
6885 no_viable_candidates: false,
6886 },
6887 "parked after `implementing`",
6888 2,
6889 );
6890 assert_eq!(parked.attempts, 0, "a park is refunded");
6891 assert!(
6892 parked.status.runnable(),
6893 "and the task stays in line so the next loop resumes its run"
6894 );
6895 assert_eq!(parked.status, TaskStatus::Parked);
6896 assert_eq!(
6897 parked.park_reason.as_deref(),
6898 Some("parked after `implementing`"),
6899 "the card says where it stopped"
6900 );
6901 assert_eq!(parked.last_error, None, "nothing failed");
6902
6903 let mut broken = task();
6907 broken.start("20260903-183634-2d98".to_owned());
6908 settle(
6909 &mut broken,
6910 Verdict {
6911 status: RunStatus::Implementing,
6912 left_pr: false,
6913 quota_hit: false,
6914 parked: false,
6915 no_viable_candidates: false,
6916 },
6917 "returned mid-flight",
6918 2,
6919 );
6920 assert_eq!(broken.attempts, 1);
6921 }
6922
6923 #[test]
6924 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
6925 let mut flaky = task();
6930 flaky.start("20260903-123023-e633".to_owned());
6931 settle(
6932 &mut flaky,
6933 Verdict {
6934 status: RunStatus::Stalled,
6935 left_pr: false,
6936 parked: false,
6937 quota_hit: false,
6938 no_viable_candidates: false,
6939 },
6940 "verdict rests on 1 of 3 judges",
6941 2,
6942 );
6943 assert_eq!(
6944 flaky.attempts, 1,
6945 "flakiness spends an attempt, so `max_attempts` still bounds it"
6946 );
6947 assert!(flaky.status.runnable(), "and it is still worth retrying");
6948
6949 let mut limited = task();
6951 limited.start("20260903-123023-e633".to_owned());
6952 settle(
6953 &mut limited,
6954 Verdict {
6955 status: RunStatus::Stalled,
6956 left_pr: false,
6957 parked: false,
6958 quota_hit: true,
6959 no_viable_candidates: false,
6960 },
6961 "judge-2, judge-3 out of quota",
6962 2,
6963 );
6964 assert_eq!(limited.attempts, 0, "a quota window is refunded");
6965 assert!(limited.status.runnable());
6966
6967 let mut worn = task();
6970 for _ in 0..2 {
6971 worn.release();
6972 }
6973 worn.start("20260903-123023-e633".to_owned());
6974 worn.attempts = 2;
6975 settle(
6976 &mut worn,
6977 Verdict {
6978 status: RunStatus::Stalled,
6979 left_pr: false,
6980 parked: false,
6981 quota_hit: false,
6982 no_viable_candidates: false,
6983 },
6984 "no quorum again",
6985 2,
6986 );
6987 assert_eq!(worn.status, TaskStatus::Held);
6988 assert!(!worn.status.runnable());
6989 }
6990
6991 #[test]
6992 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
6993 let mut wiped_out = task();
7000 wiped_out.start("20260907-025000-a1b2".to_owned());
7001 settle(
7002 &mut wiped_out,
7003 Verdict {
7004 status: RunStatus::Failed,
7005 left_pr: false,
7006 parked: false,
7007 quota_hit: true,
7008 no_viable_candidates: true,
7009 },
7010 "no candidate produced a change; nothing to judge",
7011 2,
7012 );
7013 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
7014 assert!(
7015 wiped_out.status.runnable(),
7016 "a machine problem must leave the task in line"
7017 );
7018
7019 let mut partial_progress = task();
7025 partial_progress.start("20260907-025500-c3d4".to_owned());
7026 settle(
7027 &mut partial_progress,
7028 Verdict {
7029 status: RunStatus::Failed,
7030 left_pr: false,
7031 parked: false,
7032 quota_hit: true,
7033 no_viable_candidates: false,
7034 },
7035 "gate failed on the winning candidate",
7036 2,
7037 );
7038 assert_eq!(
7039 partial_progress.attempts, 1,
7040 "a candidate that actually produced a change spends the attempt \
7041 even though some other seat hit its quota"
7042 );
7043 assert!(partial_progress.status.runnable());
7044 }
7045
7046 #[test]
7047 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
7048 let mut t = task();
7054 t.start("20260907-025000-a1b2".to_owned());
7055 let mut state = run_state(RunStatus::Failed);
7056 state.quota.push(QuotaLoss {
7057 seat: "cand-a".to_owned(),
7058 node: "implement".to_owned(),
7059 at: Timestamp::now(),
7060 reset: None,
7061 });
7062 assert!(
7063 state.viable().is_empty(),
7064 "no candidate was added, so nothing is viable"
7065 );
7066 reclaim(&mut t, Some(state), 2, "en");
7067 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
7068 assert!(t.status.runnable());
7069 }
7070
7071 #[test]
7072 fn a_held_task_is_never_offered_to_the_loop() {
7073 let dir = tempfile::tempdir().unwrap();
7074 let queue = Queue::at(dir.path().to_path_buf());
7075 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
7076 let mut t = task();
7077 t.id = format!("2026090{n}-000000-000{n}");
7078 t.priority = priority;
7079 queue.put(&mut t).unwrap();
7080 }
7081 let mut held = task();
7082 held.id = "20260909-000000-9999".to_owned();
7083 held.priority = 99;
7084 held.hold_machine(None);
7085 queue.put(&mut held).unwrap();
7086
7087 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
7088 assert_eq!(order.len(), 3);
7089 assert!(!order.contains(&held.id));
7090 assert_eq!(
7091 order.first().cloned(),
7092 queue.next_runnable().map(|t| t.id),
7093 "the loop's first candidate is exactly what the queue offers"
7094 );
7095 assert_eq!(
7096 order,
7097 vec![
7098 "20260902-000000-0002".to_owned(),
7099 "20260903-000000-0003".to_owned(),
7100 "20260901-000000-0001".to_owned(),
7101 ],
7102 "priority first, then oldest, so nothing starves"
7103 );
7104 }
7105
7106 #[test]
7107 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
7108 let dir = tempfile::tempdir().unwrap();
7109 let queue = Queue::at(dir.path().to_path_buf());
7110 let mut old = task();
7111 old.id = "20260101-000000-old0".to_owned();
7112 queue.put(&mut old).unwrap();
7113 let mut fresh = task();
7114 fresh.id = "20260101-000000-new0".to_owned();
7115 queue.put(&mut fresh).unwrap();
7116
7117 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
7121 std::thread::sleep(Duration::from_millis(60));
7122 let live = queue.claim(&fresh.id).unwrap();
7123
7124 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7125 assert_eq!(swept, vec![old.id.clone()]);
7126 assert!(
7127 queue.claim(&old.id).is_ok(),
7128 "an unparseable lock older than the threshold is swept"
7129 );
7130 assert!(
7131 queue.claim(&fresh.id).is_err(),
7132 "a live pid protects its lock regardless of age"
7133 );
7134 drop(live);
7135 }
7136
7137 #[test]
7138 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
7139 let dir = tempfile::tempdir().unwrap();
7149 let queue = Queue::at(dir.path().to_path_buf());
7150 let mut t = task();
7151 t.id = "20260101-000000-live".to_owned();
7152 queue.put(&mut t).unwrap();
7153
7154 let claim = queue.claim(&t.id).unwrap();
7155 std::thread::sleep(Duration::from_millis(60));
7156
7157 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7158 assert!(
7159 swept.is_empty(),
7160 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
7161 );
7162 assert!(
7163 queue.claim(&t.id).is_err(),
7164 "the lock still protects its task"
7165 );
7166 drop(claim);
7167 }
7168
7169 fn injected_dead_pid() -> u32 {
7172 std::process::id().checked_add(1).unwrap_or(1)
7173 }
7174
7175 #[test]
7176 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
7177 let dir = tempfile::tempdir().unwrap();
7178 let queue = Queue::at(dir.path().to_path_buf());
7179 let mut t = task();
7180 t.id = "20260101-000000-dead".to_owned();
7181 queue.put(&mut t).unwrap();
7182 let dead_pid = injected_dead_pid();
7183
7184 std::fs::write(
7189 dir.path().join(format!("{}.lock", t.id)),
7190 dead_pid.to_string(),
7191 )
7192 .unwrap();
7193
7194 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7195 pid != dead_pid
7196 });
7197 assert_eq!(
7198 swept,
7199 vec![t.id.clone()],
7200 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
7201 );
7202 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
7203 }
7204
7205 #[test]
7206 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
7207 let dir = tempfile::tempdir().unwrap();
7208 let queue = Queue::at(dir.path().to_path_buf());
7209 let mut t = task();
7210 t.id = "20260101-000000-late".to_owned();
7211 queue.put(&mut t).unwrap();
7212 let dead_pid = injected_dead_pid();
7213
7214 assert!(
7217 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
7218 "nothing has claimed the task yet"
7219 );
7220
7221 std::fs::write(
7224 dir.path().join(format!("{}.lock", t.id)),
7225 dead_pid.to_string(),
7226 )
7227 .unwrap();
7228
7229 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7233 pid != dead_pid
7234 });
7235 assert_eq!(swept, vec![t.id.clone()]);
7236 }
7237
7238 #[test]
7239 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
7240 crate::run::pin_test_home();
7245 let dir = tempfile::tempdir().unwrap();
7246 let queue = Queue::at(dir.path().to_path_buf());
7247 let mut t = task();
7248 t.id = "20260101-000000-crsh".to_owned();
7249 t.status = TaskStatus::Running;
7250 t.attempts = 1;
7251 t.runs.push("20260904-000000-4043".to_owned());
7255 queue.put(&mut t).unwrap();
7256 let dead_pid = injected_dead_pid();
7257
7258 std::fs::write(
7261 dir.path().join(format!("{}.lock", t.id)),
7262 dead_pid.to_string(),
7263 )
7264 .unwrap();
7265
7266 assert!(reclaim_orphaned_running(&queue, 2).is_empty());
7272 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
7273
7274 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7275 pid != dead_pid
7276 });
7277 assert_eq!(swept, vec![t.id.clone()]);
7278
7279 let reclaimed = reclaim_orphaned_running(&queue, 2);
7280 assert_eq!(reclaimed, vec![t.id.clone()]);
7281 let after = queue.get(&t.id).unwrap();
7282 assert_eq!(
7283 after.status,
7284 TaskStatus::Held,
7285 "no run.json to recover from, so a human is asked"
7286 );
7287 assert_eq!(
7288 after.runs,
7289 vec!["20260904-000000-4043".to_owned()],
7290 "the crashed run's id is kept as evidence, not discarded"
7291 );
7292 }
7293
7294 #[test]
7295 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
7296 let dir = tempfile::tempdir().unwrap();
7297 let queue = Queue::at(dir.path().to_path_buf());
7298 let mut t = task();
7299 t.id = "20260101-000000-unknown".to_owned();
7300 queue.put(&mut t).unwrap();
7301 let dead_pid = injected_dead_pid();
7302 std::fs::write(
7303 dir.path().join(format!("{}.lock", t.id)),
7304 dead_pid.to_string(),
7305 )
7306 .unwrap();
7307
7308 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
7309 assert!(swept.is_empty(), "an unknown pid must keep its lock");
7310 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
7311 }
7312
7313 fn run_state_in(status: RunStatus, language: &str) -> RunState {
7314 let mut s = run_state(status);
7315 s.config.graph.language = language.to_owned();
7316 s
7317 }
7318
7319 fn unstarted_verdict(status: RunStatus) -> Verdict {
7320 Verdict {
7321 status,
7322 left_pr: false,
7323 quota_hit: false,
7324 parked: false,
7325 no_viable_candidates: false,
7326 }
7327 }
7328
7329 #[test]
7330 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
7331 let mut t = task();
7332 t.attempts = 1;
7333 settle_in(
7334 &mut t,
7335 unstarted_verdict(RunStatus::AlreadyInBase),
7336 "already in main",
7337 1,
7338 phrases("en"),
7339 );
7340 assert_eq!(t.status, TaskStatus::Done);
7341 assert_eq!(t.attempts, 0);
7342 }
7343
7344 #[test]
7345 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
7346 let reason = |language: &str| {
7347 let mut t = task();
7348 settle_in(
7349 &mut t,
7350 unstarted_verdict(RunStatus::Judging),
7351 "boom",
7352 1,
7353 phrases(language),
7354 );
7355 t.last_error.or(t.hold_reason).unwrap_or_default()
7356 };
7357 assert!(
7358 reason("en").starts_with("the graph stopped at `"),
7359 "{}",
7360 reason("en")
7361 );
7362 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
7363 assert!(reason("日本語").contains("boom"));
7364 assert_eq!(reason("fr"), reason("en"));
7365 }
7366
7367 #[test]
7368 fn describe_follows_the_run_language_and_keeps_the_run_id() {
7369 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
7370 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
7371 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
7372 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
7373 assert!(ja.contains("[run "), "{ja}");
7374 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
7375 assert!(ended.starts_with("run 終了: "), "{ended}");
7376 let mut de = run_state_in(RunStatus::Failed, "de");
7377 let mut en = run_state_in(RunStatus::Failed, "en");
7378 de.id = "same".to_owned();
7379 en.id = "same".to_owned();
7380 assert_eq!(describe(&de), describe(&en));
7381 }
7382
7383 #[test]
7384 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
7385 use crate::handover::Refused;
7386 let cases = [
7387 Refused::Foreign {
7388 branch: "b".into(),
7389 path: "/w/x".into(),
7390 why: "made by hand".into(),
7391 },
7392 Refused::Unsafe {
7393 branch: "b".into(),
7394 path: "/w/x".into(),
7395 why: "its worktree has uncommitted changes (a.rs)".into(),
7396 },
7397 Refused::ReleaseFailed {
7398 branch: "b".into(),
7399 path: "/w/x".into(),
7400 run: "ab12".into(),
7401 },
7402 ];
7403 for r in &cases {
7404 let en = (phrases("en").handover_refused)(r);
7405 assert_eq!(en, r.to_string());
7406 let ja = (phrases("ja").handover_refused)(r);
7407 assert!(
7408 !ja.contains("is checked out") && !ja.contains("try again"),
7409 "{ja}"
7410 );
7411 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
7412 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
7413 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
7414 }
7415 }
7416 assert!(phrases("en").handover_hint.contains("release the task"));
7417 assert!(phrases("ja").handover_hint.contains("解放"));
7418 }
7419
7420 #[test]
7421 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
7422 let build = |lang: &str| {
7423 let mut s = run_state_in(RunStatus::Blocked, lang);
7424 s.id = "same".to_owned();
7425 s.config.graph.review_rounds = 3;
7426 let mut r = review_round(3);
7427 r.expected = 3;
7428 r.answered = 1;
7429 s.reviews.push(r);
7430 s.event(
7431 "review",
7432 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
7433 );
7434 s
7435 };
7436 let en = describe(&build("en"));
7437 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
7438 let ja = describe(&build("ja"));
7439 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
7440 assert!(!ja.contains("never answered"), "{ja}");
7441 assert!(ja.contains("[run "), "{ja}");
7442
7443 let mut failed = build("ja");
7446 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
7447 command: "cargo test".to_owned(),
7448 code: Some(1),
7449 output_tail: String::new(),
7450 duration_ms: 0,
7451 resource_blocked: false,
7452 });
7453 failed.event("review", "stopped; e2e failed: cargo test");
7454 let ja = describe(&failed);
7455 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
7456 assert!(!ja.contains("席のレビュアー"), "{ja}");
7457 }
7458
7459 #[test]
7460 fn refusals_and_recovery_prose_follow_the_language() {
7461 let t = held_task_with("r1");
7462 let q = action_question(
7463 "r1",
7464 ask::ChoiceAction::Resume {
7465 run: "r1".to_owned(),
7466 },
7467 );
7468 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
7469 ActionDecision::Refuse(s) => s,
7470 other => panic!("{other:?}"),
7471 };
7472 assert!(refuse(phrases("en")).contains("could not be read: gone"));
7473 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
7474 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
7475
7476 let mut held = task();
7477 reclaim(&mut held, None, 2, "ja");
7478 assert!(held.hold_reason.unwrap().contains("保留にしました"));
7479 let mut held = task();
7480 reclaim(&mut held, None, 2, "xx");
7481 assert!(held.hold_reason.unwrap().contains("held for a human"));
7482 }
7483
7484 fn run_state(status: RunStatus) -> RunState {
7485 let mut state = RunState::new(
7486 PathBuf::from("/repo"),
7487 "main".to_owned(),
7488 "abc1234def".to_owned(),
7489 "add retries".to_owned(),
7490 Config::default(),
7491 );
7492 state.status = status;
7493 state
7494 }
7495
7496 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
7497 Candidate {
7498 index: 0,
7499 label,
7500 agent: "claude".to_owned(),
7501 branch: format!("magi/x/{label}"),
7502 worktree: PathBuf::from("/repo"),
7503 summary: summary.to_owned(),
7504 stat: String::new(),
7505 files: 0,
7506 commits: usize::from(!empty),
7507 empty,
7508 failed: failed.map(str::to_owned),
7509 verified_noop: None,
7510 duration_ms: 0,
7511 folded: false,
7512 }
7513 }
7514
7515 #[test]
7516 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
7517 let mut state = run_state(RunStatus::Blocked);
7518 state.gate = vec![
7519 CommandOutcome {
7520 command: "cargo make check".to_owned(),
7521 code: Some(0),
7522 output_tail: "ok".to_owned(),
7523 duration_ms: 0,
7524 resource_blocked: false,
7525 },
7526 CommandOutcome {
7527 command: "cargo test".to_owned(),
7528 code: Some(101),
7529 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
7530 duration_ms: 0,
7531 resource_blocked: false,
7532 },
7533 ];
7534 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
7535 assert!(d.contains("cargo test"), "{d}");
7536 assert!(
7537 !d.contains("cargo make check"),
7538 "a passing check is not a diagnostic: {d}"
7539 );
7540 assert!(d.contains("assertion failed"), "{d}");
7541 }
7542
7543 #[test]
7544 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
7545 let mut state = run_state(RunStatus::Blocked);
7546 state.event(
7547 "land",
7548 "stopped: the fixer produced no commit while 2 check(s) were failing \
7549 (build, lint); stopping instead of looping on an unchanged tree",
7550 );
7551 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
7552 assert!(d.contains("build"), "{d}");
7553 assert!(d.contains("lint"), "{d}");
7554 assert!(d.contains("fixer produced no commit"), "{d}");
7555 }
7556
7557 #[test]
7558 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
7559 let state = run_state(RunStatus::VerifiedNoop);
7564 let d = describe(&state);
7565 assert!(
7566 d.contains("agent-verified no-op"),
7567 "expected the display label, not the wire spelling: {d}"
7568 );
7569 assert!(!d.contains("verified_noop"), "{d}");
7570 }
7571
7572 #[test]
7573 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
7574 let mut state = run_state(RunStatus::Failed);
7580 state.candidates = vec![candidate(
7581 'A',
7582 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
7583 true,
7584 None,
7585 )];
7586 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
7587 assert!(d.contains("candidate A"), "{d}");
7588 assert!(d.contains("tagged v1.2.3"), "{d}");
7589 }
7590
7591 #[test]
7592 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
7593 let mut state = run_state(RunStatus::Failed);
7594 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
7595 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
7596 assert!(d.contains("candidate A"), "{d}");
7597 assert!(d.contains("agent timed out"), "{d}");
7598 }
7599
7600 #[test]
7601 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
7602 let mut state = run_state(RunStatus::Failed);
7605 state.candidates = vec![candidate('A', "did the work", false, None)];
7606 assert!(diagnostic(&state).is_none());
7607 }
7608
7609 #[test]
7610 fn diagnostic_is_bounded_however_much_a_run_printed() {
7611 let mut state = run_state(RunStatus::Blocked);
7612 state.gate = vec![
7613 CommandOutcome {
7614 command: "cargo test".to_owned(),
7615 code: Some(101),
7616 output_tail: "x".repeat(50_000),
7617 duration_ms: 0,
7618 resource_blocked: false,
7619 },
7620 CommandOutcome {
7621 command: "cargo clippy".to_owned(),
7622 code: Some(1),
7623 output_tail: "y".repeat(50_000),
7624 duration_ms: 0,
7625 resource_blocked: false,
7626 },
7627 ];
7628 state.candidates = vec![
7629 candidate('A', &"z".repeat(50_000), true, None),
7630 candidate('B', &"w".repeat(50_000), true, None),
7631 ];
7632 let d = diagnostic(&state).expect("plenty here to diagnose");
7633 assert!(
7634 d.len() <= DIAGNOSTIC_MAX,
7635 "diagnostic grew to {} bytes, unbounded",
7636 d.len()
7637 );
7638 }
7639
7640 #[test]
7641 fn selected_delivery_guards_survive_recovery_failure_accounting() {
7642 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
7643 let mut t = task();
7644 let mut state = run_state(status);
7645 t.pr_recovery = Some(crate::pr_recovery::Selection {
7646 url: "url".into(),
7647 branch: format!("magi/task-{}", t.id),
7648 head: "a".repeat(40),
7649 source_head: None,
7650 source_run: None,
7651 source_proof: None,
7652 disposition: Some(crate::pr_recovery::Disposition::Land),
7653 instruction: t.instruction.clone(),
7654 approved_at: Timestamp::now(),
7655 });
7656 state.pr_recovery = t.pr_recovery.clone();
7657 t.start(state.id.clone());
7658 settle_and_diagnose(
7659 &mut t,
7660 Verdict {
7661 status,
7662 left_pr: false,
7663 quota_hit: false,
7664 parked: false,
7665 no_viable_candidates: false,
7666 },
7667 "delivery stopped",
7668 5,
7669 &state,
7670 );
7671 assert_eq!(t.status, TaskStatus::Held);
7672 let reason = t.hold_reason.as_deref().unwrap();
7673 if status == RunStatus::Ready {
7674 assert!(reason.contains("Selected PR delivery is incomplete"));
7675 assert!(t.recovery_budget.is_none());
7676 } else {
7677 assert!(reason.contains("Selected recovery stopped"));
7678 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
7679 }
7680 assert!(t.pr_recovery.is_some());
7681 }
7682 }
7683
7684 #[test]
7685 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
7686 let mut state = run_state(RunStatus::Blocked);
7687 state.gate = vec![CommandOutcome {
7688 command: "cargo test".to_owned(),
7689 code: Some(101),
7690 output_tail: "assertion failed".to_owned(),
7691 duration_ms: 0,
7692 resource_blocked: false,
7693 }];
7694 let verdict = Verdict {
7695 status: RunStatus::Blocked,
7696 left_pr: false,
7697 quota_hit: false,
7698 parked: false,
7699 no_viable_candidates: false,
7700 };
7701
7702 let mut t = task();
7705 t.start("run-1".to_owned());
7706 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
7707 assert_eq!(t.status, TaskStatus::Failed);
7708 assert!(t.diagnostic.is_none());
7709
7710 t.start("run-2".to_owned());
7713 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
7714 assert_eq!(t.status, TaskStatus::Held);
7715 let d = t.diagnostic.expect("a held task must carry its diagnostic");
7716 assert!(d.contains("cargo test"), "{d}");
7717 }
7718
7719 #[test]
7720 fn a_held_task_names_the_open_question_it_is_waiting_on() {
7721 crate::run::pin_test_home();
7726 let home = crate::run::home();
7727 let state = run_state(RunStatus::VerifiedNoop);
7728 let mut q = ask::Question::new(
7729 state.id.clone(),
7730 "implement".to_owned(),
7731 "impl-A".to_owned(),
7732 "is this really a no-op?".to_owned(),
7733 String::new(),
7734 Vec::new(),
7735 );
7736 Questions::at(home.join("questions")).put(&mut q).unwrap();
7737
7738 let verdict = Verdict {
7739 status: RunStatus::VerifiedNoop,
7740 left_pr: false,
7741 quota_hit: false,
7742 parked: false,
7743 no_viable_candidates: false,
7744 };
7745 let mut t = task();
7746 t.start(state.id.clone());
7747 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
7748
7749 assert_eq!(t.status, TaskStatus::Held);
7750 let reason = t.hold_reason.expect("a held task must record why");
7751 assert!(
7752 reason.starts_with("run ended agent-verified no-op"),
7753 "the original settle reason must survive unchanged: {reason}"
7754 );
7755 assert!(
7756 reason.contains(q.short()),
7757 "the open question's id must be named so the notice is actionable: {reason}"
7758 );
7759 }
7760
7761 #[test]
7762 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
7763 crate::run::pin_test_home();
7764 let state = run_state(RunStatus::VerifiedNoop);
7765
7766 let verdict = Verdict {
7767 status: RunStatus::VerifiedNoop,
7768 left_pr: false,
7769 quota_hit: false,
7770 parked: false,
7771 no_viable_candidates: false,
7772 };
7773 let mut t = task();
7774 t.start(state.id.clone());
7775 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
7776
7777 assert_eq!(t.status, TaskStatus::Held);
7778 assert_eq!(
7779 t.hold_reason.as_deref(),
7780 Some("run ended agent-verified no-op"),
7781 "nothing to append when the question was already answered or never asked"
7782 );
7783 }
7784
7785 #[test]
7786 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
7787 crate::run::pin_test_home();
7788 let mut first = run_state(RunStatus::Blocked);
7789 first.id = "20260101-000000-sup1".to_owned();
7790 first.save().unwrap();
7791 let mut second = run_state(RunStatus::Merged);
7792 second.id = "20260101-000000-sup2".to_owned();
7793 second.save().unwrap();
7794
7795 let mut t = task();
7796 t.runs = vec![first.id.clone(), second.id.clone()];
7797 t.status = TaskStatus::Done;
7798
7799 supersede_prior_runs(&t, &crate::run::home());
7800
7801 assert_eq!(
7802 RunState::load(&first.id).unwrap().status,
7803 RunStatus::Superseded,
7804 "the first attempt's Blocked no longer needs anyone's attention"
7805 );
7806 assert_eq!(
7807 RunState::load(&second.id).unwrap().status,
7808 RunStatus::Merged,
7809 "the run that actually succeeded is left exactly as it was"
7810 );
7811 }
7812
7813 #[test]
7814 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
7815 crate::run::pin_test_home();
7821 let mut first = run_state(RunStatus::Blocked);
7822 first.id = "20260101-000000-sup9".to_owned();
7823 first.driver_pid = Some(std::process::id());
7826 first.driver_started_at = Some(
7827 crate::proc::process_started_at(std::process::id())
7828 .expect("this test process's own start time must be queryable"),
7829 );
7830 first.save().unwrap();
7831 let mut second = run_state(RunStatus::Merged);
7832 second.id = "20260101-000000-supa".to_owned();
7833 second.save().unwrap();
7834
7835 let mut t = task();
7836 t.runs = vec![first.id.clone(), second.id.clone()];
7837 t.status = TaskStatus::Done;
7838
7839 supersede_prior_runs(&t, &crate::run::home());
7840
7841 assert_eq!(
7842 RunState::load(&first.id).unwrap().status,
7843 RunStatus::Blocked,
7844 "a live driver_pid means something is still actually working this run, \
7845 even though no daemon claims it - rewriting under it would just be \
7846 undone the next time that process saves"
7847 );
7848 }
7849
7850 #[test]
7851 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
7852 let dir = tempfile::tempdir().unwrap();
7858 let home = dir.path().to_path_buf();
7859 let queue = Queue::at(dir.path().join("queue"));
7860
7861 let mut first = run_state(RunStatus::Blocked);
7862 first.id = "20260101-000000-supd".to_owned();
7863 first.driver_pid = Some(std::process::id());
7864 first.driver_started_at = Some(
7865 crate::proc::process_started_at(std::process::id())
7866 .expect("this test process's own start time must be queryable"),
7867 );
7868 first.save_under(&home).unwrap();
7869 let mut second = run_state(RunStatus::Merged);
7870 second.id = "20260101-000000-supe".to_owned();
7871 second.save_under(&home).unwrap();
7872
7873 let mut t = task();
7874 t.runs = vec![first.id.clone(), second.id.clone()];
7875 t.status = TaskStatus::Done;
7876 queue.put(&mut t).unwrap();
7877
7878 resweep_superseded_attempts(&queue, &home);
7879 assert_eq!(
7880 RunState::load_under(&first.id, &home).unwrap().status,
7881 RunStatus::Blocked,
7882 "still live on the first pass, so still untouched"
7883 );
7884
7885 let mut stale = RunState::load_under(&first.id, &home).unwrap();
7891 stale.driver_started_at = Some("1".to_owned());
7892 stale.save_under(&home).unwrap();
7893
7894 resweep_superseded_attempts(&queue, &home);
7895 assert_eq!(
7896 RunState::load_under(&first.id, &home).unwrap().status,
7897 RunStatus::Superseded,
7898 "the second pass catches up what the first one correctly skipped"
7899 );
7900 }
7901
7902 #[test]
7903 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
7904 crate::run::pin_test_home();
7905 let mut first = run_state(RunStatus::Blocked);
7906 first.id = "20260101-000000-sup3".to_owned();
7907 first.save().unwrap();
7908 let mut second = run_state(RunStatus::Blocked);
7909 second.id = "20260101-000000-sup4".to_owned();
7910 second.save().unwrap();
7911
7912 let mut t = task();
7913 t.runs = vec![first.id.clone(), second.id.clone()];
7914 t.status = TaskStatus::Failed;
7918
7919 supersede_prior_runs(&t, &crate::run::home());
7920
7921 assert_eq!(
7922 RunState::load(&first.id).unwrap().status,
7923 RunStatus::Blocked
7924 );
7925 assert_eq!(
7926 RunState::load(&second.id).unwrap().status,
7927 RunStatus::Blocked
7928 );
7929 }
7930
7931 #[test]
7932 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
7933 crate::run::pin_test_home();
7937 let mut first = run_state(RunStatus::Blocked);
7938 first.id = "20260101-000000-sup5".to_owned();
7939 first.save().unwrap();
7940
7941 let mut t = task();
7942 t.runs = vec![first.id.clone()];
7943 t.status = TaskStatus::Done;
7944
7945 supersede_prior_runs(&t, &crate::run::home());
7946
7947 assert_eq!(
7948 RunState::load(&first.id).unwrap().status,
7949 RunStatus::Blocked,
7950 "a single-attempt task has no earlier run to supersede"
7951 );
7952 }
7953
7954 #[test]
7955 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
7956 crate::run::pin_test_home();
7963 let mut first = run_state(RunStatus::Blocked);
7964 first.id = "20260101-000000-supb".to_owned();
7965 first.save().unwrap();
7966 let mut second = run_state(RunStatus::Failed);
7967 second.id = "20260101-000000-supc".to_owned();
7968 second.save().unwrap();
7969
7970 let mut t = task();
7971 t.runs = vec![first.id.clone(), second.id.clone()];
7972 t.status = TaskStatus::Done;
7973
7974 supersede_prior_runs(&t, &crate::run::home());
7975
7976 assert_eq!(
7977 RunState::load(&first.id).unwrap().status,
7978 RunStatus::Blocked,
7979 "the task's last attempt never landed, so there is nothing here \
7980 actually superseding it"
7981 );
7982 }
7983
7984 #[test]
7985 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
7986 crate::run::pin_test_home();
7990 let mut failed = run_state(RunStatus::Failed);
7991 failed.id = "20260101-000000-sup6".to_owned();
7992 failed.save().unwrap();
7993 let mut noop = run_state(RunStatus::VerifiedNoop);
7994 noop.id = "20260101-000000-sup7".to_owned();
7995 noop.save().unwrap();
7996 let mut winner = run_state(RunStatus::Ready);
7997 winner.id = "20260101-000000-sup8".to_owned();
7998 winner.save().unwrap();
7999
8000 let mut t = task();
8001 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
8002 t.status = TaskStatus::Done;
8003
8004 supersede_prior_runs(&t, &crate::run::home());
8005
8006 assert_eq!(
8007 RunState::load(&failed.id).unwrap().status,
8008 RunStatus::Failed
8009 );
8010 assert_eq!(
8011 RunState::load(&noop.id).unwrap().status,
8012 RunStatus::VerifiedNoop
8013 );
8014 }
8015
8016 fn approval_question(run: &str) -> ask::Question {
8017 ask::Question::new(
8018 run.to_owned(),
8019 land::APPROVAL_NODE.to_owned(),
8020 "land".to_owned(),
8021 "merge?".to_owned(),
8022 String::new(),
8023 vec!["merge".to_owned(), "hold".to_owned()],
8024 )
8025 }
8026
8027 #[test]
8028 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
8029 crate::run::pin_test_home();
8030 let mut state = run_state(RunStatus::Landing);
8031 state.id = "20260101-000000-fre1".to_owned();
8032 state.parked = true;
8033 state.save().unwrap();
8034 ask::Questions::open()
8035 .put(&mut approval_question(&state.id))
8036 .unwrap();
8037
8038 let mut t = task();
8039 t.runs.push(state.id.clone());
8040 assert_eq!(
8041 land_resume_state(&t),
8042 LandResume::StillWaiting,
8043 "nobody has answered and the timeout has not passed"
8044 );
8045 }
8046
8047 #[test]
8048 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
8049 crate::run::pin_test_home();
8050 let mut state = run_state(RunStatus::Gating);
8051 state.id = "20260101-000000-gt01".to_owned();
8052 state.parked = true;
8053 state.config.graph.answer_timeout = 60;
8054 state.github_text = Some(crate::run::GithubTextAsk {
8055 rewrite: None,
8056 fingerprint: "f".into(),
8057 title: true,
8058 body: false,
8059 categories: vec!["title-language".into()],
8060 question: None,
8061 asks: 1,
8062 resolved: false,
8063 chosen_title: None,
8064 choice: None,
8065 });
8066 state.save().unwrap();
8067 let store = ask::Questions::open();
8068 let mut q = ask::Question::new(
8069 state.id.clone(),
8070 crate::github_text::ASK_NODE.to_owned(),
8071 crate::github_text::ASK_SEAT.to_owned(),
8072 "s".into(),
8073 String::new(),
8074 vec!["use fallback".into()],
8075 );
8076 store.put(&mut q).unwrap();
8077 let mut t = task();
8078 t.runs.push(state.id.clone());
8079 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
8080 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8081 store.put(&mut q).unwrap();
8082 assert_eq!(land_resume_state(&t), LandResume::Ready);
8083 assert!(!store.get(&q.id).unwrap().status.open());
8084 state.github_text.as_mut().unwrap().resolved = true;
8086 state.save().unwrap();
8087 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
8088 }
8089
8090 #[test]
8091 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
8092 crate::run::pin_test_home();
8097 let mut state = run_state(RunStatus::Landing);
8098 state.id = "20260101-000000-exp1".to_owned();
8099 state.parked = true;
8100 state.config.graph.answer_timeout = 60;
8101 state.save().unwrap();
8102
8103 let store = ask::Questions::open();
8104 let mut q = approval_question(&state.id);
8105 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8106 store.put(&mut q).unwrap();
8107
8108 let mut t = task();
8109 t.runs.push(state.id.clone());
8110 assert_eq!(
8111 land_resume_state(&t),
8112 LandResume::Ready,
8113 "an expired question must not be waited on forever"
8114 );
8115
8116 let after = store.get(&q.id).unwrap();
8117 assert!(
8118 !after.status.open(),
8119 "the question is abandoned, not silently ignored"
8120 );
8121 assert!(
8122 after.resolution().is_none(),
8123 "an abandoned question is not read as a decision"
8124 );
8125 }
8126
8127 #[test]
8128 fn reclaim_settles_a_running_task_against_its_last_run() {
8129 let mut t = task();
8130 t.start("20260904-000000-4043".to_owned());
8131 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
8132 assert_eq!(
8133 t.status,
8134 TaskStatus::Done,
8135 "a run that actually finished must not stay `running` forever"
8136 );
8137 }
8138
8139 #[test]
8140 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
8141 let mut t = task();
8142 t.start("20260904-000000-4043".to_owned());
8143 t.last_error = Some("earlier trouble".to_owned());
8144 let mut state = run_state(RunStatus::Judging);
8145 state.parked = true;
8146 reclaim(&mut t, Some(state), 1, "en");
8147 assert_eq!(t.status, TaskStatus::Parked);
8148 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
8149 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8150 assert!(t.park_reason.is_some());
8151 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
8152 assert!(!t.fresh_start);
8153 assert!(t.status.runnable());
8154 }
8155
8156 #[test]
8157 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
8158 let mut t = task();
8159 t.start("20260904-000000-4043".to_owned());
8160 t.last_error = Some("earlier trouble".to_owned());
8161 settle(
8162 &mut t,
8163 Verdict {
8164 status: RunStatus::Judging,
8165 left_pr: false,
8166 quota_hit: false,
8167 parked: true,
8168 no_viable_candidates: false,
8169 },
8170 "parked after `judging`",
8171 1,
8172 );
8173 assert_eq!(t.status, TaskStatus::Parked);
8174 assert_eq!(t.attempts, 0);
8175 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8176 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
8177 assert_eq!(t.schema, crate::queue::SCHEMA);
8178 assert!(
8179 crate::notices::task_held(&t).is_none(),
8180 "a park is not news"
8181 );
8182 t.start("20260904-000000-4043".to_owned());
8183 assert_eq!(t.park_reason, None);
8184 }
8185
8186 #[test]
8187 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
8188 let mut t = task();
8192 t.start("20260904-000000-4043".to_owned());
8193 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
8194 assert_eq!(t.status, TaskStatus::Failed);
8195 assert!(t.status.runnable());
8196 }
8197
8198 #[test]
8199 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
8200 let mut t = task();
8201 t.start("20260904-000000-4043".to_owned());
8202 assert_eq!(t.attempts, 1);
8203 let mut run = run_state(RunStatus::Implementing);
8204 run.id = "20260904-000000-4043".to_owned();
8205 run.begin_pass();
8206 let cause = crate::run::ParkCause::Upgrade {
8207 from: "0.121.0".to_owned(),
8208 to: Some("0.121.1".to_owned()),
8209 };
8210 run.end_pass_parked(cause.clone());
8211 run.parked = true;
8212 let verdict = Verdict {
8213 status: run.status,
8214 left_pr: false,
8215 quota_hit: false,
8216 parked: true,
8217 no_viable_candidates: false,
8218 };
8219 let mut orphan = t.clone();
8222 reclaim(&mut orphan, Some(run.clone()), 2, "en");
8223 assert_eq!(orphan.status, TaskStatus::Parked);
8224 assert_eq!(orphan.attempts, 0);
8225 assert_eq!(
8226 orphan.upgrade_resume_key(),
8227 Some((run.passes[0].started_at, orphan.id.as_str()))
8228 );
8229 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
8230 assert_eq!(t.attempts, 0);
8231 assert_eq!(t.passes.len(), 1);
8232 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8233 assert_eq!(t.passes[0].parked, Some(cause));
8234 assert_eq!(t.passes[0].pass_index, 0);
8235 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
8236 assert!(t.upgrade_resume_key().is_some());
8237
8238 t.start("20260904-000000-4043".to_owned());
8241 run.begin_pass();
8242 run.status = RunStatus::Failed;
8243 let verdict = Verdict {
8244 status: RunStatus::Failed,
8245 left_pr: false,
8246 quota_hit: false,
8247 parked: false,
8248 no_viable_candidates: false,
8249 };
8250 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
8251 assert_eq!(t.passes.len(), 2);
8252 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8253 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
8254 assert_eq!(t.passes[1].pass_index, 1);
8255 assert_eq!(t.attempts, 1);
8256 }
8257
8258 #[test]
8259 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
8260 let mut t = task();
8261 t.start("20260904-000000-4043".to_owned());
8262 reclaim(&mut t, None, 2, "en");
8263 assert_eq!(t.status, TaskStatus::Held);
8264 assert!(
8265 t.last_error
8266 .as_deref()
8267 .is_some_and(|e| e.contains("running")),
8268 "the operator needs to know why this task was held"
8269 );
8270 }
8271
8272 #[test]
8273 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
8274 let dir = tempfile::tempdir().unwrap();
8275 let queue = Queue::at(dir.path().to_path_buf());
8276
8277 let mut orphaned = task();
8279 orphaned.id = "20260904-000000-orph".to_owned();
8280 orphaned.status = TaskStatus::Running;
8281 orphaned.attempts = 1;
8282 queue.put(&mut orphaned).unwrap();
8283
8284 let mut alive = task();
8285 alive.id = "20260904-000000-live".to_owned();
8286 alive.status = TaskStatus::Running;
8287 alive.attempts = 1;
8288 queue.put(&mut alive).unwrap();
8289 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
8290
8291 let mut queued = task();
8292 queued.id = "20260904-000000-wait".to_owned();
8293 queue.put(&mut queued).unwrap();
8294
8295 let reclaimed = reclaim_orphaned_running(&queue, 2);
8296 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
8297
8298 assert_eq!(
8299 queue.get(&orphaned.id).unwrap().status,
8300 TaskStatus::Held,
8301 "nothing was driving it and there was no run to recover"
8302 );
8303 assert_eq!(
8304 queue.get(&alive.id).unwrap().status,
8305 TaskStatus::Running,
8306 "a live claim must protect the task it belongs to"
8307 );
8308 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
8309 }
8310
8311 fn read_run_under(home: &Path, id: &str) -> RunState {
8317 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
8318 serde_json::from_str(&body).unwrap()
8319 }
8320
8321 #[test]
8322 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
8323 let dir = tempfile::tempdir().unwrap();
8324 let home = dir.path().to_path_buf();
8325 let now = Timestamp::now();
8326 let overrun_seat = || crate::run::ActiveSeat {
8327 node: "implement".to_owned(),
8328 started_at: now - jiff::SignedDuration::new(21_000, 0),
8329 timeout_secs: 3_600,
8330 attempt: 0,
8331 task: None,
8332 command: None,
8333 index: None,
8334 total: None,
8335 };
8336
8337 let mut dead = run_state(RunStatus::Implementing);
8338 dead.id = "20260101-000000-dead".to_owned();
8339 dead.active.insert("impl-A".to_owned(), overrun_seat());
8340 dead.driver_pid = Some(4242);
8343 dead.save_under(&home).unwrap();
8344
8345 let mut alive = run_state(RunStatus::Implementing);
8348 alive.id = "20260101-000000-aliv".to_owned();
8349 alive.active.insert("impl-A".to_owned(), overrun_seat());
8350 alive.save_under(&home).unwrap();
8351 let mut status = Status::new();
8352 status.current = vec![Current {
8353 task: "20260101-000000-task".to_owned(),
8354 run: alive.id.clone(),
8355 }];
8356 write_status_to(&home.join("daemon.json"), &status).unwrap();
8357
8358 let questions = Questions::at(home.join("questions"));
8362 let mut q = ask::Question::new(
8363 dead.id.clone(),
8364 "implement".to_owned(),
8365 "impl-A".to_owned(),
8366 "Which storage backend?".to_owned(),
8367 String::new(),
8368 vec!["SQLite".to_owned(), "Redis".to_owned()],
8369 );
8370 questions.put(&mut q).unwrap();
8371
8372 let abandoned = reclaim_abandoned_runs_with(
8373 &home,
8374 now,
8375 |pid| if pid == 4242 { Some(false) } else { None },
8376 |_| panic!("a query answering Dead outright needs no identity corroboration"),
8377 );
8378 assert_eq!(abandoned, vec![dead.id.clone()]);
8379
8380 let reloaded = read_run_under(&home, &dead.id);
8381 assert_eq!(reloaded.status, RunStatus::Failed);
8382 assert!(reloaded.active.is_empty());
8383 assert!(
8384 !questions.get(&q.id).unwrap().status.open(),
8385 "the failed run's own open question must be settled in the same pass"
8386 );
8387
8388 let still_alive = read_run_under(&home, &alive.id);
8389 assert_eq!(
8390 still_alive.status,
8391 RunStatus::Implementing,
8392 "a live daemon's claim protects it"
8393 );
8394 assert!(!still_alive.active.is_empty());
8395 }
8396
8397 #[test]
8407 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
8408 let dir = tempfile::tempdir().unwrap();
8409 let home = dir.path().to_path_buf();
8410 let now = Timestamp::now();
8411
8412 let mut manual = run_state(RunStatus::Reviewing);
8413 manual.id = "20260101-000000-manl".to_owned();
8414 manual.active.insert(
8415 "review-1".to_owned(),
8416 crate::run::ActiveSeat {
8417 node: "review".to_owned(),
8418 started_at: now - jiff::SignedDuration::new(21_000, 0),
8419 timeout_secs: 3_600,
8420 attempt: 0,
8421 task: None,
8422 command: None,
8423 index: None,
8424 total: None,
8425 },
8426 );
8427 manual.driver_pid = Some(4242);
8431 manual.driver_started_at = Some("1790000000".to_owned());
8432 manual.save_under(&home).unwrap();
8433
8434 let abandoned = reclaim_abandoned_runs_with(
8435 &home,
8436 now,
8437 |pid| if pid == 4242 { Some(true) } else { None },
8438 |pid| {
8439 if pid == 4242 {
8440 Some("1790000000".to_owned())
8441 } else {
8442 None
8443 }
8444 },
8445 );
8446 assert!(
8447 abandoned.is_empty(),
8448 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
8449 );
8450
8451 let reloaded = read_run_under(&home, &manual.id);
8452 assert_eq!(reloaded.status, RunStatus::Reviewing);
8453 assert!(!reloaded.active.is_empty());
8454 }
8455
8456 #[test]
8457 fn an_already_claimed_task_is_skipped_rather_than_failed() {
8458 let dir = tempfile::tempdir().unwrap();
8459 let queue = Queue::at(dir.path().to_path_buf());
8460 let mut only = task();
8461 queue.put(&mut only).unwrap();
8462
8463 let _elsewhere = queue.claim(&only.id).unwrap();
8464 let candidates = runnable(&queue);
8465 assert_eq!(candidates.len(), 1, "the task is still runnable");
8466 assert!(
8467 queue.claim(&candidates[0].id).is_err(),
8468 "the loop cannot take a claim somebody else holds"
8469 );
8470
8471 let after = queue.get(&only.id).unwrap();
8472 assert_eq!(after.status, TaskStatus::Queued);
8473 assert_eq!(
8474 after.attempts, 0,
8475 "losing the race is not an attempt at the task"
8476 );
8477 assert_eq!(after.last_error, None);
8478 }
8479
8480 #[test]
8481 fn the_status_file_round_trips_and_its_heartbeat_advances() {
8482 let dir = tempfile::tempdir().unwrap();
8483 let path = dir.path().join("daemon.json");
8484
8485 let mut status = Status::new();
8486 status.idle = false;
8487 status.completed = 7;
8488 status.current = vec![Current {
8489 task: "20260902-000000-t111".to_owned(),
8490 run: "20260902-000001-r111".to_owned(),
8491 }];
8492 write_status_to(&path, &status).unwrap();
8493 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
8494 assert_eq!(first.schema, SCHEMA);
8495 assert_eq!(first.pid, std::process::id());
8496 assert!(!first.idle);
8497 assert_eq!(first.completed, 7);
8498 assert_eq!(first.current, status.current);
8499 assert!(
8500 !path.with_extension("json.tmp").exists(),
8501 "the temp file is renamed, not left behind"
8502 );
8503
8504 std::thread::sleep(Duration::from_millis(5));
8505 status.updated_at = Timestamp::now();
8506 status.polls = 3;
8507 write_status_to(&path, &status).unwrap();
8508 let second: Status =
8509 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
8510 assert!(
8511 second.updated_at > first.updated_at,
8512 "a reader can only detect staleness if the heartbeat moves"
8513 );
8514 assert_eq!(
8515 second.started_at, first.started_at,
8516 "the start time is not a heartbeat"
8517 );
8518 assert_eq!(second.polls, 3);
8519 }
8520
8521 #[test]
8522 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
8523 let dir = tempfile::tempdir().unwrap();
8524
8525 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
8526
8527 let mut status = Status::new();
8528 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
8529 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8530 let stale = read_status(dir.path()).unwrap();
8531 assert!(
8532 !stale.running(Timestamp::now()),
8533 "a minute without a heartbeat is a dead daemon, not a busy one"
8534 );
8535 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
8536
8537 status.updated_at = Timestamp::now();
8538 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8539 let fresh = read_status(dir.path()).unwrap();
8540 assert!(fresh.running(Timestamp::now()));
8541 }
8542
8543 #[test]
8544 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
8545 let dir = tempfile::tempdir().unwrap();
8546 let now = Timestamp::now();
8547 let mine = "20260903-080619-01c2";
8548
8549 assert!(
8550 !is_working_on(dir.path(), mine, now),
8551 "no status file means nobody is working on anything"
8552 );
8553
8554 let mut status = Status::new();
8555 status.current = vec![Current {
8556 task: "20260903-080340-0167".to_owned(),
8557 run: mine.to_owned(),
8558 }];
8559 status.updated_at = now;
8560 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8561 assert!(is_working_on(dir.path(), mine, now));
8562 assert!(
8563 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
8564 "a daemon busy with one run is not working on another"
8565 );
8566
8567 status.updated_at = now - jiff::SignedDuration::from_secs(600);
8570 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8571 assert!(
8572 !is_working_on(dir.path(), mine, now),
8573 "a stale heartbeat is a dead daemon, so its run is a leftover"
8574 );
8575 }
8576
8577 #[test]
8578 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
8579 let dir = tempfile::tempdir().unwrap();
8580 let now = Timestamp::now();
8581
8582 assert!(
8583 !is_working_on_short(dir.path(), "01c2", now),
8584 "no status file means nobody is working on anything"
8585 );
8586
8587 let mut status = Status::new();
8588 status.current = vec![Current {
8589 task: "20260903-080340-0167".to_owned(),
8590 run: "20260903-080619-01c2".to_owned(),
8591 }];
8592 status.updated_at = now;
8593 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8594 assert!(
8595 is_working_on_short(dir.path(), "01c2", now),
8596 "the run's short id is the last block of its full id"
8597 );
8598 assert!(
8599 !is_working_on_short(dir.path(), "3cbf", now),
8600 "a daemon busy with one worktree bay is not working on another"
8601 );
8602 }
8603
8604 #[test]
8605 fn a_newer_status_file_still_yields_a_reading() {
8606 let dir = tempfile::tempdir().unwrap();
8607 std::fs::write(
8610 dir.path().join("daemon.json"),
8611 serde_json::json!({
8612 "schema": 2,
8613 "updated_at": Timestamp::now().to_string(),
8614 "idle": true,
8615 "surprise": { "nested": [1, 2, 3] },
8616 })
8617 .to_string(),
8618 )
8619 .unwrap();
8620
8621 let reading = read_status(dir.path()).expect("a forward-compatible read");
8622 assert!(reading.running(Timestamp::now()));
8623 assert!(reading.idle);
8624 assert!(reading.current.is_empty());
8625 }
8626
8627 #[test]
8628 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
8629 let dir = tempfile::tempdir().unwrap();
8635 std::fs::write(
8636 dir.path().join("daemon.json"),
8637 serde_json::json!({
8638 "schema": 1,
8639 "pid": 4242,
8640 "updated_at": Timestamp::now().to_string(),
8641 "idle": false,
8642 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
8643 "completed": 3,
8644 "polls": 9,
8645 })
8646 .to_string(),
8647 )
8648 .unwrap();
8649
8650 let reading = read_status(dir.path()).expect("an older shape must still parse");
8651 assert!(reading.running(Timestamp::now()));
8652 assert_eq!(
8653 reading.current,
8654 vec![Current {
8655 task: "20260902-140501-aaaa".to_owned(),
8656 run: "20260902-140502-bbbb".to_owned(),
8657 }]
8658 );
8659 }
8660
8661 #[test]
8662 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
8663 let dir = tempfile::tempdir().unwrap();
8664 std::fs::write(
8665 dir.path().join("daemon.json"),
8666 serde_json::json!({
8667 "schema": 1,
8668 "updated_at": Timestamp::now().to_string(),
8669 "idle": true,
8670 "current": null,
8671 })
8672 .to_string(),
8673 )
8674 .unwrap();
8675 let with_null = read_status(dir.path()).expect("null must still parse");
8676 assert!(with_null.current.is_empty());
8677
8678 std::fs::write(
8679 dir.path().join("daemon.json"),
8680 serde_json::json!({
8681 "schema": 1,
8682 "updated_at": Timestamp::now().to_string(),
8683 "idle": true,
8684 })
8685 .to_string(),
8686 )
8687 .unwrap();
8688 let absent = read_status(dir.path()).expect("a missing field must still parse");
8689 assert!(absent.current.is_empty());
8690 }
8691
8692 #[test]
8693 fn a_task_without_a_repository_runs_in_the_daemons_default() {
8694 let fallback = Path::new("/default");
8695 let mut blank = task();
8696 blank.repo = PathBuf::new();
8697 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
8698 let mut dot = task();
8699 dot.repo = PathBuf::from(".");
8700 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
8701 assert_eq!(
8702 repo_for(&task(), fallback),
8703 PathBuf::from("/repo"),
8704 "a task that names a repository keeps it"
8705 );
8706 }
8707
8708 #[test]
8709 fn task_implementers_edit_inherits_daemon_config_and_merge() {
8710 let dir = tempfile::tempdir().unwrap();
8711 let path = dir.path().join("explicit.toml");
8712 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
8713 let opts = Opts {
8714 config: Some(path),
8715 merge: Some("none".into()),
8716 ..Default::default()
8717 };
8718 let mut t = task();
8719 t.set_implementers(
8720 2,
8721 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8722 .unwrap(),
8723 )
8724 .unwrap();
8725 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
8726 assert_eq!(cfg.graph.implementers, 2);
8727 assert_eq!(cfg.graph.judges, 1);
8728 assert_eq!(cfg.merge.mode, MergeMode::None);
8729 t.set_implementers(1, &cfg).unwrap();
8730 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
8731 apply_task_implementers(&mut cfg, &t);
8732 assert_eq!(cfg.graph.implementers, 1);
8733 assert_eq!(cfg.graph.judges, 1);
8734 assert_eq!(cfg.merge.mode, MergeMode::None);
8735 }
8736
8737 #[test]
8738 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
8739 let dir = tempfile::tempdir().unwrap();
8740 let repo = dir.path().join("repo");
8741 std::fs::create_dir(&repo).unwrap();
8742 std::fs::write(
8743 repo.join("magi.toml"),
8744 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
8745 )
8746 .unwrap();
8747 let daemon_config = dir.path().join("daemon.toml");
8748 std::fs::write(
8749 &daemon_config,
8750 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
8751 )
8752 .unwrap();
8753 let opts = Opts {
8754 config: Some(daemon_config),
8755 merge: Some("pr".into()),
8756 ..Default::default()
8757 };
8758 let q = Queue::at(dir.path().join("queue"));
8759 let validation: Config =
8760 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8761 .unwrap();
8762 for record in ["{}", r#"{"candidates":2}"#] {
8764 let overrides = serde_json::from_str(record).unwrap();
8765 let filing = crate::direct::Filing {
8766 instruction: "work".into(),
8767 title: "direct".into(),
8768 repo: repo.clone(),
8769 source: crate::queue::Source::Human,
8770 solo: false,
8771 overrides,
8772 review_of: None,
8773 sources: Default::default(),
8774 };
8775 let filed =
8776 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
8777 let crate::direct::Filed::Standalone {
8778 mut task,
8779 claim: _claim,
8780 } = filed
8781 else {
8782 panic!("local filing")
8783 };
8784 let before = prepare_for(&repo, &opts, &task).unwrap();
8785 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
8786 assert_eq!(
8787 before.merge.mode,
8788 MergeMode::None,
8789 "daemon --merge must not leak"
8790 );
8791 for n in [2, 1, 2] {
8792 task.set_implementers(n, &validation).unwrap();
8793 q.put(&mut task).unwrap();
8794 task = q.get(&task.id).unwrap();
8795 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
8796 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
8797 apply_task_implementers(&mut cfg, &task);
8798 assert_eq!(cfg.graph.implementers, usize::from(n));
8799 assert_eq!(cfg.graph.judges, 2);
8800 assert_eq!(cfg.merge.mode, MergeMode::None);
8801 }
8802 }
8803 }
8804
8805 #[test]
8806 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
8807 let cfg: Config =
8808 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8809 .unwrap();
8810 let mut t = task();
8811 t.review_of = Some("review-target".into());
8812 t.hold_manual(Some("retry".into()));
8813 let mut previous = RunState::new(
8814 PathBuf::from("/repo"),
8815 "main".into(),
8816 "abc1234".into(),
8817 "review".into(),
8818 cfg.clone(),
8819 );
8820 previous.config.graph.implementers = 2;
8821 for count in [1, 3] {
8822 t.set_implementers(count, &cfg).unwrap();
8823 t.release();
8824 assert_eq!(t.review_of.as_deref(), Some("review-target"));
8825 for proposed in [
8826 Starter::Resume(previous.id.clone()),
8827 Starter::Review("recovery-target".into()),
8828 ] {
8829 assert_eq!(
8830 starter_with_implementers(&t, Some(&previous), proposed),
8831 Starter::Review("review-target".into())
8832 );
8833 }
8834 let mut configured = cfg.clone();
8835 apply_task_implementers(&mut configured, &t);
8836 assert_eq!(configured.graph.implementers, usize::from(count));
8837 assert_eq!(previous.config.graph.implementers, 2);
8838 }
8839 }
8840
8841 #[test]
8842 fn task_implementers_dispatch_and_resume_agree() {
8843 let mut t = task();
8844 let cfg: crate::config::Config =
8845 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8846 .unwrap();
8847 let mut run = RunState::new(
8848 PathBuf::from("/repo"),
8849 "main".into(),
8850 "abc1234".into(),
8851 "work".into(),
8852 cfg.clone(),
8853 );
8854 run.config.graph.implementers = 2;
8855 for n in [2, 1, 3, 2] {
8856 t.set_implementers(n, &cfg).unwrap();
8857 let mut dispatched = cfg.clone();
8858 apply_task_implementers(&mut dispatched, &t);
8859 assert_eq!(dispatched.graph.implementers, usize::from(n));
8860 assert_eq!(implementers_match(&t, &run), n == 2);
8861 for proposed in [
8862 Starter::Resume(run.id.clone()),
8863 Starter::Review("old-branch".into()),
8864 ] {
8865 let expected = if n == 2 {
8866 proposed.clone()
8867 } else {
8868 Starter::Start
8869 };
8870 assert_eq!(
8871 starter_with_implementers(&t, Some(&run), proposed),
8872 expected
8873 );
8874 }
8875 assert_eq!(
8876 run.config.graph.implementers, 2,
8877 "history must not be rewritten"
8878 );
8879 }
8880 t.solo = false;
8881 t.overrides = None;
8882 assert!(implementers_match(&t, &run));
8883 }
8884
8885 #[test]
8886 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
8887 let mut solo_cfg = Config::default();
8893 solo_cfg.graph.implementers = 3;
8894 let mut solo_task = task();
8895 solo_task.solo = true;
8896 apply_task_implementers(&mut solo_cfg, &solo_task);
8897 assert_eq!(solo_cfg.graph.implementers, 1);
8898
8899 let mut plain_cfg = Config::default();
8900 plain_cfg.graph.implementers = 3;
8901 let plain_task = task();
8902 assert!(!plain_task.solo);
8903 apply_task_implementers(&mut plain_cfg, &plain_task);
8904 assert_eq!(
8905 plain_cfg.graph.implementers, 3,
8906 "a task that did not ask to run alone keeps the config's candidates"
8907 );
8908 }
8909
8910 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
8911 QuotaLoss {
8912 seat: seat.into(),
8913 node: "judge".into(),
8914 at: at.parse().unwrap(),
8915 reset: reset.map(str::to_string),
8916 }
8917 }
8918
8919 #[test]
8920 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
8921 let old: Vec<QuotaLoss> = (1..=4)
8922 .map(|i| {
8923 loss(
8924 &format!("judge-{i}"),
8925 "2026-09-23T05:23:00Z",
8926 Some("2:40pm (Asia/Tokyo)"),
8927 )
8928 })
8929 .collect();
8930 let fresh = losses_this_attempt(&old, &old);
8931 assert!(fresh.is_empty());
8932 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
8933 }
8935
8936 #[test]
8937 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
8938 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
8939 let now = Timestamp::now();
8940 let mut after = old.clone();
8941 after.push(loss("judge-2", &now.to_string(), None));
8942 let fresh = losses_this_attempt(&old, &after);
8943 assert_eq!(fresh, vec![after[1].clone()]);
8944 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
8945 assert_eq!(
8946 until,
8947 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
8948 );
8949 }
8950
8951 #[test]
8952 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
8953 let before = vec![
8956 loss("judge-1", "2026-09-23T05:23:00Z", None),
8957 loss("judge-2", "2026-09-23T05:24:00Z", None),
8958 ];
8959 let after = vec![
8960 loss("judge-2", "2026-09-23T05:24:00Z", None),
8961 loss("judge-1", "2026-09-24T01:00:00Z", None),
8962 ];
8963 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
8964 }
8965
8966 #[test]
8967 fn merge_overrides_are_parsed_or_refused() {
8968 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
8969 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
8970 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
8971 assert!(merge_mode("squash").is_err());
8972 }
8973
8974 #[test]
8975 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
8976 let now = Timestamp::now();
8977 let fallback = Duration::from_secs(300);
8978 let cap = Duration::from_secs(1800);
8979
8980 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
8982
8983 let soon = now + jiff::SignedDuration::from_secs(600);
8985 assert_eq!(
8986 quota_wait(Some(soon), now, fallback, cap),
8987 Duration::from_secs(600)
8988 );
8989
8990 let past = now - jiff::SignedDuration::from_secs(60);
8993 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
8994
8995 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
8998 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
8999 }
9000
9001 #[test]
9002 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
9003 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9004
9005 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
9006 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
9007
9008 let already_past =
9012 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
9013 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
9014
9015 assert!(
9016 parse_reset_hint("session limit reached", now, now).is_none(),
9017 "free text with no recognised shape is not guessed at"
9018 );
9019 assert!(
9020 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
9021 "an unresolvable zone name is not guessed at either"
9022 );
9023 }
9024
9025 #[test]
9026 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
9027 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9028
9029 let at = parse_reset_hint(
9030 "You've hit your usage limit. Visit \
9031 https://chatgpt.com/codex/settings/usage to purchase more \
9032 credits or try again at Sep 19th, 2026 5:10 PM.",
9033 now,
9034 now,
9035 )
9036 .expect("the codex reset wording is a recognised shape");
9037 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
9038
9039 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
9044 .expect("an explicit year needs no rollover");
9045 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
9046
9047 assert!(
9048 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
9049 "a two-digit year is not the documented shape and is not guessed at"
9050 );
9051 assert!(
9052 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
9053 "a four-letter month name is not the documented three-letter abbreviation"
9054 );
9055 assert!(
9056 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
9057 "an explicit zone on the dated shape is a format nobody has \
9058 documented, and is refused rather than guessed at as UTC"
9059 );
9060 }
9061
9062 #[test]
9063 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
9064 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
9065 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
9066
9067 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
9068 assert_eq!(at.as_second() - recorded.as_second(), 3769);
9069
9070 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
9071 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
9072
9073 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
9074 assert!(
9075 parse_reset_hint(bad, now, recorded).is_none(),
9076 "{bad:?} must not be guessed at"
9077 );
9078 }
9079 }
9080
9081 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
9085 let config = dir.join("magi.toml");
9086 std::fs::write(
9087 &config,
9088 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
9089 )
9090 .unwrap();
9091 let opts = Opts {
9092 poll: Duration::from_secs(30),
9093 config: Some(config),
9094 repo: dir.join("repo"),
9098 ..Opts::default()
9099 };
9100 let home = dir.join("home");
9109 let worktrees = dir.join("wt");
9110 (
9111 opts,
9112 Queue::at(dir.join("queue")),
9113 home.join("daemon.json"),
9114 home,
9115 worktrees,
9116 )
9117 }
9118
9119 #[test]
9120 fn a_stop_is_idempotent_and_once_set_stays_set() {
9121 let stop = Stop::new();
9122 assert!(!stop.stopped());
9123
9124 stop.stop();
9125 assert!(stop.stopped());
9126 stop.stop();
9127 assert!(stop.stopped(), "a second stop is not a toggle");
9128
9129 let shared = stop.clone();
9130 assert!(
9131 shared.stopped(),
9132 "a clone is the same stop; that is how the loop and its caller share one"
9133 );
9134 }
9135
9136 #[test]
9137 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
9138 let stop = Stop::new();
9139 stop.enter();
9140 assert!(
9141 !stop.finishing(),
9142 "a busy loop nobody has asked to stop is just running"
9143 );
9144
9145 stop.stop();
9146 assert!(
9147 stop.finishing(),
9148 "a stop asked for mid-run has not landed until the run is settled"
9149 );
9150
9151 stop.exit();
9152 assert!(
9153 !stop.finishing(),
9154 "once the run is settled the stop has landed and there is nothing to finish"
9155 );
9156 }
9157
9158 #[test]
9159 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
9160 let stop = Stop::new();
9161 stop.enter();
9162 stop.enter();
9163 stop.stop();
9164 assert!(stop.finishing(), "two runs still in flight");
9165
9166 stop.exit();
9167 assert!(
9168 stop.finishing(),
9169 "one run finished, but a sibling is still working"
9170 );
9171
9172 stop.exit();
9173 assert!(
9174 !stop.finishing(),
9175 "the last run out is what actually lands the stop"
9176 );
9177 }
9178
9179 #[tokio::test]
9180 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
9181 let dir = tempfile::tempdir().unwrap();
9182 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9183 let stop = Stop::new();
9184 stop.stop();
9185
9186 let began = std::time::Instant::now();
9187 tokio::time::timeout(
9188 Duration::from_secs(2),
9189 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9190 )
9191 .await
9192 .expect("a stopped loop must return, not sit out its poll interval")
9193 .expect("the loop's own setup and teardown must not fail");
9194 assert!(
9195 began.elapsed() < opts.poll,
9196 "returned only after {:?}, which is a poll interval, not a stop",
9197 began.elapsed()
9198 );
9199 }
9200
9201 #[tokio::test]
9202 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
9203 let dir = tempfile::tempdir().unwrap();
9204 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9205 let stop = Stop::new();
9206
9207 let asker = {
9210 let stop = stop.clone();
9211 tokio::spawn(async move {
9212 tokio::time::sleep(Duration::from_millis(20)).await;
9213 stop.stop();
9214 })
9215 };
9216
9217 let began = std::time::Instant::now();
9218 tokio::time::timeout(
9219 Duration::from_secs(2),
9220 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9221 )
9222 .await
9223 .expect("a stop asked for while idle must wake the wait")
9224 .expect("the loop's own setup and teardown must not fail");
9225 asker.await.unwrap();
9226 assert!(
9227 began.elapsed() < opts.poll,
9228 "returned only after {:?}, so the stop waited on the sleep",
9229 began.elapsed()
9230 );
9231 }
9232
9233 #[tokio::test]
9234 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
9235 let dir = tempfile::tempdir().unwrap();
9236 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9237 let stop = Stop::new();
9238 stop.stop();
9239
9240 tokio::time::timeout(
9241 Duration::from_secs(2),
9242 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9243 )
9244 .await
9245 .expect("a stopped loop must return")
9246 .expect("the loop's own setup and teardown must not fail");
9247
9248 assert!(
9249 home.is_dir(),
9250 "the loop did publish a status file, so its removal is the teardown and not an absence"
9251 );
9252 assert!(
9253 !status_file.exists(),
9254 "a stopped loop clears its status file"
9255 );
9256 assert!(
9257 read_status(&home).is_none(),
9258 "a reader must see no daemon at all, not a heartbeat that merely stopped"
9259 );
9260 }
9261
9262 #[tokio::test]
9263 async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
9264 let dir = tempfile::tempdir().unwrap();
9265 let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9266 opts.once = true;
9267
9268 let mut settled = RunState::new(
9269 dir.path().join("repo"),
9270 "main".to_owned(),
9271 "abc1234".to_owned(),
9272 "fixture".to_owned(),
9273 Config::default(),
9274 );
9275 settled.status = RunStatus::Ready;
9276 let run_dir = home.join("runs").join(&settled.id);
9277 std::fs::create_dir_all(&run_dir).unwrap();
9278 std::fs::write(
9279 run_dir.join("run.json"),
9280 serde_json::to_string_pretty(&settled).unwrap(),
9281 )
9282 .unwrap();
9283 let questions = Questions::at(home.join("questions"));
9284 let mut question = ask::Question::new(
9285 settled.id.clone(),
9286 "review".to_owned(),
9287 "reviewer-1".to_owned(),
9288 "Continue?".to_owned(),
9289 String::new(),
9290 Vec::new(),
9291 );
9292 questions.put(&mut question).unwrap();
9293
9294 drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
9295 .await
9296 .unwrap();
9297
9298 assert_eq!(
9299 questions.get(&question.id).unwrap().status,
9300 ask::QuestionStatus::Abandoned,
9301 "an empty --once drain still performs startup question cleanup"
9302 );
9303 }
9304
9305 #[test]
9306 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
9307 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9308
9309 assert!(
9310 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
9311 "never checked before: due at once"
9312 );
9313
9314 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
9315 assert!(
9316 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
9317 "well inside the interval: not due yet"
9318 );
9319
9320 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
9321 assert!(
9322 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
9323 "exactly at the edge: not yet due, same convention as `clean::due`"
9324 );
9325
9326 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
9327 assert!(
9328 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
9329 "past the interval: due again"
9330 );
9331 }
9332
9333 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
9338 let config = dir.join("magi.toml");
9339 std::fs::write(
9345 &config,
9346 format!(
9347 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
9348 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
9349 cache_dir.display()
9350 ),
9351 )
9352 .unwrap();
9353 Opts {
9354 config: Some(config),
9355 repo: dir.join("repo"),
9356 ..Opts::default()
9357 }
9358 }
9359
9360 #[tokio::test]
9361 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
9362 let dir = tempfile::tempdir().unwrap();
9363 let home = dir.path().join("home");
9364 let cache_dir = dir.path().join("cache");
9365 std::fs::create_dir_all(&cache_dir).unwrap();
9366 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
9367 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
9368
9369 let running = Stop::new();
9372 let mut last_checked = None;
9373 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9374 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
9375 .await;
9376 assert_eq!(
9377 crate::disk::dir_size(&cache_dir),
9378 0,
9379 "over the cap on the first check ever: pruned at once, no idle queue required"
9380 );
9381 assert_eq!(last_checked, Some(t0));
9382
9383 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
9385 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
9386 maybe_prune_cache_between_runs(
9387 &opts.repo,
9388 &opts,
9389 &home,
9390 &running,
9391 &mut last_checked,
9392 too_soon,
9393 )
9394 .await;
9395 assert_eq!(
9396 crate::disk::dir_size(&cache_dir),
9397 10,
9398 "too soon since the last check: left alone rather than rescanned every call"
9399 );
9400 assert_eq!(
9401 last_checked,
9402 Some(t0),
9403 "an idle check does not reset the clock"
9404 );
9405
9406 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
9408 maybe_prune_cache_between_runs(
9409 &opts.repo,
9410 &opts,
9411 &home,
9412 &running,
9413 &mut last_checked,
9414 due_again,
9415 )
9416 .await;
9417 assert_eq!(
9418 crate::disk::dir_size(&cache_dir),
9419 0,
9420 "due again: pruned back under the cap"
9421 );
9422 }
9423
9424 #[tokio::test]
9432 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
9433 let dir = tempfile::tempdir().unwrap();
9434 let home = dir.path().join("home");
9435 let cache_dir = dir.path().join("cache");
9436 std::fs::create_dir_all(&cache_dir).unwrap();
9437 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
9438 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
9439
9440 let stop = Stop::new();
9441 stop.stop();
9442 assert!(
9443 !stop.finishing(),
9444 "no run is in flight at a between-runs boundary, so nothing else \
9445 would tell the operator this stop had not taken effect yet"
9446 );
9447
9448 let mut last_checked = None;
9449 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9450 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
9451 .await;
9452 assert_eq!(
9453 crate::disk::dir_size(&cache_dir),
9454 10,
9455 "over its cap, and due for the first check ever, but a stop outranks \
9456 it: the cap is a standing policy the next start measures again"
9457 );
9458 assert_eq!(
9459 last_checked, None,
9460 "a check that never happened must not claim the interval"
9461 );
9462 }
9463
9464 #[tokio::test]
9475 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
9476 let dir = tempfile::tempdir().unwrap();
9477 let cache_dir = dir.path().join("cache");
9478 std::fs::create_dir_all(&cache_dir).unwrap();
9479 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
9480
9481 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
9482 opts.poll = Duration::from_millis(20);
9483 opts.max_attempts = 1_000;
9484
9485 let queue = Queue::at(dir.path().join("queue"));
9486 for i in 0..128 {
9487 let mut task = Task::new(
9488 format!("queued work {i}"),
9489 format!("queued work {i}"),
9490 opts.repo.clone(),
9491 Source::Human,
9492 );
9493 queue.put(&mut task).unwrap();
9494 }
9495
9496 let home = dir.path().join("home");
9497 let worktrees = dir.path().join("wt");
9498 let status_file = home.join("daemon.json");
9499 let stop = Stop::new();
9500 let stopper = {
9501 let stop = stop.clone();
9502 let queue = Queue::at(dir.path().join("queue"));
9503 tokio::spawn(async move {
9504 for _ in 0..2_000 {
9508 if queue.list().iter().filter(|t| t.attempts == 1).count() >= 2 {
9509 break;
9510 }
9511 tokio::time::sleep(Duration::from_millis(5)).await;
9512 }
9513 stop.stop();
9514 })
9515 };
9516
9517 tokio::time::timeout(
9518 Duration::from_secs(20),
9519 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9520 )
9521 .await
9522 .expect("the loop must not hang on a queue that keeps producing failing work")
9523 .expect("the loop's own setup and teardown must not fail");
9524 stopper.await.unwrap();
9525
9526 let after = queue.list();
9527 let completed = after.iter().filter(|t| t.attempts == 1).count();
9528 assert!(
9529 completed >= 2,
9530 "the harness must have processed multiple tasks to exercise a busy queue \
9531 (got {completed} completed task(s))"
9532 );
9533 assert!(
9534 after.iter().any(|t| t.status.runnable()),
9535 "unprocessed tasks must remain: the external stop, rather than an \
9536 exhausted backlog, must end the test"
9537 );
9538 assert!(
9539 after
9540 .iter()
9541 .filter(|t| t.attempts > 0)
9542 .all(|t| { t.attempts == 1 && t.status == TaskStatus::Held }),
9543 "a busy backlog must not retry a task's unchanged startup failure"
9544 );
9545
9546 assert_eq!(
9547 crate::disk::dir_size(&cache_dir),
9548 0,
9549 "an oversized cache must not be left to grow unboundedly just because the \
9550 queue kept the loop busy the whole time"
9551 );
9552 }
9553
9554 #[test]
9555 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
9556 let dir = tempfile::tempdir().unwrap();
9557 let queue = Queue::at(dir.path().join("queue"));
9558 let questions = Questions::at(dir.path().join("questions"));
9559 let mut task = task();
9560 queue.put(&mut task).unwrap();
9561
9562 let mut task_question = ask::Question::new(
9563 task.id.clone(),
9564 crate::conduct::NODE.to_owned(),
9565 "conduct".to_owned(),
9566 "Which backend?".to_owned(),
9567 String::new(),
9568 Vec::new(),
9569 );
9570 questions.put(&mut task_question).unwrap();
9571 task.block(vec![task_question.id.clone()], None);
9572 queue.put(&mut task).unwrap();
9573
9574 let mut run_question = ask::Question::new(
9575 "20260101-000000-run1".to_owned(),
9576 "review".to_owned(),
9577 "reviewer-1".to_owned(),
9578 "Run question".to_owned(),
9579 String::new(),
9580 Vec::new(),
9581 );
9582 questions.put(&mut run_question).unwrap();
9583
9584 let mut coincidental = ask::Question::new(
9589 task.id.clone(),
9590 "review".to_owned(),
9591 "reviewer-1".to_owned(),
9592 "Unrelated review question".to_owned(),
9593 String::new(),
9594 Vec::new(),
9595 );
9596 questions.put(&mut coincidental).unwrap();
9597
9598 reconcile_task_questions(&queue, &questions);
9599 assert!(questions.get(&task_question.id).unwrap().status.open());
9600 assert!(questions.get(&run_question.id).unwrap().status.open());
9601 assert!(questions.get(&coincidental.id).unwrap().status.open());
9602
9603 task.release();
9604 queue.put(&mut task).unwrap();
9605 reconcile_task_questions(&queue, &questions);
9606 assert_eq!(
9607 questions.get(&task_question.id).unwrap().status,
9608 ask::QuestionStatus::Abandoned
9609 );
9610 assert!(
9611 questions.get(&run_question.id).unwrap().status.open(),
9612 "run questions remain the run janitor's responsibility"
9613 );
9614 assert!(
9615 questions.get(&coincidental.id).unwrap().status.open(),
9616 "a non-conductor question must not be abandoned just because its \
9617 run id coincides with a task id"
9618 );
9619 }
9620
9621 #[test]
9622 fn a_freshly_started_running_task_is_never_stalled() {
9623 let dir = tempfile::tempdir().unwrap();
9624 let mut t = task();
9625 t.start("run-1".to_owned());
9626 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
9629 }
9630
9631 #[test]
9632 fn a_long_running_task_with_no_live_daemon_is_stalled() {
9633 let dir = tempfile::tempdir().unwrap();
9634 let mut t = task();
9635 t.start("run-1".to_owned());
9636 t.updated_at = Timestamp::now()
9637 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
9638 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
9639 assert_eq!(
9640 stalled_tasks(
9641 &Queue::at(dir.path().join("q")),
9642 dir.path(),
9643 Timestamp::now()
9644 )
9645 .len(),
9646 0,
9647 "the task was never written to this queue"
9648 );
9649 }
9650
9651 #[test]
9652 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
9653 let dir = tempfile::tempdir().unwrap();
9654 let mut t = task();
9655 t.id = "20260903-080340-0167".to_owned();
9656 t.start("20260903-080619-01c2".to_owned());
9657 t.updated_at = Timestamp::now()
9658 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
9659
9660 let mut status = Status::new();
9661 status.current = vec![Current {
9662 task: t.id.clone(),
9663 run: "20260903-080619-01c2".to_owned(),
9664 }];
9665 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9666
9667 assert!(
9668 !is_stalled(&t, dir.path(), Timestamp::now()),
9669 "a live daemon's own heartbeat rules out stalled, however long the task has run"
9670 );
9671 }
9672
9673 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
9677 let path = queue.path_of(id);
9678 let body = std::fs::read_to_string(&path).unwrap();
9679 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
9680 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
9681 v["updated_at"] = serde_json::Value::String(old.to_string());
9682 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
9683 }
9684
9685 #[test]
9686 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
9687 let dir = tempfile::tempdir().unwrap();
9700 let queue = Queue::at(dir.path().join("queue"));
9701 let home = dir.path().join("home");
9702
9703 let mut t = task();
9704 t.id = "20260101-000001-lock".to_owned();
9705 t.start("run-1".to_owned());
9706 queue.put(&mut t).unwrap();
9707 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
9708 std::fs::write(
9709 dir.path().join("queue").join(format!("{}.lock", t.id)),
9710 "not a pid",
9711 )
9712 .unwrap();
9713
9714 let now = Timestamp::now();
9715 assert!(
9716 reclaim_orphaned_running(&queue, 2).is_empty(),
9717 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
9718 and reclaim must leave the task alone"
9719 );
9720 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
9721
9722 let stalled = stalled_tasks(&queue, &home, now);
9723 assert_eq!(
9724 stalled.len(),
9725 1,
9726 "reclaim's inability to claim it yet must not hide it from the conductor"
9727 );
9728 assert_eq!(stalled[0].id, t.id);
9729 }
9730
9731 #[test]
9732 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
9733 let dir = tempfile::tempdir().unwrap();
9734 crate::run::set_home(dir.path().join("run-home"));
9735 let queue = Queue::at(dir.path().join("queue"));
9736 let home = dir.path().join("home");
9737 let questions = Questions::at(dir.path().join("questions"));
9738
9739 let mut t = task();
9740 t.id = "20260101-000003-dead".to_owned();
9741 t.start("missing-run".to_owned());
9742 queue.put(&mut t).unwrap();
9743 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
9744
9745 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
9748 assert_eq!(
9749 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
9750 [&t.id]
9751 );
9752 assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
9753 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
9754
9755 crate::conduct::apply(
9758 &queue,
9759 &questions,
9760 &crate::conduct::Verdict {
9761 decisions: vec![crate::conduct::Decision {
9762 id: t.id.clone(),
9763 recovery: Some(crate::conduct::Recovery::Requeue),
9764 ..crate::conduct::Decision::default()
9765 }],
9766 },
9767 )
9768 .unwrap();
9769 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
9770 }
9771
9772 #[test]
9773 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
9774 let dir = tempfile::tempdir().unwrap();
9775 let queue = Queue::at(dir.path().join("queue"));
9776 let home = dir.path().join("home");
9777
9778 let mut fresh = task();
9779 fresh.id = "20260101-000001-aaaa".to_owned();
9780 fresh.start("run-1".to_owned());
9781 queue.put(&mut fresh).unwrap();
9782
9783 let mut old = task();
9784 old.id = "20260101-000002-bbbb".to_owned();
9785 old.start("run-2".to_owned());
9786 queue.put(&mut old).unwrap();
9787 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
9788
9789 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
9790 assert_eq!(stalled.len(), 1);
9791 assert_eq!(stalled[0].id, old.id);
9792 }
9793
9794 #[test]
9795 fn queued_and_finished_task_views_partition_by_status() {
9796 let dir = tempfile::tempdir().unwrap();
9797 let queue = Queue::at(dir.path().join("queue"));
9798
9799 let mut queued = task();
9800 queued.id = "20260101-000001-aaaa".to_owned();
9801 queue.put(&mut queued).unwrap();
9802
9803 let mut failed = task();
9804 failed.id = "20260101-000002-bbbb".to_owned();
9805 failed.start("run-1".to_owned());
9806 failed.fail("gate red", 5);
9807 queue.put(&mut failed).unwrap();
9808
9809 let mut held = task();
9810 held.id = "20260101-000003-cccc".to_owned();
9811 held.hold_machine(None);
9812 queue.put(&mut held).unwrap();
9813
9814 let mut running = task();
9815 running.id = "20260101-000004-dddd".to_owned();
9816 running.start("run-2".to_owned());
9817 queue.put(&mut running).unwrap();
9818
9819 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
9820 assert_eq!(queued_ids, [queued.id.clone()]);
9821
9822 let mut finished_ids: Vec<String> =
9823 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
9824 finished_ids.sort_unstable();
9825 let mut want = vec![failed.id.clone(), held.id.clone()];
9826 want.sort_unstable();
9827 assert_eq!(finished_ids, want);
9828 }
9829
9830 #[test]
9831 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
9832 for short in [false, true] {
9833 let dir = tempfile::tempdir().unwrap();
9834 let queue = Queue::at(dir.path().join("queue"));
9835 let questions = ask::Questions::at(dir.path().join("questions"));
9836 let mut running = task();
9837 running.id = "20261013-192213-feb2".into();
9838 running.start("existing-run".into());
9839 queue.put(&mut running).unwrap();
9840 let mut middle = task();
9841 middle.id = "20261013-193611-ca5b".into();
9842 middle.block(
9843 vec![if short {
9844 "feb2".into()
9845 } else {
9846 running.id.clone()
9847 }],
9848 None,
9849 );
9850 queue.put(&mut middle).unwrap();
9851 let mut waiting = task();
9852 waiting.id = "20261013-202849-ee37".into();
9853 waiting.block(
9854 vec![if short {
9855 "ca5b".into()
9856 } else {
9857 middle.id.clone()
9858 }],
9859 Some("wait for implementation".into()),
9860 );
9861 queue.put(&mut waiting).unwrap();
9862 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
9863 let inv = crate::blockers::Inventory::load(&queue, &questions);
9864 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
9865 assert!(inv.stuck().is_empty());
9866 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9867 assert!(report.quarantined.is_empty());
9868 assert!(report.asked.is_empty());
9869 resolve_blockers(&queue, &questions);
9870 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9871 assert!(questions.list().is_empty());
9872 running.succeed();
9875 queue.put(&mut running).unwrap();
9876 resolve_blockers(&queue, &questions);
9877 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
9878 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9879 middle = queue.get(&middle.id).unwrap();
9880 middle.succeed();
9881 queue.put(&mut middle).unwrap();
9882 resolve_blockers(&queue, &questions);
9883 let after = queue.get(&waiting.id).unwrap();
9884 assert_eq!(after.status, TaskStatus::Queued);
9885 assert!(after.blocked_by.is_empty());
9886 assert_eq!(after.instruction, waiting.instruction);
9887 }
9888 }
9889
9890 #[test]
9891 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
9892 for future_schema in [false, true] {
9893 let name = "20261013-193611-ca5b";
9894 let mut future = task();
9895 future.id = name.into();
9896 future.schema = crate::queue::SCHEMA + 1;
9897 let contents = if future_schema {
9898 serde_json::to_string(&future).unwrap()
9899 } else {
9900 "corrupt".into()
9901 };
9902 let dir = tempfile::tempdir().unwrap();
9903 let queue = Queue::at(dir.path().join("queue"));
9904 let questions = ask::Questions::at(dir.path().join("questions"));
9905 let mut waiting = task();
9906 waiting.id = "20261013-202849-ee37".into();
9907 waiting.block(vec!["ca5b".into()], None);
9908 queue.put(&mut waiting).unwrap();
9909 std::fs::write(queue.path_of(name), contents).unwrap();
9910 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
9911 assert!(
9912 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
9913 );
9914 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9915 assert!(report.asked.is_empty());
9916 resolve_blockers(&queue, &questions);
9917 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9918 assert!(questions.list().is_empty());
9919 }
9920 let dir = tempfile::tempdir().unwrap();
9921 let queue = Queue::at(dir.path().join("queue"));
9922 let questions = ask::Questions::at(dir.path().join("questions"));
9923 let mut waiting = task();
9924 waiting.id = "20261013-202849-ee37".into();
9925 waiting.block(vec!["ca5b".into()], None);
9926 queue.put(&mut waiting).unwrap();
9927 let mut done = task();
9928 done.id = "20261013-193611-ca5b".into();
9929 done.succeed();
9930 queue.put(&mut done).unwrap();
9931 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
9934 assert!(queue.get("ca5b").is_err());
9935 assert_eq!(
9936 crate::queue::dependency(&queue, &questions, "ca5b"),
9937 crate::queue::Dependency::Ambiguous
9938 );
9939 resolve_blockers(&queue, &questions);
9940 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9941 assert!(report.asked.is_empty());
9942 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9943 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
9944 }
9945
9946 #[test]
9947 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
9948 for held in [false, true] {
9949 for question in [false, true] {
9950 for reference in ["ca5b", "20261013-193611-ca5b"] {
9951 let dir = tempfile::tempdir().unwrap();
9952 let queue = Queue::at(dir.path().join("queue"));
9953 let questions = ask::Questions::at(dir.path().join("questions"));
9954 let mut prerequisite = task();
9955 prerequisite.id = "20261013-193611-ca5b".into();
9956 prerequisite.start("existing-run".into());
9957 queue.put(&mut prerequisite).unwrap();
9958 let mut waiting = task();
9959 waiting.id = "20261013-202849-ee37".into();
9960 if held {
9961 waiting.hold_machine(Some("waiting for separate consent".into()));
9962 }
9963 queue.put(&mut waiting).unwrap();
9964 let reason = "Deleting saved data requires separate owner consent";
9965 crate::conduct::apply(
9966 &queue,
9967 &questions,
9968 &crate::conduct::Verdict {
9969 decisions: vec![crate::conduct::Decision {
9970 id: waiting.id.clone(),
9971 blocked_by: vec![reference.into()],
9972 question: question.then(|| "May the saved data be deleted?".into()),
9973 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
9974 reason: Some(reason.into()),
9975 ..crate::conduct::Decision::default()
9976 }],
9977 },
9978 )
9979 .unwrap();
9980 let before = queue.get(&waiting.id).unwrap();
9981 if question {
9982 assert_eq!(before.status, TaskStatus::Blocked);
9983 let q = questions.get(&before.blocked_by[0]).unwrap();
9984 assert_eq!(q.summary, "May the saved data be deleted?");
9985 assert!(q.status.open());
9986 } else {
9987 assert_eq!(before.status, TaskStatus::Held);
9988 let recorded_reason = before.hold_reason.as_deref().unwrap();
9989 assert!(recorded_reason.starts_with(reason));
9990 if held {
9991 assert!(recorded_reason.contains("waiting for separate consent"));
9992 }
9993 }
9994 prerequisite.succeed();
9995 queue.put(&mut prerequisite).unwrap();
9996 resolve_blockers(&queue, &questions);
9997 let after = queue.get(&waiting.id).unwrap();
9998 assert_eq!(after.status, before.status);
9999 assert_eq!(after.blocked_by, before.blocked_by);
10000 assert_eq!(after.hold_reason, before.hold_reason);
10001 assert_eq!(after.instruction, waiting.instruction);
10002 if question {
10003 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
10004 }
10005 assert!(
10006 queue.next_runnable().is_none(),
10007 "dependency completion is not consent"
10008 );
10009 }
10010 }
10011 }
10012 }
10013
10014 #[test]
10015 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
10016 let dir = tempfile::tempdir().unwrap();
10017 let queue = Queue::at(dir.path().join("queue"));
10018 let questions = ask::Questions::at(dir.path().join("questions"));
10019
10020 let mut dep = task();
10021 dep.id = "20260101-000001-dep0".to_owned();
10022 dep.succeed();
10023 queue.put(&mut dep).unwrap();
10024
10025 let mut still_going = task();
10026 still_going.id = "20260101-000002-dep1".to_owned();
10027 queue.put(&mut still_going).unwrap();
10028
10029 let mut blocked = task();
10030 blocked.id = "20260101-000003-main".to_owned();
10031 blocked.block(
10032 vec![dep.id.clone(), still_going.id.clone()],
10033 Some("waits on both".to_owned()),
10034 );
10035 queue.put(&mut blocked).unwrap();
10036
10037 resolve_blockers(&queue, &questions);
10038
10039 let after = queue.get(&blocked.id).unwrap();
10040 assert_eq!(
10041 after.status,
10042 TaskStatus::Blocked,
10043 "one dependency is still outstanding"
10044 );
10045 assert_eq!(after.blocked_by, [still_going.id.clone()]);
10046 }
10047
10048 #[test]
10049 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
10050 let dir = tempfile::tempdir().unwrap();
10051 let queue = Queue::at(dir.path().join("queue"));
10052 let questions = ask::Questions::at(dir.path().join("questions"));
10053
10054 let mut q = crate::ask::Question::new(
10055 "20260101-000001-main".to_owned(),
10056 crate::conduct::NODE.to_owned(),
10057 "conduct".to_owned(),
10058 "Which backend?".to_owned(),
10059 String::new(),
10060 Vec::new(),
10061 );
10062 questions.put(&mut q).unwrap();
10063 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
10064 .unwrap();
10065 questions.put(&mut q).unwrap();
10066
10067 let mut blocked = task();
10068 blocked.id = "20260101-000001-main".to_owned();
10069 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
10070 queue.put(&mut blocked).unwrap();
10071
10072 resolve_blockers(&queue, &questions);
10073
10074 let after = queue.get(&blocked.id).unwrap();
10075 assert_eq!(
10076 after.status,
10077 TaskStatus::Queued,
10078 "the only blocker resolved"
10079 );
10080 assert_eq!(after.answers.len(), 1);
10081 assert_eq!(after.answers[0].question, "Which backend?");
10082 assert_eq!(after.answers[0].answer, "SQLite");
10083
10084 let instruction = instruction_for(&after);
10086 assert!(instruction.contains("Which backend?"));
10087 assert!(instruction.contains("SQLite"));
10088 }
10089
10090 #[test]
10091 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
10092 let dir = tempfile::tempdir().unwrap();
10093 let queue = Queue::at(dir.path().join("queue"));
10094 let questions = ask::Questions::at(dir.path().join("questions"));
10095
10096 let mut q = crate::ask::Question::new(
10097 "20260101-000001-main".to_owned(),
10098 crate::conduct::NODE.to_owned(),
10099 "conduct".to_owned(),
10100 "Is the setup done?".to_owned(),
10101 String::new(),
10102 Vec::new(),
10103 );
10104 q.abandon("no answer within 60s of asking");
10105 questions.put(&mut q).unwrap();
10106
10107 let mut blocked = task();
10108 blocked.id = "20260101-000001-main".to_owned();
10109 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
10110 queue.put(&mut blocked).unwrap();
10111
10112 resolve_blockers(&queue, &questions);
10113
10114 let after = queue.get(&blocked.id).unwrap();
10115 assert_eq!(
10116 after.status,
10117 TaskStatus::Held,
10118 "never left blocked on nothing"
10119 );
10120 assert!(!after.operator_held(), "a machine hold, for triage");
10121 assert!(
10122 after
10123 .hold_reason
10124 .as_deref()
10125 .unwrap_or_default()
10126 .contains("went unanswered")
10127 );
10128 }
10129
10130 #[test]
10131 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
10132 let dir = tempfile::tempdir().unwrap();
10138 let queue = Queue::at(dir.path().join("queue"));
10139 let questions = ask::Questions::at(dir.path().join("questions"));
10140
10141 let mut q = crate::ask::Question::new(
10142 "20260101-000001-main".to_owned(),
10143 crate::conduct::NODE.to_owned(),
10144 "conduct".to_owned(),
10145 "How should this be handled?".to_owned(),
10146 String::new(),
10147 Vec::new(),
10148 );
10149 questions.put(&mut q).unwrap();
10150 q.answer(crate::ask::Answer::Text(
10151 "leave it held, a human will look at it later".to_owned(),
10152 ))
10153 .unwrap();
10154 questions.put(&mut q).unwrap();
10155
10156 let mut held = task();
10157 held.id = "20260101-000001-main".to_owned();
10158 held.hold_machine(Some("out of attempts".to_owned()));
10159 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
10160 queue.put(&mut held).unwrap();
10161
10162 resolve_blockers(&queue, &questions);
10163
10164 let after = queue.get(&held.id).unwrap();
10165 assert_eq!(after.status, TaskStatus::Held);
10166 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
10167 assert_eq!(
10168 after.answers[0].answer,
10169 "leave it held, a human will look at it later"
10170 );
10171 }
10172
10173 #[test]
10174 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
10175 let dir = tempfile::tempdir().unwrap();
10176 let queue = Queue::at(dir.path().join("queue"));
10177 let questions = ask::Questions::at(dir.path().join("questions"));
10178 let mut dep = task();
10179 dep.id = "20260101-000001-gone".to_owned();
10180 queue.put(&mut dep).unwrap();
10181 let mut blocked = task();
10182 blocked.id = "20260101-000003-main".to_owned();
10183 blocked.block(vec![dep.id.clone()], None);
10184 queue.put(&mut blocked).unwrap();
10185
10186 let claim = queue.claim(&blocked.id).unwrap();
10188 queue.remove(&dep.id, false, &questions).unwrap();
10189 drop(claim);
10190 resolve_blockers(&queue, &questions);
10191
10192 let after = queue.get(&blocked.id).unwrap();
10193 assert_eq!(after.status, TaskStatus::Queued);
10194 assert!(after.blocked_by.is_empty());
10195 }
10196
10197 #[test]
10198 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
10199 let dir = tempfile::tempdir().unwrap();
10205 let queue = Queue::at(dir.path().join("queue"));
10206 let questions = ask::Questions::at(dir.path().join("questions"));
10207
10208 let mut still_going = task();
10209 still_going.id = "20260101-000002-dep1".to_owned();
10210 queue.put(&mut still_going).unwrap();
10211
10212 let mut blocked = task();
10213 blocked.id = "20260101-000003-main".to_owned();
10214 blocked.block(
10215 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
10216 Some("waits on both".to_owned()),
10217 );
10218 queue.put(&mut blocked).unwrap();
10219
10220 resolve_blockers(&queue, &questions);
10221
10222 let after = queue.get(&blocked.id).unwrap();
10223 assert_eq!(
10224 after.status,
10225 TaskStatus::Held,
10226 "a missing dependency must not leave the task blocked forever"
10227 );
10228 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
10229 assert!(after.blocked_by.is_empty());
10230 let reason = after.hold_reason.as_deref().unwrap_or_default();
10231 assert!(
10232 reason.contains("20260101-000001-gone"),
10233 "the missing id must be named so an operator can tell what happened: {reason}"
10234 );
10235 assert!(
10236 reason.contains(&still_going.id),
10237 "the still-valid dependency must not silently vanish from the record: {reason}"
10238 );
10239 }
10240
10241 #[test]
10242 fn instruction_for_is_unchanged_without_any_answers() {
10243 let t = task();
10244 assert_eq!(instruction_for(&t), t.instruction);
10245 }
10246
10247 #[test]
10248 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
10249 let dir = tempfile::tempdir().unwrap();
10250 let q = Queue::at(dir.path().join("queue"));
10251 let src = dir.path().join("shot.png");
10252 std::fs::write(&src, "x").unwrap();
10253 let mut t = task();
10254 q.attach(&mut t, &[src]).unwrap();
10255 let paths = task_attachments(&q, &t).unwrap();
10256 assert_eq!(paths.len(), 1);
10257 assert!(paths[0].is_absolute() && paths[0].is_file());
10258 std::fs::remove_file(&paths[0]).unwrap();
10259 let err = task_attachments(&q, &t).unwrap_err().to_string();
10260 assert!(err.contains("shot.png"), "{err}");
10261 }
10262
10263 #[tokio::test]
10264 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
10265 let dir = tempfile::tempdir().unwrap();
10266 let queue_path = dir.path().join("queue");
10267 let queue = Queue::at(queue_path.clone());
10268 let mut t = task();
10269 t.source = Source::Agent {
10270 run: "originating-chat".into(),
10271 node: crate::queue::CHAT_NODE.into(),
10272 };
10273 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
10274 stage: "config-fetch",
10275 target: "origin/main".into(),
10276 });
10277 record_config_failure(&mut t, &fetch);
10278 t.hold_machine(Some(format!("{fetch:#}")));
10279 let fetch_key = t.chat_notice_due().unwrap();
10280 t.chat_notices.push(fetch_key.clone());
10281 queue.put(&mut t).unwrap();
10282
10283 let queue = Queue::at(queue_path);
10284 let mut t = queue.get(&t.id).unwrap();
10285 let a = dir.path().join("a.toml");
10286 let b = dir.path().join("b.toml");
10287 for (path, text, new_cause) in [
10288 (&a, "[graph", true),
10289 (&a, "[verify", false),
10290 (&a, "[graph]\nimplementers = 'invalid-type'", true),
10291 (&b, "[graph]\nimplementers = 'invalid-type'", true),
10292 ] {
10293 t.release_operator("test", None);
10294 std::fs::write(path, text).unwrap();
10295 let error = Config::discover_fetched(dir.path(), Some(path))
10296 .await
10297 .unwrap_err();
10298 record_config_failure(&mut t, &error);
10299 t.hold_machine(Some(format!("{error:#}")));
10300 let due = t.chat_notice_due();
10301 assert_eq!(due.is_some(), new_cause, "{error:#}");
10302 if let Some(key) = due {
10303 assert_ne!(key, fetch_key);
10304 t.chat_notices.push(key);
10305 }
10306 }
10307 t.succeed();
10308 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
10309 }
10310
10311 #[test]
10312 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
10313 let dir = tempfile::tempdir().unwrap();
10314 let queue_path = dir.path().join("queue");
10315 let queue = Queue::at(queue_path.clone());
10316 let mut t = task();
10317 t.source = crate::queue::Source::Agent {
10318 run: "originating-chat".into(),
10319 node: crate::queue::CHAT_NODE.into(),
10320 };
10321 let a = dir.path().join("a.png");
10322 let b = dir.path().join("b.png");
10323 std::fs::write(&a, "a").unwrap();
10324 std::fs::write(&b, "b").unwrap();
10325 queue.attach(&mut t, &[a, b]).unwrap();
10326 let paths = task_attachments(&queue, &t).unwrap();
10327 for path in &paths {
10328 std::fs::remove_file(path).unwrap();
10329 }
10330 let missing = task_attachments(&queue, &t).unwrap_err();
10331 t.attempts += 1;
10332 let reason = missing.to_string();
10333 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
10334 let first = t.chat_notice_due().expect("first attachment notice");
10335 t.chat_notices.push(first.clone());
10336 queue.put(&mut t).unwrap();
10337
10338 let queue = Queue::at(queue_path.clone());
10339 let mut t = queue.get(&t.id).unwrap();
10340 assert!(t.chat_notice_due().is_none(), "restart does not replay");
10341 t.release_operator("test", None);
10342 t.attempts += 1;
10343 let missing = task_attachments(&queue, &t).unwrap_err();
10344 t.fail_startup_target(
10345 "startup-attachments",
10346 missing.name,
10347 "same missing attachment with regenerated wording",
10348 1,
10349 );
10350 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
10351
10352 std::fs::write(&paths[0], "restored").unwrap();
10353 t.release_operator("test", None);
10354 t.attempts += 1;
10355 let missing = task_attachments(&queue, &t).unwrap_err();
10356 let reason = missing.to_string();
10357 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
10358 let second = t
10359 .chat_notice_due()
10360 .expect("different attachment must notify");
10361 assert_ne!(first, second);
10362 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
10363 }
10364
10365 #[test]
10366 fn resumed_instruction_is_unchanged_without_any_answers() {
10367 let t = task();
10368 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
10369 }
10370
10371 #[test]
10372 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
10373 let mut t = task();
10374 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10375 let old = t.instruction.clone();
10379
10380 let refreshed = resumed_instruction(&old, &t);
10381 assert!(refreshed.starts_with(&old), "the original text is kept");
10382 assert!(refreshed.contains("Which backend?"));
10383 assert!(refreshed.contains("SQLite"));
10384 }
10385
10386 #[test]
10387 fn resumed_instruction_keeps_an_original_answers_heading() {
10388 let mut t = task();
10389 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
10390 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10391
10392 let refreshed = resumed_instruction(&t.instruction, &t);
10393
10394 assert!(
10395 refreshed.starts_with(&t.instruction),
10396 "an answers heading in the original instruction is not the appended block"
10397 );
10398 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
10399 assert!(refreshed.contains("Which backend?"));
10400 assert!(refreshed.contains("SQLite"));
10401
10402 let repeated = resumed_instruction(&refreshed, &t);
10403 assert_eq!(
10404 repeated, refreshed,
10405 "only the final appended block is refreshed"
10406 );
10407 }
10408
10409 #[test]
10410 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
10411 let mut t = task();
10412 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10413
10414 let once = resumed_instruction(&t.instruction, &t);
10418 let twice = resumed_instruction(&once, &t);
10419 assert_eq!(once, twice);
10420 assert_eq!(once.matches("Which backend?").count(), 1);
10421
10422 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
10424 let refreshed = resumed_instruction(&once, &t);
10425 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
10426 assert!(refreshed.contains("Which backend?"));
10427 assert!(refreshed.contains("Which cache?"));
10428 }
10429
10430 #[test]
10431 fn prepare_instruction_covers_all_three_starters() {
10432 let mut t = task();
10433 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10434
10435 assert_eq!(
10438 prepare_instruction(&Starter::Start, None, &t),
10439 Some(instruction_for(&t))
10440 );
10441
10442 let old = t.instruction.clone();
10445 assert_eq!(
10446 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
10447 Some(resumed_instruction(&old, &t))
10448 );
10449
10450 assert_eq!(
10454 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
10455 None
10456 );
10457 }
10458
10459 #[test]
10460 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
10461 assert_eq!(
10462 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
10463 Starter::Review("magi/eba2/A".to_owned())
10464 );
10465 }
10466
10467 #[test]
10468 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
10469 assert_eq!(
10470 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
10471 Starter::Start,
10472 "a vanished review branch must not fall back to resuming the old run either"
10473 );
10474 }
10475
10476 #[test]
10477 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
10478 let mut t = task();
10479 t.start("old-run".to_owned());
10480 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
10481 t.release();
10482 let branch = t.review_branch.take();
10483 assert_eq!(
10484 choose_starter(branch.as_deref(), true, Some("old-run")),
10485 Starter::Review("magi/eba2/A".to_owned()),
10486 "a review wins over resuming the old run"
10487 );
10488 }
10489
10490 #[test]
10491 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
10492 assert_eq!(
10493 choose_starter(None, false, Some("some-run")),
10494 Starter::Resume("some-run".to_owned())
10495 );
10496 assert_eq!(choose_starter(None, false, None), Starter::Start);
10497 }
10498
10499 #[test]
10500 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
10501 let mut released = task();
10502 released.start("stalled-run".to_owned());
10503 released.requeue();
10504 let unfinished = (!released.fresh_start)
10505 .then(|| Some("stalled-run".to_owned()))
10506 .flatten();
10507 assert_eq!(
10508 choose_starter(None, false, unfinished.as_deref()),
10509 Starter::Start,
10510 "release keeps run history but must not resume it"
10511 );
10512 assert_eq!(released.runs, ["stalled-run"]);
10513 }
10514
10515 fn stopped_public_run(status: RunStatus) -> RunState {
10516 let mut state = run_state(status);
10517 state.driver_exited = true;
10518 state.config.graph.land = true;
10519 state.config.graph.review_rounds = 1;
10520 state.reviews = vec![review_round(1)];
10521 state.merge = Some(crate::run::MergeOutcome {
10522 mode: MergeMode::Pr,
10523 ok: false,
10524 detail: "landing stopped".into(),
10525 empty: false,
10526 });
10527 state.pr = Some(crate::run::PrRecord {
10528 url: "https://example.test/pull/1".into(),
10529 number: 1,
10530 state: "open".into(),
10531 checks: "red".into(),
10532 round: 1,
10533 rounds: 1,
10534 red_at_merge: Vec::new(),
10535 });
10536 state
10537 }
10538
10539 #[test]
10540 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
10541 for status in [RunStatus::Blocked, RunStatus::Gating] {
10542 let home = tempfile::tempdir().unwrap();
10543 let mut state = stopped_public_run(status);
10544 state.save_under(home.path()).unwrap();
10545 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
10546 assert!(!exhausted_review_budget(&state));
10547 let mut task = held_task_with(&state.id);
10548 task.release();
10549 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
10550 assert_eq!(
10551 choose_starter(None, false, unfinished.as_deref()),
10552 Starter::Resume(state.id.clone())
10553 );
10554 let question = action_question(&state.id, resume_action(&state.id));
10555 assert_eq!(
10556 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
10557 ActionDecision::Resume(state.id.clone())
10558 );
10559 assert_eq!(
10561 crate::resume::validate(home.path(), &state.id).unwrap().id,
10562 state.id
10563 );
10564 let queue = Queue::at(home.path().join("queue"));
10565 let mut controlled = Task::new(
10566 "resume land".into(),
10567 state.instruction.clone(),
10568 state.repo.clone(),
10569 Source::Agent {
10570 run: "talk".into(),
10571 node: crate::queue::CHAT_NODE.into(),
10572 },
10573 );
10574 controlled.runs.push(state.id.clone());
10575 controlled.hold_machine(Some("land stopped".into()));
10576 queue.put(&mut controlled).unwrap();
10577 let resumed =
10578 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
10579 assert_eq!(resumed.status, TaskStatus::Queued);
10580 assert_eq!(
10581 resumed.resume_override.unwrap().pinned_run,
10582 Some(state.id.clone())
10583 );
10584 assert_eq!(resumed.attempts, 0);
10585 }
10586 }
10587
10588 #[test]
10589 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
10590 for review_rounds in [0, 1] {
10591 let home = tempfile::tempdir().unwrap();
10592 let mut state = run_state(RunStatus::Blocked);
10593 state.config.graph.review_rounds = review_rounds;
10594 if review_rounds > 0 {
10595 let mut review = review_round(1);
10596 review.clean = true;
10597 state.reviews = vec![review];
10598 }
10599 state.gate_ran = true;
10600 state.gate = vec![crate::run::CommandOutcome {
10601 command: "verify".into(),
10602 code: Some(1),
10603 output_tail: "gate failed after repair budget exhausted".into(),
10604 duration_ms: 0,
10605 resource_blocked: false,
10606 }];
10607 state.save_under(home.path()).unwrap();
10608 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
10609 assert!(exhausted_review_budget(&state));
10610 let mut task = held_task_with(&state.id);
10611 task.release();
10612 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
10613 let question = action_question(&state.id, resume_action(&state.id));
10614 assert_ne!(
10615 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
10616 ActionDecision::Resume(state.id.clone())
10617 );
10618 assert!(crate::resume::validate(home.path(), &state.id).is_err());
10619
10620 state.gate_ran = false;
10622 state.gate.clear();
10623 assert!(!exhausted_review_budget(&state));
10624 assert_eq!(
10625 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
10626 Some(state.id)
10627 );
10628 }
10629 }
10630
10631 #[test]
10632 fn resource_blocked_verification_resumes_the_same_run() {
10633 for (kind, stage) in [
10634 ("cache", "clean"),
10635 ("coordination", "clean"),
10636 ("cache", "review-e2e"),
10637 ("coordination", "no-reviews"),
10638 ("coordination", "handoff"),
10639 ] {
10640 let home = tempfile::tempdir().unwrap();
10641 let mut state = run_state(RunStatus::Blocked);
10642 state.config.graph.review_rounds = 1;
10643 let mut review = review_round(1);
10644 review.clean = true;
10645 state.reviews = vec![review];
10646 if stage == "review-e2e" {
10647 let review = &mut state.reviews[0];
10648 review.clean = false;
10649 review.blocking = 1;
10650 review.e2e = vec![crate::run::CommandOutcome {
10651 command: "verify".into(),
10652 code: None,
10653 output_tail: "resource wait expired".into(),
10654 duration_ms: 0,
10655 resource_blocked: true,
10656 }];
10657 } else if stage == "no-reviews" {
10658 state.config.graph.review_rounds = 0;
10659 state.reviews.clear();
10660 } else if stage == "handoff" {
10661 let review = &mut state.reviews[0];
10664 review.clean = false;
10665 review.blocking = 1;
10666 review.e2e = vec![crate::run::CommandOutcome {
10667 command: "verify".into(),
10668 code: Some(0),
10669 output_tail: String::new(),
10670 duration_ms: 0,
10671 resource_blocked: false,
10672 }];
10673 }
10674 state.gate_ran = false;
10675 state.resource_block = Some(crate::run::ResourceBlock {
10676 node: if stage == "review-e2e" {
10677 "review"
10678 } else {
10679 "gate"
10680 }
10681 .into(),
10682 kind: kind.into(),
10683 detail: "resource wait expired".into(),
10684 at: jiff::Timestamp::now(),
10685 });
10686 state.save_under(home.path()).unwrap();
10687 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
10688 assert!(!exhausted_review_budget(&state));
10689 let mut task = held_task_with(&state.id);
10690 task.release();
10691 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
10692 assert_eq!(
10693 choose_starter(None, false, unfinished.as_deref()),
10694 Starter::Resume(state.id.clone())
10695 );
10696 let question = action_question(&state.id, resume_action(&state.id));
10697 assert_eq!(
10698 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
10699 ActionDecision::Resume(state.id.clone())
10700 );
10701 assert_eq!(
10702 crate::resume::validate(home.path(), &state.id).unwrap().id,
10703 state.id
10704 );
10705 let queue = Queue::at(home.path().join("queue"));
10706 let mut controlled = Task::new(
10707 "resume verification".into(),
10708 state.instruction.clone(),
10709 state.repo.clone(),
10710 Source::Agent {
10711 run: "talk".into(),
10712 node: crate::queue::CHAT_NODE.into(),
10713 },
10714 );
10715 controlled.runs.push(state.id.clone());
10716 controlled.hold_machine(Some("verification resource unavailable".into()));
10717 queue.put(&mut controlled).unwrap();
10718 let resumed =
10719 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
10720 assert_eq!(resumed.status, TaskStatus::Queued);
10721 assert_eq!(
10722 resumed.resume_override.unwrap().pinned_run,
10723 Some(state.id.clone())
10724 );
10725 assert_eq!(resumed.attempts, 0);
10726
10727 state.config.graph.review_rounds = 1;
10729 let mut unresolved = review_round(1);
10730 unresolved.blocking = 1;
10731 state.reviews = vec![unresolved];
10732 state.resource_block = None;
10733 state.save_under(home.path()).unwrap();
10734 assert!(exhausted_review_budget(&state));
10735 assert!(crate::resume::validate(home.path(), &state.id).is_err());
10736 }
10737 }
10738
10739 #[tokio::test]
10740 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
10741 let home = tempfile::tempdir().unwrap();
10742 let mut owner = stopped_public_run(RunStatus::Blocked);
10743 let repo = tempfile::tempdir().unwrap();
10744 crate::git::git(repo.path(), &["init"]).await.unwrap();
10745 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
10746 owner.id = "20260101-000000-aaaa".into();
10747 owner.save_under(home.path()).unwrap();
10748 let mut successor = run_state(RunStatus::Blocked);
10749 successor.id = "20260101-000001-bbbb".into();
10750 successor.save_under(home.path()).unwrap();
10751 let runs = vec![owner.id.clone(), successor.id.clone()];
10752 assert_eq!(
10753 public_resume_owner(home.path(), &runs, &owner.repo)
10754 .await
10755 .unwrap()
10756 .unwrap()
10757 .id,
10758 owner.id
10759 );
10760 let subdir = owner.repo.join("nested");
10761 std::fs::create_dir(&subdir).unwrap();
10762 assert_eq!(
10763 public_resume_owner(home.path(), &runs, &subdir)
10764 .await
10765 .unwrap()
10766 .unwrap()
10767 .id,
10768 owner.id
10769 );
10770 #[cfg(unix)]
10771 {
10772 let alias = home.path().join("repo-alias");
10773 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
10774 assert_eq!(
10775 public_resume_owner(home.path(), &runs, &alias.join("nested"))
10776 .await
10777 .unwrap()
10778 .unwrap()
10779 .id,
10780 owner.id
10781 );
10782 }
10783 let different = tempfile::tempdir().unwrap();
10784 crate::git::git(different.path(), &["init"]).await.unwrap();
10785 assert!(
10786 public_resume_owner(home.path(), &runs, different.path())
10787 .await
10788 .is_err()
10789 );
10790 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
10791 assert!(
10792 public_resume_owner(home.path(), &runs, &owner.repo)
10793 .await
10794 .is_err()
10795 );
10796 drop(lock);
10797 std::fs::remove_file(
10798 home.path()
10799 .join("runs")
10800 .join(&successor.id)
10801 .join("run.json"),
10802 )
10803 .unwrap();
10804 assert!(
10805 public_resume_owner(home.path(), &runs, &owner.repo)
10806 .await
10807 .is_err()
10808 );
10809 }
10810
10811 #[test]
10812 fn an_ordinary_release_keeps_a_resumable_run_available() {
10813 let mut released = task();
10814 released.start("stalled-run".to_owned());
10815 released.release();
10816 let unfinished = (!released.fresh_start)
10817 .then(|| Some("stalled-run".to_owned()))
10818 .flatten();
10819 assert_eq!(
10820 choose_starter(None, false, unfinished.as_deref()),
10821 Starter::Resume("stalled-run".to_owned()),
10822 "manual release must preserve the normal resume path"
10823 );
10824 }
10825
10826 #[test]
10827 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
10828 let mut state = run_state(RunStatus::Blocked);
10829 state.config.graph.review_rounds = 3;
10830 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
10831 assert!(exhausted_review_budget(&state));
10832
10833 state.reviews.pop();
10835 assert!(!exhausted_review_budget(&state));
10836
10837 let mut stalled = run_state(RunStatus::Stalled);
10840 stalled.config.graph.review_rounds = 1;
10841 stalled.reviews = vec![review_round(1)];
10842 assert!(!exhausted_review_budget(&stalled));
10843 }
10844
10845 fn review_round(round: usize) -> crate::run::ReviewRound {
10846 crate::run::ReviewRound {
10847 round,
10848 head: "deadbeef".to_owned(),
10849 verified_head: None,
10850 verified_at: None,
10851 reviews: Vec::new(),
10852 e2e: Vec::new(),
10853 verify_retried: false,
10854 e2e_deferred: false,
10855 e2e_defer_reason: None,
10856 fix: None,
10857 blocking: 0,
10858 answered: 1,
10859 expected: 1,
10860 clean: false,
10861 progressed: true,
10862 vote_split: false,
10863 reconsideration: Vec::new(),
10864 verdict: None,
10865 }
10866 }
10867
10868 #[test]
10869 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
10870 let mut run = RunState::new(
10871 PathBuf::from("/repo"),
10872 "main".to_owned(),
10873 "abc1234def".to_owned(),
10874 "add retries".to_owned(),
10875 Config::default(),
10876 );
10877 run.status = RunStatus::Judging;
10878 run.parked = true;
10879 let mut task = Task::new(
10880 "add retries".to_owned(),
10881 "add retries".to_owned(),
10882 PathBuf::from("/repo"),
10883 crate::queue::Source::Human,
10884 );
10885 task.status = TaskStatus::Failed;
10886 task.runs = vec![run.id.clone()];
10887 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
10888 assert!(with(&task, &run), "parked after judging is the case");
10889 let mut parked_task = task.clone();
10890 parked_task.status = TaskStatus::Parked;
10891 assert!(with(&parked_task, &run), "the Parked status is the case");
10892
10893 let mut not_parked = run.clone();
10894 not_parked.parked = false;
10895 not_parked.status = RunStatus::Stalled;
10896 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
10897
10898 let mut fresh = task.clone();
10899 fresh.fresh_start = true;
10900 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
10901
10902 let mut review = task.clone();
10903 review.review_branch = Some("magi/x/A".to_owned());
10904 assert!(!with(&review, &run), "review is ranked before resume");
10905
10906 let mut held = task.clone();
10907 held.status = TaskStatus::Held;
10908 assert!(!with(&held, &run), "a hold stays visible to the conductor");
10909
10910 let mut released = run.clone();
10911 released.released_to = Some("20260901-000000-new1".to_owned());
10912 assert!(!with(&task, &released), "nothing left to resume into");
10913
10914 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
10915 "unreadable"
10916 )));
10917 }
10918
10919 #[test]
10920 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
10921 let mut released = RunState::new(
10922 PathBuf::from("/repo"),
10923 "main".to_owned(),
10924 "abc1234def".to_owned(),
10925 "add retries".to_owned(),
10926 Config::default(),
10927 );
10928 released.status = RunStatus::Blocked;
10929 assert_eq!(
10930 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
10931 Some(released.id.clone())
10932 );
10933 released.released_to = Some("20260901-000000-new1".to_owned());
10934 assert_eq!(
10935 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
10936 None,
10937 "there is nothing left to resume it into"
10938 );
10939 }
10940
10941 #[test]
10942 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
10943 let mut exhausted = RunState::new(
10952 PathBuf::from("/repo"),
10953 "main".to_owned(),
10954 "abc1234def".to_owned(),
10955 "add retries".to_owned(),
10956 Config::default(),
10957 );
10958 exhausted.status = RunStatus::Blocked;
10959 exhausted.config.graph.review_rounds = 1;
10960 exhausted.reviews = vec![review_round(1)];
10961
10962 assert_eq!(
10963 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
10964 None,
10965 "an exhausted `Blocked` run must not be offered as resumable"
10966 );
10967
10968 let mut has_budget_left = RunState::new(
10971 PathBuf::from("/repo"),
10972 "main".to_owned(),
10973 "abc1234def".to_owned(),
10974 "add retries".to_owned(),
10975 Config::default(),
10976 );
10977 has_budget_left.status = RunStatus::Blocked;
10978 has_budget_left.config.graph.review_rounds = 3;
10979 has_budget_left.reviews = vec![review_round(1)];
10980
10981 assert_eq!(
10982 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
10983 Ok(has_budget_left.clone())
10984 }),
10985 Some(has_budget_left.id.clone())
10986 );
10987 }
10988
10989 #[test]
10990 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
10991 let mut run = RunState::new(
10992 PathBuf::from("/repo"),
10993 "main".to_owned(),
10994 "abc1234def".to_owned(),
10995 "add retries".to_owned(),
10996 Config::default(),
10997 );
10998 run.status = RunStatus::Blocked;
10999 run.config.graph.review_rounds = 1;
11000 run.reviews = vec![review_round(1)];
11001 run.pr_publication_pending = true;
11002 assert_eq!(
11003 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
11004 Some(run.id.clone()),
11005 "a spent review budget must not turn a failed publication into a re-competition"
11006 );
11007 }
11008
11009 #[test]
11010 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
11011 let mut task = Task::new(
11012 "recover work".into(),
11013 "recover work".into(),
11014 PathBuf::from("/repo"),
11015 crate::queue::Source::Human,
11016 );
11017 let mut history = std::collections::BTreeMap::new();
11018 for index in 0..8 {
11019 let mut run = run_state(RunStatus::Blocked);
11020 run.id = format!("20260101-000000-{index:04}");
11021 run.base_commit = format!("upstream-{index}");
11022 if index > 0 {
11024 run.candidates.clear();
11025 }
11026 task.runs.extend([run.id.clone(), run.id.clone()]);
11027 history.insert(run.id.clone(), run);
11028 task.requeue();
11029 }
11030 let check = |task: &Task, starter: &Starter, limit| {
11031 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
11032 };
11033 assert!(
11034 check(&task, &Starter::Start, 8)
11035 .unwrap()
11036 .contains("8 unretired")
11037 );
11038 assert!(check(&task, &Starter::Start, 9).is_none());
11039 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
11040 task.release_operator("owner", None);
11041 assert_eq!(task.attempts, 0);
11042 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
11043 assert!(check(&restored, &Starter::Start, 8).is_some());
11044 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
11045 assert!(check(&restored, &Starter::Start, 0).is_none());
11046 }
11047
11048 #[test]
11049 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
11050 let mut task = Task::new(
11051 "recover work".into(),
11052 "recover work".into(),
11053 PathBuf::from("/repo"),
11054 crate::queue::Source::Human,
11055 );
11056 let mut run = run_state(RunStatus::Superseded);
11057 run.candidates.clear();
11058 task.runs.push(run.id.clone());
11059 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
11060 run.retired_to = Some("authority".into());
11061 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11062 run.retired_to = None;
11063 run.released_to = Some("authority".into());
11064 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11065 assert!(
11066 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
11067 .unwrap()
11068 .contains("Cannot verify")
11069 );
11070 assert!(
11072 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
11073 panic!("resume must not load historical attempts")
11074 })
11075 .is_none()
11076 );
11077 }
11078
11079 #[test]
11080 fn retired_resume_requires_a_valid_same_repository_authority() {
11081 let mut source = run_state(RunStatus::Blocked);
11082 let mut empty = run_state(RunStatus::Blocked);
11083 empty.retired_to = Some(source.id.clone());
11084 let runs = vec![source.id.clone(), empty.id.clone()];
11085 let resolve = |source: &RunState, empty: &RunState| {
11086 unfinished_run_with(&runs, "fixture", |id| {
11087 if id == source.id {
11088 Ok(source.clone())
11089 } else if id == empty.id {
11090 Ok(empty.clone())
11091 } else {
11092 bail!("unreadable authority")
11093 }
11094 })
11095 };
11096 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
11097 assert_eq!(
11098 choose_starter(None, false, resolve(&source, &empty).as_deref()),
11099 Starter::Resume(source.id.clone())
11100 );
11101 source.retired_to = Some(empty.id.clone());
11102 assert!(
11103 resolve(&source, &empty).is_none(),
11104 "cycles do not revive obsolete runs"
11105 );
11106 source.retired_to = None;
11107 source.repo = PathBuf::from("/another-repository");
11108 assert!(resolve(&source, &empty).is_none());
11109 source.repo = empty.repo.clone();
11110 source.released_to = Some("later".into());
11111 assert!(resolve(&source, &empty).is_none());
11112 empty.retired_to = Some("unlisted-run".into());
11113 assert!(resolve(&source, &empty).is_none());
11114 }
11115
11116 #[test]
11117 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
11118 let mut older_stalled = RunState::new(
11126 PathBuf::from("/repo"),
11127 "main".to_owned(),
11128 "abc1234def".to_owned(),
11129 "add retries".to_owned(),
11130 Config::default(),
11131 );
11132 older_stalled.status = RunStatus::Stalled;
11133
11134 let mut newest_exhausted = RunState::new(
11135 PathBuf::from("/repo"),
11136 "main".to_owned(),
11137 "abc1234def".to_owned(),
11138 "add retries".to_owned(),
11139 Config::default(),
11140 );
11141 newest_exhausted.status = RunStatus::Blocked;
11142 newest_exhausted.config.graph.review_rounds = 1;
11143 newest_exhausted.reviews = vec![review_round(1)];
11144
11145 assert_eq!(
11146 unfinished_run_with(
11147 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
11148 "t",
11149 |_| Ok(newest_exhausted.clone())
11150 ),
11151 None,
11152 "the newest run is exhausted, so nothing here is worth resuming - \
11153 least of all the older, already-superseded run"
11154 );
11155 }
11156
11157 #[test]
11158 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
11159 assert_eq!(
11160 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
11161 Err(anyhow::anyhow!("fixture is absent"))
11162 }),
11163 None
11164 );
11165 }
11166
11167 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
11168 let mut q = ask::Question::new(
11169 run.to_owned(),
11170 "implement".to_owned(),
11171 "impl-A".to_owned(),
11172 "continue?".to_owned(),
11173 String::new(),
11174 vec!["resume で続行する".to_owned(), "other".to_owned()],
11175 );
11176 q.actions.insert("resume で続行する".to_owned(), action);
11177 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
11178 .unwrap();
11179 q
11180 }
11181
11182 fn held_task_with(run: &str) -> Task {
11183 let mut t = task();
11184 t.runs = vec![run.to_owned()];
11185 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
11186 t
11187 }
11188
11189 fn resume_action(run: &str) -> ask::ChoiceAction {
11190 ask::ChoiceAction::Resume { run: run.into() }
11191 }
11192
11193 #[test]
11194 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
11195 for parked in [false, true] {
11196 let mut t = task();
11197 t.start("A".into());
11198 t.start("B".into());
11199 assert_eq!(t.runs, vec!["A", "B"]);
11200 t.start("A".into());
11201 let mut resumed = run_state(RunStatus::Blocked);
11202 resumed.id = "A".into();
11203 resumed.parked = parked;
11204 resumed.driver_exited = true;
11205 t.hold_machine(Some("resumed run stopped".into()));
11206 let q = action_question("A", resume_action("A"));
11207 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
11208 assert_eq!(
11209 decide_action(&t, &q, &PHRASES_EN, |id| {
11210 assert_eq!(id, "A");
11211 Ok(resumed.clone())
11212 }),
11213 ActionDecision::Resume("A".into())
11214 );
11215 assert_eq!(
11216 action_standing(&t, &action_question("B", resume_action("B"))),
11217 ActionStanding::Stale
11218 );
11219 t.release_operator("cli", None);
11220 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
11221 assert_eq!(id, "A");
11222 Ok(resumed.clone())
11223 });
11224 assert_eq!(
11225 choose_starter(None, false, unfinished.as_deref()),
11226 Starter::Resume("A".into())
11227 );
11228 assert_eq!(t.runs, vec!["B", "A"]);
11229 }
11230 }
11231
11232 #[test]
11233 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
11234 let mut state = crate::land::tests::repair_state(true, true);
11235 state.status = RunStatus::Landing;
11236 state.parked = true;
11237 state.driver_exited = true;
11238 let mut q = ask::Question::new(
11239 state.id.clone(),
11240 land::APPROVAL_NODE.into(),
11241 "land".into(),
11242 "Merge?".into(),
11243 String::new(),
11244 vec![land::APPROVE.into(), land::HOLD.into()],
11245 );
11246 q.id = state.id.clone();
11247 state.land_approval = Some(crate::run::LandApproval {
11248 question: q.id.clone(),
11249 head: "current-head".into(),
11250 });
11251 assert!(land::upgrade_repair_choice(&mut q, &state));
11252 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11253 let mut task = held_task_with(&state.id);
11254 task.status = TaskStatus::Parked;
11255 assert_eq!(
11256 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11257 ActionDecision::Resume(state.id.clone())
11258 );
11259 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
11260 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
11261 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
11262 assert_eq!(
11263 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11264 ActionDecision::Resume(state.id.clone()),
11265 "a repair answer newer than the manual hold resumes the existing owner"
11266 );
11267 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
11268 for answered_at in [q.answered_at, None] {
11269 let mut older = q.clone();
11270 older.answered_at = answered_at;
11271 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
11272 panic!("a later manual hold must prevent dispatch")
11273 });
11274 let ActionDecision::Refuse(reason) = refused else {
11275 panic!("expected an actionable refusal, got {refused:?}");
11276 };
11277 assert!(reason.contains("later operator decision"));
11278 assert!(reason.contains(&format!("magi task release {}", task.id)));
11279 }
11280 task.operator_decided_at = None;
11281 let mut superseded = state.clone();
11282 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
11283 assert_eq!(
11284 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
11285 ActionDecision::Stale
11286 );
11287 for case in 1..4 {
11288 let mut changed = state.clone();
11289 match case {
11290 1 => changed.status = RunStatus::Blocked,
11291 2 => changed.released_to = Some("new-owner".into()),
11292 _ => changed.land_approval.as_mut().unwrap().head.clear(),
11293 }
11294 assert!(matches!(
11295 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
11296 ActionDecision::Refuse(_)
11297 ));
11298 }
11299 assert!(matches!(
11300 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
11301 ActionDecision::Refuse(_)
11302 ));
11303 task.runs.push("replacement".into());
11304 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
11305 assert_eq!(
11306 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
11307 "old owner must not load"
11308 )),
11309 ActionDecision::Stale
11310 );
11311 }
11312
11313 #[test]
11314 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
11315 let mut state = crate::land::tests::repair_state(true, true);
11316 state.status = RunStatus::Landing;
11317 state.parked = true;
11318 state.driver_exited = true;
11319 let mut q = ask::Question::new(
11320 state.id.clone(),
11321 land::APPROVAL_NODE.into(),
11322 "land".into(),
11323 "Merge?".into(),
11324 String::new(),
11325 vec![land::APPROVE.into(), land::HOLD.into()],
11326 );
11327 q.id = state.id.clone();
11328 state.land_approval = Some(crate::run::LandApproval {
11329 question: q.id.clone(),
11330 head: "current-head".into(),
11331 });
11332 state.save().unwrap();
11333 land::upgrade_repair_choice(&mut q, &state);
11334 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11335 Questions::open().put(&mut q).unwrap();
11336 let tmp = tempfile::tempdir().unwrap();
11337 let queue = Queue::at(tmp.path().join("queue"));
11338 let mut task = held_task_with(&state.id);
11339 task.status = TaskStatus::Parked;
11340 queue.put(&mut task).unwrap();
11341 let empty = Arc::new(tokio::sync::Semaphore::new(0));
11342 let free = Arc::new(tokio::sync::Semaphore::new(1));
11343 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
11344 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
11345 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
11346 assert!(claim.permit.is_some());
11347 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
11348 drop(claim);
11349 Questions::open()
11350 .update(&q.id, |q| {
11351 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
11352 Ok(())
11353 })
11354 .unwrap();
11355 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
11356 assert!(claim.permit.is_none());
11357 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
11358 }
11359
11360 #[test]
11361 fn decide_action_resumes_only_the_latest_resumable_run() {
11362 let t = held_task_with("r1");
11363 let q = action_question("r1", resume_action("r1"));
11364 let load = |s: RunState| move |_: &str| Ok(s);
11365 assert_eq!(
11366 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
11367 ActionDecision::Resume("r1".into())
11368 );
11369 let q_other = action_question("r1", resume_action("r0"));
11371 assert!(matches!(
11372 decide_action(
11373 &t,
11374 &q_other,
11375 &PHRASES_EN,
11376 load(run_state(RunStatus::Blocked))
11377 ),
11378 ActionDecision::Refuse(_)
11379 ));
11380 assert!(matches!(
11382 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
11383 ActionDecision::Refuse(_)
11384 ));
11385 let mut released = run_state(RunStatus::Blocked);
11387 released.released_to = Some("elsewhere".into());
11388 assert!(matches!(
11389 decide_action(&t, &q, &PHRASES_EN, load(released)),
11390 ActionDecision::Refuse(_)
11391 ));
11392 assert!(matches!(
11394 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
11395 ActionDecision::Refuse(_)
11396 ));
11397 }
11398
11399 #[test]
11400 fn decide_action_ignores_a_question_about_an_earlier_run() {
11401 let mut t = held_task_with("r1");
11402 t.runs.push("r2".to_owned());
11403 let never = |_: &str| -> Result<RunState> { bail!("not read") };
11404 assert_eq!(
11405 decide_action(
11406 &t,
11407 &action_question("r1", ask::ChoiceAction::Done),
11408 &PHRASES_EN,
11409 never
11410 ),
11411 ActionDecision::Stale
11412 );
11413 }
11414
11415 #[test]
11416 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
11417 let mut t = held_task_with("r1");
11418 let never = |_: &str| -> Result<RunState> { bail!("not read") };
11419 assert_eq!(
11420 decide_action(
11421 &t,
11422 &action_question("r1", ask::ChoiceAction::Requeue),
11423 &PHRASES_EN,
11424 never
11425 ),
11426 ActionDecision::Requeue
11427 );
11428 let done_q = action_question("r1", ask::ChoiceAction::Done);
11429 assert_eq!(
11430 decide_action(&t, &done_q, &PHRASES_EN, never),
11431 ActionDecision::Done
11432 );
11433 t.mark_action_applied(&done_q.id);
11434 assert_eq!(
11435 decide_action(&t, &done_q, &PHRASES_EN, never),
11436 ActionDecision::Skip
11437 );
11438
11439 let mut plain = action_question("r1", ask::ChoiceAction::Done);
11441 plain.actions.clear();
11442 assert_eq!(
11443 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
11444 ActionDecision::Skip
11445 );
11446 let mut running = held_task_with("r1");
11448 running.status = TaskStatus::Running;
11449 assert_eq!(
11450 decide_action(
11451 &running,
11452 &action_question("r1", ask::ChoiceAction::Done),
11453 &PHRASES_EN,
11454 never
11455 ),
11456 ActionDecision::Skip
11457 );
11458 }
11459
11460 #[test]
11461 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
11462 crate::run::pin_test_home();
11463 for legacy in [false, true] {
11464 let dir = tempfile::tempdir().unwrap();
11465 let queue = Queue::at(dir.path().join("queue"));
11466 let questions = Questions::at(dir.path().join("questions"));
11467 let run = "20260101-000000-hold";
11468 let mut task = held_task_with(run);
11469 task.status = TaskStatus::Parked;
11470 let mut q = ask::Question::new(
11471 run.into(),
11472 land::APPROVAL_NODE.into(),
11473 "land".into(),
11474 "Repair?".into(),
11475 String::new(),
11476 vec![land::REPAIR.into()],
11477 );
11478 q.actions.insert(land::REPAIR.into(), resume_action(run));
11479 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11480 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
11482 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
11483 questions.put(&mut q).unwrap();
11484 task.hold_manual(Some("Keep this PR paused".into()));
11485 if legacy {
11486 task.hold_source = None;
11487 }
11488 queue.put(&mut task).unwrap();
11489
11490 apply_task_choice_actions(&queue, &questions, dir.path());
11491 let after = queue.get(&task.id).unwrap();
11492 assert!(after.operator_held());
11493 assert_eq!(after.hold_source, task.hold_source);
11494 assert_eq!(after.operator_decided_at, task.operator_decided_at);
11495 assert_eq!(after.runs, task.runs);
11496 assert_eq!(after.instruction, task.instruction);
11497 assert!(after.resume_override.is_none());
11498 assert!(after.action_applied(&q.id));
11499 let reason = after.hold_reason.as_deref().unwrap();
11500 assert!(reason.starts_with("Keep this PR paused"));
11501 assert!(reason.contains("later operator decision"));
11502 assert!(reason.contains(&format!("magi task release {}", task.id)));
11503
11504 for recovery in [
11505 crate::conduct::Recovery::Review,
11506 crate::conduct::Recovery::Requeue,
11507 ] {
11508 crate::conduct::apply(
11509 &queue,
11510 &questions,
11511 &crate::conduct::Verdict {
11512 decisions: vec![crate::conduct::Decision {
11513 id: task.id.clone(),
11514 recovery: Some(recovery),
11515 ..Default::default()
11516 }],
11517 },
11518 )
11519 .unwrap();
11520 }
11521 apply_task_choice_actions(&queue, &questions, dir.path());
11524 let still_held = queue.get(&task.id).unwrap();
11525 assert!(still_held.operator_held());
11526 assert_eq!(still_held.hold_reason, after.hold_reason);
11527 assert_eq!(still_held.runs, task.runs);
11528 assert!(still_held.review_branch.is_none());
11529 assert!(!still_held.fresh_start);
11530 assert!(queue.next_runnable().is_none());
11531 }
11532 }
11533
11534 #[test]
11535 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
11536 for held in [false, true] {
11537 for retry in [false, true] {
11538 let dir = tempfile::tempdir().unwrap();
11539 let queue = Queue::at(dir.path().join("queue"));
11540 let questions = Questions::at(dir.path().join("questions"));
11541 let home = dir.path().join("home");
11542 let mut t = task();
11543 if held {
11544 t.hold_machine(Some("driver state unknown".into()));
11545 }
11546 queue.put(&mut t).unwrap();
11547 let verdict = serde_json::from_value(serde_json::json!({
11548 "decisions": [{
11549 "id": t.id,
11550 "question": "Driver state unknown; retry the checks or hold?",
11551 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
11552 "actions": {
11553 "daemon: recheck and retry": {"do": "requeue"},
11554 "daemon: hold the task": {"do": "hold"}
11555 }
11556 }]
11557 }))
11558 .unwrap();
11559 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
11560 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
11561 let mut q = questions.list().pop().unwrap();
11562 let choice = if retry {
11563 "daemon: recheck and retry"
11564 } else {
11565 "daemon: hold the task"
11566 };
11567 assert_eq!(q.actions.len(), 2);
11568 q.answer(ask::Answer::Choice(choice.into())).unwrap();
11569 questions.put(&mut q).unwrap();
11570
11571 questions.beat(&q.id, ask::WaiterKind::Deputy);
11573 for _ in 0..2 {
11574 resolve_blockers(&queue, &questions);
11575 apply_task_choice_actions(&queue, &questions, &home);
11576 let waiting = queue.get(&t.id).unwrap();
11577 assert_eq!(waiting.status, TaskStatus::Blocked);
11578 assert_eq!(waiting.blocked_by, [q.id.clone()]);
11579 assert!(waiting.answers.is_empty());
11580 assert!(!waiting.action_applied(&q.id));
11581 assert!(runnable(&queue).is_empty());
11582 let free = Arc::new(tokio::sync::Semaphore::new(1));
11583 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
11584 }
11585 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
11586
11587 resolve_blockers(&queue, &questions);
11589 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
11590 apply_task_choice_actions(&queue, &questions, &home);
11591 let after = queue.get(&t.id).unwrap();
11592 assert_eq!(
11593 after.status,
11594 if retry {
11595 TaskStatus::Queued
11596 } else {
11597 TaskStatus::Held
11598 }
11599 );
11600 assert!(after.action_applied(&q.id));
11601 assert_eq!(after.operator_held(), !retry);
11602 assert_eq!(after.answers.len(), 1);
11603
11604 let mut changed = after;
11606 if retry {
11607 changed.hold_manual(Some("later hold".into()));
11608 } else {
11609 changed.release_operator("test", None);
11610 }
11611 let expected = changed.status;
11612 queue.put(&mut changed).unwrap();
11613 apply_task_choice_actions(&queue, &questions, &home);
11614 assert_eq!(queue.get(&t.id).unwrap().status, expected);
11615 }
11616 }
11617 }
11618
11619 #[test]
11620 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
11621 let dir = tempfile::tempdir().unwrap();
11622 let queue = Queue::at(dir.path().join("queue"));
11623 let questions = Questions::at(dir.path().join("questions"));
11624 let home = dir.path().join("home");
11625 let mut state = run_state(RunStatus::Blocked);
11626 state.id = "20260101-000000-act1".to_owned();
11627 state.save_under(&home).unwrap();
11628
11629 let mut t = held_task_with(&state.id);
11630 queue.put(&mut t).unwrap();
11631 let mut q = action_question(&state.id, resume_action(&state.id));
11632 questions.put(&mut q).unwrap();
11633
11634 apply_task_choice_actions(&queue, &questions, &home);
11635 let after = queue.get(&t.id).unwrap();
11636 assert_eq!(after.status, TaskStatus::Queued);
11637 assert!(!after.fresh_start);
11638 assert!(after.action_applied(&q.id));
11639 let pin = after.resume_override.clone().unwrap();
11640 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
11641 assert!(pin.forced);
11642
11643 let mut again = queue.get(&t.id).unwrap();
11645 again.hold_machine(Some("later".into()));
11646 queue.put(&mut again).unwrap();
11647 apply_task_choice_actions(&queue, &questions, &home);
11648 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11649 }
11650
11651 #[test]
11652 fn a_delivered_answer_is_still_acted_on_exactly_once() {
11653 let dir = tempfile::tempdir().unwrap();
11654 let queue = Queue::at(dir.path().join("queue"));
11655 let questions = Questions::at(dir.path().join("questions"));
11656 let home = dir.path().join("home");
11657 let mut state = run_state(RunStatus::Blocked);
11658 state.id = "20260101-000000-act2".to_owned();
11659 state.save_under(&home).unwrap();
11660
11661 let mut t = held_task_with(&state.id);
11662 queue.put(&mut t).unwrap();
11663 let mut q = action_question(&state.id, resume_action(&state.id));
11664 q.answer_delivered = true;
11665 questions.put(&mut q).unwrap();
11666
11667 apply_task_choice_actions(&queue, &questions, &home);
11670 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
11671 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
11672
11673 let mut again = queue.get(&t.id).unwrap();
11675 again.hold_machine(Some("later".into()));
11676 queue.put(&mut again).unwrap();
11677 apply_task_choice_actions(&queue, &questions, &home);
11678 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11679 }
11680
11681 #[test]
11682 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
11683 let dir = tempfile::tempdir().unwrap();
11684 let queue = Queue::at(dir.path().join("queue"));
11685 let questions = Questions::at(dir.path().join("questions"));
11686 let home = dir.path().join("home");
11687 let mut state = run_state(RunStatus::Blocked);
11688 state.id = "20260101-000000-act3".to_owned();
11689 state.save_under(&home).unwrap();
11690 let mut t = held_task_with(&state.id);
11691 queue.put(&mut t).unwrap();
11692 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
11693 questions.put(&mut q).unwrap();
11694
11695 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
11696 apply_task_choice_actions(&queue, &questions, &home);
11697 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11698
11699 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
11700 apply_task_choice_actions(&queue, &questions, &home);
11701 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
11702 }
11703
11704 #[test]
11705 fn action_standing_tells_the_reasons_apart() {
11706 let mut t = held_task_with("r1");
11707 let q = action_question("r1", ask::ChoiceAction::Requeue);
11708 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
11709 let mut plain = q.clone();
11710 plain.actions.clear();
11711 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
11712
11713 t.mark_action_applied(&q.id);
11715 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
11716
11717 let mut t = held_task_with("r1");
11719 t.start("r2".to_owned());
11720 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
11721 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
11723 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
11724 t.status = TaskStatus::Blocked;
11725 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
11726
11727 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
11729 c.node = crate::conduct::NODE.to_owned();
11730 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
11731 assert!(!ActionStanding::Busy.daemon_owns());
11732 }
11733
11734 #[test]
11735 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
11736 let dir = tempfile::tempdir().unwrap();
11737 let queue = Queue::at(dir.path().join("queue"));
11738 let questions = Questions::at(dir.path().join("questions"));
11739 let home = dir.path().join("home");
11740 let mut state = run_state(RunStatus::Blocked);
11741 state.id = "20260101-000000-act4".to_owned();
11742 state.save_under(&home).unwrap();
11743 let mut t = held_task_with(&state.id);
11744 t.status = TaskStatus::Running;
11745 queue.put(&mut t).unwrap();
11746 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
11747 q.cwd = Some(dir.path().display().to_string());
11748 questions.put(&mut q).unwrap();
11749
11750 let running = queue.get(&t.id).unwrap();
11753 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
11754 assert!(matches!(
11755 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
11756 crate::waiter::Action::Deliver(_)
11757 ));
11758 apply_task_choice_actions(&queue, &questions, &home);
11760 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
11761 let mut back = queue.get(&t.id).unwrap();
11762 back.hold_machine(Some("later".into()));
11763 queue.put(&mut back).unwrap();
11764 apply_task_choice_actions(&queue, &questions, &home);
11765 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
11766 let mut again = queue.get(&t.id).unwrap();
11767 again.hold_machine(Some("again".into()));
11768 queue.put(&mut again).unwrap();
11769 apply_task_choice_actions(&queue, &questions, &home);
11770 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11771 }
11772}