1use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::sync::atomic::{AtomicBool, Ordering};
52use std::sync::{Mutex, MutexGuard};
53use std::time::Duration;
54
55use anyhow::{Context, Result, bail};
56use jiff::Timestamp;
57use serde::{Deserialize, Serialize};
58use tokio::sync::Notify;
59
60use crate::ask::{self, Questions};
61use crate::clean;
62use crate::conduct::Conductor;
63use crate::config::{Config, MergeMode};
64use crate::graph::Runner;
65use crate::land;
66use crate::notices::{self, Link, Notice};
67use crate::queue::{Queue, Task, TaskStatus};
68use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
69use crate::triage;
70
71pub const SCHEMA: u32 = 1;
73
74pub const HEARTBEAT: Duration = Duration::from_secs(5);
78
79pub const STALE_SECS: i64 = 30;
88
89pub const POLL: Duration = Duration::from_secs(5);
91
92pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
96
97pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
119#[serde(default)]
120pub struct Current {
121 pub task: String,
123 pub run: String,
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct Status {
135 pub schema: u32,
137 pub pid: u32,
139 pub started_at: Timestamp,
141 pub updated_at: Timestamp,
143 pub idle: bool,
145 pub current: Vec<Current>,
151 pub completed: usize,
153 pub polls: u64,
155 #[serde(default)]
157 pub question_recovery: Option<crate::question_delivery::Recovery>,
158}
159
160impl Status {
161 #[must_use]
163 pub fn new() -> Self {
164 let now = Timestamp::now();
165 Self {
166 schema: SCHEMA,
167 pid: std::process::id(),
168 started_at: now,
169 updated_at: now,
170 idle: true,
171 current: Vec::new(),
172 completed: 0,
173 polls: 0,
174 question_recovery: None,
175 }
176 }
177}
178
179impl Default for Status {
180 fn default() -> Self {
181 Self::new()
182 }
183}
184
185#[derive(Debug, Clone)]
187pub struct Opts {
188 pub repo: PathBuf,
190 pub config: Option<PathBuf>,
192 pub poll: Duration,
194 pub max_attempts: usize,
196 pub once: bool,
198 pub merge: Option<String>,
200 pub worktrees_root: Option<PathBuf>,
209}
210
211impl Default for Opts {
212 fn default() -> Self {
213 Self {
214 repo: PathBuf::from("."),
215 config: None,
216 poll: POLL,
217 max_attempts: 2,
218 once: false,
219 merge: None,
220 worktrees_root: None,
221 }
222 }
223}
224
225fn max_concurrent(n: usize) -> usize {
230 n.max(1)
231}
232
233#[must_use]
235pub fn status_path() -> PathBuf {
236 crate::run::home().join("daemon.json")
237}
238
239pub fn write_status(status: &Status) -> Result<()> {
241 write_status_to(&status_path(), status)
242}
243
244pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
249 if let Some(parent) = path.parent() {
250 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
251 }
252 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
253 let tmp = path.with_extension("json.tmp");
254 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
255 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
256 Ok(())
257}
258
259pub fn clear_status() {
262 clear_status_at(&status_path());
263}
264
265fn clear_status_at(path: &Path) {
269 let _ = std::fs::remove_file(path);
270}
271
272#[derive(Debug, Clone, Default)]
285pub struct Stop {
286 stopped: Arc<AtomicBool>,
290 busy: Arc<std::sync::atomic::AtomicUsize>,
296 wake: Arc<Notify>,
300 pause: crate::graph::Pause,
303}
304
305impl Stop {
306 #[must_use]
308 pub fn new() -> Self {
309 Self::default()
310 }
311
312 pub fn stop(&self) {
315 self.stopped.store(true, Ordering::SeqCst);
316 self.wake.notify_one();
320 }
321
322 #[must_use]
324 pub fn stopped(&self) -> bool {
325 self.stopped.load(Ordering::SeqCst)
326 }
327
328 #[must_use]
336 pub fn finishing(&self) -> bool {
337 self.stopped() && self.busy_now()
338 }
339
340 pub fn park(&self, cause: crate::run::ParkCause) {
351 self.pause.park_for(cause);
352 self.stop();
353 }
354
355 #[must_use]
357 pub fn parking(&self) -> bool {
358 self.pause.parked()
359 }
360
361 #[must_use]
363 pub fn pause(&self) -> crate::graph::Pause {
364 self.pause.clone()
365 }
366
367 #[must_use]
373 pub fn busy_now(&self) -> bool {
374 self.busy.load(Ordering::SeqCst) > 0
375 }
376
377 fn enter(&self) {
379 self.busy.fetch_add(1, Ordering::SeqCst);
380 }
381
382 fn exit(&self) {
385 self.busy.fetch_sub(1, Ordering::SeqCst);
386 }
387
388 async fn idle(&self, poll: Duration) {
390 tokio::select! {
391 () = tokio::time::sleep(poll) => {}
392 () = self.wake.notified() => {}
393 }
394 }
395}
396
397#[derive(Debug, Clone, Default, Deserialize)]
404#[serde(default)]
405pub struct Reading {
406 pub schema: u32,
408 pub pid: Option<u32>,
410 pub started_at: Option<Timestamp>,
412 pub updated_at: Option<Timestamp>,
414 pub idle: bool,
416 #[serde(deserialize_with = "de_current")]
428 pub current: Vec<Current>,
429 pub completed: u64,
431 pub polls: u64,
433 #[serde(default)]
435 pub question_recovery: Option<crate::question_delivery::Recovery>,
436}
437
438fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
441where
442 D: serde::Deserializer<'de>,
443{
444 #[derive(Deserialize)]
445 #[serde(untagged)]
446 enum Shape {
447 Many(Vec<Current>),
448 One(Current),
449 }
450 Ok(
451 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
452 Shape::Many(v) => v,
453 Shape::One(c) => vec![c],
454 }),
455 )
456}
457
458impl Reading {
459 #[must_use]
462 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
463 self.updated_at
464 .map(|at| (now.as_second() - at.as_second()).max(0))
465 }
466
467 #[must_use]
471 pub fn running(&self, now: Timestamp) -> bool {
472 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
473 }
474}
475
476#[must_use]
483pub fn read_status(home: &Path) -> Option<Reading> {
484 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
485 serde_json::from_str(&body).ok()
486}
487
488#[must_use]
500pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
501 read_status(home)
502 .filter(|reading| reading.running(now))
503 .map(|reading| reading.current)
504 .unwrap_or_default()
505}
506
507#[must_use]
509pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
510 current_work(home, now).iter().any(|c| c.run == run)
511}
512
513#[must_use]
523pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
524 current_work(home, now)
525 .iter()
526 .any(|c| crate::run::short_of(&c.run) == short)
527}
528
529#[must_use]
531pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
532 current_work(home, now).iter().any(|c| c.task == task)
533}
534
535pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
579 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
580}
581
582fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
586where
587 F: Fn(u32) -> bool,
588{
589 let this_process = std::process::id();
590 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
591 .into_iter()
592 .flatten()
593 .flatten()
594 .map(|e| e.path())
595 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
596 .filter(|p| {
597 match std::fs::read_to_string(p)
598 .ok()
599 .and_then(|body| body.trim().parse::<u32>().ok())
600 {
601 Some(pid) if pid == this_process => false,
605 Some(pid) => !pid_alive(pid),
606 None => p
607 .metadata()
608 .and_then(|m| m.modified())
609 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
610 .is_ok_and(|age| age >= older_than),
611 }
612 })
613 .filter(|p| std::fs::remove_file(p).is_ok())
614 .filter_map(|p| {
615 p.file_stem()
616 .and_then(|s| s.to_str())
617 .map(std::borrow::ToOwned::to_owned)
618 })
619 .collect();
620 swept.sort_unstable();
621 swept
622}
623
624fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
629 task.status == TaskStatus::Running
630 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
631 && !is_working_on_task(home, &task.id, now)
632}
633
634fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
637 queue
638 .list()
639 .into_iter()
640 .filter(|t| is_stalled(t, home, now))
641 .collect()
642}
643
644fn queued_tasks(queue: &Queue) -> Vec<Task> {
650 queue
651 .list()
652 .into_iter()
653 .filter(|t| t.status == TaskStatus::Queued)
654 .collect()
655}
656
657fn finished_tasks(queue: &Queue) -> Vec<Task> {
660 queue
661 .list()
662 .into_iter()
663 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
664 .collect()
665}
666
667fn resolve_blockers(queue: &Queue, questions: &Questions) {
684 for listed in queue.list() {
685 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
686 continue;
687 }
688 let Ok(_claim) = queue.claim(&listed.id) else {
689 continue;
690 };
691 let Ok(mut task) = queue.get(&listed.id) else {
692 continue;
693 };
694 if task.status != TaskStatus::Blocked {
695 continue;
696 }
697 let deleted = queue.apply_deleted_blockers(&mut task, questions);
701 if !deleted.is_empty() {
702 record(queue, &mut task);
703 for id in &deleted {
704 queue.note_dependency_deleted(&task, id);
705 }
706 if task.status != TaskStatus::Blocked {
707 continue;
708 }
709 }
710 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
711 if !missing.is_empty() {
712 let language = language_of(&task, Path::new("."));
713 let mut cause = missing.clone();
714 cause.sort();
715 cause.dedup();
716 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
717 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
718 &task.blocked_by,
719 &missing,
720 &language,
721 )));
722 record(queue, &mut task);
723 continue;
724 }
725 if let Some(q) = task.blocked_by.iter().find_map(|id| {
728 let crate::queue::Dependency::Question(full) =
729 crate::queue::dependency(queue, questions, id)
730 else {
731 return None;
732 };
733 questions.get(&full).ok().filter(|q| {
734 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
735 })
736 }) {
737 let language = language_of(&task, Path::new("."));
738 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
739 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
740 record(queue, &mut task);
741 continue;
742 }
743 let mut changed = false;
744 for id in task.blocked_by.clone() {
745 let resolved = crate::queue::dependency(queue, questions, &id);
746 if let crate::queue::Dependency::Task(full) = &resolved {
747 if queue
748 .get(full)
749 .is_ok_and(|dep| dep.status == TaskStatus::Done)
750 {
751 task.unblock(&id);
752 changed = true;
753 }
754 continue;
755 }
756 if let crate::queue::Dependency::Question(full) = resolved
757 && let Ok(q) = questions.get(&full)
758 && q.status == ask::QuestionStatus::Answered
759 {
760 if q.node == crate::conduct::NODE
764 && q.chosen_action().is_some()
765 && !task.action_applied(&q.id)
766 {
767 continue;
768 }
769 let answer = match &q.answer {
770 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
771 None => String::new(),
772 };
773 task.record_answer(q.summary.clone(), answer);
774 task.unblock(&id);
775 changed = true;
776 }
777 }
778 if changed {
779 record(queue, &mut task);
780 }
781 }
782}
783
784fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
786 if crate::lang::is_japanese(language) {
787 format!(
788 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
789 q.short(),
790 q.summary
791 )
792 } else {
793 format!(
794 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
795 q.short(),
796 q.summary
797 )
798 }
799}
800
801#[derive(Debug, Clone, Copy, PartialEq, Eq)]
804pub(crate) enum ActionStanding {
805 NoAction,
807 Pending,
809 Applied,
812 Stale,
814 Busy,
816}
817
818impl ActionStanding {
819 pub(crate) fn daemon_owns(self) -> bool {
821 matches!(self, Self::Pending | Self::Applied | Self::Stale)
822 }
823}
824
825pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
829 if q.chosen_action().is_none() {
830 ActionStanding::NoAction
831 } else if task.action_applied(&q.id) {
832 ActionStanding::Applied
833 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
834 ActionStanding::Stale
835 } else if matches!(
836 task.status,
837 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
838 ) {
839 ActionStanding::Busy
840 } else {
841 ActionStanding::Pending
842 }
843}
844
845#[derive(Debug, Clone, PartialEq, Eq)]
847enum ActionDecision {
848 Skip,
850 Resume(String),
852 Requeue,
854 Hold,
856 Done,
858 Stale,
861 Refuse(String),
864}
865
866fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
874where
875 F: FnOnce(&str) -> Result<RunState>,
876{
877 let Some(action) = q.chosen_action() else {
878 return ActionDecision::Skip;
879 };
880 if task.action_applied(&q.id)
881 || matches!(
882 task.status,
883 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
884 )
885 {
886 return ActionDecision::Skip;
887 }
888 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
891 return ActionDecision::Stale;
892 }
893 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
894 let stopping = q.node == land::APPROVAL_NODE
895 && q.resolution()
896 .as_deref()
897 .is_some_and(crate::merge_decision::stops);
898 let decision_at = if repair || stopping {
902 q.answered_at.unwrap_or(q.asked_at)
903 } else {
904 q.asked_at
905 };
906 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
907 return if repair || stopping {
908 ActionDecision::Refuse(format!(
909 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
910 if repair { "repair" } else { "stop" },
911 q.id,
912 q.run,
913 task.id
914 ))
915 } else {
916 ActionDecision::Stale
917 };
918 }
919 match action {
920 a if a.maintenance() => ActionDecision::Skip,
921 ask::ChoiceAction::ResumeRun { .. }
922 | ask::ChoiceAction::ReleaseTask { .. }
923 | ask::ChoiceAction::RerunChecks { .. }
924 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
925 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
926 ask::ChoiceAction::Hold => ActionDecision::Hold,
927 ask::ChoiceAction::Done => ActionDecision::Done,
928 ask::ChoiceAction::Resume { run } => {
929 if task.runs.last() != Some(run) {
930 return ActionDecision::Refuse((p.resume_not_latest)(
931 q.short(),
932 ask::short_id(run),
933 ));
934 }
935 match load(run) {
936 Ok(s) if stopping => {
937 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
938 || !s
939 .land_approval
940 .as_ref()
941 .is_some_and(|binding| binding.question == q.id)
942 {
943 ActionDecision::Stale
944 } else if s.liveness(false) == Liveness::Live {
945 ActionDecision::Skip
946 } else if s.status == RunStatus::Landing
947 && !s.released()
948 && s.retired_to.is_none()
949 {
950 ActionDecision::Resume(run.clone())
951 } else {
952 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
953 }
954 }
955 Ok(s)
956 if q.node == land::APPROVAL_NODE
957 && q.resolution().as_deref() == Some(land::REPAIR) =>
958 {
959 if s.merge_repairs.iter().any(|r| r.question == q.id)
960 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
961 {
962 ActionDecision::Stale
965 } else if s.liveness(false) == Liveness::Live {
966 ActionDecision::Skip
967 } else if s.status != RunStatus::Landing
968 || s.released()
969 || s.reviews.is_empty()
970 || !implementers_match(task, &s)
971 || !s
972 .land_approval
973 .as_ref()
974 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
975 {
976 ActionDecision::Refuse(format!(
977 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
978 q.id, run, run
979 ))
980 } else {
981 ActionDecision::Resume(run.clone())
982 }
983 }
984 Ok(s) if s.liveness(false) == Liveness::Live => {
985 ActionDecision::Refuse("run already has a live driver".into())
986 }
987 Ok(s) if crate::handover::can_resume(&s) => ActionDecision::Resume(run.clone()),
988 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
989 q.short(),
990 ask::short_id(run),
991 )),
992 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
993 q.short(),
994 ask::short_id(run),
995 &format!("{e:#}"),
996 )),
997 }
998 }
999 }
1000}
1001
1002struct Phrases {
1016 graph_stopped: fn(&str, &str) -> String,
1018 quorum_lost: &'static str,
1019 quota_took_out: &'static str,
1021 run_ended: &'static str,
1023 waiting_for_answer: &'static str,
1025 recovered_running: &'static str,
1026 no_run_to_recover: &'static str,
1027 could_not_start: &'static str,
1028 resume_not_latest: fn(&str, &str) -> String,
1029 resume_cannot_progress: fn(&str, &str) -> String,
1030 resume_unreadable: fn(&str, &str, &str) -> String,
1031 handover_refused: fn(&crate::handover::Refused) -> String,
1034 handover_hint: &'static str,
1036 reviewers_never_answered: fn(usize, usize) -> String,
1039}
1040
1041const PHRASES_EN: Phrases = Phrases {
1042 graph_stopped: |status, detail| {
1043 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1044 },
1045 quorum_lost: "the judging panel lost its quorum",
1046 quota_took_out: "; quota took out ",
1047 run_ended: "run ended ",
1048 waiting_for_answer: " - waiting for operator answer to question ",
1049 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1050 no_run_to_recover: "task was `running` with no live daemon and no readable \
1051 run to recover; held for a human to check what happened",
1052 could_not_start: "could not start the run: ",
1053 resume_not_latest: |q, run| {
1054 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1055 },
1056 resume_cannot_progress: |q, run| {
1057 format!("question {q} asked to resume run {run}, which cannot make progress")
1058 },
1059 resume_unreadable: |q, run, e| {
1060 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1061 },
1062 handover_refused: |r| r.to_string(),
1063 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1064 reviewers_never_answered: |missing, rounds| {
1065 format!(
1066 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1067 refusing to call it clean"
1068 )
1069 },
1070};
1071
1072const PHRASES_JA: Phrases = Phrases {
1073 graph_stopped: |status, detail| {
1074 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1075 },
1076 quorum_lost: "審査パネルが定足数を失いました",
1077 quota_took_out: "。クォータで脱落: ",
1078 run_ended: "run 終了: ",
1079 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1080 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1081 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1082 何が起きたか人が確認するため保留にしました",
1083 could_not_start: "run を開始できませんでした: ",
1084 resume_not_latest: |q, run| {
1085 format!(
1086 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1087 )
1088 },
1089 resume_cannot_progress: |q, run| {
1090 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1091 },
1092 resume_unreadable: |q, run, e| {
1093 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1094 },
1095 handover_refused: |r| {
1096 use crate::handover::Refused;
1097 match r {
1098 Refused::Foreign { branch, path, why } => format!(
1099 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1100 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1101 から、やり直してください(詳細: {why})"
1102 ),
1103 Refused::Unsafe { branch, path, why } => format!(
1104 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1105 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1106 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1107 ),
1108 Refused::ReleaseFailed { branch, path, run } => format!(
1109 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1110 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1111 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1112 ),
1113 }
1114 },
1115 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1116 reviewers_never_answered: |missing, rounds| {
1117 format!(
1118 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1119 クリーンとは認めません"
1120 )
1121 },
1122};
1123
1124fn phrases(language: &str) -> &'static Phrases {
1125 if crate::lang::is_japanese(language) {
1126 &PHRASES_JA
1127 } else {
1128 &PHRASES_EN
1129 }
1130}
1131
1132fn language_of(task: &Task, fallback: &Path) -> String {
1136 crate::lang::of_repo(&repo_for(task, fallback))
1137}
1138
1139pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1143 if q.node == crate::conduct::NODE {
1144 return tasks.iter().find(|t| t.id == q.run);
1145 }
1146 tasks.iter().find(|t| t.runs.contains(&q.run))
1147}
1148
1149fn repair_choice_offers(questions: &Questions) {
1152 for q in questions.list().into_iter().filter(|q| {
1153 q.status.open()
1154 && q.node == crate::conduct::NODE
1155 && q.choices
1156 .iter()
1157 .any(|c| crate::choice_effect::describe(q, c).is_none())
1158 }) {
1159 let _ = questions.update(&q.id, |current| {
1160 if current.status.open() {
1161 crate::choice_effect::executable_conductor(current);
1162 if let Some(deputy) = &mut current.deputy {
1163 deputy.brief = crate::deputy::brief(
1164 ¤t.run,
1165 ¤t.detail,
1166 ¤t.choices,
1167 ¤t.actions,
1168 );
1169 }
1170 }
1171 Ok(())
1172 });
1173 }
1174}
1175
1176pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1179 apply_maintenance_choices(queue, questions, home).await;
1180 apply_task_choice_actions(queue, questions, home);
1181}
1182
1183async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1186 for q in questions.list() {
1187 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1188 continue;
1189 };
1190 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1193 continue;
1194 }
1195 let dir = home.join("choice-actions");
1196 if std::fs::create_dir_all(&dir).is_err() {
1197 continue;
1198 }
1199 let Ok(lock) = std::fs::OpenOptions::new()
1200 .create(true)
1201 .truncate(false)
1202 .write(true)
1203 .open(dir.join(format!("{}.lock", q.id)))
1204 else {
1205 continue;
1206 };
1207 if crate::pr_head::try_lock_file(&lock).is_err() {
1208 continue;
1209 }
1210 let fresh = !home
1211 .join("choice-actions")
1212 .join(format!("{}.json", q.id))
1213 .exists();
1214 let result = async {
1215 let receipt = dir.join(format!("{}.json", q.id));
1216 if receipt.exists() {
1217 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1218 anyhow::ensure!(
1219 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1220 "previous maintenance attempt did not confirm completion; inspect its receipt"
1221 );
1222 return Ok::<_, anyhow::Error>(());
1223 }
1224 let record = |status: &str| -> anyhow::Result<()> {
1227 let tmp = receipt.with_extension("tmp");
1228 std::fs::write(
1229 &tmp,
1230 serde_json::to_vec(&serde_json::json!({
1231 "action": action, "status": status,
1232 }))?,
1233 )?;
1234 std::fs::rename(tmp, &receipt)?;
1235 Ok(())
1236 };
1237 let applied = async {
1238 match action {
1239 ask::ChoiceAction::ResumeRun { run } => {
1240 crate::resume::validate(home, run)?;
1241 record("started")?;
1242 if !crate::resume::pin_queued(queue, run, Some(q.asked_at), Some(&q.id))? {
1243 crate::resume::validate_unqueued(home, run)?;
1244 crate::resume::start(run.clone());
1245 }
1246 }
1247 ask::ChoiceAction::ReleaseTask {
1248 task,
1249 held_at,
1250 reason,
1251 } => {
1252 anyhow::ensure!(
1253 queue.resolve_id(task)? == *task,
1254 "task action requires an exact task ID"
1255 );
1256 let _claim = queue.claim(task)?;
1257 let mut task = queue.get(task)?;
1258 if task.action_applied(&q.id) {
1259 return Ok(());
1260 }
1261 if q.node == crate::conduct::NODE
1266 && q.run == task.id
1267 && task.status == TaskStatus::Blocked
1268 && task.blocked_from == Some(TaskStatus::Held)
1269 && task.blocked_by.as_slice() == [q.id.as_str()]
1270 {
1271 task.unblock(&q.id);
1272 }
1273 anyhow::ensure!(
1274 task.status == TaskStatus::Held
1275 && task.held_at == Some(*held_at)
1276 && task.hold_reason.as_deref() == reason.as_deref()
1277 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at),
1278 "task no longer has the hold this question names"
1279 );
1280 task.release_operator("choice", Some(&q.id));
1281 task.mark_action_applied(&q.id);
1282 queue.put(&mut task)?;
1283 }
1284 ask::ChoiceAction::AbandonRun { run } => {
1285 record("started")?;
1286 crate::driver::abandon_proven_dead(home, run)?;
1287 }
1288 ask::ChoiceAction::RerunChecks {
1289 repo,
1290 pr,
1291 head,
1292 runs,
1293 } => {
1294 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1295 .await?;
1296 }
1297 _ => unreachable!(),
1298 }
1299 Ok::<_, anyhow::Error>(())
1300 }
1301 .await;
1302 match applied {
1303 Ok(()) => record("applied"),
1304 Err(e) => {
1305 record(&format!("refused or failed: {e:#}"))?;
1306 Err(e)
1307 }
1308 }
1309 }
1310 .await;
1311 if home
1316 .join("choice-actions")
1317 .join(format!("{}.json", q.id))
1318 .exists()
1319 && let Some(listed) = task_of_question(&queue.list(), &q)
1320 && !listed.action_applied(&q.id)
1321 && let Ok(_claim) = queue.claim(&listed.id)
1322 && let Ok(mut origin) = queue.get(&listed.id)
1323 {
1324 if !origin.operator_held()
1325 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1326 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1327 {
1328 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1329 origin.hold_machine(Some(format!(
1330 "Maintenance choice {} settled; inspect its target before releasing this task",
1331 q.id
1332 )));
1333 } else {
1334 origin.unblock(&q.id);
1335 }
1336 }
1337 origin.mark_action_applied(&q.id);
1338 let _ = queue.put(&mut origin);
1339 }
1340 if let Err(e) = result
1341 && fresh
1342 {
1343 notices::raise_in(
1344 home,
1345 Notice::warn(
1346 &format!("action:{}", q.id),
1347 format!("Choice action {} was not completed: {e:#}", q.id),
1348 ),
1349 );
1350 }
1351 }
1352}
1353
1354pub(crate) async fn rerun_checks_once(
1361 home: &Path,
1362 request: &str,
1363 repo: &Path,
1364 pr: &str,
1365 head: &str,
1366 runs: &[String],
1367 started: impl FnOnce() -> anyhow::Result<()>,
1368) -> anyhow::Result<()> {
1369 use crate::release_watch::{GhForge, ReleaseForge};
1370 anyhow::ensure!(
1371 !runs.is_empty()
1372 && runs.len() <= 20
1373 && runs
1374 .iter()
1375 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1376 && !pr.starts_with('-'),
1377 "invalid rerun targets"
1378 );
1379 let snap = GhForge.snapshot(repo, pr).await?;
1380 anyhow::ensure!(
1381 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1382 "pull request head or lifecycle changed"
1383 );
1384 anyhow::ensure!(
1385 runs.iter().all(|r| snap
1386 .checks
1387 .iter()
1388 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1389 "named workflow is no longer failed on this pull request"
1390 );
1391 started()?;
1392 let dir = home.join("choice-actions");
1393 std::fs::create_dir_all(&dir)?;
1394 let rerun_lock = std::fs::OpenOptions::new()
1395 .create(true)
1396 .truncate(false)
1397 .write(true)
1398 .open(dir.join("reruns.lock"))?;
1399 crate::pr_head::try_lock_file(&rerun_lock)?;
1400 let ledger = dir.join("reruns.json");
1401 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1402 serde_json::from_slice(&std::fs::read(&ledger)?)?
1403 } else {
1404 Default::default()
1405 };
1406 for run in runs {
1407 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1408 if !recorded.insert(key) {
1409 continue;
1410 }
1411 let tmp = ledger.with_extension("tmp");
1412 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1413 std::fs::rename(tmp, &ledger)?;
1414 GhForge.rerun(repo, run).await?;
1415 }
1416 Ok(())
1417}
1418
1419pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1428 let tasks = queue.list();
1429 for q in questions.list() {
1430 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1432 continue;
1433 }
1434 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1435 continue;
1436 }
1437 let Some(listed) = task_of_question(&tasks, &q) else {
1438 if q.node == land::APPROVAL_NODE
1439 && q.resolution().as_deref().is_some_and(|choice| {
1440 choice == land::REPAIR || crate::merge_decision::stops(choice)
1441 })
1442 {
1443 notices::raise_in(
1444 home,
1445 Notice::warn(
1446 &format!("repair-owner:{}", q.id),
1447 format!(
1448 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1449 q.run, q.run
1450 ),
1451 ),
1452 );
1453 }
1454 continue;
1455 };
1456 if listed.action_applied(&q.id) {
1457 continue;
1458 }
1459 let Ok(_claim) = queue.claim(&listed.id) else {
1460 continue;
1461 };
1462 let Ok(mut task) = queue.get(&listed.id) else {
1463 continue;
1464 };
1465 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1468 let blockers: Vec<_> = task
1469 .blocked_by
1470 .iter()
1471 .filter(|id| {
1472 crate::queue::dependency(queue, questions, id)
1473 == crate::queue::Dependency::Question(q.id.clone())
1474 })
1475 .cloned()
1476 .collect();
1477 if !blockers.is_empty() {
1478 for id in blockers {
1479 task.unblock(&id);
1480 }
1481 let answer = match &q.answer {
1482 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1483 None => String::new(),
1484 };
1485 task.record_answer(q.summary.clone(), answer);
1486 }
1487 }
1488 let language = language_of(&task, Path::new("."));
1489 let decision = decide_action(&task, &q, phrases(&language), |id| {
1490 RunState::load_under(id, home)
1491 });
1492 let ran = matches!(
1493 decision,
1494 ActionDecision::Resume(_)
1495 | ActionDecision::Requeue
1496 | ActionDecision::Hold
1497 | ActionDecision::Done
1498 );
1499 if ran && asker_may_still_read(questions, &q, home) {
1500 continue;
1509 }
1510 let done = matches!(decision, ActionDecision::Done);
1511 match decision {
1512 ActionDecision::Skip => continue,
1513 ActionDecision::Resume(run) => {
1514 task.release_operator("choice", Some(&q.id));
1515 task.resume_override = Some(crate::queue::OperatorResume {
1516 question_id: q.id.clone(),
1517 at: Timestamp::now(),
1518 conductor_rehold: None,
1519 forced: true,
1520 pinned_run: Some(run),
1521 });
1522 }
1523 ActionDecision::Stale => {}
1524 ActionDecision::Requeue => {
1525 task.release_operator("choice-requeue", Some(&q.id));
1526 task.requeue();
1527 }
1528 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1529 ActionDecision::Done => task.succeed(),
1530 ActionDecision::Refuse(why) => {
1531 if task.operator_held() {
1532 task.hold_reason = Some(match task.hold_reason.take() {
1535 Some(reason) => format!("{reason}\n\n{why}"),
1536 None => why,
1537 });
1538 } else {
1539 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1540 task.hold_machine(Some(why));
1541 }
1542 notices::raise(
1543 Notice::warn(
1544 &format!("action:{}", q.id),
1545 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1546 )
1547 .link(Link::Task {
1548 id: task.id.clone(),
1549 }),
1550 );
1551 }
1552 }
1553 task.mark_action_applied(&q.id);
1554 record(queue, &mut task);
1557 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1558 supersede_prior_runs(&task, home);
1559 }
1560 let _ = questions.update(&q.id, |r| {
1563 r.answer_delivered = true;
1564 Ok(())
1565 });
1566 }
1567}
1568
1569fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1573 let now = Timestamp::now();
1574 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1575 return true;
1576 }
1577 q.node != crate::conduct::NODE
1578 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1579 s.seats_active()
1580 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1581 })
1582}
1583
1584fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1595 let tasks = queue.list();
1596 let by_id: std::collections::BTreeMap<&str, &Task> =
1597 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1598 let referenced: Vec<&str> = tasks
1599 .iter()
1600 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1601 .collect();
1602
1603 for mut question in questions.list() {
1604 if !question.status.open() || question.node != crate::conduct::NODE {
1605 continue;
1606 }
1607 if referenced
1612 .iter()
1613 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1614 {
1615 continue;
1616 }
1617 let Some(task) = by_id.get(question.run.as_str()) else {
1618 continue;
1619 };
1620 question.abandon(format!(
1621 "task {} no longer waits for this answer",
1622 task.short()
1623 ));
1624 if let Err(e) = questions.put(&mut question) {
1625 tracing::warn!(
1626 "could not retire question {} for task {}: {e:#}",
1627 question.short(),
1628 task.short()
1629 );
1630 }
1631 }
1632}
1633
1634#[derive(Debug, Clone, Copy)]
1641pub struct Verdict {
1642 pub status: RunStatus,
1644 pub left_pr: bool,
1646 pub quota_hit: bool,
1648 pub parked: bool,
1650 pub no_viable_candidates: bool,
1657}
1658
1659pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1716 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1717}
1718
1719fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1721 if verdict.parked {
1727 task.park(detail);
1728 return;
1729 }
1730 match verdict.status {
1731 RunStatus::Merged | RunStatus::Ready
1732 if task.pr_recovery.as_ref().is_some_and(|s| {
1733 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1734 }) =>
1735 {
1736 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1737 }
1738 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1739 RunStatus::AlreadyInBase => task.already_landed(detail),
1740 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1741 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1742 task.stall(detail)
1743 }
1744 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1745 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1746 RunStatus::Blocked => task.fail(detail, max_attempts),
1747 RunStatus::VerifiedNoop => task.handed_off(detail),
1748 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1749 }
1750}
1751
1752pub fn supersede_prior_runs(task: &Task, home: &Path) {
1801 let now = Timestamp::now();
1802 let last_run_succeeded = task
1809 .runs
1810 .last()
1811 .and_then(|id| RunState::load_under(id, home).ok())
1812 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1813 for id in task.superseded_attempts(last_run_succeeded) {
1814 let mut state = match RunState::load_under(id, home) {
1815 Ok(s) => s,
1816 Err(e) => {
1817 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1818 continue;
1819 }
1820 };
1821 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1822 continue;
1823 }
1824 let daemon_claims = is_working_on(home, id, now);
1825 if state.liveness(daemon_claims) == Liveness::Live {
1826 continue;
1827 }
1828 state.status = RunStatus::Superseded;
1829 if let Err(e) = state.save_under(home) {
1830 tracing::warn!("could not mark run {id} superseded: {e:#}");
1831 }
1832 }
1833}
1834
1835fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1857 for task in queue.list() {
1858 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1859 continue;
1860 }
1861 supersede_prior_runs(&task, home);
1862 }
1863}
1864
1865fn settle_and_diagnose(
1872 task: &mut Task,
1873 verdict: Verdict,
1874 detail: &str,
1875 max_attempts: usize,
1876 state: &RunState,
1877) {
1878 let p = phrases(&state.config.graph.language);
1879 if let Some(d) = &state.merge_decision {
1882 use crate::merge_decision as md;
1883 if task.runs.last().is_some_and(|id| id != &state.id) {
1884 return;
1885 }
1886 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1887 task.succeed();
1888 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1889 if state.liveness(false) != Liveness::Dead {
1890 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1891 return;
1892 }
1893 let replacement = md::Replacement {
1894 question: d.question.clone(),
1895 run: state.id.clone(),
1896 pr: d.target.pr.clone(),
1897 agent: agent.to_owned(),
1898 context: format!(
1899 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1900 d.target.pr,
1901 state.id,
1902 serde_json::to_string(&state.reviews).unwrap_or_default()
1903 ),
1904 };
1905 task.requeue();
1906 task.pr_recovery = None;
1907 task.review_of = None;
1908 task.replacement = Some(replacement);
1909 } else {
1910 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1911 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1912 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1913 }
1914 task.mark_action_applied(&d.question);
1915 if let Some(pass) = state.passes.last() {
1916 task.record_pass_at(
1917 &state.id,
1918 pass.index,
1919 Some(pass.started_at),
1920 task.attempts,
1921 None,
1922 );
1923 }
1924 return;
1925 }
1926 if let Some(replacement) = &task.replacement
1927 && !state.parked
1928 && state.resource_block.is_none()
1929 && (state.viable().is_empty()
1930 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1931 {
1932 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1933 return;
1934 }
1935 let attempts_before = task.attempts;
1936 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1942 task.notice_cause(
1943 &format!("resource:{}:{}", block.kind, block.node),
1944 None,
1945 None,
1946 );
1947 task.last_error = Some(detail.to_owned());
1948 task.diagnostic = None;
1949 task.attempts = task.attempts.saturating_sub(1);
1950 if block.kind == "disk pressure" {
1951 task.hold_machine(Some(format!(
1952 "{}{}",
1953 crate::disk::RUNNING_PRESSURE,
1954 block.detail
1955 )));
1956 } else {
1957 task.hold_machine(Some(format!(
1958 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1959 no agent was asked and no attempt was spent",
1960 crate::cache::RESOURCE_HOLD,
1961 block.kind,
1962 block.node
1963 )));
1964 }
1965 if let Some(pass) = state.passes.last() {
1966 task.record_pass_at(
1967 &state.id,
1968 pass.index,
1969 Some(pass.started_at),
1970 attempts_before,
1971 None,
1972 );
1973 }
1974 return;
1975 }
1976 let unresolved = task.hold_notice_cause.clone();
1977 if !verdict.parked
1978 && task
1979 .pr_recovery
1980 .as_ref()
1981 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
1982 && matches!(
1983 verdict.status,
1984 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
1985 )
1986 {
1987 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
1988 task.succeed();
1989 } else {
1990 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
1991 }
1992 } else if !verdict.parked
1993 && task.pr_recovery.is_some()
1994 && matches!(
1995 verdict.status,
1996 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
1997 )
1998 && !verdict.quota_hit
1999 {
2000 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2001 } else {
2002 settle_in(task, verdict, detail, max_attempts, p);
2003 }
2004 if !verdict.parked
2005 && !verdict.quota_hit
2006 && matches!(
2007 verdict.status,
2008 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2009 )
2010 {
2011 task.recovery_failed(state);
2012 }
2013 if state.prep_failure.is_some() {
2014 task.hold_notice_cause = unresolved;
2017 task.startup_notice_cause("startup-candidates");
2018 }
2019 if let Some(pass) = state.passes.last() {
2022 let parked = match &pass.end {
2023 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2024 _ => None,
2025 };
2026 task.record_pass_at(
2027 &state.id,
2028 pass.index,
2029 Some(pass.started_at),
2030 attempts_before,
2031 parked,
2032 );
2033 }
2034 if task.status == TaskStatus::Held {
2035 if state.prep_failure.is_none() {
2036 task.notice_cause(
2037 label(verdict.status),
2038 Some(&state.id),
2039 state.passes.last().map(|p| p.index),
2040 );
2041 }
2042 task.diagnostic = diagnostic(state);
2043 note_open_question(task, &state.id, p);
2044 }
2045}
2046
2047fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2062 let Some(home) = crate::run::try_home() else {
2063 return;
2064 };
2065 let open = Questions::at(home.join("questions")).open_for(run);
2066 let Some(q) = open.first() else {
2067 return;
2068 };
2069 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2070 decisions.sort_unstable();
2071 task.notice_cause(
2072 &format!("questions:{}", decisions.join(",")),
2073 Some(run),
2074 None,
2075 );
2076 let base = task.hold_reason.clone().unwrap_or_default();
2077 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2078}
2079
2080fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2091 match last_run {
2092 Some(state) => {
2093 let verdict = Verdict {
2094 status: state.status,
2095 left_pr: state.pr.is_some(),
2096 quota_hit: !state.quota.is_empty(),
2097 parked: state.parked,
2098 no_viable_candidates: state.viable().is_empty(),
2099 };
2100 let detail = format!(
2101 "{}{}",
2102 phrases(&state.config.graph.language).recovered_running,
2103 describe(&state)
2104 );
2105 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2106 }
2107 None => {
2108 let why = phrases(language).no_run_to_recover;
2111 task.last_error = Some(why.to_owned());
2112 task.notice_cause("missing-run", None, None);
2115 task.hold_machine(Some(why.to_owned()));
2116 }
2117 }
2118}
2119
2120fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
2142 let mut reclaimed = Vec::new();
2143 for listed in queue.list() {
2144 if listed.status != TaskStatus::Running {
2145 continue;
2146 }
2147 let Ok(_claim) = queue.claim(&listed.id) else {
2148 continue;
2149 };
2150 let Ok(mut task) = queue.get(&listed.id) else {
2154 continue;
2155 };
2156 if task.status != TaskStatus::Running {
2157 continue;
2158 }
2159 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
2160 if let Some(state) = &last_run
2170 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2171 {
2172 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2173 }
2174 let language = if last_run.is_none() {
2175 language_of(&task, Path::new("."))
2176 } else {
2177 String::new()
2178 };
2179 reclaim(&mut task, last_run, max_attempts, &language);
2180 if task.status == TaskStatus::Done {
2181 supersede_prior_runs(&task, &crate::run::home());
2182 }
2183 record(queue, &mut task);
2184 reclaimed.push(task.id.clone());
2185 }
2186 reclaimed
2187}
2188
2189fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2221 reclaim_abandoned_runs_with(
2222 home,
2223 now,
2224 crate::proc::pid_status,
2225 crate::proc::process_started_at,
2226 )
2227}
2228
2229fn reclaim_abandoned_runs_with<F, G>(
2235 home: &Path,
2236 now: Timestamp,
2237 query: F,
2238 identity: G,
2239) -> Vec<String>
2240where
2241 F: Fn(u32) -> Option<bool>,
2242 G: Fn(u32) -> Option<String>,
2243{
2244 let mut abandoned = Vec::new();
2245 for entry in std::fs::read_dir(home.join("runs"))
2246 .into_iter()
2247 .flatten()
2248 .flatten()
2249 {
2250 let id = entry.file_name().to_string_lossy().into_owned();
2251 if !crate::run::is_run_id(&id) {
2252 continue;
2253 }
2254 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2257 continue;
2258 };
2259 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2267 continue;
2268 };
2269 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2270 continue;
2271 };
2272 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2273 continue;
2274 }
2275 let daemon_claims = is_working_on(home, &id, now);
2287 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2288 != crate::run::Liveness::Dead
2289 {
2290 continue;
2291 }
2292 state.abandon("daemon");
2293 if let Err(e) = state.save_under(home) {
2294 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2295 continue;
2296 }
2297 if let Some(notice) = notices::run_ended(&state) {
2300 notices::raise_in(home, notice);
2301 }
2302 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2310 tracing::warn!("abandon questions for {id}: {e:#}");
2311 }
2312 abandoned.push(id);
2313 }
2314 abandoned
2315}
2316
2317pub async fn serve(opts: Opts) -> Result<()> {
2323 serve_until(opts, Stop::new()).await
2324}
2325
2326pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2343 let signal = {
2344 let stop = stop.clone();
2345 tokio::spawn(async move {
2346 if tokio::signal::ctrl_c().await.is_ok() {
2347 stop.stop();
2348 tracing::info!("shutdown requested; a run in flight will be finished first");
2349 }
2350 })
2351 };
2352
2353 let worktrees_root = opts
2354 .worktrees_root
2355 .clone()
2356 .unwrap_or_else(crate::run::default_worktree_root);
2357 let outcome = drive(
2358 &opts,
2359 &Queue::open(),
2360 &status_path(),
2361 &crate::run::home(),
2362 &worktrees_root,
2363 &stop,
2364 )
2365 .await;
2366
2367 signal.abort();
2368 outcome
2369}
2370
2371async fn drive(
2384 opts: &Opts,
2385 queue: &Queue,
2386 status_file: &Path,
2387 home: &Path,
2388 worktrees_root: &Path,
2389 stop: &Stop,
2390) -> Result<()> {
2391 let recovery_cfg = prepare(&opts.repo, opts).ok();
2399 let daemon_cfg = recovery_cfg
2400 .as_ref()
2401 .map(|c| c.daemon.clone())
2402 .unwrap_or_default();
2403 let reconciled = crate::driver::reconcile(home);
2404 if !reconciled.is_empty() {
2405 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2406 }
2407 let mut initial_status = Status::new();
2408 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2409 recovery_cfg.as_ref(),
2410 daemon_cfg.max_deputies,
2411 ));
2412 let status = Arc::new(Mutex::new(initial_status));
2413 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2414 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2415
2416 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2422
2423 let waiter = tokio::spawn(crate::waiter::run(
2428 crate::waiter::Waiter::new(
2429 crate::ask::Questions::at(home.join("questions")),
2430 home.to_path_buf(),
2431 recovery_cfg.clone(),
2432 ),
2433 stop.clone(),
2434 ));
2435
2436 let deputies = tokio::spawn(crate::deputy::run(
2440 crate::deputy::Deputies::new(
2441 crate::ask::Questions::at(home.join("questions")),
2442 home.to_path_buf(),
2443 recovery_cfg.clone(),
2444 opts.repo.clone(),
2445 daemon_cfg.max_deputies,
2446 {
2447 let stop = stop.clone();
2448 Arc::new(move || stop.parking())
2449 },
2450 ),
2451 stop.clone(),
2452 ));
2453
2454 let fetcher = tokio::spawn(fetch_loop(opts.repo.clone(), opts.clone(), stop.clone()));
2457
2458 let release_watch = tokio::spawn(crate::release_watch::run(
2461 crate::release_watch::Watcher::new(
2462 Box::new(crate::release_watch::GhForge),
2463 home.to_path_buf(),
2464 ),
2465 {
2466 let (repo, opts) = (opts.repo.clone(), opts.clone());
2467 move || release_watch_settings(&repo, &opts)
2468 },
2469 stop.clone(),
2470 ));
2471
2472 tracing::info!(
2473 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2474 queue.root().display(),
2475 opts.poll.as_secs(),
2476 opts.max_attempts,
2477 concurrency,
2478 if daemon_cfg.pause_for_interrupts {
2479 ", interrupts enabled"
2480 } else {
2481 ""
2482 }
2483 );
2484
2485 janitor(&opts.repo, opts, home, worktrees_root).await;
2488 resweep_superseded_attempts(queue, home);
2489
2490 let outcome = poll(
2491 opts,
2492 queue,
2493 &status,
2494 home,
2495 worktrees_root,
2496 stop,
2497 DispatchLimits {
2498 max_concurrent: concurrency,
2499 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2500 },
2501 )
2502 .await;
2503
2504 beat.abort();
2505 waiter.abort();
2506 deputies.abort();
2507 fetcher.abort();
2508 release_watch.abort();
2509 clear_status_at(status_file);
2510 outcome
2511}
2512
2513fn release_watch_settings(repo: &Path, opts: &Opts) -> (Vec<PathBuf>, u64) {
2516 let Ok(cfg) = prepare(repo, opts) else {
2517 return (Vec::new(), 0);
2518 };
2519 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2520 .into_iter()
2521 .map(|r| r.path)
2522 .collect();
2523 let mut known: Vec<PathBuf> = Queue::open().list().into_iter().map(|t| t.repo).collect();
2526 let runs = crate::run::home().join("runs");
2527 for id in crate::run::list_ids_in(&runs) {
2528 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2529 .ok()
2530 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2531 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2532 known.extend(repo);
2533 }
2534 for k in known {
2535 let k = k.canonicalize().unwrap_or(k);
2536 if k.join(".git").exists() && !paths.contains(&k) {
2537 paths.push(k);
2538 }
2539 }
2540 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2541 if !paths.contains(&own) {
2542 paths.push(own);
2543 }
2544 (paths, cfg.daemon.release_stall_minutes)
2545}
2546
2547const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2549
2550async fn fetch_loop(repo: PathBuf, opts: Opts, stop: Stop) {
2554 while !stop.stopped() {
2555 let (interval, roots) = match prepare(&repo, &opts) {
2556 Ok(c) => (c.repos.fetch_interval, c.repos.roots),
2557 Err(_) => (0, Vec::new()),
2558 };
2559 if interval > 0 && !roots.is_empty() {
2560 let r = crate::clean::fetch_origins(&roots, FETCH_TIMEOUT, || stop.stopped()).await;
2561 tracing::debug!("fetch origins: {r:?}");
2562 }
2563 let wait = if interval > 0 { interval } else { 60 };
2565 let mut slept = 0;
2566 while slept < wait && !stop.stopped() {
2567 tokio::time::sleep(Duration::from_secs(1)).await;
2568 slept += 1;
2569 }
2570 }
2571}
2572
2573async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2579 loop {
2580 tokio::time::sleep(HEARTBEAT).await;
2581 let snapshot = {
2582 let mut guard = lock(&status);
2583 guard.updated_at = Timestamp::now();
2584 guard.clone()
2585 };
2586 if let Err(e) = write_status_to(&path, &snapshot) {
2587 tracing::warn!("could not refresh the daemon status file: {e:#}");
2590 }
2591 }
2592}
2593
2594#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2597enum LandResume {
2598 NotLanding,
2601 StillWaiting,
2606 Ready,
2610}
2611
2612fn land_resume_state(task: &Task) -> LandResume {
2616 let Some(run_id) = task.runs.last() else {
2617 return LandResume::NotLanding;
2618 };
2619 let Ok(state) = RunState::load(run_id) else {
2620 return LandResume::NotLanding;
2621 };
2622 if !state.parked {
2623 return LandResume::NotLanding;
2624 }
2625 if state.status == RunStatus::Landing
2626 && state.land_approval.as_ref().is_some_and(|binding| {
2627 ask::Questions::open()
2628 .get(&binding.question)
2629 .is_ok_and(|q| {
2630 q.run == state.id
2631 && q.resolution()
2632 .as_deref()
2633 .is_some_and(crate::merge_decision::stops)
2634 })
2635 })
2636 {
2637 return LandResume::Ready;
2638 }
2639 if !implementers_match(task, &state) {
2640 return LandResume::NotLanding;
2641 }
2642 let node = if state.status == RunStatus::Landing {
2646 land::APPROVAL_NODE
2647 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2648 crate::github_text::ASK_NODE
2649 } else {
2650 return LandResume::NotLanding;
2651 };
2652 let store = ask::Questions::open();
2653 let waiting = store
2654 .list()
2655 .into_iter()
2656 .filter(|q| &q.run == run_id && q.node == node)
2657 .max_by(|a, b| a.id.cmp(&b.id));
2658 let Some(mut q) = waiting else {
2659 return LandResume::Ready;
2660 };
2661 if !q.status.open() {
2662 if q.resolution().as_deref() == Some(land::REPAIR) {
2665 return if state.liveness(false) == Liveness::Live {
2666 LandResume::StillWaiting
2667 } else {
2668 LandResume::NotLanding
2669 };
2670 }
2671 return LandResume::Ready;
2672 }
2673 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2680 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2681 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2682 q.abandon(format!(
2683 "no answer within {}s of asking",
2684 timeout.as_secs().max(1)
2685 ));
2686 if store.put(&mut q).is_ok() {
2689 return LandResume::Ready;
2690 }
2691 }
2692 LandResume::StillWaiting
2693}
2694
2695const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2703
2704const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2716
2717struct InFlightGuard<'a> {
2730 status: &'a Arc<Mutex<Status>>,
2731 stop: &'a Stop,
2732 task_id: &'a str,
2733}
2734
2735impl Drop for InFlightGuard<'_> {
2736 fn drop(&mut self) {
2737 lock(self.status).current.retain(|c| c.task != self.task_id);
2738 self.stop.exit();
2739 }
2740}
2741
2742#[derive(Debug, Clone, PartialEq, Eq)]
2764enum Interrupt {
2765 Idle,
2767 Parking {
2779 parked: Vec<String>,
2780 interrupt_task: String,
2781 },
2782 Running {
2790 parked: Vec<String>,
2791 interrupt_task: String,
2792 },
2793 Resuming { parked: Vec<String> },
2800}
2801
2802fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
2823 match state {
2824 Interrupt::Idle => {
2825 if in_flight.len() != 1 {
2837 return Interrupt::Idle;
2838 }
2839 match runnable.iter().find(|t| t.interrupt) {
2840 Some(t) => Interrupt::Parking {
2841 parked: in_flight.to_vec(),
2842 interrupt_task: t.id.clone(),
2843 },
2844 None => Interrupt::Idle,
2845 }
2846 }
2847 Interrupt::Parking {
2848 parked,
2849 interrupt_task,
2850 } => {
2851 if in_flight.iter().any(|id| parked.contains(id)) {
2852 Interrupt::Parking {
2854 parked,
2855 interrupt_task,
2856 }
2857 } else if in_flight.contains(&interrupt_task) {
2858 Interrupt::Running {
2859 parked,
2860 interrupt_task,
2861 }
2862 } else if runnable.iter().any(|t| t.id == interrupt_task) {
2863 Interrupt::Parking {
2867 parked,
2868 interrupt_task,
2869 }
2870 } else {
2871 Interrupt::Resuming { parked }
2876 }
2877 }
2878 Interrupt::Running {
2879 parked,
2880 interrupt_task,
2881 } => {
2882 if in_flight.contains(&interrupt_task) {
2883 Interrupt::Running {
2884 parked,
2885 interrupt_task,
2886 }
2887 } else {
2888 Interrupt::Resuming { parked }
2894 }
2895 }
2896 Interrupt::Resuming { parked } => {
2897 if in_flight.iter().any(|id| parked.contains(id)) {
2898 Interrupt::Idle
2904 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
2905 Interrupt::Resuming { parked }
2906 } else {
2907 Interrupt::Idle
2910 }
2911 }
2912 }
2913}
2914
2915fn advance_interrupt_tick(
2921 enabled: bool,
2922 state: Interrupt,
2923 in_flight: &[String],
2924 runnable: &[Task],
2925) -> Interrupt {
2926 if !enabled {
2927 return Interrupt::Idle;
2928 }
2929 advance_interrupt(state, in_flight, runnable)
2930}
2931
2932fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
2937 match state {
2938 Interrupt::Idle => candidates,
2939 Interrupt::Parking {
2940 parked,
2941 interrupt_task,
2942 } => {
2943 if in_flight.iter().any(|id| parked.contains(id)) {
2944 Vec::new()
2945 } else {
2946 candidates
2947 .into_iter()
2948 .filter(|t| &t.id == interrupt_task)
2949 .collect()
2950 }
2951 }
2952 Interrupt::Running { .. } => Vec::new(),
2953 Interrupt::Resuming { parked } => candidates
2961 .into_iter()
2962 .find(|t| parked.contains(&t.id))
2963 .into_iter()
2964 .collect(),
2965 }
2966}
2967
2968struct DispatchLimits {
2972 max_concurrent: usize,
2975 pause_for_interrupts: bool,
2977}
2978
2979#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2988enum PermitKind {
2989 None,
2994 Urgent,
3001 Ordinary,
3004}
3005
3006fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3009 if priority {
3010 PermitKind::None
3011 } else if urgent {
3012 PermitKind::Urgent
3013 } else {
3014 PermitKind::Ordinary
3015 }
3016}
3017
3018struct DispatchClaim {
3019 task: Task,
3020 claim: crate::queue::Claim,
3021 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3022}
3023
3024fn claim_dispatch(
3027 queue: &Queue,
3028 candidate: &Task,
3029 cooling_down: bool,
3030 ordinary: &Arc<tokio::sync::Semaphore>,
3031 urgent: &Arc<tokio::sync::Semaphore>,
3032) -> Option<DispatchClaim> {
3033 let Ok(claim) = queue.claim(&candidate.id) else {
3034 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3035 return None;
3036 };
3037 let task = match queue.get(&candidate.id) {
3038 Ok(t) if t.status.runnable() => t,
3039 Ok(_) => return None,
3040 Err(e) => {
3041 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3042 return None;
3043 }
3044 };
3045 let resume = land_resume_state(&task);
3046 if resume == LandResume::StillWaiting {
3047 return None;
3048 }
3049 let priority = resume == LandResume::Ready;
3050 if !priority && cooling_down {
3051 return None;
3052 }
3053 let permit = match permit_kind(priority, task.urgent) {
3056 PermitKind::None => None,
3057 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3058 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3059 };
3060 let stamp = Timestamp::now();
3063 let mut task = task;
3064 task.dispatched_at = Some(stamp);
3065 if let Err(e) = queue.modify(&task.id, |t| {
3066 t.dispatched_at = Some(stamp);
3067 true
3068 }) {
3069 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3070 }
3071 Some(DispatchClaim {
3072 task,
3073 claim,
3074 permit,
3075 })
3076}
3077
3078async fn poll(
3097 opts: &Opts,
3098 queue: &Queue,
3099 status: &Arc<Mutex<Status>>,
3100 home: &Path,
3101 worktrees_root: &Path,
3102 stop: &Stop,
3103 limits: DispatchLimits,
3104) -> Result<()> {
3105 let DispatchLimits {
3106 max_concurrent,
3107 pause_for_interrupts,
3108 } = limits;
3109 let mut attempted: Vec<String> = Vec::new();
3114 let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
3115 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(1));
3123 let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
3129 let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
3130 let mut conductor = Conductor::new();
3131 let mut cache_last_checked: Option<Timestamp> = None;
3134 let mut disk_last_checked: Option<Timestamp> = None;
3135 let mut interrupt = Interrupt::Idle;
3137 let mut interrupt_pauses: std::collections::HashMap<String, crate::graph::Pause> =
3143 std::collections::HashMap::new();
3144
3145 while !stop.stopped() {
3146 lock(status).polls += 1;
3147
3148 while let Some(result) = inflight.try_join_next() {
3153 if let Err(e) = result {
3154 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3155 notices::raise(Notice::error(
3156 "loop:attempt",
3157 "A queued attempt ended abnormally; check the task it was running.",
3158 ));
3159 }
3160 }
3161
3162 let swept = sweep_stale_claims(queue, STALE_CLAIM);
3163 if !swept.is_empty() {
3164 tracing::warn!(
3165 "swept {} stale claim(s) left behind by an earlier daemon: {}",
3166 swept.len(),
3167 swept.join(", ")
3168 );
3169 }
3170 let now = Timestamp::now();
3175
3176 if !stop.busy_now() {
3181 maybe_prune_cache_between_runs(
3182 &opts.repo,
3183 opts,
3184 home,
3185 stop,
3186 &mut cache_last_checked,
3187 now,
3188 )
3189 .await;
3190 }
3191
3192 if cache_check_due(disk_last_checked, now, 30) {
3193 disk_last_checked = Some(now);
3194 if let Ok(cfg) = prepare(&opts.repo, opts) {
3195 crate::retention::sweep(home, &cfg.disk, now).await;
3196 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3199 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3200 }
3201 for current in lock(status).current.clone() {
3202 if let Ok(state) = RunState::load_under(¤t.run, home)
3203 && let Some(reason) = disk_gate(&state.repo, &state.config)
3204 {
3205 if let Some(pause) = interrupt_pauses.get(¤t.task) {
3206 pause.park_because(reason);
3207 }
3208 notices::raise_in(
3209 home,
3210 Notice::warn(
3211 "disk:pressure",
3212 "Disk pressure persists after cleanup; active runs were asked to checkpoint and park. New starts and resumes wait for free space.",
3213 ),
3214 );
3215 }
3216 }
3217 }
3218 }
3219
3220 let stalled = stalled_tasks(queue, home, now);
3221 let stalled_ids: std::collections::BTreeSet<_> =
3222 stalled.iter().map(|task| task.id.clone()).collect();
3223 let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
3224 if !reclaimed.is_empty() {
3225 tracing::warn!(
3226 "reclaimed {} task(s) left `running` by a daemon that never \
3227 recorded the outcome: {}",
3228 reclaimed.len(),
3229 reclaimed.join(", ")
3230 );
3231 }
3232 let abandoned_runs = reclaim_abandoned_runs(home, now);
3233 if !abandoned_runs.is_empty() {
3234 tracing::warn!(
3235 "failed {} run(s) left behind by a killed process, past every \
3236 active seat's own timeout: {}",
3237 abandoned_runs.len(),
3238 abandoned_runs.join(", ")
3239 );
3240 }
3241
3242 let questions = Questions::at(home.join("questions"));
3247
3248 repair_choice_offers(&questions);
3251 resolve_blockers(queue, &questions);
3252 apply_choice_actions(queue, &questions, home).await;
3253 triage::apply_answers(queue, &questions);
3254 reconcile_task_questions(queue, &questions);
3255 {
3259 let talks = crate::talk::Talks::at(home.join("talks"));
3260 crate::chat_notify::sweep(queue, &talks, home);
3261 crate::chat_watch::sweep(home, &talks);
3262 crate::chat_notify::start_turns(&talks, stop.parking(), &mut inflight);
3263 }
3264
3265 let finished: Vec<Task> = finished_tasks(queue)
3271 .into_iter()
3272 .filter(|task| !stalled_ids.contains(&task.id))
3273 .filter(|task| !awaiting_resume_with(task, |id| RunState::load_under(id, home)))
3277 .collect();
3278 let queued = queued_tasks(queue);
3279 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3283 && conductor.worth_a_look(queue, &stalled, &finished)
3284 {
3285 match prepare(&opts.repo, opts) {
3286 Ok(cfg) => {
3287 conductor
3288 .maybe_run(
3289 &cfg,
3290 &opts.repo,
3291 queue,
3292 &questions,
3293 home,
3294 &queued,
3295 &stalled,
3296 &finished,
3297 opts.max_attempts,
3298 )
3299 .await;
3300 }
3301 Err(e) => {
3302 tracing::warn!("conductor: no config: {e:#}");
3303 notices::raise(Notice::warn(
3304 "loop:no-config",
3305 "The loop could not read this repository's config, so held tasks are not being triaged.",
3306 ));
3307 }
3308 }
3309 }
3310
3311 let candidates: Vec<Task> = runnable(queue)
3312 .into_iter()
3313 .filter(|t| !opts.once || !attempted.contains(&t.id))
3314 .collect();
3315
3316 let in_flight: Vec<String> = lock(status)
3321 .current
3322 .iter()
3323 .map(|c| c.task.clone())
3324 .collect();
3325 interrupt_pauses.retain(|id, _| in_flight.contains(id));
3326
3327 interrupt =
3328 advance_interrupt_tick(pause_for_interrupts, interrupt, &in_flight, &candidates);
3329 if let Interrupt::Parking {
3330 parked,
3331 interrupt_task,
3332 } = &interrupt
3333 {
3334 let reason = format!(
3335 "task {} asked to run first",
3336 crate::run::short_of(interrupt_task)
3337 );
3338 for id in parked {
3339 if let Some(pause) = interrupt_pauses.get(id) {
3340 pause.park_because(reason.clone());
3341 }
3342 }
3343 }
3344 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3345
3346 let cooling_down =
3347 lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
3348
3349 let mut started_any = false;
3350 for candidate in candidates {
3351 if stop.stopped() {
3352 break;
3353 }
3354
3355 let Some(DispatchClaim {
3356 mut task,
3357 claim,
3358 permit,
3359 }) = claim_dispatch(queue, &candidate, cooling_down, &sem, &urgent_sem)
3360 else {
3361 continue;
3362 };
3363
3364 let task_id = task.id.clone();
3365 attempted.push(task_id.clone());
3366 lock(status).idle = false;
3367 stop.enter();
3370 started_any = true;
3371
3372 let run_pause = crate::graph::Pause::new();
3376 interrupt_pauses.insert(task_id.clone(), run_pause.clone());
3377
3378 let opts = opts.clone();
3379 let queue = queue.clone();
3380 let status = Arc::clone(status);
3381 let stop = stop.clone();
3382 let quota_cooldown_until = Arc::clone("a_cooldown_until);
3383 inflight.spawn(async move {
3384 let _claim = claim;
3388 let _permit = permit;
3389 let _inflight = InFlightGuard {
3391 status: &status,
3392 stop: &stop,
3393 task_id: &task_id,
3394 };
3395 let quota = attempt(&opts, &queue, &status, &stop, run_pause, &mut task).await;
3396 lock(&status).completed += 1;
3397 let now = Timestamp::now();
3403 if let Some(until) = cooldown_until("a, now) {
3404 let wait = until.as_second() - now.as_second();
3405 *lock("a_cooldown_until) = Some(until);
3406 let hint = quota
3407 .iter()
3408 .find(|q| q.reset.is_some())
3409 .and_then(|q| q.reset.as_deref());
3410 match hint {
3411 Some(h) => tracing::warn!(
3412 "quota hit; waiting {wait}s before taking another ordinary task \
3413 (CLI reported reset: {h})"
3414 ),
3415 None => tracing::warn!(
3416 "quota hit; waiting {wait}s before taking another ordinary task \
3417 (no reset hint reported)"
3418 ),
3419 }
3420 }
3421 });
3422 }
3423
3424 if started_any {
3425 continue;
3426 }
3427
3428 if stop.busy_now() {
3429 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3434 continue;
3435 }
3436
3437 lock(status).idle = true;
3439 if opts.once {
3440 janitor(&opts.repo, opts, home, worktrees_root).await;
3444 resweep_superseded_attempts(queue, home);
3445 triage_held(queue, home, opts).await;
3446 break;
3447 }
3448 stop.idle(opts.poll).await;
3449 if stop.stopped() {
3450 continue;
3451 }
3452 janitor(&opts.repo, opts, home, worktrees_root).await;
3458 resweep_superseded_attempts(queue, home);
3459 triage_held(queue, home, opts).await;
3460 }
3461
3462 while let Some(result) = inflight.join_next().await {
3467 if let Err(e) = result {
3468 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3469 notices::raise(Notice::error(
3470 "loop:attempt",
3471 "A queued attempt ended abnormally; check the task it was running.",
3472 ));
3473 }
3474 }
3475 Ok(())
3476}
3477
3478async fn public_resume_owner(
3482 home: &Path,
3483 runs: &[String],
3484 repo: &Path,
3485) -> Result<Option<RunState>> {
3486 let mut owner = None;
3487 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
3488 let _lock = crate::pr_head::owner_lock(home, id)?;
3489 let state = RunState::load_under(id, home)?;
3490 if state.released()
3491 || state.retired_to.is_some()
3492 || state.pr_taken_over_by.is_some()
3493 || !crate::handover::landing_checkpoint(&state)
3494 {
3495 continue;
3496 }
3497 anyhow::ensure!(
3498 crate::git::toplevel(&state.repo).await?.canonicalize()?
3499 == crate::git::toplevel(repo).await?.canonicalize()?,
3500 "public owner repository differs; inspect run lineage"
3501 );
3502 anyhow::ensure!(
3503 owner.is_none(),
3504 "multiple retained public owners; inspect run lineage"
3505 );
3506 anyhow::ensure!(
3507 crate::handover::can_resume(&state),
3508 "public owner cannot resume"
3509 );
3510 let claimed = is_working_on(home, id, Timestamp::now());
3511 anyhow::ensure!(
3512 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
3513 "public owner's driver is live or unknown; wait before retrying"
3514 );
3515 owner = Some(state);
3516 }
3517 Ok(owner)
3518}
3519
3520async fn attempt(
3526 opts: &Opts,
3527 queue: &Queue,
3528 status: &Arc<Mutex<Status>>,
3529 stop: &Stop,
3530 interrupt_pause: crate::graph::Pause,
3531 task: &mut Task,
3532) -> Vec<QuotaLoss> {
3533 let repo = repo_for(task, &opts.repo);
3534 tracing::info!(
3535 "task {} — {} (repo {})",
3536 task.short(),
3537 task.title,
3538 repo.display()
3539 );
3540
3541 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
3542 Ok(owner) => owner,
3543 Err(e) => {
3544 task.hold_machine(Some(format!(
3545 "Cannot verify existing public owner: {e:#}; no attempt started."
3546 )));
3547 record(queue, task);
3548 return Vec::new();
3549 }
3550 };
3551 let resume_runs = public_owner
3552 .as_ref()
3553 .map(|s| vec![s.id.clone()])
3554 .unwrap_or_else(|| task.runs.clone());
3555
3556 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
3560 && previous
3561 .merge_decision
3562 .as_ref()
3563 .is_some_and(|decision| !decision.closed)
3564 && previous.status != RunStatus::Landing
3565 && !crate::merge_decision::resumable_hold(&previous)
3566 {
3567 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3568 record(queue, task);
3569 return Vec::new();
3570 }
3571 let stop_owner = resume_runs
3572 .last()
3573 .and_then(|id| RunState::load(id).ok())
3574 .filter(|state| {
3575 crate::merge_decision::resumable_hold(state)
3576 || state.status == RunStatus::Landing
3577 && state.land_approval.as_ref().is_some_and(|binding| {
3578 Questions::open().get(&binding.question).is_ok_and(|q| {
3579 q.run == state.id
3580 && q.resolution()
3581 .as_deref()
3582 .is_some_and(crate::merge_decision::stops)
3583 })
3584 })
3585 });
3586 if let Some(owner) = stop_owner {
3587 let mut runner = match Runner::resume(&owner.id) {
3588 Ok(runner) => runner,
3589 Err(e) => {
3590 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
3591 record(queue, task);
3592 return Vec::new();
3593 }
3594 };
3595 runner.on_pause(stop.pause());
3596 runner.watch_interrupt(interrupt_pause);
3597 let quota_before = runner.state.quota.clone();
3598 task.start(owner.id.clone());
3599 record(queue, task);
3600 lock(status).current.push(Current {
3601 task: task.id.clone(),
3602 run: owner.id,
3603 });
3604 runner.state.next_pass_task_entry = true;
3605 let result = runner.execute().await;
3606 return finish_attempt(
3607 opts.max_attempts,
3608 queue,
3609 task,
3610 &runner.state,
3611 "a_before,
3612 result,
3613 );
3614 }
3615
3616 if let Some(owner) = &public_owner
3617 && (task.fresh_start || !may_resume_run(owner, task))
3618 {
3619 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
3620 record(queue, task);
3621 return Vec::new();
3622 }
3623
3624 let unfinished = (!task.fresh_start)
3625 .then(|| unfinished_run(&resume_runs, task.short()))
3626 .flatten()
3627 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
3628 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
3629 if !task.fresh_start
3630 && task.pr_recovery.is_none()
3631 && task.review_branch.is_none()
3632 && task.resume_override.is_none()
3633 && previous
3634 .as_ref()
3635 .is_some_and(|run| run.retired_to.is_some())
3636 && unfinished.is_none()
3637 {
3638 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
3639 record(queue, task);
3640 return Vec::new();
3641 }
3642
3643 let saved_recovery = previous.as_ref().filter(|run| {
3644 task.replacement.is_none()
3645 && task.pr_recovery.as_ref().is_some_and(|selected| {
3646 selected.check_task(&task.id, &task.instruction).is_ok()
3647 && selected_starter(selected, Some(run), unfinished.as_deref())
3648 == Starter::Resume(run.id.clone())
3649 })
3650 });
3651
3652 let explicit = match &task.overrides {
3656 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
3657 Some(_) => opts.config.as_deref(),
3658 None => opts.config.as_deref(),
3659 };
3660 let (mut config, layers) = if let Some(run) = saved_recovery {
3661 let mut saved = run.clone();
3664 if let Err(e) = saved.configure_recovery_resume() {
3665 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
3666 record(queue, task);
3667 return Vec::new();
3668 }
3669 (saved.config, Vec::new())
3670 } else {
3671 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
3672 Ok(loaded) => loaded,
3673 Err(e) => {
3674 let reason = format!("[config] {e:#}");
3675 record_config_failure(task, &e);
3676 task.last_error = Some(reason.clone());
3677 task.hold_machine(Some(reason));
3678 record(queue, task);
3679 return Vec::new();
3680 }
3681 };
3682 if let Err(e) = apply_options(&mut config, opts, task) {
3683 task.hold_manual(Some(format!(
3684 "[config] {e:#}; correct the merge override before releasing the task"
3685 )));
3686 record(queue, task);
3687 return Vec::new();
3688 }
3689 if let Some(selected) = &task.pr_recovery
3690 && let Err(e) = selected.configure(&mut config)
3691 {
3692 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
3693 record(queue, task);
3694 return Vec::new();
3695 }
3696 (config, layers)
3697 };
3698 let config_evidence = crate::pr_recovery::ConfigEvidence {
3699 layers,
3700 overrides: task.overrides.clone(),
3701 daemon_merge: task
3702 .overrides
3703 .as_ref()
3704 .is_none_or(|o| o.inherit_daemon_options)
3705 .then(|| opts.merge.clone())
3706 .flatten(),
3707 };
3708 apply_task_implementers(&mut config, task);
3709 if let Some(replacement) = &task.replacement {
3710 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
3711 old.liveness(false) == Liveness::Dead
3712 && old.merge_decision.as_ref().is_some_and(|d| {
3713 d.closed
3714 && d.question == replacement.question
3715 && d.replacement_run.as_ref().is_none_or(|run| {
3716 task.runs.last() == Some(run)
3717 && RunState::load(run).is_ok_and(|current| {
3718 current.replaces.as_ref().is_some_and(|pin| {
3719 pin.question == replacement.question
3720 && pin.agent == replacement.agent
3721 })
3722 })
3723 })
3724 })
3725 });
3726 if !safe {
3727 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
3728 record(queue, task);
3729 return Vec::new();
3730 }
3731 match config.agent(&replacement.agent) {
3732 Ok(agent) if crate::agent::installed(agent) => {}
3733 _ => {
3734 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
3735 record(queue, task);
3736 return Vec::new();
3737 }
3738 }
3739 config.graph.implementers = 1;
3740 config.graph.land_approval = true;
3741 config.merge.mode = MergeMode::Pr;
3742 config.graph.selected_implementer = Some(replacement.agent.clone());
3743 }
3744 let p = phrases(&config.graph.language);
3745 let start_failed = p.could_not_start;
3746 if let Ok(base) =
3751 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
3752 .await
3753 && let Ok(head) =
3754 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
3755 {
3756 let mut inputs = crate::queue::recovery_inputs(&task.sources);
3757 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
3758 let source = crate::git::rev_parse(&repo, branch)
3759 .await
3760 .unwrap_or_default();
3761 inputs.push_str(&format!(";review:{branch}:{source}"));
3762 }
3763 if let Some(reason) = task.recovery_exhausted(
3764 &head,
3765 &inputs,
3766 task.attempt_limit
3767 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
3768 ) {
3769 task.hold_unrequested_retry(Some(reason));
3770 record(queue, task);
3771 return Vec::new();
3772 }
3773 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
3774 }
3775
3776 if let Some(reason) = disk_gate(&repo, &config) {
3784 task.notice_cause("disk-space", None, None);
3785 task.last_error = Some(reason.clone());
3786 task.hold_machine(Some(reason.clone()));
3787 record(queue, task);
3788 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
3789 notices::raise(
3792 Notice::warn(
3793 &format!("disk:{}", repo.display()),
3794 "A task was held for want of free disk space; free some, then release it from the queue.",
3795 )
3796 .link(Link::Task {
3797 id: task.id.clone(),
3798 }),
3799 );
3800 return Vec::new();
3801 }
3802
3803 let review_branch = take_review_branch(task, previous.as_ref());
3828 let branch_exists = match &review_branch {
3829 Some(branch) => crate::git::branch_exists(&repo, branch)
3830 .await
3831 .unwrap_or(false),
3832 None => false,
3833 };
3834 let proposed = match (&review_branch, &unfinished, &task.review_of) {
3838 (None, None, Some(branch)) => Starter::Review(branch.clone()),
3839 _ => choose_starter(
3840 review_branch.as_deref(),
3841 branch_exists,
3842 unfinished.as_deref(),
3843 ),
3844 };
3845 let retry_selected = task.replacement.is_some()
3846 && task.resumption.is_some()
3847 && !task.fresh_start
3848 && previous.as_ref().is_some_and(|run| {
3849 run.replaces.is_some()
3850 && ((run.status == RunStatus::Failed && run.viable().is_empty())
3851 || (run.status == RunStatus::Blocked && run.tally.is_none()))
3852 });
3853 let stop_pin = task
3854 .resume_override
3855 .as_ref()
3856 .filter(|pin| {
3857 Questions::open().get(&pin.question_id).is_ok_and(|q| {
3858 q.node == land::APPROVAL_NODE
3859 && q.resolution()
3860 .as_deref()
3861 .is_some_and(crate::merge_decision::stops)
3862 })
3863 })
3864 .and_then(|pin| pin.pinned_run.clone());
3865 let starter = if let Some(owner) = &public_owner {
3866 Starter::Resume(owner.id.clone())
3867 } else if let Some(run) = stop_pin {
3868 Starter::Resume(run)
3871 } else if retry_selected {
3872 Starter::Resume(previous.as_ref().unwrap().id.clone())
3873 } else if let Some(selected) = &task.pr_recovery {
3874 if selected.check_task(&task.id, &task.instruction).is_ok() {
3875 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
3876 } else {
3877 Starter::Review(selected.branch.clone())
3878 }
3879 } else {
3880 starter_with_implementers(task, previous.as_ref(), proposed)
3881 };
3882 if let Some(pin) = &task.resume_override
3885 && let Some(run) = &pin.pinned_run
3886 && starter != Starter::Resume(run.clone())
3887 {
3888 let run = run.clone();
3889 task.notice_cause("resume-owner", Some(&run), None);
3890 task.hold_machine(Some(format!(
3891 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
3892 task.id
3893 )));
3894 record(queue, task);
3895 return Vec::new();
3896 }
3897 if task.replacement.is_some()
3898 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
3899 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
3900 {
3901 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
3902 record(queue, task);
3903 return Vec::new();
3904 }
3905 if let Some(reason) = retained_attempt_hold(
3906 task,
3907 &starter,
3908 config.disk.retained_attempt_limit,
3909 RunState::load,
3910 ) {
3911 if let Starter::Review(branch) = &starter {
3914 task.review_branch = Some(branch.clone());
3915 }
3916 task.hold_unrequested_retry(Some(reason));
3917 record(queue, task);
3918 return Vec::new();
3919 }
3920 let attachments = match task_attachments(queue, task) {
3925 Ok(a) => a,
3926 Err(e) => {
3927 task.attempts += 1;
3928 task.fail_startup_target(
3929 "startup-attachments",
3930 e.name.clone(),
3931 format!("{start_failed}{e:#}"),
3932 opts.max_attempts,
3933 );
3934 record(queue, task);
3935 return Vec::new();
3936 }
3937 };
3938 let startup_pause = stop.pause();
3939 let startup_interrupt = interrupt_pause.clone();
3940 let wait = crate::repo_lock::WaitPolicy::new(
3941 Duration::from_secs(config.graph.timeout_resource_wait),
3942 move || startup_pause.parked() || startup_interrupt.parked(),
3943 );
3944 let started = wait.scope(Box::pin(async { match &starter {
3947 Starter::Review(_) if task.pr_recovery.is_some() => {
3948 let selected = task.pr_recovery.clone().expect("selected recovery");
3949 match selected.check_task(&task.id, &task.instruction) {
3950 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
3951 .await
3952 .map(|mut r| {
3953 r.state.attachments = attachments.clone();
3954 r
3955 }),
3956 Err(e) => Err(e),
3957 }
3958 }
3959 Starter::Review(branch) => {
3960 tracing::info!(
3961 "task {} reopens `{branch}` as a review-only pass",
3962 task.short()
3963 );
3964 let takeover = crate::handover::Takeover {
3967 earlier: task.earlier_attempts().to_vec(),
3968 home: crate::run::home(),
3969 choice: take_divergence_answer(branch, &config.merge.remote, task),
3970 task_instruction: Some(instruction_for(task)),
3971 };
3972 Runner::review_taking_over(
3973 &repo,
3974 branch,
3975 config,
3976 Some(takeover),
3977 crate::run::Origin::queue(&task.id),
3978 )
3979 .await
3980 }
3981 Starter::Resume(id) => {
3982 tracing::info!("resuming run {id} rather than competing again");
3983 Runner::resume(id).and_then(|mut r| {
3984 if retry_selected {
3985 for candidate in &mut r.state.candidates {
3988 candidate.failed = None;
3989 candidate.empty = false;
3990 candidate.commits = 0;
3991 }
3992 r.state.status = RunStatus::Implementing;
3993 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
3994 }
3995 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
3996 r.state.sources = task.sources.clone();
3997 }
3998 if r.state.prep_failure.is_some() {
3999 let approval = task
4000 .resumption
4001 .as_ref()
4002 .context("preparation retry requires an explicit task release")?;
4003 r.retry_preparation(approval.at)?;
4004 }
4005 if let Some(instruction) =
4006 prepare_instruction(&starter, Some(&r.state.instruction), task)
4007 {
4008 r.state.instruction = instruction;
4009 }
4010 r.state.attachments = attachments.clone();
4011 if task.replacement.is_some() {
4012 r.state.config.merge.mode = MergeMode::Pr;
4013 r.state.config.graph.land_approval = true;
4014 }
4015 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4019 .overrides
4020 .as_ref()
4021 .and_then(|o| o.merge.as_deref())
4022 .and_then(|m| merge_mode(m).ok())
4023 {
4024 r.state.config.merge.mode = mode;
4025 }
4026 Ok(r)
4027 })
4028 }
4029 Starter::Start => {
4030 if let Some(branch) = &review_branch {
4031 tracing::warn!(
4032 "conductor chose review for task {} but branch `{branch}` no longer \
4033 exists; requeuing as a fresh competition instead",
4034 task.short()
4035 );
4036 }
4037 let instruction = prepare_instruction(&starter, None, task)
4038 .unwrap_or_else(|| task.instruction.clone());
4039 Runner::start_naming(
4040 &repo,
4041 instruction,
4042 &task.title,
4043 config,
4044 crate::run::Origin::queue(&task.id),
4045 )
4046 .await
4047 .map(|mut r| {
4048 r.state.attachments = attachments.clone();
4049 r.state.sources = task.sources.clone();
4050 r
4051 })
4052 }
4053 }})).await;
4054 let mut runner = match started {
4055 Ok(r) => r,
4056 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4057 task.last_error = Some(e.to_string());
4058 if stop.parking() || interrupt_pause.parked() {
4059 let attempts = task.attempts;
4060 task.park(e.to_string());
4061 task.attempts = attempts;
4063 } else {
4064 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4065 }
4066 record(queue, task);
4067 return Vec::new();
4068 }
4069 Err(e) if task.pr_recovery.is_some() => {
4073 task.startup_notice_cause("startup-public-head");
4074 task.attempts += 1;
4075 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
4076 task.link_run(&partial.run);
4077 task.record_pass(&partial.run, 0, task.attempts, None);
4078 }
4079 task.last_error = Some(format!("{start_failed}{e:#}"));
4080 task.hold_unrequested_retry(Some(format!(
4081 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4082 )));
4083 record(queue, task);
4084 return Vec::new();
4085 }
4086 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4087 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4088 Some(r) => (p.handover_refused)(r),
4089 None => format!("{e:#}"),
4090 };
4091 task.startup_notice_cause("startup-handover");
4092 let reason = format!("{start_failed}{detail}");
4093 task.last_error = Some(reason.clone());
4094 let branch = match &starter {
4097 Starter::Review(branch) => Some(branch.clone()),
4098 _ => None,
4099 };
4100 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4104 record(queue, task);
4105 tracing::warn!(
4106 "holding {} for a branch it cannot take over: {e:#}",
4107 task.short()
4108 );
4109 return Vec::new();
4110 }
4111 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4116 let d = e
4117 .downcast_ref::<crate::reconcile::Diverged>()
4118 .expect("checked by the guard");
4119 let mut q = ask::Question::new(
4120 task.id.clone(),
4121 crate::reconcile::NODE.to_owned(),
4122 crate::reconcile::SEAT.to_owned(),
4123 d.summary(),
4124 d.detail(),
4125 d.choices(),
4126 );
4127 match Questions::open().put(&mut q) {
4128 Ok(()) => {
4129 task.last_error = Some(format!("{e:#}"));
4130 task.review_branch = Some(d.branch.clone());
4133 task.review_instruction = Some(task.instruction.clone());
4134 task.block(vec![q.id.clone()], Some(d.summary()));
4135 }
4136 Err(put) => {
4137 tracing::warn!("could not file the divergence question: {put:#}");
4138 task.attempts += 1;
4139 task.fail_startup(
4140 "startup-question-store",
4141 format!("{start_failed}{e:#}"),
4142 opts.max_attempts,
4143 );
4144 }
4145 }
4146 record(queue, task);
4147 return Vec::new();
4148 }
4149 Err(e) => {
4150 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4156 && let Starter::Review(branch) = &starter
4157 {
4158 task.review_branch = Some(branch.clone());
4159 task.review_instruction = Some(task.instruction.clone());
4160 task.last_error = Some(format!("{start_failed}{e:#}"));
4161 task.status = crate::queue::TaskStatus::Failed;
4162 record(queue, task);
4163 return Vec::new();
4164 }
4165 task.attempts += 1;
4166 task.last_error = Some(format!("{start_failed}{e:#}"));
4167 task.startup_notice_cause("startup-source");
4168 task.hold_unrequested_retry(Some(format!(
4169 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4170 )));
4171 record(queue, task);
4172 return Vec::new();
4173 }
4174 };
4175 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4176 runner.state.recovery_config = Some(config_evidence);
4177 }
4178 if let Some(replacement) = &task.replacement {
4179 if matches!(starter, Starter::Start) {
4180 runner.state.instruction.push_str(&format!(
4181 "\n\n# Replacement review context\n{}",
4182 replacement.context
4183 ));
4184 }
4185 runner.state.replaces = Some(replacement.clone());
4186 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4187 if let Ok(mut old) = RunState::load(&replacement.run)
4188 && let Some(d) = &mut old.merge_decision
4189 {
4190 d.replacement_run = Some(runner.state.id.clone());
4191 old.event(
4192 "replacement",
4193 format!(
4194 "replacement run {} owns the next attempt; old PR remains closed",
4195 runner.state.id
4196 ),
4197 );
4198 if let Err(e) = old.save() {
4199 task.hold_manual(Some(format!(
4200 "Could not link replacement history: {e}; no agents dispatched"
4201 )));
4202 record(queue, task);
4203 return Vec::new();
4204 }
4205 }
4206 }
4207 runner.state.task_instruction = Some(task.instruction.clone());
4208 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4211 if runner.state.origin_chat.is_none() {
4212 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4213 }
4214 runner.on_pause(stop.pause());
4216 runner.watch_interrupt(interrupt_pause);
4220
4221 let run = runner.state.id.clone();
4224 task.start(run.clone());
4225 record(queue, task);
4226 lock(status).current.push(Current {
4227 task: task.id.clone(),
4228 run,
4229 });
4230
4231 let quota_before = runner.state.quota.clone();
4234 runner.state.next_pass_task_entry = true;
4235 let result = runner.execute().await;
4236 finish_attempt(
4237 opts.max_attempts,
4238 queue,
4239 task,
4240 &runner.state,
4241 "a_before,
4242 result,
4243 )
4244}
4245
4246pub fn finish_attempt(
4251 max_attempts: usize,
4252 queue: &Queue,
4253 task: &mut Task,
4254 state: &RunState,
4255 quota_before: &[QuotaLoss],
4256 result: Result<()>,
4257) -> Vec<QuotaLoss> {
4258 let detail = match result {
4259 Ok(()) => describe(state),
4260 Err(e) => format!("{e:#}"),
4261 };
4262 let fresh = losses_this_attempt(quota_before, &state.quota);
4263 let verdict = Verdict {
4264 status: state.status,
4265 left_pr: state.pr.is_some(),
4268 quota_hit: !fresh.is_empty(),
4274 parked: state.parked,
4278 no_viable_candidates: state.viable().is_empty(),
4281 };
4282 if let Some(replacement) = &state.replaces
4283 && let Some(pr) = &state.pr
4284 && let Ok(mut old) = RunState::load(&replacement.run)
4285 && let Some(d) = &mut old.merge_decision
4286 {
4287 d.replacement_pr = Some(pr.url.clone());
4288 let _ = old.save();
4289 }
4290 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4291 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4292 task.hold_unrequested_retry(Some(format!(
4293 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4294 task.short()
4295 )));
4296 }
4297 if task.status == TaskStatus::Done {
4298 supersede_prior_runs(task, &crate::run::home());
4299 }
4300 record(queue, task);
4301 tracing::info!(
4302 "task {} is {} after run {} ({})",
4303 task.short(),
4304 task.status.as_str(),
4305 state.short(),
4306 label(state.status)
4307 );
4308 fresh
4309}
4310
4311pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4315 let waits_on_owner = withheld_text_wait(task).is_some();
4320 if task.status.runnable()
4321 && !waits_on_owner
4322 && !(task.fresh_start && task.replacement.is_some())
4323 {
4324 let why = task.last_error.clone().map_or_else(
4325 || "the run did not finish".to_owned(),
4326 |e| format!("the run did not finish: {e}"),
4327 );
4328 task.hold_unrequested_retry(Some(format!(
4329 "{why}. It was started by hand, so it is not retried \
4330 automatically; `magi task release` retries it."
4331 )));
4332 record(queue, task);
4333 }
4334}
4335
4336#[must_use]
4340pub fn loop_would_resume(run: &str) -> bool {
4341 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
4342}
4343
4344#[must_use]
4354pub fn foreign_loop(
4355 reading: Option<&Reading>,
4356 now: Timestamp,
4357 own_pid: u32,
4358) -> Option<Option<u32>> {
4359 let reading = reading.filter(|r| r.running(now))?;
4360 match reading.pid {
4361 Some(pid) if pid == own_pid => None,
4362 pid => Some(pid),
4363 }
4364}
4365
4366pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
4381 let status = Arc::new(Mutex::new(Status::new()));
4382 let stop = Stop::new();
4383 Box::pin(attempt(
4386 opts,
4387 queue,
4388 &status,
4389 &stop,
4390 crate::graph::Pause::new(),
4391 task,
4392 ))
4393 .await;
4394 let mut resumes = 0;
4403 while resumes < MAX_WITHHELD_RESUMES {
4404 let Some(timeout) = withheld_text_wait(task) else {
4405 break;
4406 };
4407 eprintln!(
4408 "waiting for the owner's answer on a withheld pull request text \
4409 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
4410 );
4411 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
4414 let ready = loop {
4415 match land_resume_state(task) {
4416 LandResume::StillWaiting if std::time::Instant::now() < limit => {
4417 tokio::time::sleep(WITHHELD_POLL).await;
4418 }
4419 LandResume::Ready => break true,
4420 _ => break false,
4421 }
4422 };
4423 if !ready {
4424 break;
4425 }
4426 resumes += 1;
4427 Box::pin(attempt(
4428 opts,
4429 queue,
4430 &status,
4431 &stop,
4432 crate::graph::Pause::new(),
4433 task,
4434 ))
4435 .await;
4436 }
4437 hold_if_runnable(queue, task);
4438}
4439
4440const MAX_WITHHELD_RESUMES: u32 = 3;
4443
4444const WITHHELD_POLL: Duration = Duration::from_secs(15);
4446
4447fn withheld_text_wait(task: &Task) -> Option<u64> {
4450 let id = task.runs.last()?;
4451 let s = RunState::load(id).ok()?;
4452 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
4453 .then_some(s.config.graph.answer_timeout)
4454}
4455
4456fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
4466 after
4467 .iter()
4468 .filter(|q| !before.contains(q))
4469 .cloned()
4470 .collect()
4471}
4472
4473fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
4476 if quota.is_empty() {
4477 return None;
4478 }
4479 let with_hint = quota.iter().find(|q| q.reset.is_some());
4480 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
4481 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
4482 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
4483 Some(
4484 now.checked_add(jiff::SignedDuration::from_secs(secs))
4485 .unwrap_or(Timestamp::MAX),
4486 )
4487}
4488
4489fn implementers_match(task: &Task, state: &RunState) -> bool {
4491 task.replacement.as_ref().map_or_else(
4492 || {
4493 task.implementers()
4494 .is_none_or(|n| n == state.config.graph.implementers)
4495 },
4496 |replacement| {
4497 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
4498 && state.config.graph.implementers == 1
4499 },
4500 )
4501}
4502
4503fn starter_with_implementers(
4505 task: &Task,
4506 previous: Option<&RunState>,
4507 proposed: Starter,
4508) -> Starter {
4509 if previous.is_some_and(|s| !implementers_match(task, s)) {
4510 task.review_of
4511 .as_ref()
4512 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
4513 } else {
4514 proposed
4515 }
4516}
4517
4518fn apply_task_implementers(config: &mut Config, task: &Task) {
4520 if let Some(n) = task.implementers() {
4521 config.graph.implementers = n;
4522 }
4523}
4524
4525#[cfg(test)]
4530fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
4531 let explicit = match &task.overrides {
4532 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4533 _ => opts.config.as_deref(),
4534 };
4535 let (mut config, _) = Config::discover(repo, explicit)?;
4536 apply_options(&mut config, opts, task)?;
4537 Ok(config)
4538}
4539
4540fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
4541 if task
4542 .overrides
4543 .as_ref()
4544 .is_none_or(|o| o.inherit_daemon_options)
4545 && let Some(mode) = opts.merge.as_deref()
4546 {
4547 config.merge.mode = merge_mode(mode)?;
4548 }
4549 if let Some(o) = &task.overrides {
4550 o.apply(config);
4551 }
4552 Ok(())
4553}
4554
4555fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
4557 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
4558 if let Some(mode) = &opts.merge {
4559 config.merge.mode = merge_mode(mode)?;
4560 }
4561 Ok(config)
4562}
4563
4564async fn maybe_prune_cache_between_runs(
4595 repo: &Path,
4596 opts: &Opts,
4597 home: &Path,
4598 stop: &Stop,
4599 last_checked: &mut Option<Timestamp>,
4600 now: Timestamp,
4601) {
4602 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
4603 return;
4604 }
4605 *last_checked = Some(now);
4606 let cfg = match prepare(repo, opts) {
4607 Ok(cfg) => cfg,
4608 Err(e) => {
4609 tracing::warn!("cache check: no config: {e:#}");
4610 return;
4611 }
4612 };
4613 match clean::prune_cache_if_over_limit(&cfg, home) {
4614 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
4615 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
4616 pruned.files,
4617 pruned.freed
4618 ),
4619 Ok(_) => {}
4620 Err(e) => {
4621 tracing::warn!("housekeep: prune cache: {e:#}");
4622 notices::raise_in(
4623 home,
4624 Notice::warn(
4625 "housekeep:cache",
4626 "Pruning the shared build cache failed; disk usage may keep growing.",
4627 ),
4628 );
4629 }
4630 }
4631}
4632
4633fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
4637 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
4638}
4639
4640async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
4659 let cfg = match prepare(repo, opts) {
4660 Ok(cfg) => cfg,
4661 Err(e) => {
4662 tracing::warn!("housekeep: no config: {e:#}");
4663 return;
4664 }
4665 };
4666 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
4675 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
4676 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
4681 let mut extra = Vec::new();
4682 if out.unreadable > 0 {
4683 extra.push(format!("{} unreadable", out.unreadable));
4684 }
4685 if out.orphaned_worktrees > 0 {
4686 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
4687 }
4688 let detail = if extra.is_empty() {
4689 String::new()
4690 } else {
4691 format!(" ({})", extra.join(", "))
4692 };
4693 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
4694 }
4695 if out.external_merges_recorded > 0 {
4696 tracing::info!(
4697 "housekeep: recorded {} run(s) as merged externally",
4698 out.external_merges_recorded
4699 );
4700 }
4701 if out.stale_pr_states_repaired > 0 {
4702 tracing::info!(
4703 "housekeep: rewrote {} run record(s) whose pull request had already settled",
4704 out.stale_pr_states_repaired
4705 );
4706 }
4707 if out.cache_files > 0 {
4708 tracing::info!(
4709 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
4710 out.cache_files,
4711 out.cache_freed
4712 );
4713 }
4714 if out.questions_abandoned > 0 {
4715 tracing::info!(
4716 "housekeep: abandoned {} question(s) left open by a finished run",
4717 out.questions_abandoned
4718 );
4719 }
4720}
4721
4722async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
4731 let questions = Questions::at(home.join("questions"));
4732 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
4733 if report.is_empty() {
4734 return;
4735 }
4736 if !report.quarantined.is_empty() {
4737 tracing::info!(
4738 "triage: held {} blocked task(s) whose blocked-on task or \
4739 question no longer exists: {}",
4740 report.quarantined.len(),
4741 report.quarantined.join(", ")
4742 );
4743 }
4744 if !report.resumed.is_empty() {
4745 tracing::info!(
4746 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
4747 report.resumed.len(),
4748 report.resumed.join(", ")
4749 );
4750 }
4751 if !report.asked.is_empty() {
4752 tracing::info!(
4753 "triage: asked about {} held task(s): {}",
4754 report.asked.len(),
4755 report.asked.join(", ")
4756 );
4757 }
4758 if !report.answered.is_empty() {
4759 tracing::info!(
4760 "triage: applied {} operator answer(s): {}",
4761 report.answered.len(),
4762 report.answered.join(", ")
4763 );
4764 }
4765}
4766
4767pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
4774 disk_gate_with(repo, config, crate::disk::free_bytes)
4775}
4776
4777fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
4781 repo: &Path,
4782 config: &Config,
4783 free_bytes: F,
4784) -> Option<String> {
4785 let min = config.disk.min_free_bytes;
4786 if min == 0 {
4787 return None;
4788 }
4789 let mut paths = vec![repo.to_owned()];
4790 if let Some(cache) = config.cache_dir() {
4791 paths.push(cache);
4792 }
4793 if let Some(root) = &config.graph.worktree_root {
4794 paths.push(root.clone());
4795 }
4796 for path in paths {
4797 let probe = if path == repo {
4798 path.as_path()
4799 } else {
4800 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
4801 };
4802 let reason = match free_bytes(probe) {
4803 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
4804 Err(e) => Some(crate::disk::unmeasured_in(
4805 probe,
4806 &e,
4807 &config.graph.language,
4808 )),
4809 };
4810 if reason.is_some() {
4811 return reason;
4812 }
4813 }
4814 None
4815}
4816
4817const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
4824
4825const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
4829
4830fn quota_wait(
4839 reset_at: Option<Timestamp>,
4840 now: Timestamp,
4841 fallback: Duration,
4842 cap: Duration,
4843) -> Duration {
4844 match reset_at {
4845 Some(at) if at > now => {
4846 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
4847 Duration::from_secs(secs).min(cap)
4848 }
4849 _ => fallback,
4850 }
4851}
4852
4853fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
4867 parse_reset_hint_zoned(text, now)
4868 .or_else(|| parse_reset_hint_dated(text))
4869 .or_else(|| parse_reset_hint_relative(text, recorded))
4870}
4871
4872fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
4876 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
4877 let mut rest = rest.trim();
4878 if rest.is_empty() {
4879 return None;
4880 }
4881 let mut total: i64 = 0;
4882 let mut matched = false;
4883 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
4884 if let Some((digits, tail)) = rest.split_once(unit)
4885 && !digits.is_empty()
4886 && digits.bytes().all(|b| b.is_ascii_digit())
4887 {
4888 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
4889 rest = tail;
4890 matched = true;
4891 }
4892 }
4893 if !rest.is_empty() || !matched {
4894 return None;
4895 }
4896 recorded
4897 .checked_add(jiff::SignedDuration::from_secs(total))
4898 .ok()
4899}
4900
4901fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
4905 let clock = clock.trim().to_lowercase();
4906 let (digits, pm) = clock
4907 .strip_suffix("am")
4908 .map(|d| (d, false))
4909 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
4910 let (h, m) = digits.trim().split_once(':')?;
4911 let mut hour: i8 = h.trim().parse().ok()?;
4912 let minute: i8 = m.trim().parse().ok()?;
4913 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
4914 return None;
4915 }
4916 if pm && hour != 12 {
4917 hour += 12;
4918 } else if !pm && hour == 12 {
4919 hour = 0;
4920 }
4921 Some((hour, minute))
4922}
4923
4924fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
4929 let open = text.find('(')?;
4930 let close = text.rfind(')')?;
4931 if close <= open {
4932 return None;
4933 }
4934 let zone = text[open + 1..close].trim();
4935 let (hour, minute) = parse_12h_clock(&text[..open])?;
4936 let tz = jiff::tz::TimeZone::get(zone).ok()?;
4937 let candidate = now
4938 .to_zoned(tz)
4939 .with()
4940 .hour(hour)
4941 .minute(minute)
4942 .second(0)
4943 .millisecond(0)
4944 .microsecond(0)
4945 .nanosecond(0)
4946 .build()
4947 .ok()?;
4948 let mut at = candidate.timestamp();
4949 if at <= now {
4950 at += jiff::SignedDuration::from_hours(24);
4951 }
4952 Some(at)
4953}
4954
4955fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
4964 let words: Vec<&str> = text.split_whitespace().collect();
4965 if words.len() < 5 {
4966 return None;
4967 }
4968 (0..=words.len() - 5)
4969 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
4970}
4971
4972fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
4978 if trailing.is_some_and(|next| next.starts_with('(')) {
4979 return None;
4980 }
4981 let month = month_number(window[0])?;
4982 let day_token = window[1].strip_suffix(',')?.to_lowercase();
4983 let day_digits = ["st", "nd", "rd", "th"]
4984 .iter()
4985 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
4986 let day: i8 = day_digits.parse().ok()?;
4987 let year_token = window[2];
4988 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
4989 return None;
4990 }
4991 let year: i16 = year_token.parse().ok()?;
4992 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
4996 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
4997 let date = jiff::civil::Date::new(year, month, day).ok()?;
4998 let candidate = date
4999 .at(hour, minute, 0, 0)
5000 .to_zoned(jiff::tz::TimeZone::UTC)
5001 .ok()?;
5002 Some(candidate.timestamp())
5003}
5004
5005fn month_number(name: &str) -> Option<i8> {
5008 const NAMES: [&str; 12] = [
5009 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5010 ];
5011 let lower = name.to_lowercase();
5012 NAMES
5013 .iter()
5014 .position(|n| *n == lower.as_str())
5015 .map(|i| i as i8 + 1)
5016}
5017
5018#[cfg(test)]
5020fn exhausted_review_budget(state: &RunState) -> bool {
5021 !crate::handover::can_resume(state)
5024 && state.status == RunStatus::Blocked
5025 && !state.released()
5026 && state.retired_to.is_none()
5027}
5028
5029fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5066 unfinished_run_with(runs, short, RunState::load)
5067}
5068
5069pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5072where
5073 F: FnMut(&str) -> Result<RunState>,
5074{
5075 let mut id = runs.last()?.clone();
5076 let mut visited = std::collections::BTreeSet::new();
5077 let mut repo = None;
5078 loop {
5079 if !runs.contains(&id) || !visited.insert(id.clone()) {
5080 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5081 return None;
5082 }
5083 let state = match load(&id) {
5084 Ok(state) if state.id == id => state,
5085 Ok(_) => return None,
5086 Err(e) => {
5087 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5088 return None;
5089 }
5090 };
5091 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5092 return None;
5093 }
5094 repo = Some(state.repo.clone());
5095 if let Some(authority) = state.retired_to {
5096 id = authority;
5097 continue;
5098 }
5099 return (crate::handover::can_resume(&state)
5100 && state.liveness(false) != crate::run::Liveness::Live)
5101 .then_some(id);
5102 }
5103}
5104
5105fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5113where
5114 F: FnOnce(&str) -> Result<RunState>,
5115{
5116 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5118 || task.fresh_start
5119 || task.review_branch.is_some()
5120 {
5121 return false;
5122 }
5123 let Some(id) = task.runs.last() else {
5124 return false;
5125 };
5126 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5127}
5128
5129#[derive(Debug, Clone, PartialEq, Eq)]
5132enum Starter {
5133 Review(String),
5136 Resume(String),
5138 Start,
5140}
5141
5142fn retained_attempt_hold(
5146 task: &Task,
5147 starter: &Starter,
5148 limit: usize,
5149 mut load: impl FnMut(&str) -> Result<RunState>,
5150) -> Option<String> {
5151 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5152 return None;
5153 }
5154 let mut retained = 0;
5155 for id in task.runs.iter().collect::<std::collections::BTreeSet<_>>() {
5156 let state = match load(id) {
5157 Ok(state) if state.id == *id => state,
5158 _ => {
5159 return Some(format!(
5160 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5161 ));
5162 }
5163 };
5164 if state.retired_to.is_some() || state.released() {
5165 continue;
5166 }
5167 retained += 1;
5170 }
5171 (retained >= limit).then(|| format!(
5172 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5173 ))
5174}
5175
5176fn selected_starter(
5178 selected: &crate::pr_recovery::Selection,
5179 previous: Option<&RunState>,
5180 unfinished: Option<&str>,
5181) -> Starter {
5182 if let Some(id) = unfinished
5183 && previous.is_some_and(|r| {
5184 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5185 })
5186 {
5187 Starter::Resume(id.to_owned())
5188 } else {
5189 Starter::Review(selected.branch.clone())
5190 }
5191}
5192
5193fn take_divergence_answer(
5197 branch: &str,
5198 remote: &str,
5199 task: &mut Task,
5200) -> Option<crate::reconcile::Choice> {
5201 let summary = crate::reconcile::summary_for(branch, remote);
5202 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5203 (a.question == summary)
5204 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5205 .flatten()
5206 .map(|c| (i, c))
5207 })?;
5208 task.answers.remove(idx);
5209 Some(choice)
5210}
5211
5212fn choose_starter(
5224 review_branch: Option<&str>,
5225 branch_exists: bool,
5226 unfinished: Option<&str>,
5227) -> Starter {
5228 match review_branch {
5229 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5230 Some(_) => Starter::Start,
5231 None => match unfinished {
5232 Some(id) => Starter::Resume(id.to_owned()),
5233 None => Starter::Start,
5234 },
5235 }
5236}
5237
5238fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5240 let branch = task.review_branch.take();
5241 let instruction = task.review_instruction.take();
5242 branch.filter(|branch| match &instruction {
5243 Some(authored) => *authored == task.instruction,
5244 None => previous.map_or_else(
5247 || task.review_of.as_ref() == Some(branch),
5248 |run| compatible_run_instruction(run, task),
5249 ),
5250 })
5251}
5252
5253fn may_resume_run(run: &RunState, task: &Task) -> bool {
5256 task.resume_override
5257 .as_ref()
5258 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5259 || compatible_run_instruction(run, task)
5260}
5261
5262pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5264 if let Some(authored) = &run.task_instruction {
5265 return compatible_instruction(authored, task);
5266 }
5267 if compatible_instruction(&run.instruction, task) {
5268 return true;
5269 }
5270 if run.reviewed_commits.is_some() {
5273 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5274 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5275 }
5276 if let Some(branch) = &task.review_of {
5277 let canonical = format!(
5278 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5279 );
5280 let generated = format!(
5281 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5282 );
5283 return task.instruction == canonical && run.instruction.starts_with(&generated);
5284 }
5285 }
5286 false
5287}
5288
5289pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5291 strip_answers_block(old, task) == task.instruction
5292}
5293
5294fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5297 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5298 return fallback.to_path_buf();
5299 }
5300 task.repo.clone()
5301}
5302
5303const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
5307
5308fn answers_block(task: &Task, count: usize) -> String {
5310 let mut s = ANSWERS_HEADER.to_owned();
5311 for a in &task.answers[..count] {
5312 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
5313 }
5314 s
5315}
5316
5317fn resumption_block(r: &crate::queue::Resumption) -> String {
5320 format!(
5321 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
5322 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
5323 r.generation, r.at, r.instruction, r.answers_through,
5324 )
5325}
5326
5327fn append_answers(base: &str, task: &Task) -> String {
5329 let mut s = base.to_owned();
5330 if !task.answers.is_empty() {
5331 s.push_str(&answers_block(task, task.answers.len()));
5332 }
5333 if let Some(r) = &task.resumption {
5334 s.push_str(&resumption_block(r));
5335 }
5336 s
5337}
5338
5339fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
5343 let mut instruction = instruction;
5344 for approval in task
5345 .resumption
5346 .iter()
5347 .chain(task.resumption_history.iter().rev())
5348 {
5349 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
5350 instruction = base;
5351 break;
5352 }
5353 }
5354 for count in (1..=task.answers.len()).rev() {
5355 let block = answers_block(task, count);
5356 if let Some(base) = instruction.strip_suffix(&block) {
5357 return base;
5358 }
5359 }
5360 instruction
5361}
5362
5363fn instruction_for(task: &Task) -> String {
5371 append_answers(&task.instruction, task)
5372}
5373
5374fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
5386 append_answers(strip_answers_block(old_instruction, task), task)
5387}
5388
5389fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
5391 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
5392 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
5393 if let Some(cause) = task.hold_notice_cause.as_mut() {
5394 cause.target = failure.map(|f| f.target.clone());
5395 }
5396}
5397
5398#[derive(Debug, thiserror::Error)]
5399#[error("attachment `{name}` is recorded on the task but {path} is missing")]
5400struct MissingAttachment {
5401 name: String,
5402 path: PathBuf,
5403}
5404
5405fn task_attachments(
5407 queue: &Queue,
5408 task: &Task,
5409) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
5410 let paths = queue.attachment_paths(task);
5411 for (name, path) in task.attachments.iter().zip(&paths) {
5412 if !path.is_file() {
5413 return Err(MissingAttachment {
5414 name: name.clone(),
5415 path: path.clone(),
5416 });
5417 }
5418 }
5419 Ok(paths)
5420}
5421
5422fn prepare_instruction(
5433 starter: &Starter,
5434 old_instruction: Option<&str>,
5435 task: &Task,
5436) -> Option<String> {
5437 match starter {
5438 Starter::Start => Some(instruction_for(task)),
5439 Starter::Resume(_) => Some(resumed_instruction(
5440 old_instruction.expect("a resumed run always has a prior instruction"),
5441 task,
5442 )),
5443 Starter::Review(_) => None,
5444 }
5445}
5446
5447fn record(queue: &Queue, task: &mut Task) {
5451 if let Err(e) = queue.put(task) {
5452 tracing::error!("could not record task {}: {e:#}", task.short());
5453 let notice = Notice::error(
5454 "loop:record",
5455 "The loop could not save a task's state; check the disk.",
5456 )
5457 .link(Link::Task {
5458 id: task.id.clone(),
5459 });
5460 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
5462 Some(home) => notices::raise_in(home, notice),
5463 None => notices::raise(notice),
5464 }
5465 }
5466}
5467
5468fn runnable(queue: &Queue) -> Vec<Task> {
5474 let mut tasks: Vec<Task> = queue
5475 .list()
5476 .into_iter()
5477 .filter(|t| t.status.runnable())
5478 .collect();
5479 tasks.sort_unstable_by(crate::queue::dispatch_order);
5480 tasks
5481}
5482
5483fn describe(state: &RunState) -> String {
5497 let p = phrases(&state.config.graph.language);
5498 let mut detail = if state.status == RunStatus::Stalled {
5499 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
5500 seats.sort_unstable();
5501 seats.dedup();
5502 if seats.is_empty() {
5503 p.quorum_lost.to_owned()
5504 } else {
5505 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
5506 }
5507 } else {
5508 format!("{}{}", p.run_ended, state.status.display_label())
5509 };
5510 if let Some(last) = state.events.last() {
5511 let unanswered = state
5515 .reviews
5516 .last()
5517 .filter(|r| {
5518 state.status == RunStatus::Blocked
5519 && last.node == "review"
5520 && r.incomplete()
5521 && r.blocking == 0
5522 && r.round == state.config.graph.review_rounds
5523 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
5524 })
5525 .map(|r| (r.expected - r.answered, r.round));
5526 match unanswered {
5527 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
5528 detail.push_str(&format!(
5529 " ({}: {})",
5530 last.node,
5531 (p.reviewers_never_answered)(missing, rounds)
5532 ));
5533 }
5534 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
5535 }
5536 }
5537 detail.push_str(&format!(" [run {}]", state.id));
5538 detail
5539}
5540
5541const DIAGNOSTIC_MAX: usize = 4_000;
5547
5548const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
5553
5554fn diagnostic(state: &RunState) -> Option<String> {
5568 let mut parts: Vec<String> = Vec::new();
5569
5570 for o in state.gate.iter().filter(|o| !o.ok()) {
5572 parts.push(format!(
5573 "gate `{}` failed ({}):\n{}",
5574 o.command,
5575 notices::exit_status(o.code),
5576 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
5577 ));
5578 }
5579
5580 if let Some(last) = state
5583 .events
5584 .iter()
5585 .rev()
5586 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
5587 {
5588 parts.push(last.message.clone());
5589 }
5590
5591 if state.viable().is_empty() {
5598 for c in &state.candidates {
5599 if let Some(evidence) = &c.verified_noop {
5600 parts.push(format!(
5601 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
5602 c.label
5603 ));
5604 } else if !c.summary.trim().is_empty() {
5605 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
5606 } else if let Some(why) = &c.failed {
5607 parts.push(format!("candidate {}: {why}", c.label));
5608 }
5609 }
5610 }
5611
5612 if parts.is_empty() {
5613 return None;
5614 }
5615 Some(crate::run::tail(
5620 &parts.join("\n\n"),
5621 DIAGNOSTIC_MAX.saturating_sub(100),
5622 ))
5623}
5624
5625fn label(status: RunStatus) -> &'static str {
5633 status.as_str()
5634}
5635
5636pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
5638 match mode {
5639 "none" => Ok(MergeMode::None),
5640 "local" => Ok(MergeMode::Local),
5641 "pr" => Ok(MergeMode::Pr),
5642 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
5643 }
5644}
5645
5646fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
5651 mutex
5652 .lock()
5653 .unwrap_or_else(std::sync::PoisonError::into_inner)
5654}
5655
5656#[cfg(test)]
5657mod tests {
5658 use super::*;
5659 use crate::queue::{Source, TaskStatus};
5660 use crate::run::{Candidate, CommandOutcome};
5661 use pretty_assertions::assert_eq;
5662
5663 fn task() -> Task {
5664 Task::new(
5665 "add retries".to_owned(),
5666 "add retries".to_owned(),
5667 PathBuf::from("/repo"),
5668 Source::Human,
5669 )
5670 }
5671
5672 #[test]
5673 fn selected_public_head_overrides_historical_resume_and_pin() {
5674 let mut t = task();
5675 let mut old = run_state(RunStatus::Blocked);
5676 let selection = crate::pr_recovery::Selection {
5677 url: "url".into(),
5678 branch: format!("magi/task-{}", t.id),
5679 head: "a".repeat(40),
5680 source_head: None,
5681 source_run: None,
5682 source_proof: None,
5683 disposition: Some(crate::pr_recovery::Disposition::Review),
5684 instruction: t.instruction.clone(),
5685 approved_at: Timestamp::now(),
5686 };
5687 t.resume_override = Some(crate::queue::OperatorResume {
5688 pinned_run: Some(old.id.clone()),
5689 question_id: "choice".into(),
5690 at: Timestamp::now(),
5691 conductor_rehold: None,
5692 forced: false,
5693 });
5694 assert_eq!(
5695 selected_starter(&selection, Some(&old), Some(&old.id)),
5696 Starter::Review(selection.branch.clone())
5697 );
5698 old.pr_recovery = Some(selection.clone());
5699 assert_eq!(
5700 selected_starter(&selection, Some(&old), Some(&old.id)),
5701 Starter::Review(selection.branch.clone())
5702 );
5703 old.reviewed_commits = Some(vec!["implementation".into()]);
5704 assert_eq!(
5705 selected_starter(&selection, Some(&old), Some(&old.id)),
5706 Starter::Resume(old.id.clone())
5707 );
5708 let mut changed = selection.clone();
5709 changed.head = "b".repeat(40);
5710 assert_eq!(
5711 selected_starter(&changed, Some(&old), Some(&old.id)),
5712 Starter::Review(changed.branch)
5713 );
5714 }
5715
5716 #[test]
5717 fn candidate_transplant_failure_is_one_actionable_hold() {
5718 let dir = tempfile::tempdir().unwrap();
5719 let q = Queue::at(dir.path().join("queue"));
5720 let mut t = task();
5721 let mut state = run_state(RunStatus::Blocked);
5722 state.event("prep", "cannot apply referenced commit: conflict");
5723 state.prep_failure = Some("conflict".into());
5724 t.start(state.id.clone());
5725 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
5726 assert_eq!(t.attempts, 1);
5727 assert_eq!(t.status, TaskStatus::Held);
5728 assert!(t.operator_held());
5729 assert!(!t.status.runnable());
5730 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
5731 let first = t.chat_notice_key().unwrap();
5732 t.chat_notices.push(first.clone());
5733 t.release_operator("test", None);
5734 state.id = "another-unprepared-run".into();
5735 t.start(state.id.clone());
5736 finish_attempt(
5737 3,
5738 &q,
5739 &mut t,
5740 &state,
5741 &[],
5742 Err(anyhow::anyhow!("still conflicts")),
5743 );
5744 assert_eq!(
5745 t.chat_notice_key().as_ref(),
5746 Some(&first),
5747 "no work happened"
5748 );
5749 t.release_operator("test", None);
5750 state.prep_failure = None;
5751 state.status = RunStatus::Failed;
5752 state.begin_pass();
5753 t.start(state.id.clone());
5754 finish_attempt(
5755 1,
5756 &q,
5757 &mut t,
5758 &state,
5759 &[],
5760 Err(anyhow::anyhow!("new gate failure")),
5761 );
5762 let failure = t.chat_notice_key().unwrap();
5763 assert_ne!(failure, first, "approved recovery reached a real failure");
5764 t.release_operator("test", None);
5765 state.begin_pass();
5766 t.start(state.id.clone());
5767 finish_attempt(
5768 1,
5769 &q,
5770 &mut t,
5771 &state,
5772 &[],
5773 Err(anyhow::anyhow!("new failure on resume")),
5774 );
5775 assert_ne!(
5776 t.chat_notice_key().as_ref(),
5777 Some(&failure),
5778 "a new settled pass"
5779 );
5780 }
5781
5782 #[test]
5783 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
5784 for source in ["cli", "web"] {
5785 let mut t = task();
5786 let mut run = run_state(RunStatus::Blocked);
5787 run.instruction = "Run A original criteria".into();
5788 run.task_instruction = Some(run.instruction.clone());
5789 t.start(run.id.clone());
5790 t.release_operator("choice", Some("resume-question"));
5791 t.resume_override = Some(crate::queue::OperatorResume {
5792 question_id: "resume-question".into(),
5793 at: Timestamp::now(),
5794 conductor_rehold: None,
5795 forced: true,
5796 pinned_run: Some(run.id.clone()),
5797 });
5798 t.hold_machine(Some("not enough free space to start a run".into()));
5799 let mut unchanged = t.clone();
5800 unchanged
5801 .edit("title changed".into(), unchanged.instruction.clone())
5802 .unwrap();
5803 unchanged.release_operator(source, None);
5804 assert_eq!(
5805 unchanged
5806 .resume_override
5807 .as_ref()
5808 .unwrap()
5809 .pinned_run
5810 .as_deref(),
5811 Some(run.id.as_str())
5812 );
5813 assert!(!compatible_run_instruction(&run, &unchanged));
5814 assert!(may_resume_run(&run, &unchanged));
5815
5816 t.edit(
5817 "approved recovery".into(),
5818 "Reuse measured PR head B; fix Clippy; review".into(),
5819 )
5820 .unwrap();
5821 assert!(t.resume_override.is_none());
5822 t.release_operator(source, None);
5823 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
5824 assert!(reloaded.resume_override.is_none());
5825 assert!(!may_resume_run(&run, &reloaded));
5826 let unfinished =
5827 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
5828 .filter(|_| may_resume_run(&run, &reloaded));
5829 assert_eq!(
5830 choose_starter(None, false, unfinished.as_deref()),
5831 Starter::Start
5832 );
5833 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
5834 assert_eq!(
5835 reloaded.resumption_history[0].question_id.as_deref(),
5836 Some("resume-question")
5837 );
5838
5839 unchanged.instruction = "Reuse measured PR head B".into();
5842 unchanged.hold_machine(Some("disk hold".into()));
5843 unchanged.release_operator(source, None);
5844 assert!(unchanged.resume_override.is_none());
5845 assert!(!may_resume_run(&run, &unchanged));
5846 }
5847 }
5848
5849 #[test]
5850 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
5851 let mut t = task();
5852 let mut run = run_state(RunStatus::Blocked);
5853 t.review_of = Some("approved-head".into());
5854 t.instruction =
5855 "Review the branch `approved-head` (review-only: no implementation, no judging)."
5856 .into();
5857 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
5858 run.reviewed_commits = Some(vec!["Commit log".into()]);
5859 t.start(run.id.clone());
5860 t.hold_machine(Some("interrupted review".into()));
5861 t.release_operator("cli", None);
5862 assert!(
5863 compatible_run_instruction(&run, &t),
5864 "legacy explicit review remains resumable"
5865 );
5866 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
5867 .filter(|_| compatible_run_instruction(&run, &t));
5868 assert_eq!(
5869 choose_starter(None, false, unfinished.as_deref()),
5870 Starter::Resume(run.id.clone())
5871 );
5872 run.task_instruction = Some(t.instruction.clone());
5873 assert!(compatible_run_instruction(&run, &t));
5874 let reloaded: RunState =
5875 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
5876 assert!(compatible_run_instruction(&reloaded, &t));
5877 t.edit(
5878 "new approved head".into(),
5879 "Reuse measured PR head B".into(),
5880 )
5881 .unwrap();
5882 assert!(!compatible_run_instruction(&run, &t));
5883
5884 t.review_of = None;
5886 t.instruction = "Fix Clippy on approved head".into();
5887 run.task_instruction = None;
5888 run.instruction = format!(
5889 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
5890 t.instruction
5891 );
5892 assert!(compatible_run_instruction(&run, &t));
5893 t.instruction.push('\n');
5894 assert!(
5895 compatible_run_instruction(&run, &t),
5896 "legacy generated review trimmed the original criteria"
5897 );
5898 t.hold_for_handover(Some("approved-head".into()), "refused".into());
5899 let mut unchanged = t.clone();
5900 assert_eq!(
5901 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
5902 Some("approved-head")
5903 );
5904 t.edit("new plan".into(), "Reuse measured PR head B".into())
5905 .unwrap();
5906 assert!(take_review_branch(&mut t, Some(&run)).is_none());
5907 assert!(!compatible_run_instruction(&run, &t));
5908 t.review_branch = Some("approved-head".into());
5909 t.review_instruction = None;
5910 assert!(
5911 take_review_branch(&mut t, Some(&run)).is_none(),
5912 "legacy retained candidate cannot replace a revised approved head"
5913 );
5914 }
5915
5916 #[test]
5917 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
5918 crate::run::pin_test_home();
5919 let dir = tempfile::tempdir().unwrap();
5920 let queue = Queue::at(dir.path().join("queue"));
5921 for resume in [true, false] {
5922 let state = run_state(RunStatus::Blocked);
5923 let mut t = task();
5924 t.start(state.id.clone());
5925 let action = if resume {
5926 ask::ChoiceAction::Resume {
5927 run: state.id.clone(),
5928 }
5929 } else {
5930 ask::ChoiceAction::Requeue
5931 };
5932 let mut q = action_question(&state.id, action);
5933 q.status = ask::QuestionStatus::Open;
5934 q.answer = None;
5935 q.answered_at = None;
5936 t.fail("run stopped with a question", 3);
5937 hold_if_runnable(&queue, &mut t);
5938 assert!(t.operator_decided_at.is_none());
5939 assert!(t.operator_held());
5940 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
5941 let expected = if resume {
5942 ActionDecision::Resume(state.id.clone())
5943 } else {
5944 ActionDecision::Requeue
5945 };
5946 assert_eq!(
5947 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5948 expected
5949 );
5950 t.hold_manual(Some("later explicit hold".into()));
5951 assert_eq!(
5952 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5953 ActionDecision::Stale
5954 );
5955 }
5956 }
5957
5958 #[test]
5959 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
5960 let mut t = task();
5961 t.answers.push(crate::queue::AnsweredQuestion {
5962 question: "Inspect the remote?".into(),
5963 answer: "Inspect then wait".into(),
5964 });
5965 let old = instruction_for(&t);
5966 assert!(compatible_instruction(&old, &t));
5967 t.instruction = "Reuse approved PR head and review".into();
5968 t.release_operator("cli", None);
5969 assert!(!compatible_instruction(&old, &t));
5970 let approved = instruction_for(&t);
5971 assert!(approved.contains("waiting premises are superseded"));
5972 assert!(compatible_instruction(&approved, &t));
5973 assert_eq!(resumed_instruction(&approved, &t), approved);
5974 t.release_operator("web", None);
5975 assert!(compatible_instruction(&approved, &t));
5976 assert_eq!(t.resumption_history.len(), 1);
5977 t.start("run".into());
5978 assert!(instruction_for(&t).contains("Explicit operator resumption"));
5979 }
5980
5981 #[test]
5985 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
5986 let mut state = RunState::new(
5987 PathBuf::from("/repo"),
5988 "main".to_owned(),
5989 "abc1234".to_owned(),
5990 "task".to_owned(),
5991 Config::default(),
5992 );
5993 state.status = RunStatus::Blocked;
5994 state.resource_block = Some(crate::run::ResourceBlock {
5995 node: "graft".to_owned(),
5996 kind: "shared build cache".to_owned(),
5997 detail: "timed out".to_owned(),
5998 at: jiff::Timestamp::now(),
5999 });
6000 let mut t = task();
6001 t.start("20260902-000000-aaaa".to_owned());
6002 assert_eq!(t.attempts, 1);
6003 let verdict = Verdict {
6004 status: RunStatus::Blocked,
6005 left_pr: false,
6006 quota_hit: false,
6007 parked: false,
6008 no_viable_candidates: false,
6009 };
6010 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
6011 assert_eq!(t.status, TaskStatus::Held);
6012 assert_eq!(t.attempts, 0, "the attempt is refunded");
6013 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
6014 assert!(
6015 !t.operator_held(),
6016 "a machine hold, released without a human"
6017 );
6018
6019 let mut selected = task();
6020 selected.start("20260902-000000-cccc".into());
6021 selected.replacement = Some(crate::merge_decision::Replacement {
6022 question: "fixture".into(),
6023 run: "earlier".into(),
6024 pr: "https://example.invalid/pr/1".into(),
6025 agent: "selected".into(),
6026 context: "fixture".into(),
6027 });
6028 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
6029 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
6030 assert_eq!(selected.attempts, 0);
6031 assert!(
6032 !selected.operator_held(),
6033 "a selected agent also retains a machine resource hold"
6034 );
6035 assert!(
6036 selected
6037 .hold_reason
6038 .as_deref()
6039 .unwrap()
6040 .starts_with("[resource] ")
6041 );
6042
6043 let mut p = task();
6045 p.start("20260902-000000-bbbb".to_owned());
6046 settle_and_diagnose(
6047 &mut p,
6048 Verdict {
6049 parked: true,
6050 ..verdict
6051 },
6052 "parked",
6053 2,
6054 &state,
6055 );
6056 assert_eq!(p.status, TaskStatus::Parked);
6057 }
6058
6059 #[test]
6060 fn record_failure_notice_links_the_task() {
6061 let dir = tempfile::tempdir().unwrap();
6062 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
6065 let queue = Queue::at(dir.path().join("queue"));
6066 let mut t = interrupt_task("rec-fail-1");
6067 record(&queue, &mut t);
6068
6069 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
6070 assert_eq!(found.len(), 1);
6071 assert_eq!(found[0].key, "loop:record");
6072 assert_eq!(
6073 found[0].link,
6074 Some(Link::Task {
6075 id: "rec-fail-1".to_owned()
6076 })
6077 );
6078 }
6079
6080 fn interrupt_task(id: &str) -> Task {
6083 let mut t = task();
6084 t.id = id.to_owned();
6085 t.interrupt = true;
6086 t
6087 }
6088
6089 fn task_with_id(id: &str) -> Task {
6091 let mut t = task();
6092 t.id = id.to_owned();
6093 t
6094 }
6095
6096 fn urgent_task(id: &str) -> Task {
6098 let mut t = task();
6099 t.id = id.to_owned();
6100 t.urgent = true;
6101 t
6102 }
6103
6104 #[test]
6105 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
6106 let dir = tempfile::tempdir().unwrap();
6107 let queue = Queue::at(dir.path().join("queue"));
6108 let mut parked = task_with_id("parked");
6109 parked.start("run".into());
6110 parked.park("upgrade");
6111 parked.record_pass_at(
6112 "run",
6113 0,
6114 Some(Timestamp::now()),
6115 1,
6116 Some(crate::run::ParkCause::Upgrade {
6117 from: "old".into(),
6118 to: None,
6119 }),
6120 );
6121 let mut fresh = task_with_id("fresh");
6122 fresh.priority = 100;
6123 let mut urgent_task = urgent_task("urgent");
6124 for t in [&mut parked, &mut fresh, &mut urgent_task] {
6125 queue.put(t).unwrap();
6126 }
6127 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6128 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6129 let candidates = runnable(&queue);
6130 assert_eq!(candidates[0].id, parked.id);
6131 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
6132 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
6133 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
6134 drop(first);
6135 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
6136 let next = runnable(&queue)
6137 .into_iter()
6138 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
6139 .unwrap();
6140 assert_eq!(next.task.id, fresh.id);
6141 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
6142 drop(claimed_elsewhere);
6143 drop(next);
6144 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
6145 }
6146
6147 #[test]
6148 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
6149 let dir = tempfile::tempdir().unwrap();
6150 let queue = Queue::at(dir.path().join("queue"));
6151 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6152 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6153 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
6154 let mut first = task();
6155 queue.put(&mut first).unwrap();
6156 let stale_plain = queue.get(&first.id).unwrap();
6157 first.set_urgent(true).unwrap();
6158 queue.put(&mut first).unwrap();
6159 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
6160 let mut second = urgent_task("second");
6161 queue.put(&mut second).unwrap();
6162 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
6163 drop(queue.claim(&second.id).unwrap());
6164 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
6165 drop(dispatched);
6166 let stale_urgent = queue.get(&first.id).unwrap();
6167 first.set_urgent(false).unwrap();
6168 queue.put(&mut first).unwrap();
6169 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
6170 assert_eq!(urgent.available_permits(), 1);
6171 drop(queue.claim(&first.id).unwrap());
6172 drop(occupied);
6173 }
6174
6175 #[test]
6176 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
6177 let dir = tempfile::tempdir().unwrap();
6178 let queue = Queue::at(dir.path().join("queue"));
6179 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6180 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6181 let mut t = task();
6182 queue.put(&mut t).unwrap();
6183 for enabled in [true, false] {
6184 let stale = queue.get(&t.id).unwrap();
6185 t.set_urgent(enabled).unwrap();
6186 queue.put(&mut t).unwrap();
6187 assert_ne!(stale.urgent, enabled);
6188 let DispatchClaim {
6189 task: fresh,
6190 claim,
6191 permit,
6192 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
6193 assert_eq!(fresh.urgent, enabled);
6194 assert_eq!(ordinary.available_permits(), usize::from(enabled));
6195 assert_eq!(urgent.available_permits(), usize::from(!enabled));
6196 assert!(queue.claim(&t.id).is_err());
6197 assert_eq!(fresh.attempts, stale.attempts);
6198 drop((claim, permit));
6199 assert_eq!(ordinary.available_permits(), 1);
6200 assert_eq!(urgent.available_permits(), 1);
6201 }
6202 t.set_urgent(true).unwrap();
6203 queue.put(&mut t).unwrap();
6204 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
6205 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6206 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
6207 drop(queue.claim(&t.id).unwrap());
6208 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
6209 drop(occupied);
6210 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
6211 drop(queue.claim(&t.id).unwrap());
6212 let held = queue.claim(&t.id).unwrap();
6213 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6214 drop(held);
6215 t.status = TaskStatus::Held;
6216 queue.put(&mut t).unwrap();
6217 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6218 drop(queue.claim(&t.id).unwrap());
6219 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
6220 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6221 drop(queue.claim(&t.id).unwrap());
6222 assert_eq!(urgent.available_permits(), 1);
6223 }
6224
6225 #[test]
6230 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
6231 assert_eq!(permit_kind(true, false), PermitKind::None);
6232 assert_eq!(permit_kind(true, true), PermitKind::None);
6233 }
6234
6235 #[test]
6240 fn permit_kind_separates_urgent_from_ordinary() {
6241 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
6242 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
6243 }
6244
6245 #[test]
6252 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
6253 let cfg = Config::default();
6254 let repo = Path::new("/any/repo/path");
6255
6256 let reason =
6257 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
6258 assert!(reason.contains("1024"), "{reason}");
6259 assert!(
6260 reason.contains(&cfg.disk.min_free_bytes.to_string()),
6261 "{reason}"
6262 );
6263
6264 assert_eq!(
6265 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
6266 None,
6267 "exactly at the floor is open"
6268 );
6269 assert_eq!(
6270 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
6271 None,
6272 "comfortably above the floor is open"
6273 );
6274 }
6275
6276 #[test]
6277 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
6278 let mut cfg = Config::default();
6279 cfg.disk.min_free_bytes = 0;
6280 let repo = Path::new("/any/repo/path");
6281 assert_eq!(
6282 disk_gate_with(repo, &cfg, |_| Ok(0)),
6283 None,
6284 "a zero floor never measures at all"
6285 );
6286 }
6287
6288 #[test]
6289 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
6290 let cfg = Config::default();
6291 let repo = Path::new("/any/repo/path");
6292 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
6293 .expect("a measurement failure must close the gate, not open it");
6294 assert!(reason.contains("could not measure"), "{reason}");
6295 }
6296
6297 #[test]
6298 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
6299 let ordinary = task();
6300 let next = advance_interrupt(
6301 Interrupt::Idle,
6302 std::slice::from_ref(&ordinary.id),
6303 std::slice::from_ref(&ordinary),
6304 );
6305 assert_eq!(next, Interrupt::Idle);
6306 }
6307
6308 #[test]
6309 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
6310 let marked = interrupt_task("marked");
6313 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
6314 assert_eq!(next, Interrupt::Idle);
6315 }
6316
6317 #[test]
6318 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
6319 let marked = interrupt_task("marked");
6320 let next = advance_interrupt(
6321 Interrupt::Idle,
6322 &["running".to_owned()],
6323 std::slice::from_ref(&marked),
6324 );
6325 assert_eq!(
6326 next,
6327 Interrupt::Parking {
6328 parked: vec!["running".to_owned()],
6329 interrupt_task: "marked".to_owned(),
6330 }
6331 );
6332 }
6333
6334 #[test]
6343 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
6344 let marked = interrupt_task("marked");
6345
6346 let two = advance_interrupt(
6347 Interrupt::Idle,
6348 &["a".to_owned(), "b".to_owned()],
6349 std::slice::from_ref(&marked),
6350 );
6351 assert_eq!(two, Interrupt::Idle);
6352
6353 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
6354 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
6355 }
6356
6357 #[test]
6358 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
6359 let state = Interrupt::Parking {
6360 parked: vec!["running".to_owned()],
6361 interrupt_task: "marked".to_owned(),
6362 };
6363 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
6365 assert_eq!(still_going, state);
6366
6367 let stopped_but_not_yet_dispatched =
6371 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
6372 assert_eq!(stopped_but_not_yet_dispatched, state);
6373
6374 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
6376 assert_eq!(
6377 dispatched,
6378 Interrupt::Running {
6379 parked: vec!["running".to_owned()],
6380 interrupt_task: "marked".to_owned(),
6381 }
6382 );
6383 }
6384
6385 #[test]
6386 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
6387 let state = Interrupt::Running {
6388 parked: vec!["running".to_owned()],
6389 interrupt_task: "marked".to_owned(),
6390 };
6391 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
6392 assert_eq!(still_running, state);
6393
6394 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
6401 assert_eq!(
6402 ended,
6403 Interrupt::Resuming {
6404 parked: vec!["running".to_owned()]
6405 }
6406 );
6407 }
6408
6409 #[test]
6410 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
6411 let state = Interrupt::Resuming {
6412 parked: vec!["running".to_owned()],
6413 };
6414 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
6415 assert_eq!(still_waiting, state);
6416
6417 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
6418 assert_eq!(dispatched, Interrupt::Idle);
6419 }
6420
6421 #[test]
6427 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
6428 {
6429 let state = Interrupt::Parking {
6430 parked: vec!["running".to_owned()],
6431 interrupt_task: "marked".to_owned(),
6432 };
6433 let next = advance_interrupt(state, &[], &[]);
6436 assert_eq!(
6437 next,
6438 Interrupt::Resuming {
6439 parked: vec!["running".to_owned()]
6440 },
6441 "abandoning the interrupt must not abandon the resume it owes"
6442 );
6443 }
6444
6445 #[test]
6448 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
6449 let state = Interrupt::Resuming {
6450 parked: vec!["running".to_owned()],
6451 };
6452 let next = advance_interrupt(state, &[], &[]);
6453 assert_eq!(
6454 next,
6455 Interrupt::Idle,
6456 "nothing is left to wait for; the loop must not stay wedged"
6457 );
6458 }
6459
6460 #[test]
6461 fn disabled_by_config_the_sequence_can_never_leave_idle() {
6462 let marked = interrupt_task("marked");
6463 let next = advance_interrupt_tick(
6464 false,
6465 Interrupt::Idle,
6466 &["running".to_owned()],
6467 std::slice::from_ref(&marked),
6468 );
6469 assert_eq!(
6470 next,
6471 Interrupt::Idle,
6472 "an unmarked, unconfigured daemon must behave exactly as before"
6473 );
6474 }
6475
6476 #[test]
6477 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
6478 let state = Interrupt::Parking {
6479 parked: vec!["running".to_owned()],
6480 interrupt_task: "marked".to_owned(),
6481 };
6482 let candidates = vec![interrupt_task("marked"), task()];
6483 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
6484 assert!(
6485 allowed.is_empty(),
6486 "nothing may dispatch - not even the interrupt task itself - \
6487 until the parked run has actually stopped"
6488 );
6489 }
6490
6491 #[test]
6505 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
6506 for state in [
6507 Interrupt::Parking {
6508 parked: vec!["running".to_owned()],
6509 interrupt_task: "marked".to_owned(),
6510 },
6511 Interrupt::Running {
6512 parked: vec!["running".to_owned()],
6513 interrupt_task: "marked".to_owned(),
6514 },
6515 Interrupt::Resuming {
6516 parked: vec!["running".to_owned()],
6517 },
6518 ] {
6519 let candidates = vec![
6520 interrupt_task("marked"),
6521 urgent_task("hot"),
6522 task_with_id("ordinary"),
6523 ];
6524 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
6525 assert!(
6526 !allowed.iter().any(|t| t.id == "hot"),
6527 "an urgent candidate must wait out the same gate as anything \
6528 else while the run it would run alongside has not actually \
6529 left flight, for state {state:?}: {allowed:?}"
6530 );
6531 }
6532 }
6533
6534 #[test]
6540 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
6541 let state = Interrupt::Resuming {
6542 parked: vec!["hot".to_owned()],
6543 };
6544 let candidates = vec![urgent_task("hot"), task()];
6545 let allowed = interrupt_gate(&state, &[], candidates);
6546 assert_eq!(
6547 allowed.iter().filter(|t| t.id == "hot").count(),
6548 1,
6549 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
6550 );
6551 }
6552
6553 #[test]
6554 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
6555 let state = Interrupt::Parking {
6556 parked: vec!["running".to_owned()],
6557 interrupt_task: "marked".to_owned(),
6558 };
6559 let other = task();
6560 let candidates = vec![interrupt_task("marked"), other.clone()];
6561 let allowed = interrupt_gate(&state, &[], candidates);
6562 assert_eq!(allowed.len(), 1);
6563 assert_eq!(allowed[0].id, "marked");
6564 }
6565
6566 #[test]
6567 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
6568 let state = Interrupt::Running {
6569 parked: vec!["running".to_owned()],
6570 interrupt_task: "marked".to_owned(),
6571 };
6572 let candidates = vec![task(), task()];
6573 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
6574 assert!(allowed.is_empty());
6575 }
6576
6577 #[test]
6584 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
6585 let state = Interrupt::Resuming {
6586 parked: vec!["a".to_owned(), "c".to_owned()],
6587 };
6588 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
6589 let allowed = interrupt_gate(&state, &[], candidates);
6590 assert_eq!(
6591 allowed.len(),
6592 1,
6593 "at most one candidate may be offered while resuming: {allowed:?}"
6594 );
6595 assert_eq!(allowed[0].id, "a");
6596 }
6597
6598 #[test]
6599 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
6600 let state = Interrupt::Resuming {
6601 parked: vec!["a".to_owned()],
6602 };
6603 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
6604 assert!(allowed.is_empty());
6605 }
6606
6607 #[test]
6613 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
6614 let running = task(); let marked = interrupt_task("marked");
6616
6617 let mut state = Interrupt::Idle;
6618 let in_flight = vec![running.id.clone()];
6620 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
6621 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
6622 assert!(gated.is_empty(), "still waiting on `running` to park");
6623
6624 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
6626 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
6627 assert_eq!(
6628 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6629 vec!["marked"],
6630 "only the interrupt task may be offered to the dispatcher now"
6631 );
6632
6633 state = advance_interrupt_tick(
6635 true,
6636 state,
6637 &["marked".to_owned()],
6638 std::slice::from_ref(&running),
6639 );
6640 let gated = interrupt_gate(
6641 &state,
6642 &["marked".to_owned()],
6643 vec![marked.clone(), running.clone()],
6644 );
6645 assert!(
6646 gated.is_empty(),
6647 "the parked run must not be offered back while the interrupt \
6648 task is still running"
6649 );
6650
6651 let other = task_with_id("other");
6655 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
6656 assert_eq!(
6657 state,
6658 Interrupt::Resuming {
6659 parked: vec![running.id.clone()]
6660 }
6661 );
6662 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
6663 assert_eq!(
6664 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6665 vec![running.id.as_str()],
6666 "exactly the parked run resumes - not the unrelated task, even \
6667 though it was offered first"
6668 );
6669
6670 state = advance_interrupt_tick(
6674 true,
6675 state,
6676 std::slice::from_ref(&running.id),
6677 std::slice::from_ref(&other),
6678 );
6679 assert_eq!(state, Interrupt::Idle);
6680 let gated = interrupt_gate(
6681 &state,
6682 std::slice::from_ref(&running.id),
6683 vec![other.clone()],
6684 );
6685 assert_eq!(
6686 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6687 vec![other.id.as_str()],
6688 "ordinary dispatch is unrestricted again"
6689 );
6690 }
6691
6692 #[test]
6693 fn every_run_status_settles_the_task_it_came_from() {
6694 let table = [
6696 (RunStatus::Merged, TaskStatus::Done, 1),
6697 (RunStatus::Ready, TaskStatus::Done, 1),
6698 (RunStatus::Stalled, TaskStatus::Failed, 0),
6699 (RunStatus::Blocked, TaskStatus::Failed, 1),
6700 (RunStatus::Failed, TaskStatus::Failed, 1),
6701 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
6702 (RunStatus::Prep, TaskStatus::Failed, 1),
6703 (RunStatus::Implementing, TaskStatus::Failed, 1),
6704 (RunStatus::Judging, TaskStatus::Failed, 1),
6705 (RunStatus::Deliberating, TaskStatus::Failed, 1),
6706 (RunStatus::Voting, TaskStatus::Failed, 1),
6707 (RunStatus::Reviewing, TaskStatus::Failed, 1),
6708 (RunStatus::Gating, TaskStatus::Failed, 1),
6709 ];
6710 for (run, want, attempts) in table {
6711 let mut t = task();
6712 t.start("20260902-000000-aaaa".to_owned());
6713 settle(
6714 &mut t,
6715 Verdict {
6716 status: run,
6717 left_pr: false,
6718 parked: false,
6719 quota_hit: matches!(run, RunStatus::Stalled),
6720 no_viable_candidates: false,
6721 },
6722 "why",
6723 2,
6724 );
6725 assert_eq!(t.status, want, "task status after {}", label(run));
6726 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
6727 }
6728 }
6729
6730 #[test]
6731 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
6732 let mut stalled = task();
6733 stalled.start("20260902-000000-aaaa".to_owned());
6734 settle(
6735 &mut stalled,
6736 Verdict {
6737 status: RunStatus::Stalled,
6738 left_pr: false,
6739 parked: false,
6740 quota_hit: true,
6741 no_viable_candidates: false,
6742 },
6743 "quota",
6744 1,
6745 );
6746 assert_eq!(stalled.attempts, 0);
6747 assert!(
6748 stalled.status.runnable(),
6749 "a machine problem must leave the task in line"
6750 );
6751
6752 let mut blocked = task();
6753 blocked.start("20260902-000000-aaaa".to_owned());
6754 settle(
6755 &mut blocked,
6756 Verdict {
6757 status: RunStatus::Blocked,
6758 left_pr: false,
6759 parked: false,
6760 quota_hit: false,
6761 no_viable_candidates: false,
6762 },
6763 "findings open",
6764 1,
6765 );
6766 assert_eq!(blocked.attempts, 1);
6767 assert_eq!(
6768 blocked.status,
6769 TaskStatus::Held,
6770 "the last attempt hands the task to a human"
6771 );
6772 }
6773
6774 #[test]
6775 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
6776 let mut delivered = task();
6779 delivered.start("20260903-080619-01c2".to_owned());
6780 settle(
6781 &mut delivered,
6782 Verdict {
6783 status: RunStatus::Blocked,
6784 left_pr: true,
6785 parked: false,
6786 quota_hit: false,
6787 no_viable_candidates: false,
6788 },
6789 "no check status",
6790 4,
6791 );
6792 assert_eq!(
6793 delivered.status,
6794 TaskStatus::Held,
6795 "a pull request waiting on CI or a person is not a retryable failure"
6796 );
6797 assert!(
6798 !delivered.status.runnable(),
6799 "the loop must not pick this task up again"
6800 );
6801 assert_eq!(
6802 delivered.last_error.as_deref(),
6803 Some("no check status"),
6804 "the operator needs to be told what the gate was waiting for"
6805 );
6806
6807 let mut empty_handed = task();
6810 empty_handed.start("20260903-080619-01c2".to_owned());
6811 settle(
6812 &mut empty_handed,
6813 Verdict {
6814 status: RunStatus::Blocked,
6815 left_pr: false,
6816 parked: false,
6817 quota_hit: false,
6818 no_viable_candidates: false,
6819 },
6820 "findings open",
6821 4,
6822 );
6823 assert_eq!(empty_handed.status, TaskStatus::Failed);
6824 assert!(empty_handed.status.runnable());
6825 }
6826
6827 #[test]
6828 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
6829 let mut noop = task();
6835 noop.start("20260912-131304-391f".to_owned());
6836 settle(
6837 &mut noop,
6838 Verdict {
6839 status: RunStatus::VerifiedNoop,
6840 left_pr: false,
6841 parked: false,
6842 quota_hit: false,
6843 no_viable_candidates: true,
6844 },
6845 "candidate A: already fixed by b32cfc4, on main",
6846 4,
6847 );
6848 assert_eq!(
6849 noop.status,
6850 TaskStatus::Held,
6851 "an unverified claim is a request for a human, not a failure"
6852 );
6853 assert!(
6854 !noop.status.runnable(),
6855 "the loop must not requeue this on the same unverified claim"
6856 );
6857 assert_eq!(noop.attempts, 1);
6862 }
6863
6864 #[test]
6865 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
6866 let mut parked = task();
6871 parked.start("20260903-183634-2d98".to_owned());
6872 settle(
6873 &mut parked,
6874 Verdict {
6875 status: RunStatus::Implementing,
6876 left_pr: false,
6877 quota_hit: false,
6878 parked: true,
6879 no_viable_candidates: false,
6880 },
6881 "parked after `implementing`",
6882 2,
6883 );
6884 assert_eq!(parked.attempts, 0, "a park is refunded");
6885 assert!(
6886 parked.status.runnable(),
6887 "and the task stays in line so the next loop resumes its run"
6888 );
6889 assert_eq!(parked.status, TaskStatus::Parked);
6890 assert_eq!(
6891 parked.park_reason.as_deref(),
6892 Some("parked after `implementing`"),
6893 "the card says where it stopped"
6894 );
6895 assert_eq!(parked.last_error, None, "nothing failed");
6896
6897 let mut broken = task();
6901 broken.start("20260903-183634-2d98".to_owned());
6902 settle(
6903 &mut broken,
6904 Verdict {
6905 status: RunStatus::Implementing,
6906 left_pr: false,
6907 quota_hit: false,
6908 parked: false,
6909 no_viable_candidates: false,
6910 },
6911 "returned mid-flight",
6912 2,
6913 );
6914 assert_eq!(broken.attempts, 1);
6915 }
6916
6917 #[test]
6918 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
6919 let mut flaky = task();
6924 flaky.start("20260903-123023-e633".to_owned());
6925 settle(
6926 &mut flaky,
6927 Verdict {
6928 status: RunStatus::Stalled,
6929 left_pr: false,
6930 parked: false,
6931 quota_hit: false,
6932 no_viable_candidates: false,
6933 },
6934 "verdict rests on 1 of 3 judges",
6935 2,
6936 );
6937 assert_eq!(
6938 flaky.attempts, 1,
6939 "flakiness spends an attempt, so `max_attempts` still bounds it"
6940 );
6941 assert!(flaky.status.runnable(), "and it is still worth retrying");
6942
6943 let mut limited = task();
6945 limited.start("20260903-123023-e633".to_owned());
6946 settle(
6947 &mut limited,
6948 Verdict {
6949 status: RunStatus::Stalled,
6950 left_pr: false,
6951 parked: false,
6952 quota_hit: true,
6953 no_viable_candidates: false,
6954 },
6955 "judge-2, judge-3 out of quota",
6956 2,
6957 );
6958 assert_eq!(limited.attempts, 0, "a quota window is refunded");
6959 assert!(limited.status.runnable());
6960
6961 let mut worn = task();
6964 for _ in 0..2 {
6965 worn.release();
6966 }
6967 worn.start("20260903-123023-e633".to_owned());
6968 worn.attempts = 2;
6969 settle(
6970 &mut worn,
6971 Verdict {
6972 status: RunStatus::Stalled,
6973 left_pr: false,
6974 parked: false,
6975 quota_hit: false,
6976 no_viable_candidates: false,
6977 },
6978 "no quorum again",
6979 2,
6980 );
6981 assert_eq!(worn.status, TaskStatus::Held);
6982 assert!(!worn.status.runnable());
6983 }
6984
6985 #[test]
6986 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
6987 let mut wiped_out = task();
6994 wiped_out.start("20260907-025000-a1b2".to_owned());
6995 settle(
6996 &mut wiped_out,
6997 Verdict {
6998 status: RunStatus::Failed,
6999 left_pr: false,
7000 parked: false,
7001 quota_hit: true,
7002 no_viable_candidates: true,
7003 },
7004 "no candidate produced a change; nothing to judge",
7005 2,
7006 );
7007 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
7008 assert!(
7009 wiped_out.status.runnable(),
7010 "a machine problem must leave the task in line"
7011 );
7012
7013 let mut partial_progress = task();
7019 partial_progress.start("20260907-025500-c3d4".to_owned());
7020 settle(
7021 &mut partial_progress,
7022 Verdict {
7023 status: RunStatus::Failed,
7024 left_pr: false,
7025 parked: false,
7026 quota_hit: true,
7027 no_viable_candidates: false,
7028 },
7029 "gate failed on the winning candidate",
7030 2,
7031 );
7032 assert_eq!(
7033 partial_progress.attempts, 1,
7034 "a candidate that actually produced a change spends the attempt \
7035 even though some other seat hit its quota"
7036 );
7037 assert!(partial_progress.status.runnable());
7038 }
7039
7040 #[test]
7041 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
7042 let mut t = task();
7048 t.start("20260907-025000-a1b2".to_owned());
7049 let mut state = run_state(RunStatus::Failed);
7050 state.quota.push(QuotaLoss {
7051 seat: "cand-a".to_owned(),
7052 node: "implement".to_owned(),
7053 at: Timestamp::now(),
7054 reset: None,
7055 });
7056 assert!(
7057 state.viable().is_empty(),
7058 "no candidate was added, so nothing is viable"
7059 );
7060 reclaim(&mut t, Some(state), 2, "en");
7061 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
7062 assert!(t.status.runnable());
7063 }
7064
7065 #[test]
7066 fn a_held_task_is_never_offered_to_the_loop() {
7067 let dir = tempfile::tempdir().unwrap();
7068 let queue = Queue::at(dir.path().to_path_buf());
7069 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
7070 let mut t = task();
7071 t.id = format!("2026090{n}-000000-000{n}");
7072 t.priority = priority;
7073 queue.put(&mut t).unwrap();
7074 }
7075 let mut held = task();
7076 held.id = "20260909-000000-9999".to_owned();
7077 held.priority = 99;
7078 held.hold_machine(None);
7079 queue.put(&mut held).unwrap();
7080
7081 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
7082 assert_eq!(order.len(), 3);
7083 assert!(!order.contains(&held.id));
7084 assert_eq!(
7085 order.first().cloned(),
7086 queue.next_runnable().map(|t| t.id),
7087 "the loop's first candidate is exactly what the queue offers"
7088 );
7089 assert_eq!(
7090 order,
7091 vec![
7092 "20260902-000000-0002".to_owned(),
7093 "20260903-000000-0003".to_owned(),
7094 "20260901-000000-0001".to_owned(),
7095 ],
7096 "priority first, then oldest, so nothing starves"
7097 );
7098 }
7099
7100 #[test]
7101 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
7102 let dir = tempfile::tempdir().unwrap();
7103 let queue = Queue::at(dir.path().to_path_buf());
7104 let mut old = task();
7105 old.id = "20260101-000000-old0".to_owned();
7106 queue.put(&mut old).unwrap();
7107 let mut fresh = task();
7108 fresh.id = "20260101-000000-new0".to_owned();
7109 queue.put(&mut fresh).unwrap();
7110
7111 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
7115 std::thread::sleep(Duration::from_millis(60));
7116 let live = queue.claim(&fresh.id).unwrap();
7117
7118 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7119 assert_eq!(swept, vec![old.id.clone()]);
7120 assert!(
7121 queue.claim(&old.id).is_ok(),
7122 "an unparseable lock older than the threshold is swept"
7123 );
7124 assert!(
7125 queue.claim(&fresh.id).is_err(),
7126 "a live pid protects its lock regardless of age"
7127 );
7128 drop(live);
7129 }
7130
7131 #[test]
7132 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
7133 let dir = tempfile::tempdir().unwrap();
7143 let queue = Queue::at(dir.path().to_path_buf());
7144 let mut t = task();
7145 t.id = "20260101-000000-live".to_owned();
7146 queue.put(&mut t).unwrap();
7147
7148 let claim = queue.claim(&t.id).unwrap();
7149 std::thread::sleep(Duration::from_millis(60));
7150
7151 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7152 assert!(
7153 swept.is_empty(),
7154 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
7155 );
7156 assert!(
7157 queue.claim(&t.id).is_err(),
7158 "the lock still protects its task"
7159 );
7160 drop(claim);
7161 }
7162
7163 fn injected_dead_pid() -> u32 {
7166 std::process::id().checked_add(1).unwrap_or(1)
7167 }
7168
7169 #[test]
7170 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
7171 let dir = tempfile::tempdir().unwrap();
7172 let queue = Queue::at(dir.path().to_path_buf());
7173 let mut t = task();
7174 t.id = "20260101-000000-dead".to_owned();
7175 queue.put(&mut t).unwrap();
7176 let dead_pid = injected_dead_pid();
7177
7178 std::fs::write(
7183 dir.path().join(format!("{}.lock", t.id)),
7184 dead_pid.to_string(),
7185 )
7186 .unwrap();
7187
7188 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7189 pid != dead_pid
7190 });
7191 assert_eq!(
7192 swept,
7193 vec![t.id.clone()],
7194 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
7195 );
7196 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
7197 }
7198
7199 #[test]
7200 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
7201 let dir = tempfile::tempdir().unwrap();
7202 let queue = Queue::at(dir.path().to_path_buf());
7203 let mut t = task();
7204 t.id = "20260101-000000-late".to_owned();
7205 queue.put(&mut t).unwrap();
7206 let dead_pid = injected_dead_pid();
7207
7208 assert!(
7211 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
7212 "nothing has claimed the task yet"
7213 );
7214
7215 std::fs::write(
7218 dir.path().join(format!("{}.lock", t.id)),
7219 dead_pid.to_string(),
7220 )
7221 .unwrap();
7222
7223 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7227 pid != dead_pid
7228 });
7229 assert_eq!(swept, vec![t.id.clone()]);
7230 }
7231
7232 #[test]
7233 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
7234 crate::run::pin_test_home();
7239 let dir = tempfile::tempdir().unwrap();
7240 let queue = Queue::at(dir.path().to_path_buf());
7241 let mut t = task();
7242 t.id = "20260101-000000-crsh".to_owned();
7243 t.status = TaskStatus::Running;
7244 t.attempts = 1;
7245 t.runs.push("20260904-000000-4043".to_owned());
7249 queue.put(&mut t).unwrap();
7250 let dead_pid = injected_dead_pid();
7251
7252 std::fs::write(
7255 dir.path().join(format!("{}.lock", t.id)),
7256 dead_pid.to_string(),
7257 )
7258 .unwrap();
7259
7260 assert!(reclaim_orphaned_running(&queue, 2).is_empty());
7266 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
7267
7268 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7269 pid != dead_pid
7270 });
7271 assert_eq!(swept, vec![t.id.clone()]);
7272
7273 let reclaimed = reclaim_orphaned_running(&queue, 2);
7274 assert_eq!(reclaimed, vec![t.id.clone()]);
7275 let after = queue.get(&t.id).unwrap();
7276 assert_eq!(
7277 after.status,
7278 TaskStatus::Held,
7279 "no run.json to recover from, so a human is asked"
7280 );
7281 assert_eq!(
7282 after.runs,
7283 vec!["20260904-000000-4043".to_owned()],
7284 "the crashed run's id is kept as evidence, not discarded"
7285 );
7286 }
7287
7288 #[test]
7289 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
7290 let dir = tempfile::tempdir().unwrap();
7291 let queue = Queue::at(dir.path().to_path_buf());
7292 let mut t = task();
7293 t.id = "20260101-000000-unknown".to_owned();
7294 queue.put(&mut t).unwrap();
7295 let dead_pid = injected_dead_pid();
7296 std::fs::write(
7297 dir.path().join(format!("{}.lock", t.id)),
7298 dead_pid.to_string(),
7299 )
7300 .unwrap();
7301
7302 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
7303 assert!(swept.is_empty(), "an unknown pid must keep its lock");
7304 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
7305 }
7306
7307 fn run_state_in(status: RunStatus, language: &str) -> RunState {
7308 let mut s = run_state(status);
7309 s.config.graph.language = language.to_owned();
7310 s
7311 }
7312
7313 fn unstarted_verdict(status: RunStatus) -> Verdict {
7314 Verdict {
7315 status,
7316 left_pr: false,
7317 quota_hit: false,
7318 parked: false,
7319 no_viable_candidates: false,
7320 }
7321 }
7322
7323 #[test]
7324 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
7325 let mut t = task();
7326 t.attempts = 1;
7327 settle_in(
7328 &mut t,
7329 unstarted_verdict(RunStatus::AlreadyInBase),
7330 "already in main",
7331 1,
7332 phrases("en"),
7333 );
7334 assert_eq!(t.status, TaskStatus::Done);
7335 assert_eq!(t.attempts, 0);
7336 }
7337
7338 #[test]
7339 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
7340 let reason = |language: &str| {
7341 let mut t = task();
7342 settle_in(
7343 &mut t,
7344 unstarted_verdict(RunStatus::Judging),
7345 "boom",
7346 1,
7347 phrases(language),
7348 );
7349 t.last_error.or(t.hold_reason).unwrap_or_default()
7350 };
7351 assert!(
7352 reason("en").starts_with("the graph stopped at `"),
7353 "{}",
7354 reason("en")
7355 );
7356 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
7357 assert!(reason("日本語").contains("boom"));
7358 assert_eq!(reason("fr"), reason("en"));
7359 }
7360
7361 #[test]
7362 fn describe_follows_the_run_language_and_keeps_the_run_id() {
7363 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
7364 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
7365 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
7366 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
7367 assert!(ja.contains("[run "), "{ja}");
7368 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
7369 assert!(ended.starts_with("run 終了: "), "{ended}");
7370 let mut de = run_state_in(RunStatus::Failed, "de");
7371 let mut en = run_state_in(RunStatus::Failed, "en");
7372 de.id = "same".to_owned();
7373 en.id = "same".to_owned();
7374 assert_eq!(describe(&de), describe(&en));
7375 }
7376
7377 #[test]
7378 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
7379 use crate::handover::Refused;
7380 let cases = [
7381 Refused::Foreign {
7382 branch: "b".into(),
7383 path: "/w/x".into(),
7384 why: "made by hand".into(),
7385 },
7386 Refused::Unsafe {
7387 branch: "b".into(),
7388 path: "/w/x".into(),
7389 why: "its worktree has uncommitted changes (a.rs)".into(),
7390 },
7391 Refused::ReleaseFailed {
7392 branch: "b".into(),
7393 path: "/w/x".into(),
7394 run: "ab12".into(),
7395 },
7396 ];
7397 for r in &cases {
7398 let en = (phrases("en").handover_refused)(r);
7399 assert_eq!(en, r.to_string());
7400 let ja = (phrases("ja").handover_refused)(r);
7401 assert!(
7402 !ja.contains("is checked out") && !ja.contains("try again"),
7403 "{ja}"
7404 );
7405 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
7406 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
7407 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
7408 }
7409 }
7410 assert!(phrases("en").handover_hint.contains("release the task"));
7411 assert!(phrases("ja").handover_hint.contains("解放"));
7412 }
7413
7414 #[test]
7415 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
7416 let build = |lang: &str| {
7417 let mut s = run_state_in(RunStatus::Blocked, lang);
7418 s.id = "same".to_owned();
7419 s.config.graph.review_rounds = 3;
7420 let mut r = review_round(3);
7421 r.expected = 3;
7422 r.answered = 1;
7423 s.reviews.push(r);
7424 s.event(
7425 "review",
7426 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
7427 );
7428 s
7429 };
7430 let en = describe(&build("en"));
7431 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
7432 let ja = describe(&build("ja"));
7433 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
7434 assert!(!ja.contains("never answered"), "{ja}");
7435 assert!(ja.contains("[run "), "{ja}");
7436
7437 let mut failed = build("ja");
7440 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
7441 command: "cargo test".to_owned(),
7442 code: Some(1),
7443 output_tail: String::new(),
7444 duration_ms: 0,
7445 resource_blocked: false,
7446 });
7447 failed.event("review", "stopped; e2e failed: cargo test");
7448 let ja = describe(&failed);
7449 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
7450 assert!(!ja.contains("席のレビュアー"), "{ja}");
7451 }
7452
7453 #[test]
7454 fn refusals_and_recovery_prose_follow_the_language() {
7455 let t = held_task_with("r1");
7456 let q = action_question(
7457 "r1",
7458 ask::ChoiceAction::Resume {
7459 run: "r1".to_owned(),
7460 },
7461 );
7462 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
7463 ActionDecision::Refuse(s) => s,
7464 other => panic!("{other:?}"),
7465 };
7466 assert!(refuse(phrases("en")).contains("could not be read: gone"));
7467 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
7468 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
7469
7470 let mut held = task();
7471 reclaim(&mut held, None, 2, "ja");
7472 assert!(held.hold_reason.unwrap().contains("保留にしました"));
7473 let mut held = task();
7474 reclaim(&mut held, None, 2, "xx");
7475 assert!(held.hold_reason.unwrap().contains("held for a human"));
7476 }
7477
7478 fn run_state(status: RunStatus) -> RunState {
7479 let mut state = RunState::new(
7480 PathBuf::from("/repo"),
7481 "main".to_owned(),
7482 "abc1234def".to_owned(),
7483 "add retries".to_owned(),
7484 Config::default(),
7485 );
7486 state.status = status;
7487 state
7488 }
7489
7490 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
7491 Candidate {
7492 index: 0,
7493 label,
7494 agent: "claude".to_owned(),
7495 branch: format!("magi/x/{label}"),
7496 worktree: PathBuf::from("/repo"),
7497 summary: summary.to_owned(),
7498 stat: String::new(),
7499 files: 0,
7500 commits: usize::from(!empty),
7501 empty,
7502 failed: failed.map(str::to_owned),
7503 verified_noop: None,
7504 duration_ms: 0,
7505 folded: false,
7506 }
7507 }
7508
7509 #[test]
7510 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
7511 let mut state = run_state(RunStatus::Blocked);
7512 state.gate = vec![
7513 CommandOutcome {
7514 command: "cargo make check".to_owned(),
7515 code: Some(0),
7516 output_tail: "ok".to_owned(),
7517 duration_ms: 0,
7518 resource_blocked: false,
7519 },
7520 CommandOutcome {
7521 command: "cargo test".to_owned(),
7522 code: Some(101),
7523 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
7524 duration_ms: 0,
7525 resource_blocked: false,
7526 },
7527 ];
7528 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
7529 assert!(d.contains("cargo test"), "{d}");
7530 assert!(
7531 !d.contains("cargo make check"),
7532 "a passing check is not a diagnostic: {d}"
7533 );
7534 assert!(d.contains("assertion failed"), "{d}");
7535 }
7536
7537 #[test]
7538 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
7539 let mut state = run_state(RunStatus::Blocked);
7540 state.event(
7541 "land",
7542 "stopped: the fixer produced no commit while 2 check(s) were failing \
7543 (build, lint); stopping instead of looping on an unchanged tree",
7544 );
7545 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
7546 assert!(d.contains("build"), "{d}");
7547 assert!(d.contains("lint"), "{d}");
7548 assert!(d.contains("fixer produced no commit"), "{d}");
7549 }
7550
7551 #[test]
7552 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
7553 let state = run_state(RunStatus::VerifiedNoop);
7558 let d = describe(&state);
7559 assert!(
7560 d.contains("agent-verified no-op"),
7561 "expected the display label, not the wire spelling: {d}"
7562 );
7563 assert!(!d.contains("verified_noop"), "{d}");
7564 }
7565
7566 #[test]
7567 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
7568 let mut state = run_state(RunStatus::Failed);
7574 state.candidates = vec![candidate(
7575 'A',
7576 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
7577 true,
7578 None,
7579 )];
7580 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
7581 assert!(d.contains("candidate A"), "{d}");
7582 assert!(d.contains("tagged v1.2.3"), "{d}");
7583 }
7584
7585 #[test]
7586 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
7587 let mut state = run_state(RunStatus::Failed);
7588 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
7589 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
7590 assert!(d.contains("candidate A"), "{d}");
7591 assert!(d.contains("agent timed out"), "{d}");
7592 }
7593
7594 #[test]
7595 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
7596 let mut state = run_state(RunStatus::Failed);
7599 state.candidates = vec![candidate('A', "did the work", false, None)];
7600 assert!(diagnostic(&state).is_none());
7601 }
7602
7603 #[test]
7604 fn diagnostic_is_bounded_however_much_a_run_printed() {
7605 let mut state = run_state(RunStatus::Blocked);
7606 state.gate = vec![
7607 CommandOutcome {
7608 command: "cargo test".to_owned(),
7609 code: Some(101),
7610 output_tail: "x".repeat(50_000),
7611 duration_ms: 0,
7612 resource_blocked: false,
7613 },
7614 CommandOutcome {
7615 command: "cargo clippy".to_owned(),
7616 code: Some(1),
7617 output_tail: "y".repeat(50_000),
7618 duration_ms: 0,
7619 resource_blocked: false,
7620 },
7621 ];
7622 state.candidates = vec![
7623 candidate('A', &"z".repeat(50_000), true, None),
7624 candidate('B', &"w".repeat(50_000), true, None),
7625 ];
7626 let d = diagnostic(&state).expect("plenty here to diagnose");
7627 assert!(
7628 d.len() <= DIAGNOSTIC_MAX,
7629 "diagnostic grew to {} bytes, unbounded",
7630 d.len()
7631 );
7632 }
7633
7634 #[test]
7635 fn selected_delivery_guards_survive_recovery_failure_accounting() {
7636 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
7637 let mut t = task();
7638 let mut state = run_state(status);
7639 t.pr_recovery = Some(crate::pr_recovery::Selection {
7640 url: "url".into(),
7641 branch: format!("magi/task-{}", t.id),
7642 head: "a".repeat(40),
7643 source_head: None,
7644 source_run: None,
7645 source_proof: None,
7646 disposition: Some(crate::pr_recovery::Disposition::Land),
7647 instruction: t.instruction.clone(),
7648 approved_at: Timestamp::now(),
7649 });
7650 state.pr_recovery = t.pr_recovery.clone();
7651 t.start(state.id.clone());
7652 settle_and_diagnose(
7653 &mut t,
7654 Verdict {
7655 status,
7656 left_pr: false,
7657 quota_hit: false,
7658 parked: false,
7659 no_viable_candidates: false,
7660 },
7661 "delivery stopped",
7662 5,
7663 &state,
7664 );
7665 assert_eq!(t.status, TaskStatus::Held);
7666 let reason = t.hold_reason.as_deref().unwrap();
7667 if status == RunStatus::Ready {
7668 assert!(reason.contains("Selected PR delivery is incomplete"));
7669 assert!(t.recovery_budget.is_none());
7670 } else {
7671 assert!(reason.contains("Selected recovery stopped"));
7672 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
7673 }
7674 assert!(t.pr_recovery.is_some());
7675 }
7676 }
7677
7678 #[test]
7679 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
7680 let mut state = run_state(RunStatus::Blocked);
7681 state.gate = vec![CommandOutcome {
7682 command: "cargo test".to_owned(),
7683 code: Some(101),
7684 output_tail: "assertion failed".to_owned(),
7685 duration_ms: 0,
7686 resource_blocked: false,
7687 }];
7688 let verdict = Verdict {
7689 status: RunStatus::Blocked,
7690 left_pr: false,
7691 quota_hit: false,
7692 parked: false,
7693 no_viable_candidates: false,
7694 };
7695
7696 let mut t = task();
7699 t.start("run-1".to_owned());
7700 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
7701 assert_eq!(t.status, TaskStatus::Failed);
7702 assert!(t.diagnostic.is_none());
7703
7704 t.start("run-2".to_owned());
7707 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
7708 assert_eq!(t.status, TaskStatus::Held);
7709 let d = t.diagnostic.expect("a held task must carry its diagnostic");
7710 assert!(d.contains("cargo test"), "{d}");
7711 }
7712
7713 #[test]
7714 fn a_held_task_names_the_open_question_it_is_waiting_on() {
7715 crate::run::pin_test_home();
7720 let home = crate::run::home();
7721 let state = run_state(RunStatus::VerifiedNoop);
7722 let mut q = ask::Question::new(
7723 state.id.clone(),
7724 "implement".to_owned(),
7725 "impl-A".to_owned(),
7726 "is this really a no-op?".to_owned(),
7727 String::new(),
7728 Vec::new(),
7729 );
7730 Questions::at(home.join("questions")).put(&mut q).unwrap();
7731
7732 let verdict = Verdict {
7733 status: RunStatus::VerifiedNoop,
7734 left_pr: false,
7735 quota_hit: false,
7736 parked: false,
7737 no_viable_candidates: false,
7738 };
7739 let mut t = task();
7740 t.start(state.id.clone());
7741 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
7742
7743 assert_eq!(t.status, TaskStatus::Held);
7744 let reason = t.hold_reason.expect("a held task must record why");
7745 assert!(
7746 reason.starts_with("run ended agent-verified no-op"),
7747 "the original settle reason must survive unchanged: {reason}"
7748 );
7749 assert!(
7750 reason.contains(q.short()),
7751 "the open question's id must be named so the notice is actionable: {reason}"
7752 );
7753 }
7754
7755 #[test]
7756 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
7757 crate::run::pin_test_home();
7758 let state = run_state(RunStatus::VerifiedNoop);
7759
7760 let verdict = Verdict {
7761 status: RunStatus::VerifiedNoop,
7762 left_pr: false,
7763 quota_hit: false,
7764 parked: false,
7765 no_viable_candidates: false,
7766 };
7767 let mut t = task();
7768 t.start(state.id.clone());
7769 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
7770
7771 assert_eq!(t.status, TaskStatus::Held);
7772 assert_eq!(
7773 t.hold_reason.as_deref(),
7774 Some("run ended agent-verified no-op"),
7775 "nothing to append when the question was already answered or never asked"
7776 );
7777 }
7778
7779 #[test]
7780 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
7781 crate::run::pin_test_home();
7782 let mut first = run_state(RunStatus::Blocked);
7783 first.id = "20260101-000000-sup1".to_owned();
7784 first.save().unwrap();
7785 let mut second = run_state(RunStatus::Merged);
7786 second.id = "20260101-000000-sup2".to_owned();
7787 second.save().unwrap();
7788
7789 let mut t = task();
7790 t.runs = vec![first.id.clone(), second.id.clone()];
7791 t.status = TaskStatus::Done;
7792
7793 supersede_prior_runs(&t, &crate::run::home());
7794
7795 assert_eq!(
7796 RunState::load(&first.id).unwrap().status,
7797 RunStatus::Superseded,
7798 "the first attempt's Blocked no longer needs anyone's attention"
7799 );
7800 assert_eq!(
7801 RunState::load(&second.id).unwrap().status,
7802 RunStatus::Merged,
7803 "the run that actually succeeded is left exactly as it was"
7804 );
7805 }
7806
7807 #[test]
7808 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
7809 crate::run::pin_test_home();
7815 let mut first = run_state(RunStatus::Blocked);
7816 first.id = "20260101-000000-sup9".to_owned();
7817 first.driver_pid = Some(std::process::id());
7820 first.driver_started_at = Some(
7821 crate::proc::process_started_at(std::process::id())
7822 .expect("this test process's own start time must be queryable"),
7823 );
7824 first.save().unwrap();
7825 let mut second = run_state(RunStatus::Merged);
7826 second.id = "20260101-000000-supa".to_owned();
7827 second.save().unwrap();
7828
7829 let mut t = task();
7830 t.runs = vec![first.id.clone(), second.id.clone()];
7831 t.status = TaskStatus::Done;
7832
7833 supersede_prior_runs(&t, &crate::run::home());
7834
7835 assert_eq!(
7836 RunState::load(&first.id).unwrap().status,
7837 RunStatus::Blocked,
7838 "a live driver_pid means something is still actually working this run, \
7839 even though no daemon claims it - rewriting under it would just be \
7840 undone the next time that process saves"
7841 );
7842 }
7843
7844 #[test]
7845 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
7846 let dir = tempfile::tempdir().unwrap();
7852 let home = dir.path().to_path_buf();
7853 let queue = Queue::at(dir.path().join("queue"));
7854
7855 let mut first = run_state(RunStatus::Blocked);
7856 first.id = "20260101-000000-supd".to_owned();
7857 first.driver_pid = Some(std::process::id());
7858 first.driver_started_at = Some(
7859 crate::proc::process_started_at(std::process::id())
7860 .expect("this test process's own start time must be queryable"),
7861 );
7862 first.save_under(&home).unwrap();
7863 let mut second = run_state(RunStatus::Merged);
7864 second.id = "20260101-000000-supe".to_owned();
7865 second.save_under(&home).unwrap();
7866
7867 let mut t = task();
7868 t.runs = vec![first.id.clone(), second.id.clone()];
7869 t.status = TaskStatus::Done;
7870 queue.put(&mut t).unwrap();
7871
7872 resweep_superseded_attempts(&queue, &home);
7873 assert_eq!(
7874 RunState::load_under(&first.id, &home).unwrap().status,
7875 RunStatus::Blocked,
7876 "still live on the first pass, so still untouched"
7877 );
7878
7879 let mut stale = RunState::load_under(&first.id, &home).unwrap();
7885 stale.driver_started_at = Some("1".to_owned());
7886 stale.save_under(&home).unwrap();
7887
7888 resweep_superseded_attempts(&queue, &home);
7889 assert_eq!(
7890 RunState::load_under(&first.id, &home).unwrap().status,
7891 RunStatus::Superseded,
7892 "the second pass catches up what the first one correctly skipped"
7893 );
7894 }
7895
7896 #[test]
7897 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
7898 crate::run::pin_test_home();
7899 let mut first = run_state(RunStatus::Blocked);
7900 first.id = "20260101-000000-sup3".to_owned();
7901 first.save().unwrap();
7902 let mut second = run_state(RunStatus::Blocked);
7903 second.id = "20260101-000000-sup4".to_owned();
7904 second.save().unwrap();
7905
7906 let mut t = task();
7907 t.runs = vec![first.id.clone(), second.id.clone()];
7908 t.status = TaskStatus::Failed;
7912
7913 supersede_prior_runs(&t, &crate::run::home());
7914
7915 assert_eq!(
7916 RunState::load(&first.id).unwrap().status,
7917 RunStatus::Blocked
7918 );
7919 assert_eq!(
7920 RunState::load(&second.id).unwrap().status,
7921 RunStatus::Blocked
7922 );
7923 }
7924
7925 #[test]
7926 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
7927 crate::run::pin_test_home();
7931 let mut first = run_state(RunStatus::Blocked);
7932 first.id = "20260101-000000-sup5".to_owned();
7933 first.save().unwrap();
7934
7935 let mut t = task();
7936 t.runs = vec![first.id.clone()];
7937 t.status = TaskStatus::Done;
7938
7939 supersede_prior_runs(&t, &crate::run::home());
7940
7941 assert_eq!(
7942 RunState::load(&first.id).unwrap().status,
7943 RunStatus::Blocked,
7944 "a single-attempt task has no earlier run to supersede"
7945 );
7946 }
7947
7948 #[test]
7949 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
7950 crate::run::pin_test_home();
7957 let mut first = run_state(RunStatus::Blocked);
7958 first.id = "20260101-000000-supb".to_owned();
7959 first.save().unwrap();
7960 let mut second = run_state(RunStatus::Failed);
7961 second.id = "20260101-000000-supc".to_owned();
7962 second.save().unwrap();
7963
7964 let mut t = task();
7965 t.runs = vec![first.id.clone(), second.id.clone()];
7966 t.status = TaskStatus::Done;
7967
7968 supersede_prior_runs(&t, &crate::run::home());
7969
7970 assert_eq!(
7971 RunState::load(&first.id).unwrap().status,
7972 RunStatus::Blocked,
7973 "the task's last attempt never landed, so there is nothing here \
7974 actually superseding it"
7975 );
7976 }
7977
7978 #[test]
7979 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
7980 crate::run::pin_test_home();
7984 let mut failed = run_state(RunStatus::Failed);
7985 failed.id = "20260101-000000-sup6".to_owned();
7986 failed.save().unwrap();
7987 let mut noop = run_state(RunStatus::VerifiedNoop);
7988 noop.id = "20260101-000000-sup7".to_owned();
7989 noop.save().unwrap();
7990 let mut winner = run_state(RunStatus::Ready);
7991 winner.id = "20260101-000000-sup8".to_owned();
7992 winner.save().unwrap();
7993
7994 let mut t = task();
7995 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
7996 t.status = TaskStatus::Done;
7997
7998 supersede_prior_runs(&t, &crate::run::home());
7999
8000 assert_eq!(
8001 RunState::load(&failed.id).unwrap().status,
8002 RunStatus::Failed
8003 );
8004 assert_eq!(
8005 RunState::load(&noop.id).unwrap().status,
8006 RunStatus::VerifiedNoop
8007 );
8008 }
8009
8010 fn approval_question(run: &str) -> ask::Question {
8011 ask::Question::new(
8012 run.to_owned(),
8013 land::APPROVAL_NODE.to_owned(),
8014 "land".to_owned(),
8015 "merge?".to_owned(),
8016 String::new(),
8017 vec!["merge".to_owned(), "hold".to_owned()],
8018 )
8019 }
8020
8021 #[test]
8022 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
8023 crate::run::pin_test_home();
8024 let mut state = run_state(RunStatus::Landing);
8025 state.id = "20260101-000000-fre1".to_owned();
8026 state.parked = true;
8027 state.save().unwrap();
8028 ask::Questions::open()
8029 .put(&mut approval_question(&state.id))
8030 .unwrap();
8031
8032 let mut t = task();
8033 t.runs.push(state.id.clone());
8034 assert_eq!(
8035 land_resume_state(&t),
8036 LandResume::StillWaiting,
8037 "nobody has answered and the timeout has not passed"
8038 );
8039 }
8040
8041 #[test]
8042 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
8043 crate::run::pin_test_home();
8044 let mut state = run_state(RunStatus::Gating);
8045 state.id = "20260101-000000-gt01".to_owned();
8046 state.parked = true;
8047 state.config.graph.answer_timeout = 60;
8048 state.github_text = Some(crate::run::GithubTextAsk {
8049 rewrite: None,
8050 fingerprint: "f".into(),
8051 title: true,
8052 body: false,
8053 categories: vec!["title-language".into()],
8054 question: None,
8055 asks: 1,
8056 resolved: false,
8057 chosen_title: None,
8058 choice: None,
8059 });
8060 state.save().unwrap();
8061 let store = ask::Questions::open();
8062 let mut q = ask::Question::new(
8063 state.id.clone(),
8064 crate::github_text::ASK_NODE.to_owned(),
8065 crate::github_text::ASK_SEAT.to_owned(),
8066 "s".into(),
8067 String::new(),
8068 vec!["use fallback".into()],
8069 );
8070 store.put(&mut q).unwrap();
8071 let mut t = task();
8072 t.runs.push(state.id.clone());
8073 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
8074 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8075 store.put(&mut q).unwrap();
8076 assert_eq!(land_resume_state(&t), LandResume::Ready);
8077 assert!(!store.get(&q.id).unwrap().status.open());
8078 state.github_text.as_mut().unwrap().resolved = true;
8080 state.save().unwrap();
8081 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
8082 }
8083
8084 #[test]
8085 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
8086 crate::run::pin_test_home();
8091 let mut state = run_state(RunStatus::Landing);
8092 state.id = "20260101-000000-exp1".to_owned();
8093 state.parked = true;
8094 state.config.graph.answer_timeout = 60;
8095 state.save().unwrap();
8096
8097 let store = ask::Questions::open();
8098 let mut q = approval_question(&state.id);
8099 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8100 store.put(&mut q).unwrap();
8101
8102 let mut t = task();
8103 t.runs.push(state.id.clone());
8104 assert_eq!(
8105 land_resume_state(&t),
8106 LandResume::Ready,
8107 "an expired question must not be waited on forever"
8108 );
8109
8110 let after = store.get(&q.id).unwrap();
8111 assert!(
8112 !after.status.open(),
8113 "the question is abandoned, not silently ignored"
8114 );
8115 assert!(
8116 after.resolution().is_none(),
8117 "an abandoned question is not read as a decision"
8118 );
8119 }
8120
8121 #[test]
8122 fn reclaim_settles_a_running_task_against_its_last_run() {
8123 let mut t = task();
8124 t.start("20260904-000000-4043".to_owned());
8125 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
8126 assert_eq!(
8127 t.status,
8128 TaskStatus::Done,
8129 "a run that actually finished must not stay `running` forever"
8130 );
8131 }
8132
8133 #[test]
8134 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
8135 let mut t = task();
8136 t.start("20260904-000000-4043".to_owned());
8137 t.last_error = Some("earlier trouble".to_owned());
8138 let mut state = run_state(RunStatus::Judging);
8139 state.parked = true;
8140 reclaim(&mut t, Some(state), 1, "en");
8141 assert_eq!(t.status, TaskStatus::Parked);
8142 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
8143 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8144 assert!(t.park_reason.is_some());
8145 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
8146 assert!(!t.fresh_start);
8147 assert!(t.status.runnable());
8148 }
8149
8150 #[test]
8151 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
8152 let mut t = task();
8153 t.start("20260904-000000-4043".to_owned());
8154 t.last_error = Some("earlier trouble".to_owned());
8155 settle(
8156 &mut t,
8157 Verdict {
8158 status: RunStatus::Judging,
8159 left_pr: false,
8160 quota_hit: false,
8161 parked: true,
8162 no_viable_candidates: false,
8163 },
8164 "parked after `judging`",
8165 1,
8166 );
8167 assert_eq!(t.status, TaskStatus::Parked);
8168 assert_eq!(t.attempts, 0);
8169 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8170 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
8171 assert_eq!(t.schema, crate::queue::SCHEMA);
8172 assert!(
8173 crate::notices::task_held(&t).is_none(),
8174 "a park is not news"
8175 );
8176 t.start("20260904-000000-4043".to_owned());
8177 assert_eq!(t.park_reason, None);
8178 }
8179
8180 #[test]
8181 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
8182 let mut t = task();
8186 t.start("20260904-000000-4043".to_owned());
8187 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
8188 assert_eq!(t.status, TaskStatus::Failed);
8189 assert!(t.status.runnable());
8190 }
8191
8192 #[test]
8193 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
8194 let mut t = task();
8195 t.start("20260904-000000-4043".to_owned());
8196 assert_eq!(t.attempts, 1);
8197 let mut run = run_state(RunStatus::Implementing);
8198 run.id = "20260904-000000-4043".to_owned();
8199 run.begin_pass();
8200 let cause = crate::run::ParkCause::Upgrade {
8201 from: "0.121.0".to_owned(),
8202 to: Some("0.121.1".to_owned()),
8203 };
8204 run.end_pass_parked(cause.clone());
8205 run.parked = true;
8206 let verdict = Verdict {
8207 status: run.status,
8208 left_pr: false,
8209 quota_hit: false,
8210 parked: true,
8211 no_viable_candidates: false,
8212 };
8213 let mut orphan = t.clone();
8216 reclaim(&mut orphan, Some(run.clone()), 2, "en");
8217 assert_eq!(orphan.status, TaskStatus::Parked);
8218 assert_eq!(orphan.attempts, 0);
8219 assert_eq!(
8220 orphan.upgrade_resume_key(),
8221 Some((run.passes[0].started_at, orphan.id.as_str()))
8222 );
8223 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
8224 assert_eq!(t.attempts, 0);
8225 assert_eq!(t.passes.len(), 1);
8226 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8227 assert_eq!(t.passes[0].parked, Some(cause));
8228 assert_eq!(t.passes[0].pass_index, 0);
8229 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
8230 assert!(t.upgrade_resume_key().is_some());
8231
8232 t.start("20260904-000000-4043".to_owned());
8235 run.begin_pass();
8236 run.status = RunStatus::Failed;
8237 let verdict = Verdict {
8238 status: RunStatus::Failed,
8239 left_pr: false,
8240 quota_hit: false,
8241 parked: false,
8242 no_viable_candidates: false,
8243 };
8244 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
8245 assert_eq!(t.passes.len(), 2);
8246 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8247 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
8248 assert_eq!(t.passes[1].pass_index, 1);
8249 assert_eq!(t.attempts, 1);
8250 }
8251
8252 #[test]
8253 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
8254 let mut t = task();
8255 t.start("20260904-000000-4043".to_owned());
8256 reclaim(&mut t, None, 2, "en");
8257 assert_eq!(t.status, TaskStatus::Held);
8258 assert!(
8259 t.last_error
8260 .as_deref()
8261 .is_some_and(|e| e.contains("running")),
8262 "the operator needs to know why this task was held"
8263 );
8264 }
8265
8266 #[test]
8267 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
8268 let dir = tempfile::tempdir().unwrap();
8269 let queue = Queue::at(dir.path().to_path_buf());
8270
8271 let mut orphaned = task();
8273 orphaned.id = "20260904-000000-orph".to_owned();
8274 orphaned.status = TaskStatus::Running;
8275 orphaned.attempts = 1;
8276 queue.put(&mut orphaned).unwrap();
8277
8278 let mut alive = task();
8279 alive.id = "20260904-000000-live".to_owned();
8280 alive.status = TaskStatus::Running;
8281 alive.attempts = 1;
8282 queue.put(&mut alive).unwrap();
8283 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
8284
8285 let mut queued = task();
8286 queued.id = "20260904-000000-wait".to_owned();
8287 queue.put(&mut queued).unwrap();
8288
8289 let reclaimed = reclaim_orphaned_running(&queue, 2);
8290 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
8291
8292 assert_eq!(
8293 queue.get(&orphaned.id).unwrap().status,
8294 TaskStatus::Held,
8295 "nothing was driving it and there was no run to recover"
8296 );
8297 assert_eq!(
8298 queue.get(&alive.id).unwrap().status,
8299 TaskStatus::Running,
8300 "a live claim must protect the task it belongs to"
8301 );
8302 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
8303 }
8304
8305 fn read_run_under(home: &Path, id: &str) -> RunState {
8311 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
8312 serde_json::from_str(&body).unwrap()
8313 }
8314
8315 #[test]
8316 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
8317 let dir = tempfile::tempdir().unwrap();
8318 let home = dir.path().to_path_buf();
8319 let now = Timestamp::now();
8320 let overrun_seat = || crate::run::ActiveSeat {
8321 node: "implement".to_owned(),
8322 started_at: now - jiff::SignedDuration::new(21_000, 0),
8323 timeout_secs: 3_600,
8324 attempt: 0,
8325 task: None,
8326 command: None,
8327 index: None,
8328 total: None,
8329 };
8330
8331 let mut dead = run_state(RunStatus::Implementing);
8332 dead.id = "20260101-000000-dead".to_owned();
8333 dead.active.insert("impl-A".to_owned(), overrun_seat());
8334 dead.driver_pid = Some(4242);
8337 dead.save_under(&home).unwrap();
8338
8339 let mut alive = run_state(RunStatus::Implementing);
8342 alive.id = "20260101-000000-aliv".to_owned();
8343 alive.active.insert("impl-A".to_owned(), overrun_seat());
8344 alive.save_under(&home).unwrap();
8345 let mut status = Status::new();
8346 status.current = vec![Current {
8347 task: "20260101-000000-task".to_owned(),
8348 run: alive.id.clone(),
8349 }];
8350 write_status_to(&home.join("daemon.json"), &status).unwrap();
8351
8352 let questions = Questions::at(home.join("questions"));
8356 let mut q = ask::Question::new(
8357 dead.id.clone(),
8358 "implement".to_owned(),
8359 "impl-A".to_owned(),
8360 "Which storage backend?".to_owned(),
8361 String::new(),
8362 vec!["SQLite".to_owned(), "Redis".to_owned()],
8363 );
8364 questions.put(&mut q).unwrap();
8365
8366 let abandoned = reclaim_abandoned_runs_with(
8367 &home,
8368 now,
8369 |pid| if pid == 4242 { Some(false) } else { None },
8370 |_| panic!("a query answering Dead outright needs no identity corroboration"),
8371 );
8372 assert_eq!(abandoned, vec![dead.id.clone()]);
8373
8374 let reloaded = read_run_under(&home, &dead.id);
8375 assert_eq!(reloaded.status, RunStatus::Failed);
8376 assert!(reloaded.active.is_empty());
8377 assert!(
8378 !questions.get(&q.id).unwrap().status.open(),
8379 "the failed run's own open question must be settled in the same pass"
8380 );
8381
8382 let still_alive = read_run_under(&home, &alive.id);
8383 assert_eq!(
8384 still_alive.status,
8385 RunStatus::Implementing,
8386 "a live daemon's claim protects it"
8387 );
8388 assert!(!still_alive.active.is_empty());
8389 }
8390
8391 #[test]
8401 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
8402 let dir = tempfile::tempdir().unwrap();
8403 let home = dir.path().to_path_buf();
8404 let now = Timestamp::now();
8405
8406 let mut manual = run_state(RunStatus::Reviewing);
8407 manual.id = "20260101-000000-manl".to_owned();
8408 manual.active.insert(
8409 "review-1".to_owned(),
8410 crate::run::ActiveSeat {
8411 node: "review".to_owned(),
8412 started_at: now - jiff::SignedDuration::new(21_000, 0),
8413 timeout_secs: 3_600,
8414 attempt: 0,
8415 task: None,
8416 command: None,
8417 index: None,
8418 total: None,
8419 },
8420 );
8421 manual.driver_pid = Some(4242);
8425 manual.driver_started_at = Some("1790000000".to_owned());
8426 manual.save_under(&home).unwrap();
8427
8428 let abandoned = reclaim_abandoned_runs_with(
8429 &home,
8430 now,
8431 |pid| if pid == 4242 { Some(true) } else { None },
8432 |pid| {
8433 if pid == 4242 {
8434 Some("1790000000".to_owned())
8435 } else {
8436 None
8437 }
8438 },
8439 );
8440 assert!(
8441 abandoned.is_empty(),
8442 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
8443 );
8444
8445 let reloaded = read_run_under(&home, &manual.id);
8446 assert_eq!(reloaded.status, RunStatus::Reviewing);
8447 assert!(!reloaded.active.is_empty());
8448 }
8449
8450 #[test]
8451 fn an_already_claimed_task_is_skipped_rather_than_failed() {
8452 let dir = tempfile::tempdir().unwrap();
8453 let queue = Queue::at(dir.path().to_path_buf());
8454 let mut only = task();
8455 queue.put(&mut only).unwrap();
8456
8457 let _elsewhere = queue.claim(&only.id).unwrap();
8458 let candidates = runnable(&queue);
8459 assert_eq!(candidates.len(), 1, "the task is still runnable");
8460 assert!(
8461 queue.claim(&candidates[0].id).is_err(),
8462 "the loop cannot take a claim somebody else holds"
8463 );
8464
8465 let after = queue.get(&only.id).unwrap();
8466 assert_eq!(after.status, TaskStatus::Queued);
8467 assert_eq!(
8468 after.attempts, 0,
8469 "losing the race is not an attempt at the task"
8470 );
8471 assert_eq!(after.last_error, None);
8472 }
8473
8474 #[test]
8475 fn the_status_file_round_trips_and_its_heartbeat_advances() {
8476 let dir = tempfile::tempdir().unwrap();
8477 let path = dir.path().join("daemon.json");
8478
8479 let mut status = Status::new();
8480 status.idle = false;
8481 status.completed = 7;
8482 status.current = vec![Current {
8483 task: "20260902-000000-t111".to_owned(),
8484 run: "20260902-000001-r111".to_owned(),
8485 }];
8486 write_status_to(&path, &status).unwrap();
8487 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
8488 assert_eq!(first.schema, SCHEMA);
8489 assert_eq!(first.pid, std::process::id());
8490 assert!(!first.idle);
8491 assert_eq!(first.completed, 7);
8492 assert_eq!(first.current, status.current);
8493 assert!(
8494 !path.with_extension("json.tmp").exists(),
8495 "the temp file is renamed, not left behind"
8496 );
8497
8498 std::thread::sleep(Duration::from_millis(5));
8499 status.updated_at = Timestamp::now();
8500 status.polls = 3;
8501 write_status_to(&path, &status).unwrap();
8502 let second: Status =
8503 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
8504 assert!(
8505 second.updated_at > first.updated_at,
8506 "a reader can only detect staleness if the heartbeat moves"
8507 );
8508 assert_eq!(
8509 second.started_at, first.started_at,
8510 "the start time is not a heartbeat"
8511 );
8512 assert_eq!(second.polls, 3);
8513 }
8514
8515 #[test]
8516 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
8517 let dir = tempfile::tempdir().unwrap();
8518
8519 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
8520
8521 let mut status = Status::new();
8522 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
8523 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8524 let stale = read_status(dir.path()).unwrap();
8525 assert!(
8526 !stale.running(Timestamp::now()),
8527 "a minute without a heartbeat is a dead daemon, not a busy one"
8528 );
8529 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
8530
8531 status.updated_at = Timestamp::now();
8532 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8533 let fresh = read_status(dir.path()).unwrap();
8534 assert!(fresh.running(Timestamp::now()));
8535 }
8536
8537 #[test]
8538 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
8539 let dir = tempfile::tempdir().unwrap();
8540 let now = Timestamp::now();
8541 let mine = "20260903-080619-01c2";
8542
8543 assert!(
8544 !is_working_on(dir.path(), mine, now),
8545 "no status file means nobody is working on anything"
8546 );
8547
8548 let mut status = Status::new();
8549 status.current = vec![Current {
8550 task: "20260903-080340-0167".to_owned(),
8551 run: mine.to_owned(),
8552 }];
8553 status.updated_at = now;
8554 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8555 assert!(is_working_on(dir.path(), mine, now));
8556 assert!(
8557 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
8558 "a daemon busy with one run is not working on another"
8559 );
8560
8561 status.updated_at = now - jiff::SignedDuration::from_secs(600);
8564 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8565 assert!(
8566 !is_working_on(dir.path(), mine, now),
8567 "a stale heartbeat is a dead daemon, so its run is a leftover"
8568 );
8569 }
8570
8571 #[test]
8572 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
8573 let dir = tempfile::tempdir().unwrap();
8574 let now = Timestamp::now();
8575
8576 assert!(
8577 !is_working_on_short(dir.path(), "01c2", now),
8578 "no status file means nobody is working on anything"
8579 );
8580
8581 let mut status = Status::new();
8582 status.current = vec![Current {
8583 task: "20260903-080340-0167".to_owned(),
8584 run: "20260903-080619-01c2".to_owned(),
8585 }];
8586 status.updated_at = now;
8587 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8588 assert!(
8589 is_working_on_short(dir.path(), "01c2", now),
8590 "the run's short id is the last block of its full id"
8591 );
8592 assert!(
8593 !is_working_on_short(dir.path(), "3cbf", now),
8594 "a daemon busy with one worktree bay is not working on another"
8595 );
8596 }
8597
8598 #[test]
8599 fn a_newer_status_file_still_yields_a_reading() {
8600 let dir = tempfile::tempdir().unwrap();
8601 std::fs::write(
8604 dir.path().join("daemon.json"),
8605 serde_json::json!({
8606 "schema": 2,
8607 "updated_at": Timestamp::now().to_string(),
8608 "idle": true,
8609 "surprise": { "nested": [1, 2, 3] },
8610 })
8611 .to_string(),
8612 )
8613 .unwrap();
8614
8615 let reading = read_status(dir.path()).expect("a forward-compatible read");
8616 assert!(reading.running(Timestamp::now()));
8617 assert!(reading.idle);
8618 assert!(reading.current.is_empty());
8619 }
8620
8621 #[test]
8622 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
8623 let dir = tempfile::tempdir().unwrap();
8629 std::fs::write(
8630 dir.path().join("daemon.json"),
8631 serde_json::json!({
8632 "schema": 1,
8633 "pid": 4242,
8634 "updated_at": Timestamp::now().to_string(),
8635 "idle": false,
8636 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
8637 "completed": 3,
8638 "polls": 9,
8639 })
8640 .to_string(),
8641 )
8642 .unwrap();
8643
8644 let reading = read_status(dir.path()).expect("an older shape must still parse");
8645 assert!(reading.running(Timestamp::now()));
8646 assert_eq!(
8647 reading.current,
8648 vec![Current {
8649 task: "20260902-140501-aaaa".to_owned(),
8650 run: "20260902-140502-bbbb".to_owned(),
8651 }]
8652 );
8653 }
8654
8655 #[test]
8656 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
8657 let dir = tempfile::tempdir().unwrap();
8658 std::fs::write(
8659 dir.path().join("daemon.json"),
8660 serde_json::json!({
8661 "schema": 1,
8662 "updated_at": Timestamp::now().to_string(),
8663 "idle": true,
8664 "current": null,
8665 })
8666 .to_string(),
8667 )
8668 .unwrap();
8669 let with_null = read_status(dir.path()).expect("null must still parse");
8670 assert!(with_null.current.is_empty());
8671
8672 std::fs::write(
8673 dir.path().join("daemon.json"),
8674 serde_json::json!({
8675 "schema": 1,
8676 "updated_at": Timestamp::now().to_string(),
8677 "idle": true,
8678 })
8679 .to_string(),
8680 )
8681 .unwrap();
8682 let absent = read_status(dir.path()).expect("a missing field must still parse");
8683 assert!(absent.current.is_empty());
8684 }
8685
8686 #[test]
8687 fn a_task_without_a_repository_runs_in_the_daemons_default() {
8688 let fallback = Path::new("/default");
8689 let mut blank = task();
8690 blank.repo = PathBuf::new();
8691 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
8692 let mut dot = task();
8693 dot.repo = PathBuf::from(".");
8694 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
8695 assert_eq!(
8696 repo_for(&task(), fallback),
8697 PathBuf::from("/repo"),
8698 "a task that names a repository keeps it"
8699 );
8700 }
8701
8702 #[test]
8703 fn task_implementers_edit_inherits_daemon_config_and_merge() {
8704 let dir = tempfile::tempdir().unwrap();
8705 let path = dir.path().join("explicit.toml");
8706 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
8707 let opts = Opts {
8708 config: Some(path),
8709 merge: Some("none".into()),
8710 ..Default::default()
8711 };
8712 let mut t = task();
8713 t.set_implementers(
8714 2,
8715 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8716 .unwrap(),
8717 )
8718 .unwrap();
8719 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
8720 assert_eq!(cfg.graph.implementers, 2);
8721 assert_eq!(cfg.graph.judges, 1);
8722 assert_eq!(cfg.merge.mode, MergeMode::None);
8723 t.set_implementers(1, &cfg).unwrap();
8724 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
8725 apply_task_implementers(&mut cfg, &t);
8726 assert_eq!(cfg.graph.implementers, 1);
8727 assert_eq!(cfg.graph.judges, 1);
8728 assert_eq!(cfg.merge.mode, MergeMode::None);
8729 }
8730
8731 #[test]
8732 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
8733 let dir = tempfile::tempdir().unwrap();
8734 let repo = dir.path().join("repo");
8735 std::fs::create_dir(&repo).unwrap();
8736 std::fs::write(
8737 repo.join("magi.toml"),
8738 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
8739 )
8740 .unwrap();
8741 let daemon_config = dir.path().join("daemon.toml");
8742 std::fs::write(
8743 &daemon_config,
8744 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
8745 )
8746 .unwrap();
8747 let opts = Opts {
8748 config: Some(daemon_config),
8749 merge: Some("pr".into()),
8750 ..Default::default()
8751 };
8752 let q = Queue::at(dir.path().join("queue"));
8753 let validation: Config =
8754 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8755 .unwrap();
8756 for record in ["{}", r#"{"candidates":2}"#] {
8758 let overrides = serde_json::from_str(record).unwrap();
8759 let filing = crate::direct::Filing {
8760 instruction: "work".into(),
8761 title: "direct".into(),
8762 repo: repo.clone(),
8763 source: crate::queue::Source::Human,
8764 solo: false,
8765 overrides,
8766 review_of: None,
8767 sources: Default::default(),
8768 };
8769 let filed =
8770 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
8771 let crate::direct::Filed::Standalone {
8772 mut task,
8773 claim: _claim,
8774 } = filed
8775 else {
8776 panic!("local filing")
8777 };
8778 let before = prepare_for(&repo, &opts, &task).unwrap();
8779 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
8780 assert_eq!(
8781 before.merge.mode,
8782 MergeMode::None,
8783 "daemon --merge must not leak"
8784 );
8785 for n in [2, 1, 2] {
8786 task.set_implementers(n, &validation).unwrap();
8787 q.put(&mut task).unwrap();
8788 task = q.get(&task.id).unwrap();
8789 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
8790 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
8791 apply_task_implementers(&mut cfg, &task);
8792 assert_eq!(cfg.graph.implementers, usize::from(n));
8793 assert_eq!(cfg.graph.judges, 2);
8794 assert_eq!(cfg.merge.mode, MergeMode::None);
8795 }
8796 }
8797 }
8798
8799 #[test]
8800 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
8801 let cfg: Config =
8802 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8803 .unwrap();
8804 let mut t = task();
8805 t.review_of = Some("review-target".into());
8806 t.hold_manual(Some("retry".into()));
8807 let mut previous = RunState::new(
8808 PathBuf::from("/repo"),
8809 "main".into(),
8810 "abc1234".into(),
8811 "review".into(),
8812 cfg.clone(),
8813 );
8814 previous.config.graph.implementers = 2;
8815 for count in [1, 3] {
8816 t.set_implementers(count, &cfg).unwrap();
8817 t.release();
8818 assert_eq!(t.review_of.as_deref(), Some("review-target"));
8819 for proposed in [
8820 Starter::Resume(previous.id.clone()),
8821 Starter::Review("recovery-target".into()),
8822 ] {
8823 assert_eq!(
8824 starter_with_implementers(&t, Some(&previous), proposed),
8825 Starter::Review("review-target".into())
8826 );
8827 }
8828 let mut configured = cfg.clone();
8829 apply_task_implementers(&mut configured, &t);
8830 assert_eq!(configured.graph.implementers, usize::from(count));
8831 assert_eq!(previous.config.graph.implementers, 2);
8832 }
8833 }
8834
8835 #[test]
8836 fn task_implementers_dispatch_and_resume_agree() {
8837 let mut t = task();
8838 let cfg: crate::config::Config =
8839 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8840 .unwrap();
8841 let mut run = RunState::new(
8842 PathBuf::from("/repo"),
8843 "main".into(),
8844 "abc1234".into(),
8845 "work".into(),
8846 cfg.clone(),
8847 );
8848 run.config.graph.implementers = 2;
8849 for n in [2, 1, 3, 2] {
8850 t.set_implementers(n, &cfg).unwrap();
8851 let mut dispatched = cfg.clone();
8852 apply_task_implementers(&mut dispatched, &t);
8853 assert_eq!(dispatched.graph.implementers, usize::from(n));
8854 assert_eq!(implementers_match(&t, &run), n == 2);
8855 for proposed in [
8856 Starter::Resume(run.id.clone()),
8857 Starter::Review("old-branch".into()),
8858 ] {
8859 let expected = if n == 2 {
8860 proposed.clone()
8861 } else {
8862 Starter::Start
8863 };
8864 assert_eq!(
8865 starter_with_implementers(&t, Some(&run), proposed),
8866 expected
8867 );
8868 }
8869 assert_eq!(
8870 run.config.graph.implementers, 2,
8871 "history must not be rewritten"
8872 );
8873 }
8874 t.solo = false;
8875 t.overrides = None;
8876 assert!(implementers_match(&t, &run));
8877 }
8878
8879 #[test]
8880 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
8881 let mut solo_cfg = Config::default();
8887 solo_cfg.graph.implementers = 3;
8888 let mut solo_task = task();
8889 solo_task.solo = true;
8890 apply_task_implementers(&mut solo_cfg, &solo_task);
8891 assert_eq!(solo_cfg.graph.implementers, 1);
8892
8893 let mut plain_cfg = Config::default();
8894 plain_cfg.graph.implementers = 3;
8895 let plain_task = task();
8896 assert!(!plain_task.solo);
8897 apply_task_implementers(&mut plain_cfg, &plain_task);
8898 assert_eq!(
8899 plain_cfg.graph.implementers, 3,
8900 "a task that did not ask to run alone keeps the config's candidates"
8901 );
8902 }
8903
8904 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
8905 QuotaLoss {
8906 seat: seat.into(),
8907 node: "judge".into(),
8908 at: at.parse().unwrap(),
8909 reset: reset.map(str::to_string),
8910 }
8911 }
8912
8913 #[test]
8914 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
8915 let old: Vec<QuotaLoss> = (1..=4)
8916 .map(|i| {
8917 loss(
8918 &format!("judge-{i}"),
8919 "2026-09-23T05:23:00Z",
8920 Some("2:40pm (Asia/Tokyo)"),
8921 )
8922 })
8923 .collect();
8924 let fresh = losses_this_attempt(&old, &old);
8925 assert!(fresh.is_empty());
8926 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
8927 }
8929
8930 #[test]
8931 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
8932 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
8933 let now = Timestamp::now();
8934 let mut after = old.clone();
8935 after.push(loss("judge-2", &now.to_string(), None));
8936 let fresh = losses_this_attempt(&old, &after);
8937 assert_eq!(fresh, vec![after[1].clone()]);
8938 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
8939 assert_eq!(
8940 until,
8941 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
8942 );
8943 }
8944
8945 #[test]
8946 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
8947 let before = vec![
8950 loss("judge-1", "2026-09-23T05:23:00Z", None),
8951 loss("judge-2", "2026-09-23T05:24:00Z", None),
8952 ];
8953 let after = vec![
8954 loss("judge-2", "2026-09-23T05:24:00Z", None),
8955 loss("judge-1", "2026-09-24T01:00:00Z", None),
8956 ];
8957 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
8958 }
8959
8960 #[test]
8961 fn merge_overrides_are_parsed_or_refused() {
8962 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
8963 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
8964 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
8965 assert!(merge_mode("squash").is_err());
8966 }
8967
8968 #[test]
8969 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
8970 let now = Timestamp::now();
8971 let fallback = Duration::from_secs(300);
8972 let cap = Duration::from_secs(1800);
8973
8974 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
8976
8977 let soon = now + jiff::SignedDuration::from_secs(600);
8979 assert_eq!(
8980 quota_wait(Some(soon), now, fallback, cap),
8981 Duration::from_secs(600)
8982 );
8983
8984 let past = now - jiff::SignedDuration::from_secs(60);
8987 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
8988
8989 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
8992 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
8993 }
8994
8995 #[test]
8996 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
8997 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8998
8999 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
9000 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
9001
9002 let already_past =
9006 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
9007 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
9008
9009 assert!(
9010 parse_reset_hint("session limit reached", now, now).is_none(),
9011 "free text with no recognised shape is not guessed at"
9012 );
9013 assert!(
9014 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
9015 "an unresolvable zone name is not guessed at either"
9016 );
9017 }
9018
9019 #[test]
9020 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
9021 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9022
9023 let at = parse_reset_hint(
9024 "You've hit your usage limit. Visit \
9025 https://chatgpt.com/codex/settings/usage to purchase more \
9026 credits or try again at Sep 19th, 2026 5:10 PM.",
9027 now,
9028 now,
9029 )
9030 .expect("the codex reset wording is a recognised shape");
9031 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
9032
9033 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
9038 .expect("an explicit year needs no rollover");
9039 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
9040
9041 assert!(
9042 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
9043 "a two-digit year is not the documented shape and is not guessed at"
9044 );
9045 assert!(
9046 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
9047 "a four-letter month name is not the documented three-letter abbreviation"
9048 );
9049 assert!(
9050 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
9051 "an explicit zone on the dated shape is a format nobody has \
9052 documented, and is refused rather than guessed at as UTC"
9053 );
9054 }
9055
9056 #[test]
9057 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
9058 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
9059 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
9060
9061 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
9062 assert_eq!(at.as_second() - recorded.as_second(), 3769);
9063
9064 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
9065 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
9066
9067 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
9068 assert!(
9069 parse_reset_hint(bad, now, recorded).is_none(),
9070 "{bad:?} must not be guessed at"
9071 );
9072 }
9073 }
9074
9075 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
9079 let config = dir.join("magi.toml");
9080 std::fs::write(
9081 &config,
9082 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
9083 )
9084 .unwrap();
9085 let opts = Opts {
9086 poll: Duration::from_secs(30),
9087 config: Some(config),
9088 repo: dir.join("repo"),
9092 ..Opts::default()
9093 };
9094 let home = dir.join("home");
9103 let worktrees = dir.join("wt");
9104 (
9105 opts,
9106 Queue::at(dir.join("queue")),
9107 home.join("daemon.json"),
9108 home,
9109 worktrees,
9110 )
9111 }
9112
9113 #[test]
9114 fn a_stop_is_idempotent_and_once_set_stays_set() {
9115 let stop = Stop::new();
9116 assert!(!stop.stopped());
9117
9118 stop.stop();
9119 assert!(stop.stopped());
9120 stop.stop();
9121 assert!(stop.stopped(), "a second stop is not a toggle");
9122
9123 let shared = stop.clone();
9124 assert!(
9125 shared.stopped(),
9126 "a clone is the same stop; that is how the loop and its caller share one"
9127 );
9128 }
9129
9130 #[test]
9131 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
9132 let stop = Stop::new();
9133 stop.enter();
9134 assert!(
9135 !stop.finishing(),
9136 "a busy loop nobody has asked to stop is just running"
9137 );
9138
9139 stop.stop();
9140 assert!(
9141 stop.finishing(),
9142 "a stop asked for mid-run has not landed until the run is settled"
9143 );
9144
9145 stop.exit();
9146 assert!(
9147 !stop.finishing(),
9148 "once the run is settled the stop has landed and there is nothing to finish"
9149 );
9150 }
9151
9152 #[test]
9153 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
9154 let stop = Stop::new();
9155 stop.enter();
9156 stop.enter();
9157 stop.stop();
9158 assert!(stop.finishing(), "two runs still in flight");
9159
9160 stop.exit();
9161 assert!(
9162 stop.finishing(),
9163 "one run finished, but a sibling is still working"
9164 );
9165
9166 stop.exit();
9167 assert!(
9168 !stop.finishing(),
9169 "the last run out is what actually lands the stop"
9170 );
9171 }
9172
9173 #[tokio::test]
9174 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
9175 let dir = tempfile::tempdir().unwrap();
9176 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9177 let stop = Stop::new();
9178 stop.stop();
9179
9180 let began = std::time::Instant::now();
9181 tokio::time::timeout(
9182 Duration::from_secs(2),
9183 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9184 )
9185 .await
9186 .expect("a stopped loop must return, not sit out its poll interval")
9187 .expect("the loop's own setup and teardown must not fail");
9188 assert!(
9189 began.elapsed() < opts.poll,
9190 "returned only after {:?}, which is a poll interval, not a stop",
9191 began.elapsed()
9192 );
9193 }
9194
9195 #[tokio::test]
9196 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
9197 let dir = tempfile::tempdir().unwrap();
9198 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9199 let stop = Stop::new();
9200
9201 let asker = {
9204 let stop = stop.clone();
9205 tokio::spawn(async move {
9206 tokio::time::sleep(Duration::from_millis(20)).await;
9207 stop.stop();
9208 })
9209 };
9210
9211 let began = std::time::Instant::now();
9212 tokio::time::timeout(
9213 Duration::from_secs(2),
9214 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9215 )
9216 .await
9217 .expect("a stop asked for while idle must wake the wait")
9218 .expect("the loop's own setup and teardown must not fail");
9219 asker.await.unwrap();
9220 assert!(
9221 began.elapsed() < opts.poll,
9222 "returned only after {:?}, so the stop waited on the sleep",
9223 began.elapsed()
9224 );
9225 }
9226
9227 #[tokio::test]
9228 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
9229 let dir = tempfile::tempdir().unwrap();
9230 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9231 let stop = Stop::new();
9232 stop.stop();
9233
9234 tokio::time::timeout(
9235 Duration::from_secs(2),
9236 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9237 )
9238 .await
9239 .expect("a stopped loop must return")
9240 .expect("the loop's own setup and teardown must not fail");
9241
9242 assert!(
9243 home.is_dir(),
9244 "the loop did publish a status file, so its removal is the teardown and not an absence"
9245 );
9246 assert!(
9247 !status_file.exists(),
9248 "a stopped loop clears its status file"
9249 );
9250 assert!(
9251 read_status(&home).is_none(),
9252 "a reader must see no daemon at all, not a heartbeat that merely stopped"
9253 );
9254 }
9255
9256 #[tokio::test]
9257 async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
9258 let dir = tempfile::tempdir().unwrap();
9259 let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9260 opts.once = true;
9261
9262 let mut settled = RunState::new(
9263 dir.path().join("repo"),
9264 "main".to_owned(),
9265 "abc1234".to_owned(),
9266 "fixture".to_owned(),
9267 Config::default(),
9268 );
9269 settled.status = RunStatus::Ready;
9270 let run_dir = home.join("runs").join(&settled.id);
9271 std::fs::create_dir_all(&run_dir).unwrap();
9272 std::fs::write(
9273 run_dir.join("run.json"),
9274 serde_json::to_string_pretty(&settled).unwrap(),
9275 )
9276 .unwrap();
9277 let questions = Questions::at(home.join("questions"));
9278 let mut question = ask::Question::new(
9279 settled.id.clone(),
9280 "review".to_owned(),
9281 "reviewer-1".to_owned(),
9282 "Continue?".to_owned(),
9283 String::new(),
9284 Vec::new(),
9285 );
9286 questions.put(&mut question).unwrap();
9287
9288 drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
9289 .await
9290 .unwrap();
9291
9292 assert_eq!(
9293 questions.get(&question.id).unwrap().status,
9294 ask::QuestionStatus::Abandoned,
9295 "an empty --once drain still performs startup question cleanup"
9296 );
9297 }
9298
9299 #[test]
9300 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
9301 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9302
9303 assert!(
9304 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
9305 "never checked before: due at once"
9306 );
9307
9308 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
9309 assert!(
9310 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
9311 "well inside the interval: not due yet"
9312 );
9313
9314 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
9315 assert!(
9316 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
9317 "exactly at the edge: not yet due, same convention as `clean::due`"
9318 );
9319
9320 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
9321 assert!(
9322 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
9323 "past the interval: due again"
9324 );
9325 }
9326
9327 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
9332 let config = dir.join("magi.toml");
9333 std::fs::write(
9339 &config,
9340 format!(
9341 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
9342 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
9343 cache_dir.display()
9344 ),
9345 )
9346 .unwrap();
9347 Opts {
9348 config: Some(config),
9349 repo: dir.join("repo"),
9350 ..Opts::default()
9351 }
9352 }
9353
9354 #[tokio::test]
9355 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
9356 let dir = tempfile::tempdir().unwrap();
9357 let home = dir.path().join("home");
9358 let cache_dir = dir.path().join("cache");
9359 std::fs::create_dir_all(&cache_dir).unwrap();
9360 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
9361 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
9362
9363 let running = Stop::new();
9366 let mut last_checked = None;
9367 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9368 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
9369 .await;
9370 assert_eq!(
9371 crate::disk::dir_size(&cache_dir),
9372 0,
9373 "over the cap on the first check ever: pruned at once, no idle queue required"
9374 );
9375 assert_eq!(last_checked, Some(t0));
9376
9377 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
9379 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
9380 maybe_prune_cache_between_runs(
9381 &opts.repo,
9382 &opts,
9383 &home,
9384 &running,
9385 &mut last_checked,
9386 too_soon,
9387 )
9388 .await;
9389 assert_eq!(
9390 crate::disk::dir_size(&cache_dir),
9391 10,
9392 "too soon since the last check: left alone rather than rescanned every call"
9393 );
9394 assert_eq!(
9395 last_checked,
9396 Some(t0),
9397 "an idle check does not reset the clock"
9398 );
9399
9400 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
9402 maybe_prune_cache_between_runs(
9403 &opts.repo,
9404 &opts,
9405 &home,
9406 &running,
9407 &mut last_checked,
9408 due_again,
9409 )
9410 .await;
9411 assert_eq!(
9412 crate::disk::dir_size(&cache_dir),
9413 0,
9414 "due again: pruned back under the cap"
9415 );
9416 }
9417
9418 #[tokio::test]
9426 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
9427 let dir = tempfile::tempdir().unwrap();
9428 let home = dir.path().join("home");
9429 let cache_dir = dir.path().join("cache");
9430 std::fs::create_dir_all(&cache_dir).unwrap();
9431 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
9432 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
9433
9434 let stop = Stop::new();
9435 stop.stop();
9436 assert!(
9437 !stop.finishing(),
9438 "no run is in flight at a between-runs boundary, so nothing else \
9439 would tell the operator this stop had not taken effect yet"
9440 );
9441
9442 let mut last_checked = None;
9443 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9444 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
9445 .await;
9446 assert_eq!(
9447 crate::disk::dir_size(&cache_dir),
9448 10,
9449 "over its cap, and due for the first check ever, but a stop outranks \
9450 it: the cap is a standing policy the next start measures again"
9451 );
9452 assert_eq!(
9453 last_checked, None,
9454 "a check that never happened must not claim the interval"
9455 );
9456 }
9457
9458 #[tokio::test]
9469 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
9470 let dir = tempfile::tempdir().unwrap();
9471 let cache_dir = dir.path().join("cache");
9472 std::fs::create_dir_all(&cache_dir).unwrap();
9473 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
9474
9475 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
9476 opts.poll = Duration::from_millis(20);
9477 opts.max_attempts = 1_000;
9478
9479 let queue = Queue::at(dir.path().join("queue"));
9480 for i in 0..128 {
9481 let mut task = Task::new(
9482 format!("queued work {i}"),
9483 format!("queued work {i}"),
9484 opts.repo.clone(),
9485 Source::Human,
9486 );
9487 queue.put(&mut task).unwrap();
9488 }
9489
9490 let home = dir.path().join("home");
9491 let worktrees = dir.path().join("wt");
9492 let status_file = home.join("daemon.json");
9493 let stop = Stop::new();
9494 let stopper = {
9495 let stop = stop.clone();
9496 let queue = Queue::at(dir.path().join("queue"));
9497 tokio::spawn(async move {
9498 for _ in 0..2_000 {
9502 if queue.list().iter().filter(|t| t.attempts == 1).count() >= 2 {
9503 break;
9504 }
9505 tokio::time::sleep(Duration::from_millis(5)).await;
9506 }
9507 stop.stop();
9508 })
9509 };
9510
9511 tokio::time::timeout(
9512 Duration::from_secs(20),
9513 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9514 )
9515 .await
9516 .expect("the loop must not hang on a queue that keeps producing failing work")
9517 .expect("the loop's own setup and teardown must not fail");
9518 stopper.await.unwrap();
9519
9520 let after = queue.list();
9521 let completed = after.iter().filter(|t| t.attempts == 1).count();
9522 assert!(
9523 completed >= 2,
9524 "the harness must have processed multiple tasks to exercise a busy queue \
9525 (got {completed} completed task(s))"
9526 );
9527 assert!(
9528 after.iter().any(|t| t.status.runnable()),
9529 "unprocessed tasks must remain: the external stop, rather than an \
9530 exhausted backlog, must end the test"
9531 );
9532 assert!(
9533 after
9534 .iter()
9535 .filter(|t| t.attempts > 0)
9536 .all(|t| { t.attempts == 1 && t.status == TaskStatus::Held }),
9537 "a busy backlog must not retry a task's unchanged startup failure"
9538 );
9539
9540 assert_eq!(
9541 crate::disk::dir_size(&cache_dir),
9542 0,
9543 "an oversized cache must not be left to grow unboundedly just because the \
9544 queue kept the loop busy the whole time"
9545 );
9546 }
9547
9548 #[test]
9549 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
9550 let dir = tempfile::tempdir().unwrap();
9551 let queue = Queue::at(dir.path().join("queue"));
9552 let questions = Questions::at(dir.path().join("questions"));
9553 let mut task = task();
9554 queue.put(&mut task).unwrap();
9555
9556 let mut task_question = ask::Question::new(
9557 task.id.clone(),
9558 crate::conduct::NODE.to_owned(),
9559 "conduct".to_owned(),
9560 "Which backend?".to_owned(),
9561 String::new(),
9562 Vec::new(),
9563 );
9564 questions.put(&mut task_question).unwrap();
9565 task.block(vec![task_question.id.clone()], None);
9566 queue.put(&mut task).unwrap();
9567
9568 let mut run_question = ask::Question::new(
9569 "20260101-000000-run1".to_owned(),
9570 "review".to_owned(),
9571 "reviewer-1".to_owned(),
9572 "Run question".to_owned(),
9573 String::new(),
9574 Vec::new(),
9575 );
9576 questions.put(&mut run_question).unwrap();
9577
9578 let mut coincidental = ask::Question::new(
9583 task.id.clone(),
9584 "review".to_owned(),
9585 "reviewer-1".to_owned(),
9586 "Unrelated review question".to_owned(),
9587 String::new(),
9588 Vec::new(),
9589 );
9590 questions.put(&mut coincidental).unwrap();
9591
9592 reconcile_task_questions(&queue, &questions);
9593 assert!(questions.get(&task_question.id).unwrap().status.open());
9594 assert!(questions.get(&run_question.id).unwrap().status.open());
9595 assert!(questions.get(&coincidental.id).unwrap().status.open());
9596
9597 task.release();
9598 queue.put(&mut task).unwrap();
9599 reconcile_task_questions(&queue, &questions);
9600 assert_eq!(
9601 questions.get(&task_question.id).unwrap().status,
9602 ask::QuestionStatus::Abandoned
9603 );
9604 assert!(
9605 questions.get(&run_question.id).unwrap().status.open(),
9606 "run questions remain the run janitor's responsibility"
9607 );
9608 assert!(
9609 questions.get(&coincidental.id).unwrap().status.open(),
9610 "a non-conductor question must not be abandoned just because its \
9611 run id coincides with a task id"
9612 );
9613 }
9614
9615 #[test]
9616 fn a_freshly_started_running_task_is_never_stalled() {
9617 let dir = tempfile::tempdir().unwrap();
9618 let mut t = task();
9619 t.start("run-1".to_owned());
9620 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
9623 }
9624
9625 #[test]
9626 fn a_long_running_task_with_no_live_daemon_is_stalled() {
9627 let dir = tempfile::tempdir().unwrap();
9628 let mut t = task();
9629 t.start("run-1".to_owned());
9630 t.updated_at = Timestamp::now()
9631 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
9632 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
9633 assert_eq!(
9634 stalled_tasks(
9635 &Queue::at(dir.path().join("q")),
9636 dir.path(),
9637 Timestamp::now()
9638 )
9639 .len(),
9640 0,
9641 "the task was never written to this queue"
9642 );
9643 }
9644
9645 #[test]
9646 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
9647 let dir = tempfile::tempdir().unwrap();
9648 let mut t = task();
9649 t.id = "20260903-080340-0167".to_owned();
9650 t.start("20260903-080619-01c2".to_owned());
9651 t.updated_at = Timestamp::now()
9652 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
9653
9654 let mut status = Status::new();
9655 status.current = vec![Current {
9656 task: t.id.clone(),
9657 run: "20260903-080619-01c2".to_owned(),
9658 }];
9659 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9660
9661 assert!(
9662 !is_stalled(&t, dir.path(), Timestamp::now()),
9663 "a live daemon's own heartbeat rules out stalled, however long the task has run"
9664 );
9665 }
9666
9667 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
9671 let path = queue.path_of(id);
9672 let body = std::fs::read_to_string(&path).unwrap();
9673 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
9674 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
9675 v["updated_at"] = serde_json::Value::String(old.to_string());
9676 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
9677 }
9678
9679 #[test]
9680 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
9681 let dir = tempfile::tempdir().unwrap();
9694 let queue = Queue::at(dir.path().join("queue"));
9695 let home = dir.path().join("home");
9696
9697 let mut t = task();
9698 t.id = "20260101-000001-lock".to_owned();
9699 t.start("run-1".to_owned());
9700 queue.put(&mut t).unwrap();
9701 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
9702 std::fs::write(
9703 dir.path().join("queue").join(format!("{}.lock", t.id)),
9704 "not a pid",
9705 )
9706 .unwrap();
9707
9708 let now = Timestamp::now();
9709 assert!(
9710 reclaim_orphaned_running(&queue, 2).is_empty(),
9711 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
9712 and reclaim must leave the task alone"
9713 );
9714 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
9715
9716 let stalled = stalled_tasks(&queue, &home, now);
9717 assert_eq!(
9718 stalled.len(),
9719 1,
9720 "reclaim's inability to claim it yet must not hide it from the conductor"
9721 );
9722 assert_eq!(stalled[0].id, t.id);
9723 }
9724
9725 #[test]
9726 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
9727 let dir = tempfile::tempdir().unwrap();
9728 crate::run::set_home(dir.path().join("run-home"));
9729 let queue = Queue::at(dir.path().join("queue"));
9730 let home = dir.path().join("home");
9731 let questions = Questions::at(dir.path().join("questions"));
9732
9733 let mut t = task();
9734 t.id = "20260101-000003-dead".to_owned();
9735 t.start("missing-run".to_owned());
9736 queue.put(&mut t).unwrap();
9737 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
9738
9739 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
9742 assert_eq!(
9743 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
9744 [&t.id]
9745 );
9746 assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
9747 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
9748
9749 crate::conduct::apply(
9752 &queue,
9753 &questions,
9754 &crate::conduct::Verdict {
9755 decisions: vec![crate::conduct::Decision {
9756 id: t.id.clone(),
9757 recovery: Some(crate::conduct::Recovery::Requeue),
9758 ..crate::conduct::Decision::default()
9759 }],
9760 },
9761 )
9762 .unwrap();
9763 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
9764 }
9765
9766 #[test]
9767 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
9768 let dir = tempfile::tempdir().unwrap();
9769 let queue = Queue::at(dir.path().join("queue"));
9770 let home = dir.path().join("home");
9771
9772 let mut fresh = task();
9773 fresh.id = "20260101-000001-aaaa".to_owned();
9774 fresh.start("run-1".to_owned());
9775 queue.put(&mut fresh).unwrap();
9776
9777 let mut old = task();
9778 old.id = "20260101-000002-bbbb".to_owned();
9779 old.start("run-2".to_owned());
9780 queue.put(&mut old).unwrap();
9781 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
9782
9783 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
9784 assert_eq!(stalled.len(), 1);
9785 assert_eq!(stalled[0].id, old.id);
9786 }
9787
9788 #[test]
9789 fn queued_and_finished_task_views_partition_by_status() {
9790 let dir = tempfile::tempdir().unwrap();
9791 let queue = Queue::at(dir.path().join("queue"));
9792
9793 let mut queued = task();
9794 queued.id = "20260101-000001-aaaa".to_owned();
9795 queue.put(&mut queued).unwrap();
9796
9797 let mut failed = task();
9798 failed.id = "20260101-000002-bbbb".to_owned();
9799 failed.start("run-1".to_owned());
9800 failed.fail("gate red", 5);
9801 queue.put(&mut failed).unwrap();
9802
9803 let mut held = task();
9804 held.id = "20260101-000003-cccc".to_owned();
9805 held.hold_machine(None);
9806 queue.put(&mut held).unwrap();
9807
9808 let mut running = task();
9809 running.id = "20260101-000004-dddd".to_owned();
9810 running.start("run-2".to_owned());
9811 queue.put(&mut running).unwrap();
9812
9813 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
9814 assert_eq!(queued_ids, [queued.id.clone()]);
9815
9816 let mut finished_ids: Vec<String> =
9817 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
9818 finished_ids.sort_unstable();
9819 let mut want = vec![failed.id.clone(), held.id.clone()];
9820 want.sort_unstable();
9821 assert_eq!(finished_ids, want);
9822 }
9823
9824 #[test]
9825 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
9826 for short in [false, true] {
9827 let dir = tempfile::tempdir().unwrap();
9828 let queue = Queue::at(dir.path().join("queue"));
9829 let questions = ask::Questions::at(dir.path().join("questions"));
9830 let mut running = task();
9831 running.id = "20261013-192213-feb2".into();
9832 running.start("existing-run".into());
9833 queue.put(&mut running).unwrap();
9834 let mut middle = task();
9835 middle.id = "20261013-193611-ca5b".into();
9836 middle.block(
9837 vec![if short {
9838 "feb2".into()
9839 } else {
9840 running.id.clone()
9841 }],
9842 None,
9843 );
9844 queue.put(&mut middle).unwrap();
9845 let mut waiting = task();
9846 waiting.id = "20261013-202849-ee37".into();
9847 waiting.block(
9848 vec![if short {
9849 "ca5b".into()
9850 } else {
9851 middle.id.clone()
9852 }],
9853 Some("wait for implementation".into()),
9854 );
9855 queue.put(&mut waiting).unwrap();
9856 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
9857 let inv = crate::blockers::Inventory::load(&queue, &questions);
9858 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
9859 assert!(inv.stuck().is_empty());
9860 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9861 assert!(report.quarantined.is_empty());
9862 assert!(report.asked.is_empty());
9863 resolve_blockers(&queue, &questions);
9864 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9865 assert!(questions.list().is_empty());
9866 running.succeed();
9869 queue.put(&mut running).unwrap();
9870 resolve_blockers(&queue, &questions);
9871 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
9872 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9873 middle = queue.get(&middle.id).unwrap();
9874 middle.succeed();
9875 queue.put(&mut middle).unwrap();
9876 resolve_blockers(&queue, &questions);
9877 let after = queue.get(&waiting.id).unwrap();
9878 assert_eq!(after.status, TaskStatus::Queued);
9879 assert!(after.blocked_by.is_empty());
9880 assert_eq!(after.instruction, waiting.instruction);
9881 }
9882 }
9883
9884 #[test]
9885 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
9886 for future_schema in [false, true] {
9887 let name = "20261013-193611-ca5b";
9888 let mut future = task();
9889 future.id = name.into();
9890 future.schema = crate::queue::SCHEMA + 1;
9891 let contents = if future_schema {
9892 serde_json::to_string(&future).unwrap()
9893 } else {
9894 "corrupt".into()
9895 };
9896 let dir = tempfile::tempdir().unwrap();
9897 let queue = Queue::at(dir.path().join("queue"));
9898 let questions = ask::Questions::at(dir.path().join("questions"));
9899 let mut waiting = task();
9900 waiting.id = "20261013-202849-ee37".into();
9901 waiting.block(vec!["ca5b".into()], None);
9902 queue.put(&mut waiting).unwrap();
9903 std::fs::write(queue.path_of(name), contents).unwrap();
9904 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
9905 assert!(
9906 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
9907 );
9908 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9909 assert!(report.asked.is_empty());
9910 resolve_blockers(&queue, &questions);
9911 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9912 assert!(questions.list().is_empty());
9913 }
9914 let dir = tempfile::tempdir().unwrap();
9915 let queue = Queue::at(dir.path().join("queue"));
9916 let questions = ask::Questions::at(dir.path().join("questions"));
9917 let mut waiting = task();
9918 waiting.id = "20261013-202849-ee37".into();
9919 waiting.block(vec!["ca5b".into()], None);
9920 queue.put(&mut waiting).unwrap();
9921 let mut done = task();
9922 done.id = "20261013-193611-ca5b".into();
9923 done.succeed();
9924 queue.put(&mut done).unwrap();
9925 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
9928 assert!(queue.get("ca5b").is_err());
9929 assert_eq!(
9930 crate::queue::dependency(&queue, &questions, "ca5b"),
9931 crate::queue::Dependency::Ambiguous
9932 );
9933 resolve_blockers(&queue, &questions);
9934 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9935 assert!(report.asked.is_empty());
9936 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9937 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
9938 }
9939
9940 #[test]
9941 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
9942 for held in [false, true] {
9943 for question in [false, true] {
9944 for reference in ["ca5b", "20261013-193611-ca5b"] {
9945 let dir = tempfile::tempdir().unwrap();
9946 let queue = Queue::at(dir.path().join("queue"));
9947 let questions = ask::Questions::at(dir.path().join("questions"));
9948 let mut prerequisite = task();
9949 prerequisite.id = "20261013-193611-ca5b".into();
9950 prerequisite.start("existing-run".into());
9951 queue.put(&mut prerequisite).unwrap();
9952 let mut waiting = task();
9953 waiting.id = "20261013-202849-ee37".into();
9954 if held {
9955 waiting.hold_machine(Some("waiting for separate consent".into()));
9956 }
9957 queue.put(&mut waiting).unwrap();
9958 let reason = "Deleting saved data requires separate owner consent";
9959 crate::conduct::apply(
9960 &queue,
9961 &questions,
9962 &crate::conduct::Verdict {
9963 decisions: vec![crate::conduct::Decision {
9964 id: waiting.id.clone(),
9965 blocked_by: vec![reference.into()],
9966 question: question.then(|| "May the saved data be deleted?".into()),
9967 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
9968 reason: Some(reason.into()),
9969 ..crate::conduct::Decision::default()
9970 }],
9971 },
9972 )
9973 .unwrap();
9974 let before = queue.get(&waiting.id).unwrap();
9975 if question {
9976 assert_eq!(before.status, TaskStatus::Blocked);
9977 let q = questions.get(&before.blocked_by[0]).unwrap();
9978 assert_eq!(q.summary, "May the saved data be deleted?");
9979 assert!(q.status.open());
9980 } else {
9981 assert_eq!(before.status, TaskStatus::Held);
9982 let recorded_reason = before.hold_reason.as_deref().unwrap();
9983 assert!(recorded_reason.starts_with(reason));
9984 if held {
9985 assert!(recorded_reason.contains("waiting for separate consent"));
9986 }
9987 }
9988 prerequisite.succeed();
9989 queue.put(&mut prerequisite).unwrap();
9990 resolve_blockers(&queue, &questions);
9991 let after = queue.get(&waiting.id).unwrap();
9992 assert_eq!(after.status, before.status);
9993 assert_eq!(after.blocked_by, before.blocked_by);
9994 assert_eq!(after.hold_reason, before.hold_reason);
9995 assert_eq!(after.instruction, waiting.instruction);
9996 if question {
9997 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
9998 }
9999 assert!(
10000 queue.next_runnable().is_none(),
10001 "dependency completion is not consent"
10002 );
10003 }
10004 }
10005 }
10006 }
10007
10008 #[test]
10009 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
10010 let dir = tempfile::tempdir().unwrap();
10011 let queue = Queue::at(dir.path().join("queue"));
10012 let questions = ask::Questions::at(dir.path().join("questions"));
10013
10014 let mut dep = task();
10015 dep.id = "20260101-000001-dep0".to_owned();
10016 dep.succeed();
10017 queue.put(&mut dep).unwrap();
10018
10019 let mut still_going = task();
10020 still_going.id = "20260101-000002-dep1".to_owned();
10021 queue.put(&mut still_going).unwrap();
10022
10023 let mut blocked = task();
10024 blocked.id = "20260101-000003-main".to_owned();
10025 blocked.block(
10026 vec![dep.id.clone(), still_going.id.clone()],
10027 Some("waits on both".to_owned()),
10028 );
10029 queue.put(&mut blocked).unwrap();
10030
10031 resolve_blockers(&queue, &questions);
10032
10033 let after = queue.get(&blocked.id).unwrap();
10034 assert_eq!(
10035 after.status,
10036 TaskStatus::Blocked,
10037 "one dependency is still outstanding"
10038 );
10039 assert_eq!(after.blocked_by, [still_going.id.clone()]);
10040 }
10041
10042 #[test]
10043 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
10044 let dir = tempfile::tempdir().unwrap();
10045 let queue = Queue::at(dir.path().join("queue"));
10046 let questions = ask::Questions::at(dir.path().join("questions"));
10047
10048 let mut q = crate::ask::Question::new(
10049 "20260101-000001-main".to_owned(),
10050 crate::conduct::NODE.to_owned(),
10051 "conduct".to_owned(),
10052 "Which backend?".to_owned(),
10053 String::new(),
10054 Vec::new(),
10055 );
10056 questions.put(&mut q).unwrap();
10057 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
10058 .unwrap();
10059 questions.put(&mut q).unwrap();
10060
10061 let mut blocked = task();
10062 blocked.id = "20260101-000001-main".to_owned();
10063 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
10064 queue.put(&mut blocked).unwrap();
10065
10066 resolve_blockers(&queue, &questions);
10067
10068 let after = queue.get(&blocked.id).unwrap();
10069 assert_eq!(
10070 after.status,
10071 TaskStatus::Queued,
10072 "the only blocker resolved"
10073 );
10074 assert_eq!(after.answers.len(), 1);
10075 assert_eq!(after.answers[0].question, "Which backend?");
10076 assert_eq!(after.answers[0].answer, "SQLite");
10077
10078 let instruction = instruction_for(&after);
10080 assert!(instruction.contains("Which backend?"));
10081 assert!(instruction.contains("SQLite"));
10082 }
10083
10084 #[test]
10085 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
10086 let dir = tempfile::tempdir().unwrap();
10087 let queue = Queue::at(dir.path().join("queue"));
10088 let questions = ask::Questions::at(dir.path().join("questions"));
10089
10090 let mut q = crate::ask::Question::new(
10091 "20260101-000001-main".to_owned(),
10092 crate::conduct::NODE.to_owned(),
10093 "conduct".to_owned(),
10094 "Is the setup done?".to_owned(),
10095 String::new(),
10096 Vec::new(),
10097 );
10098 q.abandon("no answer within 60s of asking");
10099 questions.put(&mut q).unwrap();
10100
10101 let mut blocked = task();
10102 blocked.id = "20260101-000001-main".to_owned();
10103 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
10104 queue.put(&mut blocked).unwrap();
10105
10106 resolve_blockers(&queue, &questions);
10107
10108 let after = queue.get(&blocked.id).unwrap();
10109 assert_eq!(
10110 after.status,
10111 TaskStatus::Held,
10112 "never left blocked on nothing"
10113 );
10114 assert!(!after.operator_held(), "a machine hold, for triage");
10115 assert!(
10116 after
10117 .hold_reason
10118 .as_deref()
10119 .unwrap_or_default()
10120 .contains("went unanswered")
10121 );
10122 }
10123
10124 #[test]
10125 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
10126 let dir = tempfile::tempdir().unwrap();
10132 let queue = Queue::at(dir.path().join("queue"));
10133 let questions = ask::Questions::at(dir.path().join("questions"));
10134
10135 let mut q = crate::ask::Question::new(
10136 "20260101-000001-main".to_owned(),
10137 crate::conduct::NODE.to_owned(),
10138 "conduct".to_owned(),
10139 "How should this be handled?".to_owned(),
10140 String::new(),
10141 Vec::new(),
10142 );
10143 questions.put(&mut q).unwrap();
10144 q.answer(crate::ask::Answer::Text(
10145 "leave it held, a human will look at it later".to_owned(),
10146 ))
10147 .unwrap();
10148 questions.put(&mut q).unwrap();
10149
10150 let mut held = task();
10151 held.id = "20260101-000001-main".to_owned();
10152 held.hold_machine(Some("out of attempts".to_owned()));
10153 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
10154 queue.put(&mut held).unwrap();
10155
10156 resolve_blockers(&queue, &questions);
10157
10158 let after = queue.get(&held.id).unwrap();
10159 assert_eq!(after.status, TaskStatus::Held);
10160 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
10161 assert_eq!(
10162 after.answers[0].answer,
10163 "leave it held, a human will look at it later"
10164 );
10165 }
10166
10167 #[test]
10168 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
10169 let dir = tempfile::tempdir().unwrap();
10170 let queue = Queue::at(dir.path().join("queue"));
10171 let questions = ask::Questions::at(dir.path().join("questions"));
10172 let mut dep = task();
10173 dep.id = "20260101-000001-gone".to_owned();
10174 queue.put(&mut dep).unwrap();
10175 let mut blocked = task();
10176 blocked.id = "20260101-000003-main".to_owned();
10177 blocked.block(vec![dep.id.clone()], None);
10178 queue.put(&mut blocked).unwrap();
10179
10180 let claim = queue.claim(&blocked.id).unwrap();
10182 queue.remove(&dep.id, false, &questions).unwrap();
10183 drop(claim);
10184 resolve_blockers(&queue, &questions);
10185
10186 let after = queue.get(&blocked.id).unwrap();
10187 assert_eq!(after.status, TaskStatus::Queued);
10188 assert!(after.blocked_by.is_empty());
10189 }
10190
10191 #[test]
10192 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
10193 let dir = tempfile::tempdir().unwrap();
10199 let queue = Queue::at(dir.path().join("queue"));
10200 let questions = ask::Questions::at(dir.path().join("questions"));
10201
10202 let mut still_going = task();
10203 still_going.id = "20260101-000002-dep1".to_owned();
10204 queue.put(&mut still_going).unwrap();
10205
10206 let mut blocked = task();
10207 blocked.id = "20260101-000003-main".to_owned();
10208 blocked.block(
10209 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
10210 Some("waits on both".to_owned()),
10211 );
10212 queue.put(&mut blocked).unwrap();
10213
10214 resolve_blockers(&queue, &questions);
10215
10216 let after = queue.get(&blocked.id).unwrap();
10217 assert_eq!(
10218 after.status,
10219 TaskStatus::Held,
10220 "a missing dependency must not leave the task blocked forever"
10221 );
10222 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
10223 assert!(after.blocked_by.is_empty());
10224 let reason = after.hold_reason.as_deref().unwrap_or_default();
10225 assert!(
10226 reason.contains("20260101-000001-gone"),
10227 "the missing id must be named so an operator can tell what happened: {reason}"
10228 );
10229 assert!(
10230 reason.contains(&still_going.id),
10231 "the still-valid dependency must not silently vanish from the record: {reason}"
10232 );
10233 }
10234
10235 #[test]
10236 fn instruction_for_is_unchanged_without_any_answers() {
10237 let t = task();
10238 assert_eq!(instruction_for(&t), t.instruction);
10239 }
10240
10241 #[test]
10242 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
10243 let dir = tempfile::tempdir().unwrap();
10244 let q = Queue::at(dir.path().join("queue"));
10245 let src = dir.path().join("shot.png");
10246 std::fs::write(&src, "x").unwrap();
10247 let mut t = task();
10248 q.attach(&mut t, &[src]).unwrap();
10249 let paths = task_attachments(&q, &t).unwrap();
10250 assert_eq!(paths.len(), 1);
10251 assert!(paths[0].is_absolute() && paths[0].is_file());
10252 std::fs::remove_file(&paths[0]).unwrap();
10253 let err = task_attachments(&q, &t).unwrap_err().to_string();
10254 assert!(err.contains("shot.png"), "{err}");
10255 }
10256
10257 #[tokio::test]
10258 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
10259 let dir = tempfile::tempdir().unwrap();
10260 let queue_path = dir.path().join("queue");
10261 let queue = Queue::at(queue_path.clone());
10262 let mut t = task();
10263 t.source = Source::Agent {
10264 run: "originating-chat".into(),
10265 node: crate::queue::CHAT_NODE.into(),
10266 };
10267 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
10268 stage: "config-fetch",
10269 target: "origin/main".into(),
10270 });
10271 record_config_failure(&mut t, &fetch);
10272 t.hold_machine(Some(format!("{fetch:#}")));
10273 let fetch_key = t.chat_notice_due().unwrap();
10274 t.chat_notices.push(fetch_key.clone());
10275 queue.put(&mut t).unwrap();
10276
10277 let queue = Queue::at(queue_path);
10278 let mut t = queue.get(&t.id).unwrap();
10279 let a = dir.path().join("a.toml");
10280 let b = dir.path().join("b.toml");
10281 for (path, text, new_cause) in [
10282 (&a, "[graph", true),
10283 (&a, "[verify", false),
10284 (&a, "[graph]\nimplementers = 'invalid-type'", true),
10285 (&b, "[graph]\nimplementers = 'invalid-type'", true),
10286 ] {
10287 t.release_operator("test", None);
10288 std::fs::write(path, text).unwrap();
10289 let error = Config::discover_fetched(dir.path(), Some(path))
10290 .await
10291 .unwrap_err();
10292 record_config_failure(&mut t, &error);
10293 t.hold_machine(Some(format!("{error:#}")));
10294 let due = t.chat_notice_due();
10295 assert_eq!(due.is_some(), new_cause, "{error:#}");
10296 if let Some(key) = due {
10297 assert_ne!(key, fetch_key);
10298 t.chat_notices.push(key);
10299 }
10300 }
10301 t.succeed();
10302 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
10303 }
10304
10305 #[test]
10306 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
10307 let dir = tempfile::tempdir().unwrap();
10308 let queue_path = dir.path().join("queue");
10309 let queue = Queue::at(queue_path.clone());
10310 let mut t = task();
10311 t.source = crate::queue::Source::Agent {
10312 run: "originating-chat".into(),
10313 node: crate::queue::CHAT_NODE.into(),
10314 };
10315 let a = dir.path().join("a.png");
10316 let b = dir.path().join("b.png");
10317 std::fs::write(&a, "a").unwrap();
10318 std::fs::write(&b, "b").unwrap();
10319 queue.attach(&mut t, &[a, b]).unwrap();
10320 let paths = task_attachments(&queue, &t).unwrap();
10321 for path in &paths {
10322 std::fs::remove_file(path).unwrap();
10323 }
10324 let missing = task_attachments(&queue, &t).unwrap_err();
10325 t.attempts += 1;
10326 let reason = missing.to_string();
10327 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
10328 let first = t.chat_notice_due().expect("first attachment notice");
10329 t.chat_notices.push(first.clone());
10330 queue.put(&mut t).unwrap();
10331
10332 let queue = Queue::at(queue_path.clone());
10333 let mut t = queue.get(&t.id).unwrap();
10334 assert!(t.chat_notice_due().is_none(), "restart does not replay");
10335 t.release_operator("test", None);
10336 t.attempts += 1;
10337 let missing = task_attachments(&queue, &t).unwrap_err();
10338 t.fail_startup_target(
10339 "startup-attachments",
10340 missing.name,
10341 "same missing attachment with regenerated wording",
10342 1,
10343 );
10344 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
10345
10346 std::fs::write(&paths[0], "restored").unwrap();
10347 t.release_operator("test", None);
10348 t.attempts += 1;
10349 let missing = task_attachments(&queue, &t).unwrap_err();
10350 let reason = missing.to_string();
10351 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
10352 let second = t
10353 .chat_notice_due()
10354 .expect("different attachment must notify");
10355 assert_ne!(first, second);
10356 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
10357 }
10358
10359 #[test]
10360 fn resumed_instruction_is_unchanged_without_any_answers() {
10361 let t = task();
10362 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
10363 }
10364
10365 #[test]
10366 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
10367 let mut t = task();
10368 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10369 let old = t.instruction.clone();
10373
10374 let refreshed = resumed_instruction(&old, &t);
10375 assert!(refreshed.starts_with(&old), "the original text is kept");
10376 assert!(refreshed.contains("Which backend?"));
10377 assert!(refreshed.contains("SQLite"));
10378 }
10379
10380 #[test]
10381 fn resumed_instruction_keeps_an_original_answers_heading() {
10382 let mut t = task();
10383 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
10384 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10385
10386 let refreshed = resumed_instruction(&t.instruction, &t);
10387
10388 assert!(
10389 refreshed.starts_with(&t.instruction),
10390 "an answers heading in the original instruction is not the appended block"
10391 );
10392 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
10393 assert!(refreshed.contains("Which backend?"));
10394 assert!(refreshed.contains("SQLite"));
10395
10396 let repeated = resumed_instruction(&refreshed, &t);
10397 assert_eq!(
10398 repeated, refreshed,
10399 "only the final appended block is refreshed"
10400 );
10401 }
10402
10403 #[test]
10404 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
10405 let mut t = task();
10406 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10407
10408 let once = resumed_instruction(&t.instruction, &t);
10412 let twice = resumed_instruction(&once, &t);
10413 assert_eq!(once, twice);
10414 assert_eq!(once.matches("Which backend?").count(), 1);
10415
10416 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
10418 let refreshed = resumed_instruction(&once, &t);
10419 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
10420 assert!(refreshed.contains("Which backend?"));
10421 assert!(refreshed.contains("Which cache?"));
10422 }
10423
10424 #[test]
10425 fn prepare_instruction_covers_all_three_starters() {
10426 let mut t = task();
10427 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10428
10429 assert_eq!(
10432 prepare_instruction(&Starter::Start, None, &t),
10433 Some(instruction_for(&t))
10434 );
10435
10436 let old = t.instruction.clone();
10439 assert_eq!(
10440 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
10441 Some(resumed_instruction(&old, &t))
10442 );
10443
10444 assert_eq!(
10448 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
10449 None
10450 );
10451 }
10452
10453 #[test]
10454 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
10455 assert_eq!(
10456 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
10457 Starter::Review("magi/eba2/A".to_owned())
10458 );
10459 }
10460
10461 #[test]
10462 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
10463 assert_eq!(
10464 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
10465 Starter::Start,
10466 "a vanished review branch must not fall back to resuming the old run either"
10467 );
10468 }
10469
10470 #[test]
10471 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
10472 let mut t = task();
10473 t.start("old-run".to_owned());
10474 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
10475 t.release();
10476 let branch = t.review_branch.take();
10477 assert_eq!(
10478 choose_starter(branch.as_deref(), true, Some("old-run")),
10479 Starter::Review("magi/eba2/A".to_owned()),
10480 "a review wins over resuming the old run"
10481 );
10482 }
10483
10484 #[test]
10485 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
10486 assert_eq!(
10487 choose_starter(None, false, Some("some-run")),
10488 Starter::Resume("some-run".to_owned())
10489 );
10490 assert_eq!(choose_starter(None, false, None), Starter::Start);
10491 }
10492
10493 #[test]
10494 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
10495 let mut released = task();
10496 released.start("stalled-run".to_owned());
10497 released.requeue();
10498 let unfinished = (!released.fresh_start)
10499 .then(|| Some("stalled-run".to_owned()))
10500 .flatten();
10501 assert_eq!(
10502 choose_starter(None, false, unfinished.as_deref()),
10503 Starter::Start,
10504 "release keeps run history but must not resume it"
10505 );
10506 assert_eq!(released.runs, ["stalled-run"]);
10507 }
10508
10509 fn stopped_public_run(status: RunStatus) -> RunState {
10510 let mut state = run_state(status);
10511 state.driver_exited = true;
10512 state.config.graph.land = true;
10513 state.config.graph.review_rounds = 1;
10514 state.reviews = vec![review_round(1)];
10515 state.merge = Some(crate::run::MergeOutcome {
10516 mode: MergeMode::Pr,
10517 ok: false,
10518 detail: "landing stopped".into(),
10519 empty: false,
10520 });
10521 state.pr = Some(crate::run::PrRecord {
10522 url: "https://example.test/pull/1".into(),
10523 number: 1,
10524 state: "open".into(),
10525 checks: "red".into(),
10526 round: 1,
10527 rounds: 1,
10528 red_at_merge: Vec::new(),
10529 });
10530 state
10531 }
10532
10533 #[test]
10534 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
10535 for status in [RunStatus::Blocked, RunStatus::Gating] {
10536 let home = tempfile::tempdir().unwrap();
10537 let mut state = stopped_public_run(status);
10538 state.save_under(home.path()).unwrap();
10539 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
10540 assert!(!exhausted_review_budget(&state));
10541 let mut task = held_task_with(&state.id);
10542 task.release();
10543 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
10544 assert_eq!(
10545 choose_starter(None, false, unfinished.as_deref()),
10546 Starter::Resume(state.id.clone())
10547 );
10548 let question = action_question(&state.id, resume_action(&state.id));
10549 assert_eq!(
10550 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
10551 ActionDecision::Resume(state.id.clone())
10552 );
10553 assert_eq!(
10555 crate::resume::validate(home.path(), &state.id).unwrap().id,
10556 state.id
10557 );
10558 let queue = Queue::at(home.path().join("queue"));
10559 let mut controlled = Task::new(
10560 "resume land".into(),
10561 state.instruction.clone(),
10562 state.repo.clone(),
10563 Source::Agent {
10564 run: "talk".into(),
10565 node: crate::queue::CHAT_NODE.into(),
10566 },
10567 );
10568 controlled.runs.push(state.id.clone());
10569 controlled.hold_machine(Some("land stopped".into()));
10570 queue.put(&mut controlled).unwrap();
10571 let resumed =
10572 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
10573 assert_eq!(resumed.status, TaskStatus::Queued);
10574 assert_eq!(
10575 resumed.resume_override.unwrap().pinned_run,
10576 Some(state.id.clone())
10577 );
10578 assert_eq!(resumed.attempts, 0);
10579 }
10580 }
10581
10582 #[test]
10583 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
10584 for review_rounds in [0, 1] {
10585 let home = tempfile::tempdir().unwrap();
10586 let mut state = run_state(RunStatus::Blocked);
10587 state.config.graph.review_rounds = review_rounds;
10588 if review_rounds > 0 {
10589 let mut review = review_round(1);
10590 review.clean = true;
10591 state.reviews = vec![review];
10592 }
10593 state.gate_ran = true;
10594 state.gate = vec![crate::run::CommandOutcome {
10595 command: "verify".into(),
10596 code: Some(1),
10597 output_tail: "gate failed after repair budget exhausted".into(),
10598 duration_ms: 0,
10599 resource_blocked: false,
10600 }];
10601 state.save_under(home.path()).unwrap();
10602 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
10603 assert!(exhausted_review_budget(&state));
10604 let mut task = held_task_with(&state.id);
10605 task.release();
10606 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
10607 let question = action_question(&state.id, resume_action(&state.id));
10608 assert_ne!(
10609 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
10610 ActionDecision::Resume(state.id.clone())
10611 );
10612 assert!(crate::resume::validate(home.path(), &state.id).is_err());
10613
10614 state.gate_ran = false;
10616 state.gate.clear();
10617 assert!(!exhausted_review_budget(&state));
10618 assert_eq!(
10619 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
10620 Some(state.id)
10621 );
10622 }
10623 }
10624
10625 #[test]
10626 fn resource_blocked_verification_resumes_the_same_run() {
10627 for (kind, stage) in [
10628 ("cache", "clean"),
10629 ("coordination", "clean"),
10630 ("cache", "review-e2e"),
10631 ("coordination", "no-reviews"),
10632 ("coordination", "handoff"),
10633 ] {
10634 let home = tempfile::tempdir().unwrap();
10635 let mut state = run_state(RunStatus::Blocked);
10636 state.config.graph.review_rounds = 1;
10637 let mut review = review_round(1);
10638 review.clean = true;
10639 state.reviews = vec![review];
10640 if stage == "review-e2e" {
10641 let review = &mut state.reviews[0];
10642 review.clean = false;
10643 review.blocking = 1;
10644 review.e2e = vec![crate::run::CommandOutcome {
10645 command: "verify".into(),
10646 code: None,
10647 output_tail: "resource wait expired".into(),
10648 duration_ms: 0,
10649 resource_blocked: true,
10650 }];
10651 } else if stage == "no-reviews" {
10652 state.config.graph.review_rounds = 0;
10653 state.reviews.clear();
10654 } else if stage == "handoff" {
10655 let review = &mut state.reviews[0];
10658 review.clean = false;
10659 review.blocking = 1;
10660 review.e2e = vec![crate::run::CommandOutcome {
10661 command: "verify".into(),
10662 code: Some(0),
10663 output_tail: String::new(),
10664 duration_ms: 0,
10665 resource_blocked: false,
10666 }];
10667 }
10668 state.gate_ran = false;
10669 state.resource_block = Some(crate::run::ResourceBlock {
10670 node: if stage == "review-e2e" {
10671 "review"
10672 } else {
10673 "gate"
10674 }
10675 .into(),
10676 kind: kind.into(),
10677 detail: "resource wait expired".into(),
10678 at: jiff::Timestamp::now(),
10679 });
10680 state.save_under(home.path()).unwrap();
10681 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
10682 assert!(!exhausted_review_budget(&state));
10683 let mut task = held_task_with(&state.id);
10684 task.release();
10685 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
10686 assert_eq!(
10687 choose_starter(None, false, unfinished.as_deref()),
10688 Starter::Resume(state.id.clone())
10689 );
10690 let question = action_question(&state.id, resume_action(&state.id));
10691 assert_eq!(
10692 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
10693 ActionDecision::Resume(state.id.clone())
10694 );
10695 assert_eq!(
10696 crate::resume::validate(home.path(), &state.id).unwrap().id,
10697 state.id
10698 );
10699 let queue = Queue::at(home.path().join("queue"));
10700 let mut controlled = Task::new(
10701 "resume verification".into(),
10702 state.instruction.clone(),
10703 state.repo.clone(),
10704 Source::Agent {
10705 run: "talk".into(),
10706 node: crate::queue::CHAT_NODE.into(),
10707 },
10708 );
10709 controlled.runs.push(state.id.clone());
10710 controlled.hold_machine(Some("verification resource unavailable".into()));
10711 queue.put(&mut controlled).unwrap();
10712 let resumed =
10713 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
10714 assert_eq!(resumed.status, TaskStatus::Queued);
10715 assert_eq!(
10716 resumed.resume_override.unwrap().pinned_run,
10717 Some(state.id.clone())
10718 );
10719 assert_eq!(resumed.attempts, 0);
10720
10721 state.config.graph.review_rounds = 1;
10723 let mut unresolved = review_round(1);
10724 unresolved.blocking = 1;
10725 state.reviews = vec![unresolved];
10726 state.resource_block = None;
10727 state.save_under(home.path()).unwrap();
10728 assert!(exhausted_review_budget(&state));
10729 assert!(crate::resume::validate(home.path(), &state.id).is_err());
10730 }
10731 }
10732
10733 #[tokio::test]
10734 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
10735 let home = tempfile::tempdir().unwrap();
10736 let mut owner = stopped_public_run(RunStatus::Blocked);
10737 let repo = tempfile::tempdir().unwrap();
10738 crate::git::git(repo.path(), &["init"]).await.unwrap();
10739 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
10740 owner.id = "20260101-000000-aaaa".into();
10741 owner.save_under(home.path()).unwrap();
10742 let mut successor = run_state(RunStatus::Blocked);
10743 successor.id = "20260101-000001-bbbb".into();
10744 successor.save_under(home.path()).unwrap();
10745 let runs = vec![owner.id.clone(), successor.id.clone()];
10746 assert_eq!(
10747 public_resume_owner(home.path(), &runs, &owner.repo)
10748 .await
10749 .unwrap()
10750 .unwrap()
10751 .id,
10752 owner.id
10753 );
10754 let subdir = owner.repo.join("nested");
10755 std::fs::create_dir(&subdir).unwrap();
10756 assert_eq!(
10757 public_resume_owner(home.path(), &runs, &subdir)
10758 .await
10759 .unwrap()
10760 .unwrap()
10761 .id,
10762 owner.id
10763 );
10764 #[cfg(unix)]
10765 {
10766 let alias = home.path().join("repo-alias");
10767 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
10768 assert_eq!(
10769 public_resume_owner(home.path(), &runs, &alias.join("nested"))
10770 .await
10771 .unwrap()
10772 .unwrap()
10773 .id,
10774 owner.id
10775 );
10776 }
10777 let different = tempfile::tempdir().unwrap();
10778 crate::git::git(different.path(), &["init"]).await.unwrap();
10779 assert!(
10780 public_resume_owner(home.path(), &runs, different.path())
10781 .await
10782 .is_err()
10783 );
10784 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
10785 assert!(
10786 public_resume_owner(home.path(), &runs, &owner.repo)
10787 .await
10788 .is_err()
10789 );
10790 drop(lock);
10791 std::fs::remove_file(
10792 home.path()
10793 .join("runs")
10794 .join(&successor.id)
10795 .join("run.json"),
10796 )
10797 .unwrap();
10798 assert!(
10799 public_resume_owner(home.path(), &runs, &owner.repo)
10800 .await
10801 .is_err()
10802 );
10803 }
10804
10805 #[test]
10806 fn an_ordinary_release_keeps_a_resumable_run_available() {
10807 let mut released = task();
10808 released.start("stalled-run".to_owned());
10809 released.release();
10810 let unfinished = (!released.fresh_start)
10811 .then(|| Some("stalled-run".to_owned()))
10812 .flatten();
10813 assert_eq!(
10814 choose_starter(None, false, unfinished.as_deref()),
10815 Starter::Resume("stalled-run".to_owned()),
10816 "manual release must preserve the normal resume path"
10817 );
10818 }
10819
10820 #[test]
10821 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
10822 let mut state = run_state(RunStatus::Blocked);
10823 state.config.graph.review_rounds = 3;
10824 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
10825 assert!(exhausted_review_budget(&state));
10826
10827 state.reviews.pop();
10829 assert!(!exhausted_review_budget(&state));
10830
10831 let mut stalled = run_state(RunStatus::Stalled);
10834 stalled.config.graph.review_rounds = 1;
10835 stalled.reviews = vec![review_round(1)];
10836 assert!(!exhausted_review_budget(&stalled));
10837 }
10838
10839 fn review_round(round: usize) -> crate::run::ReviewRound {
10840 crate::run::ReviewRound {
10841 round,
10842 head: "deadbeef".to_owned(),
10843 verified_head: None,
10844 verified_at: None,
10845 reviews: Vec::new(),
10846 e2e: Vec::new(),
10847 verify_retried: false,
10848 e2e_deferred: false,
10849 e2e_defer_reason: None,
10850 fix: None,
10851 blocking: 0,
10852 answered: 1,
10853 expected: 1,
10854 clean: false,
10855 progressed: true,
10856 vote_split: false,
10857 reconsideration: Vec::new(),
10858 verdict: None,
10859 }
10860 }
10861
10862 #[test]
10863 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
10864 let mut run = RunState::new(
10865 PathBuf::from("/repo"),
10866 "main".to_owned(),
10867 "abc1234def".to_owned(),
10868 "add retries".to_owned(),
10869 Config::default(),
10870 );
10871 run.status = RunStatus::Judging;
10872 run.parked = true;
10873 let mut task = Task::new(
10874 "add retries".to_owned(),
10875 "add retries".to_owned(),
10876 PathBuf::from("/repo"),
10877 crate::queue::Source::Human,
10878 );
10879 task.status = TaskStatus::Failed;
10880 task.runs = vec![run.id.clone()];
10881 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
10882 assert!(with(&task, &run), "parked after judging is the case");
10883 let mut parked_task = task.clone();
10884 parked_task.status = TaskStatus::Parked;
10885 assert!(with(&parked_task, &run), "the Parked status is the case");
10886
10887 let mut not_parked = run.clone();
10888 not_parked.parked = false;
10889 not_parked.status = RunStatus::Stalled;
10890 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
10891
10892 let mut fresh = task.clone();
10893 fresh.fresh_start = true;
10894 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
10895
10896 let mut review = task.clone();
10897 review.review_branch = Some("magi/x/A".to_owned());
10898 assert!(!with(&review, &run), "review is ranked before resume");
10899
10900 let mut held = task.clone();
10901 held.status = TaskStatus::Held;
10902 assert!(!with(&held, &run), "a hold stays visible to the conductor");
10903
10904 let mut released = run.clone();
10905 released.released_to = Some("20260901-000000-new1".to_owned());
10906 assert!(!with(&task, &released), "nothing left to resume into");
10907
10908 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
10909 "unreadable"
10910 )));
10911 }
10912
10913 #[test]
10914 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
10915 let mut released = RunState::new(
10916 PathBuf::from("/repo"),
10917 "main".to_owned(),
10918 "abc1234def".to_owned(),
10919 "add retries".to_owned(),
10920 Config::default(),
10921 );
10922 released.status = RunStatus::Blocked;
10923 assert_eq!(
10924 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
10925 Some(released.id.clone())
10926 );
10927 released.released_to = Some("20260901-000000-new1".to_owned());
10928 assert_eq!(
10929 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
10930 None,
10931 "there is nothing left to resume it into"
10932 );
10933 }
10934
10935 #[test]
10936 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
10937 let mut exhausted = RunState::new(
10946 PathBuf::from("/repo"),
10947 "main".to_owned(),
10948 "abc1234def".to_owned(),
10949 "add retries".to_owned(),
10950 Config::default(),
10951 );
10952 exhausted.status = RunStatus::Blocked;
10953 exhausted.config.graph.review_rounds = 1;
10954 exhausted.reviews = vec![review_round(1)];
10955
10956 assert_eq!(
10957 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
10958 None,
10959 "an exhausted `Blocked` run must not be offered as resumable"
10960 );
10961
10962 let mut has_budget_left = RunState::new(
10965 PathBuf::from("/repo"),
10966 "main".to_owned(),
10967 "abc1234def".to_owned(),
10968 "add retries".to_owned(),
10969 Config::default(),
10970 );
10971 has_budget_left.status = RunStatus::Blocked;
10972 has_budget_left.config.graph.review_rounds = 3;
10973 has_budget_left.reviews = vec![review_round(1)];
10974
10975 assert_eq!(
10976 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
10977 Ok(has_budget_left.clone())
10978 }),
10979 Some(has_budget_left.id.clone())
10980 );
10981 }
10982
10983 #[test]
10984 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
10985 let mut run = RunState::new(
10986 PathBuf::from("/repo"),
10987 "main".to_owned(),
10988 "abc1234def".to_owned(),
10989 "add retries".to_owned(),
10990 Config::default(),
10991 );
10992 run.status = RunStatus::Blocked;
10993 run.config.graph.review_rounds = 1;
10994 run.reviews = vec![review_round(1)];
10995 run.pr_publication_pending = true;
10996 assert_eq!(
10997 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
10998 Some(run.id.clone()),
10999 "a spent review budget must not turn a failed publication into a re-competition"
11000 );
11001 }
11002
11003 #[test]
11004 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
11005 let mut task = Task::new(
11006 "recover work".into(),
11007 "recover work".into(),
11008 PathBuf::from("/repo"),
11009 crate::queue::Source::Human,
11010 );
11011 let mut history = std::collections::BTreeMap::new();
11012 for index in 0..8 {
11013 let mut run = run_state(RunStatus::Blocked);
11014 run.id = format!("20260101-000000-{index:04}");
11015 run.base_commit = format!("upstream-{index}");
11016 if index > 0 {
11018 run.candidates.clear();
11019 }
11020 task.runs.extend([run.id.clone(), run.id.clone()]);
11021 history.insert(run.id.clone(), run);
11022 task.requeue();
11023 }
11024 let check = |task: &Task, starter: &Starter, limit| {
11025 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
11026 };
11027 assert!(
11028 check(&task, &Starter::Start, 8)
11029 .unwrap()
11030 .contains("8 unretired")
11031 );
11032 assert!(check(&task, &Starter::Start, 9).is_none());
11033 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
11034 task.release_operator("owner", None);
11035 assert_eq!(task.attempts, 0);
11036 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
11037 assert!(check(&restored, &Starter::Start, 8).is_some());
11038 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
11039 assert!(check(&restored, &Starter::Start, 0).is_none());
11040 }
11041
11042 #[test]
11043 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
11044 let mut task = Task::new(
11045 "recover work".into(),
11046 "recover work".into(),
11047 PathBuf::from("/repo"),
11048 crate::queue::Source::Human,
11049 );
11050 let mut run = run_state(RunStatus::Superseded);
11051 run.candidates.clear();
11052 task.runs.push(run.id.clone());
11053 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
11054 run.retired_to = Some("authority".into());
11055 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11056 run.retired_to = None;
11057 run.released_to = Some("authority".into());
11058 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11059 assert!(
11060 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
11061 .unwrap()
11062 .contains("Cannot verify")
11063 );
11064 assert!(
11066 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
11067 panic!("resume must not load historical attempts")
11068 })
11069 .is_none()
11070 );
11071 }
11072
11073 #[test]
11074 fn retired_resume_requires_a_valid_same_repository_authority() {
11075 let mut source = run_state(RunStatus::Blocked);
11076 let mut empty = run_state(RunStatus::Blocked);
11077 empty.retired_to = Some(source.id.clone());
11078 let runs = vec![source.id.clone(), empty.id.clone()];
11079 let resolve = |source: &RunState, empty: &RunState| {
11080 unfinished_run_with(&runs, "fixture", |id| {
11081 if id == source.id {
11082 Ok(source.clone())
11083 } else if id == empty.id {
11084 Ok(empty.clone())
11085 } else {
11086 bail!("unreadable authority")
11087 }
11088 })
11089 };
11090 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
11091 assert_eq!(
11092 choose_starter(None, false, resolve(&source, &empty).as_deref()),
11093 Starter::Resume(source.id.clone())
11094 );
11095 source.retired_to = Some(empty.id.clone());
11096 assert!(
11097 resolve(&source, &empty).is_none(),
11098 "cycles do not revive obsolete runs"
11099 );
11100 source.retired_to = None;
11101 source.repo = PathBuf::from("/another-repository");
11102 assert!(resolve(&source, &empty).is_none());
11103 source.repo = empty.repo.clone();
11104 source.released_to = Some("later".into());
11105 assert!(resolve(&source, &empty).is_none());
11106 empty.retired_to = Some("unlisted-run".into());
11107 assert!(resolve(&source, &empty).is_none());
11108 }
11109
11110 #[test]
11111 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
11112 let mut older_stalled = RunState::new(
11120 PathBuf::from("/repo"),
11121 "main".to_owned(),
11122 "abc1234def".to_owned(),
11123 "add retries".to_owned(),
11124 Config::default(),
11125 );
11126 older_stalled.status = RunStatus::Stalled;
11127
11128 let mut newest_exhausted = RunState::new(
11129 PathBuf::from("/repo"),
11130 "main".to_owned(),
11131 "abc1234def".to_owned(),
11132 "add retries".to_owned(),
11133 Config::default(),
11134 );
11135 newest_exhausted.status = RunStatus::Blocked;
11136 newest_exhausted.config.graph.review_rounds = 1;
11137 newest_exhausted.reviews = vec![review_round(1)];
11138
11139 assert_eq!(
11140 unfinished_run_with(
11141 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
11142 "t",
11143 |_| Ok(newest_exhausted.clone())
11144 ),
11145 None,
11146 "the newest run is exhausted, so nothing here is worth resuming - \
11147 least of all the older, already-superseded run"
11148 );
11149 }
11150
11151 #[test]
11152 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
11153 assert_eq!(
11154 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
11155 Err(anyhow::anyhow!("fixture is absent"))
11156 }),
11157 None
11158 );
11159 }
11160
11161 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
11162 let mut q = ask::Question::new(
11163 run.to_owned(),
11164 "implement".to_owned(),
11165 "impl-A".to_owned(),
11166 "continue?".to_owned(),
11167 String::new(),
11168 vec!["resume で続行する".to_owned(), "other".to_owned()],
11169 );
11170 q.actions.insert("resume で続行する".to_owned(), action);
11171 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
11172 .unwrap();
11173 q
11174 }
11175
11176 fn held_task_with(run: &str) -> Task {
11177 let mut t = task();
11178 t.runs = vec![run.to_owned()];
11179 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
11180 t
11181 }
11182
11183 fn resume_action(run: &str) -> ask::ChoiceAction {
11184 ask::ChoiceAction::Resume { run: run.into() }
11185 }
11186
11187 #[test]
11188 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
11189 for parked in [false, true] {
11190 let mut t = task();
11191 t.start("A".into());
11192 t.start("B".into());
11193 assert_eq!(t.runs, vec!["A", "B"]);
11194 t.start("A".into());
11195 let mut resumed = run_state(RunStatus::Blocked);
11196 resumed.id = "A".into();
11197 resumed.parked = parked;
11198 resumed.driver_exited = true;
11199 t.hold_machine(Some("resumed run stopped".into()));
11200 let q = action_question("A", resume_action("A"));
11201 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
11202 assert_eq!(
11203 decide_action(&t, &q, &PHRASES_EN, |id| {
11204 assert_eq!(id, "A");
11205 Ok(resumed.clone())
11206 }),
11207 ActionDecision::Resume("A".into())
11208 );
11209 assert_eq!(
11210 action_standing(&t, &action_question("B", resume_action("B"))),
11211 ActionStanding::Stale
11212 );
11213 t.release_operator("cli", None);
11214 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
11215 assert_eq!(id, "A");
11216 Ok(resumed.clone())
11217 });
11218 assert_eq!(
11219 choose_starter(None, false, unfinished.as_deref()),
11220 Starter::Resume("A".into())
11221 );
11222 assert_eq!(t.runs, vec!["B", "A"]);
11223 }
11224 }
11225
11226 #[test]
11227 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
11228 let mut state = crate::land::tests::repair_state(true, true);
11229 state.status = RunStatus::Landing;
11230 state.parked = true;
11231 state.driver_exited = true;
11232 let mut q = ask::Question::new(
11233 state.id.clone(),
11234 land::APPROVAL_NODE.into(),
11235 "land".into(),
11236 "Merge?".into(),
11237 String::new(),
11238 vec![land::APPROVE.into(), land::HOLD.into()],
11239 );
11240 q.id = state.id.clone();
11241 state.land_approval = Some(crate::run::LandApproval {
11242 question: q.id.clone(),
11243 head: "current-head".into(),
11244 });
11245 assert!(land::upgrade_repair_choice(&mut q, &state));
11246 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11247 let mut task = held_task_with(&state.id);
11248 task.status = TaskStatus::Parked;
11249 assert_eq!(
11250 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11251 ActionDecision::Resume(state.id.clone())
11252 );
11253 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
11254 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
11255 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
11256 assert_eq!(
11257 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11258 ActionDecision::Resume(state.id.clone()),
11259 "a repair answer newer than the manual hold resumes the existing owner"
11260 );
11261 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
11262 for answered_at in [q.answered_at, None] {
11263 let mut older = q.clone();
11264 older.answered_at = answered_at;
11265 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
11266 panic!("a later manual hold must prevent dispatch")
11267 });
11268 let ActionDecision::Refuse(reason) = refused else {
11269 panic!("expected an actionable refusal, got {refused:?}");
11270 };
11271 assert!(reason.contains("later operator decision"));
11272 assert!(reason.contains(&format!("magi task release {}", task.id)));
11273 }
11274 task.operator_decided_at = None;
11275 let mut superseded = state.clone();
11276 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
11277 assert_eq!(
11278 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
11279 ActionDecision::Stale
11280 );
11281 for case in 1..4 {
11282 let mut changed = state.clone();
11283 match case {
11284 1 => changed.status = RunStatus::Blocked,
11285 2 => changed.released_to = Some("new-owner".into()),
11286 _ => changed.land_approval.as_mut().unwrap().head.clear(),
11287 }
11288 assert!(matches!(
11289 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
11290 ActionDecision::Refuse(_)
11291 ));
11292 }
11293 assert!(matches!(
11294 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
11295 ActionDecision::Refuse(_)
11296 ));
11297 task.runs.push("replacement".into());
11298 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
11299 assert_eq!(
11300 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
11301 "old owner must not load"
11302 )),
11303 ActionDecision::Stale
11304 );
11305 }
11306
11307 #[test]
11308 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
11309 let mut state = crate::land::tests::repair_state(true, true);
11310 state.status = RunStatus::Landing;
11311 state.parked = true;
11312 state.driver_exited = true;
11313 let mut q = ask::Question::new(
11314 state.id.clone(),
11315 land::APPROVAL_NODE.into(),
11316 "land".into(),
11317 "Merge?".into(),
11318 String::new(),
11319 vec![land::APPROVE.into(), land::HOLD.into()],
11320 );
11321 q.id = state.id.clone();
11322 state.land_approval = Some(crate::run::LandApproval {
11323 question: q.id.clone(),
11324 head: "current-head".into(),
11325 });
11326 state.save().unwrap();
11327 land::upgrade_repair_choice(&mut q, &state);
11328 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11329 Questions::open().put(&mut q).unwrap();
11330 let tmp = tempfile::tempdir().unwrap();
11331 let queue = Queue::at(tmp.path().join("queue"));
11332 let mut task = held_task_with(&state.id);
11333 task.status = TaskStatus::Parked;
11334 queue.put(&mut task).unwrap();
11335 let empty = Arc::new(tokio::sync::Semaphore::new(0));
11336 let free = Arc::new(tokio::sync::Semaphore::new(1));
11337 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
11338 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
11339 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
11340 assert!(claim.permit.is_some());
11341 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
11342 drop(claim);
11343 Questions::open()
11344 .update(&q.id, |q| {
11345 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
11346 Ok(())
11347 })
11348 .unwrap();
11349 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
11350 assert!(claim.permit.is_none());
11351 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
11352 }
11353
11354 #[test]
11355 fn decide_action_resumes_only_the_latest_resumable_run() {
11356 let t = held_task_with("r1");
11357 let q = action_question("r1", resume_action("r1"));
11358 let load = |s: RunState| move |_: &str| Ok(s);
11359 assert_eq!(
11360 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
11361 ActionDecision::Resume("r1".into())
11362 );
11363 let q_other = action_question("r1", resume_action("r0"));
11365 assert!(matches!(
11366 decide_action(
11367 &t,
11368 &q_other,
11369 &PHRASES_EN,
11370 load(run_state(RunStatus::Blocked))
11371 ),
11372 ActionDecision::Refuse(_)
11373 ));
11374 assert!(matches!(
11376 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
11377 ActionDecision::Refuse(_)
11378 ));
11379 let mut released = run_state(RunStatus::Blocked);
11381 released.released_to = Some("elsewhere".into());
11382 assert!(matches!(
11383 decide_action(&t, &q, &PHRASES_EN, load(released)),
11384 ActionDecision::Refuse(_)
11385 ));
11386 assert!(matches!(
11388 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
11389 ActionDecision::Refuse(_)
11390 ));
11391 }
11392
11393 #[test]
11394 fn decide_action_ignores_a_question_about_an_earlier_run() {
11395 let mut t = held_task_with("r1");
11396 t.runs.push("r2".to_owned());
11397 let never = |_: &str| -> Result<RunState> { bail!("not read") };
11398 assert_eq!(
11399 decide_action(
11400 &t,
11401 &action_question("r1", ask::ChoiceAction::Done),
11402 &PHRASES_EN,
11403 never
11404 ),
11405 ActionDecision::Stale
11406 );
11407 }
11408
11409 #[test]
11410 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
11411 let mut t = held_task_with("r1");
11412 let never = |_: &str| -> Result<RunState> { bail!("not read") };
11413 assert_eq!(
11414 decide_action(
11415 &t,
11416 &action_question("r1", ask::ChoiceAction::Requeue),
11417 &PHRASES_EN,
11418 never
11419 ),
11420 ActionDecision::Requeue
11421 );
11422 let done_q = action_question("r1", ask::ChoiceAction::Done);
11423 assert_eq!(
11424 decide_action(&t, &done_q, &PHRASES_EN, never),
11425 ActionDecision::Done
11426 );
11427 t.mark_action_applied(&done_q.id);
11428 assert_eq!(
11429 decide_action(&t, &done_q, &PHRASES_EN, never),
11430 ActionDecision::Skip
11431 );
11432
11433 let mut plain = action_question("r1", ask::ChoiceAction::Done);
11435 plain.actions.clear();
11436 assert_eq!(
11437 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
11438 ActionDecision::Skip
11439 );
11440 let mut running = held_task_with("r1");
11442 running.status = TaskStatus::Running;
11443 assert_eq!(
11444 decide_action(
11445 &running,
11446 &action_question("r1", ask::ChoiceAction::Done),
11447 &PHRASES_EN,
11448 never
11449 ),
11450 ActionDecision::Skip
11451 );
11452 }
11453
11454 #[test]
11455 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
11456 crate::run::pin_test_home();
11457 for legacy in [false, true] {
11458 let dir = tempfile::tempdir().unwrap();
11459 let queue = Queue::at(dir.path().join("queue"));
11460 let questions = Questions::at(dir.path().join("questions"));
11461 let run = "20260101-000000-hold";
11462 let mut task = held_task_with(run);
11463 task.status = TaskStatus::Parked;
11464 let mut q = ask::Question::new(
11465 run.into(),
11466 land::APPROVAL_NODE.into(),
11467 "land".into(),
11468 "Repair?".into(),
11469 String::new(),
11470 vec![land::REPAIR.into()],
11471 );
11472 q.actions.insert(land::REPAIR.into(), resume_action(run));
11473 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11474 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
11476 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
11477 questions.put(&mut q).unwrap();
11478 task.hold_manual(Some("Keep this PR paused".into()));
11479 if legacy {
11480 task.hold_source = None;
11481 }
11482 queue.put(&mut task).unwrap();
11483
11484 apply_task_choice_actions(&queue, &questions, dir.path());
11485 let after = queue.get(&task.id).unwrap();
11486 assert!(after.operator_held());
11487 assert_eq!(after.hold_source, task.hold_source);
11488 assert_eq!(after.operator_decided_at, task.operator_decided_at);
11489 assert_eq!(after.runs, task.runs);
11490 assert_eq!(after.instruction, task.instruction);
11491 assert!(after.resume_override.is_none());
11492 assert!(after.action_applied(&q.id));
11493 let reason = after.hold_reason.as_deref().unwrap();
11494 assert!(reason.starts_with("Keep this PR paused"));
11495 assert!(reason.contains("later operator decision"));
11496 assert!(reason.contains(&format!("magi task release {}", task.id)));
11497
11498 for recovery in [
11499 crate::conduct::Recovery::Review,
11500 crate::conduct::Recovery::Requeue,
11501 ] {
11502 crate::conduct::apply(
11503 &queue,
11504 &questions,
11505 &crate::conduct::Verdict {
11506 decisions: vec![crate::conduct::Decision {
11507 id: task.id.clone(),
11508 recovery: Some(recovery),
11509 ..Default::default()
11510 }],
11511 },
11512 )
11513 .unwrap();
11514 }
11515 apply_task_choice_actions(&queue, &questions, dir.path());
11518 let still_held = queue.get(&task.id).unwrap();
11519 assert!(still_held.operator_held());
11520 assert_eq!(still_held.hold_reason, after.hold_reason);
11521 assert_eq!(still_held.runs, task.runs);
11522 assert!(still_held.review_branch.is_none());
11523 assert!(!still_held.fresh_start);
11524 assert!(queue.next_runnable().is_none());
11525 }
11526 }
11527
11528 #[test]
11529 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
11530 for held in [false, true] {
11531 for retry in [false, true] {
11532 let dir = tempfile::tempdir().unwrap();
11533 let queue = Queue::at(dir.path().join("queue"));
11534 let questions = Questions::at(dir.path().join("questions"));
11535 let home = dir.path().join("home");
11536 let mut t = task();
11537 if held {
11538 t.hold_machine(Some("driver state unknown".into()));
11539 }
11540 queue.put(&mut t).unwrap();
11541 let verdict = serde_json::from_value(serde_json::json!({
11542 "decisions": [{
11543 "id": t.id,
11544 "question": "Driver state unknown; retry the checks or hold?",
11545 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
11546 "actions": {
11547 "daemon: recheck and retry": {"do": "requeue"},
11548 "daemon: hold the task": {"do": "hold"}
11549 }
11550 }]
11551 }))
11552 .unwrap();
11553 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
11554 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
11555 let mut q = questions.list().pop().unwrap();
11556 let choice = if retry {
11557 "daemon: recheck and retry"
11558 } else {
11559 "daemon: hold the task"
11560 };
11561 assert_eq!(q.actions.len(), 2);
11562 q.answer(ask::Answer::Choice(choice.into())).unwrap();
11563 questions.put(&mut q).unwrap();
11564
11565 questions.beat(&q.id, ask::WaiterKind::Deputy);
11567 for _ in 0..2 {
11568 resolve_blockers(&queue, &questions);
11569 apply_task_choice_actions(&queue, &questions, &home);
11570 let waiting = queue.get(&t.id).unwrap();
11571 assert_eq!(waiting.status, TaskStatus::Blocked);
11572 assert_eq!(waiting.blocked_by, [q.id.clone()]);
11573 assert!(waiting.answers.is_empty());
11574 assert!(!waiting.action_applied(&q.id));
11575 assert!(runnable(&queue).is_empty());
11576 let free = Arc::new(tokio::sync::Semaphore::new(1));
11577 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
11578 }
11579 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
11580
11581 resolve_blockers(&queue, &questions);
11583 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
11584 apply_task_choice_actions(&queue, &questions, &home);
11585 let after = queue.get(&t.id).unwrap();
11586 assert_eq!(
11587 after.status,
11588 if retry {
11589 TaskStatus::Queued
11590 } else {
11591 TaskStatus::Held
11592 }
11593 );
11594 assert!(after.action_applied(&q.id));
11595 assert_eq!(after.operator_held(), !retry);
11596 assert_eq!(after.answers.len(), 1);
11597
11598 let mut changed = after;
11600 if retry {
11601 changed.hold_manual(Some("later hold".into()));
11602 } else {
11603 changed.release_operator("test", None);
11604 }
11605 let expected = changed.status;
11606 queue.put(&mut changed).unwrap();
11607 apply_task_choice_actions(&queue, &questions, &home);
11608 assert_eq!(queue.get(&t.id).unwrap().status, expected);
11609 }
11610 }
11611 }
11612
11613 #[test]
11614 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
11615 let dir = tempfile::tempdir().unwrap();
11616 let queue = Queue::at(dir.path().join("queue"));
11617 let questions = Questions::at(dir.path().join("questions"));
11618 let home = dir.path().join("home");
11619 let mut state = run_state(RunStatus::Blocked);
11620 state.id = "20260101-000000-act1".to_owned();
11621 state.save_under(&home).unwrap();
11622
11623 let mut t = held_task_with(&state.id);
11624 queue.put(&mut t).unwrap();
11625 let mut q = action_question(&state.id, resume_action(&state.id));
11626 questions.put(&mut q).unwrap();
11627
11628 apply_task_choice_actions(&queue, &questions, &home);
11629 let after = queue.get(&t.id).unwrap();
11630 assert_eq!(after.status, TaskStatus::Queued);
11631 assert!(!after.fresh_start);
11632 assert!(after.action_applied(&q.id));
11633 let pin = after.resume_override.clone().unwrap();
11634 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
11635 assert!(pin.forced);
11636
11637 let mut again = queue.get(&t.id).unwrap();
11639 again.hold_machine(Some("later".into()));
11640 queue.put(&mut again).unwrap();
11641 apply_task_choice_actions(&queue, &questions, &home);
11642 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11643 }
11644
11645 #[test]
11646 fn a_delivered_answer_is_still_acted_on_exactly_once() {
11647 let dir = tempfile::tempdir().unwrap();
11648 let queue = Queue::at(dir.path().join("queue"));
11649 let questions = Questions::at(dir.path().join("questions"));
11650 let home = dir.path().join("home");
11651 let mut state = run_state(RunStatus::Blocked);
11652 state.id = "20260101-000000-act2".to_owned();
11653 state.save_under(&home).unwrap();
11654
11655 let mut t = held_task_with(&state.id);
11656 queue.put(&mut t).unwrap();
11657 let mut q = action_question(&state.id, resume_action(&state.id));
11658 q.answer_delivered = true;
11659 questions.put(&mut q).unwrap();
11660
11661 apply_task_choice_actions(&queue, &questions, &home);
11664 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
11665 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
11666
11667 let mut again = queue.get(&t.id).unwrap();
11669 again.hold_machine(Some("later".into()));
11670 queue.put(&mut again).unwrap();
11671 apply_task_choice_actions(&queue, &questions, &home);
11672 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11673 }
11674
11675 #[test]
11676 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
11677 let dir = tempfile::tempdir().unwrap();
11678 let queue = Queue::at(dir.path().join("queue"));
11679 let questions = Questions::at(dir.path().join("questions"));
11680 let home = dir.path().join("home");
11681 let mut state = run_state(RunStatus::Blocked);
11682 state.id = "20260101-000000-act3".to_owned();
11683 state.save_under(&home).unwrap();
11684 let mut t = held_task_with(&state.id);
11685 queue.put(&mut t).unwrap();
11686 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
11687 questions.put(&mut q).unwrap();
11688
11689 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
11690 apply_task_choice_actions(&queue, &questions, &home);
11691 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11692
11693 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
11694 apply_task_choice_actions(&queue, &questions, &home);
11695 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
11696 }
11697
11698 #[test]
11699 fn action_standing_tells_the_reasons_apart() {
11700 let mut t = held_task_with("r1");
11701 let q = action_question("r1", ask::ChoiceAction::Requeue);
11702 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
11703 let mut plain = q.clone();
11704 plain.actions.clear();
11705 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
11706
11707 t.mark_action_applied(&q.id);
11709 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
11710
11711 let mut t = held_task_with("r1");
11713 t.start("r2".to_owned());
11714 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
11715 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
11717 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
11718 t.status = TaskStatus::Blocked;
11719 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
11720
11721 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
11723 c.node = crate::conduct::NODE.to_owned();
11724 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
11725 assert!(!ActionStanding::Busy.daemon_owns());
11726 }
11727
11728 #[test]
11729 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
11730 let dir = tempfile::tempdir().unwrap();
11731 let queue = Queue::at(dir.path().join("queue"));
11732 let questions = Questions::at(dir.path().join("questions"));
11733 let home = dir.path().join("home");
11734 let mut state = run_state(RunStatus::Blocked);
11735 state.id = "20260101-000000-act4".to_owned();
11736 state.save_under(&home).unwrap();
11737 let mut t = held_task_with(&state.id);
11738 t.status = TaskStatus::Running;
11739 queue.put(&mut t).unwrap();
11740 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
11741 q.cwd = Some(dir.path().display().to_string());
11742 questions.put(&mut q).unwrap();
11743
11744 let running = queue.get(&t.id).unwrap();
11747 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
11748 assert!(matches!(
11749 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
11750 crate::waiter::Action::Deliver(_)
11751 ));
11752 apply_task_choice_actions(&queue, &questions, &home);
11754 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
11755 let mut back = queue.get(&t.id).unwrap();
11756 back.hold_machine(Some("later".into()));
11757 queue.put(&mut back).unwrap();
11758 apply_task_choice_actions(&queue, &questions, &home);
11759 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
11760 let mut again = queue.get(&t.id).unwrap();
11761 again.hold_machine(Some("again".into()));
11762 queue.put(&mut again).unwrap();
11763 apply_task_choice_actions(&queue, &questions, &home);
11764 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11765 }
11766}