1use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::sync::atomic::{AtomicBool, Ordering};
52use std::sync::{Mutex, MutexGuard};
53use std::time::Duration;
54
55use anyhow::{Context, Result, bail};
56use jiff::Timestamp;
57use serde::{Deserialize, Serialize};
58use tokio::sync::Notify;
59
60use crate::ask::{self, Questions};
61use crate::clean;
62use crate::conduct::Conductor;
63use crate::config::{Config, MergeMode};
64use crate::graph::Runner;
65use crate::land;
66use crate::notices::{self, Link, Notice};
67use crate::queue::{Queue, Task, TaskStatus};
68use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
69use crate::triage;
70
71pub const SCHEMA: u32 = 1;
73
74pub const HEARTBEAT: Duration = Duration::from_secs(5);
78
79pub const STALE_SECS: i64 = 30;
88
89pub const POLL: Duration = Duration::from_secs(5);
91
92pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
96
97pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
119#[serde(default)]
120pub struct Current {
121 pub task: String,
123 pub run: String,
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct Status {
135 pub schema: u32,
137 pub pid: u32,
139 pub started_at: Timestamp,
141 pub updated_at: Timestamp,
143 pub idle: bool,
145 pub current: Vec<Current>,
151 pub completed: usize,
153 pub polls: u64,
155 #[serde(default)]
157 pub question_recovery: Option<crate::question_delivery::Recovery>,
158}
159
160impl Status {
161 #[must_use]
163 pub fn new() -> Self {
164 let now = Timestamp::now();
165 Self {
166 schema: SCHEMA,
167 pid: std::process::id(),
168 started_at: now,
169 updated_at: now,
170 idle: true,
171 current: Vec::new(),
172 completed: 0,
173 polls: 0,
174 question_recovery: None,
175 }
176 }
177}
178
179impl Default for Status {
180 fn default() -> Self {
181 Self::new()
182 }
183}
184
185#[derive(Debug, Clone)]
187pub struct Opts {
188 pub repo: PathBuf,
190 pub config: Option<PathBuf>,
192 pub poll: Duration,
194 pub max_attempts: usize,
196 pub once: bool,
198 pub merge: Option<String>,
200 pub worktrees_root: Option<PathBuf>,
209}
210
211impl Default for Opts {
212 fn default() -> Self {
213 Self {
214 repo: PathBuf::from("."),
215 config: None,
216 poll: POLL,
217 max_attempts: 2,
218 once: false,
219 merge: None,
220 worktrees_root: None,
221 }
222 }
223}
224
225fn max_concurrent(n: usize) -> usize {
230 n.max(1)
231}
232
233#[must_use]
235pub fn status_path() -> PathBuf {
236 crate::run::home().join("daemon.json")
237}
238
239pub fn write_status(status: &Status) -> Result<()> {
241 write_status_to(&status_path(), status)
242}
243
244pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
249 if let Some(parent) = path.parent() {
250 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
251 }
252 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
253 let tmp = path.with_extension("json.tmp");
254 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
255 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
256 Ok(())
257}
258
259pub fn clear_status() {
262 clear_status_at(&status_path());
263}
264
265fn clear_status_at(path: &Path) {
269 let _ = std::fs::remove_file(path);
270}
271
272#[derive(Debug, Clone, Default)]
285pub struct Stop {
286 stopped: Arc<AtomicBool>,
290 busy: Arc<std::sync::atomic::AtomicUsize>,
296 wake: Arc<Notify>,
300 pause: crate::graph::Pause,
303}
304
305impl Stop {
306 #[must_use]
308 pub fn new() -> Self {
309 Self::default()
310 }
311
312 pub fn stop(&self) {
315 self.stopped.store(true, Ordering::SeqCst);
316 self.wake.notify_one();
320 }
321
322 #[must_use]
324 pub fn stopped(&self) -> bool {
325 self.stopped.load(Ordering::SeqCst)
326 }
327
328 #[must_use]
336 pub fn finishing(&self) -> bool {
337 self.stopped() && self.busy_now()
338 }
339
340 pub fn park(&self, cause: crate::run::ParkCause) {
351 self.pause.park_for(cause);
352 self.stop();
353 }
354
355 #[must_use]
357 pub fn parking(&self) -> bool {
358 self.pause.parked()
359 }
360
361 #[must_use]
363 pub fn pause(&self) -> crate::graph::Pause {
364 self.pause.clone()
365 }
366
367 #[must_use]
373 pub fn busy_now(&self) -> bool {
374 self.busy.load(Ordering::SeqCst) > 0
375 }
376
377 fn enter(&self) {
379 self.busy.fetch_add(1, Ordering::SeqCst);
380 }
381
382 fn exit(&self) {
385 self.busy.fetch_sub(1, Ordering::SeqCst);
386 }
387
388 async fn idle(&self, poll: Duration) {
390 tokio::select! {
391 () = tokio::time::sleep(poll) => {}
392 () = self.wake.notified() => {}
393 }
394 }
395}
396
397#[derive(Debug, Clone, Default, Deserialize)]
404#[serde(default)]
405pub struct Reading {
406 pub schema: u32,
408 pub pid: Option<u32>,
410 pub started_at: Option<Timestamp>,
412 pub updated_at: Option<Timestamp>,
414 pub idle: bool,
416 #[serde(deserialize_with = "de_current")]
428 pub current: Vec<Current>,
429 pub completed: u64,
431 pub polls: u64,
433 #[serde(default)]
435 pub question_recovery: Option<crate::question_delivery::Recovery>,
436}
437
438fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
441where
442 D: serde::Deserializer<'de>,
443{
444 #[derive(Deserialize)]
445 #[serde(untagged)]
446 enum Shape {
447 Many(Vec<Current>),
448 One(Current),
449 }
450 Ok(
451 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
452 Shape::Many(v) => v,
453 Shape::One(c) => vec![c],
454 }),
455 )
456}
457
458impl Reading {
459 #[must_use]
462 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
463 self.updated_at
464 .map(|at| (now.as_second() - at.as_second()).max(0))
465 }
466
467 #[must_use]
471 pub fn running(&self, now: Timestamp) -> bool {
472 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
473 }
474}
475
476#[must_use]
483pub fn read_status(home: &Path) -> Option<Reading> {
484 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
485 serde_json::from_str(&body).ok()
486}
487
488#[must_use]
500pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
501 read_status(home)
502 .filter(|reading| reading.running(now))
503 .map(|reading| reading.current)
504 .unwrap_or_default()
505}
506
507#[must_use]
509pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
510 current_work(home, now).iter().any(|c| c.run == run)
511}
512
513#[must_use]
523pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
524 current_work(home, now)
525 .iter()
526 .any(|c| crate::run::short_of(&c.run) == short)
527}
528
529#[must_use]
531pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
532 current_work(home, now).iter().any(|c| c.task == task)
533}
534
535pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
579 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
580}
581
582fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
586where
587 F: Fn(u32) -> bool,
588{
589 let this_process = std::process::id();
590 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
591 .into_iter()
592 .flatten()
593 .flatten()
594 .map(|e| e.path())
595 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
596 .filter(|p| {
597 match std::fs::read_to_string(p)
598 .ok()
599 .and_then(|body| body.trim().parse::<u32>().ok())
600 {
601 Some(pid) if pid == this_process => false,
605 Some(pid) => !pid_alive(pid),
606 None => p
607 .metadata()
608 .and_then(|m| m.modified())
609 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
610 .is_ok_and(|age| age >= older_than),
611 }
612 })
613 .filter(|p| std::fs::remove_file(p).is_ok())
614 .filter_map(|p| {
615 p.file_stem()
616 .and_then(|s| s.to_str())
617 .map(std::borrow::ToOwned::to_owned)
618 })
619 .collect();
620 swept.sort_unstable();
621 swept
622}
623
624fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
629 task.status == TaskStatus::Running
630 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
631 && !is_working_on_task(home, &task.id, now)
632}
633
634fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
637 queue
638 .list()
639 .into_iter()
640 .filter(|t| is_stalled(t, home, now))
641 .collect()
642}
643
644fn queued_tasks(queue: &Queue) -> Vec<Task> {
650 queue
651 .list()
652 .into_iter()
653 .filter(|t| t.status == TaskStatus::Queued)
654 .collect()
655}
656
657fn finished_tasks(queue: &Queue) -> Vec<Task> {
660 queue
661 .list()
662 .into_iter()
663 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
664 .collect()
665}
666
667fn resolve_blockers(queue: &Queue, questions: &Questions) {
684 for listed in queue.list() {
685 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
686 continue;
687 }
688 let Ok(_claim) = queue.claim(&listed.id) else {
689 continue;
690 };
691 let Ok(mut task) = queue.get(&listed.id) else {
692 continue;
693 };
694 if task.status != TaskStatus::Blocked {
695 continue;
696 }
697 let deleted = queue.apply_deleted_blockers(&mut task, questions);
701 if !deleted.is_empty() {
702 record(queue, &mut task);
703 for id in &deleted {
704 queue.note_dependency_deleted(&task, id);
705 }
706 if task.status != TaskStatus::Blocked {
707 continue;
708 }
709 }
710 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
711 if !missing.is_empty() {
712 let language = language_of(&task, Path::new("."));
713 let mut cause = missing.clone();
714 cause.sort();
715 cause.dedup();
716 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
717 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
718 &task.blocked_by,
719 &missing,
720 &language,
721 )));
722 record(queue, &mut task);
723 continue;
724 }
725 if let Some(q) = task.blocked_by.iter().find_map(|id| {
728 let crate::queue::Dependency::Question(full) =
729 crate::queue::dependency(queue, questions, id)
730 else {
731 return None;
732 };
733 questions.get(&full).ok().filter(|q| {
734 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
735 })
736 }) {
737 let language = language_of(&task, Path::new("."));
738 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
739 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
740 record(queue, &mut task);
741 continue;
742 }
743 let mut changed = false;
744 for id in task.blocked_by.clone() {
745 let resolved = crate::queue::dependency(queue, questions, &id);
746 if let crate::queue::Dependency::Task(full) = &resolved {
747 if queue
748 .get(full)
749 .is_ok_and(|dep| dep.status == TaskStatus::Done)
750 {
751 task.unblock(&id);
752 changed = true;
753 }
754 continue;
755 }
756 if let crate::queue::Dependency::Question(full) = resolved
757 && let Ok(q) = questions.get(&full)
758 && q.status == ask::QuestionStatus::Answered
759 {
760 let answer = match &q.answer {
761 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
762 None => String::new(),
763 };
764 task.record_answer(q.summary.clone(), answer);
765 task.unblock(&id);
766 changed = true;
767 }
768 }
769 if changed {
770 record(queue, &mut task);
771 }
772 }
773}
774
775fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
777 if crate::lang::is_japanese(language) {
778 format!(
779 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
780 q.short(),
781 q.summary
782 )
783 } else {
784 format!(
785 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
786 q.short(),
787 q.summary
788 )
789 }
790}
791
792#[derive(Debug, Clone, Copy, PartialEq, Eq)]
795pub(crate) enum ActionStanding {
796 NoAction,
798 Pending,
800 Applied,
803 Stale,
805 Busy,
807}
808
809impl ActionStanding {
810 pub(crate) fn daemon_owns(self) -> bool {
812 matches!(self, Self::Pending | Self::Applied | Self::Stale)
813 }
814}
815
816pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
820 if q.chosen_action().is_none() {
821 ActionStanding::NoAction
822 } else if task.action_applied(&q.id) {
823 ActionStanding::Applied
824 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
825 ActionStanding::Stale
826 } else if matches!(
827 task.status,
828 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
829 ) {
830 ActionStanding::Busy
831 } else {
832 ActionStanding::Pending
833 }
834}
835
836#[derive(Debug, Clone, PartialEq, Eq)]
838enum ActionDecision {
839 Skip,
841 Resume(String),
843 Requeue,
845 Done,
847 Stale,
850 Refuse(String),
853}
854
855fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
863where
864 F: FnOnce(&str) -> Result<RunState>,
865{
866 let Some(action) = q.chosen_action() else {
867 return ActionDecision::Skip;
868 };
869 if task.action_applied(&q.id)
870 || matches!(
871 task.status,
872 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
873 )
874 {
875 return ActionDecision::Skip;
876 }
877 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
880 return ActionDecision::Stale;
881 }
882 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
883 let stopping = q.node == land::APPROVAL_NODE
884 && q.resolution()
885 .as_deref()
886 .is_some_and(crate::merge_decision::stops);
887 let decision_at = if repair || stopping {
891 q.answered_at.unwrap_or(q.asked_at)
892 } else {
893 q.asked_at
894 };
895 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
896 return if repair || stopping {
897 ActionDecision::Refuse(format!(
898 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
899 if repair { "repair" } else { "stop" },
900 q.id,
901 q.run,
902 task.id
903 ))
904 } else {
905 ActionDecision::Stale
906 };
907 }
908 match action {
909 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
910 ask::ChoiceAction::Done => ActionDecision::Done,
911 ask::ChoiceAction::Resume { run } => {
912 if task.runs.last() != Some(run) {
913 return ActionDecision::Refuse((p.resume_not_latest)(
914 q.short(),
915 ask::short_id(run),
916 ));
917 }
918 match load(run) {
919 Ok(s) if stopping => {
920 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
921 || !s
922 .land_approval
923 .as_ref()
924 .is_some_and(|binding| binding.question == q.id)
925 {
926 ActionDecision::Stale
927 } else if s.liveness(false) == Liveness::Live {
928 ActionDecision::Skip
929 } else if s.status == RunStatus::Landing && !s.released() {
930 ActionDecision::Resume(run.clone())
931 } else {
932 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
933 }
934 }
935 Ok(s)
936 if q.node == land::APPROVAL_NODE
937 && q.resolution().as_deref() == Some(land::REPAIR) =>
938 {
939 if s.merge_repairs.iter().any(|r| r.question == q.id)
940 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
941 {
942 ActionDecision::Stale
945 } else if s.liveness(false) == Liveness::Live {
946 ActionDecision::Skip
947 } else if s.status != RunStatus::Landing
948 || s.released()
949 || s.reviews.is_empty()
950 || !implementers_match(task, &s)
951 || !s
952 .land_approval
953 .as_ref()
954 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
955 {
956 ActionDecision::Refuse(format!(
957 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
958 q.id, run, run
959 ))
960 } else {
961 ActionDecision::Resume(run.clone())
962 }
963 }
964 Ok(s) if s.status.resumable() && !s.released() && !exhausted_review_budget(&s) => {
965 ActionDecision::Resume(run.clone())
966 }
967 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
968 q.short(),
969 ask::short_id(run),
970 )),
971 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
972 q.short(),
973 ask::short_id(run),
974 &format!("{e:#}"),
975 )),
976 }
977 }
978 }
979}
980
981struct Phrases {
995 graph_stopped: fn(&str, &str) -> String,
997 quorum_lost: &'static str,
998 quota_took_out: &'static str,
1000 run_ended: &'static str,
1002 waiting_for_answer: &'static str,
1004 recovered_running: &'static str,
1005 no_run_to_recover: &'static str,
1006 could_not_start: &'static str,
1007 resume_not_latest: fn(&str, &str) -> String,
1008 resume_cannot_progress: fn(&str, &str) -> String,
1009 resume_unreadable: fn(&str, &str, &str) -> String,
1010 handover_refused: fn(&crate::handover::Refused) -> String,
1013 handover_hint: &'static str,
1015 reviewers_never_answered: fn(usize, usize) -> String,
1018}
1019
1020const PHRASES_EN: Phrases = Phrases {
1021 graph_stopped: |status, detail| {
1022 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1023 },
1024 quorum_lost: "the judging panel lost its quorum",
1025 quota_took_out: "; quota took out ",
1026 run_ended: "run ended ",
1027 waiting_for_answer: " - waiting for operator answer to question ",
1028 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1029 no_run_to_recover: "task was `running` with no live daemon and no readable \
1030 run to recover; held for a human to check what happened",
1031 could_not_start: "could not start the run: ",
1032 resume_not_latest: |q, run| {
1033 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1034 },
1035 resume_cannot_progress: |q, run| {
1036 format!("question {q} asked to resume run {run}, which cannot make progress")
1037 },
1038 resume_unreadable: |q, run, e| {
1039 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1040 },
1041 handover_refused: |r| r.to_string(),
1042 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1043 reviewers_never_answered: |missing, rounds| {
1044 format!(
1045 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1046 refusing to call it clean"
1047 )
1048 },
1049};
1050
1051const PHRASES_JA: Phrases = Phrases {
1052 graph_stopped: |status, detail| {
1053 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1054 },
1055 quorum_lost: "審査パネルが定足数を失いました",
1056 quota_took_out: "。クォータで脱落: ",
1057 run_ended: "run 終了: ",
1058 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1059 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1060 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1061 何が起きたか人が確認するため保留にしました",
1062 could_not_start: "run を開始できませんでした: ",
1063 resume_not_latest: |q, run| {
1064 format!(
1065 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1066 )
1067 },
1068 resume_cannot_progress: |q, run| {
1069 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1070 },
1071 resume_unreadable: |q, run, e| {
1072 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1073 },
1074 handover_refused: |r| {
1075 use crate::handover::Refused;
1076 match r {
1077 Refused::Foreign { branch, path, why } => format!(
1078 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1079 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1080 から、やり直してください(詳細: {why})"
1081 ),
1082 Refused::Unsafe { branch, path, why } => format!(
1083 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1084 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1085 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1086 ),
1087 Refused::ReleaseFailed { branch, path, run } => format!(
1088 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1089 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1090 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1091 ),
1092 }
1093 },
1094 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1095 reviewers_never_answered: |missing, rounds| {
1096 format!(
1097 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1098 クリーンとは認めません"
1099 )
1100 },
1101};
1102
1103fn phrases(language: &str) -> &'static Phrases {
1104 if crate::lang::is_japanese(language) {
1105 &PHRASES_JA
1106 } else {
1107 &PHRASES_EN
1108 }
1109}
1110
1111fn language_of(task: &Task, fallback: &Path) -> String {
1115 crate::lang::of_repo(&repo_for(task, fallback))
1116}
1117
1118pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1122 if q.node == crate::conduct::NODE {
1123 return tasks.iter().find(|t| t.id == q.run);
1124 }
1125 tasks.iter().find(|t| t.runs.contains(&q.run))
1126}
1127
1128pub(crate) fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1137 let tasks = queue.list();
1138 for q in questions.list() {
1139 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1141 continue;
1142 }
1143 if q.chosen_action().is_none() {
1144 continue;
1145 }
1146 let Some(listed) = task_of_question(&tasks, &q) else {
1147 if q.node == land::APPROVAL_NODE
1148 && q.resolution().as_deref().is_some_and(|choice| {
1149 choice == land::REPAIR || crate::merge_decision::stops(choice)
1150 })
1151 {
1152 notices::raise_in(
1153 home,
1154 Notice::warn(
1155 &format!("repair-owner:{}", q.id),
1156 format!(
1157 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1158 q.run, q.run
1159 ),
1160 ),
1161 );
1162 }
1163 continue;
1164 };
1165 if listed.action_applied(&q.id) {
1166 continue;
1167 }
1168 let Ok(_claim) = queue.claim(&listed.id) else {
1169 continue;
1170 };
1171 let Ok(mut task) = queue.get(&listed.id) else {
1172 continue;
1173 };
1174 let language = language_of(&task, Path::new("."));
1175 let decision = decide_action(&task, &q, phrases(&language), |id| {
1176 RunState::load_under(id, home)
1177 });
1178 let ran = matches!(
1179 decision,
1180 ActionDecision::Resume(_) | ActionDecision::Requeue | ActionDecision::Done
1181 );
1182 if ran && asker_may_still_read(questions, &q, home) {
1183 continue;
1192 }
1193 let done = matches!(decision, ActionDecision::Done);
1194 match decision {
1195 ActionDecision::Skip => continue,
1196 ActionDecision::Resume(run) => {
1197 task.release_operator("choice", Some(&q.id));
1198 task.resume_override = Some(crate::queue::OperatorResume {
1199 question_id: q.id.clone(),
1200 at: Timestamp::now(),
1201 conductor_rehold: None,
1202 forced: true,
1203 pinned_run: Some(run),
1204 });
1205 }
1206 ActionDecision::Stale => {}
1207 ActionDecision::Requeue => {
1208 task.release_operator("choice-requeue", Some(&q.id));
1209 task.requeue();
1210 }
1211 ActionDecision::Done => task.succeed(),
1212 ActionDecision::Refuse(why) => {
1213 if task.operator_held() {
1214 task.hold_reason = Some(match task.hold_reason.take() {
1217 Some(reason) => format!("{reason}\n\n{why}"),
1218 None => why,
1219 });
1220 } else {
1221 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1222 task.hold_machine(Some(why));
1223 }
1224 notices::raise(
1225 Notice::warn(
1226 &format!("action:{}", q.id),
1227 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1228 )
1229 .link(Link::Task {
1230 id: task.id.clone(),
1231 }),
1232 );
1233 }
1234 }
1235 task.mark_action_applied(&q.id);
1236 record(queue, &mut task);
1239 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1240 supersede_prior_runs(&task, home);
1241 }
1242 let _ = questions.update(&q.id, |r| {
1245 r.answer_delivered = true;
1246 Ok(())
1247 });
1248 }
1249}
1250
1251fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1255 let now = Timestamp::now();
1256 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1257 return true;
1258 }
1259 q.node != crate::conduct::NODE
1260 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1261 s.seats_active()
1262 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1263 })
1264}
1265
1266fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1277 let tasks = queue.list();
1278 let by_id: std::collections::BTreeMap<&str, &Task> =
1279 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1280 let referenced: Vec<&str> = tasks
1281 .iter()
1282 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1283 .collect();
1284
1285 for mut question in questions.list() {
1286 if !question.status.open() || question.node != crate::conduct::NODE {
1287 continue;
1288 }
1289 if referenced
1294 .iter()
1295 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1296 {
1297 continue;
1298 }
1299 let Some(task) = by_id.get(question.run.as_str()) else {
1300 continue;
1301 };
1302 question.abandon(format!(
1303 "task {} no longer waits for this answer",
1304 task.short()
1305 ));
1306 if let Err(e) = questions.put(&mut question) {
1307 tracing::warn!(
1308 "could not retire question {} for task {}: {e:#}",
1309 question.short(),
1310 task.short()
1311 );
1312 }
1313 }
1314}
1315
1316#[derive(Debug, Clone, Copy)]
1323pub struct Verdict {
1324 pub status: RunStatus,
1326 pub left_pr: bool,
1328 pub quota_hit: bool,
1330 pub parked: bool,
1332 pub no_viable_candidates: bool,
1339}
1340
1341pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1398 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1399}
1400
1401fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1403 if verdict.parked {
1409 task.park(detail);
1410 return;
1411 }
1412 match verdict.status {
1413 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1414 RunStatus::AlreadyInBase => task.already_landed(detail),
1415 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1416 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1417 task.stall(detail)
1418 }
1419 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1420 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1421 RunStatus::Blocked => task.fail(detail, max_attempts),
1422 RunStatus::VerifiedNoop => task.handed_off(detail),
1423 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1424 }
1425}
1426
1427pub fn supersede_prior_runs(task: &Task, home: &Path) {
1476 let now = Timestamp::now();
1477 let last_run_succeeded = task
1484 .runs
1485 .last()
1486 .and_then(|id| RunState::load_under(id, home).ok())
1487 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1488 for id in task.superseded_attempts(last_run_succeeded) {
1489 let mut state = match RunState::load_under(id, home) {
1490 Ok(s) => s,
1491 Err(e) => {
1492 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1493 continue;
1494 }
1495 };
1496 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1497 continue;
1498 }
1499 let daemon_claims = is_working_on(home, id, now);
1500 if state.liveness(daemon_claims) == Liveness::Live {
1501 continue;
1502 }
1503 state.status = RunStatus::Superseded;
1504 if let Err(e) = state.save_under(home) {
1505 tracing::warn!("could not mark run {id} superseded: {e:#}");
1506 }
1507 }
1508}
1509
1510fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1532 for task in queue.list() {
1533 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1534 continue;
1535 }
1536 supersede_prior_runs(&task, home);
1537 }
1538}
1539
1540fn settle_and_diagnose(
1547 task: &mut Task,
1548 verdict: Verdict,
1549 detail: &str,
1550 max_attempts: usize,
1551 state: &RunState,
1552) {
1553 let p = phrases(&state.config.graph.language);
1554 if let Some(d) = &state.merge_decision {
1557 use crate::merge_decision as md;
1558 if task.runs.last().is_some_and(|id| id != &state.id) {
1559 return;
1560 }
1561 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1562 task.succeed();
1563 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1564 if state.liveness(false) != Liveness::Dead {
1565 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1566 return;
1567 }
1568 let replacement = md::Replacement {
1569 question: d.question.clone(),
1570 run: state.id.clone(),
1571 pr: d.target.pr.clone(),
1572 agent: agent.to_owned(),
1573 context: format!(
1574 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1575 d.target.pr,
1576 state.id,
1577 serde_json::to_string(&state.reviews).unwrap_or_default()
1578 ),
1579 };
1580 task.requeue();
1581 task.pr_recovery = None;
1582 task.review_of = None;
1583 task.replacement = Some(replacement);
1584 } else {
1585 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1586 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1587 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1588 }
1589 task.mark_action_applied(&d.question);
1590 if let Some(pass) = state.passes.last() {
1591 task.record_pass(&state.id, pass.index, task.attempts, None);
1592 }
1593 return;
1594 }
1595 if let Some(replacement) = &task.replacement
1596 && !state.parked
1597 && state.resource_block.is_none()
1598 && (state.viable().is_empty()
1599 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1600 {
1601 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1602 return;
1603 }
1604 let attempts_before = task.attempts;
1605 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1611 task.notice_cause(
1612 &format!("resource:{}:{}", block.kind, block.node),
1613 None,
1614 None,
1615 );
1616 task.last_error = Some(detail.to_owned());
1617 task.diagnostic = None;
1618 task.attempts = task.attempts.saturating_sub(1);
1619 task.hold_machine(Some(format!(
1620 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1621 no agent was asked and no attempt was spent",
1622 crate::cache::RESOURCE_HOLD,
1623 block.kind,
1624 block.node
1625 )));
1626 if let Some(pass) = state.passes.last() {
1627 task.record_pass(&state.id, pass.index, attempts_before, None);
1628 }
1629 return;
1630 }
1631 let unresolved = task.hold_notice_cause.clone();
1632 settle_in(task, verdict, detail, max_attempts, p);
1633 if state.prep_failure.is_some() {
1634 task.hold_notice_cause = unresolved;
1637 task.startup_notice_cause("startup-candidates");
1638 }
1639 if let Some(pass) = state.passes.last() {
1642 let parked = match &pass.end {
1643 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
1644 _ => None,
1645 };
1646 task.record_pass(&state.id, pass.index, attempts_before, parked);
1647 }
1648 if task.status == TaskStatus::Held {
1649 if state.prep_failure.is_none() {
1650 task.notice_cause(
1651 label(verdict.status),
1652 Some(&state.id),
1653 state.passes.last().map(|p| p.index),
1654 );
1655 }
1656 task.diagnostic = diagnostic(state);
1657 note_open_question(task, &state.id, p);
1658 }
1659}
1660
1661fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
1676 let Some(home) = crate::run::try_home() else {
1677 return;
1678 };
1679 let open = Questions::at(home.join("questions")).open_for(run);
1680 let Some(q) = open.first() else {
1681 return;
1682 };
1683 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
1684 decisions.sort_unstable();
1685 task.notice_cause(
1686 &format!("questions:{}", decisions.join(",")),
1687 Some(run),
1688 None,
1689 );
1690 let base = task.hold_reason.clone().unwrap_or_default();
1691 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
1692}
1693
1694fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
1705 match last_run {
1706 Some(state) => {
1707 let verdict = Verdict {
1708 status: state.status,
1709 left_pr: state.pr.is_some(),
1710 quota_hit: !state.quota.is_empty(),
1711 parked: state.parked,
1712 no_viable_candidates: state.viable().is_empty(),
1713 };
1714 let detail = format!(
1715 "{}{}",
1716 phrases(&state.config.graph.language).recovered_running,
1717 describe(&state)
1718 );
1719 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
1720 }
1721 None => {
1722 let why = phrases(language).no_run_to_recover;
1725 task.last_error = Some(why.to_owned());
1726 task.notice_cause("missing-run", None, None);
1729 task.hold_machine(Some(why.to_owned()));
1730 }
1731 }
1732}
1733
1734fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
1756 let mut reclaimed = Vec::new();
1757 for listed in queue.list() {
1758 if listed.status != TaskStatus::Running {
1759 continue;
1760 }
1761 let Ok(_claim) = queue.claim(&listed.id) else {
1762 continue;
1763 };
1764 let Ok(mut task) = queue.get(&listed.id) else {
1768 continue;
1769 };
1770 if task.status != TaskStatus::Running {
1771 continue;
1772 }
1773 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
1774 if let Some(state) = &last_run
1784 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
1785 {
1786 tracing::warn!("abandon questions for {}: {e:#}", state.id);
1787 }
1788 let language = if last_run.is_none() {
1789 language_of(&task, Path::new("."))
1790 } else {
1791 String::new()
1792 };
1793 reclaim(&mut task, last_run, max_attempts, &language);
1794 if task.status == TaskStatus::Done {
1795 supersede_prior_runs(&task, &crate::run::home());
1796 }
1797 record(queue, &mut task);
1798 reclaimed.push(task.id.clone());
1799 }
1800 reclaimed
1801}
1802
1803fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
1835 reclaim_abandoned_runs_with(
1836 home,
1837 now,
1838 crate::proc::pid_status,
1839 crate::proc::process_started_at,
1840 )
1841}
1842
1843fn reclaim_abandoned_runs_with<F, G>(
1849 home: &Path,
1850 now: Timestamp,
1851 query: F,
1852 identity: G,
1853) -> Vec<String>
1854where
1855 F: Fn(u32) -> Option<bool>,
1856 G: Fn(u32) -> Option<String>,
1857{
1858 let mut abandoned = Vec::new();
1859 for entry in std::fs::read_dir(home.join("runs"))
1860 .into_iter()
1861 .flatten()
1862 .flatten()
1863 {
1864 let id = entry.file_name().to_string_lossy().into_owned();
1865 if !crate::run::is_run_id(&id) {
1866 continue;
1867 }
1868 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
1876 continue;
1877 };
1878 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
1879 continue;
1880 };
1881 if state.status.done() || !state.active_all_overrun(now) {
1882 continue;
1883 }
1884 let daemon_claims = is_working_on(home, &id, now);
1896 if state.liveness_with(daemon_claims, &query, &identity) != crate::run::Liveness::Dead {
1897 continue;
1898 }
1899 state.abandon("daemon");
1900 if let Err(e) = state.save_under(home) {
1901 tracing::warn!("could not persist abandoned run {id}: {e:#}");
1902 continue;
1903 }
1904 if let Some(notice) = notices::run_ended(&state) {
1907 notices::raise_in(home, notice);
1908 }
1909 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
1917 tracing::warn!("abandon questions for {id}: {e:#}");
1918 }
1919 abandoned.push(id);
1920 }
1921 abandoned
1922}
1923
1924pub async fn serve(opts: Opts) -> Result<()> {
1930 serve_until(opts, Stop::new()).await
1931}
1932
1933pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
1950 let signal = {
1951 let stop = stop.clone();
1952 tokio::spawn(async move {
1953 if tokio::signal::ctrl_c().await.is_ok() {
1954 stop.stop();
1955 tracing::info!("shutdown requested; a run in flight will be finished first");
1956 }
1957 })
1958 };
1959
1960 let worktrees_root = opts
1961 .worktrees_root
1962 .clone()
1963 .unwrap_or_else(crate::run::default_worktree_root);
1964 let outcome = drive(
1965 &opts,
1966 &Queue::open(),
1967 &status_path(),
1968 &crate::run::home(),
1969 &worktrees_root,
1970 &stop,
1971 )
1972 .await;
1973
1974 signal.abort();
1975 outcome
1976}
1977
1978async fn drive(
1991 opts: &Opts,
1992 queue: &Queue,
1993 status_file: &Path,
1994 home: &Path,
1995 worktrees_root: &Path,
1996 stop: &Stop,
1997) -> Result<()> {
1998 let recovery_cfg = prepare(&opts.repo, opts).ok();
2006 let daemon_cfg = recovery_cfg
2007 .as_ref()
2008 .map(|c| c.daemon.clone())
2009 .unwrap_or_default();
2010 let mut initial_status = Status::new();
2011 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2012 recovery_cfg.as_ref(),
2013 daemon_cfg.max_deputies,
2014 ));
2015 let status = Arc::new(Mutex::new(initial_status));
2016 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2017 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2018
2019 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2025
2026 let waiter = tokio::spawn(crate::waiter::run(
2031 crate::waiter::Waiter::new(
2032 crate::ask::Questions::at(home.join("questions")),
2033 home.to_path_buf(),
2034 recovery_cfg.clone(),
2035 ),
2036 stop.clone(),
2037 ));
2038
2039 let deputies = tokio::spawn(crate::deputy::run(
2043 crate::deputy::Deputies::new(
2044 crate::ask::Questions::at(home.join("questions")),
2045 home.to_path_buf(),
2046 recovery_cfg.clone(),
2047 opts.repo.clone(),
2048 daemon_cfg.max_deputies,
2049 {
2050 let stop = stop.clone();
2051 Arc::new(move || stop.parking())
2052 },
2053 ),
2054 stop.clone(),
2055 ));
2056
2057 let fetcher = tokio::spawn(fetch_loop(opts.repo.clone(), opts.clone(), stop.clone()));
2060
2061 let release_watch = tokio::spawn(crate::release_watch::run(
2064 crate::release_watch::Watcher::new(
2065 Box::new(crate::release_watch::GhForge),
2066 home.to_path_buf(),
2067 ),
2068 {
2069 let (repo, opts) = (opts.repo.clone(), opts.clone());
2070 move || release_watch_settings(&repo, &opts)
2071 },
2072 stop.clone(),
2073 ));
2074
2075 tracing::info!(
2076 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2077 queue.root().display(),
2078 opts.poll.as_secs(),
2079 opts.max_attempts,
2080 concurrency,
2081 if daemon_cfg.pause_for_interrupts {
2082 ", interrupts enabled"
2083 } else {
2084 ""
2085 }
2086 );
2087
2088 janitor(&opts.repo, opts, home, worktrees_root).await;
2091 resweep_superseded_attempts(queue, home);
2092
2093 let outcome = poll(
2094 opts,
2095 queue,
2096 &status,
2097 home,
2098 worktrees_root,
2099 stop,
2100 DispatchLimits {
2101 max_concurrent: concurrency,
2102 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2103 },
2104 )
2105 .await;
2106
2107 beat.abort();
2108 waiter.abort();
2109 deputies.abort();
2110 fetcher.abort();
2111 release_watch.abort();
2112 clear_status_at(status_file);
2113 outcome
2114}
2115
2116fn release_watch_settings(repo: &Path, opts: &Opts) -> (Vec<PathBuf>, u64) {
2119 let Ok(cfg) = prepare(repo, opts) else {
2120 return (Vec::new(), 0);
2121 };
2122 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2123 .into_iter()
2124 .map(|r| r.path)
2125 .collect();
2126 let mut known: Vec<PathBuf> = Queue::open().list().into_iter().map(|t| t.repo).collect();
2129 let runs = crate::run::home().join("runs");
2130 for id in crate::run::list_ids_in(&runs) {
2131 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2132 .ok()
2133 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2134 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2135 known.extend(repo);
2136 }
2137 for k in known {
2138 let k = k.canonicalize().unwrap_or(k);
2139 if k.join(".git").exists() && !paths.contains(&k) {
2140 paths.push(k);
2141 }
2142 }
2143 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2144 if !paths.contains(&own) {
2145 paths.push(own);
2146 }
2147 (paths, cfg.daemon.release_stall_minutes)
2148}
2149
2150const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2152
2153async fn fetch_loop(repo: PathBuf, opts: Opts, stop: Stop) {
2157 while !stop.stopped() {
2158 let (interval, roots) = match prepare(&repo, &opts) {
2159 Ok(c) => (c.repos.fetch_interval, c.repos.roots),
2160 Err(_) => (0, Vec::new()),
2161 };
2162 if interval > 0 && !roots.is_empty() {
2163 let r = crate::clean::fetch_origins(&roots, FETCH_TIMEOUT, || stop.stopped()).await;
2164 tracing::debug!("fetch origins: {r:?}");
2165 }
2166 let wait = if interval > 0 { interval } else { 60 };
2168 let mut slept = 0;
2169 while slept < wait && !stop.stopped() {
2170 tokio::time::sleep(Duration::from_secs(1)).await;
2171 slept += 1;
2172 }
2173 }
2174}
2175
2176async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2182 loop {
2183 tokio::time::sleep(HEARTBEAT).await;
2184 let snapshot = {
2185 let mut guard = lock(&status);
2186 guard.updated_at = Timestamp::now();
2187 guard.clone()
2188 };
2189 if let Err(e) = write_status_to(&path, &snapshot) {
2190 tracing::warn!("could not refresh the daemon status file: {e:#}");
2193 }
2194 }
2195}
2196
2197#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2200enum LandResume {
2201 NotLanding,
2204 StillWaiting,
2209 Ready,
2213}
2214
2215fn land_resume_state(task: &Task) -> LandResume {
2219 let Some(run_id) = task.runs.last() else {
2220 return LandResume::NotLanding;
2221 };
2222 let Ok(state) = RunState::load(run_id) else {
2223 return LandResume::NotLanding;
2224 };
2225 if !state.parked {
2226 return LandResume::NotLanding;
2227 }
2228 if state.status == RunStatus::Landing
2229 && state.land_approval.as_ref().is_some_and(|binding| {
2230 ask::Questions::open()
2231 .get(&binding.question)
2232 .is_ok_and(|q| {
2233 q.run == state.id
2234 && q.resolution()
2235 .as_deref()
2236 .is_some_and(crate::merge_decision::stops)
2237 })
2238 })
2239 {
2240 return LandResume::Ready;
2241 }
2242 if !implementers_match(task, &state) {
2243 return LandResume::NotLanding;
2244 }
2245 let node = if state.status == RunStatus::Landing {
2249 land::APPROVAL_NODE
2250 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2251 crate::github_text::ASK_NODE
2252 } else {
2253 return LandResume::NotLanding;
2254 };
2255 let store = ask::Questions::open();
2256 let waiting = store
2257 .list()
2258 .into_iter()
2259 .filter(|q| &q.run == run_id && q.node == node)
2260 .max_by(|a, b| a.id.cmp(&b.id));
2261 let Some(mut q) = waiting else {
2262 return LandResume::Ready;
2263 };
2264 if !q.status.open() {
2265 if q.resolution().as_deref() == Some(land::REPAIR) {
2268 return if state.liveness(false) == Liveness::Live {
2269 LandResume::StillWaiting
2270 } else {
2271 LandResume::NotLanding
2272 };
2273 }
2274 return LandResume::Ready;
2275 }
2276 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2283 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2284 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2285 q.abandon(format!(
2286 "no answer within {}s of asking",
2287 timeout.as_secs().max(1)
2288 ));
2289 if store.put(&mut q).is_ok() {
2292 return LandResume::Ready;
2293 }
2294 }
2295 LandResume::StillWaiting
2296}
2297
2298const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2306
2307const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2319
2320struct InFlightGuard<'a> {
2333 status: &'a Arc<Mutex<Status>>,
2334 stop: &'a Stop,
2335 task_id: &'a str,
2336}
2337
2338impl Drop for InFlightGuard<'_> {
2339 fn drop(&mut self) {
2340 lock(self.status).current.retain(|c| c.task != self.task_id);
2341 self.stop.exit();
2342 }
2343}
2344
2345#[derive(Debug, Clone, PartialEq, Eq)]
2367enum Interrupt {
2368 Idle,
2370 Parking {
2382 parked: Vec<String>,
2383 interrupt_task: String,
2384 },
2385 Running {
2393 parked: Vec<String>,
2394 interrupt_task: String,
2395 },
2396 Resuming { parked: Vec<String> },
2403}
2404
2405fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
2426 match state {
2427 Interrupt::Idle => {
2428 if in_flight.len() != 1 {
2440 return Interrupt::Idle;
2441 }
2442 match runnable.iter().find(|t| t.interrupt) {
2443 Some(t) => Interrupt::Parking {
2444 parked: in_flight.to_vec(),
2445 interrupt_task: t.id.clone(),
2446 },
2447 None => Interrupt::Idle,
2448 }
2449 }
2450 Interrupt::Parking {
2451 parked,
2452 interrupt_task,
2453 } => {
2454 if in_flight.iter().any(|id| parked.contains(id)) {
2455 Interrupt::Parking {
2457 parked,
2458 interrupt_task,
2459 }
2460 } else if in_flight.contains(&interrupt_task) {
2461 Interrupt::Running {
2462 parked,
2463 interrupt_task,
2464 }
2465 } else if runnable.iter().any(|t| t.id == interrupt_task) {
2466 Interrupt::Parking {
2470 parked,
2471 interrupt_task,
2472 }
2473 } else {
2474 Interrupt::Resuming { parked }
2479 }
2480 }
2481 Interrupt::Running {
2482 parked,
2483 interrupt_task,
2484 } => {
2485 if in_flight.contains(&interrupt_task) {
2486 Interrupt::Running {
2487 parked,
2488 interrupt_task,
2489 }
2490 } else {
2491 Interrupt::Resuming { parked }
2497 }
2498 }
2499 Interrupt::Resuming { parked } => {
2500 if in_flight.iter().any(|id| parked.contains(id)) {
2501 Interrupt::Idle
2507 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
2508 Interrupt::Resuming { parked }
2509 } else {
2510 Interrupt::Idle
2513 }
2514 }
2515 }
2516}
2517
2518fn advance_interrupt_tick(
2524 enabled: bool,
2525 state: Interrupt,
2526 in_flight: &[String],
2527 runnable: &[Task],
2528) -> Interrupt {
2529 if !enabled {
2530 return Interrupt::Idle;
2531 }
2532 advance_interrupt(state, in_flight, runnable)
2533}
2534
2535fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
2540 match state {
2541 Interrupt::Idle => candidates,
2542 Interrupt::Parking {
2543 parked,
2544 interrupt_task,
2545 } => {
2546 if in_flight.iter().any(|id| parked.contains(id)) {
2547 Vec::new()
2548 } else {
2549 candidates
2550 .into_iter()
2551 .filter(|t| &t.id == interrupt_task)
2552 .collect()
2553 }
2554 }
2555 Interrupt::Running { .. } => Vec::new(),
2556 Interrupt::Resuming { parked } => candidates
2564 .into_iter()
2565 .find(|t| parked.contains(&t.id))
2566 .into_iter()
2567 .collect(),
2568 }
2569}
2570
2571struct DispatchLimits {
2575 max_concurrent: usize,
2578 pause_for_interrupts: bool,
2580}
2581
2582#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2591enum PermitKind {
2592 None,
2597 Urgent,
2604 Ordinary,
2607}
2608
2609fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
2612 if priority {
2613 PermitKind::None
2614 } else if urgent {
2615 PermitKind::Urgent
2616 } else {
2617 PermitKind::Ordinary
2618 }
2619}
2620
2621struct DispatchClaim {
2622 task: Task,
2623 claim: crate::queue::Claim,
2624 permit: Option<tokio::sync::OwnedSemaphorePermit>,
2625}
2626
2627fn claim_dispatch(
2630 queue: &Queue,
2631 candidate: &Task,
2632 cooling_down: bool,
2633 ordinary: &Arc<tokio::sync::Semaphore>,
2634 urgent: &Arc<tokio::sync::Semaphore>,
2635) -> Option<DispatchClaim> {
2636 let Ok(claim) = queue.claim(&candidate.id) else {
2637 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
2638 return None;
2639 };
2640 let task = match queue.get(&candidate.id) {
2641 Ok(t) if t.status.runnable() => t,
2642 Ok(_) => return None,
2643 Err(e) => {
2644 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
2645 return None;
2646 }
2647 };
2648 let resume = land_resume_state(&task);
2649 if resume == LandResume::StillWaiting {
2650 return None;
2651 }
2652 let priority = resume == LandResume::Ready;
2653 if !priority && cooling_down {
2654 return None;
2655 }
2656 let permit = match permit_kind(priority, task.urgent) {
2659 PermitKind::None => None,
2660 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
2661 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
2662 };
2663 Some(DispatchClaim {
2664 task,
2665 claim,
2666 permit,
2667 })
2668}
2669
2670async fn poll(
2689 opts: &Opts,
2690 queue: &Queue,
2691 status: &Arc<Mutex<Status>>,
2692 home: &Path,
2693 worktrees_root: &Path,
2694 stop: &Stop,
2695 limits: DispatchLimits,
2696) -> Result<()> {
2697 let DispatchLimits {
2698 max_concurrent,
2699 pause_for_interrupts,
2700 } = limits;
2701 let mut attempted: Vec<String> = Vec::new();
2706 let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
2707 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(1));
2715 let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
2721 let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
2722 let mut conductor = Conductor::new();
2723 let mut cache_last_checked: Option<Timestamp> = None;
2726 let mut interrupt = Interrupt::Idle;
2728 let mut interrupt_pauses: std::collections::HashMap<String, crate::graph::Pause> =
2734 std::collections::HashMap::new();
2735
2736 while !stop.stopped() {
2737 lock(status).polls += 1;
2738
2739 while let Some(result) = inflight.try_join_next() {
2744 if let Err(e) = result {
2745 tracing::error!("a spawned attempt did not finish cleanly: {e}");
2746 notices::raise(Notice::error(
2747 "loop:attempt",
2748 "A queued attempt ended abnormally; check the task it was running.",
2749 ));
2750 }
2751 }
2752
2753 let swept = sweep_stale_claims(queue, STALE_CLAIM);
2754 if !swept.is_empty() {
2755 tracing::warn!(
2756 "swept {} stale claim(s) left behind by an earlier daemon: {}",
2757 swept.len(),
2758 swept.join(", ")
2759 );
2760 }
2761 let now = Timestamp::now();
2766
2767 if !stop.busy_now() {
2772 maybe_prune_cache_between_runs(
2773 &opts.repo,
2774 opts,
2775 home,
2776 stop,
2777 &mut cache_last_checked,
2778 now,
2779 )
2780 .await;
2781 }
2782
2783 let stalled = stalled_tasks(queue, home, now);
2784 let stalled_ids: std::collections::BTreeSet<_> =
2785 stalled.iter().map(|task| task.id.clone()).collect();
2786 let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
2787 if !reclaimed.is_empty() {
2788 tracing::warn!(
2789 "reclaimed {} task(s) left `running` by a daemon that never \
2790 recorded the outcome: {}",
2791 reclaimed.len(),
2792 reclaimed.join(", ")
2793 );
2794 }
2795 let abandoned_runs = reclaim_abandoned_runs(home, now);
2796 if !abandoned_runs.is_empty() {
2797 tracing::warn!(
2798 "failed {} run(s) left behind by a killed process, past every \
2799 active seat's own timeout: {}",
2800 abandoned_runs.len(),
2801 abandoned_runs.join(", ")
2802 );
2803 }
2804
2805 let questions = Questions::at(home.join("questions"));
2810
2811 resolve_blockers(queue, &questions);
2814 apply_choice_actions(queue, &questions, home);
2815 triage::apply_answers(queue, &questions);
2816 reconcile_task_questions(queue, &questions);
2817 {
2821 let talks = crate::talk::Talks::at(home.join("talks"));
2822 crate::chat_notify::sweep(queue, &talks, home);
2823 crate::chat_watch::sweep(home, &talks);
2824 crate::chat_notify::start_turns(&talks, stop.parking(), &mut inflight);
2825 }
2826
2827 let finished: Vec<Task> = finished_tasks(queue)
2833 .into_iter()
2834 .filter(|task| !stalled_ids.contains(&task.id))
2835 .filter(|task| !awaiting_resume_with(task, |id| RunState::load_under(id, home)))
2839 .collect();
2840 let queued = queued_tasks(queue);
2841 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
2845 && conductor.worth_a_look(queue, &stalled, &finished)
2846 {
2847 match prepare(&opts.repo, opts) {
2848 Ok(cfg) => {
2849 conductor
2850 .maybe_run(
2851 &cfg,
2852 &opts.repo,
2853 queue,
2854 &questions,
2855 home,
2856 &queued,
2857 &stalled,
2858 &finished,
2859 opts.max_attempts,
2860 )
2861 .await;
2862 }
2863 Err(e) => {
2864 tracing::warn!("conductor: no config: {e:#}");
2865 notices::raise(Notice::warn(
2866 "loop:no-config",
2867 "The loop could not read this repository's config, so held tasks are not being triaged.",
2868 ));
2869 }
2870 }
2871 }
2872
2873 let candidates: Vec<Task> = runnable(queue)
2874 .into_iter()
2875 .filter(|t| !opts.once || !attempted.contains(&t.id))
2876 .collect();
2877
2878 let in_flight: Vec<String> = lock(status)
2883 .current
2884 .iter()
2885 .map(|c| c.task.clone())
2886 .collect();
2887 interrupt_pauses.retain(|id, _| in_flight.contains(id));
2888
2889 interrupt =
2890 advance_interrupt_tick(pause_for_interrupts, interrupt, &in_flight, &candidates);
2891 if let Interrupt::Parking {
2892 parked,
2893 interrupt_task,
2894 } = &interrupt
2895 {
2896 let reason = format!(
2897 "task {} asked to run first",
2898 crate::run::short_of(interrupt_task)
2899 );
2900 for id in parked {
2901 if let Some(pause) = interrupt_pauses.get(id) {
2902 pause.park_because(reason.clone());
2903 }
2904 }
2905 }
2906 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
2907
2908 let cooling_down =
2909 lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
2910
2911 let mut started_any = false;
2912 for candidate in candidates {
2913 if stop.stopped() {
2914 break;
2915 }
2916
2917 let Some(DispatchClaim {
2918 mut task,
2919 claim,
2920 permit,
2921 }) = claim_dispatch(queue, &candidate, cooling_down, &sem, &urgent_sem)
2922 else {
2923 continue;
2924 };
2925
2926 let task_id = task.id.clone();
2927 attempted.push(task_id.clone());
2928 lock(status).idle = false;
2929 stop.enter();
2932 started_any = true;
2933
2934 let run_pause = crate::graph::Pause::new();
2938 interrupt_pauses.insert(task_id.clone(), run_pause.clone());
2939
2940 let opts = opts.clone();
2941 let queue = queue.clone();
2942 let status = Arc::clone(status);
2943 let stop = stop.clone();
2944 let quota_cooldown_until = Arc::clone("a_cooldown_until);
2945 inflight.spawn(async move {
2946 let _claim = claim;
2950 let _permit = permit;
2951 let _inflight = InFlightGuard {
2953 status: &status,
2954 stop: &stop,
2955 task_id: &task_id,
2956 };
2957 let quota = attempt(&opts, &queue, &status, &stop, run_pause, &mut task).await;
2958 lock(&status).completed += 1;
2959 let now = Timestamp::now();
2965 if let Some(until) = cooldown_until("a, now) {
2966 let wait = until.as_second() - now.as_second();
2967 *lock("a_cooldown_until) = Some(until);
2968 let hint = quota
2969 .iter()
2970 .find(|q| q.reset.is_some())
2971 .and_then(|q| q.reset.as_deref());
2972 match hint {
2973 Some(h) => tracing::warn!(
2974 "quota hit; waiting {wait}s before taking another ordinary task \
2975 (CLI reported reset: {h})"
2976 ),
2977 None => tracing::warn!(
2978 "quota hit; waiting {wait}s before taking another ordinary task \
2979 (no reset hint reported)"
2980 ),
2981 }
2982 }
2983 });
2984 }
2985
2986 if started_any {
2987 continue;
2988 }
2989
2990 if stop.busy_now() {
2991 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
2996 continue;
2997 }
2998
2999 lock(status).idle = true;
3001 if opts.once {
3002 janitor(&opts.repo, opts, home, worktrees_root).await;
3006 resweep_superseded_attempts(queue, home);
3007 triage_held(queue, home, opts).await;
3008 break;
3009 }
3010 stop.idle(opts.poll).await;
3011 if stop.stopped() {
3012 continue;
3013 }
3014 janitor(&opts.repo, opts, home, worktrees_root).await;
3020 resweep_superseded_attempts(queue, home);
3021 triage_held(queue, home, opts).await;
3022 }
3023
3024 while let Some(result) = inflight.join_next().await {
3029 if let Err(e) = result {
3030 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3031 notices::raise(Notice::error(
3032 "loop:attempt",
3033 "A queued attempt ended abnormally; check the task it was running.",
3034 ));
3035 }
3036 }
3037 Ok(())
3038}
3039
3040async fn attempt(
3046 opts: &Opts,
3047 queue: &Queue,
3048 status: &Arc<Mutex<Status>>,
3049 stop: &Stop,
3050 interrupt_pause: crate::graph::Pause,
3051 task: &mut Task,
3052) -> Vec<QuotaLoss> {
3053 let repo = repo_for(task, &opts.repo);
3054 tracing::info!(
3055 "task {} — {} (repo {})",
3056 task.short(),
3057 task.title,
3058 repo.display()
3059 );
3060
3061 if let Some(previous) = task.runs.last().and_then(|id| RunState::load(id).ok())
3065 && previous
3066 .merge_decision
3067 .as_ref()
3068 .is_some_and(|decision| !decision.closed)
3069 && previous.status != RunStatus::Landing
3070 && !crate::merge_decision::resumable_hold(&previous)
3071 {
3072 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3073 record(queue, task);
3074 return Vec::new();
3075 }
3076 let stop_owner = task
3077 .runs
3078 .last()
3079 .and_then(|id| RunState::load(id).ok())
3080 .filter(|state| {
3081 crate::merge_decision::resumable_hold(state)
3082 || state.status == RunStatus::Landing
3083 && state.land_approval.as_ref().is_some_and(|binding| {
3084 Questions::open().get(&binding.question).is_ok_and(|q| {
3085 q.run == state.id
3086 && q.resolution()
3087 .as_deref()
3088 .is_some_and(crate::merge_decision::stops)
3089 })
3090 })
3091 });
3092 if let Some(owner) = stop_owner {
3093 let mut runner = match Runner::resume(&owner.id) {
3094 Ok(runner) => runner,
3095 Err(e) => {
3096 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
3097 record(queue, task);
3098 return Vec::new();
3099 }
3100 };
3101 runner.on_pause(stop.pause());
3102 runner.watch_interrupt(interrupt_pause);
3103 let quota_before = runner.state.quota.clone();
3104 task.start(owner.id.clone());
3105 record(queue, task);
3106 lock(status).current.push(Current {
3107 task: task.id.clone(),
3108 run: owner.id,
3109 });
3110 runner.state.next_pass_task_entry = true;
3111 let result = runner.execute().await;
3112 return finish_attempt(
3113 opts.max_attempts,
3114 queue,
3115 task,
3116 &runner.state,
3117 "a_before,
3118 result,
3119 );
3120 }
3121
3122 let explicit = match &task.overrides {
3126 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
3127 Some(_) => opts.config.as_deref(),
3128 None => opts.config.as_deref(),
3129 };
3130 if let Err(e) = Config::discover_fetched(&repo, explicit).await {
3131 let reason = format!("[config] {e:#}");
3132 record_config_failure(task, &e);
3133 task.last_error = Some(reason.clone());
3134 task.hold_machine(Some(reason.clone()));
3135 record(queue, task);
3136 tracing::warn!("holding {}: {reason}", task.short());
3137 return Vec::new();
3138 }
3139 let mut config = match prepare_for(&repo, opts, task) {
3140 Ok(c) => c,
3141 Err(e) => {
3142 task.attempts += 1;
3146 task.fail_startup(
3147 "startup-config",
3148 format!("config: {e:#}"),
3149 opts.max_attempts,
3150 );
3151 record(queue, task);
3152 return Vec::new();
3153 }
3154 };
3155 apply_task_implementers(&mut config, task);
3156 if let Some(replacement) = &task.replacement {
3157 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
3158 old.liveness(false) == Liveness::Dead
3159 && old.merge_decision.as_ref().is_some_and(|d| {
3160 d.closed
3161 && d.question == replacement.question
3162 && d.replacement_run.as_ref().is_none_or(|run| {
3163 task.runs.last() == Some(run)
3164 && RunState::load(run).is_ok_and(|current| {
3165 current.replaces.as_ref().is_some_and(|pin| {
3166 pin.question == replacement.question
3167 && pin.agent == replacement.agent
3168 })
3169 })
3170 })
3171 })
3172 });
3173 if !safe {
3174 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
3175 record(queue, task);
3176 return Vec::new();
3177 }
3178 match config.agent(&replacement.agent) {
3179 Ok(agent) if crate::agent::installed(agent) => {}
3180 _ => {
3181 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
3182 record(queue, task);
3183 return Vec::new();
3184 }
3185 }
3186 config.graph.implementers = 1;
3187 config.graph.land_approval = true;
3188 config.merge.mode = MergeMode::Pr;
3189 config.graph.selected_implementer = Some(replacement.agent.clone());
3190 }
3191 let p = phrases(&config.graph.language);
3192 let start_failed = p.could_not_start;
3193
3194 if let Some(reason) = disk_gate(&repo, &config) {
3202 task.notice_cause("disk-space", None, None);
3203 task.last_error = Some(reason.clone());
3204 task.hold_machine(Some(reason.clone()));
3205 record(queue, task);
3206 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
3207 notices::raise(
3210 Notice::warn(
3211 &format!("disk:{}", repo.display()),
3212 "A task was held for want of free disk space; free some, then release it from the queue.",
3213 )
3214 .link(Link::Task {
3215 id: task.id.clone(),
3216 }),
3217 );
3218 return Vec::new();
3219 }
3220
3221 let unfinished = (!task.fresh_start)
3241 .then(|| unfinished_run(&task.runs, task.short()))
3242 .flatten()
3243 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
3244 let previous = task.runs.last().and_then(|id| RunState::load(id).ok());
3250 let review_branch = take_review_branch(task, previous.as_ref());
3251 let branch_exists = match &review_branch {
3252 Some(branch) => crate::git::branch_exists(&repo, branch)
3253 .await
3254 .unwrap_or(false),
3255 None => false,
3256 };
3257 let proposed = match (&review_branch, &unfinished, &task.review_of) {
3261 (None, None, Some(branch)) => Starter::Review(branch.clone()),
3262 _ => choose_starter(
3263 review_branch.as_deref(),
3264 branch_exists,
3265 unfinished.as_deref(),
3266 ),
3267 };
3268 let retry_selected = task.replacement.is_some()
3269 && task.resumption.is_some()
3270 && !task.fresh_start
3271 && previous.as_ref().is_some_and(|run| {
3272 run.replaces.is_some()
3273 && ((run.status == RunStatus::Failed && run.viable().is_empty())
3274 || (run.status == RunStatus::Blocked && run.tally.is_none()))
3275 });
3276 let stop_pin = task
3277 .resume_override
3278 .as_ref()
3279 .filter(|pin| {
3280 Questions::open().get(&pin.question_id).is_ok_and(|q| {
3281 q.node == land::APPROVAL_NODE
3282 && q.resolution()
3283 .as_deref()
3284 .is_some_and(crate::merge_decision::stops)
3285 })
3286 })
3287 .and_then(|pin| pin.pinned_run.clone());
3288 let starter = if let Some(run) = stop_pin {
3289 Starter::Resume(run)
3292 } else if retry_selected {
3293 Starter::Resume(previous.as_ref().unwrap().id.clone())
3294 } else if let Some(selected) = &task.pr_recovery {
3295 if selected.check_task(&task.id, &task.instruction).is_ok() {
3296 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
3297 } else {
3298 Starter::Review(selected.branch.clone())
3299 }
3300 } else {
3301 starter_with_implementers(task, previous.as_ref(), proposed)
3302 };
3303 if let Some(pin) = &task.resume_override
3306 && let Some(run) = &pin.pinned_run
3307 && Questions::open().get(&pin.question_id).map_or(true, |q| {
3308 q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR)
3309 })
3310 && starter != Starter::Resume(run.clone())
3311 {
3312 let run = run.clone();
3313 task.notice_cause("repair-owner", Some(&run), None);
3314 task.hold_machine(Some(format!(
3315 "repair owner cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
3316 task.id
3317 )));
3318 record(queue, task);
3319 return Vec::new();
3320 }
3321 if task.replacement.is_some()
3322 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
3323 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
3324 {
3325 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
3326 record(queue, task);
3327 return Vec::new();
3328 }
3329 let attachments = match task_attachments(queue, task) {
3334 Ok(a) => a,
3335 Err(e) => {
3336 task.attempts += 1;
3337 task.fail_startup_target(
3338 "startup-attachments",
3339 e.name.clone(),
3340 format!("{start_failed}{e:#}"),
3341 opts.max_attempts,
3342 );
3343 record(queue, task);
3344 return Vec::new();
3345 }
3346 };
3347 let startup_pause = stop.pause();
3348 let startup_interrupt = interrupt_pause.clone();
3349 let wait = crate::repo_lock::WaitPolicy::new(
3350 Duration::from_secs(config.graph.timeout_resource_wait),
3351 move || startup_pause.parked() || startup_interrupt.parked(),
3352 );
3353 let started = wait.scope(async { match &starter {
3354 Starter::Review(_) if task.pr_recovery.is_some() => {
3355 let selected = task.pr_recovery.clone().expect("selected recovery");
3356 match selected.check_task(&task.id, &task.instruction) {
3357 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
3358 .await
3359 .map(|mut r| {
3360 r.state.attachments = attachments.clone();
3361 r
3362 }),
3363 Err(e) => Err(e),
3364 }
3365 }
3366 Starter::Review(branch) => {
3367 tracing::info!(
3368 "task {} reopens `{branch}` as a review-only pass",
3369 task.short()
3370 );
3371 let takeover = crate::handover::Takeover {
3374 earlier: task.earlier_attempts().to_vec(),
3375 home: crate::run::home(),
3376 choice: take_divergence_answer(branch, &config.merge.remote, task),
3377 task_instruction: Some(instruction_for(task)),
3378 };
3379 Runner::review_taking_over(
3380 &repo,
3381 branch,
3382 config,
3383 Some(takeover),
3384 crate::run::Origin::queue(&task.id),
3385 )
3386 .await
3387 }
3388 Starter::Resume(id) => {
3389 tracing::info!("resuming run {id} rather than competing again");
3390 Runner::resume(id).and_then(|mut r| {
3391 if retry_selected {
3392 for candidate in &mut r.state.candidates {
3395 candidate.failed = None;
3396 candidate.empty = false;
3397 candidate.commits = 0;
3398 }
3399 r.state.status = RunStatus::Implementing;
3400 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
3401 }
3402 if r.state.prep_failure.is_some() {
3403 let approval = task
3404 .resumption
3405 .as_ref()
3406 .context("preparation retry requires an explicit task release")?;
3407 r.retry_preparation(approval.at)?;
3408 }
3409 if let Some(instruction) =
3410 prepare_instruction(&starter, Some(&r.state.instruction), task)
3411 {
3412 r.state.instruction = instruction;
3413 }
3414 r.state.attachments = attachments.clone();
3415 if task.replacement.is_some() {
3416 r.state.config.merge.mode = MergeMode::Pr;
3417 r.state.config.graph.land_approval = true;
3418 }
3419 if task.replacement.is_none() && let Some(mode) = task
3422 .overrides
3423 .as_ref()
3424 .and_then(|o| o.merge.as_deref())
3425 .and_then(|m| merge_mode(m).ok())
3426 {
3427 r.state.config.merge.mode = mode;
3428 }
3429 Ok(r)
3430 })
3431 }
3432 Starter::Start => {
3433 if let Some(branch) = &review_branch {
3434 tracing::warn!(
3435 "conductor chose review for task {} but branch `{branch}` no longer \
3436 exists; requeuing as a fresh competition instead",
3437 task.short()
3438 );
3439 }
3440 let instruction = prepare_instruction(&starter, None, task)
3441 .unwrap_or_else(|| task.instruction.clone());
3442 Runner::start_naming(
3443 &repo,
3444 instruction,
3445 &task.title,
3446 config,
3447 crate::run::Origin::queue(&task.id),
3448 )
3449 .await
3450 .map(|mut r| {
3451 r.state.attachments = attachments.clone();
3452 r
3453 })
3454 }
3455 }}).await;
3456 let mut runner = match started {
3457 Ok(r) => r,
3458 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
3459 task.last_error = Some(e.to_string());
3460 if stop.parking() || interrupt_pause.parked() {
3461 let attempts = task.attempts;
3462 task.park(e.to_string());
3463 task.attempts = attempts;
3465 } else {
3466 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
3467 }
3468 record(queue, task);
3469 return Vec::new();
3470 }
3471 Err(e) if task.pr_recovery.is_some() => {
3475 task.startup_notice_cause("startup-public-head");
3476 task.attempts += 1;
3477 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
3478 task.link_run(&partial.run);
3479 task.record_pass(&partial.run, 0, task.attempts, None);
3480 }
3481 task.last_error = Some(format!("{start_failed}{e:#}"));
3482 task.hold_unrequested_retry(Some(format!(
3483 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
3484 )));
3485 record(queue, task);
3486 return Vec::new();
3487 }
3488 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
3489 let detail = match e.downcast_ref::<crate::handover::Refused>() {
3490 Some(r) => (p.handover_refused)(r),
3491 None => format!("{e:#}"),
3492 };
3493 task.startup_notice_cause("startup-handover");
3494 let reason = format!("{start_failed}{detail}");
3495 task.last_error = Some(reason.clone());
3496 let branch = match &starter {
3499 Starter::Review(branch) => Some(branch.clone()),
3500 _ => None,
3501 };
3502 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
3506 record(queue, task);
3507 tracing::warn!(
3508 "holding {} for a branch it cannot take over: {e:#}",
3509 task.short()
3510 );
3511 return Vec::new();
3512 }
3513 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
3518 let d = e
3519 .downcast_ref::<crate::reconcile::Diverged>()
3520 .expect("checked by the guard");
3521 let mut q = ask::Question::new(
3522 task.id.clone(),
3523 crate::reconcile::NODE.to_owned(),
3524 crate::reconcile::SEAT.to_owned(),
3525 d.summary(),
3526 d.detail(),
3527 d.choices(),
3528 );
3529 match Questions::open().put(&mut q) {
3530 Ok(()) => {
3531 task.last_error = Some(format!("{e:#}"));
3532 task.review_branch = Some(d.branch.clone());
3535 task.review_instruction = Some(task.instruction.clone());
3536 task.block(vec![q.id.clone()], Some(d.summary()));
3537 }
3538 Err(put) => {
3539 tracing::warn!("could not file the divergence question: {put:#}");
3540 task.attempts += 1;
3541 task.fail_startup(
3542 "startup-question-store",
3543 format!("{start_failed}{e:#}"),
3544 opts.max_attempts,
3545 );
3546 }
3547 }
3548 record(queue, task);
3549 return Vec::new();
3550 }
3551 Err(e) => {
3552 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
3558 && let Starter::Review(branch) = &starter
3559 {
3560 task.review_branch = Some(branch.clone());
3561 task.review_instruction = Some(task.instruction.clone());
3562 task.last_error = Some(format!("{start_failed}{e:#}"));
3563 task.status = crate::queue::TaskStatus::Failed;
3564 record(queue, task);
3565 return Vec::new();
3566 }
3567 task.attempts += 1;
3568 task.last_error = Some(format!("{start_failed}{e:#}"));
3569 task.startup_notice_cause("startup-source");
3570 task.hold_unrequested_retry(Some(format!(
3571 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
3572 )));
3573 record(queue, task);
3574 return Vec::new();
3575 }
3576 };
3577 if let Some(replacement) = &task.replacement {
3578 if matches!(starter, Starter::Start) {
3579 runner.state.instruction.push_str(&format!(
3580 "\n\n# Replacement review context\n{}",
3581 replacement.context
3582 ));
3583 }
3584 runner.state.replaces = Some(replacement.clone());
3585 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
3586 if let Ok(mut old) = RunState::load(&replacement.run)
3587 && let Some(d) = &mut old.merge_decision
3588 {
3589 d.replacement_run = Some(runner.state.id.clone());
3590 old.event(
3591 "replacement",
3592 format!(
3593 "replacement run {} owns the next attempt; old PR remains closed",
3594 runner.state.id
3595 ),
3596 );
3597 if let Err(e) = old.save() {
3598 task.hold_manual(Some(format!(
3599 "Could not link replacement history: {e}; no agents dispatched"
3600 )));
3601 record(queue, task);
3602 return Vec::new();
3603 }
3604 }
3605 }
3606 runner.state.task_instruction = Some(task.instruction.clone());
3607 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
3610 if runner.state.origin_chat.is_none() {
3611 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
3612 }
3613 runner.on_pause(stop.pause());
3615 runner.watch_interrupt(interrupt_pause);
3619
3620 let run = runner.state.id.clone();
3623 task.start(run.clone());
3624 record(queue, task);
3625 lock(status).current.push(Current {
3626 task: task.id.clone(),
3627 run,
3628 });
3629
3630 let quota_before = runner.state.quota.clone();
3633 runner.state.next_pass_task_entry = true;
3634 let result = runner.execute().await;
3635 finish_attempt(
3636 opts.max_attempts,
3637 queue,
3638 task,
3639 &runner.state,
3640 "a_before,
3641 result,
3642 )
3643}
3644
3645pub fn finish_attempt(
3650 max_attempts: usize,
3651 queue: &Queue,
3652 task: &mut Task,
3653 state: &RunState,
3654 quota_before: &[QuotaLoss],
3655 result: Result<()>,
3656) -> Vec<QuotaLoss> {
3657 let detail = match result {
3658 Ok(()) => describe(state),
3659 Err(e) => format!("{e:#}"),
3660 };
3661 let fresh = losses_this_attempt(quota_before, &state.quota);
3662 let verdict = Verdict {
3663 status: state.status,
3664 left_pr: state.pr.is_some(),
3667 quota_hit: !fresh.is_empty(),
3673 parked: state.parked,
3677 no_viable_candidates: state.viable().is_empty(),
3680 };
3681 if let Some(replacement) = &state.replaces
3682 && let Some(pr) = &state.pr
3683 && let Ok(mut old) = RunState::load(&replacement.run)
3684 && let Some(d) = &mut old.merge_decision
3685 {
3686 d.replacement_pr = Some(pr.url.clone());
3687 let _ = old.save();
3688 }
3689 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
3690 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
3691 task.hold_unrequested_retry(Some(format!(
3692 "Candidate setup failed: {detail}. Select a complete implementation with `magi task recover-pr`, or correct the referenced source before explicitly releasing the task."
3693 )));
3694 }
3695 if task.status == TaskStatus::Done {
3696 supersede_prior_runs(task, &crate::run::home());
3697 }
3698 record(queue, task);
3699 tracing::info!(
3700 "task {} is {} after run {} ({})",
3701 task.short(),
3702 task.status.as_str(),
3703 state.short(),
3704 label(state.status)
3705 );
3706 fresh
3707}
3708
3709pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
3713 let waits_on_owner = withheld_text_wait(task).is_some();
3718 if task.status.runnable()
3719 && !waits_on_owner
3720 && !(task.fresh_start && task.replacement.is_some())
3721 {
3722 let why = task.last_error.clone().map_or_else(
3723 || "the run did not finish".to_owned(),
3724 |e| format!("the run did not finish: {e}"),
3725 );
3726 task.hold_unrequested_retry(Some(format!(
3727 "{why}. It was started by hand, so it is not retried \
3728 automatically; `magi task release` retries it."
3729 )));
3730 record(queue, task);
3731 }
3732}
3733
3734#[must_use]
3738pub fn loop_would_resume(run: &str) -> bool {
3739 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
3740}
3741
3742#[must_use]
3752pub fn foreign_loop(
3753 reading: Option<&Reading>,
3754 now: Timestamp,
3755 own_pid: u32,
3756) -> Option<Option<u32>> {
3757 let reading = reading.filter(|r| r.running(now))?;
3758 match reading.pid {
3759 Some(pid) if pid == own_pid => None,
3760 pid => Some(pid),
3761 }
3762}
3763
3764pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
3779 let status = Arc::new(Mutex::new(Status::new()));
3780 let stop = Stop::new();
3781 attempt(
3782 opts,
3783 queue,
3784 &status,
3785 &stop,
3786 crate::graph::Pause::new(),
3787 task,
3788 )
3789 .await;
3790 let mut resumes = 0;
3799 while resumes < MAX_WITHHELD_RESUMES {
3800 let Some(timeout) = withheld_text_wait(task) else {
3801 break;
3802 };
3803 eprintln!(
3804 "waiting for the owner's answer on a withheld pull request text \
3805 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
3806 );
3807 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
3810 let ready = loop {
3811 match land_resume_state(task) {
3812 LandResume::StillWaiting if std::time::Instant::now() < limit => {
3813 tokio::time::sleep(WITHHELD_POLL).await;
3814 }
3815 LandResume::Ready => break true,
3816 _ => break false,
3817 }
3818 };
3819 if !ready {
3820 break;
3821 }
3822 resumes += 1;
3823 attempt(
3824 opts,
3825 queue,
3826 &status,
3827 &stop,
3828 crate::graph::Pause::new(),
3829 task,
3830 )
3831 .await;
3832 }
3833 hold_if_runnable(queue, task);
3834}
3835
3836const MAX_WITHHELD_RESUMES: u32 = 3;
3839
3840const WITHHELD_POLL: Duration = Duration::from_secs(15);
3842
3843fn withheld_text_wait(task: &Task) -> Option<u64> {
3846 let id = task.runs.last()?;
3847 let s = RunState::load(id).ok()?;
3848 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
3849 .then_some(s.config.graph.answer_timeout)
3850}
3851
3852fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
3862 after
3863 .iter()
3864 .filter(|q| !before.contains(q))
3865 .cloned()
3866 .collect()
3867}
3868
3869fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
3872 if quota.is_empty() {
3873 return None;
3874 }
3875 let with_hint = quota.iter().find(|q| q.reset.is_some());
3876 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
3877 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
3878 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
3879 Some(
3880 now.checked_add(jiff::SignedDuration::from_secs(secs))
3881 .unwrap_or(Timestamp::MAX),
3882 )
3883}
3884
3885fn implementers_match(task: &Task, state: &RunState) -> bool {
3887 task.replacement.as_ref().map_or_else(
3888 || {
3889 task.implementers()
3890 .is_none_or(|n| n == state.config.graph.implementers)
3891 },
3892 |replacement| {
3893 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
3894 && state.config.graph.implementers == 1
3895 },
3896 )
3897}
3898
3899fn starter_with_implementers(
3901 task: &Task,
3902 previous: Option<&RunState>,
3903 proposed: Starter,
3904) -> Starter {
3905 if previous.is_some_and(|s| !implementers_match(task, s)) {
3906 task.review_of
3907 .as_ref()
3908 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
3909 } else {
3910 proposed
3911 }
3912}
3913
3914fn apply_task_implementers(config: &mut Config, task: &Task) {
3916 if let Some(n) = task.implementers() {
3917 config.graph.implementers = n;
3918 }
3919}
3920
3921fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
3926 let Some(o) = &task.overrides else {
3927 return prepare(repo, opts);
3928 };
3929 let own = Opts {
3933 config: o.config.clone(),
3934 merge: None,
3935 ..opts.clone()
3936 };
3937 let mut config = prepare(repo, if o.inherit_daemon_options { opts } else { &own })?;
3938 o.apply(&mut config);
3939 Ok(config)
3940}
3941
3942fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
3944 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
3945 if let Some(mode) = &opts.merge {
3946 config.merge.mode = merge_mode(mode)?;
3947 }
3948 Ok(config)
3949}
3950
3951async fn maybe_prune_cache_between_runs(
3982 repo: &Path,
3983 opts: &Opts,
3984 home: &Path,
3985 stop: &Stop,
3986 last_checked: &mut Option<Timestamp>,
3987 now: Timestamp,
3988) {
3989 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
3990 return;
3991 }
3992 *last_checked = Some(now);
3993 let cfg = match prepare(repo, opts) {
3994 Ok(cfg) => cfg,
3995 Err(e) => {
3996 tracing::warn!("cache check: no config: {e:#}");
3997 return;
3998 }
3999 };
4000 match clean::prune_cache_if_over_limit(&cfg, home) {
4001 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
4002 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
4003 pruned.files,
4004 pruned.freed
4005 ),
4006 Ok(_) => {}
4007 Err(e) => {
4008 tracing::warn!("housekeep: prune cache: {e:#}");
4009 notices::raise_in(
4010 home,
4011 Notice::warn(
4012 "housekeep:cache",
4013 "Pruning the shared build cache failed; disk usage may keep growing.",
4014 ),
4015 );
4016 }
4017 }
4018}
4019
4020fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
4024 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
4025}
4026
4027async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
4046 let cfg = match prepare(repo, opts) {
4047 Ok(cfg) => cfg,
4048 Err(e) => {
4049 tracing::warn!("housekeep: no config: {e:#}");
4050 return;
4051 }
4052 };
4053 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
4062 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
4063 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
4068 let mut extra = Vec::new();
4069 if out.unreadable > 0 {
4070 extra.push(format!("{} unreadable", out.unreadable));
4071 }
4072 if out.orphaned_worktrees > 0 {
4073 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
4074 }
4075 let detail = if extra.is_empty() {
4076 String::new()
4077 } else {
4078 format!(" ({})", extra.join(", "))
4079 };
4080 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
4081 }
4082 if out.external_merges_recorded > 0 {
4083 tracing::info!(
4084 "housekeep: recorded {} run(s) as merged externally",
4085 out.external_merges_recorded
4086 );
4087 }
4088 if out.stale_pr_states_repaired > 0 {
4089 tracing::info!(
4090 "housekeep: rewrote {} run record(s) whose pull request had already settled",
4091 out.stale_pr_states_repaired
4092 );
4093 }
4094 if out.cache_files > 0 {
4095 tracing::info!(
4096 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
4097 out.cache_files,
4098 out.cache_freed
4099 );
4100 }
4101 if out.questions_abandoned > 0 {
4102 tracing::info!(
4103 "housekeep: abandoned {} question(s) left open by a finished run",
4104 out.questions_abandoned
4105 );
4106 }
4107}
4108
4109async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
4118 let questions = Questions::at(home.join("questions"));
4119 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
4120 if report.is_empty() {
4121 return;
4122 }
4123 if !report.quarantined.is_empty() {
4124 tracing::info!(
4125 "triage: held {} blocked task(s) whose blocked-on task or \
4126 question no longer exists: {}",
4127 report.quarantined.len(),
4128 report.quarantined.join(", ")
4129 );
4130 }
4131 if !report.resumed.is_empty() {
4132 tracing::info!(
4133 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
4134 report.resumed.len(),
4135 report.resumed.join(", ")
4136 );
4137 }
4138 if !report.asked.is_empty() {
4139 tracing::info!(
4140 "triage: asked about {} held task(s): {}",
4141 report.asked.len(),
4142 report.asked.join(", ")
4143 );
4144 }
4145 if !report.answered.is_empty() {
4146 tracing::info!(
4147 "triage: applied {} operator answer(s): {}",
4148 report.answered.len(),
4149 report.answered.join(", ")
4150 );
4151 }
4152}
4153
4154fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
4161 disk_gate_with(repo, config, crate::disk::free_bytes)
4162}
4163
4164fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
4168 repo: &Path,
4169 config: &Config,
4170 free_bytes: F,
4171) -> Option<String> {
4172 let min = config.disk.min_free_bytes;
4173 if min == 0 {
4174 return None;
4175 }
4176 match free_bytes(repo) {
4177 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
4178 Err(e) => Some(crate::disk::unmeasured_in(repo, &e, &config.graph.language)),
4179 }
4180}
4181
4182const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
4189
4190const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
4194
4195fn quota_wait(
4204 reset_at: Option<Timestamp>,
4205 now: Timestamp,
4206 fallback: Duration,
4207 cap: Duration,
4208) -> Duration {
4209 match reset_at {
4210 Some(at) if at > now => {
4211 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
4212 Duration::from_secs(secs).min(cap)
4213 }
4214 _ => fallback,
4215 }
4216}
4217
4218fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
4232 parse_reset_hint_zoned(text, now)
4233 .or_else(|| parse_reset_hint_dated(text))
4234 .or_else(|| parse_reset_hint_relative(text, recorded))
4235}
4236
4237fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
4241 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
4242 let mut rest = rest.trim();
4243 if rest.is_empty() {
4244 return None;
4245 }
4246 let mut total: i64 = 0;
4247 let mut matched = false;
4248 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
4249 if let Some((digits, tail)) = rest.split_once(unit)
4250 && !digits.is_empty()
4251 && digits.bytes().all(|b| b.is_ascii_digit())
4252 {
4253 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
4254 rest = tail;
4255 matched = true;
4256 }
4257 }
4258 if !rest.is_empty() || !matched {
4259 return None;
4260 }
4261 recorded
4262 .checked_add(jiff::SignedDuration::from_secs(total))
4263 .ok()
4264}
4265
4266fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
4270 let clock = clock.trim().to_lowercase();
4271 let (digits, pm) = clock
4272 .strip_suffix("am")
4273 .map(|d| (d, false))
4274 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
4275 let (h, m) = digits.trim().split_once(':')?;
4276 let mut hour: i8 = h.trim().parse().ok()?;
4277 let minute: i8 = m.trim().parse().ok()?;
4278 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
4279 return None;
4280 }
4281 if pm && hour != 12 {
4282 hour += 12;
4283 } else if !pm && hour == 12 {
4284 hour = 0;
4285 }
4286 Some((hour, minute))
4287}
4288
4289fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
4294 let open = text.find('(')?;
4295 let close = text.rfind(')')?;
4296 if close <= open {
4297 return None;
4298 }
4299 let zone = text[open + 1..close].trim();
4300 let (hour, minute) = parse_12h_clock(&text[..open])?;
4301 let tz = jiff::tz::TimeZone::get(zone).ok()?;
4302 let candidate = now
4303 .to_zoned(tz)
4304 .with()
4305 .hour(hour)
4306 .minute(minute)
4307 .second(0)
4308 .millisecond(0)
4309 .microsecond(0)
4310 .nanosecond(0)
4311 .build()
4312 .ok()?;
4313 let mut at = candidate.timestamp();
4314 if at <= now {
4315 at += jiff::SignedDuration::from_hours(24);
4316 }
4317 Some(at)
4318}
4319
4320fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
4329 let words: Vec<&str> = text.split_whitespace().collect();
4330 if words.len() < 5 {
4331 return None;
4332 }
4333 (0..=words.len() - 5)
4334 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
4335}
4336
4337fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
4343 if trailing.is_some_and(|next| next.starts_with('(')) {
4344 return None;
4345 }
4346 let month = month_number(window[0])?;
4347 let day_token = window[1].strip_suffix(',')?.to_lowercase();
4348 let day_digits = ["st", "nd", "rd", "th"]
4349 .iter()
4350 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
4351 let day: i8 = day_digits.parse().ok()?;
4352 let year_token = window[2];
4353 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
4354 return None;
4355 }
4356 let year: i16 = year_token.parse().ok()?;
4357 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
4361 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
4362 let date = jiff::civil::Date::new(year, month, day).ok()?;
4363 let candidate = date
4364 .at(hour, minute, 0, 0)
4365 .to_zoned(jiff::tz::TimeZone::UTC)
4366 .ok()?;
4367 Some(candidate.timestamp())
4368}
4369
4370fn month_number(name: &str) -> Option<i8> {
4373 const NAMES: [&str; 12] = [
4374 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
4375 ];
4376 let lower = name.to_lowercase();
4377 NAMES
4378 .iter()
4379 .position(|n| *n == lower.as_str())
4380 .map(|i| i as i8 + 1)
4381}
4382
4383fn exhausted_review_budget(state: &RunState) -> bool {
4395 state.status == RunStatus::Blocked
4398 && !state.pr_publication_pending
4399 && state.reviews.len() >= state.config.graph.review_rounds
4400}
4401
4402fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
4439 unfinished_run_with(runs, short, RunState::load)
4440}
4441
4442fn unfinished_run_with<F>(runs: &[String], short: &str, load: F) -> Option<String>
4445where
4446 F: FnOnce(&str) -> Result<RunState>,
4447{
4448 let id = runs.last()?;
4449 match load(id) {
4450 Ok(s)
4455 if s.status.resumable()
4456 && !s.released()
4457 && !exhausted_review_budget(&s)
4458 && s.liveness(false) != crate::run::Liveness::Live =>
4459 {
4460 Some(id.clone())
4461 }
4462 Ok(_) => None,
4463 Err(e) => {
4464 tracing::warn!("could not read run {id} for task {short}: {e:#}");
4465 None
4466 }
4467 }
4468}
4469
4470fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
4478where
4479 F: FnOnce(&str) -> Result<RunState>,
4480{
4481 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
4483 || task.fresh_start
4484 || task.review_branch.is_some()
4485 {
4486 return false;
4487 }
4488 let Some(id) = task.runs.last() else {
4489 return false;
4490 };
4491 load(id).is_ok_and(|s| {
4492 s.parked && s.status.resumable() && !s.released() && !exhausted_review_budget(&s)
4493 })
4494}
4495
4496#[derive(Debug, Clone, PartialEq, Eq)]
4499enum Starter {
4500 Review(String),
4503 Resume(String),
4505 Start,
4507}
4508
4509fn selected_starter(
4511 selected: &crate::pr_recovery::Selection,
4512 previous: Option<&RunState>,
4513 unfinished: Option<&str>,
4514) -> Starter {
4515 if let Some(id) = unfinished
4516 && previous.is_some_and(|r| {
4517 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
4518 })
4519 {
4520 Starter::Resume(id.to_owned())
4521 } else {
4522 Starter::Review(selected.branch.clone())
4523 }
4524}
4525
4526fn take_divergence_answer(
4530 branch: &str,
4531 remote: &str,
4532 task: &mut Task,
4533) -> Option<crate::reconcile::Choice> {
4534 let summary = crate::reconcile::summary_for(branch, remote);
4535 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
4536 (a.question == summary)
4537 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
4538 .flatten()
4539 .map(|c| (i, c))
4540 })?;
4541 task.answers.remove(idx);
4542 Some(choice)
4543}
4544
4545fn choose_starter(
4557 review_branch: Option<&str>,
4558 branch_exists: bool,
4559 unfinished: Option<&str>,
4560) -> Starter {
4561 match review_branch {
4562 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
4563 Some(_) => Starter::Start,
4564 None => match unfinished {
4565 Some(id) => Starter::Resume(id.to_owned()),
4566 None => Starter::Start,
4567 },
4568 }
4569}
4570
4571fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
4573 let branch = task.review_branch.take();
4574 let instruction = task.review_instruction.take();
4575 branch.filter(|branch| match &instruction {
4576 Some(authored) => *authored == task.instruction,
4577 None => previous.map_or_else(
4580 || task.review_of.as_ref() == Some(branch),
4581 |run| compatible_run_instruction(run, task),
4582 ),
4583 })
4584}
4585
4586fn may_resume_run(run: &RunState, task: &Task) -> bool {
4589 task.resume_override
4590 .as_ref()
4591 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
4592 || compatible_run_instruction(run, task)
4593}
4594
4595pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
4597 if let Some(authored) = &run.task_instruction {
4598 return compatible_instruction(authored, task);
4599 }
4600 if compatible_instruction(&run.instruction, task) {
4601 return true;
4602 }
4603 if run.reviewed_commits.is_some() {
4606 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
4607 return strip_answers_block(authored, task).trim() == task.instruction.trim();
4608 }
4609 if let Some(branch) = &task.review_of {
4610 let canonical = format!(
4611 "Review the branch `{branch}` (review-only: no implementation, no judging)."
4612 );
4613 let generated = format!(
4614 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
4615 );
4616 return task.instruction == canonical && run.instruction.starts_with(&generated);
4617 }
4618 }
4619 false
4620}
4621
4622pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
4624 strip_answers_block(old, task) == task.instruction
4625}
4626
4627fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
4630 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
4631 return fallback.to_path_buf();
4632 }
4633 task.repo.clone()
4634}
4635
4636const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
4640
4641fn answers_block(task: &Task, count: usize) -> String {
4643 let mut s = ANSWERS_HEADER.to_owned();
4644 for a in &task.answers[..count] {
4645 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
4646 }
4647 s
4648}
4649
4650fn resumption_block(r: &crate::queue::Resumption) -> String {
4653 format!(
4654 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
4655 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
4656 r.generation, r.at, r.instruction, r.answers_through,
4657 )
4658}
4659
4660fn append_answers(base: &str, task: &Task) -> String {
4662 let mut s = base.to_owned();
4663 if !task.answers.is_empty() {
4664 s.push_str(&answers_block(task, task.answers.len()));
4665 }
4666 if let Some(r) = &task.resumption {
4667 s.push_str(&resumption_block(r));
4668 }
4669 s
4670}
4671
4672fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
4676 let mut instruction = instruction;
4677 for approval in task
4678 .resumption
4679 .iter()
4680 .chain(task.resumption_history.iter().rev())
4681 {
4682 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
4683 instruction = base;
4684 break;
4685 }
4686 }
4687 for count in (1..=task.answers.len()).rev() {
4688 let block = answers_block(task, count);
4689 if let Some(base) = instruction.strip_suffix(&block) {
4690 return base;
4691 }
4692 }
4693 instruction
4694}
4695
4696fn instruction_for(task: &Task) -> String {
4704 append_answers(&task.instruction, task)
4705}
4706
4707fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
4719 append_answers(strip_answers_block(old_instruction, task), task)
4720}
4721
4722fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
4724 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
4725 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
4726 if let Some(cause) = task.hold_notice_cause.as_mut() {
4727 cause.target = failure.map(|f| f.target.clone());
4728 }
4729}
4730
4731#[derive(Debug, thiserror::Error)]
4732#[error("attachment `{name}` is recorded on the task but {path} is missing")]
4733struct MissingAttachment {
4734 name: String,
4735 path: PathBuf,
4736}
4737
4738fn task_attachments(
4740 queue: &Queue,
4741 task: &Task,
4742) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
4743 let paths = queue.attachment_paths(task);
4744 for (name, path) in task.attachments.iter().zip(&paths) {
4745 if !path.is_file() {
4746 return Err(MissingAttachment {
4747 name: name.clone(),
4748 path: path.clone(),
4749 });
4750 }
4751 }
4752 Ok(paths)
4753}
4754
4755fn prepare_instruction(
4766 starter: &Starter,
4767 old_instruction: Option<&str>,
4768 task: &Task,
4769) -> Option<String> {
4770 match starter {
4771 Starter::Start => Some(instruction_for(task)),
4772 Starter::Resume(_) => Some(resumed_instruction(
4773 old_instruction.expect("a resumed run always has a prior instruction"),
4774 task,
4775 )),
4776 Starter::Review(_) => None,
4777 }
4778}
4779
4780fn record(queue: &Queue, task: &mut Task) {
4784 if let Err(e) = queue.put(task) {
4785 tracing::error!("could not record task {}: {e:#}", task.short());
4786 let notice = Notice::error(
4787 "loop:record",
4788 "The loop could not save a task's state; check the disk.",
4789 )
4790 .link(Link::Task {
4791 id: task.id.clone(),
4792 });
4793 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
4795 Some(home) => notices::raise_in(home, notice),
4796 None => notices::raise(notice),
4797 }
4798 }
4799}
4800
4801fn runnable(queue: &Queue) -> Vec<Task> {
4807 let mut tasks: Vec<Task> = queue
4808 .list()
4809 .into_iter()
4810 .filter(|t| t.status.runnable())
4811 .collect();
4812 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)));
4813 tasks
4814}
4815
4816fn describe(state: &RunState) -> String {
4830 let p = phrases(&state.config.graph.language);
4831 let mut detail = if state.status == RunStatus::Stalled {
4832 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
4833 seats.sort_unstable();
4834 seats.dedup();
4835 if seats.is_empty() {
4836 p.quorum_lost.to_owned()
4837 } else {
4838 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
4839 }
4840 } else {
4841 format!("{}{}", p.run_ended, state.status.display_label())
4842 };
4843 if let Some(last) = state.events.last() {
4844 let unanswered = state
4848 .reviews
4849 .last()
4850 .filter(|r| {
4851 state.status == RunStatus::Blocked
4852 && last.node == "review"
4853 && r.incomplete()
4854 && r.blocking == 0
4855 && r.round == state.config.graph.review_rounds
4856 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
4857 })
4858 .map(|r| (r.expected - r.answered, r.round));
4859 match unanswered {
4860 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
4861 detail.push_str(&format!(
4862 " ({}: {})",
4863 last.node,
4864 (p.reviewers_never_answered)(missing, rounds)
4865 ));
4866 }
4867 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
4868 }
4869 }
4870 detail.push_str(&format!(" [run {}]", state.id));
4871 detail
4872}
4873
4874const DIAGNOSTIC_MAX: usize = 4_000;
4880
4881const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
4886
4887fn diagnostic(state: &RunState) -> Option<String> {
4901 let mut parts: Vec<String> = Vec::new();
4902
4903 for o in state.gate.iter().filter(|o| !o.ok()) {
4905 parts.push(format!(
4906 "gate `{}` failed ({:?}):\n{}",
4907 o.command,
4908 o.code,
4909 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
4910 ));
4911 }
4912
4913 if let Some(last) = state
4916 .events
4917 .iter()
4918 .rev()
4919 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
4920 {
4921 parts.push(last.message.clone());
4922 }
4923
4924 if state.viable().is_empty() {
4931 for c in &state.candidates {
4932 if let Some(evidence) = &c.verified_noop {
4933 parts.push(format!(
4934 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
4935 c.label
4936 ));
4937 } else if !c.summary.trim().is_empty() {
4938 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
4939 } else if let Some(why) = &c.failed {
4940 parts.push(format!("candidate {}: {why}", c.label));
4941 }
4942 }
4943 }
4944
4945 if parts.is_empty() {
4946 return None;
4947 }
4948 Some(crate::run::tail(
4953 &parts.join("\n\n"),
4954 DIAGNOSTIC_MAX.saturating_sub(100),
4955 ))
4956}
4957
4958fn label(status: RunStatus) -> &'static str {
4966 status.as_str()
4967}
4968
4969pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
4971 match mode {
4972 "none" => Ok(MergeMode::None),
4973 "local" => Ok(MergeMode::Local),
4974 "pr" => Ok(MergeMode::Pr),
4975 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
4976 }
4977}
4978
4979fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
4984 mutex
4985 .lock()
4986 .unwrap_or_else(std::sync::PoisonError::into_inner)
4987}
4988
4989#[cfg(test)]
4990mod tests {
4991 use super::*;
4992 use crate::queue::{Source, TaskStatus};
4993 use crate::run::{Candidate, CommandOutcome};
4994 use pretty_assertions::assert_eq;
4995
4996 fn task() -> Task {
4997 Task::new(
4998 "add retries".to_owned(),
4999 "add retries".to_owned(),
5000 PathBuf::from("/repo"),
5001 Source::Human,
5002 )
5003 }
5004
5005 #[test]
5006 fn selected_public_head_overrides_historical_resume_and_pin() {
5007 let mut t = task();
5008 let mut old = run_state(RunStatus::Blocked);
5009 let selection = crate::pr_recovery::Selection {
5010 url: "url".into(),
5011 branch: format!("magi/task-{}", t.id),
5012 head: "a".repeat(40),
5013 instruction: t.instruction.clone(),
5014 approved_at: Timestamp::now(),
5015 };
5016 t.resume_override = Some(crate::queue::OperatorResume {
5017 pinned_run: Some(old.id.clone()),
5018 question_id: "choice".into(),
5019 at: Timestamp::now(),
5020 conductor_rehold: None,
5021 forced: false,
5022 });
5023 assert_eq!(
5024 selected_starter(&selection, Some(&old), Some(&old.id)),
5025 Starter::Review(selection.branch.clone())
5026 );
5027 old.pr_recovery = Some(selection.clone());
5028 assert_eq!(
5029 selected_starter(&selection, Some(&old), Some(&old.id)),
5030 Starter::Review(selection.branch.clone())
5031 );
5032 old.reviewed_commits = Some(vec!["implementation".into()]);
5033 assert_eq!(
5034 selected_starter(&selection, Some(&old), Some(&old.id)),
5035 Starter::Resume(old.id.clone())
5036 );
5037 let mut changed = selection.clone();
5038 changed.head = "b".repeat(40);
5039 assert_eq!(
5040 selected_starter(&changed, Some(&old), Some(&old.id)),
5041 Starter::Review(changed.branch)
5042 );
5043 }
5044
5045 #[test]
5046 fn candidate_transplant_failure_is_one_actionable_hold() {
5047 let dir = tempfile::tempdir().unwrap();
5048 let q = Queue::at(dir.path().join("queue"));
5049 let mut t = task();
5050 let mut state = run_state(RunStatus::Blocked);
5051 state.event("prep", "cannot apply referenced commit: conflict");
5052 state.prep_failure = Some("conflict".into());
5053 t.start(state.id.clone());
5054 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
5055 assert_eq!(t.attempts, 1);
5056 assert_eq!(t.status, TaskStatus::Held);
5057 assert!(t.operator_held());
5058 assert!(!t.status.runnable());
5059 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
5060 let first = t.chat_notice_key().unwrap();
5061 t.chat_notices.push(first.clone());
5062 t.release_operator("test", None);
5063 state.id = "another-unprepared-run".into();
5064 t.start(state.id.clone());
5065 finish_attempt(
5066 3,
5067 &q,
5068 &mut t,
5069 &state,
5070 &[],
5071 Err(anyhow::anyhow!("still conflicts")),
5072 );
5073 assert_eq!(
5074 t.chat_notice_key().as_ref(),
5075 Some(&first),
5076 "no work happened"
5077 );
5078 t.release_operator("test", None);
5079 state.prep_failure = None;
5080 state.status = RunStatus::Failed;
5081 state.begin_pass();
5082 t.start(state.id.clone());
5083 finish_attempt(
5084 1,
5085 &q,
5086 &mut t,
5087 &state,
5088 &[],
5089 Err(anyhow::anyhow!("new gate failure")),
5090 );
5091 let failure = t.chat_notice_key().unwrap();
5092 assert_ne!(failure, first, "approved recovery reached a real failure");
5093 t.release_operator("test", None);
5094 state.begin_pass();
5095 t.start(state.id.clone());
5096 finish_attempt(
5097 1,
5098 &q,
5099 &mut t,
5100 &state,
5101 &[],
5102 Err(anyhow::anyhow!("new failure on resume")),
5103 );
5104 assert_ne!(
5105 t.chat_notice_key().as_ref(),
5106 Some(&failure),
5107 "a new settled pass"
5108 );
5109 }
5110
5111 #[test]
5112 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
5113 for source in ["cli", "web"] {
5114 let mut t = task();
5115 let mut run = run_state(RunStatus::Blocked);
5116 run.instruction = "Run A original criteria".into();
5117 run.task_instruction = Some(run.instruction.clone());
5118 t.start(run.id.clone());
5119 t.release_operator("choice", Some("resume-question"));
5120 t.resume_override = Some(crate::queue::OperatorResume {
5121 question_id: "resume-question".into(),
5122 at: Timestamp::now(),
5123 conductor_rehold: None,
5124 forced: true,
5125 pinned_run: Some(run.id.clone()),
5126 });
5127 t.hold_machine(Some("not enough free space to start a run".into()));
5128 let mut unchanged = t.clone();
5129 unchanged
5130 .edit("title changed".into(), unchanged.instruction.clone())
5131 .unwrap();
5132 unchanged.release_operator(source, None);
5133 assert_eq!(
5134 unchanged
5135 .resume_override
5136 .as_ref()
5137 .unwrap()
5138 .pinned_run
5139 .as_deref(),
5140 Some(run.id.as_str())
5141 );
5142 assert!(!compatible_run_instruction(&run, &unchanged));
5143 assert!(may_resume_run(&run, &unchanged));
5144
5145 t.edit(
5146 "approved recovery".into(),
5147 "Reuse measured PR head B; fix Clippy; review".into(),
5148 )
5149 .unwrap();
5150 assert!(t.resume_override.is_none());
5151 t.release_operator(source, None);
5152 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
5153 assert!(reloaded.resume_override.is_none());
5154 assert!(!may_resume_run(&run, &reloaded));
5155 let unfinished =
5156 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
5157 .filter(|_| may_resume_run(&run, &reloaded));
5158 assert_eq!(
5159 choose_starter(None, false, unfinished.as_deref()),
5160 Starter::Start
5161 );
5162 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
5163 assert_eq!(
5164 reloaded.resumption_history[0].question_id.as_deref(),
5165 Some("resume-question")
5166 );
5167
5168 unchanged.instruction = "Reuse measured PR head B".into();
5171 unchanged.hold_machine(Some("disk hold".into()));
5172 unchanged.release_operator(source, None);
5173 assert!(unchanged.resume_override.is_none());
5174 assert!(!may_resume_run(&run, &unchanged));
5175 }
5176 }
5177
5178 #[test]
5179 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
5180 let mut t = task();
5181 let mut run = run_state(RunStatus::Blocked);
5182 t.review_of = Some("approved-head".into());
5183 t.instruction =
5184 "Review the branch `approved-head` (review-only: no implementation, no judging)."
5185 .into();
5186 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
5187 run.reviewed_commits = Some(vec!["Commit log".into()]);
5188 t.start(run.id.clone());
5189 t.hold_machine(Some("interrupted review".into()));
5190 t.release_operator("cli", None);
5191 assert!(
5192 compatible_run_instruction(&run, &t),
5193 "legacy explicit review remains resumable"
5194 );
5195 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
5196 .filter(|_| compatible_run_instruction(&run, &t));
5197 assert_eq!(
5198 choose_starter(None, false, unfinished.as_deref()),
5199 Starter::Resume(run.id.clone())
5200 );
5201 run.task_instruction = Some(t.instruction.clone());
5202 assert!(compatible_run_instruction(&run, &t));
5203 let reloaded: RunState =
5204 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
5205 assert!(compatible_run_instruction(&reloaded, &t));
5206 t.edit(
5207 "new approved head".into(),
5208 "Reuse measured PR head B".into(),
5209 )
5210 .unwrap();
5211 assert!(!compatible_run_instruction(&run, &t));
5212
5213 t.review_of = None;
5215 t.instruction = "Fix Clippy on approved head".into();
5216 run.task_instruction = None;
5217 run.instruction = format!(
5218 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
5219 t.instruction
5220 );
5221 assert!(compatible_run_instruction(&run, &t));
5222 t.instruction.push('\n');
5223 assert!(
5224 compatible_run_instruction(&run, &t),
5225 "legacy generated review trimmed the original criteria"
5226 );
5227 t.hold_for_handover(Some("approved-head".into()), "refused".into());
5228 let mut unchanged = t.clone();
5229 assert_eq!(
5230 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
5231 Some("approved-head")
5232 );
5233 t.edit("new plan".into(), "Reuse measured PR head B".into())
5234 .unwrap();
5235 assert!(take_review_branch(&mut t, Some(&run)).is_none());
5236 assert!(!compatible_run_instruction(&run, &t));
5237 t.review_branch = Some("approved-head".into());
5238 t.review_instruction = None;
5239 assert!(
5240 take_review_branch(&mut t, Some(&run)).is_none(),
5241 "legacy retained candidate cannot replace a revised approved head"
5242 );
5243 }
5244
5245 #[test]
5246 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
5247 crate::run::pin_test_home();
5248 let dir = tempfile::tempdir().unwrap();
5249 let queue = Queue::at(dir.path().join("queue"));
5250 for resume in [true, false] {
5251 let state = run_state(RunStatus::Blocked);
5252 let mut t = task();
5253 t.start(state.id.clone());
5254 let action = if resume {
5255 ask::ChoiceAction::Resume {
5256 run: state.id.clone(),
5257 }
5258 } else {
5259 ask::ChoiceAction::Requeue
5260 };
5261 let mut q = action_question(&state.id, action);
5262 q.status = ask::QuestionStatus::Open;
5263 q.answer = None;
5264 q.answered_at = None;
5265 t.fail("run stopped with a question", 3);
5266 hold_if_runnable(&queue, &mut t);
5267 assert!(t.operator_decided_at.is_none());
5268 assert!(t.operator_held());
5269 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
5270 let expected = if resume {
5271 ActionDecision::Resume(state.id.clone())
5272 } else {
5273 ActionDecision::Requeue
5274 };
5275 assert_eq!(
5276 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5277 expected
5278 );
5279 t.hold_manual(Some("later explicit hold".into()));
5280 assert_eq!(
5281 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5282 ActionDecision::Stale
5283 );
5284 }
5285 }
5286
5287 #[test]
5288 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
5289 let mut t = task();
5290 t.answers.push(crate::queue::AnsweredQuestion {
5291 question: "Inspect the remote?".into(),
5292 answer: "Inspect then wait".into(),
5293 });
5294 let old = instruction_for(&t);
5295 assert!(compatible_instruction(&old, &t));
5296 t.instruction = "Reuse approved PR head and review".into();
5297 t.release_operator("cli", None);
5298 assert!(!compatible_instruction(&old, &t));
5299 let approved = instruction_for(&t);
5300 assert!(approved.contains("waiting premises are superseded"));
5301 assert!(compatible_instruction(&approved, &t));
5302 assert_eq!(resumed_instruction(&approved, &t), approved);
5303 t.release_operator("web", None);
5304 assert!(compatible_instruction(&approved, &t));
5305 assert_eq!(t.resumption_history.len(), 1);
5306 t.start("run".into());
5307 assert!(instruction_for(&t).contains("Explicit operator resumption"));
5308 }
5309
5310 #[test]
5314 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
5315 let mut state = RunState::new(
5316 PathBuf::from("/repo"),
5317 "main".to_owned(),
5318 "abc1234".to_owned(),
5319 "task".to_owned(),
5320 Config::default(),
5321 );
5322 state.status = RunStatus::Blocked;
5323 state.resource_block = Some(crate::run::ResourceBlock {
5324 node: "graft".to_owned(),
5325 kind: "shared build cache".to_owned(),
5326 detail: "timed out".to_owned(),
5327 at: jiff::Timestamp::now(),
5328 });
5329 let mut t = task();
5330 t.start("20260902-000000-aaaa".to_owned());
5331 assert_eq!(t.attempts, 1);
5332 let verdict = Verdict {
5333 status: RunStatus::Blocked,
5334 left_pr: false,
5335 quota_hit: false,
5336 parked: false,
5337 no_viable_candidates: false,
5338 };
5339 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
5340 assert_eq!(t.status, TaskStatus::Held);
5341 assert_eq!(t.attempts, 0, "the attempt is refunded");
5342 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
5343 assert!(
5344 !t.operator_held(),
5345 "a machine hold, released without a human"
5346 );
5347
5348 let mut selected = task();
5349 selected.start("20260902-000000-cccc".into());
5350 selected.replacement = Some(crate::merge_decision::Replacement {
5351 question: "fixture".into(),
5352 run: "earlier".into(),
5353 pr: "https://example.invalid/pr/1".into(),
5354 agent: "selected".into(),
5355 context: "fixture".into(),
5356 });
5357 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
5358 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
5359 assert_eq!(selected.attempts, 0);
5360 assert!(
5361 !selected.operator_held(),
5362 "a selected agent also retains a machine resource hold"
5363 );
5364 assert!(
5365 selected
5366 .hold_reason
5367 .as_deref()
5368 .unwrap()
5369 .starts_with("[resource] ")
5370 );
5371
5372 let mut p = task();
5374 p.start("20260902-000000-bbbb".to_owned());
5375 settle_and_diagnose(
5376 &mut p,
5377 Verdict {
5378 parked: true,
5379 ..verdict
5380 },
5381 "parked",
5382 2,
5383 &state,
5384 );
5385 assert_eq!(p.status, TaskStatus::Parked);
5386 }
5387
5388 #[test]
5389 fn record_failure_notice_links_the_task() {
5390 let dir = tempfile::tempdir().unwrap();
5391 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
5394 let queue = Queue::at(dir.path().join("queue"));
5395 let mut t = interrupt_task("rec-fail-1");
5396 record(&queue, &mut t);
5397
5398 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
5399 assert_eq!(found.len(), 1);
5400 assert_eq!(found[0].key, "loop:record");
5401 assert_eq!(
5402 found[0].link,
5403 Some(Link::Task {
5404 id: "rec-fail-1".to_owned()
5405 })
5406 );
5407 }
5408
5409 fn interrupt_task(id: &str) -> Task {
5412 let mut t = task();
5413 t.id = id.to_owned();
5414 t.interrupt = true;
5415 t
5416 }
5417
5418 fn task_with_id(id: &str) -> Task {
5420 let mut t = task();
5421 t.id = id.to_owned();
5422 t
5423 }
5424
5425 fn urgent_task(id: &str) -> Task {
5427 let mut t = task();
5428 t.id = id.to_owned();
5429 t.urgent = true;
5430 t
5431 }
5432
5433 #[test]
5434 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
5435 let dir = tempfile::tempdir().unwrap();
5436 let queue = Queue::at(dir.path().join("queue"));
5437 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5438 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5439 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
5440 let mut first = task();
5441 queue.put(&mut first).unwrap();
5442 let stale_plain = queue.get(&first.id).unwrap();
5443 first.set_urgent(true).unwrap();
5444 queue.put(&mut first).unwrap();
5445 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
5446 let mut second = urgent_task("second");
5447 queue.put(&mut second).unwrap();
5448 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
5449 drop(queue.claim(&second.id).unwrap());
5450 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
5451 drop(dispatched);
5452 let stale_urgent = queue.get(&first.id).unwrap();
5453 first.set_urgent(false).unwrap();
5454 queue.put(&mut first).unwrap();
5455 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
5456 assert_eq!(urgent.available_permits(), 1);
5457 drop(queue.claim(&first.id).unwrap());
5458 drop(occupied);
5459 }
5460
5461 #[test]
5462 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
5463 let dir = tempfile::tempdir().unwrap();
5464 let queue = Queue::at(dir.path().join("queue"));
5465 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5466 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5467 let mut t = task();
5468 queue.put(&mut t).unwrap();
5469 for enabled in [true, false] {
5470 let stale = queue.get(&t.id).unwrap();
5471 t.set_urgent(enabled).unwrap();
5472 queue.put(&mut t).unwrap();
5473 assert_ne!(stale.urgent, enabled);
5474 let DispatchClaim {
5475 task: fresh,
5476 claim,
5477 permit,
5478 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
5479 assert_eq!(fresh.urgent, enabled);
5480 assert_eq!(ordinary.available_permits(), usize::from(enabled));
5481 assert_eq!(urgent.available_permits(), usize::from(!enabled));
5482 assert!(queue.claim(&t.id).is_err());
5483 assert_eq!(fresh.attempts, stale.attempts);
5484 drop((claim, permit));
5485 assert_eq!(ordinary.available_permits(), 1);
5486 assert_eq!(urgent.available_permits(), 1);
5487 }
5488 t.set_urgent(true).unwrap();
5489 queue.put(&mut t).unwrap();
5490 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
5491 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5492 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
5493 drop(queue.claim(&t.id).unwrap());
5494 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
5495 drop(occupied);
5496 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
5497 drop(queue.claim(&t.id).unwrap());
5498 let held = queue.claim(&t.id).unwrap();
5499 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5500 drop(held);
5501 t.status = TaskStatus::Held;
5502 queue.put(&mut t).unwrap();
5503 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5504 drop(queue.claim(&t.id).unwrap());
5505 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
5506 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5507 drop(queue.claim(&t.id).unwrap());
5508 assert_eq!(urgent.available_permits(), 1);
5509 }
5510
5511 #[test]
5516 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
5517 assert_eq!(permit_kind(true, false), PermitKind::None);
5518 assert_eq!(permit_kind(true, true), PermitKind::None);
5519 }
5520
5521 #[test]
5526 fn permit_kind_separates_urgent_from_ordinary() {
5527 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
5528 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
5529 }
5530
5531 #[test]
5538 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
5539 let cfg = Config::default();
5540 let repo = Path::new("/any/repo/path");
5541
5542 let reason =
5543 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
5544 assert!(reason.contains("1024"), "{reason}");
5545 assert!(
5546 reason.contains(&cfg.disk.min_free_bytes.to_string()),
5547 "{reason}"
5548 );
5549
5550 assert_eq!(
5551 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
5552 None,
5553 "exactly at the floor is open"
5554 );
5555 assert_eq!(
5556 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
5557 None,
5558 "comfortably above the floor is open"
5559 );
5560 }
5561
5562 #[test]
5563 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
5564 let mut cfg = Config::default();
5565 cfg.disk.min_free_bytes = 0;
5566 let repo = Path::new("/any/repo/path");
5567 assert_eq!(
5568 disk_gate_with(repo, &cfg, |_| Ok(0)),
5569 None,
5570 "a zero floor never measures at all"
5571 );
5572 }
5573
5574 #[test]
5575 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
5576 let cfg = Config::default();
5577 let repo = Path::new("/any/repo/path");
5578 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
5579 .expect("a measurement failure must close the gate, not open it");
5580 assert!(reason.contains("could not measure"), "{reason}");
5581 }
5582
5583 #[test]
5584 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
5585 let ordinary = task();
5586 let next = advance_interrupt(
5587 Interrupt::Idle,
5588 std::slice::from_ref(&ordinary.id),
5589 std::slice::from_ref(&ordinary),
5590 );
5591 assert_eq!(next, Interrupt::Idle);
5592 }
5593
5594 #[test]
5595 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
5596 let marked = interrupt_task("marked");
5599 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5600 assert_eq!(next, Interrupt::Idle);
5601 }
5602
5603 #[test]
5604 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
5605 let marked = interrupt_task("marked");
5606 let next = advance_interrupt(
5607 Interrupt::Idle,
5608 &["running".to_owned()],
5609 std::slice::from_ref(&marked),
5610 );
5611 assert_eq!(
5612 next,
5613 Interrupt::Parking {
5614 parked: vec!["running".to_owned()],
5615 interrupt_task: "marked".to_owned(),
5616 }
5617 );
5618 }
5619
5620 #[test]
5629 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
5630 let marked = interrupt_task("marked");
5631
5632 let two = advance_interrupt(
5633 Interrupt::Idle,
5634 &["a".to_owned(), "b".to_owned()],
5635 std::slice::from_ref(&marked),
5636 );
5637 assert_eq!(two, Interrupt::Idle);
5638
5639 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5640 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
5641 }
5642
5643 #[test]
5644 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
5645 let state = Interrupt::Parking {
5646 parked: vec!["running".to_owned()],
5647 interrupt_task: "marked".to_owned(),
5648 };
5649 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
5651 assert_eq!(still_going, state);
5652
5653 let stopped_but_not_yet_dispatched =
5657 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
5658 assert_eq!(stopped_but_not_yet_dispatched, state);
5659
5660 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
5662 assert_eq!(
5663 dispatched,
5664 Interrupt::Running {
5665 parked: vec!["running".to_owned()],
5666 interrupt_task: "marked".to_owned(),
5667 }
5668 );
5669 }
5670
5671 #[test]
5672 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
5673 let state = Interrupt::Running {
5674 parked: vec!["running".to_owned()],
5675 interrupt_task: "marked".to_owned(),
5676 };
5677 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
5678 assert_eq!(still_running, state);
5679
5680 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
5687 assert_eq!(
5688 ended,
5689 Interrupt::Resuming {
5690 parked: vec!["running".to_owned()]
5691 }
5692 );
5693 }
5694
5695 #[test]
5696 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
5697 let state = Interrupt::Resuming {
5698 parked: vec!["running".to_owned()],
5699 };
5700 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
5701 assert_eq!(still_waiting, state);
5702
5703 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
5704 assert_eq!(dispatched, Interrupt::Idle);
5705 }
5706
5707 #[test]
5713 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
5714 {
5715 let state = Interrupt::Parking {
5716 parked: vec!["running".to_owned()],
5717 interrupt_task: "marked".to_owned(),
5718 };
5719 let next = advance_interrupt(state, &[], &[]);
5722 assert_eq!(
5723 next,
5724 Interrupt::Resuming {
5725 parked: vec!["running".to_owned()]
5726 },
5727 "abandoning the interrupt must not abandon the resume it owes"
5728 );
5729 }
5730
5731 #[test]
5734 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
5735 let state = Interrupt::Resuming {
5736 parked: vec!["running".to_owned()],
5737 };
5738 let next = advance_interrupt(state, &[], &[]);
5739 assert_eq!(
5740 next,
5741 Interrupt::Idle,
5742 "nothing is left to wait for; the loop must not stay wedged"
5743 );
5744 }
5745
5746 #[test]
5747 fn disabled_by_config_the_sequence_can_never_leave_idle() {
5748 let marked = interrupt_task("marked");
5749 let next = advance_interrupt_tick(
5750 false,
5751 Interrupt::Idle,
5752 &["running".to_owned()],
5753 std::slice::from_ref(&marked),
5754 );
5755 assert_eq!(
5756 next,
5757 Interrupt::Idle,
5758 "an unmarked, unconfigured daemon must behave exactly as before"
5759 );
5760 }
5761
5762 #[test]
5763 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
5764 let state = Interrupt::Parking {
5765 parked: vec!["running".to_owned()],
5766 interrupt_task: "marked".to_owned(),
5767 };
5768 let candidates = vec![interrupt_task("marked"), task()];
5769 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5770 assert!(
5771 allowed.is_empty(),
5772 "nothing may dispatch - not even the interrupt task itself - \
5773 until the parked run has actually stopped"
5774 );
5775 }
5776
5777 #[test]
5791 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
5792 for state in [
5793 Interrupt::Parking {
5794 parked: vec!["running".to_owned()],
5795 interrupt_task: "marked".to_owned(),
5796 },
5797 Interrupt::Running {
5798 parked: vec!["running".to_owned()],
5799 interrupt_task: "marked".to_owned(),
5800 },
5801 Interrupt::Resuming {
5802 parked: vec!["running".to_owned()],
5803 },
5804 ] {
5805 let candidates = vec![
5806 interrupt_task("marked"),
5807 urgent_task("hot"),
5808 task_with_id("ordinary"),
5809 ];
5810 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5811 assert!(
5812 !allowed.iter().any(|t| t.id == "hot"),
5813 "an urgent candidate must wait out the same gate as anything \
5814 else while the run it would run alongside has not actually \
5815 left flight, for state {state:?}: {allowed:?}"
5816 );
5817 }
5818 }
5819
5820 #[test]
5826 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
5827 let state = Interrupt::Resuming {
5828 parked: vec!["hot".to_owned()],
5829 };
5830 let candidates = vec![urgent_task("hot"), task()];
5831 let allowed = interrupt_gate(&state, &[], candidates);
5832 assert_eq!(
5833 allowed.iter().filter(|t| t.id == "hot").count(),
5834 1,
5835 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
5836 );
5837 }
5838
5839 #[test]
5840 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
5841 let state = Interrupt::Parking {
5842 parked: vec!["running".to_owned()],
5843 interrupt_task: "marked".to_owned(),
5844 };
5845 let other = task();
5846 let candidates = vec![interrupt_task("marked"), other.clone()];
5847 let allowed = interrupt_gate(&state, &[], candidates);
5848 assert_eq!(allowed.len(), 1);
5849 assert_eq!(allowed[0].id, "marked");
5850 }
5851
5852 #[test]
5853 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
5854 let state = Interrupt::Running {
5855 parked: vec!["running".to_owned()],
5856 interrupt_task: "marked".to_owned(),
5857 };
5858 let candidates = vec![task(), task()];
5859 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
5860 assert!(allowed.is_empty());
5861 }
5862
5863 #[test]
5870 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
5871 let state = Interrupt::Resuming {
5872 parked: vec!["a".to_owned(), "c".to_owned()],
5873 };
5874 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
5875 let allowed = interrupt_gate(&state, &[], candidates);
5876 assert_eq!(
5877 allowed.len(),
5878 1,
5879 "at most one candidate may be offered while resuming: {allowed:?}"
5880 );
5881 assert_eq!(allowed[0].id, "a");
5882 }
5883
5884 #[test]
5885 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
5886 let state = Interrupt::Resuming {
5887 parked: vec!["a".to_owned()],
5888 };
5889 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
5890 assert!(allowed.is_empty());
5891 }
5892
5893 #[test]
5899 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
5900 let running = task(); let marked = interrupt_task("marked");
5902
5903 let mut state = Interrupt::Idle;
5904 let in_flight = vec![running.id.clone()];
5906 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
5907 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
5908 assert!(gated.is_empty(), "still waiting on `running` to park");
5909
5910 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
5912 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
5913 assert_eq!(
5914 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5915 vec!["marked"],
5916 "only the interrupt task may be offered to the dispatcher now"
5917 );
5918
5919 state = advance_interrupt_tick(
5921 true,
5922 state,
5923 &["marked".to_owned()],
5924 std::slice::from_ref(&running),
5925 );
5926 let gated = interrupt_gate(
5927 &state,
5928 &["marked".to_owned()],
5929 vec![marked.clone(), running.clone()],
5930 );
5931 assert!(
5932 gated.is_empty(),
5933 "the parked run must not be offered back while the interrupt \
5934 task is still running"
5935 );
5936
5937 let other = task_with_id("other");
5941 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
5942 assert_eq!(
5943 state,
5944 Interrupt::Resuming {
5945 parked: vec![running.id.clone()]
5946 }
5947 );
5948 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
5949 assert_eq!(
5950 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5951 vec![running.id.as_str()],
5952 "exactly the parked run resumes - not the unrelated task, even \
5953 though it was offered first"
5954 );
5955
5956 state = advance_interrupt_tick(
5960 true,
5961 state,
5962 std::slice::from_ref(&running.id),
5963 std::slice::from_ref(&other),
5964 );
5965 assert_eq!(state, Interrupt::Idle);
5966 let gated = interrupt_gate(
5967 &state,
5968 std::slice::from_ref(&running.id),
5969 vec![other.clone()],
5970 );
5971 assert_eq!(
5972 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5973 vec![other.id.as_str()],
5974 "ordinary dispatch is unrestricted again"
5975 );
5976 }
5977
5978 #[test]
5979 fn every_run_status_settles_the_task_it_came_from() {
5980 let table = [
5982 (RunStatus::Merged, TaskStatus::Done, 1),
5983 (RunStatus::Ready, TaskStatus::Done, 1),
5984 (RunStatus::Stalled, TaskStatus::Failed, 0),
5985 (RunStatus::Blocked, TaskStatus::Failed, 1),
5986 (RunStatus::Failed, TaskStatus::Failed, 1),
5987 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
5988 (RunStatus::Prep, TaskStatus::Failed, 1),
5989 (RunStatus::Implementing, TaskStatus::Failed, 1),
5990 (RunStatus::Judging, TaskStatus::Failed, 1),
5991 (RunStatus::Deliberating, TaskStatus::Failed, 1),
5992 (RunStatus::Voting, TaskStatus::Failed, 1),
5993 (RunStatus::Reviewing, TaskStatus::Failed, 1),
5994 (RunStatus::Gating, TaskStatus::Failed, 1),
5995 ];
5996 for (run, want, attempts) in table {
5997 let mut t = task();
5998 t.start("20260902-000000-aaaa".to_owned());
5999 settle(
6000 &mut t,
6001 Verdict {
6002 status: run,
6003 left_pr: false,
6004 parked: false,
6005 quota_hit: matches!(run, RunStatus::Stalled),
6006 no_viable_candidates: false,
6007 },
6008 "why",
6009 2,
6010 );
6011 assert_eq!(t.status, want, "task status after {}", label(run));
6012 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
6013 }
6014 }
6015
6016 #[test]
6017 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
6018 let mut stalled = task();
6019 stalled.start("20260902-000000-aaaa".to_owned());
6020 settle(
6021 &mut stalled,
6022 Verdict {
6023 status: RunStatus::Stalled,
6024 left_pr: false,
6025 parked: false,
6026 quota_hit: true,
6027 no_viable_candidates: false,
6028 },
6029 "quota",
6030 1,
6031 );
6032 assert_eq!(stalled.attempts, 0);
6033 assert!(
6034 stalled.status.runnable(),
6035 "a machine problem must leave the task in line"
6036 );
6037
6038 let mut blocked = task();
6039 blocked.start("20260902-000000-aaaa".to_owned());
6040 settle(
6041 &mut blocked,
6042 Verdict {
6043 status: RunStatus::Blocked,
6044 left_pr: false,
6045 parked: false,
6046 quota_hit: false,
6047 no_viable_candidates: false,
6048 },
6049 "findings open",
6050 1,
6051 );
6052 assert_eq!(blocked.attempts, 1);
6053 assert_eq!(
6054 blocked.status,
6055 TaskStatus::Held,
6056 "the last attempt hands the task to a human"
6057 );
6058 }
6059
6060 #[test]
6061 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
6062 let mut delivered = task();
6065 delivered.start("20260903-080619-01c2".to_owned());
6066 settle(
6067 &mut delivered,
6068 Verdict {
6069 status: RunStatus::Blocked,
6070 left_pr: true,
6071 parked: false,
6072 quota_hit: false,
6073 no_viable_candidates: false,
6074 },
6075 "no check status",
6076 4,
6077 );
6078 assert_eq!(
6079 delivered.status,
6080 TaskStatus::Held,
6081 "a pull request waiting on CI or a person is not a retryable failure"
6082 );
6083 assert!(
6084 !delivered.status.runnable(),
6085 "the loop must not pick this task up again"
6086 );
6087 assert_eq!(
6088 delivered.last_error.as_deref(),
6089 Some("no check status"),
6090 "the operator needs to be told what the gate was waiting for"
6091 );
6092
6093 let mut empty_handed = task();
6096 empty_handed.start("20260903-080619-01c2".to_owned());
6097 settle(
6098 &mut empty_handed,
6099 Verdict {
6100 status: RunStatus::Blocked,
6101 left_pr: false,
6102 parked: false,
6103 quota_hit: false,
6104 no_viable_candidates: false,
6105 },
6106 "findings open",
6107 4,
6108 );
6109 assert_eq!(empty_handed.status, TaskStatus::Failed);
6110 assert!(empty_handed.status.runnable());
6111 }
6112
6113 #[test]
6114 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
6115 let mut noop = task();
6121 noop.start("20260912-131304-391f".to_owned());
6122 settle(
6123 &mut noop,
6124 Verdict {
6125 status: RunStatus::VerifiedNoop,
6126 left_pr: false,
6127 parked: false,
6128 quota_hit: false,
6129 no_viable_candidates: true,
6130 },
6131 "candidate A: already fixed by b32cfc4, on main",
6132 4,
6133 );
6134 assert_eq!(
6135 noop.status,
6136 TaskStatus::Held,
6137 "an unverified claim is a request for a human, not a failure"
6138 );
6139 assert!(
6140 !noop.status.runnable(),
6141 "the loop must not requeue this on the same unverified claim"
6142 );
6143 assert_eq!(noop.attempts, 1);
6148 }
6149
6150 #[test]
6151 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
6152 let mut parked = task();
6157 parked.start("20260903-183634-2d98".to_owned());
6158 settle(
6159 &mut parked,
6160 Verdict {
6161 status: RunStatus::Implementing,
6162 left_pr: false,
6163 quota_hit: false,
6164 parked: true,
6165 no_viable_candidates: false,
6166 },
6167 "parked after `implementing`",
6168 2,
6169 );
6170 assert_eq!(parked.attempts, 0, "a park is refunded");
6171 assert!(
6172 parked.status.runnable(),
6173 "and the task stays in line so the next loop resumes its run"
6174 );
6175 assert_eq!(parked.status, TaskStatus::Parked);
6176 assert_eq!(
6177 parked.park_reason.as_deref(),
6178 Some("parked after `implementing`"),
6179 "the card says where it stopped"
6180 );
6181 assert_eq!(parked.last_error, None, "nothing failed");
6182
6183 let mut broken = task();
6187 broken.start("20260903-183634-2d98".to_owned());
6188 settle(
6189 &mut broken,
6190 Verdict {
6191 status: RunStatus::Implementing,
6192 left_pr: false,
6193 quota_hit: false,
6194 parked: false,
6195 no_viable_candidates: false,
6196 },
6197 "returned mid-flight",
6198 2,
6199 );
6200 assert_eq!(broken.attempts, 1);
6201 }
6202
6203 #[test]
6204 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
6205 let mut flaky = task();
6210 flaky.start("20260903-123023-e633".to_owned());
6211 settle(
6212 &mut flaky,
6213 Verdict {
6214 status: RunStatus::Stalled,
6215 left_pr: false,
6216 parked: false,
6217 quota_hit: false,
6218 no_viable_candidates: false,
6219 },
6220 "verdict rests on 1 of 3 judges",
6221 2,
6222 );
6223 assert_eq!(
6224 flaky.attempts, 1,
6225 "flakiness spends an attempt, so `max_attempts` still bounds it"
6226 );
6227 assert!(flaky.status.runnable(), "and it is still worth retrying");
6228
6229 let mut limited = task();
6231 limited.start("20260903-123023-e633".to_owned());
6232 settle(
6233 &mut limited,
6234 Verdict {
6235 status: RunStatus::Stalled,
6236 left_pr: false,
6237 parked: false,
6238 quota_hit: true,
6239 no_viable_candidates: false,
6240 },
6241 "judge-2, judge-3 out of quota",
6242 2,
6243 );
6244 assert_eq!(limited.attempts, 0, "a quota window is refunded");
6245 assert!(limited.status.runnable());
6246
6247 let mut worn = task();
6250 for _ in 0..2 {
6251 worn.release();
6252 }
6253 worn.start("20260903-123023-e633".to_owned());
6254 worn.attempts = 2;
6255 settle(
6256 &mut worn,
6257 Verdict {
6258 status: RunStatus::Stalled,
6259 left_pr: false,
6260 parked: false,
6261 quota_hit: false,
6262 no_viable_candidates: false,
6263 },
6264 "no quorum again",
6265 2,
6266 );
6267 assert_eq!(worn.status, TaskStatus::Held);
6268 assert!(!worn.status.runnable());
6269 }
6270
6271 #[test]
6272 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
6273 let mut wiped_out = task();
6280 wiped_out.start("20260907-025000-a1b2".to_owned());
6281 settle(
6282 &mut wiped_out,
6283 Verdict {
6284 status: RunStatus::Failed,
6285 left_pr: false,
6286 parked: false,
6287 quota_hit: true,
6288 no_viable_candidates: true,
6289 },
6290 "no candidate produced a change; nothing to judge",
6291 2,
6292 );
6293 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
6294 assert!(
6295 wiped_out.status.runnable(),
6296 "a machine problem must leave the task in line"
6297 );
6298
6299 let mut partial_progress = task();
6305 partial_progress.start("20260907-025500-c3d4".to_owned());
6306 settle(
6307 &mut partial_progress,
6308 Verdict {
6309 status: RunStatus::Failed,
6310 left_pr: false,
6311 parked: false,
6312 quota_hit: true,
6313 no_viable_candidates: false,
6314 },
6315 "gate failed on the winning candidate",
6316 2,
6317 );
6318 assert_eq!(
6319 partial_progress.attempts, 1,
6320 "a candidate that actually produced a change spends the attempt \
6321 even though some other seat hit its quota"
6322 );
6323 assert!(partial_progress.status.runnable());
6324 }
6325
6326 #[test]
6327 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
6328 let mut t = task();
6334 t.start("20260907-025000-a1b2".to_owned());
6335 let mut state = run_state(RunStatus::Failed);
6336 state.quota.push(QuotaLoss {
6337 seat: "cand-a".to_owned(),
6338 node: "implement".to_owned(),
6339 at: Timestamp::now(),
6340 reset: None,
6341 });
6342 assert!(
6343 state.viable().is_empty(),
6344 "no candidate was added, so nothing is viable"
6345 );
6346 reclaim(&mut t, Some(state), 2, "en");
6347 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
6348 assert!(t.status.runnable());
6349 }
6350
6351 #[test]
6352 fn a_held_task_is_never_offered_to_the_loop() {
6353 let dir = tempfile::tempdir().unwrap();
6354 let queue = Queue::at(dir.path().to_path_buf());
6355 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
6356 let mut t = task();
6357 t.id = format!("2026090{n}-000000-000{n}");
6358 t.priority = priority;
6359 queue.put(&mut t).unwrap();
6360 }
6361 let mut held = task();
6362 held.id = "20260909-000000-9999".to_owned();
6363 held.priority = 99;
6364 held.hold_machine(None);
6365 queue.put(&mut held).unwrap();
6366
6367 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
6368 assert_eq!(order.len(), 3);
6369 assert!(!order.contains(&held.id));
6370 assert_eq!(
6371 order.first().cloned(),
6372 queue.next_runnable().map(|t| t.id),
6373 "the loop's first candidate is exactly what the queue offers"
6374 );
6375 assert_eq!(
6376 order,
6377 vec![
6378 "20260902-000000-0002".to_owned(),
6379 "20260903-000000-0003".to_owned(),
6380 "20260901-000000-0001".to_owned(),
6381 ],
6382 "priority first, then oldest, so nothing starves"
6383 );
6384 }
6385
6386 #[test]
6387 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
6388 let dir = tempfile::tempdir().unwrap();
6389 let queue = Queue::at(dir.path().to_path_buf());
6390 let mut old = task();
6391 old.id = "20260101-000000-old0".to_owned();
6392 queue.put(&mut old).unwrap();
6393 let mut fresh = task();
6394 fresh.id = "20260101-000000-new0".to_owned();
6395 queue.put(&mut fresh).unwrap();
6396
6397 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
6401 std::thread::sleep(Duration::from_millis(60));
6402 let live = queue.claim(&fresh.id).unwrap();
6403
6404 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6405 assert_eq!(swept, vec![old.id.clone()]);
6406 assert!(
6407 queue.claim(&old.id).is_ok(),
6408 "an unparseable lock older than the threshold is swept"
6409 );
6410 assert!(
6411 queue.claim(&fresh.id).is_err(),
6412 "a live pid protects its lock regardless of age"
6413 );
6414 drop(live);
6415 }
6416
6417 #[test]
6418 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
6419 let dir = tempfile::tempdir().unwrap();
6429 let queue = Queue::at(dir.path().to_path_buf());
6430 let mut t = task();
6431 t.id = "20260101-000000-live".to_owned();
6432 queue.put(&mut t).unwrap();
6433
6434 let claim = queue.claim(&t.id).unwrap();
6435 std::thread::sleep(Duration::from_millis(60));
6436
6437 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6438 assert!(
6439 swept.is_empty(),
6440 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
6441 );
6442 assert!(
6443 queue.claim(&t.id).is_err(),
6444 "the lock still protects its task"
6445 );
6446 drop(claim);
6447 }
6448
6449 fn injected_dead_pid() -> u32 {
6452 std::process::id().checked_add(1).unwrap_or(1)
6453 }
6454
6455 #[test]
6456 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
6457 let dir = tempfile::tempdir().unwrap();
6458 let queue = Queue::at(dir.path().to_path_buf());
6459 let mut t = task();
6460 t.id = "20260101-000000-dead".to_owned();
6461 queue.put(&mut t).unwrap();
6462 let dead_pid = injected_dead_pid();
6463
6464 std::fs::write(
6469 dir.path().join(format!("{}.lock", t.id)),
6470 dead_pid.to_string(),
6471 )
6472 .unwrap();
6473
6474 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6475 pid != dead_pid
6476 });
6477 assert_eq!(
6478 swept,
6479 vec![t.id.clone()],
6480 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
6481 );
6482 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
6483 }
6484
6485 #[test]
6486 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
6487 let dir = tempfile::tempdir().unwrap();
6488 let queue = Queue::at(dir.path().to_path_buf());
6489 let mut t = task();
6490 t.id = "20260101-000000-late".to_owned();
6491 queue.put(&mut t).unwrap();
6492 let dead_pid = injected_dead_pid();
6493
6494 assert!(
6497 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
6498 "nothing has claimed the task yet"
6499 );
6500
6501 std::fs::write(
6504 dir.path().join(format!("{}.lock", t.id)),
6505 dead_pid.to_string(),
6506 )
6507 .unwrap();
6508
6509 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6513 pid != dead_pid
6514 });
6515 assert_eq!(swept, vec![t.id.clone()]);
6516 }
6517
6518 #[test]
6519 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
6520 crate::run::pin_test_home();
6525 let dir = tempfile::tempdir().unwrap();
6526 let queue = Queue::at(dir.path().to_path_buf());
6527 let mut t = task();
6528 t.id = "20260101-000000-crsh".to_owned();
6529 t.status = TaskStatus::Running;
6530 t.attempts = 1;
6531 t.runs.push("20260904-000000-4043".to_owned());
6535 queue.put(&mut t).unwrap();
6536 let dead_pid = injected_dead_pid();
6537
6538 std::fs::write(
6541 dir.path().join(format!("{}.lock", t.id)),
6542 dead_pid.to_string(),
6543 )
6544 .unwrap();
6545
6546 assert!(reclaim_orphaned_running(&queue, 2).is_empty());
6552 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
6553
6554 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6555 pid != dead_pid
6556 });
6557 assert_eq!(swept, vec![t.id.clone()]);
6558
6559 let reclaimed = reclaim_orphaned_running(&queue, 2);
6560 assert_eq!(reclaimed, vec![t.id.clone()]);
6561 let after = queue.get(&t.id).unwrap();
6562 assert_eq!(
6563 after.status,
6564 TaskStatus::Held,
6565 "no run.json to recover from, so a human is asked"
6566 );
6567 assert_eq!(
6568 after.runs,
6569 vec!["20260904-000000-4043".to_owned()],
6570 "the crashed run's id is kept as evidence, not discarded"
6571 );
6572 }
6573
6574 #[test]
6575 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
6576 let dir = tempfile::tempdir().unwrap();
6577 let queue = Queue::at(dir.path().to_path_buf());
6578 let mut t = task();
6579 t.id = "20260101-000000-unknown".to_owned();
6580 queue.put(&mut t).unwrap();
6581 let dead_pid = injected_dead_pid();
6582 std::fs::write(
6583 dir.path().join(format!("{}.lock", t.id)),
6584 dead_pid.to_string(),
6585 )
6586 .unwrap();
6587
6588 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
6589 assert!(swept.is_empty(), "an unknown pid must keep its lock");
6590 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
6591 }
6592
6593 fn run_state_in(status: RunStatus, language: &str) -> RunState {
6594 let mut s = run_state(status);
6595 s.config.graph.language = language.to_owned();
6596 s
6597 }
6598
6599 fn unstarted_verdict(status: RunStatus) -> Verdict {
6600 Verdict {
6601 status,
6602 left_pr: false,
6603 quota_hit: false,
6604 parked: false,
6605 no_viable_candidates: false,
6606 }
6607 }
6608
6609 #[test]
6610 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
6611 let mut t = task();
6612 t.attempts = 1;
6613 settle_in(
6614 &mut t,
6615 unstarted_verdict(RunStatus::AlreadyInBase),
6616 "already in main",
6617 1,
6618 phrases("en"),
6619 );
6620 assert_eq!(t.status, TaskStatus::Done);
6621 assert_eq!(t.attempts, 0);
6622 }
6623
6624 #[test]
6625 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
6626 let reason = |language: &str| {
6627 let mut t = task();
6628 settle_in(
6629 &mut t,
6630 unstarted_verdict(RunStatus::Judging),
6631 "boom",
6632 1,
6633 phrases(language),
6634 );
6635 t.last_error.or(t.hold_reason).unwrap_or_default()
6636 };
6637 assert!(
6638 reason("en").starts_with("the graph stopped at `"),
6639 "{}",
6640 reason("en")
6641 );
6642 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
6643 assert!(reason("日本語").contains("boom"));
6644 assert_eq!(reason("fr"), reason("en"));
6645 }
6646
6647 #[test]
6648 fn describe_follows_the_run_language_and_keeps_the_run_id() {
6649 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
6650 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
6651 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
6652 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
6653 assert!(ja.contains("[run "), "{ja}");
6654 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
6655 assert!(ended.starts_with("run 終了: "), "{ended}");
6656 let mut de = run_state_in(RunStatus::Failed, "de");
6657 let mut en = run_state_in(RunStatus::Failed, "en");
6658 de.id = "same".to_owned();
6659 en.id = "same".to_owned();
6660 assert_eq!(describe(&de), describe(&en));
6661 }
6662
6663 #[test]
6664 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
6665 use crate::handover::Refused;
6666 let cases = [
6667 Refused::Foreign {
6668 branch: "b".into(),
6669 path: "/w/x".into(),
6670 why: "made by hand".into(),
6671 },
6672 Refused::Unsafe {
6673 branch: "b".into(),
6674 path: "/w/x".into(),
6675 why: "its worktree has uncommitted changes (a.rs)".into(),
6676 },
6677 Refused::ReleaseFailed {
6678 branch: "b".into(),
6679 path: "/w/x".into(),
6680 run: "ab12".into(),
6681 },
6682 ];
6683 for r in &cases {
6684 let en = (phrases("en").handover_refused)(r);
6685 assert_eq!(en, r.to_string());
6686 let ja = (phrases("ja").handover_refused)(r);
6687 assert!(
6688 !ja.contains("is checked out") && !ja.contains("try again"),
6689 "{ja}"
6690 );
6691 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
6692 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
6693 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
6694 }
6695 }
6696 assert!(phrases("en").handover_hint.contains("release the task"));
6697 assert!(phrases("ja").handover_hint.contains("解放"));
6698 }
6699
6700 #[test]
6701 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
6702 let build = |lang: &str| {
6703 let mut s = run_state_in(RunStatus::Blocked, lang);
6704 s.id = "same".to_owned();
6705 s.config.graph.review_rounds = 3;
6706 let mut r = review_round(3);
6707 r.expected = 3;
6708 r.answered = 1;
6709 s.reviews.push(r);
6710 s.event(
6711 "review",
6712 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
6713 );
6714 s
6715 };
6716 let en = describe(&build("en"));
6717 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
6718 let ja = describe(&build("ja"));
6719 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
6720 assert!(!ja.contains("never answered"), "{ja}");
6721 assert!(ja.contains("[run "), "{ja}");
6722
6723 let mut failed = build("ja");
6726 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
6727 command: "cargo test".to_owned(),
6728 code: Some(1),
6729 output_tail: String::new(),
6730 duration_ms: 0,
6731 resource_blocked: false,
6732 });
6733 failed.event("review", "stopped; e2e failed: cargo test");
6734 let ja = describe(&failed);
6735 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
6736 assert!(!ja.contains("席のレビュアー"), "{ja}");
6737 }
6738
6739 #[test]
6740 fn refusals_and_recovery_prose_follow_the_language() {
6741 let t = held_task_with("r1");
6742 let q = action_question(
6743 "r1",
6744 ask::ChoiceAction::Resume {
6745 run: "r1".to_owned(),
6746 },
6747 );
6748 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
6749 ActionDecision::Refuse(s) => s,
6750 other => panic!("{other:?}"),
6751 };
6752 assert!(refuse(phrases("en")).contains("could not be read: gone"));
6753 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
6754 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
6755
6756 let mut held = task();
6757 reclaim(&mut held, None, 2, "ja");
6758 assert!(held.hold_reason.unwrap().contains("保留にしました"));
6759 let mut held = task();
6760 reclaim(&mut held, None, 2, "xx");
6761 assert!(held.hold_reason.unwrap().contains("held for a human"));
6762 }
6763
6764 fn run_state(status: RunStatus) -> RunState {
6765 let mut state = RunState::new(
6766 PathBuf::from("/repo"),
6767 "main".to_owned(),
6768 "abc1234def".to_owned(),
6769 "add retries".to_owned(),
6770 Config::default(),
6771 );
6772 state.status = status;
6773 state
6774 }
6775
6776 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
6777 Candidate {
6778 index: 0,
6779 label,
6780 agent: "claude".to_owned(),
6781 branch: format!("magi/x/{label}"),
6782 worktree: PathBuf::from("/repo"),
6783 summary: summary.to_owned(),
6784 stat: String::new(),
6785 files: 0,
6786 commits: usize::from(!empty),
6787 empty,
6788 failed: failed.map(str::to_owned),
6789 verified_noop: None,
6790 duration_ms: 0,
6791 folded: false,
6792 }
6793 }
6794
6795 #[test]
6796 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
6797 let mut state = run_state(RunStatus::Blocked);
6798 state.gate = vec![
6799 CommandOutcome {
6800 command: "cargo make check".to_owned(),
6801 code: Some(0),
6802 output_tail: "ok".to_owned(),
6803 duration_ms: 0,
6804 resource_blocked: false,
6805 },
6806 CommandOutcome {
6807 command: "cargo test".to_owned(),
6808 code: Some(101),
6809 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
6810 duration_ms: 0,
6811 resource_blocked: false,
6812 },
6813 ];
6814 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
6815 assert!(d.contains("cargo test"), "{d}");
6816 assert!(
6817 !d.contains("cargo make check"),
6818 "a passing check is not a diagnostic: {d}"
6819 );
6820 assert!(d.contains("assertion failed"), "{d}");
6821 }
6822
6823 #[test]
6824 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
6825 let mut state = run_state(RunStatus::Blocked);
6826 state.event(
6827 "land",
6828 "stopped: the fixer produced no commit while 2 check(s) were failing \
6829 (build, lint); stopping instead of looping on an unchanged tree",
6830 );
6831 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
6832 assert!(d.contains("build"), "{d}");
6833 assert!(d.contains("lint"), "{d}");
6834 assert!(d.contains("fixer produced no commit"), "{d}");
6835 }
6836
6837 #[test]
6838 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
6839 let state = run_state(RunStatus::VerifiedNoop);
6844 let d = describe(&state);
6845 assert!(
6846 d.contains("agent-verified no-op"),
6847 "expected the display label, not the wire spelling: {d}"
6848 );
6849 assert!(!d.contains("verified_noop"), "{d}");
6850 }
6851
6852 #[test]
6853 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
6854 let mut state = run_state(RunStatus::Failed);
6860 state.candidates = vec![candidate(
6861 'A',
6862 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
6863 true,
6864 None,
6865 )];
6866 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
6867 assert!(d.contains("candidate A"), "{d}");
6868 assert!(d.contains("tagged v1.2.3"), "{d}");
6869 }
6870
6871 #[test]
6872 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
6873 let mut state = run_state(RunStatus::Failed);
6874 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
6875 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
6876 assert!(d.contains("candidate A"), "{d}");
6877 assert!(d.contains("agent timed out"), "{d}");
6878 }
6879
6880 #[test]
6881 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
6882 let mut state = run_state(RunStatus::Failed);
6885 state.candidates = vec![candidate('A', "did the work", false, None)];
6886 assert!(diagnostic(&state).is_none());
6887 }
6888
6889 #[test]
6890 fn diagnostic_is_bounded_however_much_a_run_printed() {
6891 let mut state = run_state(RunStatus::Blocked);
6892 state.gate = vec![
6893 CommandOutcome {
6894 command: "cargo test".to_owned(),
6895 code: Some(101),
6896 output_tail: "x".repeat(50_000),
6897 duration_ms: 0,
6898 resource_blocked: false,
6899 },
6900 CommandOutcome {
6901 command: "cargo clippy".to_owned(),
6902 code: Some(1),
6903 output_tail: "y".repeat(50_000),
6904 duration_ms: 0,
6905 resource_blocked: false,
6906 },
6907 ];
6908 state.candidates = vec![
6909 candidate('A', &"z".repeat(50_000), true, None),
6910 candidate('B', &"w".repeat(50_000), true, None),
6911 ];
6912 let d = diagnostic(&state).expect("plenty here to diagnose");
6913 assert!(
6914 d.len() <= DIAGNOSTIC_MAX,
6915 "diagnostic grew to {} bytes, unbounded",
6916 d.len()
6917 );
6918 }
6919
6920 #[test]
6921 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
6922 let mut state = run_state(RunStatus::Blocked);
6923 state.gate = vec![CommandOutcome {
6924 command: "cargo test".to_owned(),
6925 code: Some(101),
6926 output_tail: "assertion failed".to_owned(),
6927 duration_ms: 0,
6928 resource_blocked: false,
6929 }];
6930 let verdict = Verdict {
6931 status: RunStatus::Blocked,
6932 left_pr: false,
6933 quota_hit: false,
6934 parked: false,
6935 no_viable_candidates: false,
6936 };
6937
6938 let mut t = task();
6941 t.start("run-1".to_owned());
6942 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
6943 assert_eq!(t.status, TaskStatus::Failed);
6944 assert!(t.diagnostic.is_none());
6945
6946 t.start("run-2".to_owned());
6949 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
6950 assert_eq!(t.status, TaskStatus::Held);
6951 let d = t.diagnostic.expect("a held task must carry its diagnostic");
6952 assert!(d.contains("cargo test"), "{d}");
6953 }
6954
6955 #[test]
6956 fn a_held_task_names_the_open_question_it_is_waiting_on() {
6957 crate::run::pin_test_home();
6962 let home = crate::run::home();
6963 let state = run_state(RunStatus::VerifiedNoop);
6964 let mut q = ask::Question::new(
6965 state.id.clone(),
6966 "implement".to_owned(),
6967 "impl-A".to_owned(),
6968 "is this really a no-op?".to_owned(),
6969 String::new(),
6970 Vec::new(),
6971 );
6972 Questions::at(home.join("questions")).put(&mut q).unwrap();
6973
6974 let verdict = Verdict {
6975 status: RunStatus::VerifiedNoop,
6976 left_pr: false,
6977 quota_hit: false,
6978 parked: false,
6979 no_viable_candidates: false,
6980 };
6981 let mut t = task();
6982 t.start(state.id.clone());
6983 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
6984
6985 assert_eq!(t.status, TaskStatus::Held);
6986 let reason = t.hold_reason.expect("a held task must record why");
6987 assert!(
6988 reason.starts_with("run ended agent-verified no-op"),
6989 "the original settle reason must survive unchanged: {reason}"
6990 );
6991 assert!(
6992 reason.contains(q.short()),
6993 "the open question's id must be named so the notice is actionable: {reason}"
6994 );
6995 }
6996
6997 #[test]
6998 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
6999 crate::run::pin_test_home();
7000 let state = run_state(RunStatus::VerifiedNoop);
7001
7002 let verdict = Verdict {
7003 status: RunStatus::VerifiedNoop,
7004 left_pr: false,
7005 quota_hit: false,
7006 parked: false,
7007 no_viable_candidates: false,
7008 };
7009 let mut t = task();
7010 t.start(state.id.clone());
7011 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
7012
7013 assert_eq!(t.status, TaskStatus::Held);
7014 assert_eq!(
7015 t.hold_reason.as_deref(),
7016 Some("run ended agent-verified no-op"),
7017 "nothing to append when the question was already answered or never asked"
7018 );
7019 }
7020
7021 #[test]
7022 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
7023 crate::run::pin_test_home();
7024 let mut first = run_state(RunStatus::Blocked);
7025 first.id = "20260101-000000-sup1".to_owned();
7026 first.save().unwrap();
7027 let mut second = run_state(RunStatus::Merged);
7028 second.id = "20260101-000000-sup2".to_owned();
7029 second.save().unwrap();
7030
7031 let mut t = task();
7032 t.runs = vec![first.id.clone(), second.id.clone()];
7033 t.status = TaskStatus::Done;
7034
7035 supersede_prior_runs(&t, &crate::run::home());
7036
7037 assert_eq!(
7038 RunState::load(&first.id).unwrap().status,
7039 RunStatus::Superseded,
7040 "the first attempt's Blocked no longer needs anyone's attention"
7041 );
7042 assert_eq!(
7043 RunState::load(&second.id).unwrap().status,
7044 RunStatus::Merged,
7045 "the run that actually succeeded is left exactly as it was"
7046 );
7047 }
7048
7049 #[test]
7050 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
7051 crate::run::pin_test_home();
7057 let mut first = run_state(RunStatus::Blocked);
7058 first.id = "20260101-000000-sup9".to_owned();
7059 first.driver_pid = Some(std::process::id());
7062 first.driver_started_at = Some(
7063 crate::proc::process_started_at(std::process::id())
7064 .expect("this test process's own start time must be queryable"),
7065 );
7066 first.save().unwrap();
7067 let mut second = run_state(RunStatus::Merged);
7068 second.id = "20260101-000000-supa".to_owned();
7069 second.save().unwrap();
7070
7071 let mut t = task();
7072 t.runs = vec![first.id.clone(), second.id.clone()];
7073 t.status = TaskStatus::Done;
7074
7075 supersede_prior_runs(&t, &crate::run::home());
7076
7077 assert_eq!(
7078 RunState::load(&first.id).unwrap().status,
7079 RunStatus::Blocked,
7080 "a live driver_pid means something is still actually working this run, \
7081 even though no daemon claims it - rewriting under it would just be \
7082 undone the next time that process saves"
7083 );
7084 }
7085
7086 #[test]
7087 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
7088 let dir = tempfile::tempdir().unwrap();
7094 let home = dir.path().to_path_buf();
7095 let queue = Queue::at(dir.path().join("queue"));
7096
7097 let mut first = run_state(RunStatus::Blocked);
7098 first.id = "20260101-000000-supd".to_owned();
7099 first.driver_pid = Some(std::process::id());
7100 first.driver_started_at = Some(
7101 crate::proc::process_started_at(std::process::id())
7102 .expect("this test process's own start time must be queryable"),
7103 );
7104 first.save_under(&home).unwrap();
7105 let mut second = run_state(RunStatus::Merged);
7106 second.id = "20260101-000000-supe".to_owned();
7107 second.save_under(&home).unwrap();
7108
7109 let mut t = task();
7110 t.runs = vec![first.id.clone(), second.id.clone()];
7111 t.status = TaskStatus::Done;
7112 queue.put(&mut t).unwrap();
7113
7114 resweep_superseded_attempts(&queue, &home);
7115 assert_eq!(
7116 RunState::load_under(&first.id, &home).unwrap().status,
7117 RunStatus::Blocked,
7118 "still live on the first pass, so still untouched"
7119 );
7120
7121 let mut stale = RunState::load_under(&first.id, &home).unwrap();
7127 stale.driver_started_at = Some("1".to_owned());
7128 stale.save_under(&home).unwrap();
7129
7130 resweep_superseded_attempts(&queue, &home);
7131 assert_eq!(
7132 RunState::load_under(&first.id, &home).unwrap().status,
7133 RunStatus::Superseded,
7134 "the second pass catches up what the first one correctly skipped"
7135 );
7136 }
7137
7138 #[test]
7139 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
7140 crate::run::pin_test_home();
7141 let mut first = run_state(RunStatus::Blocked);
7142 first.id = "20260101-000000-sup3".to_owned();
7143 first.save().unwrap();
7144 let mut second = run_state(RunStatus::Blocked);
7145 second.id = "20260101-000000-sup4".to_owned();
7146 second.save().unwrap();
7147
7148 let mut t = task();
7149 t.runs = vec![first.id.clone(), second.id.clone()];
7150 t.status = TaskStatus::Failed;
7154
7155 supersede_prior_runs(&t, &crate::run::home());
7156
7157 assert_eq!(
7158 RunState::load(&first.id).unwrap().status,
7159 RunStatus::Blocked
7160 );
7161 assert_eq!(
7162 RunState::load(&second.id).unwrap().status,
7163 RunStatus::Blocked
7164 );
7165 }
7166
7167 #[test]
7168 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
7169 crate::run::pin_test_home();
7173 let mut first = run_state(RunStatus::Blocked);
7174 first.id = "20260101-000000-sup5".to_owned();
7175 first.save().unwrap();
7176
7177 let mut t = task();
7178 t.runs = vec![first.id.clone()];
7179 t.status = TaskStatus::Done;
7180
7181 supersede_prior_runs(&t, &crate::run::home());
7182
7183 assert_eq!(
7184 RunState::load(&first.id).unwrap().status,
7185 RunStatus::Blocked,
7186 "a single-attempt task has no earlier run to supersede"
7187 );
7188 }
7189
7190 #[test]
7191 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
7192 crate::run::pin_test_home();
7199 let mut first = run_state(RunStatus::Blocked);
7200 first.id = "20260101-000000-supb".to_owned();
7201 first.save().unwrap();
7202 let mut second = run_state(RunStatus::Failed);
7203 second.id = "20260101-000000-supc".to_owned();
7204 second.save().unwrap();
7205
7206 let mut t = task();
7207 t.runs = vec![first.id.clone(), second.id.clone()];
7208 t.status = TaskStatus::Done;
7209
7210 supersede_prior_runs(&t, &crate::run::home());
7211
7212 assert_eq!(
7213 RunState::load(&first.id).unwrap().status,
7214 RunStatus::Blocked,
7215 "the task's last attempt never landed, so there is nothing here \
7216 actually superseding it"
7217 );
7218 }
7219
7220 #[test]
7221 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
7222 crate::run::pin_test_home();
7226 let mut failed = run_state(RunStatus::Failed);
7227 failed.id = "20260101-000000-sup6".to_owned();
7228 failed.save().unwrap();
7229 let mut noop = run_state(RunStatus::VerifiedNoop);
7230 noop.id = "20260101-000000-sup7".to_owned();
7231 noop.save().unwrap();
7232 let mut winner = run_state(RunStatus::Ready);
7233 winner.id = "20260101-000000-sup8".to_owned();
7234 winner.save().unwrap();
7235
7236 let mut t = task();
7237 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
7238 t.status = TaskStatus::Done;
7239
7240 supersede_prior_runs(&t, &crate::run::home());
7241
7242 assert_eq!(
7243 RunState::load(&failed.id).unwrap().status,
7244 RunStatus::Failed
7245 );
7246 assert_eq!(
7247 RunState::load(&noop.id).unwrap().status,
7248 RunStatus::VerifiedNoop
7249 );
7250 }
7251
7252 fn approval_question(run: &str) -> ask::Question {
7253 ask::Question::new(
7254 run.to_owned(),
7255 land::APPROVAL_NODE.to_owned(),
7256 "land".to_owned(),
7257 "merge?".to_owned(),
7258 String::new(),
7259 vec!["merge".to_owned(), "hold".to_owned()],
7260 )
7261 }
7262
7263 #[test]
7264 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
7265 crate::run::pin_test_home();
7266 let mut state = run_state(RunStatus::Landing);
7267 state.id = "20260101-000000-fre1".to_owned();
7268 state.parked = true;
7269 state.save().unwrap();
7270 ask::Questions::open()
7271 .put(&mut approval_question(&state.id))
7272 .unwrap();
7273
7274 let mut t = task();
7275 t.runs.push(state.id.clone());
7276 assert_eq!(
7277 land_resume_state(&t),
7278 LandResume::StillWaiting,
7279 "nobody has answered and the timeout has not passed"
7280 );
7281 }
7282
7283 #[test]
7284 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
7285 crate::run::pin_test_home();
7286 let mut state = run_state(RunStatus::Gating);
7287 state.id = "20260101-000000-gt01".to_owned();
7288 state.parked = true;
7289 state.config.graph.answer_timeout = 60;
7290 state.github_text = Some(crate::run::GithubTextAsk {
7291 rewrite: None,
7292 fingerprint: "f".into(),
7293 title: true,
7294 body: false,
7295 categories: vec!["title-language".into()],
7296 question: None,
7297 asks: 1,
7298 resolved: false,
7299 chosen_title: None,
7300 choice: None,
7301 });
7302 state.save().unwrap();
7303 let store = ask::Questions::open();
7304 let mut q = ask::Question::new(
7305 state.id.clone(),
7306 crate::github_text::ASK_NODE.to_owned(),
7307 crate::github_text::ASK_SEAT.to_owned(),
7308 "s".into(),
7309 String::new(),
7310 vec!["use fallback".into()],
7311 );
7312 store.put(&mut q).unwrap();
7313 let mut t = task();
7314 t.runs.push(state.id.clone());
7315 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
7316 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7317 store.put(&mut q).unwrap();
7318 assert_eq!(land_resume_state(&t), LandResume::Ready);
7319 assert!(!store.get(&q.id).unwrap().status.open());
7320 state.github_text.as_mut().unwrap().resolved = true;
7322 state.save().unwrap();
7323 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
7324 }
7325
7326 #[test]
7327 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
7328 crate::run::pin_test_home();
7333 let mut state = run_state(RunStatus::Landing);
7334 state.id = "20260101-000000-exp1".to_owned();
7335 state.parked = true;
7336 state.config.graph.answer_timeout = 60;
7337 state.save().unwrap();
7338
7339 let store = ask::Questions::open();
7340 let mut q = approval_question(&state.id);
7341 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7342 store.put(&mut q).unwrap();
7343
7344 let mut t = task();
7345 t.runs.push(state.id.clone());
7346 assert_eq!(
7347 land_resume_state(&t),
7348 LandResume::Ready,
7349 "an expired question must not be waited on forever"
7350 );
7351
7352 let after = store.get(&q.id).unwrap();
7353 assert!(
7354 !after.status.open(),
7355 "the question is abandoned, not silently ignored"
7356 );
7357 assert!(
7358 after.resolution().is_none(),
7359 "an abandoned question is not read as a decision"
7360 );
7361 }
7362
7363 #[test]
7364 fn reclaim_settles_a_running_task_against_its_last_run() {
7365 let mut t = task();
7366 t.start("20260904-000000-4043".to_owned());
7367 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
7368 assert_eq!(
7369 t.status,
7370 TaskStatus::Done,
7371 "a run that actually finished must not stay `running` forever"
7372 );
7373 }
7374
7375 #[test]
7376 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
7377 let mut t = task();
7378 t.start("20260904-000000-4043".to_owned());
7379 t.last_error = Some("earlier trouble".to_owned());
7380 let mut state = run_state(RunStatus::Judging);
7381 state.parked = true;
7382 reclaim(&mut t, Some(state), 1, "en");
7383 assert_eq!(t.status, TaskStatus::Parked);
7384 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
7385 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7386 assert!(t.park_reason.is_some());
7387 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
7388 assert!(!t.fresh_start);
7389 assert!(t.status.runnable());
7390 }
7391
7392 #[test]
7393 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
7394 let mut t = task();
7395 t.start("20260904-000000-4043".to_owned());
7396 t.last_error = Some("earlier trouble".to_owned());
7397 settle(
7398 &mut t,
7399 Verdict {
7400 status: RunStatus::Judging,
7401 left_pr: false,
7402 quota_hit: false,
7403 parked: true,
7404 no_viable_candidates: false,
7405 },
7406 "parked after `judging`",
7407 1,
7408 );
7409 assert_eq!(t.status, TaskStatus::Parked);
7410 assert_eq!(t.attempts, 0);
7411 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7412 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
7413 assert_eq!(t.schema, crate::queue::SCHEMA);
7414 assert!(
7415 crate::notices::task_held(&t).is_none(),
7416 "a park is not news"
7417 );
7418 t.start("20260904-000000-4043".to_owned());
7419 assert_eq!(t.park_reason, None);
7420 }
7421
7422 #[test]
7423 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
7424 let mut t = task();
7428 t.start("20260904-000000-4043".to_owned());
7429 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
7430 assert_eq!(t.status, TaskStatus::Failed);
7431 assert!(t.status.runnable());
7432 }
7433
7434 #[test]
7435 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
7436 let mut t = task();
7437 t.start("20260904-000000-4043".to_owned());
7438 assert_eq!(t.attempts, 1);
7439 let mut run = run_state(RunStatus::Implementing);
7440 run.id = "20260904-000000-4043".to_owned();
7441 run.begin_pass();
7442 let cause = crate::run::ParkCause::Upgrade {
7443 from: "0.121.0".to_owned(),
7444 to: Some("0.121.1".to_owned()),
7445 };
7446 run.end_pass_parked(cause.clone());
7447 run.parked = true;
7448 let verdict = Verdict {
7449 status: run.status,
7450 left_pr: false,
7451 quota_hit: false,
7452 parked: true,
7453 no_viable_candidates: false,
7454 };
7455 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
7456 assert_eq!(t.attempts, 0);
7457 assert_eq!(t.passes.len(), 1);
7458 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7459 assert_eq!(t.passes[0].parked, Some(cause));
7460 assert_eq!(t.passes[0].pass_index, 0);
7461
7462 t.start("20260904-000000-4043".to_owned());
7465 run.begin_pass();
7466 run.status = RunStatus::Failed;
7467 let verdict = Verdict {
7468 status: RunStatus::Failed,
7469 left_pr: false,
7470 quota_hit: false,
7471 parked: false,
7472 no_viable_candidates: false,
7473 };
7474 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
7475 assert_eq!(t.passes.len(), 2);
7476 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7477 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
7478 assert_eq!(t.passes[1].pass_index, 1);
7479 assert_eq!(t.attempts, 1);
7480 }
7481
7482 #[test]
7483 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
7484 let mut t = task();
7485 t.start("20260904-000000-4043".to_owned());
7486 reclaim(&mut t, None, 2, "en");
7487 assert_eq!(t.status, TaskStatus::Held);
7488 assert!(
7489 t.last_error
7490 .as_deref()
7491 .is_some_and(|e| e.contains("running")),
7492 "the operator needs to know why this task was held"
7493 );
7494 }
7495
7496 #[test]
7497 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
7498 let dir = tempfile::tempdir().unwrap();
7499 let queue = Queue::at(dir.path().to_path_buf());
7500
7501 let mut orphaned = task();
7503 orphaned.id = "20260904-000000-orph".to_owned();
7504 orphaned.status = TaskStatus::Running;
7505 orphaned.attempts = 1;
7506 queue.put(&mut orphaned).unwrap();
7507
7508 let mut alive = task();
7509 alive.id = "20260904-000000-live".to_owned();
7510 alive.status = TaskStatus::Running;
7511 alive.attempts = 1;
7512 queue.put(&mut alive).unwrap();
7513 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
7514
7515 let mut queued = task();
7516 queued.id = "20260904-000000-wait".to_owned();
7517 queue.put(&mut queued).unwrap();
7518
7519 let reclaimed = reclaim_orphaned_running(&queue, 2);
7520 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
7521
7522 assert_eq!(
7523 queue.get(&orphaned.id).unwrap().status,
7524 TaskStatus::Held,
7525 "nothing was driving it and there was no run to recover"
7526 );
7527 assert_eq!(
7528 queue.get(&alive.id).unwrap().status,
7529 TaskStatus::Running,
7530 "a live claim must protect the task it belongs to"
7531 );
7532 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
7533 }
7534
7535 fn read_run_under(home: &Path, id: &str) -> RunState {
7541 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
7542 serde_json::from_str(&body).unwrap()
7543 }
7544
7545 #[test]
7546 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
7547 let dir = tempfile::tempdir().unwrap();
7548 let home = dir.path().to_path_buf();
7549 let now = Timestamp::now();
7550 let overrun_seat = || crate::run::ActiveSeat {
7551 node: "implement".to_owned(),
7552 started_at: now - jiff::SignedDuration::new(21_000, 0),
7553 timeout_secs: 3_600,
7554 attempt: 0,
7555 task: None,
7556 command: None,
7557 index: None,
7558 total: None,
7559 };
7560
7561 let mut dead = run_state(RunStatus::Implementing);
7562 dead.id = "20260101-000000-dead".to_owned();
7563 dead.active.insert("impl-A".to_owned(), overrun_seat());
7564 dead.driver_pid = Some(4242);
7567 dead.save_under(&home).unwrap();
7568
7569 let mut alive = run_state(RunStatus::Implementing);
7572 alive.id = "20260101-000000-aliv".to_owned();
7573 alive.active.insert("impl-A".to_owned(), overrun_seat());
7574 alive.save_under(&home).unwrap();
7575 let mut status = Status::new();
7576 status.current = vec![Current {
7577 task: "20260101-000000-task".to_owned(),
7578 run: alive.id.clone(),
7579 }];
7580 write_status_to(&home.join("daemon.json"), &status).unwrap();
7581
7582 let questions = Questions::at(home.join("questions"));
7586 let mut q = ask::Question::new(
7587 dead.id.clone(),
7588 "implement".to_owned(),
7589 "impl-A".to_owned(),
7590 "Which storage backend?".to_owned(),
7591 String::new(),
7592 vec!["SQLite".to_owned(), "Redis".to_owned()],
7593 );
7594 questions.put(&mut q).unwrap();
7595
7596 let abandoned = reclaim_abandoned_runs_with(
7597 &home,
7598 now,
7599 |pid| if pid == 4242 { Some(false) } else { None },
7600 |_| panic!("a query answering Dead outright needs no identity corroboration"),
7601 );
7602 assert_eq!(abandoned, vec![dead.id.clone()]);
7603
7604 let reloaded = read_run_under(&home, &dead.id);
7605 assert_eq!(reloaded.status, RunStatus::Failed);
7606 assert!(reloaded.active.is_empty());
7607 assert!(
7608 !questions.get(&q.id).unwrap().status.open(),
7609 "the failed run's own open question must be settled in the same pass"
7610 );
7611
7612 let still_alive = read_run_under(&home, &alive.id);
7613 assert_eq!(
7614 still_alive.status,
7615 RunStatus::Implementing,
7616 "a live daemon's claim protects it"
7617 );
7618 assert!(!still_alive.active.is_empty());
7619 }
7620
7621 #[test]
7631 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
7632 let dir = tempfile::tempdir().unwrap();
7633 let home = dir.path().to_path_buf();
7634 let now = Timestamp::now();
7635
7636 let mut manual = run_state(RunStatus::Reviewing);
7637 manual.id = "20260101-000000-manl".to_owned();
7638 manual.active.insert(
7639 "review-1".to_owned(),
7640 crate::run::ActiveSeat {
7641 node: "review".to_owned(),
7642 started_at: now - jiff::SignedDuration::new(21_000, 0),
7643 timeout_secs: 3_600,
7644 attempt: 0,
7645 task: None,
7646 command: None,
7647 index: None,
7648 total: None,
7649 },
7650 );
7651 manual.driver_pid = Some(4242);
7655 manual.driver_started_at = Some("1790000000".to_owned());
7656 manual.save_under(&home).unwrap();
7657
7658 let abandoned = reclaim_abandoned_runs_with(
7659 &home,
7660 now,
7661 |pid| if pid == 4242 { Some(true) } else { None },
7662 |pid| {
7663 if pid == 4242 {
7664 Some("1790000000".to_owned())
7665 } else {
7666 None
7667 }
7668 },
7669 );
7670 assert!(
7671 abandoned.is_empty(),
7672 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
7673 );
7674
7675 let reloaded = read_run_under(&home, &manual.id);
7676 assert_eq!(reloaded.status, RunStatus::Reviewing);
7677 assert!(!reloaded.active.is_empty());
7678 }
7679
7680 #[test]
7681 fn an_already_claimed_task_is_skipped_rather_than_failed() {
7682 let dir = tempfile::tempdir().unwrap();
7683 let queue = Queue::at(dir.path().to_path_buf());
7684 let mut only = task();
7685 queue.put(&mut only).unwrap();
7686
7687 let _elsewhere = queue.claim(&only.id).unwrap();
7688 let candidates = runnable(&queue);
7689 assert_eq!(candidates.len(), 1, "the task is still runnable");
7690 assert!(
7691 queue.claim(&candidates[0].id).is_err(),
7692 "the loop cannot take a claim somebody else holds"
7693 );
7694
7695 let after = queue.get(&only.id).unwrap();
7696 assert_eq!(after.status, TaskStatus::Queued);
7697 assert_eq!(
7698 after.attempts, 0,
7699 "losing the race is not an attempt at the task"
7700 );
7701 assert_eq!(after.last_error, None);
7702 }
7703
7704 #[test]
7705 fn the_status_file_round_trips_and_its_heartbeat_advances() {
7706 let dir = tempfile::tempdir().unwrap();
7707 let path = dir.path().join("daemon.json");
7708
7709 let mut status = Status::new();
7710 status.idle = false;
7711 status.completed = 7;
7712 status.current = vec![Current {
7713 task: "20260902-000000-t111".to_owned(),
7714 run: "20260902-000001-r111".to_owned(),
7715 }];
7716 write_status_to(&path, &status).unwrap();
7717 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7718 assert_eq!(first.schema, SCHEMA);
7719 assert_eq!(first.pid, std::process::id());
7720 assert!(!first.idle);
7721 assert_eq!(first.completed, 7);
7722 assert_eq!(first.current, status.current);
7723 assert!(
7724 !path.with_extension("json.tmp").exists(),
7725 "the temp file is renamed, not left behind"
7726 );
7727
7728 std::thread::sleep(Duration::from_millis(5));
7729 status.updated_at = Timestamp::now();
7730 status.polls = 3;
7731 write_status_to(&path, &status).unwrap();
7732 let second: Status =
7733 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7734 assert!(
7735 second.updated_at > first.updated_at,
7736 "a reader can only detect staleness if the heartbeat moves"
7737 );
7738 assert_eq!(
7739 second.started_at, first.started_at,
7740 "the start time is not a heartbeat"
7741 );
7742 assert_eq!(second.polls, 3);
7743 }
7744
7745 #[test]
7746 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
7747 let dir = tempfile::tempdir().unwrap();
7748
7749 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
7750
7751 let mut status = Status::new();
7752 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
7753 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7754 let stale = read_status(dir.path()).unwrap();
7755 assert!(
7756 !stale.running(Timestamp::now()),
7757 "a minute without a heartbeat is a dead daemon, not a busy one"
7758 );
7759 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
7760
7761 status.updated_at = Timestamp::now();
7762 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7763 let fresh = read_status(dir.path()).unwrap();
7764 assert!(fresh.running(Timestamp::now()));
7765 }
7766
7767 #[test]
7768 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
7769 let dir = tempfile::tempdir().unwrap();
7770 let now = Timestamp::now();
7771 let mine = "20260903-080619-01c2";
7772
7773 assert!(
7774 !is_working_on(dir.path(), mine, now),
7775 "no status file means nobody is working on anything"
7776 );
7777
7778 let mut status = Status::new();
7779 status.current = vec![Current {
7780 task: "20260903-080340-0167".to_owned(),
7781 run: mine.to_owned(),
7782 }];
7783 status.updated_at = now;
7784 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7785 assert!(is_working_on(dir.path(), mine, now));
7786 assert!(
7787 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
7788 "a daemon busy with one run is not working on another"
7789 );
7790
7791 status.updated_at = now - jiff::SignedDuration::from_secs(600);
7794 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7795 assert!(
7796 !is_working_on(dir.path(), mine, now),
7797 "a stale heartbeat is a dead daemon, so its run is a leftover"
7798 );
7799 }
7800
7801 #[test]
7802 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
7803 let dir = tempfile::tempdir().unwrap();
7804 let now = Timestamp::now();
7805
7806 assert!(
7807 !is_working_on_short(dir.path(), "01c2", now),
7808 "no status file means nobody is working on anything"
7809 );
7810
7811 let mut status = Status::new();
7812 status.current = vec![Current {
7813 task: "20260903-080340-0167".to_owned(),
7814 run: "20260903-080619-01c2".to_owned(),
7815 }];
7816 status.updated_at = now;
7817 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7818 assert!(
7819 is_working_on_short(dir.path(), "01c2", now),
7820 "the run's short id is the last block of its full id"
7821 );
7822 assert!(
7823 !is_working_on_short(dir.path(), "3cbf", now),
7824 "a daemon busy with one worktree bay is not working on another"
7825 );
7826 }
7827
7828 #[test]
7829 fn a_newer_status_file_still_yields_a_reading() {
7830 let dir = tempfile::tempdir().unwrap();
7831 std::fs::write(
7834 dir.path().join("daemon.json"),
7835 serde_json::json!({
7836 "schema": 2,
7837 "updated_at": Timestamp::now().to_string(),
7838 "idle": true,
7839 "surprise": { "nested": [1, 2, 3] },
7840 })
7841 .to_string(),
7842 )
7843 .unwrap();
7844
7845 let reading = read_status(dir.path()).expect("a forward-compatible read");
7846 assert!(reading.running(Timestamp::now()));
7847 assert!(reading.idle);
7848 assert!(reading.current.is_empty());
7849 }
7850
7851 #[test]
7852 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
7853 let dir = tempfile::tempdir().unwrap();
7859 std::fs::write(
7860 dir.path().join("daemon.json"),
7861 serde_json::json!({
7862 "schema": 1,
7863 "pid": 4242,
7864 "updated_at": Timestamp::now().to_string(),
7865 "idle": false,
7866 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
7867 "completed": 3,
7868 "polls": 9,
7869 })
7870 .to_string(),
7871 )
7872 .unwrap();
7873
7874 let reading = read_status(dir.path()).expect("an older shape must still parse");
7875 assert!(reading.running(Timestamp::now()));
7876 assert_eq!(
7877 reading.current,
7878 vec![Current {
7879 task: "20260902-140501-aaaa".to_owned(),
7880 run: "20260902-140502-bbbb".to_owned(),
7881 }]
7882 );
7883 }
7884
7885 #[test]
7886 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
7887 let dir = tempfile::tempdir().unwrap();
7888 std::fs::write(
7889 dir.path().join("daemon.json"),
7890 serde_json::json!({
7891 "schema": 1,
7892 "updated_at": Timestamp::now().to_string(),
7893 "idle": true,
7894 "current": null,
7895 })
7896 .to_string(),
7897 )
7898 .unwrap();
7899 let with_null = read_status(dir.path()).expect("null must still parse");
7900 assert!(with_null.current.is_empty());
7901
7902 std::fs::write(
7903 dir.path().join("daemon.json"),
7904 serde_json::json!({
7905 "schema": 1,
7906 "updated_at": Timestamp::now().to_string(),
7907 "idle": true,
7908 })
7909 .to_string(),
7910 )
7911 .unwrap();
7912 let absent = read_status(dir.path()).expect("a missing field must still parse");
7913 assert!(absent.current.is_empty());
7914 }
7915
7916 #[test]
7917 fn a_task_without_a_repository_runs_in_the_daemons_default() {
7918 let fallback = Path::new("/default");
7919 let mut blank = task();
7920 blank.repo = PathBuf::new();
7921 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
7922 let mut dot = task();
7923 dot.repo = PathBuf::from(".");
7924 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
7925 assert_eq!(
7926 repo_for(&task(), fallback),
7927 PathBuf::from("/repo"),
7928 "a task that names a repository keeps it"
7929 );
7930 }
7931
7932 #[test]
7933 fn task_implementers_edit_inherits_daemon_config_and_merge() {
7934 let dir = tempfile::tempdir().unwrap();
7935 let path = dir.path().join("explicit.toml");
7936 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
7937 let opts = Opts {
7938 config: Some(path),
7939 merge: Some("none".into()),
7940 ..Default::default()
7941 };
7942 let mut t = task();
7943 t.set_implementers(
7944 2,
7945 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7946 .unwrap(),
7947 )
7948 .unwrap();
7949 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
7950 assert_eq!(cfg.graph.implementers, 2);
7951 assert_eq!(cfg.graph.judges, 1);
7952 assert_eq!(cfg.merge.mode, MergeMode::None);
7953 t.set_implementers(1, &cfg).unwrap();
7954 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
7955 apply_task_implementers(&mut cfg, &t);
7956 assert_eq!(cfg.graph.implementers, 1);
7957 assert_eq!(cfg.graph.judges, 1);
7958 assert_eq!(cfg.merge.mode, MergeMode::None);
7959 }
7960
7961 #[test]
7962 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
7963 let dir = tempfile::tempdir().unwrap();
7964 let repo = dir.path().join("repo");
7965 std::fs::create_dir(&repo).unwrap();
7966 std::fs::write(
7967 repo.join("magi.toml"),
7968 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
7969 )
7970 .unwrap();
7971 let daemon_config = dir.path().join("daemon.toml");
7972 std::fs::write(
7973 &daemon_config,
7974 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
7975 )
7976 .unwrap();
7977 let opts = Opts {
7978 config: Some(daemon_config),
7979 merge: Some("pr".into()),
7980 ..Default::default()
7981 };
7982 let q = Queue::at(dir.path().join("queue"));
7983 let validation: Config =
7984 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7985 .unwrap();
7986 for record in ["{}", r#"{"candidates":2}"#] {
7988 let overrides = serde_json::from_str(record).unwrap();
7989 let filing = crate::direct::Filing {
7990 instruction: "work".into(),
7991 title: "direct".into(),
7992 repo: repo.clone(),
7993 source: crate::queue::Source::Human,
7994 solo: false,
7995 overrides,
7996 review_of: None,
7997 };
7998 let filed =
7999 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
8000 let crate::direct::Filed::Standalone {
8001 mut task,
8002 claim: _claim,
8003 } = filed
8004 else {
8005 panic!("local filing")
8006 };
8007 let before = prepare_for(&repo, &opts, &task).unwrap();
8008 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
8009 assert_eq!(
8010 before.merge.mode,
8011 MergeMode::None,
8012 "daemon --merge must not leak"
8013 );
8014 for n in [2, 1, 2] {
8015 task.set_implementers(n, &validation).unwrap();
8016 q.put(&mut task).unwrap();
8017 task = q.get(&task.id).unwrap();
8018 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
8019 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
8020 apply_task_implementers(&mut cfg, &task);
8021 assert_eq!(cfg.graph.implementers, usize::from(n));
8022 assert_eq!(cfg.graph.judges, 2);
8023 assert_eq!(cfg.merge.mode, MergeMode::None);
8024 }
8025 }
8026 }
8027
8028 #[test]
8029 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
8030 let cfg: Config =
8031 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8032 .unwrap();
8033 let mut t = task();
8034 t.review_of = Some("review-target".into());
8035 t.hold_manual(Some("retry".into()));
8036 let mut previous = RunState::new(
8037 PathBuf::from("/repo"),
8038 "main".into(),
8039 "abc1234".into(),
8040 "review".into(),
8041 cfg.clone(),
8042 );
8043 previous.config.graph.implementers = 2;
8044 for count in [1, 3] {
8045 t.set_implementers(count, &cfg).unwrap();
8046 t.release();
8047 assert_eq!(t.review_of.as_deref(), Some("review-target"));
8048 for proposed in [
8049 Starter::Resume(previous.id.clone()),
8050 Starter::Review("recovery-target".into()),
8051 ] {
8052 assert_eq!(
8053 starter_with_implementers(&t, Some(&previous), proposed),
8054 Starter::Review("review-target".into())
8055 );
8056 }
8057 let mut configured = cfg.clone();
8058 apply_task_implementers(&mut configured, &t);
8059 assert_eq!(configured.graph.implementers, usize::from(count));
8060 assert_eq!(previous.config.graph.implementers, 2);
8061 }
8062 }
8063
8064 #[test]
8065 fn task_implementers_dispatch_and_resume_agree() {
8066 let mut t = task();
8067 let cfg: crate::config::Config =
8068 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8069 .unwrap();
8070 let mut run = RunState::new(
8071 PathBuf::from("/repo"),
8072 "main".into(),
8073 "abc1234".into(),
8074 "work".into(),
8075 cfg.clone(),
8076 );
8077 run.config.graph.implementers = 2;
8078 for n in [2, 1, 3, 2] {
8079 t.set_implementers(n, &cfg).unwrap();
8080 let mut dispatched = cfg.clone();
8081 apply_task_implementers(&mut dispatched, &t);
8082 assert_eq!(dispatched.graph.implementers, usize::from(n));
8083 assert_eq!(implementers_match(&t, &run), n == 2);
8084 for proposed in [
8085 Starter::Resume(run.id.clone()),
8086 Starter::Review("old-branch".into()),
8087 ] {
8088 let expected = if n == 2 {
8089 proposed.clone()
8090 } else {
8091 Starter::Start
8092 };
8093 assert_eq!(
8094 starter_with_implementers(&t, Some(&run), proposed),
8095 expected
8096 );
8097 }
8098 assert_eq!(
8099 run.config.graph.implementers, 2,
8100 "history must not be rewritten"
8101 );
8102 }
8103 t.solo = false;
8104 t.overrides = None;
8105 assert!(implementers_match(&t, &run));
8106 }
8107
8108 #[test]
8109 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
8110 let mut solo_cfg = Config::default();
8116 solo_cfg.graph.implementers = 3;
8117 let mut solo_task = task();
8118 solo_task.solo = true;
8119 apply_task_implementers(&mut solo_cfg, &solo_task);
8120 assert_eq!(solo_cfg.graph.implementers, 1);
8121
8122 let mut plain_cfg = Config::default();
8123 plain_cfg.graph.implementers = 3;
8124 let plain_task = task();
8125 assert!(!plain_task.solo);
8126 apply_task_implementers(&mut plain_cfg, &plain_task);
8127 assert_eq!(
8128 plain_cfg.graph.implementers, 3,
8129 "a task that did not ask to run alone keeps the config's candidates"
8130 );
8131 }
8132
8133 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
8134 QuotaLoss {
8135 seat: seat.into(),
8136 node: "judge".into(),
8137 at: at.parse().unwrap(),
8138 reset: reset.map(str::to_string),
8139 }
8140 }
8141
8142 #[test]
8143 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
8144 let old: Vec<QuotaLoss> = (1..=4)
8145 .map(|i| {
8146 loss(
8147 &format!("judge-{i}"),
8148 "2026-09-23T05:23:00Z",
8149 Some("2:40pm (Asia/Tokyo)"),
8150 )
8151 })
8152 .collect();
8153 let fresh = losses_this_attempt(&old, &old);
8154 assert!(fresh.is_empty());
8155 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
8156 }
8158
8159 #[test]
8160 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
8161 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
8162 let now = Timestamp::now();
8163 let mut after = old.clone();
8164 after.push(loss("judge-2", &now.to_string(), None));
8165 let fresh = losses_this_attempt(&old, &after);
8166 assert_eq!(fresh, vec![after[1].clone()]);
8167 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
8168 assert_eq!(
8169 until,
8170 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
8171 );
8172 }
8173
8174 #[test]
8175 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
8176 let before = vec![
8179 loss("judge-1", "2026-09-23T05:23:00Z", None),
8180 loss("judge-2", "2026-09-23T05:24:00Z", None),
8181 ];
8182 let after = vec![
8183 loss("judge-2", "2026-09-23T05:24:00Z", None),
8184 loss("judge-1", "2026-09-24T01:00:00Z", None),
8185 ];
8186 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
8187 }
8188
8189 #[test]
8190 fn merge_overrides_are_parsed_or_refused() {
8191 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
8192 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
8193 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
8194 assert!(merge_mode("squash").is_err());
8195 }
8196
8197 #[test]
8198 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
8199 let now = Timestamp::now();
8200 let fallback = Duration::from_secs(300);
8201 let cap = Duration::from_secs(1800);
8202
8203 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
8205
8206 let soon = now + jiff::SignedDuration::from_secs(600);
8208 assert_eq!(
8209 quota_wait(Some(soon), now, fallback, cap),
8210 Duration::from_secs(600)
8211 );
8212
8213 let past = now - jiff::SignedDuration::from_secs(60);
8216 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
8217
8218 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
8221 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
8222 }
8223
8224 #[test]
8225 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
8226 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8227
8228 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
8229 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
8230
8231 let already_past =
8235 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
8236 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
8237
8238 assert!(
8239 parse_reset_hint("session limit reached", now, now).is_none(),
8240 "free text with no recognised shape is not guessed at"
8241 );
8242 assert!(
8243 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
8244 "an unresolvable zone name is not guessed at either"
8245 );
8246 }
8247
8248 #[test]
8249 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
8250 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8251
8252 let at = parse_reset_hint(
8253 "You've hit your usage limit. Visit \
8254 https://chatgpt.com/codex/settings/usage to purchase more \
8255 credits or try again at Sep 19th, 2026 5:10 PM.",
8256 now,
8257 now,
8258 )
8259 .expect("the codex reset wording is a recognised shape");
8260 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
8261
8262 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
8267 .expect("an explicit year needs no rollover");
8268 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
8269
8270 assert!(
8271 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
8272 "a two-digit year is not the documented shape and is not guessed at"
8273 );
8274 assert!(
8275 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
8276 "a four-letter month name is not the documented three-letter abbreviation"
8277 );
8278 assert!(
8279 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
8280 "an explicit zone on the dated shape is a format nobody has \
8281 documented, and is refused rather than guessed at as UTC"
8282 );
8283 }
8284
8285 #[test]
8286 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
8287 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
8288 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
8289
8290 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
8291 assert_eq!(at.as_second() - recorded.as_second(), 3769);
8292
8293 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
8294 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
8295
8296 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
8297 assert!(
8298 parse_reset_hint(bad, now, recorded).is_none(),
8299 "{bad:?} must not be guessed at"
8300 );
8301 }
8302 }
8303
8304 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
8308 let config = dir.join("magi.toml");
8309 std::fs::write(
8310 &config,
8311 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
8312 )
8313 .unwrap();
8314 let opts = Opts {
8315 poll: Duration::from_secs(30),
8316 config: Some(config),
8317 repo: dir.join("repo"),
8321 ..Opts::default()
8322 };
8323 let home = dir.join("home");
8332 let worktrees = dir.join("wt");
8333 (
8334 opts,
8335 Queue::at(dir.join("queue")),
8336 home.join("daemon.json"),
8337 home,
8338 worktrees,
8339 )
8340 }
8341
8342 #[test]
8343 fn a_stop_is_idempotent_and_once_set_stays_set() {
8344 let stop = Stop::new();
8345 assert!(!stop.stopped());
8346
8347 stop.stop();
8348 assert!(stop.stopped());
8349 stop.stop();
8350 assert!(stop.stopped(), "a second stop is not a toggle");
8351
8352 let shared = stop.clone();
8353 assert!(
8354 shared.stopped(),
8355 "a clone is the same stop; that is how the loop and its caller share one"
8356 );
8357 }
8358
8359 #[test]
8360 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
8361 let stop = Stop::new();
8362 stop.enter();
8363 assert!(
8364 !stop.finishing(),
8365 "a busy loop nobody has asked to stop is just running"
8366 );
8367
8368 stop.stop();
8369 assert!(
8370 stop.finishing(),
8371 "a stop asked for mid-run has not landed until the run is settled"
8372 );
8373
8374 stop.exit();
8375 assert!(
8376 !stop.finishing(),
8377 "once the run is settled the stop has landed and there is nothing to finish"
8378 );
8379 }
8380
8381 #[test]
8382 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
8383 let stop = Stop::new();
8384 stop.enter();
8385 stop.enter();
8386 stop.stop();
8387 assert!(stop.finishing(), "two runs still in flight");
8388
8389 stop.exit();
8390 assert!(
8391 stop.finishing(),
8392 "one run finished, but a sibling is still working"
8393 );
8394
8395 stop.exit();
8396 assert!(
8397 !stop.finishing(),
8398 "the last run out is what actually lands the stop"
8399 );
8400 }
8401
8402 #[tokio::test]
8403 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
8404 let dir = tempfile::tempdir().unwrap();
8405 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8406 let stop = Stop::new();
8407 stop.stop();
8408
8409 let began = std::time::Instant::now();
8410 tokio::time::timeout(
8411 Duration::from_secs(2),
8412 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8413 )
8414 .await
8415 .expect("a stopped loop must return, not sit out its poll interval")
8416 .expect("the loop's own setup and teardown must not fail");
8417 assert!(
8418 began.elapsed() < opts.poll,
8419 "returned only after {:?}, which is a poll interval, not a stop",
8420 began.elapsed()
8421 );
8422 }
8423
8424 #[tokio::test]
8425 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
8426 let dir = tempfile::tempdir().unwrap();
8427 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8428 let stop = Stop::new();
8429
8430 let asker = {
8433 let stop = stop.clone();
8434 tokio::spawn(async move {
8435 tokio::time::sleep(Duration::from_millis(20)).await;
8436 stop.stop();
8437 })
8438 };
8439
8440 let began = std::time::Instant::now();
8441 tokio::time::timeout(
8442 Duration::from_secs(2),
8443 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8444 )
8445 .await
8446 .expect("a stop asked for while idle must wake the wait")
8447 .expect("the loop's own setup and teardown must not fail");
8448 asker.await.unwrap();
8449 assert!(
8450 began.elapsed() < opts.poll,
8451 "returned only after {:?}, so the stop waited on the sleep",
8452 began.elapsed()
8453 );
8454 }
8455
8456 #[tokio::test]
8457 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
8458 let dir = tempfile::tempdir().unwrap();
8459 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8460 let stop = Stop::new();
8461 stop.stop();
8462
8463 tokio::time::timeout(
8464 Duration::from_secs(2),
8465 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8466 )
8467 .await
8468 .expect("a stopped loop must return")
8469 .expect("the loop's own setup and teardown must not fail");
8470
8471 assert!(
8472 home.is_dir(),
8473 "the loop did publish a status file, so its removal is the teardown and not an absence"
8474 );
8475 assert!(
8476 !status_file.exists(),
8477 "a stopped loop clears its status file"
8478 );
8479 assert!(
8480 read_status(&home).is_none(),
8481 "a reader must see no daemon at all, not a heartbeat that merely stopped"
8482 );
8483 }
8484
8485 #[tokio::test]
8486 async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
8487 let dir = tempfile::tempdir().unwrap();
8488 let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8489 opts.once = true;
8490
8491 let mut settled = RunState::new(
8492 dir.path().join("repo"),
8493 "main".to_owned(),
8494 "abc1234".to_owned(),
8495 "fixture".to_owned(),
8496 Config::default(),
8497 );
8498 settled.status = RunStatus::Ready;
8499 let run_dir = home.join("runs").join(&settled.id);
8500 std::fs::create_dir_all(&run_dir).unwrap();
8501 std::fs::write(
8502 run_dir.join("run.json"),
8503 serde_json::to_string_pretty(&settled).unwrap(),
8504 )
8505 .unwrap();
8506 let questions = Questions::at(home.join("questions"));
8507 let mut question = ask::Question::new(
8508 settled.id.clone(),
8509 "review".to_owned(),
8510 "reviewer-1".to_owned(),
8511 "Continue?".to_owned(),
8512 String::new(),
8513 Vec::new(),
8514 );
8515 questions.put(&mut question).unwrap();
8516
8517 drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
8518 .await
8519 .unwrap();
8520
8521 assert_eq!(
8522 questions.get(&question.id).unwrap().status,
8523 ask::QuestionStatus::Abandoned,
8524 "an empty --once drain still performs startup question cleanup"
8525 );
8526 }
8527
8528 #[test]
8529 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
8530 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8531
8532 assert!(
8533 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
8534 "never checked before: due at once"
8535 );
8536
8537 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
8538 assert!(
8539 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
8540 "well inside the interval: not due yet"
8541 );
8542
8543 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
8544 assert!(
8545 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
8546 "exactly at the edge: not yet due, same convention as `clean::due`"
8547 );
8548
8549 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8550 assert!(
8551 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
8552 "past the interval: due again"
8553 );
8554 }
8555
8556 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
8561 let config = dir.join("magi.toml");
8562 std::fs::write(
8568 &config,
8569 format!(
8570 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
8571 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
8572 cache_dir.display()
8573 ),
8574 )
8575 .unwrap();
8576 Opts {
8577 config: Some(config),
8578 repo: dir.join("repo"),
8579 ..Opts::default()
8580 }
8581 }
8582
8583 #[tokio::test]
8584 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
8585 let dir = tempfile::tempdir().unwrap();
8586 let home = dir.path().join("home");
8587 let cache_dir = dir.path().join("cache");
8588 std::fs::create_dir_all(&cache_dir).unwrap();
8589 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8590 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8591
8592 let running = Stop::new();
8595 let mut last_checked = None;
8596 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8597 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
8598 .await;
8599 assert_eq!(
8600 crate::disk::dir_size(&cache_dir),
8601 0,
8602 "over the cap on the first check ever: pruned at once, no idle queue required"
8603 );
8604 assert_eq!(last_checked, Some(t0));
8605
8606 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
8608 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
8609 maybe_prune_cache_between_runs(
8610 &opts.repo,
8611 &opts,
8612 &home,
8613 &running,
8614 &mut last_checked,
8615 too_soon,
8616 )
8617 .await;
8618 assert_eq!(
8619 crate::disk::dir_size(&cache_dir),
8620 10,
8621 "too soon since the last check: left alone rather than rescanned every call"
8622 );
8623 assert_eq!(
8624 last_checked,
8625 Some(t0),
8626 "an idle check does not reset the clock"
8627 );
8628
8629 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8631 maybe_prune_cache_between_runs(
8632 &opts.repo,
8633 &opts,
8634 &home,
8635 &running,
8636 &mut last_checked,
8637 due_again,
8638 )
8639 .await;
8640 assert_eq!(
8641 crate::disk::dir_size(&cache_dir),
8642 0,
8643 "due again: pruned back under the cap"
8644 );
8645 }
8646
8647 #[tokio::test]
8655 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
8656 let dir = tempfile::tempdir().unwrap();
8657 let home = dir.path().join("home");
8658 let cache_dir = dir.path().join("cache");
8659 std::fs::create_dir_all(&cache_dir).unwrap();
8660 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8661 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8662
8663 let stop = Stop::new();
8664 stop.stop();
8665 assert!(
8666 !stop.finishing(),
8667 "no run is in flight at a between-runs boundary, so nothing else \
8668 would tell the operator this stop had not taken effect yet"
8669 );
8670
8671 let mut last_checked = None;
8672 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8673 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
8674 .await;
8675 assert_eq!(
8676 crate::disk::dir_size(&cache_dir),
8677 10,
8678 "over its cap, and due for the first check ever, but a stop outranks \
8679 it: the cap is a standing policy the next start measures again"
8680 );
8681 assert_eq!(
8682 last_checked, None,
8683 "a check that never happened must not claim the interval"
8684 );
8685 }
8686
8687 #[tokio::test]
8698 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
8699 let dir = tempfile::tempdir().unwrap();
8700 let cache_dir = dir.path().join("cache");
8701 std::fs::create_dir_all(&cache_dir).unwrap();
8702 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
8703
8704 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
8705 opts.poll = Duration::from_millis(20);
8706 opts.max_attempts = 1_000;
8707
8708 let queue = Queue::at(dir.path().join("queue"));
8709 for i in 0..128 {
8710 let mut task = Task::new(
8711 format!("queued work {i}"),
8712 format!("queued work {i}"),
8713 opts.repo.clone(),
8714 Source::Human,
8715 );
8716 queue.put(&mut task).unwrap();
8717 }
8718
8719 let home = dir.path().join("home");
8720 let worktrees = dir.path().join("wt");
8721 let status_file = home.join("daemon.json");
8722 let stop = Stop::new();
8723 let stopper = {
8724 let stop = stop.clone();
8725 let queue = Queue::at(dir.path().join("queue"));
8726 tokio::spawn(async move {
8727 for _ in 0..2_000 {
8731 if queue.list().iter().filter(|t| t.attempts == 1).count() >= 2 {
8732 break;
8733 }
8734 tokio::time::sleep(Duration::from_millis(5)).await;
8735 }
8736 stop.stop();
8737 })
8738 };
8739
8740 tokio::time::timeout(
8741 Duration::from_secs(20),
8742 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8743 )
8744 .await
8745 .expect("the loop must not hang on a queue that keeps producing failing work")
8746 .expect("the loop's own setup and teardown must not fail");
8747 stopper.await.unwrap();
8748
8749 let after = queue.list();
8750 let completed = after.iter().filter(|t| t.attempts == 1).count();
8751 assert!(
8752 completed >= 2,
8753 "the harness must have processed multiple tasks to exercise a busy queue \
8754 (got {completed} completed task(s))"
8755 );
8756 assert!(
8757 after.iter().any(|t| t.status.runnable()),
8758 "unprocessed tasks must remain: the external stop, rather than an \
8759 exhausted backlog, must end the test"
8760 );
8761 assert!(
8762 after
8763 .iter()
8764 .filter(|t| t.attempts > 0)
8765 .all(|t| { t.attempts == 1 && t.status == TaskStatus::Held }),
8766 "a busy backlog must not retry a task's unchanged startup failure"
8767 );
8768
8769 assert_eq!(
8770 crate::disk::dir_size(&cache_dir),
8771 0,
8772 "an oversized cache must not be left to grow unboundedly just because the \
8773 queue kept the loop busy the whole time"
8774 );
8775 }
8776
8777 #[test]
8778 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
8779 let dir = tempfile::tempdir().unwrap();
8780 let queue = Queue::at(dir.path().join("queue"));
8781 let questions = Questions::at(dir.path().join("questions"));
8782 let mut task = task();
8783 queue.put(&mut task).unwrap();
8784
8785 let mut task_question = ask::Question::new(
8786 task.id.clone(),
8787 crate::conduct::NODE.to_owned(),
8788 "conduct".to_owned(),
8789 "Which backend?".to_owned(),
8790 String::new(),
8791 Vec::new(),
8792 );
8793 questions.put(&mut task_question).unwrap();
8794 task.block(vec![task_question.id.clone()], None);
8795 queue.put(&mut task).unwrap();
8796
8797 let mut run_question = ask::Question::new(
8798 "20260101-000000-run1".to_owned(),
8799 "review".to_owned(),
8800 "reviewer-1".to_owned(),
8801 "Run question".to_owned(),
8802 String::new(),
8803 Vec::new(),
8804 );
8805 questions.put(&mut run_question).unwrap();
8806
8807 let mut coincidental = ask::Question::new(
8812 task.id.clone(),
8813 "review".to_owned(),
8814 "reviewer-1".to_owned(),
8815 "Unrelated review question".to_owned(),
8816 String::new(),
8817 Vec::new(),
8818 );
8819 questions.put(&mut coincidental).unwrap();
8820
8821 reconcile_task_questions(&queue, &questions);
8822 assert!(questions.get(&task_question.id).unwrap().status.open());
8823 assert!(questions.get(&run_question.id).unwrap().status.open());
8824 assert!(questions.get(&coincidental.id).unwrap().status.open());
8825
8826 task.release();
8827 queue.put(&mut task).unwrap();
8828 reconcile_task_questions(&queue, &questions);
8829 assert_eq!(
8830 questions.get(&task_question.id).unwrap().status,
8831 ask::QuestionStatus::Abandoned
8832 );
8833 assert!(
8834 questions.get(&run_question.id).unwrap().status.open(),
8835 "run questions remain the run janitor's responsibility"
8836 );
8837 assert!(
8838 questions.get(&coincidental.id).unwrap().status.open(),
8839 "a non-conductor question must not be abandoned just because its \
8840 run id coincides with a task id"
8841 );
8842 }
8843
8844 #[test]
8845 fn a_freshly_started_running_task_is_never_stalled() {
8846 let dir = tempfile::tempdir().unwrap();
8847 let mut t = task();
8848 t.start("run-1".to_owned());
8849 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
8852 }
8853
8854 #[test]
8855 fn a_long_running_task_with_no_live_daemon_is_stalled() {
8856 let dir = tempfile::tempdir().unwrap();
8857 let mut t = task();
8858 t.start("run-1".to_owned());
8859 t.updated_at = Timestamp::now()
8860 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8861 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
8862 assert_eq!(
8863 stalled_tasks(
8864 &Queue::at(dir.path().join("q")),
8865 dir.path(),
8866 Timestamp::now()
8867 )
8868 .len(),
8869 0,
8870 "the task was never written to this queue"
8871 );
8872 }
8873
8874 #[test]
8875 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
8876 let dir = tempfile::tempdir().unwrap();
8877 let mut t = task();
8878 t.id = "20260903-080340-0167".to_owned();
8879 t.start("20260903-080619-01c2".to_owned());
8880 t.updated_at = Timestamp::now()
8881 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8882
8883 let mut status = Status::new();
8884 status.current = vec![Current {
8885 task: t.id.clone(),
8886 run: "20260903-080619-01c2".to_owned(),
8887 }];
8888 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8889
8890 assert!(
8891 !is_stalled(&t, dir.path(), Timestamp::now()),
8892 "a live daemon's own heartbeat rules out stalled, however long the task has run"
8893 );
8894 }
8895
8896 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
8900 let path = queue.path_of(id);
8901 let body = std::fs::read_to_string(&path).unwrap();
8902 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
8903 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
8904 v["updated_at"] = serde_json::Value::String(old.to_string());
8905 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
8906 }
8907
8908 #[test]
8909 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
8910 let dir = tempfile::tempdir().unwrap();
8923 let queue = Queue::at(dir.path().join("queue"));
8924 let home = dir.path().join("home");
8925
8926 let mut t = task();
8927 t.id = "20260101-000001-lock".to_owned();
8928 t.start("run-1".to_owned());
8929 queue.put(&mut t).unwrap();
8930 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
8931 std::fs::write(
8932 dir.path().join("queue").join(format!("{}.lock", t.id)),
8933 "not a pid",
8934 )
8935 .unwrap();
8936
8937 let now = Timestamp::now();
8938 assert!(
8939 reclaim_orphaned_running(&queue, 2).is_empty(),
8940 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
8941 and reclaim must leave the task alone"
8942 );
8943 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8944
8945 let stalled = stalled_tasks(&queue, &home, now);
8946 assert_eq!(
8947 stalled.len(),
8948 1,
8949 "reclaim's inability to claim it yet must not hide it from the conductor"
8950 );
8951 assert_eq!(stalled[0].id, t.id);
8952 }
8953
8954 #[test]
8955 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
8956 let dir = tempfile::tempdir().unwrap();
8957 crate::run::set_home(dir.path().join("run-home"));
8958 let queue = Queue::at(dir.path().join("queue"));
8959 let home = dir.path().join("home");
8960 let questions = Questions::at(dir.path().join("questions"));
8961
8962 let mut t = task();
8963 t.id = "20260101-000003-dead".to_owned();
8964 t.start("missing-run".to_owned());
8965 queue.put(&mut t).unwrap();
8966 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
8967
8968 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
8971 assert_eq!(
8972 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
8973 [&t.id]
8974 );
8975 assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
8976 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
8977
8978 crate::conduct::apply(
8981 &queue,
8982 &questions,
8983 &crate::conduct::Verdict {
8984 decisions: vec![crate::conduct::Decision {
8985 id: t.id.clone(),
8986 recovery: Some(crate::conduct::Recovery::Requeue),
8987 ..crate::conduct::Decision::default()
8988 }],
8989 },
8990 )
8991 .unwrap();
8992 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
8993 }
8994
8995 #[test]
8996 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
8997 let dir = tempfile::tempdir().unwrap();
8998 let queue = Queue::at(dir.path().join("queue"));
8999 let home = dir.path().join("home");
9000
9001 let mut fresh = task();
9002 fresh.id = "20260101-000001-aaaa".to_owned();
9003 fresh.start("run-1".to_owned());
9004 queue.put(&mut fresh).unwrap();
9005
9006 let mut old = task();
9007 old.id = "20260101-000002-bbbb".to_owned();
9008 old.start("run-2".to_owned());
9009 queue.put(&mut old).unwrap();
9010 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
9011
9012 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
9013 assert_eq!(stalled.len(), 1);
9014 assert_eq!(stalled[0].id, old.id);
9015 }
9016
9017 #[test]
9018 fn queued_and_finished_task_views_partition_by_status() {
9019 let dir = tempfile::tempdir().unwrap();
9020 let queue = Queue::at(dir.path().join("queue"));
9021
9022 let mut queued = task();
9023 queued.id = "20260101-000001-aaaa".to_owned();
9024 queue.put(&mut queued).unwrap();
9025
9026 let mut failed = task();
9027 failed.id = "20260101-000002-bbbb".to_owned();
9028 failed.start("run-1".to_owned());
9029 failed.fail("gate red", 5);
9030 queue.put(&mut failed).unwrap();
9031
9032 let mut held = task();
9033 held.id = "20260101-000003-cccc".to_owned();
9034 held.hold_machine(None);
9035 queue.put(&mut held).unwrap();
9036
9037 let mut running = task();
9038 running.id = "20260101-000004-dddd".to_owned();
9039 running.start("run-2".to_owned());
9040 queue.put(&mut running).unwrap();
9041
9042 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
9043 assert_eq!(queued_ids, [queued.id.clone()]);
9044
9045 let mut finished_ids: Vec<String> =
9046 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
9047 finished_ids.sort_unstable();
9048 let mut want = vec![failed.id.clone(), held.id.clone()];
9049 want.sort_unstable();
9050 assert_eq!(finished_ids, want);
9051 }
9052
9053 #[test]
9054 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
9055 for short in [false, true] {
9056 let dir = tempfile::tempdir().unwrap();
9057 let queue = Queue::at(dir.path().join("queue"));
9058 let questions = ask::Questions::at(dir.path().join("questions"));
9059 let mut running = task();
9060 running.id = "20261013-192213-feb2".into();
9061 running.start("existing-run".into());
9062 queue.put(&mut running).unwrap();
9063 let mut middle = task();
9064 middle.id = "20261013-193611-ca5b".into();
9065 middle.block(
9066 vec![if short {
9067 "feb2".into()
9068 } else {
9069 running.id.clone()
9070 }],
9071 None,
9072 );
9073 queue.put(&mut middle).unwrap();
9074 let mut waiting = task();
9075 waiting.id = "20261013-202849-ee37".into();
9076 waiting.block(
9077 vec![if short {
9078 "ca5b".into()
9079 } else {
9080 middle.id.clone()
9081 }],
9082 Some("wait for implementation".into()),
9083 );
9084 queue.put(&mut waiting).unwrap();
9085 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
9086 let inv = crate::blockers::Inventory::load(&queue, &questions);
9087 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
9088 assert!(inv.stuck().is_empty());
9089 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9090 assert!(report.quarantined.is_empty());
9091 assert!(report.asked.is_empty());
9092 resolve_blockers(&queue, &questions);
9093 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9094 assert!(questions.list().is_empty());
9095 running.succeed();
9098 queue.put(&mut running).unwrap();
9099 resolve_blockers(&queue, &questions);
9100 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
9101 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9102 middle = queue.get(&middle.id).unwrap();
9103 middle.succeed();
9104 queue.put(&mut middle).unwrap();
9105 resolve_blockers(&queue, &questions);
9106 let after = queue.get(&waiting.id).unwrap();
9107 assert_eq!(after.status, TaskStatus::Queued);
9108 assert!(after.blocked_by.is_empty());
9109 assert_eq!(after.instruction, waiting.instruction);
9110 }
9111 }
9112
9113 #[test]
9114 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
9115 for future_schema in [false, true] {
9116 let name = "20261013-193611-ca5b";
9117 let mut future = task();
9118 future.id = name.into();
9119 future.schema = crate::queue::SCHEMA + 1;
9120 let contents = if future_schema {
9121 serde_json::to_string(&future).unwrap()
9122 } else {
9123 "corrupt".into()
9124 };
9125 let dir = tempfile::tempdir().unwrap();
9126 let queue = Queue::at(dir.path().join("queue"));
9127 let questions = ask::Questions::at(dir.path().join("questions"));
9128 let mut waiting = task();
9129 waiting.id = "20261013-202849-ee37".into();
9130 waiting.block(vec!["ca5b".into()], None);
9131 queue.put(&mut waiting).unwrap();
9132 std::fs::write(queue.path_of(name), contents).unwrap();
9133 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
9134 assert!(
9135 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
9136 );
9137 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9138 assert!(report.asked.is_empty());
9139 resolve_blockers(&queue, &questions);
9140 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9141 assert!(questions.list().is_empty());
9142 }
9143 let dir = tempfile::tempdir().unwrap();
9144 let queue = Queue::at(dir.path().join("queue"));
9145 let questions = ask::Questions::at(dir.path().join("questions"));
9146 let mut waiting = task();
9147 waiting.id = "20261013-202849-ee37".into();
9148 waiting.block(vec!["ca5b".into()], None);
9149 queue.put(&mut waiting).unwrap();
9150 let mut done = task();
9151 done.id = "20261013-193611-ca5b".into();
9152 done.succeed();
9153 queue.put(&mut done).unwrap();
9154 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
9157 assert!(queue.get("ca5b").is_err());
9158 assert_eq!(
9159 crate::queue::dependency(&queue, &questions, "ca5b"),
9160 crate::queue::Dependency::Ambiguous
9161 );
9162 resolve_blockers(&queue, &questions);
9163 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9164 assert!(report.asked.is_empty());
9165 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9166 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
9167 }
9168
9169 #[test]
9170 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
9171 for held in [false, true] {
9172 for question in [false, true] {
9173 for reference in ["ca5b", "20261013-193611-ca5b"] {
9174 let dir = tempfile::tempdir().unwrap();
9175 let queue = Queue::at(dir.path().join("queue"));
9176 let questions = ask::Questions::at(dir.path().join("questions"));
9177 let mut prerequisite = task();
9178 prerequisite.id = "20261013-193611-ca5b".into();
9179 prerequisite.start("existing-run".into());
9180 queue.put(&mut prerequisite).unwrap();
9181 let mut waiting = task();
9182 waiting.id = "20261013-202849-ee37".into();
9183 if held {
9184 waiting.hold_machine(Some("waiting for separate consent".into()));
9185 }
9186 queue.put(&mut waiting).unwrap();
9187 let reason = "Deleting saved data requires separate owner consent";
9188 crate::conduct::apply(
9189 &queue,
9190 &questions,
9191 &crate::conduct::Verdict {
9192 decisions: vec![crate::conduct::Decision {
9193 id: waiting.id.clone(),
9194 blocked_by: vec![reference.into()],
9195 question: question.then(|| "May the saved data be deleted?".into()),
9196 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
9197 reason: Some(reason.into()),
9198 ..crate::conduct::Decision::default()
9199 }],
9200 },
9201 )
9202 .unwrap();
9203 let before = queue.get(&waiting.id).unwrap();
9204 if question {
9205 assert_eq!(before.status, TaskStatus::Blocked);
9206 let q = questions.get(&before.blocked_by[0]).unwrap();
9207 assert_eq!(q.summary, "May the saved data be deleted?");
9208 assert!(q.status.open());
9209 } else {
9210 assert_eq!(before.status, TaskStatus::Held);
9211 let recorded_reason = before.hold_reason.as_deref().unwrap();
9212 assert!(recorded_reason.starts_with(reason));
9213 if held {
9214 assert!(recorded_reason.contains("waiting for separate consent"));
9215 }
9216 }
9217 prerequisite.succeed();
9218 queue.put(&mut prerequisite).unwrap();
9219 resolve_blockers(&queue, &questions);
9220 let after = queue.get(&waiting.id).unwrap();
9221 assert_eq!(after.status, before.status);
9222 assert_eq!(after.blocked_by, before.blocked_by);
9223 assert_eq!(after.hold_reason, before.hold_reason);
9224 assert_eq!(after.instruction, waiting.instruction);
9225 if question {
9226 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
9227 }
9228 assert!(
9229 queue.next_runnable().is_none(),
9230 "dependency completion is not consent"
9231 );
9232 }
9233 }
9234 }
9235 }
9236
9237 #[test]
9238 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
9239 let dir = tempfile::tempdir().unwrap();
9240 let queue = Queue::at(dir.path().join("queue"));
9241 let questions = ask::Questions::at(dir.path().join("questions"));
9242
9243 let mut dep = task();
9244 dep.id = "20260101-000001-dep0".to_owned();
9245 dep.succeed();
9246 queue.put(&mut dep).unwrap();
9247
9248 let mut still_going = task();
9249 still_going.id = "20260101-000002-dep1".to_owned();
9250 queue.put(&mut still_going).unwrap();
9251
9252 let mut blocked = task();
9253 blocked.id = "20260101-000003-main".to_owned();
9254 blocked.block(
9255 vec![dep.id.clone(), still_going.id.clone()],
9256 Some("waits on both".to_owned()),
9257 );
9258 queue.put(&mut blocked).unwrap();
9259
9260 resolve_blockers(&queue, &questions);
9261
9262 let after = queue.get(&blocked.id).unwrap();
9263 assert_eq!(
9264 after.status,
9265 TaskStatus::Blocked,
9266 "one dependency is still outstanding"
9267 );
9268 assert_eq!(after.blocked_by, [still_going.id.clone()]);
9269 }
9270
9271 #[test]
9272 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
9273 let dir = tempfile::tempdir().unwrap();
9274 let queue = Queue::at(dir.path().join("queue"));
9275 let questions = ask::Questions::at(dir.path().join("questions"));
9276
9277 let mut q = crate::ask::Question::new(
9278 "20260101-000001-main".to_owned(),
9279 crate::conduct::NODE.to_owned(),
9280 "conduct".to_owned(),
9281 "Which backend?".to_owned(),
9282 String::new(),
9283 Vec::new(),
9284 );
9285 questions.put(&mut q).unwrap();
9286 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
9287 .unwrap();
9288 questions.put(&mut q).unwrap();
9289
9290 let mut blocked = task();
9291 blocked.id = "20260101-000001-main".to_owned();
9292 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
9293 queue.put(&mut blocked).unwrap();
9294
9295 resolve_blockers(&queue, &questions);
9296
9297 let after = queue.get(&blocked.id).unwrap();
9298 assert_eq!(
9299 after.status,
9300 TaskStatus::Queued,
9301 "the only blocker resolved"
9302 );
9303 assert_eq!(after.answers.len(), 1);
9304 assert_eq!(after.answers[0].question, "Which backend?");
9305 assert_eq!(after.answers[0].answer, "SQLite");
9306
9307 let instruction = instruction_for(&after);
9309 assert!(instruction.contains("Which backend?"));
9310 assert!(instruction.contains("SQLite"));
9311 }
9312
9313 #[test]
9314 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
9315 let dir = tempfile::tempdir().unwrap();
9316 let queue = Queue::at(dir.path().join("queue"));
9317 let questions = ask::Questions::at(dir.path().join("questions"));
9318
9319 let mut q = crate::ask::Question::new(
9320 "20260101-000001-main".to_owned(),
9321 crate::conduct::NODE.to_owned(),
9322 "conduct".to_owned(),
9323 "Is the setup done?".to_owned(),
9324 String::new(),
9325 Vec::new(),
9326 );
9327 q.abandon("no answer within 60s of asking");
9328 questions.put(&mut q).unwrap();
9329
9330 let mut blocked = task();
9331 blocked.id = "20260101-000001-main".to_owned();
9332 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
9333 queue.put(&mut blocked).unwrap();
9334
9335 resolve_blockers(&queue, &questions);
9336
9337 let after = queue.get(&blocked.id).unwrap();
9338 assert_eq!(
9339 after.status,
9340 TaskStatus::Held,
9341 "never left blocked on nothing"
9342 );
9343 assert!(!after.operator_held(), "a machine hold, for triage");
9344 assert!(
9345 after
9346 .hold_reason
9347 .as_deref()
9348 .unwrap_or_default()
9349 .contains("went unanswered")
9350 );
9351 }
9352
9353 #[test]
9354 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
9355 let dir = tempfile::tempdir().unwrap();
9361 let queue = Queue::at(dir.path().join("queue"));
9362 let questions = ask::Questions::at(dir.path().join("questions"));
9363
9364 let mut q = crate::ask::Question::new(
9365 "20260101-000001-main".to_owned(),
9366 crate::conduct::NODE.to_owned(),
9367 "conduct".to_owned(),
9368 "How should this be handled?".to_owned(),
9369 String::new(),
9370 Vec::new(),
9371 );
9372 questions.put(&mut q).unwrap();
9373 q.answer(crate::ask::Answer::Text(
9374 "leave it held, a human will look at it later".to_owned(),
9375 ))
9376 .unwrap();
9377 questions.put(&mut q).unwrap();
9378
9379 let mut held = task();
9380 held.id = "20260101-000001-main".to_owned();
9381 held.hold_machine(Some("out of attempts".to_owned()));
9382 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
9383 queue.put(&mut held).unwrap();
9384
9385 resolve_blockers(&queue, &questions);
9386
9387 let after = queue.get(&held.id).unwrap();
9388 assert_eq!(after.status, TaskStatus::Held);
9389 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
9390 assert_eq!(
9391 after.answers[0].answer,
9392 "leave it held, a human will look at it later"
9393 );
9394 }
9395
9396 #[test]
9397 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
9398 let dir = tempfile::tempdir().unwrap();
9399 let queue = Queue::at(dir.path().join("queue"));
9400 let questions = ask::Questions::at(dir.path().join("questions"));
9401 let mut dep = task();
9402 dep.id = "20260101-000001-gone".to_owned();
9403 queue.put(&mut dep).unwrap();
9404 let mut blocked = task();
9405 blocked.id = "20260101-000003-main".to_owned();
9406 blocked.block(vec![dep.id.clone()], None);
9407 queue.put(&mut blocked).unwrap();
9408
9409 let claim = queue.claim(&blocked.id).unwrap();
9411 queue.remove(&dep.id, false, &questions).unwrap();
9412 drop(claim);
9413 resolve_blockers(&queue, &questions);
9414
9415 let after = queue.get(&blocked.id).unwrap();
9416 assert_eq!(after.status, TaskStatus::Queued);
9417 assert!(after.blocked_by.is_empty());
9418 }
9419
9420 #[test]
9421 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
9422 let dir = tempfile::tempdir().unwrap();
9428 let queue = Queue::at(dir.path().join("queue"));
9429 let questions = ask::Questions::at(dir.path().join("questions"));
9430
9431 let mut still_going = task();
9432 still_going.id = "20260101-000002-dep1".to_owned();
9433 queue.put(&mut still_going).unwrap();
9434
9435 let mut blocked = task();
9436 blocked.id = "20260101-000003-main".to_owned();
9437 blocked.block(
9438 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
9439 Some("waits on both".to_owned()),
9440 );
9441 queue.put(&mut blocked).unwrap();
9442
9443 resolve_blockers(&queue, &questions);
9444
9445 let after = queue.get(&blocked.id).unwrap();
9446 assert_eq!(
9447 after.status,
9448 TaskStatus::Held,
9449 "a missing dependency must not leave the task blocked forever"
9450 );
9451 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
9452 assert!(after.blocked_by.is_empty());
9453 let reason = after.hold_reason.as_deref().unwrap_or_default();
9454 assert!(
9455 reason.contains("20260101-000001-gone"),
9456 "the missing id must be named so an operator can tell what happened: {reason}"
9457 );
9458 assert!(
9459 reason.contains(&still_going.id),
9460 "the still-valid dependency must not silently vanish from the record: {reason}"
9461 );
9462 }
9463
9464 #[test]
9465 fn instruction_for_is_unchanged_without_any_answers() {
9466 let t = task();
9467 assert_eq!(instruction_for(&t), t.instruction);
9468 }
9469
9470 #[test]
9471 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
9472 let dir = tempfile::tempdir().unwrap();
9473 let q = Queue::at(dir.path().join("queue"));
9474 let src = dir.path().join("shot.png");
9475 std::fs::write(&src, "x").unwrap();
9476 let mut t = task();
9477 q.attach(&mut t, &[src]).unwrap();
9478 let paths = task_attachments(&q, &t).unwrap();
9479 assert_eq!(paths.len(), 1);
9480 assert!(paths[0].is_absolute() && paths[0].is_file());
9481 std::fs::remove_file(&paths[0]).unwrap();
9482 let err = task_attachments(&q, &t).unwrap_err().to_string();
9483 assert!(err.contains("shot.png"), "{err}");
9484 }
9485
9486 #[tokio::test]
9487 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
9488 let dir = tempfile::tempdir().unwrap();
9489 let queue_path = dir.path().join("queue");
9490 let queue = Queue::at(queue_path.clone());
9491 let mut t = task();
9492 t.source = Source::Agent {
9493 run: "originating-chat".into(),
9494 node: crate::queue::CHAT_NODE.into(),
9495 };
9496 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
9497 stage: "config-fetch",
9498 target: "origin/main".into(),
9499 });
9500 record_config_failure(&mut t, &fetch);
9501 t.hold_machine(Some(format!("{fetch:#}")));
9502 let fetch_key = t.chat_notice_due().unwrap();
9503 t.chat_notices.push(fetch_key.clone());
9504 queue.put(&mut t).unwrap();
9505
9506 let queue = Queue::at(queue_path);
9507 let mut t = queue.get(&t.id).unwrap();
9508 let a = dir.path().join("a.toml");
9509 let b = dir.path().join("b.toml");
9510 for (path, text, new_cause) in [
9511 (&a, "[graph", true),
9512 (&a, "[verify", false),
9513 (&a, "[graph]\nimplementers = 'invalid-type'", true),
9514 (&b, "[graph]\nimplementers = 'invalid-type'", true),
9515 ] {
9516 t.release_operator("test", None);
9517 std::fs::write(path, text).unwrap();
9518 let error = Config::discover_fetched(dir.path(), Some(path))
9519 .await
9520 .unwrap_err();
9521 record_config_failure(&mut t, &error);
9522 t.hold_machine(Some(format!("{error:#}")));
9523 let due = t.chat_notice_due();
9524 assert_eq!(due.is_some(), new_cause, "{error:#}");
9525 if let Some(key) = due {
9526 assert_ne!(key, fetch_key);
9527 t.chat_notices.push(key);
9528 }
9529 }
9530 t.succeed();
9531 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
9532 }
9533
9534 #[test]
9535 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
9536 let dir = tempfile::tempdir().unwrap();
9537 let queue_path = dir.path().join("queue");
9538 let queue = Queue::at(queue_path.clone());
9539 let mut t = task();
9540 t.source = crate::queue::Source::Agent {
9541 run: "originating-chat".into(),
9542 node: crate::queue::CHAT_NODE.into(),
9543 };
9544 let a = dir.path().join("a.png");
9545 let b = dir.path().join("b.png");
9546 std::fs::write(&a, "a").unwrap();
9547 std::fs::write(&b, "b").unwrap();
9548 queue.attach(&mut t, &[a, b]).unwrap();
9549 let paths = task_attachments(&queue, &t).unwrap();
9550 for path in &paths {
9551 std::fs::remove_file(path).unwrap();
9552 }
9553 let missing = task_attachments(&queue, &t).unwrap_err();
9554 t.attempts += 1;
9555 let reason = missing.to_string();
9556 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9557 let first = t.chat_notice_due().expect("first attachment notice");
9558 t.chat_notices.push(first.clone());
9559 queue.put(&mut t).unwrap();
9560
9561 let queue = Queue::at(queue_path.clone());
9562 let mut t = queue.get(&t.id).unwrap();
9563 assert!(t.chat_notice_due().is_none(), "restart does not replay");
9564 t.release_operator("test", None);
9565 t.attempts += 1;
9566 let missing = task_attachments(&queue, &t).unwrap_err();
9567 t.fail_startup_target(
9568 "startup-attachments",
9569 missing.name,
9570 "same missing attachment with regenerated wording",
9571 1,
9572 );
9573 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
9574
9575 std::fs::write(&paths[0], "restored").unwrap();
9576 t.release_operator("test", None);
9577 t.attempts += 1;
9578 let missing = task_attachments(&queue, &t).unwrap_err();
9579 let reason = missing.to_string();
9580 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9581 let second = t
9582 .chat_notice_due()
9583 .expect("different attachment must notify");
9584 assert_ne!(first, second);
9585 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
9586 }
9587
9588 #[test]
9589 fn resumed_instruction_is_unchanged_without_any_answers() {
9590 let t = task();
9591 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
9592 }
9593
9594 #[test]
9595 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
9596 let mut t = task();
9597 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9598 let old = t.instruction.clone();
9602
9603 let refreshed = resumed_instruction(&old, &t);
9604 assert!(refreshed.starts_with(&old), "the original text is kept");
9605 assert!(refreshed.contains("Which backend?"));
9606 assert!(refreshed.contains("SQLite"));
9607 }
9608
9609 #[test]
9610 fn resumed_instruction_keeps_an_original_answers_heading() {
9611 let mut t = task();
9612 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
9613 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9614
9615 let refreshed = resumed_instruction(&t.instruction, &t);
9616
9617 assert!(
9618 refreshed.starts_with(&t.instruction),
9619 "an answers heading in the original instruction is not the appended block"
9620 );
9621 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
9622 assert!(refreshed.contains("Which backend?"));
9623 assert!(refreshed.contains("SQLite"));
9624
9625 let repeated = resumed_instruction(&refreshed, &t);
9626 assert_eq!(
9627 repeated, refreshed,
9628 "only the final appended block is refreshed"
9629 );
9630 }
9631
9632 #[test]
9633 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
9634 let mut t = task();
9635 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9636
9637 let once = resumed_instruction(&t.instruction, &t);
9641 let twice = resumed_instruction(&once, &t);
9642 assert_eq!(once, twice);
9643 assert_eq!(once.matches("Which backend?").count(), 1);
9644
9645 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
9647 let refreshed = resumed_instruction(&once, &t);
9648 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
9649 assert!(refreshed.contains("Which backend?"));
9650 assert!(refreshed.contains("Which cache?"));
9651 }
9652
9653 #[test]
9654 fn prepare_instruction_covers_all_three_starters() {
9655 let mut t = task();
9656 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9657
9658 assert_eq!(
9661 prepare_instruction(&Starter::Start, None, &t),
9662 Some(instruction_for(&t))
9663 );
9664
9665 let old = t.instruction.clone();
9668 assert_eq!(
9669 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
9670 Some(resumed_instruction(&old, &t))
9671 );
9672
9673 assert_eq!(
9677 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
9678 None
9679 );
9680 }
9681
9682 #[test]
9683 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
9684 assert_eq!(
9685 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
9686 Starter::Review("magi/eba2/A".to_owned())
9687 );
9688 }
9689
9690 #[test]
9691 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
9692 assert_eq!(
9693 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
9694 Starter::Start,
9695 "a vanished review branch must not fall back to resuming the old run either"
9696 );
9697 }
9698
9699 #[test]
9700 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
9701 let mut t = task();
9702 t.start("old-run".to_owned());
9703 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
9704 t.release();
9705 let branch = t.review_branch.take();
9706 assert_eq!(
9707 choose_starter(branch.as_deref(), true, Some("old-run")),
9708 Starter::Review("magi/eba2/A".to_owned()),
9709 "a review wins over resuming the old run"
9710 );
9711 }
9712
9713 #[test]
9714 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
9715 assert_eq!(
9716 choose_starter(None, false, Some("some-run")),
9717 Starter::Resume("some-run".to_owned())
9718 );
9719 assert_eq!(choose_starter(None, false, None), Starter::Start);
9720 }
9721
9722 #[test]
9723 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
9724 let mut released = task();
9725 released.start("stalled-run".to_owned());
9726 released.requeue();
9727 let unfinished = (!released.fresh_start)
9728 .then(|| Some("stalled-run".to_owned()))
9729 .flatten();
9730 assert_eq!(
9731 choose_starter(None, false, unfinished.as_deref()),
9732 Starter::Start,
9733 "release keeps run history but must not resume it"
9734 );
9735 assert_eq!(released.runs, ["stalled-run"]);
9736 }
9737
9738 #[test]
9739 fn an_ordinary_release_keeps_a_resumable_run_available() {
9740 let mut released = task();
9741 released.start("stalled-run".to_owned());
9742 released.release();
9743 let unfinished = (!released.fresh_start)
9744 .then(|| Some("stalled-run".to_owned()))
9745 .flatten();
9746 assert_eq!(
9747 choose_starter(None, false, unfinished.as_deref()),
9748 Starter::Resume("stalled-run".to_owned()),
9749 "manual release must preserve the normal resume path"
9750 );
9751 }
9752
9753 #[test]
9754 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
9755 let mut state = run_state(RunStatus::Blocked);
9756 state.config.graph.review_rounds = 3;
9757 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
9758 assert!(exhausted_review_budget(&state));
9759
9760 state.reviews.pop();
9762 assert!(!exhausted_review_budget(&state));
9763
9764 let mut stalled = run_state(RunStatus::Stalled);
9767 stalled.config.graph.review_rounds = 1;
9768 stalled.reviews = vec![review_round(1)];
9769 assert!(!exhausted_review_budget(&stalled));
9770 }
9771
9772 fn review_round(round: usize) -> crate::run::ReviewRound {
9773 crate::run::ReviewRound {
9774 round,
9775 head: "deadbeef".to_owned(),
9776 verified_head: None,
9777 verified_at: None,
9778 reviews: Vec::new(),
9779 e2e: Vec::new(),
9780 verify_retried: false,
9781 e2e_deferred: false,
9782 e2e_defer_reason: None,
9783 fix: None,
9784 blocking: 0,
9785 answered: 1,
9786 expected: 1,
9787 clean: false,
9788 progressed: true,
9789 vote_split: false,
9790 reconsideration: Vec::new(),
9791 verdict: None,
9792 }
9793 }
9794
9795 #[test]
9796 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
9797 let mut run = RunState::new(
9798 PathBuf::from("/repo"),
9799 "main".to_owned(),
9800 "abc1234def".to_owned(),
9801 "add retries".to_owned(),
9802 Config::default(),
9803 );
9804 run.status = RunStatus::Judging;
9805 run.parked = true;
9806 let mut task = Task::new(
9807 "add retries".to_owned(),
9808 "add retries".to_owned(),
9809 PathBuf::from("/repo"),
9810 crate::queue::Source::Human,
9811 );
9812 task.status = TaskStatus::Failed;
9813 task.runs = vec![run.id.clone()];
9814 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
9815 assert!(with(&task, &run), "parked after judging is the case");
9816 let mut parked_task = task.clone();
9817 parked_task.status = TaskStatus::Parked;
9818 assert!(with(&parked_task, &run), "the Parked status is the case");
9819
9820 let mut not_parked = run.clone();
9821 not_parked.parked = false;
9822 not_parked.status = RunStatus::Stalled;
9823 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
9824
9825 let mut fresh = task.clone();
9826 fresh.fresh_start = true;
9827 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
9828
9829 let mut review = task.clone();
9830 review.review_branch = Some("magi/x/A".to_owned());
9831 assert!(!with(&review, &run), "review is ranked before resume");
9832
9833 let mut held = task.clone();
9834 held.status = TaskStatus::Held;
9835 assert!(!with(&held, &run), "a hold stays visible to the conductor");
9836
9837 let mut released = run.clone();
9838 released.released_to = Some("20260901-000000-new1".to_owned());
9839 assert!(!with(&task, &released), "nothing left to resume into");
9840
9841 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
9842 "unreadable"
9843 )));
9844 }
9845
9846 #[test]
9847 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
9848 let mut released = RunState::new(
9849 PathBuf::from("/repo"),
9850 "main".to_owned(),
9851 "abc1234def".to_owned(),
9852 "add retries".to_owned(),
9853 Config::default(),
9854 );
9855 released.status = RunStatus::Blocked;
9856 assert_eq!(
9857 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9858 Some(released.id.clone())
9859 );
9860 released.released_to = Some("20260901-000000-new1".to_owned());
9861 assert_eq!(
9862 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9863 None,
9864 "there is nothing left to resume it into"
9865 );
9866 }
9867
9868 #[test]
9869 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
9870 let mut exhausted = RunState::new(
9879 PathBuf::from("/repo"),
9880 "main".to_owned(),
9881 "abc1234def".to_owned(),
9882 "add retries".to_owned(),
9883 Config::default(),
9884 );
9885 exhausted.status = RunStatus::Blocked;
9886 exhausted.config.graph.review_rounds = 1;
9887 exhausted.reviews = vec![review_round(1)];
9888
9889 assert_eq!(
9890 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
9891 None,
9892 "an exhausted `Blocked` run must not be offered as resumable"
9893 );
9894
9895 let mut has_budget_left = RunState::new(
9898 PathBuf::from("/repo"),
9899 "main".to_owned(),
9900 "abc1234def".to_owned(),
9901 "add retries".to_owned(),
9902 Config::default(),
9903 );
9904 has_budget_left.status = RunStatus::Blocked;
9905 has_budget_left.config.graph.review_rounds = 3;
9906 has_budget_left.reviews = vec![review_round(1)];
9907
9908 assert_eq!(
9909 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
9910 Ok(has_budget_left.clone())
9911 }),
9912 Some(has_budget_left.id.clone())
9913 );
9914 }
9915
9916 #[test]
9917 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
9918 let mut run = RunState::new(
9919 PathBuf::from("/repo"),
9920 "main".to_owned(),
9921 "abc1234def".to_owned(),
9922 "add retries".to_owned(),
9923 Config::default(),
9924 );
9925 run.status = RunStatus::Blocked;
9926 run.config.graph.review_rounds = 1;
9927 run.reviews = vec![review_round(1)];
9928 run.pr_publication_pending = true;
9929 assert_eq!(
9930 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
9931 Some(run.id.clone()),
9932 "a spent review budget must not turn a failed publication into a re-competition"
9933 );
9934 }
9935
9936 #[test]
9937 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
9938 let mut older_stalled = RunState::new(
9946 PathBuf::from("/repo"),
9947 "main".to_owned(),
9948 "abc1234def".to_owned(),
9949 "add retries".to_owned(),
9950 Config::default(),
9951 );
9952 older_stalled.status = RunStatus::Stalled;
9953
9954 let mut newest_exhausted = RunState::new(
9955 PathBuf::from("/repo"),
9956 "main".to_owned(),
9957 "abc1234def".to_owned(),
9958 "add retries".to_owned(),
9959 Config::default(),
9960 );
9961 newest_exhausted.status = RunStatus::Blocked;
9962 newest_exhausted.config.graph.review_rounds = 1;
9963 newest_exhausted.reviews = vec![review_round(1)];
9964
9965 assert_eq!(
9966 unfinished_run_with(
9967 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
9968 "t",
9969 |_| Ok(newest_exhausted.clone())
9970 ),
9971 None,
9972 "the newest run is exhausted, so nothing here is worth resuming - \
9973 least of all the older, already-superseded run"
9974 );
9975 }
9976
9977 #[test]
9978 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
9979 assert_eq!(
9980 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
9981 Err(anyhow::anyhow!("fixture is absent"))
9982 }),
9983 None
9984 );
9985 }
9986
9987 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
9988 let mut q = ask::Question::new(
9989 run.to_owned(),
9990 "implement".to_owned(),
9991 "impl-A".to_owned(),
9992 "continue?".to_owned(),
9993 String::new(),
9994 vec!["resume で続行する".to_owned(), "other".to_owned()],
9995 );
9996 q.actions.insert("resume で続行する".to_owned(), action);
9997 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
9998 .unwrap();
9999 q
10000 }
10001
10002 fn held_task_with(run: &str) -> Task {
10003 let mut t = task();
10004 t.runs = vec![run.to_owned()];
10005 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
10006 t
10007 }
10008
10009 fn resume_action(run: &str) -> ask::ChoiceAction {
10010 ask::ChoiceAction::Resume { run: run.into() }
10011 }
10012
10013 #[test]
10014 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
10015 for parked in [false, true] {
10016 let mut t = task();
10017 t.start("A".into());
10018 t.start("B".into());
10019 assert_eq!(t.runs, vec!["A", "B"]);
10020 t.start("A".into());
10021 let mut resumed = run_state(RunStatus::Blocked);
10022 resumed.id = "A".into();
10023 resumed.parked = parked;
10024 resumed.driver_exited = true;
10025 t.hold_machine(Some("resumed run stopped".into()));
10026 let q = action_question("A", resume_action("A"));
10027 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
10028 assert_eq!(
10029 decide_action(&t, &q, &PHRASES_EN, |id| {
10030 assert_eq!(id, "A");
10031 Ok(resumed.clone())
10032 }),
10033 ActionDecision::Resume("A".into())
10034 );
10035 assert_eq!(
10036 action_standing(&t, &action_question("B", resume_action("B"))),
10037 ActionStanding::Stale
10038 );
10039 t.release_operator("cli", None);
10040 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
10041 assert_eq!(id, "A");
10042 Ok(resumed.clone())
10043 });
10044 assert_eq!(
10045 choose_starter(None, false, unfinished.as_deref()),
10046 Starter::Resume("A".into())
10047 );
10048 assert_eq!(t.runs, vec!["B", "A"]);
10049 }
10050 }
10051
10052 #[test]
10053 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
10054 let mut state = crate::land::tests::repair_state(true, true);
10055 state.status = RunStatus::Landing;
10056 state.parked = true;
10057 state.driver_exited = true;
10058 let mut q = ask::Question::new(
10059 state.id.clone(),
10060 land::APPROVAL_NODE.into(),
10061 "land".into(),
10062 "Merge?".into(),
10063 String::new(),
10064 vec![land::APPROVE.into(), land::HOLD.into()],
10065 );
10066 q.id = state.id.clone();
10067 state.land_approval = Some(crate::run::LandApproval {
10068 question: q.id.clone(),
10069 head: "current-head".into(),
10070 });
10071 assert!(land::upgrade_repair_choice(&mut q, &state));
10072 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10073 let mut task = held_task_with(&state.id);
10074 task.status = TaskStatus::Parked;
10075 assert_eq!(
10076 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10077 ActionDecision::Resume(state.id.clone())
10078 );
10079 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10080 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
10081 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
10082 assert_eq!(
10083 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10084 ActionDecision::Resume(state.id.clone()),
10085 "a repair answer newer than the manual hold resumes the existing owner"
10086 );
10087 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10088 for answered_at in [q.answered_at, None] {
10089 let mut older = q.clone();
10090 older.answered_at = answered_at;
10091 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
10092 panic!("a later manual hold must prevent dispatch")
10093 });
10094 let ActionDecision::Refuse(reason) = refused else {
10095 panic!("expected an actionable refusal, got {refused:?}");
10096 };
10097 assert!(reason.contains("later operator decision"));
10098 assert!(reason.contains(&format!("magi task release {}", task.id)));
10099 }
10100 task.operator_decided_at = None;
10101 let mut superseded = state.clone();
10102 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
10103 assert_eq!(
10104 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
10105 ActionDecision::Stale
10106 );
10107 for case in 1..4 {
10108 let mut changed = state.clone();
10109 match case {
10110 1 => changed.status = RunStatus::Blocked,
10111 2 => changed.released_to = Some("new-owner".into()),
10112 _ => changed.land_approval.as_mut().unwrap().head.clear(),
10113 }
10114 assert!(matches!(
10115 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
10116 ActionDecision::Refuse(_)
10117 ));
10118 }
10119 assert!(matches!(
10120 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
10121 ActionDecision::Refuse(_)
10122 ));
10123 task.runs.push("replacement".into());
10124 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10125 assert_eq!(
10126 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
10127 "old owner must not load"
10128 )),
10129 ActionDecision::Stale
10130 );
10131 }
10132
10133 #[test]
10134 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
10135 let mut state = crate::land::tests::repair_state(true, true);
10136 state.status = RunStatus::Landing;
10137 state.parked = true;
10138 state.driver_exited = true;
10139 let mut q = ask::Question::new(
10140 state.id.clone(),
10141 land::APPROVAL_NODE.into(),
10142 "land".into(),
10143 "Merge?".into(),
10144 String::new(),
10145 vec![land::APPROVE.into(), land::HOLD.into()],
10146 );
10147 q.id = state.id.clone();
10148 state.land_approval = Some(crate::run::LandApproval {
10149 question: q.id.clone(),
10150 head: "current-head".into(),
10151 });
10152 state.save().unwrap();
10153 land::upgrade_repair_choice(&mut q, &state);
10154 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10155 Questions::open().put(&mut q).unwrap();
10156 let tmp = tempfile::tempdir().unwrap();
10157 let queue = Queue::at(tmp.path().join("queue"));
10158 let mut task = held_task_with(&state.id);
10159 task.status = TaskStatus::Parked;
10160 queue.put(&mut task).unwrap();
10161 let empty = Arc::new(tokio::sync::Semaphore::new(0));
10162 let free = Arc::new(tokio::sync::Semaphore::new(1));
10163 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
10164 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
10165 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
10166 assert!(claim.permit.is_some());
10167 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
10168 drop(claim);
10169 Questions::open()
10170 .update(&q.id, |q| {
10171 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
10172 Ok(())
10173 })
10174 .unwrap();
10175 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
10176 assert!(claim.permit.is_none());
10177 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
10178 }
10179
10180 #[test]
10181 fn decide_action_resumes_only_the_latest_resumable_run() {
10182 let t = held_task_with("r1");
10183 let q = action_question("r1", resume_action("r1"));
10184 let load = |s: RunState| move |_: &str| Ok(s);
10185 assert_eq!(
10186 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
10187 ActionDecision::Resume("r1".into())
10188 );
10189 let q_other = action_question("r1", resume_action("r0"));
10191 assert!(matches!(
10192 decide_action(
10193 &t,
10194 &q_other,
10195 &PHRASES_EN,
10196 load(run_state(RunStatus::Blocked))
10197 ),
10198 ActionDecision::Refuse(_)
10199 ));
10200 assert!(matches!(
10202 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
10203 ActionDecision::Refuse(_)
10204 ));
10205 let mut released = run_state(RunStatus::Blocked);
10207 released.released_to = Some("elsewhere".into());
10208 assert!(matches!(
10209 decide_action(&t, &q, &PHRASES_EN, load(released)),
10210 ActionDecision::Refuse(_)
10211 ));
10212 assert!(matches!(
10214 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
10215 ActionDecision::Refuse(_)
10216 ));
10217 }
10218
10219 #[test]
10220 fn decide_action_ignores_a_question_about_an_earlier_run() {
10221 let mut t = held_task_with("r1");
10222 t.runs.push("r2".to_owned());
10223 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10224 assert_eq!(
10225 decide_action(
10226 &t,
10227 &action_question("r1", ask::ChoiceAction::Done),
10228 &PHRASES_EN,
10229 never
10230 ),
10231 ActionDecision::Stale
10232 );
10233 }
10234
10235 #[test]
10236 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
10237 let mut t = held_task_with("r1");
10238 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10239 assert_eq!(
10240 decide_action(
10241 &t,
10242 &action_question("r1", ask::ChoiceAction::Requeue),
10243 &PHRASES_EN,
10244 never
10245 ),
10246 ActionDecision::Requeue
10247 );
10248 let done_q = action_question("r1", ask::ChoiceAction::Done);
10249 assert_eq!(
10250 decide_action(&t, &done_q, &PHRASES_EN, never),
10251 ActionDecision::Done
10252 );
10253 t.mark_action_applied(&done_q.id);
10254 assert_eq!(
10255 decide_action(&t, &done_q, &PHRASES_EN, never),
10256 ActionDecision::Skip
10257 );
10258
10259 let mut plain = action_question("r1", ask::ChoiceAction::Done);
10261 plain.actions.clear();
10262 assert_eq!(
10263 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
10264 ActionDecision::Skip
10265 );
10266 let mut running = held_task_with("r1");
10268 running.status = TaskStatus::Running;
10269 assert_eq!(
10270 decide_action(
10271 &running,
10272 &action_question("r1", ask::ChoiceAction::Done),
10273 &PHRASES_EN,
10274 never
10275 ),
10276 ActionDecision::Skip
10277 );
10278 }
10279
10280 #[test]
10281 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
10282 crate::run::pin_test_home();
10283 for legacy in [false, true] {
10284 let dir = tempfile::tempdir().unwrap();
10285 let queue = Queue::at(dir.path().join("queue"));
10286 let questions = Questions::at(dir.path().join("questions"));
10287 let run = "20260101-000000-hold";
10288 let mut task = held_task_with(run);
10289 task.status = TaskStatus::Parked;
10290 let mut q = ask::Question::new(
10291 run.into(),
10292 land::APPROVAL_NODE.into(),
10293 "land".into(),
10294 "Repair?".into(),
10295 String::new(),
10296 vec![land::REPAIR.into()],
10297 );
10298 q.actions.insert(land::REPAIR.into(), resume_action(run));
10299 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10300 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10302 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
10303 questions.put(&mut q).unwrap();
10304 task.hold_manual(Some("Keep this PR paused".into()));
10305 if legacy {
10306 task.hold_source = None;
10307 }
10308 queue.put(&mut task).unwrap();
10309
10310 apply_choice_actions(&queue, &questions, dir.path());
10311 let after = queue.get(&task.id).unwrap();
10312 assert!(after.operator_held());
10313 assert_eq!(after.hold_source, task.hold_source);
10314 assert_eq!(after.operator_decided_at, task.operator_decided_at);
10315 assert_eq!(after.runs, task.runs);
10316 assert_eq!(after.instruction, task.instruction);
10317 assert!(after.resume_override.is_none());
10318 assert!(after.action_applied(&q.id));
10319 let reason = after.hold_reason.as_deref().unwrap();
10320 assert!(reason.starts_with("Keep this PR paused"));
10321 assert!(reason.contains("later operator decision"));
10322 assert!(reason.contains(&format!("magi task release {}", task.id)));
10323
10324 for recovery in [
10325 crate::conduct::Recovery::Review,
10326 crate::conduct::Recovery::Requeue,
10327 ] {
10328 crate::conduct::apply(
10329 &queue,
10330 &questions,
10331 &crate::conduct::Verdict {
10332 decisions: vec![crate::conduct::Decision {
10333 id: task.id.clone(),
10334 recovery: Some(recovery),
10335 ..Default::default()
10336 }],
10337 },
10338 )
10339 .unwrap();
10340 }
10341 apply_choice_actions(&queue, &questions, dir.path());
10344 let still_held = queue.get(&task.id).unwrap();
10345 assert!(still_held.operator_held());
10346 assert_eq!(still_held.hold_reason, after.hold_reason);
10347 assert_eq!(still_held.runs, task.runs);
10348 assert!(still_held.review_branch.is_none());
10349 assert!(!still_held.fresh_start);
10350 assert!(queue.next_runnable().is_none());
10351 }
10352 }
10353
10354 #[test]
10355 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
10356 let dir = tempfile::tempdir().unwrap();
10357 let queue = Queue::at(dir.path().join("queue"));
10358 let questions = Questions::at(dir.path().join("questions"));
10359 let home = dir.path().join("home");
10360 let mut state = run_state(RunStatus::Blocked);
10361 state.id = "20260101-000000-act1".to_owned();
10362 state.save_under(&home).unwrap();
10363
10364 let mut t = held_task_with(&state.id);
10365 queue.put(&mut t).unwrap();
10366 let mut q = action_question(&state.id, resume_action(&state.id));
10367 questions.put(&mut q).unwrap();
10368
10369 apply_choice_actions(&queue, &questions, &home);
10370 let after = queue.get(&t.id).unwrap();
10371 assert_eq!(after.status, TaskStatus::Queued);
10372 assert!(!after.fresh_start);
10373 assert!(after.action_applied(&q.id));
10374 let pin = after.resume_override.clone().unwrap();
10375 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
10376 assert!(pin.forced);
10377
10378 let mut again = queue.get(&t.id).unwrap();
10380 again.hold_machine(Some("later".into()));
10381 queue.put(&mut again).unwrap();
10382 apply_choice_actions(&queue, &questions, &home);
10383 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10384 }
10385
10386 #[test]
10387 fn a_delivered_answer_is_still_acted_on_exactly_once() {
10388 let dir = tempfile::tempdir().unwrap();
10389 let queue = Queue::at(dir.path().join("queue"));
10390 let questions = Questions::at(dir.path().join("questions"));
10391 let home = dir.path().join("home");
10392 let mut state = run_state(RunStatus::Blocked);
10393 state.id = "20260101-000000-act2".to_owned();
10394 state.save_under(&home).unwrap();
10395
10396 let mut t = held_task_with(&state.id);
10397 queue.put(&mut t).unwrap();
10398 let mut q = action_question(&state.id, resume_action(&state.id));
10399 q.answer_delivered = true;
10400 questions.put(&mut q).unwrap();
10401
10402 apply_choice_actions(&queue, &questions, &home);
10405 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10406 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
10407
10408 let mut again = queue.get(&t.id).unwrap();
10410 again.hold_machine(Some("later".into()));
10411 queue.put(&mut again).unwrap();
10412 apply_choice_actions(&queue, &questions, &home);
10413 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10414 }
10415
10416 #[test]
10417 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
10418 let dir = tempfile::tempdir().unwrap();
10419 let queue = Queue::at(dir.path().join("queue"));
10420 let questions = Questions::at(dir.path().join("questions"));
10421 let home = dir.path().join("home");
10422 let mut state = run_state(RunStatus::Blocked);
10423 state.id = "20260101-000000-act3".to_owned();
10424 state.save_under(&home).unwrap();
10425 let mut t = held_task_with(&state.id);
10426 queue.put(&mut t).unwrap();
10427 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
10428 questions.put(&mut q).unwrap();
10429
10430 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
10431 apply_choice_actions(&queue, &questions, &home);
10432 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10433
10434 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
10435 apply_choice_actions(&queue, &questions, &home);
10436 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10437 }
10438
10439 #[test]
10440 fn action_standing_tells_the_reasons_apart() {
10441 let mut t = held_task_with("r1");
10442 let q = action_question("r1", ask::ChoiceAction::Requeue);
10443 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
10444 let mut plain = q.clone();
10445 plain.actions.clear();
10446 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
10447
10448 t.mark_action_applied(&q.id);
10450 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
10451
10452 let mut t = held_task_with("r1");
10454 t.start("r2".to_owned());
10455 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
10456 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
10458 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10459 t.status = TaskStatus::Blocked;
10460 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10461
10462 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
10464 c.node = crate::conduct::NODE.to_owned();
10465 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
10466 assert!(!ActionStanding::Busy.daemon_owns());
10467 }
10468
10469 #[test]
10470 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
10471 let dir = tempfile::tempdir().unwrap();
10472 let queue = Queue::at(dir.path().join("queue"));
10473 let questions = Questions::at(dir.path().join("questions"));
10474 let home = dir.path().join("home");
10475 let mut state = run_state(RunStatus::Blocked);
10476 state.id = "20260101-000000-act4".to_owned();
10477 state.save_under(&home).unwrap();
10478 let mut t = held_task_with(&state.id);
10479 t.status = TaskStatus::Running;
10480 queue.put(&mut t).unwrap();
10481 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
10482 q.cwd = Some(dir.path().display().to_string());
10483 questions.put(&mut q).unwrap();
10484
10485 let running = queue.get(&t.id).unwrap();
10488 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
10489 assert!(matches!(
10490 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
10491 crate::waiter::Action::Deliver(_)
10492 ));
10493 apply_choice_actions(&queue, &questions, &home);
10495 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10496 let mut back = queue.get(&t.id).unwrap();
10497 back.hold_machine(Some("later".into()));
10498 queue.put(&mut back).unwrap();
10499 apply_choice_actions(&queue, &questions, &home);
10500 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
10501 let mut again = queue.get(&t.id).unwrap();
10502 again.hold_machine(Some("again".into()));
10503 queue.put(&mut again).unwrap();
10504 apply_choice_actions(&queue, &questions, &home);
10505 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10506 }
10507}