1use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::sync::atomic::{AtomicBool, Ordering};
52use std::sync::{Mutex, MutexGuard};
53use std::time::Duration;
54
55use anyhow::{Context, Result, bail};
56use jiff::Timestamp;
57use serde::{Deserialize, Serialize};
58use tokio::sync::Notify;
59
60use crate::ask::{self, Questions};
61use crate::clean;
62use crate::conduct::Conductor;
63use crate::config::{Config, MergeMode};
64use crate::graph::Runner;
65use crate::land;
66use crate::notices::{self, Link, Notice};
67use crate::queue::{Queue, Task, TaskStatus};
68use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
69use crate::triage;
70
71pub const SCHEMA: u32 = 1;
73
74pub const HEARTBEAT: Duration = Duration::from_secs(5);
78
79pub const STALE_SECS: i64 = 30;
88
89pub const POLL: Duration = Duration::from_secs(5);
91
92pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
96
97pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
119#[serde(default)]
120pub struct Current {
121 pub task: String,
123 pub run: String,
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct Status {
135 pub schema: u32,
137 pub pid: u32,
139 pub started_at: Timestamp,
141 pub updated_at: Timestamp,
143 pub idle: bool,
145 pub current: Vec<Current>,
151 pub completed: usize,
153 pub polls: u64,
155 #[serde(default)]
157 pub question_recovery: Option<crate::question_delivery::Recovery>,
158}
159
160impl Status {
161 #[must_use]
163 pub fn new() -> Self {
164 let now = Timestamp::now();
165 Self {
166 schema: SCHEMA,
167 pid: std::process::id(),
168 started_at: now,
169 updated_at: now,
170 idle: true,
171 current: Vec::new(),
172 completed: 0,
173 polls: 0,
174 question_recovery: None,
175 }
176 }
177}
178
179impl Default for Status {
180 fn default() -> Self {
181 Self::new()
182 }
183}
184
185#[derive(Debug, Clone)]
187pub struct Opts {
188 pub repo: PathBuf,
190 pub config: Option<PathBuf>,
192 pub poll: Duration,
194 pub max_attempts: usize,
196 pub once: bool,
198 pub merge: Option<String>,
200 pub worktrees_root: Option<PathBuf>,
209}
210
211impl Default for Opts {
212 fn default() -> Self {
213 Self {
214 repo: PathBuf::from("."),
215 config: None,
216 poll: POLL,
217 max_attempts: 2,
218 once: false,
219 merge: None,
220 worktrees_root: None,
221 }
222 }
223}
224
225fn max_concurrent(n: usize) -> usize {
230 n.max(1)
231}
232
233#[must_use]
235pub fn status_path() -> PathBuf {
236 crate::run::home().join("daemon.json")
237}
238
239pub fn write_status(status: &Status) -> Result<()> {
241 write_status_to(&status_path(), status)
242}
243
244pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
249 if let Some(parent) = path.parent() {
250 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
251 }
252 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
253 let tmp = path.with_extension("json.tmp");
254 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
255 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
256 Ok(())
257}
258
259pub fn clear_status() {
262 clear_status_at(&status_path());
263}
264
265fn clear_status_at(path: &Path) {
269 let _ = std::fs::remove_file(path);
270}
271
272#[derive(Debug, Clone, Default)]
285pub struct Stop {
286 stopped: Arc<AtomicBool>,
290 busy: Arc<std::sync::atomic::AtomicUsize>,
296 wake: Arc<Notify>,
300 pause: crate::graph::Pause,
303}
304
305impl Stop {
306 #[must_use]
308 pub fn new() -> Self {
309 Self::default()
310 }
311
312 pub fn stop(&self) {
315 self.stopped.store(true, Ordering::SeqCst);
316 self.wake.notify_one();
320 }
321
322 #[must_use]
324 pub fn stopped(&self) -> bool {
325 self.stopped.load(Ordering::SeqCst)
326 }
327
328 #[must_use]
336 pub fn finishing(&self) -> bool {
337 self.stopped() && self.busy_now()
338 }
339
340 pub fn park(&self, cause: crate::run::ParkCause) {
351 self.pause.park_for(cause);
352 self.stop();
353 }
354
355 #[must_use]
357 pub fn parking(&self) -> bool {
358 self.pause.parked()
359 }
360
361 #[must_use]
363 pub fn pause(&self) -> crate::graph::Pause {
364 self.pause.clone()
365 }
366
367 #[must_use]
373 pub fn busy_now(&self) -> bool {
374 self.busy.load(Ordering::SeqCst) > 0
375 }
376
377 fn enter(&self) {
379 self.busy.fetch_add(1, Ordering::SeqCst);
380 }
381
382 fn exit(&self) {
385 self.busy.fetch_sub(1, Ordering::SeqCst);
386 }
387
388 async fn idle(&self, poll: Duration) {
390 tokio::select! {
391 () = tokio::time::sleep(poll) => {}
392 () = self.wake.notified() => {}
393 }
394 }
395}
396
397#[derive(Debug, Clone, Default, Deserialize)]
404#[serde(default)]
405pub struct Reading {
406 pub schema: u32,
408 pub pid: Option<u32>,
410 pub started_at: Option<Timestamp>,
412 pub updated_at: Option<Timestamp>,
414 pub idle: bool,
416 #[serde(deserialize_with = "de_current")]
428 pub current: Vec<Current>,
429 pub completed: u64,
431 pub polls: u64,
433 #[serde(default)]
435 pub question_recovery: Option<crate::question_delivery::Recovery>,
436}
437
438fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
441where
442 D: serde::Deserializer<'de>,
443{
444 #[derive(Deserialize)]
445 #[serde(untagged)]
446 enum Shape {
447 Many(Vec<Current>),
448 One(Current),
449 }
450 Ok(
451 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
452 Shape::Many(v) => v,
453 Shape::One(c) => vec![c],
454 }),
455 )
456}
457
458impl Reading {
459 #[must_use]
462 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
463 self.updated_at
464 .map(|at| (now.as_second() - at.as_second()).max(0))
465 }
466
467 #[must_use]
471 pub fn running(&self, now: Timestamp) -> bool {
472 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
473 }
474}
475
476#[must_use]
483pub fn read_status(home: &Path) -> Option<Reading> {
484 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
485 serde_json::from_str(&body).ok()
486}
487
488#[must_use]
500pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
501 read_status(home)
502 .filter(|reading| reading.running(now))
503 .map(|reading| reading.current)
504 .unwrap_or_default()
505}
506
507#[must_use]
509pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
510 current_work(home, now).iter().any(|c| c.run == run)
511}
512
513#[must_use]
523pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
524 current_work(home, now)
525 .iter()
526 .any(|c| crate::run::short_of(&c.run) == short)
527}
528
529#[must_use]
531pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
532 current_work(home, now).iter().any(|c| c.task == task)
533}
534
535pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
579 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
580}
581
582fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
586where
587 F: Fn(u32) -> bool,
588{
589 let this_process = std::process::id();
590 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
591 .into_iter()
592 .flatten()
593 .flatten()
594 .map(|e| e.path())
595 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
596 .filter(|p| {
597 match std::fs::read_to_string(p)
598 .ok()
599 .and_then(|body| body.trim().parse::<u32>().ok())
600 {
601 Some(pid) if pid == this_process => false,
605 Some(pid) => !pid_alive(pid),
606 None => p
607 .metadata()
608 .and_then(|m| m.modified())
609 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
610 .is_ok_and(|age| age >= older_than),
611 }
612 })
613 .filter(|p| std::fs::remove_file(p).is_ok())
614 .filter_map(|p| {
615 p.file_stem()
616 .and_then(|s| s.to_str())
617 .map(std::borrow::ToOwned::to_owned)
618 })
619 .collect();
620 swept.sort_unstable();
621 swept
622}
623
624fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
629 task.status == TaskStatus::Running
630 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
631 && !is_working_on_task(home, &task.id, now)
632}
633
634fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
637 queue
638 .list()
639 .into_iter()
640 .filter(|t| is_stalled(t, home, now))
641 .collect()
642}
643
644fn queued_tasks(queue: &Queue) -> Vec<Task> {
650 queue
651 .list()
652 .into_iter()
653 .filter(|t| t.status == TaskStatus::Queued)
654 .collect()
655}
656
657fn finished_tasks(queue: &Queue) -> Vec<Task> {
660 queue
661 .list()
662 .into_iter()
663 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
664 .collect()
665}
666
667fn resolve_blockers(queue: &Queue, questions: &Questions) {
684 for listed in queue.list() {
685 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
686 continue;
687 }
688 let Ok(_claim) = queue.claim(&listed.id) else {
689 continue;
690 };
691 let Ok(mut task) = queue.get(&listed.id) else {
692 continue;
693 };
694 if task.status != TaskStatus::Blocked {
695 continue;
696 }
697 let deleted = queue.apply_deleted_blockers(&mut task, questions);
701 if !deleted.is_empty() {
702 record(queue, &mut task);
703 for id in &deleted {
704 queue.note_dependency_deleted(&task, id);
705 }
706 if task.status != TaskStatus::Blocked {
707 continue;
708 }
709 }
710 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
711 if !missing.is_empty() {
712 let language = language_of(&task, Path::new("."));
713 let mut cause = missing.clone();
714 cause.sort();
715 cause.dedup();
716 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
717 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
718 &task.blocked_by,
719 &missing,
720 &language,
721 )));
722 record(queue, &mut task);
723 continue;
724 }
725 if let Some(q) = task.blocked_by.iter().find_map(|id| {
728 let crate::queue::Dependency::Question(full) =
729 crate::queue::dependency(queue, questions, id)
730 else {
731 return None;
732 };
733 questions.get(&full).ok().filter(|q| {
734 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
735 })
736 }) {
737 let language = language_of(&task, Path::new("."));
738 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
739 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
740 record(queue, &mut task);
741 continue;
742 }
743 let mut changed = false;
744 for id in task.blocked_by.clone() {
745 let resolved = crate::queue::dependency(queue, questions, &id);
746 if let crate::queue::Dependency::Task(full) = &resolved {
747 if queue
748 .get(full)
749 .is_ok_and(|dep| dep.status == TaskStatus::Done)
750 {
751 task.unblock(&id);
752 changed = true;
753 }
754 continue;
755 }
756 if let crate::queue::Dependency::Question(full) = resolved
757 && let Ok(q) = questions.get(&full)
758 && q.status == ask::QuestionStatus::Answered
759 {
760 let answer = match &q.answer {
761 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
762 None => String::new(),
763 };
764 task.record_answer(q.summary.clone(), answer);
765 task.unblock(&id);
766 changed = true;
767 }
768 }
769 if changed {
770 record(queue, &mut task);
771 }
772 }
773}
774
775fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
777 if crate::lang::is_japanese(language) {
778 format!(
779 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
780 q.short(),
781 q.summary
782 )
783 } else {
784 format!(
785 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
786 q.short(),
787 q.summary
788 )
789 }
790}
791
792#[derive(Debug, Clone, Copy, PartialEq, Eq)]
795pub(crate) enum ActionStanding {
796 NoAction,
798 Pending,
800 Applied,
803 Stale,
805 Busy,
807}
808
809impl ActionStanding {
810 pub(crate) fn daemon_owns(self) -> bool {
812 matches!(self, Self::Pending | Self::Applied | Self::Stale)
813 }
814}
815
816pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
820 if q.chosen_action().is_none() {
821 ActionStanding::NoAction
822 } else if task.action_applied(&q.id) {
823 ActionStanding::Applied
824 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
825 ActionStanding::Stale
826 } else if matches!(
827 task.status,
828 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
829 ) {
830 ActionStanding::Busy
831 } else {
832 ActionStanding::Pending
833 }
834}
835
836#[derive(Debug, Clone, PartialEq, Eq)]
838enum ActionDecision {
839 Skip,
841 Resume(String),
843 Requeue,
845 Done,
847 Stale,
850 Refuse(String),
853}
854
855fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
863where
864 F: FnOnce(&str) -> Result<RunState>,
865{
866 let Some(action) = q.chosen_action() else {
867 return ActionDecision::Skip;
868 };
869 if task.action_applied(&q.id)
870 || matches!(
871 task.status,
872 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
873 )
874 {
875 return ActionDecision::Skip;
876 }
877 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
880 return ActionDecision::Stale;
881 }
882 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
883 let stopping = q.node == land::APPROVAL_NODE
884 && q.resolution()
885 .as_deref()
886 .is_some_and(crate::merge_decision::stops);
887 let decision_at = if repair || stopping {
891 q.answered_at.unwrap_or(q.asked_at)
892 } else {
893 q.asked_at
894 };
895 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
896 return if repair || stopping {
897 ActionDecision::Refuse(format!(
898 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
899 if repair { "repair" } else { "stop" },
900 q.id,
901 q.run,
902 task.id
903 ))
904 } else {
905 ActionDecision::Stale
906 };
907 }
908 match action {
909 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
910 ask::ChoiceAction::Done => ActionDecision::Done,
911 ask::ChoiceAction::Resume { run } => {
912 if task.runs.last() != Some(run) {
913 return ActionDecision::Refuse((p.resume_not_latest)(
914 q.short(),
915 ask::short_id(run),
916 ));
917 }
918 match load(run) {
919 Ok(s) if stopping => {
920 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
921 || !s
922 .land_approval
923 .as_ref()
924 .is_some_and(|binding| binding.question == q.id)
925 {
926 ActionDecision::Stale
927 } else if s.liveness(false) == Liveness::Live {
928 ActionDecision::Skip
929 } else if s.status == RunStatus::Landing && !s.released() {
930 ActionDecision::Resume(run.clone())
931 } else {
932 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
933 }
934 }
935 Ok(s)
936 if q.node == land::APPROVAL_NODE
937 && q.resolution().as_deref() == Some(land::REPAIR) =>
938 {
939 if s.merge_repairs.iter().any(|r| r.question == q.id)
940 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
941 {
942 ActionDecision::Stale
945 } else if s.liveness(false) == Liveness::Live {
946 ActionDecision::Skip
947 } else if s.status != RunStatus::Landing
948 || s.released()
949 || s.reviews.is_empty()
950 || !implementers_match(task, &s)
951 || !s
952 .land_approval
953 .as_ref()
954 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
955 {
956 ActionDecision::Refuse(format!(
957 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
958 q.id, run, run
959 ))
960 } else {
961 ActionDecision::Resume(run.clone())
962 }
963 }
964 Ok(s) if s.status.resumable() && !s.released() && !exhausted_review_budget(&s) => {
965 ActionDecision::Resume(run.clone())
966 }
967 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
968 q.short(),
969 ask::short_id(run),
970 )),
971 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
972 q.short(),
973 ask::short_id(run),
974 &format!("{e:#}"),
975 )),
976 }
977 }
978 }
979}
980
981struct Phrases {
995 graph_stopped: fn(&str, &str) -> String,
997 quorum_lost: &'static str,
998 quota_took_out: &'static str,
1000 run_ended: &'static str,
1002 waiting_for_answer: &'static str,
1004 recovered_running: &'static str,
1005 no_run_to_recover: &'static str,
1006 could_not_start: &'static str,
1007 resume_not_latest: fn(&str, &str) -> String,
1008 resume_cannot_progress: fn(&str, &str) -> String,
1009 resume_unreadable: fn(&str, &str, &str) -> String,
1010 handover_refused: fn(&crate::handover::Refused) -> String,
1013 handover_hint: &'static str,
1015 reviewers_never_answered: fn(usize, usize) -> String,
1018}
1019
1020const PHRASES_EN: Phrases = Phrases {
1021 graph_stopped: |status, detail| {
1022 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1023 },
1024 quorum_lost: "the judging panel lost its quorum",
1025 quota_took_out: "; quota took out ",
1026 run_ended: "run ended ",
1027 waiting_for_answer: " - waiting for operator answer to question ",
1028 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1029 no_run_to_recover: "task was `running` with no live daemon and no readable \
1030 run to recover; held for a human to check what happened",
1031 could_not_start: "could not start the run: ",
1032 resume_not_latest: |q, run| {
1033 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1034 },
1035 resume_cannot_progress: |q, run| {
1036 format!("question {q} asked to resume run {run}, which cannot make progress")
1037 },
1038 resume_unreadable: |q, run, e| {
1039 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1040 },
1041 handover_refused: |r| r.to_string(),
1042 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1043 reviewers_never_answered: |missing, rounds| {
1044 format!(
1045 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1046 refusing to call it clean"
1047 )
1048 },
1049};
1050
1051const PHRASES_JA: Phrases = Phrases {
1052 graph_stopped: |status, detail| {
1053 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1054 },
1055 quorum_lost: "審査パネルが定足数を失いました",
1056 quota_took_out: "。クォータで脱落: ",
1057 run_ended: "run 終了: ",
1058 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1059 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1060 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1061 何が起きたか人が確認するため保留にしました",
1062 could_not_start: "run を開始できませんでした: ",
1063 resume_not_latest: |q, run| {
1064 format!(
1065 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1066 )
1067 },
1068 resume_cannot_progress: |q, run| {
1069 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1070 },
1071 resume_unreadable: |q, run, e| {
1072 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1073 },
1074 handover_refused: |r| {
1075 use crate::handover::Refused;
1076 match r {
1077 Refused::Foreign { branch, path, why } => format!(
1078 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1079 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1080 から、やり直してください(詳細: {why})"
1081 ),
1082 Refused::Unsafe { branch, path, why } => format!(
1083 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1084 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1085 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1086 ),
1087 Refused::ReleaseFailed { branch, path, run } => format!(
1088 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1089 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1090 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1091 ),
1092 }
1093 },
1094 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1095 reviewers_never_answered: |missing, rounds| {
1096 format!(
1097 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1098 クリーンとは認めません"
1099 )
1100 },
1101};
1102
1103fn phrases(language: &str) -> &'static Phrases {
1104 if crate::lang::is_japanese(language) {
1105 &PHRASES_JA
1106 } else {
1107 &PHRASES_EN
1108 }
1109}
1110
1111fn language_of(task: &Task, fallback: &Path) -> String {
1115 crate::lang::of_repo(&repo_for(task, fallback))
1116}
1117
1118pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1122 if q.node == crate::conduct::NODE {
1123 return tasks.iter().find(|t| t.id == q.run);
1124 }
1125 tasks.iter().find(|t| t.runs.contains(&q.run))
1126}
1127
1128pub(crate) fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1137 let tasks = queue.list();
1138 for q in questions.list() {
1139 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1141 continue;
1142 }
1143 if q.chosen_action().is_none() {
1144 continue;
1145 }
1146 let Some(listed) = task_of_question(&tasks, &q) else {
1147 if q.node == land::APPROVAL_NODE
1148 && q.resolution().as_deref().is_some_and(|choice| {
1149 choice == land::REPAIR || crate::merge_decision::stops(choice)
1150 })
1151 {
1152 notices::raise_in(
1153 home,
1154 Notice::warn(
1155 &format!("repair-owner:{}", q.id),
1156 format!(
1157 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1158 q.run, q.run
1159 ),
1160 ),
1161 );
1162 }
1163 continue;
1164 };
1165 if listed.action_applied(&q.id) {
1166 continue;
1167 }
1168 let Ok(_claim) = queue.claim(&listed.id) else {
1169 continue;
1170 };
1171 let Ok(mut task) = queue.get(&listed.id) else {
1172 continue;
1173 };
1174 let language = language_of(&task, Path::new("."));
1175 let decision = decide_action(&task, &q, phrases(&language), |id| {
1176 RunState::load_under(id, home)
1177 });
1178 let ran = matches!(
1179 decision,
1180 ActionDecision::Resume(_) | ActionDecision::Requeue | ActionDecision::Done
1181 );
1182 if ran && asker_may_still_read(questions, &q, home) {
1183 continue;
1192 }
1193 let done = matches!(decision, ActionDecision::Done);
1194 match decision {
1195 ActionDecision::Skip => continue,
1196 ActionDecision::Resume(run) => {
1197 task.release_operator("choice", Some(&q.id));
1198 task.resume_override = Some(crate::queue::OperatorResume {
1199 question_id: q.id.clone(),
1200 at: Timestamp::now(),
1201 conductor_rehold: None,
1202 forced: true,
1203 pinned_run: Some(run),
1204 });
1205 }
1206 ActionDecision::Stale => {}
1207 ActionDecision::Requeue => {
1208 task.release_operator("choice-requeue", Some(&q.id));
1209 task.requeue();
1210 }
1211 ActionDecision::Done => task.succeed(),
1212 ActionDecision::Refuse(why) => {
1213 if task.operator_held() {
1214 task.hold_reason = Some(match task.hold_reason.take() {
1217 Some(reason) => format!("{reason}\n\n{why}"),
1218 None => why,
1219 });
1220 } else {
1221 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1222 task.hold_machine(Some(why));
1223 }
1224 notices::raise(
1225 Notice::warn(
1226 &format!("action:{}", q.id),
1227 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1228 )
1229 .link(Link::Task {
1230 id: task.id.clone(),
1231 }),
1232 );
1233 }
1234 }
1235 task.mark_action_applied(&q.id);
1236 record(queue, &mut task);
1239 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1240 supersede_prior_runs(&task, home);
1241 }
1242 let _ = questions.update(&q.id, |r| {
1245 r.answer_delivered = true;
1246 Ok(())
1247 });
1248 }
1249}
1250
1251fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1255 let now = Timestamp::now();
1256 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1257 return true;
1258 }
1259 q.node != crate::conduct::NODE
1260 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1261 s.seats_active()
1262 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1263 })
1264}
1265
1266fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1277 let tasks = queue.list();
1278 let by_id: std::collections::BTreeMap<&str, &Task> =
1279 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1280 let referenced: Vec<&str> = tasks
1281 .iter()
1282 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1283 .collect();
1284
1285 for mut question in questions.list() {
1286 if !question.status.open() || question.node != crate::conduct::NODE {
1287 continue;
1288 }
1289 if referenced
1294 .iter()
1295 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1296 {
1297 continue;
1298 }
1299 let Some(task) = by_id.get(question.run.as_str()) else {
1300 continue;
1301 };
1302 question.abandon(format!(
1303 "task {} no longer waits for this answer",
1304 task.short()
1305 ));
1306 if let Err(e) = questions.put(&mut question) {
1307 tracing::warn!(
1308 "could not retire question {} for task {}: {e:#}",
1309 question.short(),
1310 task.short()
1311 );
1312 }
1313 }
1314}
1315
1316#[derive(Debug, Clone, Copy)]
1323pub struct Verdict {
1324 pub status: RunStatus,
1326 pub left_pr: bool,
1328 pub quota_hit: bool,
1330 pub parked: bool,
1332 pub no_viable_candidates: bool,
1339}
1340
1341pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1398 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1399}
1400
1401fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1403 if verdict.parked {
1409 task.park(detail);
1410 return;
1411 }
1412 match verdict.status {
1413 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1414 RunStatus::AlreadyInBase => task.already_landed(detail),
1415 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1416 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1417 task.stall(detail)
1418 }
1419 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1420 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1421 RunStatus::Blocked => task.fail(detail, max_attempts),
1422 RunStatus::VerifiedNoop => task.handed_off(detail),
1423 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1424 }
1425}
1426
1427pub fn supersede_prior_runs(task: &Task, home: &Path) {
1476 let now = Timestamp::now();
1477 let last_run_succeeded = task
1484 .runs
1485 .last()
1486 .and_then(|id| RunState::load_under(id, home).ok())
1487 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1488 for id in task.superseded_attempts(last_run_succeeded) {
1489 let mut state = match RunState::load_under(id, home) {
1490 Ok(s) => s,
1491 Err(e) => {
1492 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1493 continue;
1494 }
1495 };
1496 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1497 continue;
1498 }
1499 let daemon_claims = is_working_on(home, id, now);
1500 if state.liveness(daemon_claims) == Liveness::Live {
1501 continue;
1502 }
1503 state.status = RunStatus::Superseded;
1504 if let Err(e) = state.save_under(home) {
1505 tracing::warn!("could not mark run {id} superseded: {e:#}");
1506 }
1507 }
1508}
1509
1510fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1532 for task in queue.list() {
1533 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1534 continue;
1535 }
1536 supersede_prior_runs(&task, home);
1537 }
1538}
1539
1540fn settle_and_diagnose(
1547 task: &mut Task,
1548 verdict: Verdict,
1549 detail: &str,
1550 max_attempts: usize,
1551 state: &RunState,
1552) {
1553 let p = phrases(&state.config.graph.language);
1554 if let Some(d) = &state.merge_decision {
1557 use crate::merge_decision as md;
1558 if task.runs.last().is_some_and(|id| id != &state.id) {
1559 return;
1560 }
1561 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1562 task.succeed();
1563 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1564 if state.liveness(false) != Liveness::Dead {
1565 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1566 return;
1567 }
1568 let replacement = md::Replacement {
1569 question: d.question.clone(),
1570 run: state.id.clone(),
1571 pr: d.target.pr.clone(),
1572 agent: agent.to_owned(),
1573 context: format!(
1574 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1575 d.target.pr,
1576 state.id,
1577 serde_json::to_string(&state.reviews).unwrap_or_default()
1578 ),
1579 };
1580 task.requeue();
1581 task.pr_recovery = None;
1582 task.review_of = None;
1583 task.replacement = Some(replacement);
1584 } else {
1585 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1586 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1587 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1588 }
1589 task.mark_action_applied(&d.question);
1590 if let Some(pass) = state.passes.last() {
1591 task.record_pass(&state.id, pass.index, task.attempts, None);
1592 }
1593 return;
1594 }
1595 if let Some(replacement) = &task.replacement
1596 && !state.parked
1597 && (state.viable().is_empty()
1598 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1599 {
1600 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1601 return;
1602 }
1603 let attempts_before = task.attempts;
1604 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1610 task.notice_cause(
1611 &format!("resource:{}:{}", block.kind, block.node),
1612 None,
1613 None,
1614 );
1615 task.last_error = Some(detail.to_owned());
1616 task.diagnostic = None;
1617 task.attempts = task.attempts.saturating_sub(1);
1618 task.hold_machine(Some(format!(
1619 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1620 no agent was asked and no attempt was spent",
1621 crate::cache::RESOURCE_HOLD,
1622 block.kind,
1623 block.node
1624 )));
1625 if let Some(pass) = state.passes.last() {
1626 task.record_pass(&state.id, pass.index, attempts_before, None);
1627 }
1628 return;
1629 }
1630 let unresolved = task.hold_notice_cause.clone();
1631 settle_in(task, verdict, detail, max_attempts, p);
1632 if state.prep_failure.is_some() {
1633 task.hold_notice_cause = unresolved;
1636 task.startup_notice_cause("startup-candidates");
1637 }
1638 if let Some(pass) = state.passes.last() {
1641 let parked = match &pass.end {
1642 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
1643 _ => None,
1644 };
1645 task.record_pass(&state.id, pass.index, attempts_before, parked);
1646 }
1647 if task.status == TaskStatus::Held {
1648 if state.prep_failure.is_none() {
1649 task.notice_cause(
1650 label(verdict.status),
1651 Some(&state.id),
1652 state.passes.last().map(|p| p.index),
1653 );
1654 }
1655 task.diagnostic = diagnostic(state);
1656 note_open_question(task, &state.id, p);
1657 }
1658}
1659
1660fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
1675 let Some(home) = crate::run::try_home() else {
1676 return;
1677 };
1678 let open = Questions::at(home.join("questions")).open_for(run);
1679 let Some(q) = open.first() else {
1680 return;
1681 };
1682 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
1683 decisions.sort_unstable();
1684 task.notice_cause(
1685 &format!("questions:{}", decisions.join(",")),
1686 Some(run),
1687 None,
1688 );
1689 let base = task.hold_reason.clone().unwrap_or_default();
1690 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
1691}
1692
1693fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
1704 match last_run {
1705 Some(state) => {
1706 let verdict = Verdict {
1707 status: state.status,
1708 left_pr: state.pr.is_some(),
1709 quota_hit: !state.quota.is_empty(),
1710 parked: state.parked,
1711 no_viable_candidates: state.viable().is_empty(),
1712 };
1713 let detail = format!(
1714 "{}{}",
1715 phrases(&state.config.graph.language).recovered_running,
1716 describe(&state)
1717 );
1718 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
1719 }
1720 None => {
1721 let why = phrases(language).no_run_to_recover;
1724 task.last_error = Some(why.to_owned());
1725 task.notice_cause("missing-run", None, None);
1728 task.hold_machine(Some(why.to_owned()));
1729 }
1730 }
1731}
1732
1733fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
1755 let mut reclaimed = Vec::new();
1756 for listed in queue.list() {
1757 if listed.status != TaskStatus::Running {
1758 continue;
1759 }
1760 let Ok(_claim) = queue.claim(&listed.id) else {
1761 continue;
1762 };
1763 let Ok(mut task) = queue.get(&listed.id) else {
1767 continue;
1768 };
1769 if task.status != TaskStatus::Running {
1770 continue;
1771 }
1772 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
1773 if let Some(state) = &last_run
1783 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
1784 {
1785 tracing::warn!("abandon questions for {}: {e:#}", state.id);
1786 }
1787 let language = if last_run.is_none() {
1788 language_of(&task, Path::new("."))
1789 } else {
1790 String::new()
1791 };
1792 reclaim(&mut task, last_run, max_attempts, &language);
1793 if task.status == TaskStatus::Done {
1794 supersede_prior_runs(&task, &crate::run::home());
1795 }
1796 record(queue, &mut task);
1797 reclaimed.push(task.id.clone());
1798 }
1799 reclaimed
1800}
1801
1802fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
1834 reclaim_abandoned_runs_with(
1835 home,
1836 now,
1837 crate::proc::pid_status,
1838 crate::proc::process_started_at,
1839 )
1840}
1841
1842fn reclaim_abandoned_runs_with<F, G>(
1848 home: &Path,
1849 now: Timestamp,
1850 query: F,
1851 identity: G,
1852) -> Vec<String>
1853where
1854 F: Fn(u32) -> Option<bool>,
1855 G: Fn(u32) -> Option<String>,
1856{
1857 let mut abandoned = Vec::new();
1858 for entry in std::fs::read_dir(home.join("runs"))
1859 .into_iter()
1860 .flatten()
1861 .flatten()
1862 {
1863 let id = entry.file_name().to_string_lossy().into_owned();
1864 if !crate::run::is_run_id(&id) {
1865 continue;
1866 }
1867 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
1875 continue;
1876 };
1877 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
1878 continue;
1879 };
1880 if state.status.done() || !state.active_all_overrun(now) {
1881 continue;
1882 }
1883 let daemon_claims = is_working_on(home, &id, now);
1895 if state.liveness_with(daemon_claims, &query, &identity) != crate::run::Liveness::Dead {
1896 continue;
1897 }
1898 state.abandon("daemon");
1899 if let Err(e) = state.save_under(home) {
1900 tracing::warn!("could not persist abandoned run {id}: {e:#}");
1901 continue;
1902 }
1903 if let Some(notice) = notices::run_ended(&state) {
1906 notices::raise_in(home, notice);
1907 }
1908 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
1916 tracing::warn!("abandon questions for {id}: {e:#}");
1917 }
1918 abandoned.push(id);
1919 }
1920 abandoned
1921}
1922
1923pub async fn serve(opts: Opts) -> Result<()> {
1929 serve_until(opts, Stop::new()).await
1930}
1931
1932pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
1949 let signal = {
1950 let stop = stop.clone();
1951 tokio::spawn(async move {
1952 if tokio::signal::ctrl_c().await.is_ok() {
1953 stop.stop();
1954 tracing::info!("shutdown requested; a run in flight will be finished first");
1955 }
1956 })
1957 };
1958
1959 let worktrees_root = opts
1960 .worktrees_root
1961 .clone()
1962 .unwrap_or_else(crate::run::default_worktree_root);
1963 let outcome = drive(
1964 &opts,
1965 &Queue::open(),
1966 &status_path(),
1967 &crate::run::home(),
1968 &worktrees_root,
1969 &stop,
1970 )
1971 .await;
1972
1973 signal.abort();
1974 outcome
1975}
1976
1977async fn drive(
1990 opts: &Opts,
1991 queue: &Queue,
1992 status_file: &Path,
1993 home: &Path,
1994 worktrees_root: &Path,
1995 stop: &Stop,
1996) -> Result<()> {
1997 let recovery_cfg = prepare(&opts.repo, opts).ok();
2005 let daemon_cfg = recovery_cfg
2006 .as_ref()
2007 .map(|c| c.daemon.clone())
2008 .unwrap_or_default();
2009 let mut initial_status = Status::new();
2010 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2011 recovery_cfg.as_ref(),
2012 daemon_cfg.max_deputies,
2013 ));
2014 let status = Arc::new(Mutex::new(initial_status));
2015 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2016 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2017
2018 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2024
2025 let waiter = tokio::spawn(crate::waiter::run(
2030 crate::waiter::Waiter::new(
2031 crate::ask::Questions::at(home.join("questions")),
2032 home.to_path_buf(),
2033 recovery_cfg.clone(),
2034 ),
2035 stop.clone(),
2036 ));
2037
2038 let deputies = tokio::spawn(crate::deputy::run(
2042 crate::deputy::Deputies::new(
2043 crate::ask::Questions::at(home.join("questions")),
2044 home.to_path_buf(),
2045 recovery_cfg.clone(),
2046 opts.repo.clone(),
2047 daemon_cfg.max_deputies,
2048 {
2049 let stop = stop.clone();
2050 Arc::new(move || stop.parking())
2051 },
2052 ),
2053 stop.clone(),
2054 ));
2055
2056 let fetcher = tokio::spawn(fetch_loop(opts.repo.clone(), opts.clone(), stop.clone()));
2059
2060 let release_watch = tokio::spawn(crate::release_watch::run(
2063 crate::release_watch::Watcher::new(
2064 Box::new(crate::release_watch::GhForge),
2065 home.to_path_buf(),
2066 ),
2067 {
2068 let (repo, opts) = (opts.repo.clone(), opts.clone());
2069 move || release_watch_settings(&repo, &opts)
2070 },
2071 stop.clone(),
2072 ));
2073
2074 tracing::info!(
2075 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2076 queue.root().display(),
2077 opts.poll.as_secs(),
2078 opts.max_attempts,
2079 concurrency,
2080 if daemon_cfg.pause_for_interrupts {
2081 ", interrupts enabled"
2082 } else {
2083 ""
2084 }
2085 );
2086
2087 janitor(&opts.repo, opts, home, worktrees_root).await;
2090 resweep_superseded_attempts(queue, home);
2091
2092 let outcome = poll(
2093 opts,
2094 queue,
2095 &status,
2096 home,
2097 worktrees_root,
2098 stop,
2099 DispatchLimits {
2100 max_concurrent: concurrency,
2101 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2102 },
2103 )
2104 .await;
2105
2106 beat.abort();
2107 waiter.abort();
2108 deputies.abort();
2109 fetcher.abort();
2110 release_watch.abort();
2111 clear_status_at(status_file);
2112 outcome
2113}
2114
2115fn release_watch_settings(repo: &Path, opts: &Opts) -> (Vec<PathBuf>, u64) {
2118 let Ok(cfg) = prepare(repo, opts) else {
2119 return (Vec::new(), 0);
2120 };
2121 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2122 .into_iter()
2123 .map(|r| r.path)
2124 .collect();
2125 let mut known: Vec<PathBuf> = Queue::open().list().into_iter().map(|t| t.repo).collect();
2128 let runs = crate::run::home().join("runs");
2129 for id in crate::run::list_ids_in(&runs) {
2130 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2131 .ok()
2132 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2133 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2134 known.extend(repo);
2135 }
2136 for k in known {
2137 let k = k.canonicalize().unwrap_or(k);
2138 if k.join(".git").exists() && !paths.contains(&k) {
2139 paths.push(k);
2140 }
2141 }
2142 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2143 if !paths.contains(&own) {
2144 paths.push(own);
2145 }
2146 (paths, cfg.daemon.release_stall_minutes)
2147}
2148
2149const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2151
2152async fn fetch_loop(repo: PathBuf, opts: Opts, stop: Stop) {
2156 while !stop.stopped() {
2157 let (interval, roots) = match prepare(&repo, &opts) {
2158 Ok(c) => (c.repos.fetch_interval, c.repos.roots),
2159 Err(_) => (0, Vec::new()),
2160 };
2161 if interval > 0 && !roots.is_empty() {
2162 let r = crate::clean::fetch_origins(&roots, FETCH_TIMEOUT, || stop.stopped()).await;
2163 tracing::debug!("fetch origins: {r:?}");
2164 }
2165 let wait = if interval > 0 { interval } else { 60 };
2167 let mut slept = 0;
2168 while slept < wait && !stop.stopped() {
2169 tokio::time::sleep(Duration::from_secs(1)).await;
2170 slept += 1;
2171 }
2172 }
2173}
2174
2175async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2181 loop {
2182 tokio::time::sleep(HEARTBEAT).await;
2183 let snapshot = {
2184 let mut guard = lock(&status);
2185 guard.updated_at = Timestamp::now();
2186 guard.clone()
2187 };
2188 if let Err(e) = write_status_to(&path, &snapshot) {
2189 tracing::warn!("could not refresh the daemon status file: {e:#}");
2192 }
2193 }
2194}
2195
2196#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2199enum LandResume {
2200 NotLanding,
2203 StillWaiting,
2208 Ready,
2212}
2213
2214fn land_resume_state(task: &Task) -> LandResume {
2218 let Some(run_id) = task.runs.last() else {
2219 return LandResume::NotLanding;
2220 };
2221 let Ok(state) = RunState::load(run_id) else {
2222 return LandResume::NotLanding;
2223 };
2224 if !state.parked {
2225 return LandResume::NotLanding;
2226 }
2227 if state.status == RunStatus::Landing
2228 && state.land_approval.as_ref().is_some_and(|binding| {
2229 ask::Questions::open()
2230 .get(&binding.question)
2231 .is_ok_and(|q| {
2232 q.run == state.id
2233 && q.resolution()
2234 .as_deref()
2235 .is_some_and(crate::merge_decision::stops)
2236 })
2237 })
2238 {
2239 return LandResume::Ready;
2240 }
2241 if !implementers_match(task, &state) {
2242 return LandResume::NotLanding;
2243 }
2244 let node = if state.status == RunStatus::Landing {
2248 land::APPROVAL_NODE
2249 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2250 crate::github_text::ASK_NODE
2251 } else {
2252 return LandResume::NotLanding;
2253 };
2254 let store = ask::Questions::open();
2255 let waiting = store
2256 .list()
2257 .into_iter()
2258 .filter(|q| &q.run == run_id && q.node == node)
2259 .max_by(|a, b| a.id.cmp(&b.id));
2260 let Some(mut q) = waiting else {
2261 return LandResume::Ready;
2262 };
2263 if !q.status.open() {
2264 if q.resolution().as_deref() == Some(land::REPAIR) {
2267 return if state.liveness(false) == Liveness::Live {
2268 LandResume::StillWaiting
2269 } else {
2270 LandResume::NotLanding
2271 };
2272 }
2273 return LandResume::Ready;
2274 }
2275 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2282 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2283 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2284 q.abandon(format!(
2285 "no answer within {}s of asking",
2286 timeout.as_secs().max(1)
2287 ));
2288 if store.put(&mut q).is_ok() {
2291 return LandResume::Ready;
2292 }
2293 }
2294 LandResume::StillWaiting
2295}
2296
2297const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2305
2306const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2318
2319struct InFlightGuard<'a> {
2332 status: &'a Arc<Mutex<Status>>,
2333 stop: &'a Stop,
2334 task_id: &'a str,
2335}
2336
2337impl Drop for InFlightGuard<'_> {
2338 fn drop(&mut self) {
2339 lock(self.status).current.retain(|c| c.task != self.task_id);
2340 self.stop.exit();
2341 }
2342}
2343
2344#[derive(Debug, Clone, PartialEq, Eq)]
2366enum Interrupt {
2367 Idle,
2369 Parking {
2381 parked: Vec<String>,
2382 interrupt_task: String,
2383 },
2384 Running {
2392 parked: Vec<String>,
2393 interrupt_task: String,
2394 },
2395 Resuming { parked: Vec<String> },
2402}
2403
2404fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
2425 match state {
2426 Interrupt::Idle => {
2427 if in_flight.len() != 1 {
2439 return Interrupt::Idle;
2440 }
2441 match runnable.iter().find(|t| t.interrupt) {
2442 Some(t) => Interrupt::Parking {
2443 parked: in_flight.to_vec(),
2444 interrupt_task: t.id.clone(),
2445 },
2446 None => Interrupt::Idle,
2447 }
2448 }
2449 Interrupt::Parking {
2450 parked,
2451 interrupt_task,
2452 } => {
2453 if in_flight.iter().any(|id| parked.contains(id)) {
2454 Interrupt::Parking {
2456 parked,
2457 interrupt_task,
2458 }
2459 } else if in_flight.contains(&interrupt_task) {
2460 Interrupt::Running {
2461 parked,
2462 interrupt_task,
2463 }
2464 } else if runnable.iter().any(|t| t.id == interrupt_task) {
2465 Interrupt::Parking {
2469 parked,
2470 interrupt_task,
2471 }
2472 } else {
2473 Interrupt::Resuming { parked }
2478 }
2479 }
2480 Interrupt::Running {
2481 parked,
2482 interrupt_task,
2483 } => {
2484 if in_flight.contains(&interrupt_task) {
2485 Interrupt::Running {
2486 parked,
2487 interrupt_task,
2488 }
2489 } else {
2490 Interrupt::Resuming { parked }
2496 }
2497 }
2498 Interrupt::Resuming { parked } => {
2499 if in_flight.iter().any(|id| parked.contains(id)) {
2500 Interrupt::Idle
2506 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
2507 Interrupt::Resuming { parked }
2508 } else {
2509 Interrupt::Idle
2512 }
2513 }
2514 }
2515}
2516
2517fn advance_interrupt_tick(
2523 enabled: bool,
2524 state: Interrupt,
2525 in_flight: &[String],
2526 runnable: &[Task],
2527) -> Interrupt {
2528 if !enabled {
2529 return Interrupt::Idle;
2530 }
2531 advance_interrupt(state, in_flight, runnable)
2532}
2533
2534fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
2539 match state {
2540 Interrupt::Idle => candidates,
2541 Interrupt::Parking {
2542 parked,
2543 interrupt_task,
2544 } => {
2545 if in_flight.iter().any(|id| parked.contains(id)) {
2546 Vec::new()
2547 } else {
2548 candidates
2549 .into_iter()
2550 .filter(|t| &t.id == interrupt_task)
2551 .collect()
2552 }
2553 }
2554 Interrupt::Running { .. } => Vec::new(),
2555 Interrupt::Resuming { parked } => candidates
2563 .into_iter()
2564 .find(|t| parked.contains(&t.id))
2565 .into_iter()
2566 .collect(),
2567 }
2568}
2569
2570struct DispatchLimits {
2574 max_concurrent: usize,
2577 pause_for_interrupts: bool,
2579}
2580
2581#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2590enum PermitKind {
2591 None,
2596 Urgent,
2603 Ordinary,
2606}
2607
2608fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
2611 if priority {
2612 PermitKind::None
2613 } else if urgent {
2614 PermitKind::Urgent
2615 } else {
2616 PermitKind::Ordinary
2617 }
2618}
2619
2620struct DispatchClaim {
2621 task: Task,
2622 claim: crate::queue::Claim,
2623 permit: Option<tokio::sync::OwnedSemaphorePermit>,
2624}
2625
2626fn claim_dispatch(
2629 queue: &Queue,
2630 candidate: &Task,
2631 cooling_down: bool,
2632 ordinary: &Arc<tokio::sync::Semaphore>,
2633 urgent: &Arc<tokio::sync::Semaphore>,
2634) -> Option<DispatchClaim> {
2635 let Ok(claim) = queue.claim(&candidate.id) else {
2636 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
2637 return None;
2638 };
2639 let task = match queue.get(&candidate.id) {
2640 Ok(t) if t.status.runnable() => t,
2641 Ok(_) => return None,
2642 Err(e) => {
2643 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
2644 return None;
2645 }
2646 };
2647 let resume = land_resume_state(&task);
2648 if resume == LandResume::StillWaiting {
2649 return None;
2650 }
2651 let priority = resume == LandResume::Ready;
2652 if !priority && cooling_down {
2653 return None;
2654 }
2655 let permit = match permit_kind(priority, task.urgent) {
2658 PermitKind::None => None,
2659 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
2660 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
2661 };
2662 Some(DispatchClaim {
2663 task,
2664 claim,
2665 permit,
2666 })
2667}
2668
2669async fn poll(
2688 opts: &Opts,
2689 queue: &Queue,
2690 status: &Arc<Mutex<Status>>,
2691 home: &Path,
2692 worktrees_root: &Path,
2693 stop: &Stop,
2694 limits: DispatchLimits,
2695) -> Result<()> {
2696 let DispatchLimits {
2697 max_concurrent,
2698 pause_for_interrupts,
2699 } = limits;
2700 let mut attempted: Vec<String> = Vec::new();
2705 let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
2706 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(1));
2714 let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
2720 let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
2721 let mut conductor = Conductor::new();
2722 let mut cache_last_checked: Option<Timestamp> = None;
2725 let mut interrupt = Interrupt::Idle;
2727 let mut interrupt_pauses: std::collections::HashMap<String, crate::graph::Pause> =
2733 std::collections::HashMap::new();
2734
2735 while !stop.stopped() {
2736 lock(status).polls += 1;
2737
2738 while let Some(result) = inflight.try_join_next() {
2743 if let Err(e) = result {
2744 tracing::error!("a spawned attempt did not finish cleanly: {e}");
2745 notices::raise(Notice::error(
2746 "loop:attempt",
2747 "A queued attempt ended abnormally; check the task it was running.",
2748 ));
2749 }
2750 }
2751
2752 let swept = sweep_stale_claims(queue, STALE_CLAIM);
2753 if !swept.is_empty() {
2754 tracing::warn!(
2755 "swept {} stale claim(s) left behind by an earlier daemon: {}",
2756 swept.len(),
2757 swept.join(", ")
2758 );
2759 }
2760 let now = Timestamp::now();
2765
2766 if !stop.busy_now() {
2771 maybe_prune_cache_between_runs(
2772 &opts.repo,
2773 opts,
2774 home,
2775 stop,
2776 &mut cache_last_checked,
2777 now,
2778 )
2779 .await;
2780 }
2781
2782 let stalled = stalled_tasks(queue, home, now);
2783 let stalled_ids: std::collections::BTreeSet<_> =
2784 stalled.iter().map(|task| task.id.clone()).collect();
2785 let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
2786 if !reclaimed.is_empty() {
2787 tracing::warn!(
2788 "reclaimed {} task(s) left `running` by a daemon that never \
2789 recorded the outcome: {}",
2790 reclaimed.len(),
2791 reclaimed.join(", ")
2792 );
2793 }
2794 let abandoned_runs = reclaim_abandoned_runs(home, now);
2795 if !abandoned_runs.is_empty() {
2796 tracing::warn!(
2797 "failed {} run(s) left behind by a killed process, past every \
2798 active seat's own timeout: {}",
2799 abandoned_runs.len(),
2800 abandoned_runs.join(", ")
2801 );
2802 }
2803
2804 let questions = Questions::at(home.join("questions"));
2809
2810 resolve_blockers(queue, &questions);
2813 apply_choice_actions(queue, &questions, home);
2814 triage::apply_answers(queue, &questions);
2815 reconcile_task_questions(queue, &questions);
2816 {
2820 let talks = crate::talk::Talks::at(home.join("talks"));
2821 crate::chat_notify::sweep(queue, &talks, home);
2822 crate::chat_watch::sweep(home, &talks);
2823 crate::chat_notify::start_turns(&talks, stop.parking(), &mut inflight);
2824 }
2825
2826 let finished: Vec<Task> = finished_tasks(queue)
2832 .into_iter()
2833 .filter(|task| !stalled_ids.contains(&task.id))
2834 .filter(|task| !awaiting_resume_with(task, |id| RunState::load_under(id, home)))
2838 .collect();
2839 let queued = queued_tasks(queue);
2840 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
2844 && conductor.worth_a_look(queue, &stalled, &finished)
2845 {
2846 match prepare(&opts.repo, opts) {
2847 Ok(cfg) => {
2848 conductor
2849 .maybe_run(
2850 &cfg,
2851 &opts.repo,
2852 queue,
2853 &questions,
2854 home,
2855 &queued,
2856 &stalled,
2857 &finished,
2858 opts.max_attempts,
2859 )
2860 .await;
2861 }
2862 Err(e) => {
2863 tracing::warn!("conductor: no config: {e:#}");
2864 notices::raise(Notice::warn(
2865 "loop:no-config",
2866 "The loop could not read this repository's config, so held tasks are not being triaged.",
2867 ));
2868 }
2869 }
2870 }
2871
2872 let candidates: Vec<Task> = runnable(queue)
2873 .into_iter()
2874 .filter(|t| !opts.once || !attempted.contains(&t.id))
2875 .collect();
2876
2877 let in_flight: Vec<String> = lock(status)
2882 .current
2883 .iter()
2884 .map(|c| c.task.clone())
2885 .collect();
2886 interrupt_pauses.retain(|id, _| in_flight.contains(id));
2887
2888 interrupt =
2889 advance_interrupt_tick(pause_for_interrupts, interrupt, &in_flight, &candidates);
2890 if let Interrupt::Parking {
2891 parked,
2892 interrupt_task,
2893 } = &interrupt
2894 {
2895 let reason = format!(
2896 "task {} asked to run first",
2897 crate::run::short_of(interrupt_task)
2898 );
2899 for id in parked {
2900 if let Some(pause) = interrupt_pauses.get(id) {
2901 pause.park_because(reason.clone());
2902 }
2903 }
2904 }
2905 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
2906
2907 let cooling_down =
2908 lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
2909
2910 let mut started_any = false;
2911 for candidate in candidates {
2912 if stop.stopped() {
2913 break;
2914 }
2915
2916 let Some(DispatchClaim {
2917 mut task,
2918 claim,
2919 permit,
2920 }) = claim_dispatch(queue, &candidate, cooling_down, &sem, &urgent_sem)
2921 else {
2922 continue;
2923 };
2924
2925 let task_id = task.id.clone();
2926 attempted.push(task_id.clone());
2927 lock(status).idle = false;
2928 stop.enter();
2931 started_any = true;
2932
2933 let run_pause = crate::graph::Pause::new();
2937 interrupt_pauses.insert(task_id.clone(), run_pause.clone());
2938
2939 let opts = opts.clone();
2940 let queue = queue.clone();
2941 let status = Arc::clone(status);
2942 let stop = stop.clone();
2943 let quota_cooldown_until = Arc::clone("a_cooldown_until);
2944 inflight.spawn(async move {
2945 let _claim = claim;
2949 let _permit = permit;
2950 let _inflight = InFlightGuard {
2952 status: &status,
2953 stop: &stop,
2954 task_id: &task_id,
2955 };
2956 let quota = attempt(&opts, &queue, &status, &stop, run_pause, &mut task).await;
2957 lock(&status).completed += 1;
2958 let now = Timestamp::now();
2964 if let Some(until) = cooldown_until("a, now) {
2965 let wait = until.as_second() - now.as_second();
2966 *lock("a_cooldown_until) = Some(until);
2967 let hint = quota
2968 .iter()
2969 .find(|q| q.reset.is_some())
2970 .and_then(|q| q.reset.as_deref());
2971 match hint {
2972 Some(h) => tracing::warn!(
2973 "quota hit; waiting {wait}s before taking another ordinary task \
2974 (CLI reported reset: {h})"
2975 ),
2976 None => tracing::warn!(
2977 "quota hit; waiting {wait}s before taking another ordinary task \
2978 (no reset hint reported)"
2979 ),
2980 }
2981 }
2982 });
2983 }
2984
2985 if started_any {
2986 continue;
2987 }
2988
2989 if stop.busy_now() {
2990 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
2995 continue;
2996 }
2997
2998 lock(status).idle = true;
3000 if opts.once {
3001 janitor(&opts.repo, opts, home, worktrees_root).await;
3005 resweep_superseded_attempts(queue, home);
3006 triage_held(queue, home, opts).await;
3007 break;
3008 }
3009 stop.idle(opts.poll).await;
3010 if stop.stopped() {
3011 continue;
3012 }
3013 janitor(&opts.repo, opts, home, worktrees_root).await;
3019 resweep_superseded_attempts(queue, home);
3020 triage_held(queue, home, opts).await;
3021 }
3022
3023 while let Some(result) = inflight.join_next().await {
3028 if let Err(e) = result {
3029 tracing::error!("a spawned attempt did not finish cleanly: {e}");
3030 notices::raise(Notice::error(
3031 "loop:attempt",
3032 "A queued attempt ended abnormally; check the task it was running.",
3033 ));
3034 }
3035 }
3036 Ok(())
3037}
3038
3039async fn attempt(
3045 opts: &Opts,
3046 queue: &Queue,
3047 status: &Arc<Mutex<Status>>,
3048 stop: &Stop,
3049 interrupt_pause: crate::graph::Pause,
3050 task: &mut Task,
3051) -> Vec<QuotaLoss> {
3052 let repo = repo_for(task, &opts.repo);
3053 tracing::info!(
3054 "task {} — {} (repo {})",
3055 task.short(),
3056 task.title,
3057 repo.display()
3058 );
3059
3060 if let Some(previous) = task.runs.last().and_then(|id| RunState::load(id).ok())
3064 && previous
3065 .merge_decision
3066 .as_ref()
3067 .is_some_and(|decision| !decision.closed)
3068 && previous.status != RunStatus::Landing
3069 && !crate::merge_decision::resumable_hold(&previous)
3070 {
3071 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3072 record(queue, task);
3073 return Vec::new();
3074 }
3075 let stop_owner = task
3076 .runs
3077 .last()
3078 .and_then(|id| RunState::load(id).ok())
3079 .filter(|state| {
3080 crate::merge_decision::resumable_hold(state)
3081 || state.status == RunStatus::Landing
3082 && state.land_approval.as_ref().is_some_and(|binding| {
3083 Questions::open().get(&binding.question).is_ok_and(|q| {
3084 q.run == state.id
3085 && q.resolution()
3086 .as_deref()
3087 .is_some_and(crate::merge_decision::stops)
3088 })
3089 })
3090 });
3091 if let Some(owner) = stop_owner {
3092 let mut runner = match Runner::resume(&owner.id) {
3093 Ok(runner) => runner,
3094 Err(e) => {
3095 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
3096 record(queue, task);
3097 return Vec::new();
3098 }
3099 };
3100 runner.on_pause(stop.pause());
3101 runner.watch_interrupt(interrupt_pause);
3102 let quota_before = runner.state.quota.clone();
3103 task.start(owner.id.clone());
3104 record(queue, task);
3105 lock(status).current.push(Current {
3106 task: task.id.clone(),
3107 run: owner.id,
3108 });
3109 runner.state.next_pass_task_entry = true;
3110 let result = runner.execute().await;
3111 return finish_attempt(
3112 opts.max_attempts,
3113 queue,
3114 task,
3115 &runner.state,
3116 "a_before,
3117 result,
3118 );
3119 }
3120
3121 let explicit = match &task.overrides {
3125 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
3126 Some(_) => opts.config.as_deref(),
3127 None => opts.config.as_deref(),
3128 };
3129 if let Err(e) = Config::discover_fetched(&repo, explicit).await {
3130 let reason = format!("[config] {e:#}");
3131 record_config_failure(task, &e);
3132 task.last_error = Some(reason.clone());
3133 task.hold_machine(Some(reason.clone()));
3134 record(queue, task);
3135 tracing::warn!("holding {}: {reason}", task.short());
3136 return Vec::new();
3137 }
3138 let mut config = match prepare_for(&repo, opts, task) {
3139 Ok(c) => c,
3140 Err(e) => {
3141 task.attempts += 1;
3145 task.fail_startup(
3146 "startup-config",
3147 format!("config: {e:#}"),
3148 opts.max_attempts,
3149 );
3150 record(queue, task);
3151 return Vec::new();
3152 }
3153 };
3154 apply_task_implementers(&mut config, task);
3155 if let Some(replacement) = &task.replacement {
3156 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
3157 old.liveness(false) == Liveness::Dead
3158 && old.merge_decision.as_ref().is_some_and(|d| {
3159 d.closed
3160 && d.question == replacement.question
3161 && d.replacement_run.as_ref().is_none_or(|run| {
3162 task.runs.last() == Some(run)
3163 && RunState::load(run).is_ok_and(|current| {
3164 current.replaces.as_ref().is_some_and(|pin| {
3165 pin.question == replacement.question
3166 && pin.agent == replacement.agent
3167 })
3168 })
3169 })
3170 })
3171 });
3172 if !safe {
3173 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
3174 record(queue, task);
3175 return Vec::new();
3176 }
3177 match config.agent(&replacement.agent) {
3178 Ok(agent) if crate::agent::installed(agent) => {}
3179 _ => {
3180 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
3181 record(queue, task);
3182 return Vec::new();
3183 }
3184 }
3185 config.graph.implementers = 1;
3186 config.graph.land_approval = true;
3187 config.merge.mode = MergeMode::Pr;
3188 config.graph.selected_implementer = Some(replacement.agent.clone());
3189 }
3190 let p = phrases(&config.graph.language);
3191 let start_failed = p.could_not_start;
3192
3193 if let Some(reason) = disk_gate(&repo, &config) {
3201 task.notice_cause("disk-space", None, None);
3202 task.last_error = Some(reason.clone());
3203 task.hold_machine(Some(reason.clone()));
3204 record(queue, task);
3205 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
3206 notices::raise(
3209 Notice::warn(
3210 &format!("disk:{}", repo.display()),
3211 "A task was held for want of free disk space; free some, then release it from the queue.",
3212 )
3213 .link(Link::Task {
3214 id: task.id.clone(),
3215 }),
3216 );
3217 return Vec::new();
3218 }
3219
3220 let unfinished = (!task.fresh_start)
3240 .then(|| unfinished_run(&task.runs, task.short()))
3241 .flatten()
3242 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
3243 let previous = task.runs.last().and_then(|id| RunState::load(id).ok());
3249 let review_branch = take_review_branch(task, previous.as_ref());
3250 let branch_exists = match &review_branch {
3251 Some(branch) => crate::git::branch_exists(&repo, branch)
3252 .await
3253 .unwrap_or(false),
3254 None => false,
3255 };
3256 let proposed = match (&review_branch, &unfinished, &task.review_of) {
3260 (None, None, Some(branch)) => Starter::Review(branch.clone()),
3261 _ => choose_starter(
3262 review_branch.as_deref(),
3263 branch_exists,
3264 unfinished.as_deref(),
3265 ),
3266 };
3267 let retry_selected = task.replacement.is_some()
3268 && task.resumption.is_some()
3269 && !task.fresh_start
3270 && previous.as_ref().is_some_and(|run| {
3271 run.replaces.is_some()
3272 && ((run.status == RunStatus::Failed && run.viable().is_empty())
3273 || (run.status == RunStatus::Blocked && run.tally.is_none()))
3274 });
3275 let stop_pin = task
3276 .resume_override
3277 .as_ref()
3278 .filter(|pin| {
3279 Questions::open().get(&pin.question_id).is_ok_and(|q| {
3280 q.node == land::APPROVAL_NODE
3281 && q.resolution()
3282 .as_deref()
3283 .is_some_and(crate::merge_decision::stops)
3284 })
3285 })
3286 .and_then(|pin| pin.pinned_run.clone());
3287 let starter = if let Some(run) = stop_pin {
3288 Starter::Resume(run)
3291 } else if retry_selected {
3292 Starter::Resume(previous.as_ref().unwrap().id.clone())
3293 } else if let Some(selected) = &task.pr_recovery {
3294 if selected.check_task(&task.id, &task.instruction).is_ok() {
3295 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
3296 } else {
3297 Starter::Review(selected.branch.clone())
3298 }
3299 } else {
3300 starter_with_implementers(task, previous.as_ref(), proposed)
3301 };
3302 if let Some(pin) = &task.resume_override
3305 && let Some(run) = &pin.pinned_run
3306 && Questions::open().get(&pin.question_id).map_or(true, |q| {
3307 q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR)
3308 })
3309 && starter != Starter::Resume(run.clone())
3310 {
3311 let run = run.clone();
3312 task.notice_cause("repair-owner", Some(&run), None);
3313 task.hold_machine(Some(format!(
3314 "repair owner cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
3315 task.id
3316 )));
3317 record(queue, task);
3318 return Vec::new();
3319 }
3320 if task.replacement.is_some()
3321 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
3322 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
3323 {
3324 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
3325 record(queue, task);
3326 return Vec::new();
3327 }
3328 let attachments = match task_attachments(queue, task) {
3333 Ok(a) => a,
3334 Err(e) => {
3335 task.attempts += 1;
3336 task.fail_startup_target(
3337 "startup-attachments",
3338 e.name.clone(),
3339 format!("{start_failed}{e:#}"),
3340 opts.max_attempts,
3341 );
3342 record(queue, task);
3343 return Vec::new();
3344 }
3345 };
3346 let started = match &starter {
3347 Starter::Review(_) if task.pr_recovery.is_some() => {
3348 let selected = task.pr_recovery.clone().expect("selected recovery");
3349 match selected.check_task(&task.id, &task.instruction) {
3350 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
3351 .await
3352 .map(|mut r| {
3353 r.state.attachments = attachments.clone();
3354 r
3355 }),
3356 Err(e) => Err(e),
3357 }
3358 }
3359 Starter::Review(branch) => {
3360 tracing::info!(
3361 "task {} reopens `{branch}` as a review-only pass",
3362 task.short()
3363 );
3364 let takeover = crate::handover::Takeover {
3367 earlier: task.earlier_attempts().to_vec(),
3368 home: crate::run::home(),
3369 choice: take_divergence_answer(branch, &config.merge.remote, task),
3370 task_instruction: Some(instruction_for(task)),
3371 };
3372 Runner::review_taking_over(
3373 &repo,
3374 branch,
3375 config,
3376 Some(takeover),
3377 crate::run::Origin::queue(&task.id),
3378 )
3379 .await
3380 }
3381 Starter::Resume(id) => {
3382 tracing::info!("resuming run {id} rather than competing again");
3383 Runner::resume(id).and_then(|mut r| {
3384 if retry_selected {
3385 for candidate in &mut r.state.candidates {
3388 candidate.failed = None;
3389 candidate.empty = false;
3390 candidate.commits = 0;
3391 }
3392 r.state.status = RunStatus::Implementing;
3393 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
3394 }
3395 if r.state.prep_failure.is_some() {
3396 let approval = task
3397 .resumption
3398 .as_ref()
3399 .context("preparation retry requires an explicit task release")?;
3400 r.retry_preparation(approval.at)?;
3401 }
3402 if let Some(instruction) =
3403 prepare_instruction(&starter, Some(&r.state.instruction), task)
3404 {
3405 r.state.instruction = instruction;
3406 }
3407 r.state.attachments = attachments.clone();
3408 if task.replacement.is_some() {
3409 r.state.config.merge.mode = MergeMode::Pr;
3410 r.state.config.graph.land_approval = true;
3411 }
3412 if task.replacement.is_none() && let Some(mode) = task
3415 .overrides
3416 .as_ref()
3417 .and_then(|o| o.merge.as_deref())
3418 .and_then(|m| merge_mode(m).ok())
3419 {
3420 r.state.config.merge.mode = mode;
3421 }
3422 Ok(r)
3423 })
3424 }
3425 Starter::Start => {
3426 if let Some(branch) = &review_branch {
3427 tracing::warn!(
3428 "conductor chose review for task {} but branch `{branch}` no longer \
3429 exists; requeuing as a fresh competition instead",
3430 task.short()
3431 );
3432 }
3433 let instruction = prepare_instruction(&starter, None, task)
3434 .unwrap_or_else(|| task.instruction.clone());
3435 Runner::start_naming(
3436 &repo,
3437 instruction,
3438 &task.title,
3439 config,
3440 crate::run::Origin::queue(&task.id),
3441 )
3442 .await
3443 .map(|mut r| {
3444 r.state.attachments = attachments.clone();
3445 r
3446 })
3447 }
3448 };
3449 let mut runner = match started {
3450 Ok(r) => r,
3451 Err(e) if task.pr_recovery.is_some() => {
3455 task.startup_notice_cause("startup-public-head");
3456 task.attempts += 1;
3457 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
3458 task.link_run(&partial.run);
3459 task.record_pass(&partial.run, 0, task.attempts, None);
3460 }
3461 task.last_error = Some(format!("{start_failed}{e:#}"));
3462 task.hold_unrequested_retry(Some(format!(
3463 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
3464 )));
3465 record(queue, task);
3466 return Vec::new();
3467 }
3468 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
3469 let detail = match e.downcast_ref::<crate::handover::Refused>() {
3470 Some(r) => (p.handover_refused)(r),
3471 None => format!("{e:#}"),
3472 };
3473 task.startup_notice_cause("startup-handover");
3474 let reason = format!("{start_failed}{detail}");
3475 task.last_error = Some(reason.clone());
3476 let branch = match &starter {
3479 Starter::Review(branch) => Some(branch.clone()),
3480 _ => None,
3481 };
3482 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
3486 record(queue, task);
3487 tracing::warn!(
3488 "holding {} for a branch it cannot take over: {e:#}",
3489 task.short()
3490 );
3491 return Vec::new();
3492 }
3493 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
3498 let d = e
3499 .downcast_ref::<crate::reconcile::Diverged>()
3500 .expect("checked by the guard");
3501 let mut q = ask::Question::new(
3502 task.id.clone(),
3503 crate::reconcile::NODE.to_owned(),
3504 crate::reconcile::SEAT.to_owned(),
3505 d.summary(),
3506 d.detail(),
3507 d.choices(),
3508 );
3509 match Questions::open().put(&mut q) {
3510 Ok(()) => {
3511 task.last_error = Some(format!("{e:#}"));
3512 task.review_branch = Some(d.branch.clone());
3515 task.review_instruction = Some(task.instruction.clone());
3516 task.block(vec![q.id.clone()], Some(d.summary()));
3517 }
3518 Err(put) => {
3519 tracing::warn!("could not file the divergence question: {put:#}");
3520 task.attempts += 1;
3521 task.fail_startup(
3522 "startup-question-store",
3523 format!("{start_failed}{e:#}"),
3524 opts.max_attempts,
3525 );
3526 }
3527 }
3528 record(queue, task);
3529 return Vec::new();
3530 }
3531 Err(e) => {
3532 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
3538 && let Starter::Review(branch) = &starter
3539 {
3540 task.review_branch = Some(branch.clone());
3541 task.review_instruction = Some(task.instruction.clone());
3542 task.last_error = Some(format!("{start_failed}{e:#}"));
3543 task.status = crate::queue::TaskStatus::Failed;
3544 record(queue, task);
3545 return Vec::new();
3546 }
3547 task.attempts += 1;
3548 task.last_error = Some(format!("{start_failed}{e:#}"));
3549 task.startup_notice_cause("startup-source");
3550 task.hold_unrequested_retry(Some(format!(
3551 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
3552 )));
3553 record(queue, task);
3554 return Vec::new();
3555 }
3556 };
3557 if let Some(replacement) = &task.replacement {
3558 if matches!(starter, Starter::Start) {
3559 runner.state.instruction.push_str(&format!(
3560 "\n\n# Replacement review context\n{}",
3561 replacement.context
3562 ));
3563 }
3564 runner.state.replaces = Some(replacement.clone());
3565 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
3566 if let Ok(mut old) = RunState::load(&replacement.run)
3567 && let Some(d) = &mut old.merge_decision
3568 {
3569 d.replacement_run = Some(runner.state.id.clone());
3570 old.event(
3571 "replacement",
3572 format!(
3573 "replacement run {} owns the next attempt; old PR remains closed",
3574 runner.state.id
3575 ),
3576 );
3577 if let Err(e) = old.save() {
3578 task.hold_manual(Some(format!(
3579 "Could not link replacement history: {e}; no agents dispatched"
3580 )));
3581 record(queue, task);
3582 return Vec::new();
3583 }
3584 }
3585 }
3586 runner.state.task_instruction = Some(task.instruction.clone());
3587 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
3590 if runner.state.origin_chat.is_none() {
3591 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
3592 }
3593 runner.on_pause(stop.pause());
3595 runner.watch_interrupt(interrupt_pause);
3599
3600 let run = runner.state.id.clone();
3603 task.start(run.clone());
3604 record(queue, task);
3605 lock(status).current.push(Current {
3606 task: task.id.clone(),
3607 run,
3608 });
3609
3610 let quota_before = runner.state.quota.clone();
3613 runner.state.next_pass_task_entry = true;
3614 let result = runner.execute().await;
3615 finish_attempt(
3616 opts.max_attempts,
3617 queue,
3618 task,
3619 &runner.state,
3620 "a_before,
3621 result,
3622 )
3623}
3624
3625pub fn finish_attempt(
3630 max_attempts: usize,
3631 queue: &Queue,
3632 task: &mut Task,
3633 state: &RunState,
3634 quota_before: &[QuotaLoss],
3635 result: Result<()>,
3636) -> Vec<QuotaLoss> {
3637 let detail = match result {
3638 Ok(()) => describe(state),
3639 Err(e) => format!("{e:#}"),
3640 };
3641 let fresh = losses_this_attempt(quota_before, &state.quota);
3642 let verdict = Verdict {
3643 status: state.status,
3644 left_pr: state.pr.is_some(),
3647 quota_hit: !fresh.is_empty(),
3653 parked: state.parked,
3657 no_viable_candidates: state.viable().is_empty(),
3660 };
3661 if let Some(replacement) = &state.replaces
3662 && let Some(pr) = &state.pr
3663 && let Ok(mut old) = RunState::load(&replacement.run)
3664 && let Some(d) = &mut old.merge_decision
3665 {
3666 d.replacement_pr = Some(pr.url.clone());
3667 let _ = old.save();
3668 }
3669 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
3670 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
3671 task.hold_unrequested_retry(Some(format!(
3672 "Candidate setup failed: {detail}. Select a complete implementation with `magi task recover-pr`, or correct the referenced source before explicitly releasing the task."
3673 )));
3674 }
3675 if task.status == TaskStatus::Done {
3676 supersede_prior_runs(task, &crate::run::home());
3677 }
3678 record(queue, task);
3679 tracing::info!(
3680 "task {} is {} after run {} ({})",
3681 task.short(),
3682 task.status.as_str(),
3683 state.short(),
3684 label(state.status)
3685 );
3686 fresh
3687}
3688
3689pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
3693 let waits_on_owner = withheld_text_wait(task).is_some();
3698 if task.status.runnable()
3699 && !waits_on_owner
3700 && !(task.fresh_start && task.replacement.is_some())
3701 {
3702 let why = task.last_error.clone().map_or_else(
3703 || "the run did not finish".to_owned(),
3704 |e| format!("the run did not finish: {e}"),
3705 );
3706 task.hold_unrequested_retry(Some(format!(
3707 "{why}. It was started by hand, so it is not retried \
3708 automatically; `magi task release` retries it."
3709 )));
3710 record(queue, task);
3711 }
3712}
3713
3714#[must_use]
3718pub fn loop_would_resume(run: &str) -> bool {
3719 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
3720}
3721
3722#[must_use]
3732pub fn foreign_loop(
3733 reading: Option<&Reading>,
3734 now: Timestamp,
3735 own_pid: u32,
3736) -> Option<Option<u32>> {
3737 let reading = reading.filter(|r| r.running(now))?;
3738 match reading.pid {
3739 Some(pid) if pid == own_pid => None,
3740 pid => Some(pid),
3741 }
3742}
3743
3744pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
3759 let status = Arc::new(Mutex::new(Status::new()));
3760 let stop = Stop::new();
3761 attempt(
3762 opts,
3763 queue,
3764 &status,
3765 &stop,
3766 crate::graph::Pause::new(),
3767 task,
3768 )
3769 .await;
3770 let mut resumes = 0;
3779 while resumes < MAX_WITHHELD_RESUMES {
3780 let Some(timeout) = withheld_text_wait(task) else {
3781 break;
3782 };
3783 eprintln!(
3784 "waiting for the owner's answer on a withheld pull request text \
3785 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
3786 );
3787 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
3790 let ready = loop {
3791 match land_resume_state(task) {
3792 LandResume::StillWaiting if std::time::Instant::now() < limit => {
3793 tokio::time::sleep(WITHHELD_POLL).await;
3794 }
3795 LandResume::Ready => break true,
3796 _ => break false,
3797 }
3798 };
3799 if !ready {
3800 break;
3801 }
3802 resumes += 1;
3803 attempt(
3804 opts,
3805 queue,
3806 &status,
3807 &stop,
3808 crate::graph::Pause::new(),
3809 task,
3810 )
3811 .await;
3812 }
3813 hold_if_runnable(queue, task);
3814}
3815
3816const MAX_WITHHELD_RESUMES: u32 = 3;
3819
3820const WITHHELD_POLL: Duration = Duration::from_secs(15);
3822
3823fn withheld_text_wait(task: &Task) -> Option<u64> {
3826 let id = task.runs.last()?;
3827 let s = RunState::load(id).ok()?;
3828 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
3829 .then_some(s.config.graph.answer_timeout)
3830}
3831
3832fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
3842 after
3843 .iter()
3844 .filter(|q| !before.contains(q))
3845 .cloned()
3846 .collect()
3847}
3848
3849fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
3852 if quota.is_empty() {
3853 return None;
3854 }
3855 let with_hint = quota.iter().find(|q| q.reset.is_some());
3856 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
3857 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
3858 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
3859 Some(
3860 now.checked_add(jiff::SignedDuration::from_secs(secs))
3861 .unwrap_or(Timestamp::MAX),
3862 )
3863}
3864
3865fn implementers_match(task: &Task, state: &RunState) -> bool {
3867 task.replacement.as_ref().map_or_else(
3868 || {
3869 task.implementers()
3870 .is_none_or(|n| n == state.config.graph.implementers)
3871 },
3872 |replacement| {
3873 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
3874 && state.config.graph.implementers == 1
3875 },
3876 )
3877}
3878
3879fn starter_with_implementers(
3881 task: &Task,
3882 previous: Option<&RunState>,
3883 proposed: Starter,
3884) -> Starter {
3885 if previous.is_some_and(|s| !implementers_match(task, s)) {
3886 task.review_of
3887 .as_ref()
3888 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
3889 } else {
3890 proposed
3891 }
3892}
3893
3894fn apply_task_implementers(config: &mut Config, task: &Task) {
3896 if let Some(n) = task.implementers() {
3897 config.graph.implementers = n;
3898 }
3899}
3900
3901fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
3906 let Some(o) = &task.overrides else {
3907 return prepare(repo, opts);
3908 };
3909 let own = Opts {
3913 config: o.config.clone(),
3914 merge: None,
3915 ..opts.clone()
3916 };
3917 let mut config = prepare(repo, if o.inherit_daemon_options { opts } else { &own })?;
3918 o.apply(&mut config);
3919 Ok(config)
3920}
3921
3922fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
3924 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
3925 if let Some(mode) = &opts.merge {
3926 config.merge.mode = merge_mode(mode)?;
3927 }
3928 Ok(config)
3929}
3930
3931async fn maybe_prune_cache_between_runs(
3962 repo: &Path,
3963 opts: &Opts,
3964 home: &Path,
3965 stop: &Stop,
3966 last_checked: &mut Option<Timestamp>,
3967 now: Timestamp,
3968) {
3969 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
3970 return;
3971 }
3972 *last_checked = Some(now);
3973 let cfg = match prepare(repo, opts) {
3974 Ok(cfg) => cfg,
3975 Err(e) => {
3976 tracing::warn!("cache check: no config: {e:#}");
3977 return;
3978 }
3979 };
3980 match clean::prune_cache_if_over_limit(&cfg, home) {
3981 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
3982 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
3983 pruned.files,
3984 pruned.freed
3985 ),
3986 Ok(_) => {}
3987 Err(e) => {
3988 tracing::warn!("housekeep: prune cache: {e:#}");
3989 notices::raise_in(
3990 home,
3991 Notice::warn(
3992 "housekeep:cache",
3993 "Pruning the shared build cache failed; disk usage may keep growing.",
3994 ),
3995 );
3996 }
3997 }
3998}
3999
4000fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
4004 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
4005}
4006
4007async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
4026 let cfg = match prepare(repo, opts) {
4027 Ok(cfg) => cfg,
4028 Err(e) => {
4029 tracing::warn!("housekeep: no config: {e:#}");
4030 return;
4031 }
4032 };
4033 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
4042 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
4043 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
4048 let mut extra = Vec::new();
4049 if out.unreadable > 0 {
4050 extra.push(format!("{} unreadable", out.unreadable));
4051 }
4052 if out.orphaned_worktrees > 0 {
4053 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
4054 }
4055 let detail = if extra.is_empty() {
4056 String::new()
4057 } else {
4058 format!(" ({})", extra.join(", "))
4059 };
4060 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
4061 }
4062 if out.external_merges_recorded > 0 {
4063 tracing::info!(
4064 "housekeep: recorded {} run(s) as merged externally",
4065 out.external_merges_recorded
4066 );
4067 }
4068 if out.stale_pr_states_repaired > 0 {
4069 tracing::info!(
4070 "housekeep: rewrote {} run record(s) whose pull request had already settled",
4071 out.stale_pr_states_repaired
4072 );
4073 }
4074 if out.cache_files > 0 {
4075 tracing::info!(
4076 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
4077 out.cache_files,
4078 out.cache_freed
4079 );
4080 }
4081 if out.questions_abandoned > 0 {
4082 tracing::info!(
4083 "housekeep: abandoned {} question(s) left open by a finished run",
4084 out.questions_abandoned
4085 );
4086 }
4087}
4088
4089async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
4098 let questions = Questions::at(home.join("questions"));
4099 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
4100 if report.is_empty() {
4101 return;
4102 }
4103 if !report.quarantined.is_empty() {
4104 tracing::info!(
4105 "triage: held {} blocked task(s) whose blocked-on task or \
4106 question no longer exists: {}",
4107 report.quarantined.len(),
4108 report.quarantined.join(", ")
4109 );
4110 }
4111 if !report.resumed.is_empty() {
4112 tracing::info!(
4113 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
4114 report.resumed.len(),
4115 report.resumed.join(", ")
4116 );
4117 }
4118 if !report.asked.is_empty() {
4119 tracing::info!(
4120 "triage: asked about {} held task(s): {}",
4121 report.asked.len(),
4122 report.asked.join(", ")
4123 );
4124 }
4125 if !report.answered.is_empty() {
4126 tracing::info!(
4127 "triage: applied {} operator answer(s): {}",
4128 report.answered.len(),
4129 report.answered.join(", ")
4130 );
4131 }
4132}
4133
4134fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
4141 disk_gate_with(repo, config, crate::disk::free_bytes)
4142}
4143
4144fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
4148 repo: &Path,
4149 config: &Config,
4150 free_bytes: F,
4151) -> Option<String> {
4152 let min = config.disk.min_free_bytes;
4153 if min == 0 {
4154 return None;
4155 }
4156 match free_bytes(repo) {
4157 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
4158 Err(e) => Some(crate::disk::unmeasured_in(repo, &e, &config.graph.language)),
4159 }
4160}
4161
4162const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
4169
4170const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
4174
4175fn quota_wait(
4184 reset_at: Option<Timestamp>,
4185 now: Timestamp,
4186 fallback: Duration,
4187 cap: Duration,
4188) -> Duration {
4189 match reset_at {
4190 Some(at) if at > now => {
4191 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
4192 Duration::from_secs(secs).min(cap)
4193 }
4194 _ => fallback,
4195 }
4196}
4197
4198fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
4212 parse_reset_hint_zoned(text, now)
4213 .or_else(|| parse_reset_hint_dated(text))
4214 .or_else(|| parse_reset_hint_relative(text, recorded))
4215}
4216
4217fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
4221 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
4222 let mut rest = rest.trim();
4223 if rest.is_empty() {
4224 return None;
4225 }
4226 let mut total: i64 = 0;
4227 let mut matched = false;
4228 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
4229 if let Some((digits, tail)) = rest.split_once(unit)
4230 && !digits.is_empty()
4231 && digits.bytes().all(|b| b.is_ascii_digit())
4232 {
4233 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
4234 rest = tail;
4235 matched = true;
4236 }
4237 }
4238 if !rest.is_empty() || !matched {
4239 return None;
4240 }
4241 recorded
4242 .checked_add(jiff::SignedDuration::from_secs(total))
4243 .ok()
4244}
4245
4246fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
4250 let clock = clock.trim().to_lowercase();
4251 let (digits, pm) = clock
4252 .strip_suffix("am")
4253 .map(|d| (d, false))
4254 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
4255 let (h, m) = digits.trim().split_once(':')?;
4256 let mut hour: i8 = h.trim().parse().ok()?;
4257 let minute: i8 = m.trim().parse().ok()?;
4258 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
4259 return None;
4260 }
4261 if pm && hour != 12 {
4262 hour += 12;
4263 } else if !pm && hour == 12 {
4264 hour = 0;
4265 }
4266 Some((hour, minute))
4267}
4268
4269fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
4274 let open = text.find('(')?;
4275 let close = text.rfind(')')?;
4276 if close <= open {
4277 return None;
4278 }
4279 let zone = text[open + 1..close].trim();
4280 let (hour, minute) = parse_12h_clock(&text[..open])?;
4281 let tz = jiff::tz::TimeZone::get(zone).ok()?;
4282 let candidate = now
4283 .to_zoned(tz)
4284 .with()
4285 .hour(hour)
4286 .minute(minute)
4287 .second(0)
4288 .millisecond(0)
4289 .microsecond(0)
4290 .nanosecond(0)
4291 .build()
4292 .ok()?;
4293 let mut at = candidate.timestamp();
4294 if at <= now {
4295 at += jiff::SignedDuration::from_hours(24);
4296 }
4297 Some(at)
4298}
4299
4300fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
4309 let words: Vec<&str> = text.split_whitespace().collect();
4310 if words.len() < 5 {
4311 return None;
4312 }
4313 (0..=words.len() - 5)
4314 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
4315}
4316
4317fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
4323 if trailing.is_some_and(|next| next.starts_with('(')) {
4324 return None;
4325 }
4326 let month = month_number(window[0])?;
4327 let day_token = window[1].strip_suffix(',')?.to_lowercase();
4328 let day_digits = ["st", "nd", "rd", "th"]
4329 .iter()
4330 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
4331 let day: i8 = day_digits.parse().ok()?;
4332 let year_token = window[2];
4333 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
4334 return None;
4335 }
4336 let year: i16 = year_token.parse().ok()?;
4337 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
4341 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
4342 let date = jiff::civil::Date::new(year, month, day).ok()?;
4343 let candidate = date
4344 .at(hour, minute, 0, 0)
4345 .to_zoned(jiff::tz::TimeZone::UTC)
4346 .ok()?;
4347 Some(candidate.timestamp())
4348}
4349
4350fn month_number(name: &str) -> Option<i8> {
4353 const NAMES: [&str; 12] = [
4354 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
4355 ];
4356 let lower = name.to_lowercase();
4357 NAMES
4358 .iter()
4359 .position(|n| *n == lower.as_str())
4360 .map(|i| i as i8 + 1)
4361}
4362
4363fn exhausted_review_budget(state: &RunState) -> bool {
4375 state.status == RunStatus::Blocked
4378 && !state.pr_publication_pending
4379 && state.reviews.len() >= state.config.graph.review_rounds
4380}
4381
4382fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
4419 unfinished_run_with(runs, short, RunState::load)
4420}
4421
4422fn unfinished_run_with<F>(runs: &[String], short: &str, load: F) -> Option<String>
4425where
4426 F: FnOnce(&str) -> Result<RunState>,
4427{
4428 let id = runs.last()?;
4429 match load(id) {
4430 Ok(s)
4435 if s.status.resumable()
4436 && !s.released()
4437 && !exhausted_review_budget(&s)
4438 && s.liveness(false) != crate::run::Liveness::Live =>
4439 {
4440 Some(id.clone())
4441 }
4442 Ok(_) => None,
4443 Err(e) => {
4444 tracing::warn!("could not read run {id} for task {short}: {e:#}");
4445 None
4446 }
4447 }
4448}
4449
4450fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
4458where
4459 F: FnOnce(&str) -> Result<RunState>,
4460{
4461 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
4463 || task.fresh_start
4464 || task.review_branch.is_some()
4465 {
4466 return false;
4467 }
4468 let Some(id) = task.runs.last() else {
4469 return false;
4470 };
4471 load(id).is_ok_and(|s| {
4472 s.parked && s.status.resumable() && !s.released() && !exhausted_review_budget(&s)
4473 })
4474}
4475
4476#[derive(Debug, Clone, PartialEq, Eq)]
4479enum Starter {
4480 Review(String),
4483 Resume(String),
4485 Start,
4487}
4488
4489fn selected_starter(
4491 selected: &crate::pr_recovery::Selection,
4492 previous: Option<&RunState>,
4493 unfinished: Option<&str>,
4494) -> Starter {
4495 if let Some(id) = unfinished
4496 && previous.is_some_and(|r| {
4497 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
4498 })
4499 {
4500 Starter::Resume(id.to_owned())
4501 } else {
4502 Starter::Review(selected.branch.clone())
4503 }
4504}
4505
4506fn take_divergence_answer(
4510 branch: &str,
4511 remote: &str,
4512 task: &mut Task,
4513) -> Option<crate::reconcile::Choice> {
4514 let summary = crate::reconcile::summary_for(branch, remote);
4515 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
4516 (a.question == summary)
4517 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
4518 .flatten()
4519 .map(|c| (i, c))
4520 })?;
4521 task.answers.remove(idx);
4522 Some(choice)
4523}
4524
4525fn choose_starter(
4537 review_branch: Option<&str>,
4538 branch_exists: bool,
4539 unfinished: Option<&str>,
4540) -> Starter {
4541 match review_branch {
4542 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
4543 Some(_) => Starter::Start,
4544 None => match unfinished {
4545 Some(id) => Starter::Resume(id.to_owned()),
4546 None => Starter::Start,
4547 },
4548 }
4549}
4550
4551fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
4553 let branch = task.review_branch.take();
4554 let instruction = task.review_instruction.take();
4555 branch.filter(|branch| match &instruction {
4556 Some(authored) => *authored == task.instruction,
4557 None => previous.map_or_else(
4560 || task.review_of.as_ref() == Some(branch),
4561 |run| compatible_run_instruction(run, task),
4562 ),
4563 })
4564}
4565
4566fn may_resume_run(run: &RunState, task: &Task) -> bool {
4569 task.resume_override
4570 .as_ref()
4571 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
4572 || compatible_run_instruction(run, task)
4573}
4574
4575pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
4577 if let Some(authored) = &run.task_instruction {
4578 return compatible_instruction(authored, task);
4579 }
4580 if compatible_instruction(&run.instruction, task) {
4581 return true;
4582 }
4583 if run.reviewed_commits.is_some() {
4586 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
4587 return strip_answers_block(authored, task).trim() == task.instruction.trim();
4588 }
4589 if let Some(branch) = &task.review_of {
4590 let canonical = format!(
4591 "Review the branch `{branch}` (review-only: no implementation, no judging)."
4592 );
4593 let generated = format!(
4594 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
4595 );
4596 return task.instruction == canonical && run.instruction.starts_with(&generated);
4597 }
4598 }
4599 false
4600}
4601
4602pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
4604 strip_answers_block(old, task) == task.instruction
4605}
4606
4607fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
4610 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
4611 return fallback.to_path_buf();
4612 }
4613 task.repo.clone()
4614}
4615
4616const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
4620
4621fn answers_block(task: &Task, count: usize) -> String {
4623 let mut s = ANSWERS_HEADER.to_owned();
4624 for a in &task.answers[..count] {
4625 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
4626 }
4627 s
4628}
4629
4630fn resumption_block(r: &crate::queue::Resumption) -> String {
4633 format!(
4634 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
4635 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
4636 r.generation, r.at, r.instruction, r.answers_through,
4637 )
4638}
4639
4640fn append_answers(base: &str, task: &Task) -> String {
4642 let mut s = base.to_owned();
4643 if !task.answers.is_empty() {
4644 s.push_str(&answers_block(task, task.answers.len()));
4645 }
4646 if let Some(r) = &task.resumption {
4647 s.push_str(&resumption_block(r));
4648 }
4649 s
4650}
4651
4652fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
4656 let mut instruction = instruction;
4657 for approval in task
4658 .resumption
4659 .iter()
4660 .chain(task.resumption_history.iter().rev())
4661 {
4662 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
4663 instruction = base;
4664 break;
4665 }
4666 }
4667 for count in (1..=task.answers.len()).rev() {
4668 let block = answers_block(task, count);
4669 if let Some(base) = instruction.strip_suffix(&block) {
4670 return base;
4671 }
4672 }
4673 instruction
4674}
4675
4676fn instruction_for(task: &Task) -> String {
4684 append_answers(&task.instruction, task)
4685}
4686
4687fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
4699 append_answers(strip_answers_block(old_instruction, task), task)
4700}
4701
4702fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
4704 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
4705 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
4706 if let Some(cause) = task.hold_notice_cause.as_mut() {
4707 cause.target = failure.map(|f| f.target.clone());
4708 }
4709}
4710
4711#[derive(Debug, thiserror::Error)]
4712#[error("attachment `{name}` is recorded on the task but {path} is missing")]
4713struct MissingAttachment {
4714 name: String,
4715 path: PathBuf,
4716}
4717
4718fn task_attachments(
4720 queue: &Queue,
4721 task: &Task,
4722) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
4723 let paths = queue.attachment_paths(task);
4724 for (name, path) in task.attachments.iter().zip(&paths) {
4725 if !path.is_file() {
4726 return Err(MissingAttachment {
4727 name: name.clone(),
4728 path: path.clone(),
4729 });
4730 }
4731 }
4732 Ok(paths)
4733}
4734
4735fn prepare_instruction(
4746 starter: &Starter,
4747 old_instruction: Option<&str>,
4748 task: &Task,
4749) -> Option<String> {
4750 match starter {
4751 Starter::Start => Some(instruction_for(task)),
4752 Starter::Resume(_) => Some(resumed_instruction(
4753 old_instruction.expect("a resumed run always has a prior instruction"),
4754 task,
4755 )),
4756 Starter::Review(_) => None,
4757 }
4758}
4759
4760fn record(queue: &Queue, task: &mut Task) {
4764 if let Err(e) = queue.put(task) {
4765 tracing::error!("could not record task {}: {e:#}", task.short());
4766 let notice = Notice::error(
4767 "loop:record",
4768 "The loop could not save a task's state; check the disk.",
4769 )
4770 .link(Link::Task {
4771 id: task.id.clone(),
4772 });
4773 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
4775 Some(home) => notices::raise_in(home, notice),
4776 None => notices::raise(notice),
4777 }
4778 }
4779}
4780
4781fn runnable(queue: &Queue) -> Vec<Task> {
4787 let mut tasks: Vec<Task> = queue
4788 .list()
4789 .into_iter()
4790 .filter(|t| t.status.runnable())
4791 .collect();
4792 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)));
4793 tasks
4794}
4795
4796fn describe(state: &RunState) -> String {
4810 let p = phrases(&state.config.graph.language);
4811 let mut detail = if state.status == RunStatus::Stalled {
4812 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
4813 seats.sort_unstable();
4814 seats.dedup();
4815 if seats.is_empty() {
4816 p.quorum_lost.to_owned()
4817 } else {
4818 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
4819 }
4820 } else {
4821 format!("{}{}", p.run_ended, state.status.display_label())
4822 };
4823 if let Some(last) = state.events.last() {
4824 let unanswered = state
4828 .reviews
4829 .last()
4830 .filter(|r| {
4831 state.status == RunStatus::Blocked
4832 && last.node == "review"
4833 && r.incomplete()
4834 && r.blocking == 0
4835 && r.round == state.config.graph.review_rounds
4836 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
4837 })
4838 .map(|r| (r.expected - r.answered, r.round));
4839 match unanswered {
4840 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
4841 detail.push_str(&format!(
4842 " ({}: {})",
4843 last.node,
4844 (p.reviewers_never_answered)(missing, rounds)
4845 ));
4846 }
4847 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
4848 }
4849 }
4850 detail.push_str(&format!(" [run {}]", state.id));
4851 detail
4852}
4853
4854const DIAGNOSTIC_MAX: usize = 4_000;
4860
4861const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
4866
4867fn diagnostic(state: &RunState) -> Option<String> {
4881 let mut parts: Vec<String> = Vec::new();
4882
4883 for o in state.gate.iter().filter(|o| !o.ok()) {
4885 parts.push(format!(
4886 "gate `{}` failed ({:?}):\n{}",
4887 o.command,
4888 o.code,
4889 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
4890 ));
4891 }
4892
4893 if let Some(last) = state
4896 .events
4897 .iter()
4898 .rev()
4899 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
4900 {
4901 parts.push(last.message.clone());
4902 }
4903
4904 if state.viable().is_empty() {
4911 for c in &state.candidates {
4912 if let Some(evidence) = &c.verified_noop {
4913 parts.push(format!(
4914 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
4915 c.label
4916 ));
4917 } else if !c.summary.trim().is_empty() {
4918 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
4919 } else if let Some(why) = &c.failed {
4920 parts.push(format!("candidate {}: {why}", c.label));
4921 }
4922 }
4923 }
4924
4925 if parts.is_empty() {
4926 return None;
4927 }
4928 Some(crate::run::tail(
4933 &parts.join("\n\n"),
4934 DIAGNOSTIC_MAX.saturating_sub(100),
4935 ))
4936}
4937
4938fn label(status: RunStatus) -> &'static str {
4946 status.as_str()
4947}
4948
4949pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
4951 match mode {
4952 "none" => Ok(MergeMode::None),
4953 "local" => Ok(MergeMode::Local),
4954 "pr" => Ok(MergeMode::Pr),
4955 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
4956 }
4957}
4958
4959fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
4964 mutex
4965 .lock()
4966 .unwrap_or_else(std::sync::PoisonError::into_inner)
4967}
4968
4969#[cfg(test)]
4970mod tests {
4971 use super::*;
4972 use crate::queue::{Source, TaskStatus};
4973 use crate::run::{Candidate, CommandOutcome};
4974 use pretty_assertions::assert_eq;
4975
4976 fn task() -> Task {
4977 Task::new(
4978 "add retries".to_owned(),
4979 "add retries".to_owned(),
4980 PathBuf::from("/repo"),
4981 Source::Human,
4982 )
4983 }
4984
4985 #[test]
4986 fn selected_public_head_overrides_historical_resume_and_pin() {
4987 let mut t = task();
4988 let mut old = run_state(RunStatus::Blocked);
4989 let selection = crate::pr_recovery::Selection {
4990 url: "url".into(),
4991 branch: format!("magi/task-{}", t.id),
4992 head: "a".repeat(40),
4993 instruction: t.instruction.clone(),
4994 approved_at: Timestamp::now(),
4995 };
4996 t.resume_override = Some(crate::queue::OperatorResume {
4997 pinned_run: Some(old.id.clone()),
4998 question_id: "choice".into(),
4999 at: Timestamp::now(),
5000 conductor_rehold: None,
5001 forced: false,
5002 });
5003 assert_eq!(
5004 selected_starter(&selection, Some(&old), Some(&old.id)),
5005 Starter::Review(selection.branch.clone())
5006 );
5007 old.pr_recovery = Some(selection.clone());
5008 assert_eq!(
5009 selected_starter(&selection, Some(&old), Some(&old.id)),
5010 Starter::Review(selection.branch.clone())
5011 );
5012 old.reviewed_commits = Some(vec!["implementation".into()]);
5013 assert_eq!(
5014 selected_starter(&selection, Some(&old), Some(&old.id)),
5015 Starter::Resume(old.id.clone())
5016 );
5017 let mut changed = selection.clone();
5018 changed.head = "b".repeat(40);
5019 assert_eq!(
5020 selected_starter(&changed, Some(&old), Some(&old.id)),
5021 Starter::Review(changed.branch)
5022 );
5023 }
5024
5025 #[test]
5026 fn candidate_transplant_failure_is_one_actionable_hold() {
5027 let dir = tempfile::tempdir().unwrap();
5028 let q = Queue::at(dir.path().join("queue"));
5029 let mut t = task();
5030 let mut state = run_state(RunStatus::Blocked);
5031 state.event("prep", "cannot apply referenced commit: conflict");
5032 state.prep_failure = Some("conflict".into());
5033 t.start(state.id.clone());
5034 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
5035 assert_eq!(t.attempts, 1);
5036 assert_eq!(t.status, TaskStatus::Held);
5037 assert!(t.operator_held());
5038 assert!(!t.status.runnable());
5039 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
5040 let first = t.chat_notice_key().unwrap();
5041 t.chat_notices.push(first.clone());
5042 t.release_operator("test", None);
5043 state.id = "another-unprepared-run".into();
5044 t.start(state.id.clone());
5045 finish_attempt(
5046 3,
5047 &q,
5048 &mut t,
5049 &state,
5050 &[],
5051 Err(anyhow::anyhow!("still conflicts")),
5052 );
5053 assert_eq!(
5054 t.chat_notice_key().as_ref(),
5055 Some(&first),
5056 "no work happened"
5057 );
5058 t.release_operator("test", None);
5059 state.prep_failure = None;
5060 state.status = RunStatus::Failed;
5061 state.begin_pass();
5062 t.start(state.id.clone());
5063 finish_attempt(
5064 1,
5065 &q,
5066 &mut t,
5067 &state,
5068 &[],
5069 Err(anyhow::anyhow!("new gate failure")),
5070 );
5071 let failure = t.chat_notice_key().unwrap();
5072 assert_ne!(failure, first, "approved recovery reached a real failure");
5073 t.release_operator("test", None);
5074 state.begin_pass();
5075 t.start(state.id.clone());
5076 finish_attempt(
5077 1,
5078 &q,
5079 &mut t,
5080 &state,
5081 &[],
5082 Err(anyhow::anyhow!("new failure on resume")),
5083 );
5084 assert_ne!(
5085 t.chat_notice_key().as_ref(),
5086 Some(&failure),
5087 "a new settled pass"
5088 );
5089 }
5090
5091 #[test]
5092 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
5093 for source in ["cli", "web"] {
5094 let mut t = task();
5095 let mut run = run_state(RunStatus::Blocked);
5096 run.instruction = "Run A original criteria".into();
5097 run.task_instruction = Some(run.instruction.clone());
5098 t.start(run.id.clone());
5099 t.release_operator("choice", Some("resume-question"));
5100 t.resume_override = Some(crate::queue::OperatorResume {
5101 question_id: "resume-question".into(),
5102 at: Timestamp::now(),
5103 conductor_rehold: None,
5104 forced: true,
5105 pinned_run: Some(run.id.clone()),
5106 });
5107 t.hold_machine(Some("not enough free space to start a run".into()));
5108 let mut unchanged = t.clone();
5109 unchanged
5110 .edit("title changed".into(), unchanged.instruction.clone())
5111 .unwrap();
5112 unchanged.release_operator(source, None);
5113 assert_eq!(
5114 unchanged
5115 .resume_override
5116 .as_ref()
5117 .unwrap()
5118 .pinned_run
5119 .as_deref(),
5120 Some(run.id.as_str())
5121 );
5122 assert!(!compatible_run_instruction(&run, &unchanged));
5123 assert!(may_resume_run(&run, &unchanged));
5124
5125 t.edit(
5126 "approved recovery".into(),
5127 "Reuse measured PR head B; fix Clippy; review".into(),
5128 )
5129 .unwrap();
5130 assert!(t.resume_override.is_none());
5131 t.release_operator(source, None);
5132 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
5133 assert!(reloaded.resume_override.is_none());
5134 assert!(!may_resume_run(&run, &reloaded));
5135 let unfinished =
5136 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
5137 .filter(|_| may_resume_run(&run, &reloaded));
5138 assert_eq!(
5139 choose_starter(None, false, unfinished.as_deref()),
5140 Starter::Start
5141 );
5142 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
5143 assert_eq!(
5144 reloaded.resumption_history[0].question_id.as_deref(),
5145 Some("resume-question")
5146 );
5147
5148 unchanged.instruction = "Reuse measured PR head B".into();
5151 unchanged.hold_machine(Some("disk hold".into()));
5152 unchanged.release_operator(source, None);
5153 assert!(unchanged.resume_override.is_none());
5154 assert!(!may_resume_run(&run, &unchanged));
5155 }
5156 }
5157
5158 #[test]
5159 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
5160 let mut t = task();
5161 let mut run = run_state(RunStatus::Blocked);
5162 t.review_of = Some("approved-head".into());
5163 t.instruction =
5164 "Review the branch `approved-head` (review-only: no implementation, no judging)."
5165 .into();
5166 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
5167 run.reviewed_commits = Some(vec!["Commit log".into()]);
5168 t.start(run.id.clone());
5169 t.hold_machine(Some("interrupted review".into()));
5170 t.release_operator("cli", None);
5171 assert!(
5172 compatible_run_instruction(&run, &t),
5173 "legacy explicit review remains resumable"
5174 );
5175 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
5176 .filter(|_| compatible_run_instruction(&run, &t));
5177 assert_eq!(
5178 choose_starter(None, false, unfinished.as_deref()),
5179 Starter::Resume(run.id.clone())
5180 );
5181 run.task_instruction = Some(t.instruction.clone());
5182 assert!(compatible_run_instruction(&run, &t));
5183 let reloaded: RunState =
5184 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
5185 assert!(compatible_run_instruction(&reloaded, &t));
5186 t.edit(
5187 "new approved head".into(),
5188 "Reuse measured PR head B".into(),
5189 )
5190 .unwrap();
5191 assert!(!compatible_run_instruction(&run, &t));
5192
5193 t.review_of = None;
5195 t.instruction = "Fix Clippy on approved head".into();
5196 run.task_instruction = None;
5197 run.instruction = format!(
5198 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
5199 t.instruction
5200 );
5201 assert!(compatible_run_instruction(&run, &t));
5202 t.instruction.push('\n');
5203 assert!(
5204 compatible_run_instruction(&run, &t),
5205 "legacy generated review trimmed the original criteria"
5206 );
5207 t.hold_for_handover(Some("approved-head".into()), "refused".into());
5208 let mut unchanged = t.clone();
5209 assert_eq!(
5210 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
5211 Some("approved-head")
5212 );
5213 t.edit("new plan".into(), "Reuse measured PR head B".into())
5214 .unwrap();
5215 assert!(take_review_branch(&mut t, Some(&run)).is_none());
5216 assert!(!compatible_run_instruction(&run, &t));
5217 t.review_branch = Some("approved-head".into());
5218 t.review_instruction = None;
5219 assert!(
5220 take_review_branch(&mut t, Some(&run)).is_none(),
5221 "legacy retained candidate cannot replace a revised approved head"
5222 );
5223 }
5224
5225 #[test]
5226 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
5227 crate::run::pin_test_home();
5228 let dir = tempfile::tempdir().unwrap();
5229 let queue = Queue::at(dir.path().join("queue"));
5230 for resume in [true, false] {
5231 let state = run_state(RunStatus::Blocked);
5232 let mut t = task();
5233 t.start(state.id.clone());
5234 let action = if resume {
5235 ask::ChoiceAction::Resume {
5236 run: state.id.clone(),
5237 }
5238 } else {
5239 ask::ChoiceAction::Requeue
5240 };
5241 let mut q = action_question(&state.id, action);
5242 q.status = ask::QuestionStatus::Open;
5243 q.answer = None;
5244 q.answered_at = None;
5245 t.fail("run stopped with a question", 3);
5246 hold_if_runnable(&queue, &mut t);
5247 assert!(t.operator_decided_at.is_none());
5248 assert!(t.operator_held());
5249 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
5250 let expected = if resume {
5251 ActionDecision::Resume(state.id.clone())
5252 } else {
5253 ActionDecision::Requeue
5254 };
5255 assert_eq!(
5256 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5257 expected
5258 );
5259 t.hold_manual(Some("later explicit hold".into()));
5260 assert_eq!(
5261 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
5262 ActionDecision::Stale
5263 );
5264 }
5265 }
5266
5267 #[test]
5268 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
5269 let mut t = task();
5270 t.answers.push(crate::queue::AnsweredQuestion {
5271 question: "Inspect the remote?".into(),
5272 answer: "Inspect then wait".into(),
5273 });
5274 let old = instruction_for(&t);
5275 assert!(compatible_instruction(&old, &t));
5276 t.instruction = "Reuse approved PR head and review".into();
5277 t.release_operator("cli", None);
5278 assert!(!compatible_instruction(&old, &t));
5279 let approved = instruction_for(&t);
5280 assert!(approved.contains("waiting premises are superseded"));
5281 assert!(compatible_instruction(&approved, &t));
5282 assert_eq!(resumed_instruction(&approved, &t), approved);
5283 t.release_operator("web", None);
5284 assert!(compatible_instruction(&approved, &t));
5285 assert_eq!(t.resumption_history.len(), 1);
5286 t.start("run".into());
5287 assert!(instruction_for(&t).contains("Explicit operator resumption"));
5288 }
5289
5290 #[test]
5294 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
5295 let mut state = RunState::new(
5296 PathBuf::from("/repo"),
5297 "main".to_owned(),
5298 "abc1234".to_owned(),
5299 "task".to_owned(),
5300 Config::default(),
5301 );
5302 state.status = RunStatus::Blocked;
5303 state.resource_block = Some(crate::run::ResourceBlock {
5304 node: "graft".to_owned(),
5305 kind: "shared build cache".to_owned(),
5306 detail: "timed out".to_owned(),
5307 at: jiff::Timestamp::now(),
5308 });
5309 let mut t = task();
5310 t.start("20260902-000000-aaaa".to_owned());
5311 assert_eq!(t.attempts, 1);
5312 let verdict = Verdict {
5313 status: RunStatus::Blocked,
5314 left_pr: false,
5315 quota_hit: false,
5316 parked: false,
5317 no_viable_candidates: false,
5318 };
5319 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
5320 assert_eq!(t.status, TaskStatus::Held);
5321 assert_eq!(t.attempts, 0, "the attempt is refunded");
5322 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
5323 assert!(
5324 !t.operator_held(),
5325 "a machine hold, released without a human"
5326 );
5327
5328 let mut p = task();
5330 p.start("20260902-000000-bbbb".to_owned());
5331 settle_and_diagnose(
5332 &mut p,
5333 Verdict {
5334 parked: true,
5335 ..verdict
5336 },
5337 "parked",
5338 2,
5339 &state,
5340 );
5341 assert_eq!(p.status, TaskStatus::Parked);
5342 }
5343
5344 #[test]
5345 fn record_failure_notice_links_the_task() {
5346 let dir = tempfile::tempdir().unwrap();
5347 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
5350 let queue = Queue::at(dir.path().join("queue"));
5351 let mut t = interrupt_task("rec-fail-1");
5352 record(&queue, &mut t);
5353
5354 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
5355 assert_eq!(found.len(), 1);
5356 assert_eq!(found[0].key, "loop:record");
5357 assert_eq!(
5358 found[0].link,
5359 Some(Link::Task {
5360 id: "rec-fail-1".to_owned()
5361 })
5362 );
5363 }
5364
5365 fn interrupt_task(id: &str) -> Task {
5368 let mut t = task();
5369 t.id = id.to_owned();
5370 t.interrupt = true;
5371 t
5372 }
5373
5374 fn task_with_id(id: &str) -> Task {
5376 let mut t = task();
5377 t.id = id.to_owned();
5378 t
5379 }
5380
5381 fn urgent_task(id: &str) -> Task {
5383 let mut t = task();
5384 t.id = id.to_owned();
5385 t.urgent = true;
5386 t
5387 }
5388
5389 #[test]
5390 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
5391 let dir = tempfile::tempdir().unwrap();
5392 let queue = Queue::at(dir.path().join("queue"));
5393 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5394 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5395 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
5396 let mut first = task();
5397 queue.put(&mut first).unwrap();
5398 let stale_plain = queue.get(&first.id).unwrap();
5399 first.set_urgent(true).unwrap();
5400 queue.put(&mut first).unwrap();
5401 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
5402 let mut second = urgent_task("second");
5403 queue.put(&mut second).unwrap();
5404 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
5405 drop(queue.claim(&second.id).unwrap());
5406 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
5407 drop(dispatched);
5408 let stale_urgent = queue.get(&first.id).unwrap();
5409 first.set_urgent(false).unwrap();
5410 queue.put(&mut first).unwrap();
5411 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
5412 assert_eq!(urgent.available_permits(), 1);
5413 drop(queue.claim(&first.id).unwrap());
5414 drop(occupied);
5415 }
5416
5417 #[test]
5418 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
5419 let dir = tempfile::tempdir().unwrap();
5420 let queue = Queue::at(dir.path().join("queue"));
5421 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
5422 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
5423 let mut t = task();
5424 queue.put(&mut t).unwrap();
5425 for enabled in [true, false] {
5426 let stale = queue.get(&t.id).unwrap();
5427 t.set_urgent(enabled).unwrap();
5428 queue.put(&mut t).unwrap();
5429 assert_ne!(stale.urgent, enabled);
5430 let DispatchClaim {
5431 task: fresh,
5432 claim,
5433 permit,
5434 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
5435 assert_eq!(fresh.urgent, enabled);
5436 assert_eq!(ordinary.available_permits(), usize::from(enabled));
5437 assert_eq!(urgent.available_permits(), usize::from(!enabled));
5438 assert!(queue.claim(&t.id).is_err());
5439 assert_eq!(fresh.attempts, stale.attempts);
5440 drop((claim, permit));
5441 assert_eq!(ordinary.available_permits(), 1);
5442 assert_eq!(urgent.available_permits(), 1);
5443 }
5444 t.set_urgent(true).unwrap();
5445 queue.put(&mut t).unwrap();
5446 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
5447 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5448 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
5449 drop(queue.claim(&t.id).unwrap());
5450 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
5451 drop(occupied);
5452 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
5453 drop(queue.claim(&t.id).unwrap());
5454 let held = queue.claim(&t.id).unwrap();
5455 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5456 drop(held);
5457 t.status = TaskStatus::Held;
5458 queue.put(&mut t).unwrap();
5459 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5460 drop(queue.claim(&t.id).unwrap());
5461 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
5462 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
5463 drop(queue.claim(&t.id).unwrap());
5464 assert_eq!(urgent.available_permits(), 1);
5465 }
5466
5467 #[test]
5472 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
5473 assert_eq!(permit_kind(true, false), PermitKind::None);
5474 assert_eq!(permit_kind(true, true), PermitKind::None);
5475 }
5476
5477 #[test]
5482 fn permit_kind_separates_urgent_from_ordinary() {
5483 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
5484 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
5485 }
5486
5487 #[test]
5494 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
5495 let cfg = Config::default();
5496 let repo = Path::new("/any/repo/path");
5497
5498 let reason =
5499 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
5500 assert!(reason.contains("1024"), "{reason}");
5501 assert!(
5502 reason.contains(&cfg.disk.min_free_bytes.to_string()),
5503 "{reason}"
5504 );
5505
5506 assert_eq!(
5507 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
5508 None,
5509 "exactly at the floor is open"
5510 );
5511 assert_eq!(
5512 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
5513 None,
5514 "comfortably above the floor is open"
5515 );
5516 }
5517
5518 #[test]
5519 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
5520 let mut cfg = Config::default();
5521 cfg.disk.min_free_bytes = 0;
5522 let repo = Path::new("/any/repo/path");
5523 assert_eq!(
5524 disk_gate_with(repo, &cfg, |_| Ok(0)),
5525 None,
5526 "a zero floor never measures at all"
5527 );
5528 }
5529
5530 #[test]
5531 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
5532 let cfg = Config::default();
5533 let repo = Path::new("/any/repo/path");
5534 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
5535 .expect("a measurement failure must close the gate, not open it");
5536 assert!(reason.contains("could not measure"), "{reason}");
5537 }
5538
5539 #[test]
5540 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
5541 let ordinary = task();
5542 let next = advance_interrupt(
5543 Interrupt::Idle,
5544 std::slice::from_ref(&ordinary.id),
5545 std::slice::from_ref(&ordinary),
5546 );
5547 assert_eq!(next, Interrupt::Idle);
5548 }
5549
5550 #[test]
5551 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
5552 let marked = interrupt_task("marked");
5555 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5556 assert_eq!(next, Interrupt::Idle);
5557 }
5558
5559 #[test]
5560 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
5561 let marked = interrupt_task("marked");
5562 let next = advance_interrupt(
5563 Interrupt::Idle,
5564 &["running".to_owned()],
5565 std::slice::from_ref(&marked),
5566 );
5567 assert_eq!(
5568 next,
5569 Interrupt::Parking {
5570 parked: vec!["running".to_owned()],
5571 interrupt_task: "marked".to_owned(),
5572 }
5573 );
5574 }
5575
5576 #[test]
5585 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
5586 let marked = interrupt_task("marked");
5587
5588 let two = advance_interrupt(
5589 Interrupt::Idle,
5590 &["a".to_owned(), "b".to_owned()],
5591 std::slice::from_ref(&marked),
5592 );
5593 assert_eq!(two, Interrupt::Idle);
5594
5595 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
5596 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
5597 }
5598
5599 #[test]
5600 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
5601 let state = Interrupt::Parking {
5602 parked: vec!["running".to_owned()],
5603 interrupt_task: "marked".to_owned(),
5604 };
5605 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
5607 assert_eq!(still_going, state);
5608
5609 let stopped_but_not_yet_dispatched =
5613 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
5614 assert_eq!(stopped_but_not_yet_dispatched, state);
5615
5616 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
5618 assert_eq!(
5619 dispatched,
5620 Interrupt::Running {
5621 parked: vec!["running".to_owned()],
5622 interrupt_task: "marked".to_owned(),
5623 }
5624 );
5625 }
5626
5627 #[test]
5628 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
5629 let state = Interrupt::Running {
5630 parked: vec!["running".to_owned()],
5631 interrupt_task: "marked".to_owned(),
5632 };
5633 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
5634 assert_eq!(still_running, state);
5635
5636 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
5643 assert_eq!(
5644 ended,
5645 Interrupt::Resuming {
5646 parked: vec!["running".to_owned()]
5647 }
5648 );
5649 }
5650
5651 #[test]
5652 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
5653 let state = Interrupt::Resuming {
5654 parked: vec!["running".to_owned()],
5655 };
5656 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
5657 assert_eq!(still_waiting, state);
5658
5659 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
5660 assert_eq!(dispatched, Interrupt::Idle);
5661 }
5662
5663 #[test]
5669 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
5670 {
5671 let state = Interrupt::Parking {
5672 parked: vec!["running".to_owned()],
5673 interrupt_task: "marked".to_owned(),
5674 };
5675 let next = advance_interrupt(state, &[], &[]);
5678 assert_eq!(
5679 next,
5680 Interrupt::Resuming {
5681 parked: vec!["running".to_owned()]
5682 },
5683 "abandoning the interrupt must not abandon the resume it owes"
5684 );
5685 }
5686
5687 #[test]
5690 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
5691 let state = Interrupt::Resuming {
5692 parked: vec!["running".to_owned()],
5693 };
5694 let next = advance_interrupt(state, &[], &[]);
5695 assert_eq!(
5696 next,
5697 Interrupt::Idle,
5698 "nothing is left to wait for; the loop must not stay wedged"
5699 );
5700 }
5701
5702 #[test]
5703 fn disabled_by_config_the_sequence_can_never_leave_idle() {
5704 let marked = interrupt_task("marked");
5705 let next = advance_interrupt_tick(
5706 false,
5707 Interrupt::Idle,
5708 &["running".to_owned()],
5709 std::slice::from_ref(&marked),
5710 );
5711 assert_eq!(
5712 next,
5713 Interrupt::Idle,
5714 "an unmarked, unconfigured daemon must behave exactly as before"
5715 );
5716 }
5717
5718 #[test]
5719 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
5720 let state = Interrupt::Parking {
5721 parked: vec!["running".to_owned()],
5722 interrupt_task: "marked".to_owned(),
5723 };
5724 let candidates = vec![interrupt_task("marked"), task()];
5725 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5726 assert!(
5727 allowed.is_empty(),
5728 "nothing may dispatch - not even the interrupt task itself - \
5729 until the parked run has actually stopped"
5730 );
5731 }
5732
5733 #[test]
5747 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
5748 for state in [
5749 Interrupt::Parking {
5750 parked: vec!["running".to_owned()],
5751 interrupt_task: "marked".to_owned(),
5752 },
5753 Interrupt::Running {
5754 parked: vec!["running".to_owned()],
5755 interrupt_task: "marked".to_owned(),
5756 },
5757 Interrupt::Resuming {
5758 parked: vec!["running".to_owned()],
5759 },
5760 ] {
5761 let candidates = vec![
5762 interrupt_task("marked"),
5763 urgent_task("hot"),
5764 task_with_id("ordinary"),
5765 ];
5766 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
5767 assert!(
5768 !allowed.iter().any(|t| t.id == "hot"),
5769 "an urgent candidate must wait out the same gate as anything \
5770 else while the run it would run alongside has not actually \
5771 left flight, for state {state:?}: {allowed:?}"
5772 );
5773 }
5774 }
5775
5776 #[test]
5782 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
5783 let state = Interrupt::Resuming {
5784 parked: vec!["hot".to_owned()],
5785 };
5786 let candidates = vec![urgent_task("hot"), task()];
5787 let allowed = interrupt_gate(&state, &[], candidates);
5788 assert_eq!(
5789 allowed.iter().filter(|t| t.id == "hot").count(),
5790 1,
5791 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
5792 );
5793 }
5794
5795 #[test]
5796 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
5797 let state = Interrupt::Parking {
5798 parked: vec!["running".to_owned()],
5799 interrupt_task: "marked".to_owned(),
5800 };
5801 let other = task();
5802 let candidates = vec![interrupt_task("marked"), other.clone()];
5803 let allowed = interrupt_gate(&state, &[], candidates);
5804 assert_eq!(allowed.len(), 1);
5805 assert_eq!(allowed[0].id, "marked");
5806 }
5807
5808 #[test]
5809 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
5810 let state = Interrupt::Running {
5811 parked: vec!["running".to_owned()],
5812 interrupt_task: "marked".to_owned(),
5813 };
5814 let candidates = vec![task(), task()];
5815 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
5816 assert!(allowed.is_empty());
5817 }
5818
5819 #[test]
5826 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
5827 let state = Interrupt::Resuming {
5828 parked: vec!["a".to_owned(), "c".to_owned()],
5829 };
5830 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
5831 let allowed = interrupt_gate(&state, &[], candidates);
5832 assert_eq!(
5833 allowed.len(),
5834 1,
5835 "at most one candidate may be offered while resuming: {allowed:?}"
5836 );
5837 assert_eq!(allowed[0].id, "a");
5838 }
5839
5840 #[test]
5841 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
5842 let state = Interrupt::Resuming {
5843 parked: vec!["a".to_owned()],
5844 };
5845 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
5846 assert!(allowed.is_empty());
5847 }
5848
5849 #[test]
5855 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
5856 let running = task(); let marked = interrupt_task("marked");
5858
5859 let mut state = Interrupt::Idle;
5860 let in_flight = vec![running.id.clone()];
5862 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
5863 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
5864 assert!(gated.is_empty(), "still waiting on `running` to park");
5865
5866 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
5868 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
5869 assert_eq!(
5870 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5871 vec!["marked"],
5872 "only the interrupt task may be offered to the dispatcher now"
5873 );
5874
5875 state = advance_interrupt_tick(
5877 true,
5878 state,
5879 &["marked".to_owned()],
5880 std::slice::from_ref(&running),
5881 );
5882 let gated = interrupt_gate(
5883 &state,
5884 &["marked".to_owned()],
5885 vec![marked.clone(), running.clone()],
5886 );
5887 assert!(
5888 gated.is_empty(),
5889 "the parked run must not be offered back while the interrupt \
5890 task is still running"
5891 );
5892
5893 let other = task_with_id("other");
5897 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
5898 assert_eq!(
5899 state,
5900 Interrupt::Resuming {
5901 parked: vec![running.id.clone()]
5902 }
5903 );
5904 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
5905 assert_eq!(
5906 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5907 vec![running.id.as_str()],
5908 "exactly the parked run resumes - not the unrelated task, even \
5909 though it was offered first"
5910 );
5911
5912 state = advance_interrupt_tick(
5916 true,
5917 state,
5918 std::slice::from_ref(&running.id),
5919 std::slice::from_ref(&other),
5920 );
5921 assert_eq!(state, Interrupt::Idle);
5922 let gated = interrupt_gate(
5923 &state,
5924 std::slice::from_ref(&running.id),
5925 vec![other.clone()],
5926 );
5927 assert_eq!(
5928 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5929 vec![other.id.as_str()],
5930 "ordinary dispatch is unrestricted again"
5931 );
5932 }
5933
5934 #[test]
5935 fn every_run_status_settles_the_task_it_came_from() {
5936 let table = [
5938 (RunStatus::Merged, TaskStatus::Done, 1),
5939 (RunStatus::Ready, TaskStatus::Done, 1),
5940 (RunStatus::Stalled, TaskStatus::Failed, 0),
5941 (RunStatus::Blocked, TaskStatus::Failed, 1),
5942 (RunStatus::Failed, TaskStatus::Failed, 1),
5943 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
5944 (RunStatus::Prep, TaskStatus::Failed, 1),
5945 (RunStatus::Implementing, TaskStatus::Failed, 1),
5946 (RunStatus::Judging, TaskStatus::Failed, 1),
5947 (RunStatus::Deliberating, TaskStatus::Failed, 1),
5948 (RunStatus::Voting, TaskStatus::Failed, 1),
5949 (RunStatus::Reviewing, TaskStatus::Failed, 1),
5950 (RunStatus::Gating, TaskStatus::Failed, 1),
5951 ];
5952 for (run, want, attempts) in table {
5953 let mut t = task();
5954 t.start("20260902-000000-aaaa".to_owned());
5955 settle(
5956 &mut t,
5957 Verdict {
5958 status: run,
5959 left_pr: false,
5960 parked: false,
5961 quota_hit: matches!(run, RunStatus::Stalled),
5962 no_viable_candidates: false,
5963 },
5964 "why",
5965 2,
5966 );
5967 assert_eq!(t.status, want, "task status after {}", label(run));
5968 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
5969 }
5970 }
5971
5972 #[test]
5973 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
5974 let mut stalled = task();
5975 stalled.start("20260902-000000-aaaa".to_owned());
5976 settle(
5977 &mut stalled,
5978 Verdict {
5979 status: RunStatus::Stalled,
5980 left_pr: false,
5981 parked: false,
5982 quota_hit: true,
5983 no_viable_candidates: false,
5984 },
5985 "quota",
5986 1,
5987 );
5988 assert_eq!(stalled.attempts, 0);
5989 assert!(
5990 stalled.status.runnable(),
5991 "a machine problem must leave the task in line"
5992 );
5993
5994 let mut blocked = task();
5995 blocked.start("20260902-000000-aaaa".to_owned());
5996 settle(
5997 &mut blocked,
5998 Verdict {
5999 status: RunStatus::Blocked,
6000 left_pr: false,
6001 parked: false,
6002 quota_hit: false,
6003 no_viable_candidates: false,
6004 },
6005 "findings open",
6006 1,
6007 );
6008 assert_eq!(blocked.attempts, 1);
6009 assert_eq!(
6010 blocked.status,
6011 TaskStatus::Held,
6012 "the last attempt hands the task to a human"
6013 );
6014 }
6015
6016 #[test]
6017 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
6018 let mut delivered = task();
6021 delivered.start("20260903-080619-01c2".to_owned());
6022 settle(
6023 &mut delivered,
6024 Verdict {
6025 status: RunStatus::Blocked,
6026 left_pr: true,
6027 parked: false,
6028 quota_hit: false,
6029 no_viable_candidates: false,
6030 },
6031 "no check status",
6032 4,
6033 );
6034 assert_eq!(
6035 delivered.status,
6036 TaskStatus::Held,
6037 "a pull request waiting on CI or a person is not a retryable failure"
6038 );
6039 assert!(
6040 !delivered.status.runnable(),
6041 "the loop must not pick this task up again"
6042 );
6043 assert_eq!(
6044 delivered.last_error.as_deref(),
6045 Some("no check status"),
6046 "the operator needs to be told what the gate was waiting for"
6047 );
6048
6049 let mut empty_handed = task();
6052 empty_handed.start("20260903-080619-01c2".to_owned());
6053 settle(
6054 &mut empty_handed,
6055 Verdict {
6056 status: RunStatus::Blocked,
6057 left_pr: false,
6058 parked: false,
6059 quota_hit: false,
6060 no_viable_candidates: false,
6061 },
6062 "findings open",
6063 4,
6064 );
6065 assert_eq!(empty_handed.status, TaskStatus::Failed);
6066 assert!(empty_handed.status.runnable());
6067 }
6068
6069 #[test]
6070 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
6071 let mut noop = task();
6077 noop.start("20260912-131304-391f".to_owned());
6078 settle(
6079 &mut noop,
6080 Verdict {
6081 status: RunStatus::VerifiedNoop,
6082 left_pr: false,
6083 parked: false,
6084 quota_hit: false,
6085 no_viable_candidates: true,
6086 },
6087 "candidate A: already fixed by b32cfc4, on main",
6088 4,
6089 );
6090 assert_eq!(
6091 noop.status,
6092 TaskStatus::Held,
6093 "an unverified claim is a request for a human, not a failure"
6094 );
6095 assert!(
6096 !noop.status.runnable(),
6097 "the loop must not requeue this on the same unverified claim"
6098 );
6099 assert_eq!(noop.attempts, 1);
6104 }
6105
6106 #[test]
6107 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
6108 let mut parked = task();
6113 parked.start("20260903-183634-2d98".to_owned());
6114 settle(
6115 &mut parked,
6116 Verdict {
6117 status: RunStatus::Implementing,
6118 left_pr: false,
6119 quota_hit: false,
6120 parked: true,
6121 no_viable_candidates: false,
6122 },
6123 "parked after `implementing`",
6124 2,
6125 );
6126 assert_eq!(parked.attempts, 0, "a park is refunded");
6127 assert!(
6128 parked.status.runnable(),
6129 "and the task stays in line so the next loop resumes its run"
6130 );
6131 assert_eq!(parked.status, TaskStatus::Parked);
6132 assert_eq!(
6133 parked.park_reason.as_deref(),
6134 Some("parked after `implementing`"),
6135 "the card says where it stopped"
6136 );
6137 assert_eq!(parked.last_error, None, "nothing failed");
6138
6139 let mut broken = task();
6143 broken.start("20260903-183634-2d98".to_owned());
6144 settle(
6145 &mut broken,
6146 Verdict {
6147 status: RunStatus::Implementing,
6148 left_pr: false,
6149 quota_hit: false,
6150 parked: false,
6151 no_viable_candidates: false,
6152 },
6153 "returned mid-flight",
6154 2,
6155 );
6156 assert_eq!(broken.attempts, 1);
6157 }
6158
6159 #[test]
6160 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
6161 let mut flaky = task();
6166 flaky.start("20260903-123023-e633".to_owned());
6167 settle(
6168 &mut flaky,
6169 Verdict {
6170 status: RunStatus::Stalled,
6171 left_pr: false,
6172 parked: false,
6173 quota_hit: false,
6174 no_viable_candidates: false,
6175 },
6176 "verdict rests on 1 of 3 judges",
6177 2,
6178 );
6179 assert_eq!(
6180 flaky.attempts, 1,
6181 "flakiness spends an attempt, so `max_attempts` still bounds it"
6182 );
6183 assert!(flaky.status.runnable(), "and it is still worth retrying");
6184
6185 let mut limited = task();
6187 limited.start("20260903-123023-e633".to_owned());
6188 settle(
6189 &mut limited,
6190 Verdict {
6191 status: RunStatus::Stalled,
6192 left_pr: false,
6193 parked: false,
6194 quota_hit: true,
6195 no_viable_candidates: false,
6196 },
6197 "judge-2, judge-3 out of quota",
6198 2,
6199 );
6200 assert_eq!(limited.attempts, 0, "a quota window is refunded");
6201 assert!(limited.status.runnable());
6202
6203 let mut worn = task();
6206 for _ in 0..2 {
6207 worn.release();
6208 }
6209 worn.start("20260903-123023-e633".to_owned());
6210 worn.attempts = 2;
6211 settle(
6212 &mut worn,
6213 Verdict {
6214 status: RunStatus::Stalled,
6215 left_pr: false,
6216 parked: false,
6217 quota_hit: false,
6218 no_viable_candidates: false,
6219 },
6220 "no quorum again",
6221 2,
6222 );
6223 assert_eq!(worn.status, TaskStatus::Held);
6224 assert!(!worn.status.runnable());
6225 }
6226
6227 #[test]
6228 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
6229 let mut wiped_out = task();
6236 wiped_out.start("20260907-025000-a1b2".to_owned());
6237 settle(
6238 &mut wiped_out,
6239 Verdict {
6240 status: RunStatus::Failed,
6241 left_pr: false,
6242 parked: false,
6243 quota_hit: true,
6244 no_viable_candidates: true,
6245 },
6246 "no candidate produced a change; nothing to judge",
6247 2,
6248 );
6249 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
6250 assert!(
6251 wiped_out.status.runnable(),
6252 "a machine problem must leave the task in line"
6253 );
6254
6255 let mut partial_progress = task();
6261 partial_progress.start("20260907-025500-c3d4".to_owned());
6262 settle(
6263 &mut partial_progress,
6264 Verdict {
6265 status: RunStatus::Failed,
6266 left_pr: false,
6267 parked: false,
6268 quota_hit: true,
6269 no_viable_candidates: false,
6270 },
6271 "gate failed on the winning candidate",
6272 2,
6273 );
6274 assert_eq!(
6275 partial_progress.attempts, 1,
6276 "a candidate that actually produced a change spends the attempt \
6277 even though some other seat hit its quota"
6278 );
6279 assert!(partial_progress.status.runnable());
6280 }
6281
6282 #[test]
6283 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
6284 let mut t = task();
6290 t.start("20260907-025000-a1b2".to_owned());
6291 let mut state = run_state(RunStatus::Failed);
6292 state.quota.push(QuotaLoss {
6293 seat: "cand-a".to_owned(),
6294 node: "implement".to_owned(),
6295 at: Timestamp::now(),
6296 reset: None,
6297 });
6298 assert!(
6299 state.viable().is_empty(),
6300 "no candidate was added, so nothing is viable"
6301 );
6302 reclaim(&mut t, Some(state), 2, "en");
6303 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
6304 assert!(t.status.runnable());
6305 }
6306
6307 #[test]
6308 fn a_held_task_is_never_offered_to_the_loop() {
6309 let dir = tempfile::tempdir().unwrap();
6310 let queue = Queue::at(dir.path().to_path_buf());
6311 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
6312 let mut t = task();
6313 t.id = format!("2026090{n}-000000-000{n}");
6314 t.priority = priority;
6315 queue.put(&mut t).unwrap();
6316 }
6317 let mut held = task();
6318 held.id = "20260909-000000-9999".to_owned();
6319 held.priority = 99;
6320 held.hold_machine(None);
6321 queue.put(&mut held).unwrap();
6322
6323 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
6324 assert_eq!(order.len(), 3);
6325 assert!(!order.contains(&held.id));
6326 assert_eq!(
6327 order.first().cloned(),
6328 queue.next_runnable().map(|t| t.id),
6329 "the loop's first candidate is exactly what the queue offers"
6330 );
6331 assert_eq!(
6332 order,
6333 vec![
6334 "20260902-000000-0002".to_owned(),
6335 "20260903-000000-0003".to_owned(),
6336 "20260901-000000-0001".to_owned(),
6337 ],
6338 "priority first, then oldest, so nothing starves"
6339 );
6340 }
6341
6342 #[test]
6343 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
6344 let dir = tempfile::tempdir().unwrap();
6345 let queue = Queue::at(dir.path().to_path_buf());
6346 let mut old = task();
6347 old.id = "20260101-000000-old0".to_owned();
6348 queue.put(&mut old).unwrap();
6349 let mut fresh = task();
6350 fresh.id = "20260101-000000-new0".to_owned();
6351 queue.put(&mut fresh).unwrap();
6352
6353 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
6357 std::thread::sleep(Duration::from_millis(60));
6358 let live = queue.claim(&fresh.id).unwrap();
6359
6360 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6361 assert_eq!(swept, vec![old.id.clone()]);
6362 assert!(
6363 queue.claim(&old.id).is_ok(),
6364 "an unparseable lock older than the threshold is swept"
6365 );
6366 assert!(
6367 queue.claim(&fresh.id).is_err(),
6368 "a live pid protects its lock regardless of age"
6369 );
6370 drop(live);
6371 }
6372
6373 #[test]
6374 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
6375 let dir = tempfile::tempdir().unwrap();
6385 let queue = Queue::at(dir.path().to_path_buf());
6386 let mut t = task();
6387 t.id = "20260101-000000-live".to_owned();
6388 queue.put(&mut t).unwrap();
6389
6390 let claim = queue.claim(&t.id).unwrap();
6391 std::thread::sleep(Duration::from_millis(60));
6392
6393 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
6394 assert!(
6395 swept.is_empty(),
6396 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
6397 );
6398 assert!(
6399 queue.claim(&t.id).is_err(),
6400 "the lock still protects its task"
6401 );
6402 drop(claim);
6403 }
6404
6405 fn injected_dead_pid() -> u32 {
6408 std::process::id().checked_add(1).unwrap_or(1)
6409 }
6410
6411 #[test]
6412 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
6413 let dir = tempfile::tempdir().unwrap();
6414 let queue = Queue::at(dir.path().to_path_buf());
6415 let mut t = task();
6416 t.id = "20260101-000000-dead".to_owned();
6417 queue.put(&mut t).unwrap();
6418 let dead_pid = injected_dead_pid();
6419
6420 std::fs::write(
6425 dir.path().join(format!("{}.lock", t.id)),
6426 dead_pid.to_string(),
6427 )
6428 .unwrap();
6429
6430 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6431 pid != dead_pid
6432 });
6433 assert_eq!(
6434 swept,
6435 vec![t.id.clone()],
6436 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
6437 );
6438 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
6439 }
6440
6441 #[test]
6442 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
6443 let dir = tempfile::tempdir().unwrap();
6444 let queue = Queue::at(dir.path().to_path_buf());
6445 let mut t = task();
6446 t.id = "20260101-000000-late".to_owned();
6447 queue.put(&mut t).unwrap();
6448 let dead_pid = injected_dead_pid();
6449
6450 assert!(
6453 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
6454 "nothing has claimed the task yet"
6455 );
6456
6457 std::fs::write(
6460 dir.path().join(format!("{}.lock", t.id)),
6461 dead_pid.to_string(),
6462 )
6463 .unwrap();
6464
6465 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6469 pid != dead_pid
6470 });
6471 assert_eq!(swept, vec![t.id.clone()]);
6472 }
6473
6474 #[test]
6475 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
6476 crate::run::pin_test_home();
6481 let dir = tempfile::tempdir().unwrap();
6482 let queue = Queue::at(dir.path().to_path_buf());
6483 let mut t = task();
6484 t.id = "20260101-000000-crsh".to_owned();
6485 t.status = TaskStatus::Running;
6486 t.attempts = 1;
6487 t.runs.push("20260904-000000-4043".to_owned());
6491 queue.put(&mut t).unwrap();
6492 let dead_pid = injected_dead_pid();
6493
6494 std::fs::write(
6497 dir.path().join(format!("{}.lock", t.id)),
6498 dead_pid.to_string(),
6499 )
6500 .unwrap();
6501
6502 assert!(reclaim_orphaned_running(&queue, 2).is_empty());
6508 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
6509
6510 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
6511 pid != dead_pid
6512 });
6513 assert_eq!(swept, vec![t.id.clone()]);
6514
6515 let reclaimed = reclaim_orphaned_running(&queue, 2);
6516 assert_eq!(reclaimed, vec![t.id.clone()]);
6517 let after = queue.get(&t.id).unwrap();
6518 assert_eq!(
6519 after.status,
6520 TaskStatus::Held,
6521 "no run.json to recover from, so a human is asked"
6522 );
6523 assert_eq!(
6524 after.runs,
6525 vec!["20260904-000000-4043".to_owned()],
6526 "the crashed run's id is kept as evidence, not discarded"
6527 );
6528 }
6529
6530 #[test]
6531 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
6532 let dir = tempfile::tempdir().unwrap();
6533 let queue = Queue::at(dir.path().to_path_buf());
6534 let mut t = task();
6535 t.id = "20260101-000000-unknown".to_owned();
6536 queue.put(&mut t).unwrap();
6537 let dead_pid = injected_dead_pid();
6538 std::fs::write(
6539 dir.path().join(format!("{}.lock", t.id)),
6540 dead_pid.to_string(),
6541 )
6542 .unwrap();
6543
6544 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
6545 assert!(swept.is_empty(), "an unknown pid must keep its lock");
6546 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
6547 }
6548
6549 fn run_state_in(status: RunStatus, language: &str) -> RunState {
6550 let mut s = run_state(status);
6551 s.config.graph.language = language.to_owned();
6552 s
6553 }
6554
6555 fn unstarted_verdict(status: RunStatus) -> Verdict {
6556 Verdict {
6557 status,
6558 left_pr: false,
6559 quota_hit: false,
6560 parked: false,
6561 no_viable_candidates: false,
6562 }
6563 }
6564
6565 #[test]
6566 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
6567 let mut t = task();
6568 t.attempts = 1;
6569 settle_in(
6570 &mut t,
6571 unstarted_verdict(RunStatus::AlreadyInBase),
6572 "already in main",
6573 1,
6574 phrases("en"),
6575 );
6576 assert_eq!(t.status, TaskStatus::Done);
6577 assert_eq!(t.attempts, 0);
6578 }
6579
6580 #[test]
6581 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
6582 let reason = |language: &str| {
6583 let mut t = task();
6584 settle_in(
6585 &mut t,
6586 unstarted_verdict(RunStatus::Judging),
6587 "boom",
6588 1,
6589 phrases(language),
6590 );
6591 t.last_error.or(t.hold_reason).unwrap_or_default()
6592 };
6593 assert!(
6594 reason("en").starts_with("the graph stopped at `"),
6595 "{}",
6596 reason("en")
6597 );
6598 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
6599 assert!(reason("日本語").contains("boom"));
6600 assert_eq!(reason("fr"), reason("en"));
6601 }
6602
6603 #[test]
6604 fn describe_follows_the_run_language_and_keeps_the_run_id() {
6605 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
6606 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
6607 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
6608 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
6609 assert!(ja.contains("[run "), "{ja}");
6610 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
6611 assert!(ended.starts_with("run 終了: "), "{ended}");
6612 let mut de = run_state_in(RunStatus::Failed, "de");
6613 let mut en = run_state_in(RunStatus::Failed, "en");
6614 de.id = "same".to_owned();
6615 en.id = "same".to_owned();
6616 assert_eq!(describe(&de), describe(&en));
6617 }
6618
6619 #[test]
6620 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
6621 use crate::handover::Refused;
6622 let cases = [
6623 Refused::Foreign {
6624 branch: "b".into(),
6625 path: "/w/x".into(),
6626 why: "made by hand".into(),
6627 },
6628 Refused::Unsafe {
6629 branch: "b".into(),
6630 path: "/w/x".into(),
6631 why: "its worktree has uncommitted changes (a.rs)".into(),
6632 },
6633 Refused::ReleaseFailed {
6634 branch: "b".into(),
6635 path: "/w/x".into(),
6636 run: "ab12".into(),
6637 },
6638 ];
6639 for r in &cases {
6640 let en = (phrases("en").handover_refused)(r);
6641 assert_eq!(en, r.to_string());
6642 let ja = (phrases("ja").handover_refused)(r);
6643 assert!(
6644 !ja.contains("is checked out") && !ja.contains("try again"),
6645 "{ja}"
6646 );
6647 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
6648 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
6649 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
6650 }
6651 }
6652 assert!(phrases("en").handover_hint.contains("release the task"));
6653 assert!(phrases("ja").handover_hint.contains("解放"));
6654 }
6655
6656 #[test]
6657 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
6658 let build = |lang: &str| {
6659 let mut s = run_state_in(RunStatus::Blocked, lang);
6660 s.id = "same".to_owned();
6661 s.config.graph.review_rounds = 3;
6662 let mut r = review_round(3);
6663 r.expected = 3;
6664 r.answered = 1;
6665 s.reviews.push(r);
6666 s.event(
6667 "review",
6668 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
6669 );
6670 s
6671 };
6672 let en = describe(&build("en"));
6673 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
6674 let ja = describe(&build("ja"));
6675 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
6676 assert!(!ja.contains("never answered"), "{ja}");
6677 assert!(ja.contains("[run "), "{ja}");
6678
6679 let mut failed = build("ja");
6682 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
6683 command: "cargo test".to_owned(),
6684 code: Some(1),
6685 output_tail: String::new(),
6686 duration_ms: 0,
6687 resource_blocked: false,
6688 });
6689 failed.event("review", "stopped; e2e failed: cargo test");
6690 let ja = describe(&failed);
6691 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
6692 assert!(!ja.contains("席のレビュアー"), "{ja}");
6693 }
6694
6695 #[test]
6696 fn refusals_and_recovery_prose_follow_the_language() {
6697 let t = held_task_with("r1");
6698 let q = action_question(
6699 "r1",
6700 ask::ChoiceAction::Resume {
6701 run: "r1".to_owned(),
6702 },
6703 );
6704 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
6705 ActionDecision::Refuse(s) => s,
6706 other => panic!("{other:?}"),
6707 };
6708 assert!(refuse(phrases("en")).contains("could not be read: gone"));
6709 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
6710 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
6711
6712 let mut held = task();
6713 reclaim(&mut held, None, 2, "ja");
6714 assert!(held.hold_reason.unwrap().contains("保留にしました"));
6715 let mut held = task();
6716 reclaim(&mut held, None, 2, "xx");
6717 assert!(held.hold_reason.unwrap().contains("held for a human"));
6718 }
6719
6720 fn run_state(status: RunStatus) -> RunState {
6721 let mut state = RunState::new(
6722 PathBuf::from("/repo"),
6723 "main".to_owned(),
6724 "abc1234def".to_owned(),
6725 "add retries".to_owned(),
6726 Config::default(),
6727 );
6728 state.status = status;
6729 state
6730 }
6731
6732 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
6733 Candidate {
6734 index: 0,
6735 label,
6736 agent: "claude".to_owned(),
6737 branch: format!("magi/x/{label}"),
6738 worktree: PathBuf::from("/repo"),
6739 summary: summary.to_owned(),
6740 stat: String::new(),
6741 files: 0,
6742 commits: usize::from(!empty),
6743 empty,
6744 failed: failed.map(str::to_owned),
6745 verified_noop: None,
6746 duration_ms: 0,
6747 folded: false,
6748 }
6749 }
6750
6751 #[test]
6752 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
6753 let mut state = run_state(RunStatus::Blocked);
6754 state.gate = vec![
6755 CommandOutcome {
6756 command: "cargo make check".to_owned(),
6757 code: Some(0),
6758 output_tail: "ok".to_owned(),
6759 duration_ms: 0,
6760 resource_blocked: false,
6761 },
6762 CommandOutcome {
6763 command: "cargo test".to_owned(),
6764 code: Some(101),
6765 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
6766 duration_ms: 0,
6767 resource_blocked: false,
6768 },
6769 ];
6770 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
6771 assert!(d.contains("cargo test"), "{d}");
6772 assert!(
6773 !d.contains("cargo make check"),
6774 "a passing check is not a diagnostic: {d}"
6775 );
6776 assert!(d.contains("assertion failed"), "{d}");
6777 }
6778
6779 #[test]
6780 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
6781 let mut state = run_state(RunStatus::Blocked);
6782 state.event(
6783 "land",
6784 "stopped: the fixer produced no commit while 2 check(s) were failing \
6785 (build, lint); stopping instead of looping on an unchanged tree",
6786 );
6787 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
6788 assert!(d.contains("build"), "{d}");
6789 assert!(d.contains("lint"), "{d}");
6790 assert!(d.contains("fixer produced no commit"), "{d}");
6791 }
6792
6793 #[test]
6794 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
6795 let state = run_state(RunStatus::VerifiedNoop);
6800 let d = describe(&state);
6801 assert!(
6802 d.contains("agent-verified no-op"),
6803 "expected the display label, not the wire spelling: {d}"
6804 );
6805 assert!(!d.contains("verified_noop"), "{d}");
6806 }
6807
6808 #[test]
6809 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
6810 let mut state = run_state(RunStatus::Failed);
6816 state.candidates = vec![candidate(
6817 'A',
6818 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
6819 true,
6820 None,
6821 )];
6822 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
6823 assert!(d.contains("candidate A"), "{d}");
6824 assert!(d.contains("tagged v1.2.3"), "{d}");
6825 }
6826
6827 #[test]
6828 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
6829 let mut state = run_state(RunStatus::Failed);
6830 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
6831 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
6832 assert!(d.contains("candidate A"), "{d}");
6833 assert!(d.contains("agent timed out"), "{d}");
6834 }
6835
6836 #[test]
6837 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
6838 let mut state = run_state(RunStatus::Failed);
6841 state.candidates = vec![candidate('A', "did the work", false, None)];
6842 assert!(diagnostic(&state).is_none());
6843 }
6844
6845 #[test]
6846 fn diagnostic_is_bounded_however_much_a_run_printed() {
6847 let mut state = run_state(RunStatus::Blocked);
6848 state.gate = vec![
6849 CommandOutcome {
6850 command: "cargo test".to_owned(),
6851 code: Some(101),
6852 output_tail: "x".repeat(50_000),
6853 duration_ms: 0,
6854 resource_blocked: false,
6855 },
6856 CommandOutcome {
6857 command: "cargo clippy".to_owned(),
6858 code: Some(1),
6859 output_tail: "y".repeat(50_000),
6860 duration_ms: 0,
6861 resource_blocked: false,
6862 },
6863 ];
6864 state.candidates = vec![
6865 candidate('A', &"z".repeat(50_000), true, None),
6866 candidate('B', &"w".repeat(50_000), true, None),
6867 ];
6868 let d = diagnostic(&state).expect("plenty here to diagnose");
6869 assert!(
6870 d.len() <= DIAGNOSTIC_MAX,
6871 "diagnostic grew to {} bytes, unbounded",
6872 d.len()
6873 );
6874 }
6875
6876 #[test]
6877 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
6878 let mut state = run_state(RunStatus::Blocked);
6879 state.gate = vec![CommandOutcome {
6880 command: "cargo test".to_owned(),
6881 code: Some(101),
6882 output_tail: "assertion failed".to_owned(),
6883 duration_ms: 0,
6884 resource_blocked: false,
6885 }];
6886 let verdict = Verdict {
6887 status: RunStatus::Blocked,
6888 left_pr: false,
6889 quota_hit: false,
6890 parked: false,
6891 no_viable_candidates: false,
6892 };
6893
6894 let mut t = task();
6897 t.start("run-1".to_owned());
6898 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
6899 assert_eq!(t.status, TaskStatus::Failed);
6900 assert!(t.diagnostic.is_none());
6901
6902 t.start("run-2".to_owned());
6905 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
6906 assert_eq!(t.status, TaskStatus::Held);
6907 let d = t.diagnostic.expect("a held task must carry its diagnostic");
6908 assert!(d.contains("cargo test"), "{d}");
6909 }
6910
6911 #[test]
6912 fn a_held_task_names_the_open_question_it_is_waiting_on() {
6913 crate::run::pin_test_home();
6918 let home = crate::run::home();
6919 let state = run_state(RunStatus::VerifiedNoop);
6920 let mut q = ask::Question::new(
6921 state.id.clone(),
6922 "implement".to_owned(),
6923 "impl-A".to_owned(),
6924 "is this really a no-op?".to_owned(),
6925 String::new(),
6926 Vec::new(),
6927 );
6928 Questions::at(home.join("questions")).put(&mut q).unwrap();
6929
6930 let verdict = Verdict {
6931 status: RunStatus::VerifiedNoop,
6932 left_pr: false,
6933 quota_hit: false,
6934 parked: false,
6935 no_viable_candidates: false,
6936 };
6937 let mut t = task();
6938 t.start(state.id.clone());
6939 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
6940
6941 assert_eq!(t.status, TaskStatus::Held);
6942 let reason = t.hold_reason.expect("a held task must record why");
6943 assert!(
6944 reason.starts_with("run ended agent-verified no-op"),
6945 "the original settle reason must survive unchanged: {reason}"
6946 );
6947 assert!(
6948 reason.contains(q.short()),
6949 "the open question's id must be named so the notice is actionable: {reason}"
6950 );
6951 }
6952
6953 #[test]
6954 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
6955 crate::run::pin_test_home();
6956 let state = run_state(RunStatus::VerifiedNoop);
6957
6958 let verdict = Verdict {
6959 status: RunStatus::VerifiedNoop,
6960 left_pr: false,
6961 quota_hit: false,
6962 parked: false,
6963 no_viable_candidates: false,
6964 };
6965 let mut t = task();
6966 t.start(state.id.clone());
6967 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
6968
6969 assert_eq!(t.status, TaskStatus::Held);
6970 assert_eq!(
6971 t.hold_reason.as_deref(),
6972 Some("run ended agent-verified no-op"),
6973 "nothing to append when the question was already answered or never asked"
6974 );
6975 }
6976
6977 #[test]
6978 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
6979 crate::run::pin_test_home();
6980 let mut first = run_state(RunStatus::Blocked);
6981 first.id = "20260101-000000-sup1".to_owned();
6982 first.save().unwrap();
6983 let mut second = run_state(RunStatus::Merged);
6984 second.id = "20260101-000000-sup2".to_owned();
6985 second.save().unwrap();
6986
6987 let mut t = task();
6988 t.runs = vec![first.id.clone(), second.id.clone()];
6989 t.status = TaskStatus::Done;
6990
6991 supersede_prior_runs(&t, &crate::run::home());
6992
6993 assert_eq!(
6994 RunState::load(&first.id).unwrap().status,
6995 RunStatus::Superseded,
6996 "the first attempt's Blocked no longer needs anyone's attention"
6997 );
6998 assert_eq!(
6999 RunState::load(&second.id).unwrap().status,
7000 RunStatus::Merged,
7001 "the run that actually succeeded is left exactly as it was"
7002 );
7003 }
7004
7005 #[test]
7006 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
7007 crate::run::pin_test_home();
7013 let mut first = run_state(RunStatus::Blocked);
7014 first.id = "20260101-000000-sup9".to_owned();
7015 first.driver_pid = Some(std::process::id());
7018 first.driver_started_at = Some(
7019 crate::proc::process_started_at(std::process::id())
7020 .expect("this test process's own start time must be queryable"),
7021 );
7022 first.save().unwrap();
7023 let mut second = run_state(RunStatus::Merged);
7024 second.id = "20260101-000000-supa".to_owned();
7025 second.save().unwrap();
7026
7027 let mut t = task();
7028 t.runs = vec![first.id.clone(), second.id.clone()];
7029 t.status = TaskStatus::Done;
7030
7031 supersede_prior_runs(&t, &crate::run::home());
7032
7033 assert_eq!(
7034 RunState::load(&first.id).unwrap().status,
7035 RunStatus::Blocked,
7036 "a live driver_pid means something is still actually working this run, \
7037 even though no daemon claims it - rewriting under it would just be \
7038 undone the next time that process saves"
7039 );
7040 }
7041
7042 #[test]
7043 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
7044 let dir = tempfile::tempdir().unwrap();
7050 let home = dir.path().to_path_buf();
7051 let queue = Queue::at(dir.path().join("queue"));
7052
7053 let mut first = run_state(RunStatus::Blocked);
7054 first.id = "20260101-000000-supd".to_owned();
7055 first.driver_pid = Some(std::process::id());
7056 first.driver_started_at = Some(
7057 crate::proc::process_started_at(std::process::id())
7058 .expect("this test process's own start time must be queryable"),
7059 );
7060 first.save_under(&home).unwrap();
7061 let mut second = run_state(RunStatus::Merged);
7062 second.id = "20260101-000000-supe".to_owned();
7063 second.save_under(&home).unwrap();
7064
7065 let mut t = task();
7066 t.runs = vec![first.id.clone(), second.id.clone()];
7067 t.status = TaskStatus::Done;
7068 queue.put(&mut t).unwrap();
7069
7070 resweep_superseded_attempts(&queue, &home);
7071 assert_eq!(
7072 RunState::load_under(&first.id, &home).unwrap().status,
7073 RunStatus::Blocked,
7074 "still live on the first pass, so still untouched"
7075 );
7076
7077 let mut stale = RunState::load_under(&first.id, &home).unwrap();
7083 stale.driver_started_at = Some("1".to_owned());
7084 stale.save_under(&home).unwrap();
7085
7086 resweep_superseded_attempts(&queue, &home);
7087 assert_eq!(
7088 RunState::load_under(&first.id, &home).unwrap().status,
7089 RunStatus::Superseded,
7090 "the second pass catches up what the first one correctly skipped"
7091 );
7092 }
7093
7094 #[test]
7095 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
7096 crate::run::pin_test_home();
7097 let mut first = run_state(RunStatus::Blocked);
7098 first.id = "20260101-000000-sup3".to_owned();
7099 first.save().unwrap();
7100 let mut second = run_state(RunStatus::Blocked);
7101 second.id = "20260101-000000-sup4".to_owned();
7102 second.save().unwrap();
7103
7104 let mut t = task();
7105 t.runs = vec![first.id.clone(), second.id.clone()];
7106 t.status = TaskStatus::Failed;
7110
7111 supersede_prior_runs(&t, &crate::run::home());
7112
7113 assert_eq!(
7114 RunState::load(&first.id).unwrap().status,
7115 RunStatus::Blocked
7116 );
7117 assert_eq!(
7118 RunState::load(&second.id).unwrap().status,
7119 RunStatus::Blocked
7120 );
7121 }
7122
7123 #[test]
7124 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
7125 crate::run::pin_test_home();
7129 let mut first = run_state(RunStatus::Blocked);
7130 first.id = "20260101-000000-sup5".to_owned();
7131 first.save().unwrap();
7132
7133 let mut t = task();
7134 t.runs = vec![first.id.clone()];
7135 t.status = TaskStatus::Done;
7136
7137 supersede_prior_runs(&t, &crate::run::home());
7138
7139 assert_eq!(
7140 RunState::load(&first.id).unwrap().status,
7141 RunStatus::Blocked,
7142 "a single-attempt task has no earlier run to supersede"
7143 );
7144 }
7145
7146 #[test]
7147 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
7148 crate::run::pin_test_home();
7155 let mut first = run_state(RunStatus::Blocked);
7156 first.id = "20260101-000000-supb".to_owned();
7157 first.save().unwrap();
7158 let mut second = run_state(RunStatus::Failed);
7159 second.id = "20260101-000000-supc".to_owned();
7160 second.save().unwrap();
7161
7162 let mut t = task();
7163 t.runs = vec![first.id.clone(), second.id.clone()];
7164 t.status = TaskStatus::Done;
7165
7166 supersede_prior_runs(&t, &crate::run::home());
7167
7168 assert_eq!(
7169 RunState::load(&first.id).unwrap().status,
7170 RunStatus::Blocked,
7171 "the task's last attempt never landed, so there is nothing here \
7172 actually superseding it"
7173 );
7174 }
7175
7176 #[test]
7177 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
7178 crate::run::pin_test_home();
7182 let mut failed = run_state(RunStatus::Failed);
7183 failed.id = "20260101-000000-sup6".to_owned();
7184 failed.save().unwrap();
7185 let mut noop = run_state(RunStatus::VerifiedNoop);
7186 noop.id = "20260101-000000-sup7".to_owned();
7187 noop.save().unwrap();
7188 let mut winner = run_state(RunStatus::Ready);
7189 winner.id = "20260101-000000-sup8".to_owned();
7190 winner.save().unwrap();
7191
7192 let mut t = task();
7193 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
7194 t.status = TaskStatus::Done;
7195
7196 supersede_prior_runs(&t, &crate::run::home());
7197
7198 assert_eq!(
7199 RunState::load(&failed.id).unwrap().status,
7200 RunStatus::Failed
7201 );
7202 assert_eq!(
7203 RunState::load(&noop.id).unwrap().status,
7204 RunStatus::VerifiedNoop
7205 );
7206 }
7207
7208 fn approval_question(run: &str) -> ask::Question {
7209 ask::Question::new(
7210 run.to_owned(),
7211 land::APPROVAL_NODE.to_owned(),
7212 "land".to_owned(),
7213 "merge?".to_owned(),
7214 String::new(),
7215 vec!["merge".to_owned(), "hold".to_owned()],
7216 )
7217 }
7218
7219 #[test]
7220 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
7221 crate::run::pin_test_home();
7222 let mut state = run_state(RunStatus::Landing);
7223 state.id = "20260101-000000-fre1".to_owned();
7224 state.parked = true;
7225 state.save().unwrap();
7226 ask::Questions::open()
7227 .put(&mut approval_question(&state.id))
7228 .unwrap();
7229
7230 let mut t = task();
7231 t.runs.push(state.id.clone());
7232 assert_eq!(
7233 land_resume_state(&t),
7234 LandResume::StillWaiting,
7235 "nobody has answered and the timeout has not passed"
7236 );
7237 }
7238
7239 #[test]
7240 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
7241 crate::run::pin_test_home();
7242 let mut state = run_state(RunStatus::Gating);
7243 state.id = "20260101-000000-gt01".to_owned();
7244 state.parked = true;
7245 state.config.graph.answer_timeout = 60;
7246 state.github_text = Some(crate::run::GithubTextAsk {
7247 rewrite: None,
7248 fingerprint: "f".into(),
7249 title: true,
7250 body: false,
7251 categories: vec!["title-language".into()],
7252 question: None,
7253 asks: 1,
7254 resolved: false,
7255 chosen_title: None,
7256 choice: None,
7257 });
7258 state.save().unwrap();
7259 let store = ask::Questions::open();
7260 let mut q = ask::Question::new(
7261 state.id.clone(),
7262 crate::github_text::ASK_NODE.to_owned(),
7263 crate::github_text::ASK_SEAT.to_owned(),
7264 "s".into(),
7265 String::new(),
7266 vec!["use fallback".into()],
7267 );
7268 store.put(&mut q).unwrap();
7269 let mut t = task();
7270 t.runs.push(state.id.clone());
7271 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
7272 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7273 store.put(&mut q).unwrap();
7274 assert_eq!(land_resume_state(&t), LandResume::Ready);
7275 assert!(!store.get(&q.id).unwrap().status.open());
7276 state.github_text.as_mut().unwrap().resolved = true;
7278 state.save().unwrap();
7279 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
7280 }
7281
7282 #[test]
7283 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
7284 crate::run::pin_test_home();
7289 let mut state = run_state(RunStatus::Landing);
7290 state.id = "20260101-000000-exp1".to_owned();
7291 state.parked = true;
7292 state.config.graph.answer_timeout = 60;
7293 state.save().unwrap();
7294
7295 let store = ask::Questions::open();
7296 let mut q = approval_question(&state.id);
7297 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
7298 store.put(&mut q).unwrap();
7299
7300 let mut t = task();
7301 t.runs.push(state.id.clone());
7302 assert_eq!(
7303 land_resume_state(&t),
7304 LandResume::Ready,
7305 "an expired question must not be waited on forever"
7306 );
7307
7308 let after = store.get(&q.id).unwrap();
7309 assert!(
7310 !after.status.open(),
7311 "the question is abandoned, not silently ignored"
7312 );
7313 assert!(
7314 after.resolution().is_none(),
7315 "an abandoned question is not read as a decision"
7316 );
7317 }
7318
7319 #[test]
7320 fn reclaim_settles_a_running_task_against_its_last_run() {
7321 let mut t = task();
7322 t.start("20260904-000000-4043".to_owned());
7323 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
7324 assert_eq!(
7325 t.status,
7326 TaskStatus::Done,
7327 "a run that actually finished must not stay `running` forever"
7328 );
7329 }
7330
7331 #[test]
7332 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
7333 let mut t = task();
7334 t.start("20260904-000000-4043".to_owned());
7335 t.last_error = Some("earlier trouble".to_owned());
7336 let mut state = run_state(RunStatus::Judging);
7337 state.parked = true;
7338 reclaim(&mut t, Some(state), 1, "en");
7339 assert_eq!(t.status, TaskStatus::Parked);
7340 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
7341 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7342 assert!(t.park_reason.is_some());
7343 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
7344 assert!(!t.fresh_start);
7345 assert!(t.status.runnable());
7346 }
7347
7348 #[test]
7349 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
7350 let mut t = task();
7351 t.start("20260904-000000-4043".to_owned());
7352 t.last_error = Some("earlier trouble".to_owned());
7353 settle(
7354 &mut t,
7355 Verdict {
7356 status: RunStatus::Judging,
7357 left_pr: false,
7358 quota_hit: false,
7359 parked: true,
7360 no_viable_candidates: false,
7361 },
7362 "parked after `judging`",
7363 1,
7364 );
7365 assert_eq!(t.status, TaskStatus::Parked);
7366 assert_eq!(t.attempts, 0);
7367 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
7368 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
7369 assert_eq!(t.schema, crate::queue::SCHEMA);
7370 assert!(
7371 crate::notices::task_held(&t).is_none(),
7372 "a park is not news"
7373 );
7374 t.start("20260904-000000-4043".to_owned());
7375 assert_eq!(t.park_reason, None);
7376 }
7377
7378 #[test]
7379 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
7380 let mut t = task();
7384 t.start("20260904-000000-4043".to_owned());
7385 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
7386 assert_eq!(t.status, TaskStatus::Failed);
7387 assert!(t.status.runnable());
7388 }
7389
7390 #[test]
7391 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
7392 let mut t = task();
7393 t.start("20260904-000000-4043".to_owned());
7394 assert_eq!(t.attempts, 1);
7395 let mut run = run_state(RunStatus::Implementing);
7396 run.id = "20260904-000000-4043".to_owned();
7397 run.begin_pass();
7398 let cause = crate::run::ParkCause::Upgrade {
7399 from: "0.121.0".to_owned(),
7400 to: Some("0.121.1".to_owned()),
7401 };
7402 run.end_pass_parked(cause.clone());
7403 run.parked = true;
7404 let verdict = Verdict {
7405 status: run.status,
7406 left_pr: false,
7407 quota_hit: false,
7408 parked: true,
7409 no_viable_candidates: false,
7410 };
7411 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
7412 assert_eq!(t.attempts, 0);
7413 assert_eq!(t.passes.len(), 1);
7414 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7415 assert_eq!(t.passes[0].parked, Some(cause));
7416 assert_eq!(t.passes[0].pass_index, 0);
7417
7418 t.start("20260904-000000-4043".to_owned());
7421 run.begin_pass();
7422 run.status = RunStatus::Failed;
7423 let verdict = Verdict {
7424 status: RunStatus::Failed,
7425 left_pr: false,
7426 quota_hit: false,
7427 parked: false,
7428 no_viable_candidates: false,
7429 };
7430 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
7431 assert_eq!(t.passes.len(), 2);
7432 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7433 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
7434 assert_eq!(t.passes[1].pass_index, 1);
7435 assert_eq!(t.attempts, 1);
7436 }
7437
7438 #[test]
7439 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
7440 let mut t = task();
7441 t.start("20260904-000000-4043".to_owned());
7442 reclaim(&mut t, None, 2, "en");
7443 assert_eq!(t.status, TaskStatus::Held);
7444 assert!(
7445 t.last_error
7446 .as_deref()
7447 .is_some_and(|e| e.contains("running")),
7448 "the operator needs to know why this task was held"
7449 );
7450 }
7451
7452 #[test]
7453 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
7454 let dir = tempfile::tempdir().unwrap();
7455 let queue = Queue::at(dir.path().to_path_buf());
7456
7457 let mut orphaned = task();
7459 orphaned.id = "20260904-000000-orph".to_owned();
7460 orphaned.status = TaskStatus::Running;
7461 orphaned.attempts = 1;
7462 queue.put(&mut orphaned).unwrap();
7463
7464 let mut alive = task();
7465 alive.id = "20260904-000000-live".to_owned();
7466 alive.status = TaskStatus::Running;
7467 alive.attempts = 1;
7468 queue.put(&mut alive).unwrap();
7469 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
7470
7471 let mut queued = task();
7472 queued.id = "20260904-000000-wait".to_owned();
7473 queue.put(&mut queued).unwrap();
7474
7475 let reclaimed = reclaim_orphaned_running(&queue, 2);
7476 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
7477
7478 assert_eq!(
7479 queue.get(&orphaned.id).unwrap().status,
7480 TaskStatus::Held,
7481 "nothing was driving it and there was no run to recover"
7482 );
7483 assert_eq!(
7484 queue.get(&alive.id).unwrap().status,
7485 TaskStatus::Running,
7486 "a live claim must protect the task it belongs to"
7487 );
7488 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
7489 }
7490
7491 fn read_run_under(home: &Path, id: &str) -> RunState {
7497 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
7498 serde_json::from_str(&body).unwrap()
7499 }
7500
7501 #[test]
7502 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
7503 let dir = tempfile::tempdir().unwrap();
7504 let home = dir.path().to_path_buf();
7505 let now = Timestamp::now();
7506 let overrun_seat = || crate::run::ActiveSeat {
7507 node: "implement".to_owned(),
7508 started_at: now - jiff::SignedDuration::new(21_000, 0),
7509 timeout_secs: 3_600,
7510 attempt: 0,
7511 task: None,
7512 command: None,
7513 index: None,
7514 total: None,
7515 };
7516
7517 let mut dead = run_state(RunStatus::Implementing);
7518 dead.id = "20260101-000000-dead".to_owned();
7519 dead.active.insert("impl-A".to_owned(), overrun_seat());
7520 dead.driver_pid = Some(4242);
7523 dead.save_under(&home).unwrap();
7524
7525 let mut alive = run_state(RunStatus::Implementing);
7528 alive.id = "20260101-000000-aliv".to_owned();
7529 alive.active.insert("impl-A".to_owned(), overrun_seat());
7530 alive.save_under(&home).unwrap();
7531 let mut status = Status::new();
7532 status.current = vec![Current {
7533 task: "20260101-000000-task".to_owned(),
7534 run: alive.id.clone(),
7535 }];
7536 write_status_to(&home.join("daemon.json"), &status).unwrap();
7537
7538 let questions = Questions::at(home.join("questions"));
7542 let mut q = ask::Question::new(
7543 dead.id.clone(),
7544 "implement".to_owned(),
7545 "impl-A".to_owned(),
7546 "Which storage backend?".to_owned(),
7547 String::new(),
7548 vec!["SQLite".to_owned(), "Redis".to_owned()],
7549 );
7550 questions.put(&mut q).unwrap();
7551
7552 let abandoned = reclaim_abandoned_runs_with(
7553 &home,
7554 now,
7555 |pid| if pid == 4242 { Some(false) } else { None },
7556 |_| panic!("a query answering Dead outright needs no identity corroboration"),
7557 );
7558 assert_eq!(abandoned, vec![dead.id.clone()]);
7559
7560 let reloaded = read_run_under(&home, &dead.id);
7561 assert_eq!(reloaded.status, RunStatus::Failed);
7562 assert!(reloaded.active.is_empty());
7563 assert!(
7564 !questions.get(&q.id).unwrap().status.open(),
7565 "the failed run's own open question must be settled in the same pass"
7566 );
7567
7568 let still_alive = read_run_under(&home, &alive.id);
7569 assert_eq!(
7570 still_alive.status,
7571 RunStatus::Implementing,
7572 "a live daemon's claim protects it"
7573 );
7574 assert!(!still_alive.active.is_empty());
7575 }
7576
7577 #[test]
7587 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
7588 let dir = tempfile::tempdir().unwrap();
7589 let home = dir.path().to_path_buf();
7590 let now = Timestamp::now();
7591
7592 let mut manual = run_state(RunStatus::Reviewing);
7593 manual.id = "20260101-000000-manl".to_owned();
7594 manual.active.insert(
7595 "review-1".to_owned(),
7596 crate::run::ActiveSeat {
7597 node: "review".to_owned(),
7598 started_at: now - jiff::SignedDuration::new(21_000, 0),
7599 timeout_secs: 3_600,
7600 attempt: 0,
7601 task: None,
7602 command: None,
7603 index: None,
7604 total: None,
7605 },
7606 );
7607 manual.driver_pid = Some(4242);
7611 manual.driver_started_at = Some("1790000000".to_owned());
7612 manual.save_under(&home).unwrap();
7613
7614 let abandoned = reclaim_abandoned_runs_with(
7615 &home,
7616 now,
7617 |pid| if pid == 4242 { Some(true) } else { None },
7618 |pid| {
7619 if pid == 4242 {
7620 Some("1790000000".to_owned())
7621 } else {
7622 None
7623 }
7624 },
7625 );
7626 assert!(
7627 abandoned.is_empty(),
7628 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
7629 );
7630
7631 let reloaded = read_run_under(&home, &manual.id);
7632 assert_eq!(reloaded.status, RunStatus::Reviewing);
7633 assert!(!reloaded.active.is_empty());
7634 }
7635
7636 #[test]
7637 fn an_already_claimed_task_is_skipped_rather_than_failed() {
7638 let dir = tempfile::tempdir().unwrap();
7639 let queue = Queue::at(dir.path().to_path_buf());
7640 let mut only = task();
7641 queue.put(&mut only).unwrap();
7642
7643 let _elsewhere = queue.claim(&only.id).unwrap();
7644 let candidates = runnable(&queue);
7645 assert_eq!(candidates.len(), 1, "the task is still runnable");
7646 assert!(
7647 queue.claim(&candidates[0].id).is_err(),
7648 "the loop cannot take a claim somebody else holds"
7649 );
7650
7651 let after = queue.get(&only.id).unwrap();
7652 assert_eq!(after.status, TaskStatus::Queued);
7653 assert_eq!(
7654 after.attempts, 0,
7655 "losing the race is not an attempt at the task"
7656 );
7657 assert_eq!(after.last_error, None);
7658 }
7659
7660 #[test]
7661 fn the_status_file_round_trips_and_its_heartbeat_advances() {
7662 let dir = tempfile::tempdir().unwrap();
7663 let path = dir.path().join("daemon.json");
7664
7665 let mut status = Status::new();
7666 status.idle = false;
7667 status.completed = 7;
7668 status.current = vec![Current {
7669 task: "20260902-000000-t111".to_owned(),
7670 run: "20260902-000001-r111".to_owned(),
7671 }];
7672 write_status_to(&path, &status).unwrap();
7673 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7674 assert_eq!(first.schema, SCHEMA);
7675 assert_eq!(first.pid, std::process::id());
7676 assert!(!first.idle);
7677 assert_eq!(first.completed, 7);
7678 assert_eq!(first.current, status.current);
7679 assert!(
7680 !path.with_extension("json.tmp").exists(),
7681 "the temp file is renamed, not left behind"
7682 );
7683
7684 std::thread::sleep(Duration::from_millis(5));
7685 status.updated_at = Timestamp::now();
7686 status.polls = 3;
7687 write_status_to(&path, &status).unwrap();
7688 let second: Status =
7689 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
7690 assert!(
7691 second.updated_at > first.updated_at,
7692 "a reader can only detect staleness if the heartbeat moves"
7693 );
7694 assert_eq!(
7695 second.started_at, first.started_at,
7696 "the start time is not a heartbeat"
7697 );
7698 assert_eq!(second.polls, 3);
7699 }
7700
7701 #[test]
7702 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
7703 let dir = tempfile::tempdir().unwrap();
7704
7705 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
7706
7707 let mut status = Status::new();
7708 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
7709 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7710 let stale = read_status(dir.path()).unwrap();
7711 assert!(
7712 !stale.running(Timestamp::now()),
7713 "a minute without a heartbeat is a dead daemon, not a busy one"
7714 );
7715 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
7716
7717 status.updated_at = Timestamp::now();
7718 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7719 let fresh = read_status(dir.path()).unwrap();
7720 assert!(fresh.running(Timestamp::now()));
7721 }
7722
7723 #[test]
7724 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
7725 let dir = tempfile::tempdir().unwrap();
7726 let now = Timestamp::now();
7727 let mine = "20260903-080619-01c2";
7728
7729 assert!(
7730 !is_working_on(dir.path(), mine, now),
7731 "no status file means nobody is working on anything"
7732 );
7733
7734 let mut status = Status::new();
7735 status.current = vec![Current {
7736 task: "20260903-080340-0167".to_owned(),
7737 run: mine.to_owned(),
7738 }];
7739 status.updated_at = now;
7740 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7741 assert!(is_working_on(dir.path(), mine, now));
7742 assert!(
7743 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
7744 "a daemon busy with one run is not working on another"
7745 );
7746
7747 status.updated_at = now - jiff::SignedDuration::from_secs(600);
7750 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7751 assert!(
7752 !is_working_on(dir.path(), mine, now),
7753 "a stale heartbeat is a dead daemon, so its run is a leftover"
7754 );
7755 }
7756
7757 #[test]
7758 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
7759 let dir = tempfile::tempdir().unwrap();
7760 let now = Timestamp::now();
7761
7762 assert!(
7763 !is_working_on_short(dir.path(), "01c2", now),
7764 "no status file means nobody is working on anything"
7765 );
7766
7767 let mut status = Status::new();
7768 status.current = vec![Current {
7769 task: "20260903-080340-0167".to_owned(),
7770 run: "20260903-080619-01c2".to_owned(),
7771 }];
7772 status.updated_at = now;
7773 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
7774 assert!(
7775 is_working_on_short(dir.path(), "01c2", now),
7776 "the run's short id is the last block of its full id"
7777 );
7778 assert!(
7779 !is_working_on_short(dir.path(), "3cbf", now),
7780 "a daemon busy with one worktree bay is not working on another"
7781 );
7782 }
7783
7784 #[test]
7785 fn a_newer_status_file_still_yields_a_reading() {
7786 let dir = tempfile::tempdir().unwrap();
7787 std::fs::write(
7790 dir.path().join("daemon.json"),
7791 serde_json::json!({
7792 "schema": 2,
7793 "updated_at": Timestamp::now().to_string(),
7794 "idle": true,
7795 "surprise": { "nested": [1, 2, 3] },
7796 })
7797 .to_string(),
7798 )
7799 .unwrap();
7800
7801 let reading = read_status(dir.path()).expect("a forward-compatible read");
7802 assert!(reading.running(Timestamp::now()));
7803 assert!(reading.idle);
7804 assert!(reading.current.is_empty());
7805 }
7806
7807 #[test]
7808 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
7809 let dir = tempfile::tempdir().unwrap();
7815 std::fs::write(
7816 dir.path().join("daemon.json"),
7817 serde_json::json!({
7818 "schema": 1,
7819 "pid": 4242,
7820 "updated_at": Timestamp::now().to_string(),
7821 "idle": false,
7822 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
7823 "completed": 3,
7824 "polls": 9,
7825 })
7826 .to_string(),
7827 )
7828 .unwrap();
7829
7830 let reading = read_status(dir.path()).expect("an older shape must still parse");
7831 assert!(reading.running(Timestamp::now()));
7832 assert_eq!(
7833 reading.current,
7834 vec![Current {
7835 task: "20260902-140501-aaaa".to_owned(),
7836 run: "20260902-140502-bbbb".to_owned(),
7837 }]
7838 );
7839 }
7840
7841 #[test]
7842 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
7843 let dir = tempfile::tempdir().unwrap();
7844 std::fs::write(
7845 dir.path().join("daemon.json"),
7846 serde_json::json!({
7847 "schema": 1,
7848 "updated_at": Timestamp::now().to_string(),
7849 "idle": true,
7850 "current": null,
7851 })
7852 .to_string(),
7853 )
7854 .unwrap();
7855 let with_null = read_status(dir.path()).expect("null must still parse");
7856 assert!(with_null.current.is_empty());
7857
7858 std::fs::write(
7859 dir.path().join("daemon.json"),
7860 serde_json::json!({
7861 "schema": 1,
7862 "updated_at": Timestamp::now().to_string(),
7863 "idle": true,
7864 })
7865 .to_string(),
7866 )
7867 .unwrap();
7868 let absent = read_status(dir.path()).expect("a missing field must still parse");
7869 assert!(absent.current.is_empty());
7870 }
7871
7872 #[test]
7873 fn a_task_without_a_repository_runs_in_the_daemons_default() {
7874 let fallback = Path::new("/default");
7875 let mut blank = task();
7876 blank.repo = PathBuf::new();
7877 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
7878 let mut dot = task();
7879 dot.repo = PathBuf::from(".");
7880 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
7881 assert_eq!(
7882 repo_for(&task(), fallback),
7883 PathBuf::from("/repo"),
7884 "a task that names a repository keeps it"
7885 );
7886 }
7887
7888 #[test]
7889 fn task_implementers_edit_inherits_daemon_config_and_merge() {
7890 let dir = tempfile::tempdir().unwrap();
7891 let path = dir.path().join("explicit.toml");
7892 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
7893 let opts = Opts {
7894 config: Some(path),
7895 merge: Some("none".into()),
7896 ..Default::default()
7897 };
7898 let mut t = task();
7899 t.set_implementers(
7900 2,
7901 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7902 .unwrap(),
7903 )
7904 .unwrap();
7905 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
7906 assert_eq!(cfg.graph.implementers, 2);
7907 assert_eq!(cfg.graph.judges, 1);
7908 assert_eq!(cfg.merge.mode, MergeMode::None);
7909 t.set_implementers(1, &cfg).unwrap();
7910 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
7911 apply_task_implementers(&mut cfg, &t);
7912 assert_eq!(cfg.graph.implementers, 1);
7913 assert_eq!(cfg.graph.judges, 1);
7914 assert_eq!(cfg.merge.mode, MergeMode::None);
7915 }
7916
7917 #[test]
7918 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
7919 let dir = tempfile::tempdir().unwrap();
7920 let repo = dir.path().join("repo");
7921 std::fs::create_dir(&repo).unwrap();
7922 std::fs::write(
7923 repo.join("magi.toml"),
7924 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
7925 )
7926 .unwrap();
7927 let daemon_config = dir.path().join("daemon.toml");
7928 std::fs::write(
7929 &daemon_config,
7930 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
7931 )
7932 .unwrap();
7933 let opts = Opts {
7934 config: Some(daemon_config),
7935 merge: Some("pr".into()),
7936 ..Default::default()
7937 };
7938 let q = Queue::at(dir.path().join("queue"));
7939 let validation: Config =
7940 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7941 .unwrap();
7942 for record in ["{}", r#"{"candidates":2}"#] {
7944 let overrides = serde_json::from_str(record).unwrap();
7945 let filing = crate::direct::Filing {
7946 instruction: "work".into(),
7947 title: "direct".into(),
7948 repo: repo.clone(),
7949 source: crate::queue::Source::Human,
7950 solo: false,
7951 overrides,
7952 review_of: None,
7953 };
7954 let filed =
7955 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
7956 let crate::direct::Filed::Standalone {
7957 mut task,
7958 claim: _claim,
7959 } = filed
7960 else {
7961 panic!("local filing")
7962 };
7963 let before = prepare_for(&repo, &opts, &task).unwrap();
7964 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
7965 assert_eq!(
7966 before.merge.mode,
7967 MergeMode::None,
7968 "daemon --merge must not leak"
7969 );
7970 for n in [2, 1, 2] {
7971 task.set_implementers(n, &validation).unwrap();
7972 q.put(&mut task).unwrap();
7973 task = q.get(&task.id).unwrap();
7974 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
7975 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
7976 apply_task_implementers(&mut cfg, &task);
7977 assert_eq!(cfg.graph.implementers, usize::from(n));
7978 assert_eq!(cfg.graph.judges, 2);
7979 assert_eq!(cfg.merge.mode, MergeMode::None);
7980 }
7981 }
7982 }
7983
7984 #[test]
7985 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
7986 let cfg: Config =
7987 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
7988 .unwrap();
7989 let mut t = task();
7990 t.review_of = Some("review-target".into());
7991 t.hold_manual(Some("retry".into()));
7992 let mut previous = RunState::new(
7993 PathBuf::from("/repo"),
7994 "main".into(),
7995 "abc1234".into(),
7996 "review".into(),
7997 cfg.clone(),
7998 );
7999 previous.config.graph.implementers = 2;
8000 for count in [1, 3] {
8001 t.set_implementers(count, &cfg).unwrap();
8002 t.release();
8003 assert_eq!(t.review_of.as_deref(), Some("review-target"));
8004 for proposed in [
8005 Starter::Resume(previous.id.clone()),
8006 Starter::Review("recovery-target".into()),
8007 ] {
8008 assert_eq!(
8009 starter_with_implementers(&t, Some(&previous), proposed),
8010 Starter::Review("review-target".into())
8011 );
8012 }
8013 let mut configured = cfg.clone();
8014 apply_task_implementers(&mut configured, &t);
8015 assert_eq!(configured.graph.implementers, usize::from(count));
8016 assert_eq!(previous.config.graph.implementers, 2);
8017 }
8018 }
8019
8020 #[test]
8021 fn task_implementers_dispatch_and_resume_agree() {
8022 let mut t = task();
8023 let cfg: crate::config::Config =
8024 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
8025 .unwrap();
8026 let mut run = RunState::new(
8027 PathBuf::from("/repo"),
8028 "main".into(),
8029 "abc1234".into(),
8030 "work".into(),
8031 cfg.clone(),
8032 );
8033 run.config.graph.implementers = 2;
8034 for n in [2, 1, 3, 2] {
8035 t.set_implementers(n, &cfg).unwrap();
8036 let mut dispatched = cfg.clone();
8037 apply_task_implementers(&mut dispatched, &t);
8038 assert_eq!(dispatched.graph.implementers, usize::from(n));
8039 assert_eq!(implementers_match(&t, &run), n == 2);
8040 for proposed in [
8041 Starter::Resume(run.id.clone()),
8042 Starter::Review("old-branch".into()),
8043 ] {
8044 let expected = if n == 2 {
8045 proposed.clone()
8046 } else {
8047 Starter::Start
8048 };
8049 assert_eq!(
8050 starter_with_implementers(&t, Some(&run), proposed),
8051 expected
8052 );
8053 }
8054 assert_eq!(
8055 run.config.graph.implementers, 2,
8056 "history must not be rewritten"
8057 );
8058 }
8059 t.solo = false;
8060 t.overrides = None;
8061 assert!(implementers_match(&t, &run));
8062 }
8063
8064 #[test]
8065 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
8066 let mut solo_cfg = Config::default();
8072 solo_cfg.graph.implementers = 3;
8073 let mut solo_task = task();
8074 solo_task.solo = true;
8075 apply_task_implementers(&mut solo_cfg, &solo_task);
8076 assert_eq!(solo_cfg.graph.implementers, 1);
8077
8078 let mut plain_cfg = Config::default();
8079 plain_cfg.graph.implementers = 3;
8080 let plain_task = task();
8081 assert!(!plain_task.solo);
8082 apply_task_implementers(&mut plain_cfg, &plain_task);
8083 assert_eq!(
8084 plain_cfg.graph.implementers, 3,
8085 "a task that did not ask to run alone keeps the config's candidates"
8086 );
8087 }
8088
8089 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
8090 QuotaLoss {
8091 seat: seat.into(),
8092 node: "judge".into(),
8093 at: at.parse().unwrap(),
8094 reset: reset.map(str::to_string),
8095 }
8096 }
8097
8098 #[test]
8099 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
8100 let old: Vec<QuotaLoss> = (1..=4)
8101 .map(|i| {
8102 loss(
8103 &format!("judge-{i}"),
8104 "2026-09-23T05:23:00Z",
8105 Some("2:40pm (Asia/Tokyo)"),
8106 )
8107 })
8108 .collect();
8109 let fresh = losses_this_attempt(&old, &old);
8110 assert!(fresh.is_empty());
8111 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
8112 }
8114
8115 #[test]
8116 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
8117 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
8118 let now = Timestamp::now();
8119 let mut after = old.clone();
8120 after.push(loss("judge-2", &now.to_string(), None));
8121 let fresh = losses_this_attempt(&old, &after);
8122 assert_eq!(fresh, vec![after[1].clone()]);
8123 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
8124 assert_eq!(
8125 until,
8126 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
8127 );
8128 }
8129
8130 #[test]
8131 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
8132 let before = vec![
8135 loss("judge-1", "2026-09-23T05:23:00Z", None),
8136 loss("judge-2", "2026-09-23T05:24:00Z", None),
8137 ];
8138 let after = vec![
8139 loss("judge-2", "2026-09-23T05:24:00Z", None),
8140 loss("judge-1", "2026-09-24T01:00:00Z", None),
8141 ];
8142 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
8143 }
8144
8145 #[test]
8146 fn merge_overrides_are_parsed_or_refused() {
8147 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
8148 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
8149 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
8150 assert!(merge_mode("squash").is_err());
8151 }
8152
8153 #[test]
8154 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
8155 let now = Timestamp::now();
8156 let fallback = Duration::from_secs(300);
8157 let cap = Duration::from_secs(1800);
8158
8159 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
8161
8162 let soon = now + jiff::SignedDuration::from_secs(600);
8164 assert_eq!(
8165 quota_wait(Some(soon), now, fallback, cap),
8166 Duration::from_secs(600)
8167 );
8168
8169 let past = now - jiff::SignedDuration::from_secs(60);
8172 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
8173
8174 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
8177 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
8178 }
8179
8180 #[test]
8181 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
8182 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8183
8184 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
8185 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
8186
8187 let already_past =
8191 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
8192 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
8193
8194 assert!(
8195 parse_reset_hint("session limit reached", now, now).is_none(),
8196 "free text with no recognised shape is not guessed at"
8197 );
8198 assert!(
8199 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
8200 "an unresolvable zone name is not guessed at either"
8201 );
8202 }
8203
8204 #[test]
8205 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
8206 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
8207
8208 let at = parse_reset_hint(
8209 "You've hit your usage limit. Visit \
8210 https://chatgpt.com/codex/settings/usage to purchase more \
8211 credits or try again at Sep 19th, 2026 5:10 PM.",
8212 now,
8213 now,
8214 )
8215 .expect("the codex reset wording is a recognised shape");
8216 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
8217
8218 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
8223 .expect("an explicit year needs no rollover");
8224 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
8225
8226 assert!(
8227 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
8228 "a two-digit year is not the documented shape and is not guessed at"
8229 );
8230 assert!(
8231 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
8232 "a four-letter month name is not the documented three-letter abbreviation"
8233 );
8234 assert!(
8235 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
8236 "an explicit zone on the dated shape is a format nobody has \
8237 documented, and is refused rather than guessed at as UTC"
8238 );
8239 }
8240
8241 #[test]
8242 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
8243 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
8244 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
8245
8246 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
8247 assert_eq!(at.as_second() - recorded.as_second(), 3769);
8248
8249 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
8250 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
8251
8252 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
8253 assert!(
8254 parse_reset_hint(bad, now, recorded).is_none(),
8255 "{bad:?} must not be guessed at"
8256 );
8257 }
8258 }
8259
8260 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
8264 let config = dir.join("magi.toml");
8265 std::fs::write(
8266 &config,
8267 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
8268 )
8269 .unwrap();
8270 let opts = Opts {
8271 poll: Duration::from_secs(30),
8272 config: Some(config),
8273 repo: dir.join("repo"),
8277 ..Opts::default()
8278 };
8279 let home = dir.join("home");
8288 let worktrees = dir.join("wt");
8289 (
8290 opts,
8291 Queue::at(dir.join("queue")),
8292 home.join("daemon.json"),
8293 home,
8294 worktrees,
8295 )
8296 }
8297
8298 #[test]
8299 fn a_stop_is_idempotent_and_once_set_stays_set() {
8300 let stop = Stop::new();
8301 assert!(!stop.stopped());
8302
8303 stop.stop();
8304 assert!(stop.stopped());
8305 stop.stop();
8306 assert!(stop.stopped(), "a second stop is not a toggle");
8307
8308 let shared = stop.clone();
8309 assert!(
8310 shared.stopped(),
8311 "a clone is the same stop; that is how the loop and its caller share one"
8312 );
8313 }
8314
8315 #[test]
8316 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
8317 let stop = Stop::new();
8318 stop.enter();
8319 assert!(
8320 !stop.finishing(),
8321 "a busy loop nobody has asked to stop is just running"
8322 );
8323
8324 stop.stop();
8325 assert!(
8326 stop.finishing(),
8327 "a stop asked for mid-run has not landed until the run is settled"
8328 );
8329
8330 stop.exit();
8331 assert!(
8332 !stop.finishing(),
8333 "once the run is settled the stop has landed and there is nothing to finish"
8334 );
8335 }
8336
8337 #[test]
8338 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
8339 let stop = Stop::new();
8340 stop.enter();
8341 stop.enter();
8342 stop.stop();
8343 assert!(stop.finishing(), "two runs still in flight");
8344
8345 stop.exit();
8346 assert!(
8347 stop.finishing(),
8348 "one run finished, but a sibling is still working"
8349 );
8350
8351 stop.exit();
8352 assert!(
8353 !stop.finishing(),
8354 "the last run out is what actually lands the stop"
8355 );
8356 }
8357
8358 #[tokio::test]
8359 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
8360 let dir = tempfile::tempdir().unwrap();
8361 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8362 let stop = Stop::new();
8363 stop.stop();
8364
8365 let began = std::time::Instant::now();
8366 tokio::time::timeout(
8367 Duration::from_secs(2),
8368 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8369 )
8370 .await
8371 .expect("a stopped loop must return, not sit out its poll interval")
8372 .expect("the loop's own setup and teardown must not fail");
8373 assert!(
8374 began.elapsed() < opts.poll,
8375 "returned only after {:?}, which is a poll interval, not a stop",
8376 began.elapsed()
8377 );
8378 }
8379
8380 #[tokio::test]
8381 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
8382 let dir = tempfile::tempdir().unwrap();
8383 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8384 let stop = Stop::new();
8385
8386 let asker = {
8389 let stop = stop.clone();
8390 tokio::spawn(async move {
8391 tokio::time::sleep(Duration::from_millis(20)).await;
8392 stop.stop();
8393 })
8394 };
8395
8396 let began = std::time::Instant::now();
8397 tokio::time::timeout(
8398 Duration::from_secs(2),
8399 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8400 )
8401 .await
8402 .expect("a stop asked for while idle must wake the wait")
8403 .expect("the loop's own setup and teardown must not fail");
8404 asker.await.unwrap();
8405 assert!(
8406 began.elapsed() < opts.poll,
8407 "returned only after {:?}, so the stop waited on the sleep",
8408 began.elapsed()
8409 );
8410 }
8411
8412 #[tokio::test]
8413 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
8414 let dir = tempfile::tempdir().unwrap();
8415 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8416 let stop = Stop::new();
8417 stop.stop();
8418
8419 tokio::time::timeout(
8420 Duration::from_secs(2),
8421 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8422 )
8423 .await
8424 .expect("a stopped loop must return")
8425 .expect("the loop's own setup and teardown must not fail");
8426
8427 assert!(
8428 home.is_dir(),
8429 "the loop did publish a status file, so its removal is the teardown and not an absence"
8430 );
8431 assert!(
8432 !status_file.exists(),
8433 "a stopped loop clears its status file"
8434 );
8435 assert!(
8436 read_status(&home).is_none(),
8437 "a reader must see no daemon at all, not a heartbeat that merely stopped"
8438 );
8439 }
8440
8441 #[tokio::test]
8442 async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
8443 let dir = tempfile::tempdir().unwrap();
8444 let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
8445 opts.once = true;
8446
8447 let mut settled = RunState::new(
8448 dir.path().join("repo"),
8449 "main".to_owned(),
8450 "abc1234".to_owned(),
8451 "fixture".to_owned(),
8452 Config::default(),
8453 );
8454 settled.status = RunStatus::Ready;
8455 let run_dir = home.join("runs").join(&settled.id);
8456 std::fs::create_dir_all(&run_dir).unwrap();
8457 std::fs::write(
8458 run_dir.join("run.json"),
8459 serde_json::to_string_pretty(&settled).unwrap(),
8460 )
8461 .unwrap();
8462 let questions = Questions::at(home.join("questions"));
8463 let mut question = ask::Question::new(
8464 settled.id.clone(),
8465 "review".to_owned(),
8466 "reviewer-1".to_owned(),
8467 "Continue?".to_owned(),
8468 String::new(),
8469 Vec::new(),
8470 );
8471 questions.put(&mut question).unwrap();
8472
8473 drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
8474 .await
8475 .unwrap();
8476
8477 assert_eq!(
8478 questions.get(&question.id).unwrap().status,
8479 ask::QuestionStatus::Abandoned,
8480 "an empty --once drain still performs startup question cleanup"
8481 );
8482 }
8483
8484 #[test]
8485 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
8486 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8487
8488 assert!(
8489 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
8490 "never checked before: due at once"
8491 );
8492
8493 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
8494 assert!(
8495 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
8496 "well inside the interval: not due yet"
8497 );
8498
8499 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
8500 assert!(
8501 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
8502 "exactly at the edge: not yet due, same convention as `clean::due`"
8503 );
8504
8505 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8506 assert!(
8507 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
8508 "past the interval: due again"
8509 );
8510 }
8511
8512 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
8517 let config = dir.join("magi.toml");
8518 std::fs::write(
8524 &config,
8525 format!(
8526 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
8527 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
8528 cache_dir.display()
8529 ),
8530 )
8531 .unwrap();
8532 Opts {
8533 config: Some(config),
8534 repo: dir.join("repo"),
8535 ..Opts::default()
8536 }
8537 }
8538
8539 #[tokio::test]
8540 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
8541 let dir = tempfile::tempdir().unwrap();
8542 let home = dir.path().join("home");
8543 let cache_dir = dir.path().join("cache");
8544 std::fs::create_dir_all(&cache_dir).unwrap();
8545 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8546 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8547
8548 let running = Stop::new();
8551 let mut last_checked = None;
8552 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8553 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
8554 .await;
8555 assert_eq!(
8556 crate::disk::dir_size(&cache_dir),
8557 0,
8558 "over the cap on the first check ever: pruned at once, no idle queue required"
8559 );
8560 assert_eq!(last_checked, Some(t0));
8561
8562 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
8564 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
8565 maybe_prune_cache_between_runs(
8566 &opts.repo,
8567 &opts,
8568 &home,
8569 &running,
8570 &mut last_checked,
8571 too_soon,
8572 )
8573 .await;
8574 assert_eq!(
8575 crate::disk::dir_size(&cache_dir),
8576 10,
8577 "too soon since the last check: left alone rather than rescanned every call"
8578 );
8579 assert_eq!(
8580 last_checked,
8581 Some(t0),
8582 "an idle check does not reset the clock"
8583 );
8584
8585 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
8587 maybe_prune_cache_between_runs(
8588 &opts.repo,
8589 &opts,
8590 &home,
8591 &running,
8592 &mut last_checked,
8593 due_again,
8594 )
8595 .await;
8596 assert_eq!(
8597 crate::disk::dir_size(&cache_dir),
8598 0,
8599 "due again: pruned back under the cap"
8600 );
8601 }
8602
8603 #[tokio::test]
8611 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
8612 let dir = tempfile::tempdir().unwrap();
8613 let home = dir.path().join("home");
8614 let cache_dir = dir.path().join("cache");
8615 std::fs::create_dir_all(&cache_dir).unwrap();
8616 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
8617 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
8618
8619 let stop = Stop::new();
8620 stop.stop();
8621 assert!(
8622 !stop.finishing(),
8623 "no run is in flight at a between-runs boundary, so nothing else \
8624 would tell the operator this stop had not taken effect yet"
8625 );
8626
8627 let mut last_checked = None;
8628 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
8629 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
8630 .await;
8631 assert_eq!(
8632 crate::disk::dir_size(&cache_dir),
8633 10,
8634 "over its cap, and due for the first check ever, but a stop outranks \
8635 it: the cap is a standing policy the next start measures again"
8636 );
8637 assert_eq!(
8638 last_checked, None,
8639 "a check that never happened must not claim the interval"
8640 );
8641 }
8642
8643 #[tokio::test]
8654 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
8655 let dir = tempfile::tempdir().unwrap();
8656 let cache_dir = dir.path().join("cache");
8657 std::fs::create_dir_all(&cache_dir).unwrap();
8658 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
8659
8660 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
8661 opts.poll = Duration::from_millis(20);
8662 opts.max_attempts = 1_000;
8663
8664 let queue = Queue::at(dir.path().join("queue"));
8665 for i in 0..128 {
8666 let mut task = Task::new(
8667 format!("queued work {i}"),
8668 format!("queued work {i}"),
8669 opts.repo.clone(),
8670 Source::Human,
8671 );
8672 queue.put(&mut task).unwrap();
8673 }
8674
8675 let home = dir.path().join("home");
8676 let worktrees = dir.path().join("wt");
8677 let status_file = home.join("daemon.json");
8678 let stop = Stop::new();
8679 let stopper = {
8680 let stop = stop.clone();
8681 let queue = Queue::at(dir.path().join("queue"));
8682 tokio::spawn(async move {
8683 for _ in 0..2_000 {
8687 if queue.list().iter().filter(|t| t.attempts == 1).count() >= 2 {
8688 break;
8689 }
8690 tokio::time::sleep(Duration::from_millis(5)).await;
8691 }
8692 stop.stop();
8693 })
8694 };
8695
8696 tokio::time::timeout(
8697 Duration::from_secs(20),
8698 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
8699 )
8700 .await
8701 .expect("the loop must not hang on a queue that keeps producing failing work")
8702 .expect("the loop's own setup and teardown must not fail");
8703 stopper.await.unwrap();
8704
8705 let after = queue.list();
8706 let completed = after.iter().filter(|t| t.attempts == 1).count();
8707 assert!(
8708 completed >= 2,
8709 "the harness must have processed multiple tasks to exercise a busy queue \
8710 (got {completed} completed task(s))"
8711 );
8712 assert!(
8713 after.iter().any(|t| t.status.runnable()),
8714 "unprocessed tasks must remain: the external stop, rather than an \
8715 exhausted backlog, must end the test"
8716 );
8717 assert!(
8718 after
8719 .iter()
8720 .filter(|t| t.attempts > 0)
8721 .all(|t| { t.attempts == 1 && t.status == TaskStatus::Held }),
8722 "a busy backlog must not retry a task's unchanged startup failure"
8723 );
8724
8725 assert_eq!(
8726 crate::disk::dir_size(&cache_dir),
8727 0,
8728 "an oversized cache must not be left to grow unboundedly just because the \
8729 queue kept the loop busy the whole time"
8730 );
8731 }
8732
8733 #[test]
8734 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
8735 let dir = tempfile::tempdir().unwrap();
8736 let queue = Queue::at(dir.path().join("queue"));
8737 let questions = Questions::at(dir.path().join("questions"));
8738 let mut task = task();
8739 queue.put(&mut task).unwrap();
8740
8741 let mut task_question = ask::Question::new(
8742 task.id.clone(),
8743 crate::conduct::NODE.to_owned(),
8744 "conduct".to_owned(),
8745 "Which backend?".to_owned(),
8746 String::new(),
8747 Vec::new(),
8748 );
8749 questions.put(&mut task_question).unwrap();
8750 task.block(vec![task_question.id.clone()], None);
8751 queue.put(&mut task).unwrap();
8752
8753 let mut run_question = ask::Question::new(
8754 "20260101-000000-run1".to_owned(),
8755 "review".to_owned(),
8756 "reviewer-1".to_owned(),
8757 "Run question".to_owned(),
8758 String::new(),
8759 Vec::new(),
8760 );
8761 questions.put(&mut run_question).unwrap();
8762
8763 let mut coincidental = ask::Question::new(
8768 task.id.clone(),
8769 "review".to_owned(),
8770 "reviewer-1".to_owned(),
8771 "Unrelated review question".to_owned(),
8772 String::new(),
8773 Vec::new(),
8774 );
8775 questions.put(&mut coincidental).unwrap();
8776
8777 reconcile_task_questions(&queue, &questions);
8778 assert!(questions.get(&task_question.id).unwrap().status.open());
8779 assert!(questions.get(&run_question.id).unwrap().status.open());
8780 assert!(questions.get(&coincidental.id).unwrap().status.open());
8781
8782 task.release();
8783 queue.put(&mut task).unwrap();
8784 reconcile_task_questions(&queue, &questions);
8785 assert_eq!(
8786 questions.get(&task_question.id).unwrap().status,
8787 ask::QuestionStatus::Abandoned
8788 );
8789 assert!(
8790 questions.get(&run_question.id).unwrap().status.open(),
8791 "run questions remain the run janitor's responsibility"
8792 );
8793 assert!(
8794 questions.get(&coincidental.id).unwrap().status.open(),
8795 "a non-conductor question must not be abandoned just because its \
8796 run id coincides with a task id"
8797 );
8798 }
8799
8800 #[test]
8801 fn a_freshly_started_running_task_is_never_stalled() {
8802 let dir = tempfile::tempdir().unwrap();
8803 let mut t = task();
8804 t.start("run-1".to_owned());
8805 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
8808 }
8809
8810 #[test]
8811 fn a_long_running_task_with_no_live_daemon_is_stalled() {
8812 let dir = tempfile::tempdir().unwrap();
8813 let mut t = task();
8814 t.start("run-1".to_owned());
8815 t.updated_at = Timestamp::now()
8816 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8817 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
8818 assert_eq!(
8819 stalled_tasks(
8820 &Queue::at(dir.path().join("q")),
8821 dir.path(),
8822 Timestamp::now()
8823 )
8824 .len(),
8825 0,
8826 "the task was never written to this queue"
8827 );
8828 }
8829
8830 #[test]
8831 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
8832 let dir = tempfile::tempdir().unwrap();
8833 let mut t = task();
8834 t.id = "20260903-080340-0167".to_owned();
8835 t.start("20260903-080619-01c2".to_owned());
8836 t.updated_at = Timestamp::now()
8837 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
8838
8839 let mut status = Status::new();
8840 status.current = vec![Current {
8841 task: t.id.clone(),
8842 run: "20260903-080619-01c2".to_owned(),
8843 }];
8844 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
8845
8846 assert!(
8847 !is_stalled(&t, dir.path(), Timestamp::now()),
8848 "a live daemon's own heartbeat rules out stalled, however long the task has run"
8849 );
8850 }
8851
8852 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
8856 let path = queue.path_of(id);
8857 let body = std::fs::read_to_string(&path).unwrap();
8858 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
8859 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
8860 v["updated_at"] = serde_json::Value::String(old.to_string());
8861 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
8862 }
8863
8864 #[test]
8865 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
8866 let dir = tempfile::tempdir().unwrap();
8879 let queue = Queue::at(dir.path().join("queue"));
8880 let home = dir.path().join("home");
8881
8882 let mut t = task();
8883 t.id = "20260101-000001-lock".to_owned();
8884 t.start("run-1".to_owned());
8885 queue.put(&mut t).unwrap();
8886 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
8887 std::fs::write(
8888 dir.path().join("queue").join(format!("{}.lock", t.id)),
8889 "not a pid",
8890 )
8891 .unwrap();
8892
8893 let now = Timestamp::now();
8894 assert!(
8895 reclaim_orphaned_running(&queue, 2).is_empty(),
8896 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
8897 and reclaim must leave the task alone"
8898 );
8899 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8900
8901 let stalled = stalled_tasks(&queue, &home, now);
8902 assert_eq!(
8903 stalled.len(),
8904 1,
8905 "reclaim's inability to claim it yet must not hide it from the conductor"
8906 );
8907 assert_eq!(stalled[0].id, t.id);
8908 }
8909
8910 #[test]
8911 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
8912 let dir = tempfile::tempdir().unwrap();
8913 crate::run::set_home(dir.path().join("run-home"));
8914 let queue = Queue::at(dir.path().join("queue"));
8915 let home = dir.path().join("home");
8916 let questions = Questions::at(dir.path().join("questions"));
8917
8918 let mut t = task();
8919 t.id = "20260101-000003-dead".to_owned();
8920 t.start("missing-run".to_owned());
8921 queue.put(&mut t).unwrap();
8922 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
8923
8924 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
8927 assert_eq!(
8928 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
8929 [&t.id]
8930 );
8931 assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
8932 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
8933
8934 crate::conduct::apply(
8937 &queue,
8938 &questions,
8939 &crate::conduct::Verdict {
8940 decisions: vec![crate::conduct::Decision {
8941 id: t.id.clone(),
8942 recovery: Some(crate::conduct::Recovery::Requeue),
8943 ..crate::conduct::Decision::default()
8944 }],
8945 },
8946 )
8947 .unwrap();
8948 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
8949 }
8950
8951 #[test]
8952 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
8953 let dir = tempfile::tempdir().unwrap();
8954 let queue = Queue::at(dir.path().join("queue"));
8955 let home = dir.path().join("home");
8956
8957 let mut fresh = task();
8958 fresh.id = "20260101-000001-aaaa".to_owned();
8959 fresh.start("run-1".to_owned());
8960 queue.put(&mut fresh).unwrap();
8961
8962 let mut old = task();
8963 old.id = "20260101-000002-bbbb".to_owned();
8964 old.start("run-2".to_owned());
8965 queue.put(&mut old).unwrap();
8966 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
8967
8968 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
8969 assert_eq!(stalled.len(), 1);
8970 assert_eq!(stalled[0].id, old.id);
8971 }
8972
8973 #[test]
8974 fn queued_and_finished_task_views_partition_by_status() {
8975 let dir = tempfile::tempdir().unwrap();
8976 let queue = Queue::at(dir.path().join("queue"));
8977
8978 let mut queued = task();
8979 queued.id = "20260101-000001-aaaa".to_owned();
8980 queue.put(&mut queued).unwrap();
8981
8982 let mut failed = task();
8983 failed.id = "20260101-000002-bbbb".to_owned();
8984 failed.start("run-1".to_owned());
8985 failed.fail("gate red", 5);
8986 queue.put(&mut failed).unwrap();
8987
8988 let mut held = task();
8989 held.id = "20260101-000003-cccc".to_owned();
8990 held.hold_machine(None);
8991 queue.put(&mut held).unwrap();
8992
8993 let mut running = task();
8994 running.id = "20260101-000004-dddd".to_owned();
8995 running.start("run-2".to_owned());
8996 queue.put(&mut running).unwrap();
8997
8998 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
8999 assert_eq!(queued_ids, [queued.id.clone()]);
9000
9001 let mut finished_ids: Vec<String> =
9002 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
9003 finished_ids.sort_unstable();
9004 let mut want = vec![failed.id.clone(), held.id.clone()];
9005 want.sort_unstable();
9006 assert_eq!(finished_ids, want);
9007 }
9008
9009 #[test]
9010 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
9011 for short in [false, true] {
9012 let dir = tempfile::tempdir().unwrap();
9013 let queue = Queue::at(dir.path().join("queue"));
9014 let questions = ask::Questions::at(dir.path().join("questions"));
9015 let mut running = task();
9016 running.id = "20261013-192213-feb2".into();
9017 running.start("existing-run".into());
9018 queue.put(&mut running).unwrap();
9019 let mut middle = task();
9020 middle.id = "20261013-193611-ca5b".into();
9021 middle.block(
9022 vec![if short {
9023 "feb2".into()
9024 } else {
9025 running.id.clone()
9026 }],
9027 None,
9028 );
9029 queue.put(&mut middle).unwrap();
9030 let mut waiting = task();
9031 waiting.id = "20261013-202849-ee37".into();
9032 waiting.block(
9033 vec![if short {
9034 "ca5b".into()
9035 } else {
9036 middle.id.clone()
9037 }],
9038 Some("wait for implementation".into()),
9039 );
9040 queue.put(&mut waiting).unwrap();
9041 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
9042 let inv = crate::blockers::Inventory::load(&queue, &questions);
9043 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
9044 assert!(inv.stuck().is_empty());
9045 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9046 assert!(report.quarantined.is_empty());
9047 assert!(report.asked.is_empty());
9048 resolve_blockers(&queue, &questions);
9049 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9050 assert!(questions.list().is_empty());
9051 running.succeed();
9054 queue.put(&mut running).unwrap();
9055 resolve_blockers(&queue, &questions);
9056 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
9057 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9058 middle = queue.get(&middle.id).unwrap();
9059 middle.succeed();
9060 queue.put(&mut middle).unwrap();
9061 resolve_blockers(&queue, &questions);
9062 let after = queue.get(&waiting.id).unwrap();
9063 assert_eq!(after.status, TaskStatus::Queued);
9064 assert!(after.blocked_by.is_empty());
9065 assert_eq!(after.instruction, waiting.instruction);
9066 }
9067 }
9068
9069 #[test]
9070 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
9071 for future_schema in [false, true] {
9072 let name = "20261013-193611-ca5b";
9073 let mut future = task();
9074 future.id = name.into();
9075 future.schema = crate::queue::SCHEMA + 1;
9076 let contents = if future_schema {
9077 serde_json::to_string(&future).unwrap()
9078 } else {
9079 "corrupt".into()
9080 };
9081 let dir = tempfile::tempdir().unwrap();
9082 let queue = Queue::at(dir.path().join("queue"));
9083 let questions = ask::Questions::at(dir.path().join("questions"));
9084 let mut waiting = task();
9085 waiting.id = "20261013-202849-ee37".into();
9086 waiting.block(vec!["ca5b".into()], None);
9087 queue.put(&mut waiting).unwrap();
9088 std::fs::write(queue.path_of(name), contents).unwrap();
9089 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
9090 assert!(
9091 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
9092 );
9093 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9094 assert!(report.asked.is_empty());
9095 resolve_blockers(&queue, &questions);
9096 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9097 assert!(questions.list().is_empty());
9098 }
9099 let dir = tempfile::tempdir().unwrap();
9100 let queue = Queue::at(dir.path().join("queue"));
9101 let questions = ask::Questions::at(dir.path().join("questions"));
9102 let mut waiting = task();
9103 waiting.id = "20261013-202849-ee37".into();
9104 waiting.block(vec!["ca5b".into()], None);
9105 queue.put(&mut waiting).unwrap();
9106 let mut done = task();
9107 done.id = "20261013-193611-ca5b".into();
9108 done.succeed();
9109 queue.put(&mut done).unwrap();
9110 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
9113 assert!(queue.get("ca5b").is_err());
9114 assert_eq!(
9115 crate::queue::dependency(&queue, &questions, "ca5b"),
9116 crate::queue::Dependency::Ambiguous
9117 );
9118 resolve_blockers(&queue, &questions);
9119 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
9120 assert!(report.asked.is_empty());
9121 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
9122 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
9123 }
9124
9125 #[test]
9126 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
9127 for held in [false, true] {
9128 for question in [false, true] {
9129 for reference in ["ca5b", "20261013-193611-ca5b"] {
9130 let dir = tempfile::tempdir().unwrap();
9131 let queue = Queue::at(dir.path().join("queue"));
9132 let questions = ask::Questions::at(dir.path().join("questions"));
9133 let mut prerequisite = task();
9134 prerequisite.id = "20261013-193611-ca5b".into();
9135 prerequisite.start("existing-run".into());
9136 queue.put(&mut prerequisite).unwrap();
9137 let mut waiting = task();
9138 waiting.id = "20261013-202849-ee37".into();
9139 if held {
9140 waiting.hold_machine(Some("waiting for separate consent".into()));
9141 }
9142 queue.put(&mut waiting).unwrap();
9143 let reason = "Deleting saved data requires separate owner consent";
9144 crate::conduct::apply(
9145 &queue,
9146 &questions,
9147 &crate::conduct::Verdict {
9148 decisions: vec![crate::conduct::Decision {
9149 id: waiting.id.clone(),
9150 blocked_by: vec![reference.into()],
9151 question: question.then(|| "May the saved data be deleted?".into()),
9152 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
9153 reason: Some(reason.into()),
9154 ..crate::conduct::Decision::default()
9155 }],
9156 },
9157 )
9158 .unwrap();
9159 let before = queue.get(&waiting.id).unwrap();
9160 if question {
9161 assert_eq!(before.status, TaskStatus::Blocked);
9162 let q = questions.get(&before.blocked_by[0]).unwrap();
9163 assert_eq!(q.summary, "May the saved data be deleted?");
9164 assert!(q.status.open());
9165 } else {
9166 assert_eq!(before.status, TaskStatus::Held);
9167 let recorded_reason = before.hold_reason.as_deref().unwrap();
9168 assert!(recorded_reason.starts_with(reason));
9169 if held {
9170 assert!(recorded_reason.contains("waiting for separate consent"));
9171 }
9172 }
9173 prerequisite.succeed();
9174 queue.put(&mut prerequisite).unwrap();
9175 resolve_blockers(&queue, &questions);
9176 let after = queue.get(&waiting.id).unwrap();
9177 assert_eq!(after.status, before.status);
9178 assert_eq!(after.blocked_by, before.blocked_by);
9179 assert_eq!(after.hold_reason, before.hold_reason);
9180 assert_eq!(after.instruction, waiting.instruction);
9181 if question {
9182 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
9183 }
9184 assert!(
9185 queue.next_runnable().is_none(),
9186 "dependency completion is not consent"
9187 );
9188 }
9189 }
9190 }
9191 }
9192
9193 #[test]
9194 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
9195 let dir = tempfile::tempdir().unwrap();
9196 let queue = Queue::at(dir.path().join("queue"));
9197 let questions = ask::Questions::at(dir.path().join("questions"));
9198
9199 let mut dep = task();
9200 dep.id = "20260101-000001-dep0".to_owned();
9201 dep.succeed();
9202 queue.put(&mut dep).unwrap();
9203
9204 let mut still_going = task();
9205 still_going.id = "20260101-000002-dep1".to_owned();
9206 queue.put(&mut still_going).unwrap();
9207
9208 let mut blocked = task();
9209 blocked.id = "20260101-000003-main".to_owned();
9210 blocked.block(
9211 vec![dep.id.clone(), still_going.id.clone()],
9212 Some("waits on both".to_owned()),
9213 );
9214 queue.put(&mut blocked).unwrap();
9215
9216 resolve_blockers(&queue, &questions);
9217
9218 let after = queue.get(&blocked.id).unwrap();
9219 assert_eq!(
9220 after.status,
9221 TaskStatus::Blocked,
9222 "one dependency is still outstanding"
9223 );
9224 assert_eq!(after.blocked_by, [still_going.id.clone()]);
9225 }
9226
9227 #[test]
9228 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
9229 let dir = tempfile::tempdir().unwrap();
9230 let queue = Queue::at(dir.path().join("queue"));
9231 let questions = ask::Questions::at(dir.path().join("questions"));
9232
9233 let mut q = crate::ask::Question::new(
9234 "20260101-000001-main".to_owned(),
9235 crate::conduct::NODE.to_owned(),
9236 "conduct".to_owned(),
9237 "Which backend?".to_owned(),
9238 String::new(),
9239 Vec::new(),
9240 );
9241 questions.put(&mut q).unwrap();
9242 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
9243 .unwrap();
9244 questions.put(&mut q).unwrap();
9245
9246 let mut blocked = task();
9247 blocked.id = "20260101-000001-main".to_owned();
9248 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
9249 queue.put(&mut blocked).unwrap();
9250
9251 resolve_blockers(&queue, &questions);
9252
9253 let after = queue.get(&blocked.id).unwrap();
9254 assert_eq!(
9255 after.status,
9256 TaskStatus::Queued,
9257 "the only blocker resolved"
9258 );
9259 assert_eq!(after.answers.len(), 1);
9260 assert_eq!(after.answers[0].question, "Which backend?");
9261 assert_eq!(after.answers[0].answer, "SQLite");
9262
9263 let instruction = instruction_for(&after);
9265 assert!(instruction.contains("Which backend?"));
9266 assert!(instruction.contains("SQLite"));
9267 }
9268
9269 #[test]
9270 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
9271 let dir = tempfile::tempdir().unwrap();
9272 let queue = Queue::at(dir.path().join("queue"));
9273 let questions = ask::Questions::at(dir.path().join("questions"));
9274
9275 let mut q = crate::ask::Question::new(
9276 "20260101-000001-main".to_owned(),
9277 crate::conduct::NODE.to_owned(),
9278 "conduct".to_owned(),
9279 "Is the setup done?".to_owned(),
9280 String::new(),
9281 Vec::new(),
9282 );
9283 q.abandon("no answer within 60s of asking");
9284 questions.put(&mut q).unwrap();
9285
9286 let mut blocked = task();
9287 blocked.id = "20260101-000001-main".to_owned();
9288 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
9289 queue.put(&mut blocked).unwrap();
9290
9291 resolve_blockers(&queue, &questions);
9292
9293 let after = queue.get(&blocked.id).unwrap();
9294 assert_eq!(
9295 after.status,
9296 TaskStatus::Held,
9297 "never left blocked on nothing"
9298 );
9299 assert!(!after.operator_held(), "a machine hold, for triage");
9300 assert!(
9301 after
9302 .hold_reason
9303 .as_deref()
9304 .unwrap_or_default()
9305 .contains("went unanswered")
9306 );
9307 }
9308
9309 #[test]
9310 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
9311 let dir = tempfile::tempdir().unwrap();
9317 let queue = Queue::at(dir.path().join("queue"));
9318 let questions = ask::Questions::at(dir.path().join("questions"));
9319
9320 let mut q = crate::ask::Question::new(
9321 "20260101-000001-main".to_owned(),
9322 crate::conduct::NODE.to_owned(),
9323 "conduct".to_owned(),
9324 "How should this be handled?".to_owned(),
9325 String::new(),
9326 Vec::new(),
9327 );
9328 questions.put(&mut q).unwrap();
9329 q.answer(crate::ask::Answer::Text(
9330 "leave it held, a human will look at it later".to_owned(),
9331 ))
9332 .unwrap();
9333 questions.put(&mut q).unwrap();
9334
9335 let mut held = task();
9336 held.id = "20260101-000001-main".to_owned();
9337 held.hold_machine(Some("out of attempts".to_owned()));
9338 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
9339 queue.put(&mut held).unwrap();
9340
9341 resolve_blockers(&queue, &questions);
9342
9343 let after = queue.get(&held.id).unwrap();
9344 assert_eq!(after.status, TaskStatus::Held);
9345 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
9346 assert_eq!(
9347 after.answers[0].answer,
9348 "leave it held, a human will look at it later"
9349 );
9350 }
9351
9352 #[test]
9353 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
9354 let dir = tempfile::tempdir().unwrap();
9355 let queue = Queue::at(dir.path().join("queue"));
9356 let questions = ask::Questions::at(dir.path().join("questions"));
9357 let mut dep = task();
9358 dep.id = "20260101-000001-gone".to_owned();
9359 queue.put(&mut dep).unwrap();
9360 let mut blocked = task();
9361 blocked.id = "20260101-000003-main".to_owned();
9362 blocked.block(vec![dep.id.clone()], None);
9363 queue.put(&mut blocked).unwrap();
9364
9365 let claim = queue.claim(&blocked.id).unwrap();
9367 queue.remove(&dep.id, false, &questions).unwrap();
9368 drop(claim);
9369 resolve_blockers(&queue, &questions);
9370
9371 let after = queue.get(&blocked.id).unwrap();
9372 assert_eq!(after.status, TaskStatus::Queued);
9373 assert!(after.blocked_by.is_empty());
9374 }
9375
9376 #[test]
9377 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
9378 let dir = tempfile::tempdir().unwrap();
9384 let queue = Queue::at(dir.path().join("queue"));
9385 let questions = ask::Questions::at(dir.path().join("questions"));
9386
9387 let mut still_going = task();
9388 still_going.id = "20260101-000002-dep1".to_owned();
9389 queue.put(&mut still_going).unwrap();
9390
9391 let mut blocked = task();
9392 blocked.id = "20260101-000003-main".to_owned();
9393 blocked.block(
9394 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
9395 Some("waits on both".to_owned()),
9396 );
9397 queue.put(&mut blocked).unwrap();
9398
9399 resolve_blockers(&queue, &questions);
9400
9401 let after = queue.get(&blocked.id).unwrap();
9402 assert_eq!(
9403 after.status,
9404 TaskStatus::Held,
9405 "a missing dependency must not leave the task blocked forever"
9406 );
9407 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
9408 assert!(after.blocked_by.is_empty());
9409 let reason = after.hold_reason.as_deref().unwrap_or_default();
9410 assert!(
9411 reason.contains("20260101-000001-gone"),
9412 "the missing id must be named so an operator can tell what happened: {reason}"
9413 );
9414 assert!(
9415 reason.contains(&still_going.id),
9416 "the still-valid dependency must not silently vanish from the record: {reason}"
9417 );
9418 }
9419
9420 #[test]
9421 fn instruction_for_is_unchanged_without_any_answers() {
9422 let t = task();
9423 assert_eq!(instruction_for(&t), t.instruction);
9424 }
9425
9426 #[test]
9427 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
9428 let dir = tempfile::tempdir().unwrap();
9429 let q = Queue::at(dir.path().join("queue"));
9430 let src = dir.path().join("shot.png");
9431 std::fs::write(&src, "x").unwrap();
9432 let mut t = task();
9433 q.attach(&mut t, &[src]).unwrap();
9434 let paths = task_attachments(&q, &t).unwrap();
9435 assert_eq!(paths.len(), 1);
9436 assert!(paths[0].is_absolute() && paths[0].is_file());
9437 std::fs::remove_file(&paths[0]).unwrap();
9438 let err = task_attachments(&q, &t).unwrap_err().to_string();
9439 assert!(err.contains("shot.png"), "{err}");
9440 }
9441
9442 #[tokio::test]
9443 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
9444 let dir = tempfile::tempdir().unwrap();
9445 let queue_path = dir.path().join("queue");
9446 let queue = Queue::at(queue_path.clone());
9447 let mut t = task();
9448 t.source = Source::Agent {
9449 run: "originating-chat".into(),
9450 node: crate::queue::CHAT_NODE.into(),
9451 };
9452 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
9453 stage: "config-fetch",
9454 target: "origin/main".into(),
9455 });
9456 record_config_failure(&mut t, &fetch);
9457 t.hold_machine(Some(format!("{fetch:#}")));
9458 let fetch_key = t.chat_notice_due().unwrap();
9459 t.chat_notices.push(fetch_key.clone());
9460 queue.put(&mut t).unwrap();
9461
9462 let queue = Queue::at(queue_path);
9463 let mut t = queue.get(&t.id).unwrap();
9464 let a = dir.path().join("a.toml");
9465 let b = dir.path().join("b.toml");
9466 for (path, text, new_cause) in [
9467 (&a, "[graph", true),
9468 (&a, "[verify", false),
9469 (&a, "[graph]\nimplementers = 'invalid-type'", true),
9470 (&b, "[graph]\nimplementers = 'invalid-type'", true),
9471 ] {
9472 t.release_operator("test", None);
9473 std::fs::write(path, text).unwrap();
9474 let error = Config::discover_fetched(dir.path(), Some(path))
9475 .await
9476 .unwrap_err();
9477 record_config_failure(&mut t, &error);
9478 t.hold_machine(Some(format!("{error:#}")));
9479 let due = t.chat_notice_due();
9480 assert_eq!(due.is_some(), new_cause, "{error:#}");
9481 if let Some(key) = due {
9482 assert_ne!(key, fetch_key);
9483 t.chat_notices.push(key);
9484 }
9485 }
9486 t.succeed();
9487 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
9488 }
9489
9490 #[test]
9491 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
9492 let dir = tempfile::tempdir().unwrap();
9493 let queue_path = dir.path().join("queue");
9494 let queue = Queue::at(queue_path.clone());
9495 let mut t = task();
9496 t.source = crate::queue::Source::Agent {
9497 run: "originating-chat".into(),
9498 node: crate::queue::CHAT_NODE.into(),
9499 };
9500 let a = dir.path().join("a.png");
9501 let b = dir.path().join("b.png");
9502 std::fs::write(&a, "a").unwrap();
9503 std::fs::write(&b, "b").unwrap();
9504 queue.attach(&mut t, &[a, b]).unwrap();
9505 let paths = task_attachments(&queue, &t).unwrap();
9506 for path in &paths {
9507 std::fs::remove_file(path).unwrap();
9508 }
9509 let missing = task_attachments(&queue, &t).unwrap_err();
9510 t.attempts += 1;
9511 let reason = missing.to_string();
9512 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9513 let first = t.chat_notice_due().expect("first attachment notice");
9514 t.chat_notices.push(first.clone());
9515 queue.put(&mut t).unwrap();
9516
9517 let queue = Queue::at(queue_path.clone());
9518 let mut t = queue.get(&t.id).unwrap();
9519 assert!(t.chat_notice_due().is_none(), "restart does not replay");
9520 t.release_operator("test", None);
9521 t.attempts += 1;
9522 let missing = task_attachments(&queue, &t).unwrap_err();
9523 t.fail_startup_target(
9524 "startup-attachments",
9525 missing.name,
9526 "same missing attachment with regenerated wording",
9527 1,
9528 );
9529 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
9530
9531 std::fs::write(&paths[0], "restored").unwrap();
9532 t.release_operator("test", None);
9533 t.attempts += 1;
9534 let missing = task_attachments(&queue, &t).unwrap_err();
9535 let reason = missing.to_string();
9536 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
9537 let second = t
9538 .chat_notice_due()
9539 .expect("different attachment must notify");
9540 assert_ne!(first, second);
9541 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
9542 }
9543
9544 #[test]
9545 fn resumed_instruction_is_unchanged_without_any_answers() {
9546 let t = task();
9547 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
9548 }
9549
9550 #[test]
9551 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
9552 let mut t = task();
9553 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9554 let old = t.instruction.clone();
9558
9559 let refreshed = resumed_instruction(&old, &t);
9560 assert!(refreshed.starts_with(&old), "the original text is kept");
9561 assert!(refreshed.contains("Which backend?"));
9562 assert!(refreshed.contains("SQLite"));
9563 }
9564
9565 #[test]
9566 fn resumed_instruction_keeps_an_original_answers_heading() {
9567 let mut t = task();
9568 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
9569 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9570
9571 let refreshed = resumed_instruction(&t.instruction, &t);
9572
9573 assert!(
9574 refreshed.starts_with(&t.instruction),
9575 "an answers heading in the original instruction is not the appended block"
9576 );
9577 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
9578 assert!(refreshed.contains("Which backend?"));
9579 assert!(refreshed.contains("SQLite"));
9580
9581 let repeated = resumed_instruction(&refreshed, &t);
9582 assert_eq!(
9583 repeated, refreshed,
9584 "only the final appended block is refreshed"
9585 );
9586 }
9587
9588 #[test]
9589 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
9590 let mut t = task();
9591 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9592
9593 let once = resumed_instruction(&t.instruction, &t);
9597 let twice = resumed_instruction(&once, &t);
9598 assert_eq!(once, twice);
9599 assert_eq!(once.matches("Which backend?").count(), 1);
9600
9601 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
9603 let refreshed = resumed_instruction(&once, &t);
9604 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
9605 assert!(refreshed.contains("Which backend?"));
9606 assert!(refreshed.contains("Which cache?"));
9607 }
9608
9609 #[test]
9610 fn prepare_instruction_covers_all_three_starters() {
9611 let mut t = task();
9612 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
9613
9614 assert_eq!(
9617 prepare_instruction(&Starter::Start, None, &t),
9618 Some(instruction_for(&t))
9619 );
9620
9621 let old = t.instruction.clone();
9624 assert_eq!(
9625 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
9626 Some(resumed_instruction(&old, &t))
9627 );
9628
9629 assert_eq!(
9633 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
9634 None
9635 );
9636 }
9637
9638 #[test]
9639 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
9640 assert_eq!(
9641 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
9642 Starter::Review("magi/eba2/A".to_owned())
9643 );
9644 }
9645
9646 #[test]
9647 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
9648 assert_eq!(
9649 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
9650 Starter::Start,
9651 "a vanished review branch must not fall back to resuming the old run either"
9652 );
9653 }
9654
9655 #[test]
9656 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
9657 let mut t = task();
9658 t.start("old-run".to_owned());
9659 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
9660 t.release();
9661 let branch = t.review_branch.take();
9662 assert_eq!(
9663 choose_starter(branch.as_deref(), true, Some("old-run")),
9664 Starter::Review("magi/eba2/A".to_owned()),
9665 "a review wins over resuming the old run"
9666 );
9667 }
9668
9669 #[test]
9670 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
9671 assert_eq!(
9672 choose_starter(None, false, Some("some-run")),
9673 Starter::Resume("some-run".to_owned())
9674 );
9675 assert_eq!(choose_starter(None, false, None), Starter::Start);
9676 }
9677
9678 #[test]
9679 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
9680 let mut released = task();
9681 released.start("stalled-run".to_owned());
9682 released.requeue();
9683 let unfinished = (!released.fresh_start)
9684 .then(|| Some("stalled-run".to_owned()))
9685 .flatten();
9686 assert_eq!(
9687 choose_starter(None, false, unfinished.as_deref()),
9688 Starter::Start,
9689 "release keeps run history but must not resume it"
9690 );
9691 assert_eq!(released.runs, ["stalled-run"]);
9692 }
9693
9694 #[test]
9695 fn an_ordinary_release_keeps_a_resumable_run_available() {
9696 let mut released = task();
9697 released.start("stalled-run".to_owned());
9698 released.release();
9699 let unfinished = (!released.fresh_start)
9700 .then(|| Some("stalled-run".to_owned()))
9701 .flatten();
9702 assert_eq!(
9703 choose_starter(None, false, unfinished.as_deref()),
9704 Starter::Resume("stalled-run".to_owned()),
9705 "manual release must preserve the normal resume path"
9706 );
9707 }
9708
9709 #[test]
9710 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
9711 let mut state = run_state(RunStatus::Blocked);
9712 state.config.graph.review_rounds = 3;
9713 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
9714 assert!(exhausted_review_budget(&state));
9715
9716 state.reviews.pop();
9718 assert!(!exhausted_review_budget(&state));
9719
9720 let mut stalled = run_state(RunStatus::Stalled);
9723 stalled.config.graph.review_rounds = 1;
9724 stalled.reviews = vec![review_round(1)];
9725 assert!(!exhausted_review_budget(&stalled));
9726 }
9727
9728 fn review_round(round: usize) -> crate::run::ReviewRound {
9729 crate::run::ReviewRound {
9730 round,
9731 head: "deadbeef".to_owned(),
9732 verified_head: None,
9733 verified_at: None,
9734 reviews: Vec::new(),
9735 e2e: Vec::new(),
9736 verify_retried: false,
9737 e2e_deferred: false,
9738 e2e_defer_reason: None,
9739 fix: None,
9740 blocking: 0,
9741 answered: 1,
9742 expected: 1,
9743 clean: false,
9744 progressed: true,
9745 vote_split: false,
9746 reconsideration: Vec::new(),
9747 verdict: None,
9748 }
9749 }
9750
9751 #[test]
9752 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
9753 let mut run = RunState::new(
9754 PathBuf::from("/repo"),
9755 "main".to_owned(),
9756 "abc1234def".to_owned(),
9757 "add retries".to_owned(),
9758 Config::default(),
9759 );
9760 run.status = RunStatus::Judging;
9761 run.parked = true;
9762 let mut task = Task::new(
9763 "add retries".to_owned(),
9764 "add retries".to_owned(),
9765 PathBuf::from("/repo"),
9766 crate::queue::Source::Human,
9767 );
9768 task.status = TaskStatus::Failed;
9769 task.runs = vec![run.id.clone()];
9770 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
9771 assert!(with(&task, &run), "parked after judging is the case");
9772 let mut parked_task = task.clone();
9773 parked_task.status = TaskStatus::Parked;
9774 assert!(with(&parked_task, &run), "the Parked status is the case");
9775
9776 let mut not_parked = run.clone();
9777 not_parked.parked = false;
9778 not_parked.status = RunStatus::Stalled;
9779 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
9780
9781 let mut fresh = task.clone();
9782 fresh.fresh_start = true;
9783 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
9784
9785 let mut review = task.clone();
9786 review.review_branch = Some("magi/x/A".to_owned());
9787 assert!(!with(&review, &run), "review is ranked before resume");
9788
9789 let mut held = task.clone();
9790 held.status = TaskStatus::Held;
9791 assert!(!with(&held, &run), "a hold stays visible to the conductor");
9792
9793 let mut released = run.clone();
9794 released.released_to = Some("20260901-000000-new1".to_owned());
9795 assert!(!with(&task, &released), "nothing left to resume into");
9796
9797 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
9798 "unreadable"
9799 )));
9800 }
9801
9802 #[test]
9803 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
9804 let mut released = RunState::new(
9805 PathBuf::from("/repo"),
9806 "main".to_owned(),
9807 "abc1234def".to_owned(),
9808 "add retries".to_owned(),
9809 Config::default(),
9810 );
9811 released.status = RunStatus::Blocked;
9812 assert_eq!(
9813 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9814 Some(released.id.clone())
9815 );
9816 released.released_to = Some("20260901-000000-new1".to_owned());
9817 assert_eq!(
9818 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
9819 None,
9820 "there is nothing left to resume it into"
9821 );
9822 }
9823
9824 #[test]
9825 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
9826 let mut exhausted = RunState::new(
9835 PathBuf::from("/repo"),
9836 "main".to_owned(),
9837 "abc1234def".to_owned(),
9838 "add retries".to_owned(),
9839 Config::default(),
9840 );
9841 exhausted.status = RunStatus::Blocked;
9842 exhausted.config.graph.review_rounds = 1;
9843 exhausted.reviews = vec![review_round(1)];
9844
9845 assert_eq!(
9846 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
9847 None,
9848 "an exhausted `Blocked` run must not be offered as resumable"
9849 );
9850
9851 let mut has_budget_left = RunState::new(
9854 PathBuf::from("/repo"),
9855 "main".to_owned(),
9856 "abc1234def".to_owned(),
9857 "add retries".to_owned(),
9858 Config::default(),
9859 );
9860 has_budget_left.status = RunStatus::Blocked;
9861 has_budget_left.config.graph.review_rounds = 3;
9862 has_budget_left.reviews = vec![review_round(1)];
9863
9864 assert_eq!(
9865 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
9866 Ok(has_budget_left.clone())
9867 }),
9868 Some(has_budget_left.id.clone())
9869 );
9870 }
9871
9872 #[test]
9873 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
9874 let mut run = RunState::new(
9875 PathBuf::from("/repo"),
9876 "main".to_owned(),
9877 "abc1234def".to_owned(),
9878 "add retries".to_owned(),
9879 Config::default(),
9880 );
9881 run.status = RunStatus::Blocked;
9882 run.config.graph.review_rounds = 1;
9883 run.reviews = vec![review_round(1)];
9884 run.pr_publication_pending = true;
9885 assert_eq!(
9886 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
9887 Some(run.id.clone()),
9888 "a spent review budget must not turn a failed publication into a re-competition"
9889 );
9890 }
9891
9892 #[test]
9893 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
9894 let mut older_stalled = RunState::new(
9902 PathBuf::from("/repo"),
9903 "main".to_owned(),
9904 "abc1234def".to_owned(),
9905 "add retries".to_owned(),
9906 Config::default(),
9907 );
9908 older_stalled.status = RunStatus::Stalled;
9909
9910 let mut newest_exhausted = RunState::new(
9911 PathBuf::from("/repo"),
9912 "main".to_owned(),
9913 "abc1234def".to_owned(),
9914 "add retries".to_owned(),
9915 Config::default(),
9916 );
9917 newest_exhausted.status = RunStatus::Blocked;
9918 newest_exhausted.config.graph.review_rounds = 1;
9919 newest_exhausted.reviews = vec![review_round(1)];
9920
9921 assert_eq!(
9922 unfinished_run_with(
9923 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
9924 "t",
9925 |_| Ok(newest_exhausted.clone())
9926 ),
9927 None,
9928 "the newest run is exhausted, so nothing here is worth resuming - \
9929 least of all the older, already-superseded run"
9930 );
9931 }
9932
9933 #[test]
9934 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
9935 assert_eq!(
9936 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
9937 Err(anyhow::anyhow!("fixture is absent"))
9938 }),
9939 None
9940 );
9941 }
9942
9943 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
9944 let mut q = ask::Question::new(
9945 run.to_owned(),
9946 "implement".to_owned(),
9947 "impl-A".to_owned(),
9948 "continue?".to_owned(),
9949 String::new(),
9950 vec!["resume で続行する".to_owned(), "other".to_owned()],
9951 );
9952 q.actions.insert("resume で続行する".to_owned(), action);
9953 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
9954 .unwrap();
9955 q
9956 }
9957
9958 fn held_task_with(run: &str) -> Task {
9959 let mut t = task();
9960 t.runs = vec![run.to_owned()];
9961 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
9962 t
9963 }
9964
9965 fn resume_action(run: &str) -> ask::ChoiceAction {
9966 ask::ChoiceAction::Resume { run: run.into() }
9967 }
9968
9969 #[test]
9970 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
9971 for parked in [false, true] {
9972 let mut t = task();
9973 t.start("A".into());
9974 t.start("B".into());
9975 assert_eq!(t.runs, vec!["A", "B"]);
9976 t.start("A".into());
9977 let mut resumed = run_state(RunStatus::Blocked);
9978 resumed.id = "A".into();
9979 resumed.parked = parked;
9980 resumed.driver_exited = true;
9981 t.hold_machine(Some("resumed run stopped".into()));
9982 let q = action_question("A", resume_action("A"));
9983 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
9984 assert_eq!(
9985 decide_action(&t, &q, &PHRASES_EN, |id| {
9986 assert_eq!(id, "A");
9987 Ok(resumed.clone())
9988 }),
9989 ActionDecision::Resume("A".into())
9990 );
9991 assert_eq!(
9992 action_standing(&t, &action_question("B", resume_action("B"))),
9993 ActionStanding::Stale
9994 );
9995 t.release_operator("cli", None);
9996 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
9997 assert_eq!(id, "A");
9998 Ok(resumed.clone())
9999 });
10000 assert_eq!(
10001 choose_starter(None, false, unfinished.as_deref()),
10002 Starter::Resume("A".into())
10003 );
10004 assert_eq!(t.runs, vec!["B", "A"]);
10005 }
10006 }
10007
10008 #[test]
10009 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
10010 let mut state = crate::land::tests::repair_state(true, true);
10011 state.status = RunStatus::Landing;
10012 state.parked = true;
10013 state.driver_exited = true;
10014 let mut q = ask::Question::new(
10015 state.id.clone(),
10016 land::APPROVAL_NODE.into(),
10017 "land".into(),
10018 "Merge?".into(),
10019 String::new(),
10020 vec![land::APPROVE.into(), land::HOLD.into()],
10021 );
10022 q.id = state.id.clone();
10023 state.land_approval = Some(crate::run::LandApproval {
10024 question: q.id.clone(),
10025 head: "current-head".into(),
10026 });
10027 assert!(land::upgrade_repair_choice(&mut q, &state));
10028 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10029 let mut task = held_task_with(&state.id);
10030 task.status = TaskStatus::Parked;
10031 assert_eq!(
10032 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10033 ActionDecision::Resume(state.id.clone())
10034 );
10035 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10036 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
10037 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
10038 assert_eq!(
10039 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
10040 ActionDecision::Resume(state.id.clone()),
10041 "a repair answer newer than the manual hold resumes the existing owner"
10042 );
10043 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10044 for answered_at in [q.answered_at, None] {
10045 let mut older = q.clone();
10046 older.answered_at = answered_at;
10047 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
10048 panic!("a later manual hold must prevent dispatch")
10049 });
10050 let ActionDecision::Refuse(reason) = refused else {
10051 panic!("expected an actionable refusal, got {refused:?}");
10052 };
10053 assert!(reason.contains("later operator decision"));
10054 assert!(reason.contains(&format!("magi task release {}", task.id)));
10055 }
10056 task.operator_decided_at = None;
10057 let mut superseded = state.clone();
10058 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
10059 assert_eq!(
10060 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
10061 ActionDecision::Stale
10062 );
10063 for case in 1..4 {
10064 let mut changed = state.clone();
10065 match case {
10066 1 => changed.status = RunStatus::Blocked,
10067 2 => changed.released_to = Some("new-owner".into()),
10068 _ => changed.land_approval.as_mut().unwrap().head.clear(),
10069 }
10070 assert!(matches!(
10071 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
10072 ActionDecision::Refuse(_)
10073 ));
10074 }
10075 assert!(matches!(
10076 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
10077 ActionDecision::Refuse(_)
10078 ));
10079 task.runs.push("replacement".into());
10080 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
10081 assert_eq!(
10082 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
10083 "old owner must not load"
10084 )),
10085 ActionDecision::Stale
10086 );
10087 }
10088
10089 #[test]
10090 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
10091 let mut state = crate::land::tests::repair_state(true, true);
10092 state.status = RunStatus::Landing;
10093 state.parked = true;
10094 state.driver_exited = true;
10095 let mut q = ask::Question::new(
10096 state.id.clone(),
10097 land::APPROVAL_NODE.into(),
10098 "land".into(),
10099 "Merge?".into(),
10100 String::new(),
10101 vec![land::APPROVE.into(), land::HOLD.into()],
10102 );
10103 q.id = state.id.clone();
10104 state.land_approval = Some(crate::run::LandApproval {
10105 question: q.id.clone(),
10106 head: "current-head".into(),
10107 });
10108 state.save().unwrap();
10109 land::upgrade_repair_choice(&mut q, &state);
10110 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10111 Questions::open().put(&mut q).unwrap();
10112 let tmp = tempfile::tempdir().unwrap();
10113 let queue = Queue::at(tmp.path().join("queue"));
10114 let mut task = held_task_with(&state.id);
10115 task.status = TaskStatus::Parked;
10116 queue.put(&mut task).unwrap();
10117 let empty = Arc::new(tokio::sync::Semaphore::new(0));
10118 let free = Arc::new(tokio::sync::Semaphore::new(1));
10119 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
10120 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
10121 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
10122 assert!(claim.permit.is_some());
10123 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
10124 drop(claim);
10125 Questions::open()
10126 .update(&q.id, |q| {
10127 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
10128 Ok(())
10129 })
10130 .unwrap();
10131 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
10132 assert!(claim.permit.is_none());
10133 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
10134 }
10135
10136 #[test]
10137 fn decide_action_resumes_only_the_latest_resumable_run() {
10138 let t = held_task_with("r1");
10139 let q = action_question("r1", resume_action("r1"));
10140 let load = |s: RunState| move |_: &str| Ok(s);
10141 assert_eq!(
10142 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
10143 ActionDecision::Resume("r1".into())
10144 );
10145 let q_other = action_question("r1", resume_action("r0"));
10147 assert!(matches!(
10148 decide_action(
10149 &t,
10150 &q_other,
10151 &PHRASES_EN,
10152 load(run_state(RunStatus::Blocked))
10153 ),
10154 ActionDecision::Refuse(_)
10155 ));
10156 assert!(matches!(
10158 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
10159 ActionDecision::Refuse(_)
10160 ));
10161 let mut released = run_state(RunStatus::Blocked);
10163 released.released_to = Some("elsewhere".into());
10164 assert!(matches!(
10165 decide_action(&t, &q, &PHRASES_EN, load(released)),
10166 ActionDecision::Refuse(_)
10167 ));
10168 assert!(matches!(
10170 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
10171 ActionDecision::Refuse(_)
10172 ));
10173 }
10174
10175 #[test]
10176 fn decide_action_ignores_a_question_about_an_earlier_run() {
10177 let mut t = held_task_with("r1");
10178 t.runs.push("r2".to_owned());
10179 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10180 assert_eq!(
10181 decide_action(
10182 &t,
10183 &action_question("r1", ask::ChoiceAction::Done),
10184 &PHRASES_EN,
10185 never
10186 ),
10187 ActionDecision::Stale
10188 );
10189 }
10190
10191 #[test]
10192 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
10193 let mut t = held_task_with("r1");
10194 let never = |_: &str| -> Result<RunState> { bail!("not read") };
10195 assert_eq!(
10196 decide_action(
10197 &t,
10198 &action_question("r1", ask::ChoiceAction::Requeue),
10199 &PHRASES_EN,
10200 never
10201 ),
10202 ActionDecision::Requeue
10203 );
10204 let done_q = action_question("r1", ask::ChoiceAction::Done);
10205 assert_eq!(
10206 decide_action(&t, &done_q, &PHRASES_EN, never),
10207 ActionDecision::Done
10208 );
10209 t.mark_action_applied(&done_q.id);
10210 assert_eq!(
10211 decide_action(&t, &done_q, &PHRASES_EN, never),
10212 ActionDecision::Skip
10213 );
10214
10215 let mut plain = action_question("r1", ask::ChoiceAction::Done);
10217 plain.actions.clear();
10218 assert_eq!(
10219 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
10220 ActionDecision::Skip
10221 );
10222 let mut running = held_task_with("r1");
10224 running.status = TaskStatus::Running;
10225 assert_eq!(
10226 decide_action(
10227 &running,
10228 &action_question("r1", ask::ChoiceAction::Done),
10229 &PHRASES_EN,
10230 never
10231 ),
10232 ActionDecision::Skip
10233 );
10234 }
10235
10236 #[test]
10237 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
10238 crate::run::pin_test_home();
10239 for legacy in [false, true] {
10240 let dir = tempfile::tempdir().unwrap();
10241 let queue = Queue::at(dir.path().join("queue"));
10242 let questions = Questions::at(dir.path().join("questions"));
10243 let run = "20260101-000000-hold";
10244 let mut task = held_task_with(run);
10245 task.status = TaskStatus::Parked;
10246 let mut q = ask::Question::new(
10247 run.into(),
10248 land::APPROVAL_NODE.into(),
10249 "land".into(),
10250 "Repair?".into(),
10251 String::new(),
10252 vec![land::REPAIR.into()],
10253 );
10254 q.actions.insert(land::REPAIR.into(), resume_action(run));
10255 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
10256 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
10258 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
10259 questions.put(&mut q).unwrap();
10260 task.hold_manual(Some("Keep this PR paused".into()));
10261 if legacy {
10262 task.hold_source = None;
10263 }
10264 queue.put(&mut task).unwrap();
10265
10266 apply_choice_actions(&queue, &questions, dir.path());
10267 let after = queue.get(&task.id).unwrap();
10268 assert!(after.operator_held());
10269 assert_eq!(after.hold_source, task.hold_source);
10270 assert_eq!(after.operator_decided_at, task.operator_decided_at);
10271 assert_eq!(after.runs, task.runs);
10272 assert_eq!(after.instruction, task.instruction);
10273 assert!(after.resume_override.is_none());
10274 assert!(after.action_applied(&q.id));
10275 let reason = after.hold_reason.as_deref().unwrap();
10276 assert!(reason.starts_with("Keep this PR paused"));
10277 assert!(reason.contains("later operator decision"));
10278 assert!(reason.contains(&format!("magi task release {}", task.id)));
10279
10280 for recovery in [
10281 crate::conduct::Recovery::Review,
10282 crate::conduct::Recovery::Requeue,
10283 ] {
10284 crate::conduct::apply(
10285 &queue,
10286 &questions,
10287 &crate::conduct::Verdict {
10288 decisions: vec![crate::conduct::Decision {
10289 id: task.id.clone(),
10290 recovery: Some(recovery),
10291 ..Default::default()
10292 }],
10293 },
10294 )
10295 .unwrap();
10296 }
10297 apply_choice_actions(&queue, &questions, dir.path());
10300 let still_held = queue.get(&task.id).unwrap();
10301 assert!(still_held.operator_held());
10302 assert_eq!(still_held.hold_reason, after.hold_reason);
10303 assert_eq!(still_held.runs, task.runs);
10304 assert!(still_held.review_branch.is_none());
10305 assert!(!still_held.fresh_start);
10306 assert!(queue.next_runnable().is_none());
10307 }
10308 }
10309
10310 #[test]
10311 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
10312 let dir = tempfile::tempdir().unwrap();
10313 let queue = Queue::at(dir.path().join("queue"));
10314 let questions = Questions::at(dir.path().join("questions"));
10315 let home = dir.path().join("home");
10316 let mut state = run_state(RunStatus::Blocked);
10317 state.id = "20260101-000000-act1".to_owned();
10318 state.save_under(&home).unwrap();
10319
10320 let mut t = held_task_with(&state.id);
10321 queue.put(&mut t).unwrap();
10322 let mut q = action_question(&state.id, resume_action(&state.id));
10323 questions.put(&mut q).unwrap();
10324
10325 apply_choice_actions(&queue, &questions, &home);
10326 let after = queue.get(&t.id).unwrap();
10327 assert_eq!(after.status, TaskStatus::Queued);
10328 assert!(!after.fresh_start);
10329 assert!(after.action_applied(&q.id));
10330 let pin = after.resume_override.clone().unwrap();
10331 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
10332 assert!(pin.forced);
10333
10334 let mut again = queue.get(&t.id).unwrap();
10336 again.hold_machine(Some("later".into()));
10337 queue.put(&mut again).unwrap();
10338 apply_choice_actions(&queue, &questions, &home);
10339 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10340 }
10341
10342 #[test]
10343 fn a_delivered_answer_is_still_acted_on_exactly_once() {
10344 let dir = tempfile::tempdir().unwrap();
10345 let queue = Queue::at(dir.path().join("queue"));
10346 let questions = Questions::at(dir.path().join("questions"));
10347 let home = dir.path().join("home");
10348 let mut state = run_state(RunStatus::Blocked);
10349 state.id = "20260101-000000-act2".to_owned();
10350 state.save_under(&home).unwrap();
10351
10352 let mut t = held_task_with(&state.id);
10353 queue.put(&mut t).unwrap();
10354 let mut q = action_question(&state.id, resume_action(&state.id));
10355 q.answer_delivered = true;
10356 questions.put(&mut q).unwrap();
10357
10358 apply_choice_actions(&queue, &questions, &home);
10361 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10362 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
10363
10364 let mut again = queue.get(&t.id).unwrap();
10366 again.hold_machine(Some("later".into()));
10367 queue.put(&mut again).unwrap();
10368 apply_choice_actions(&queue, &questions, &home);
10369 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10370 }
10371
10372 #[test]
10373 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
10374 let dir = tempfile::tempdir().unwrap();
10375 let queue = Queue::at(dir.path().join("queue"));
10376 let questions = Questions::at(dir.path().join("questions"));
10377 let home = dir.path().join("home");
10378 let mut state = run_state(RunStatus::Blocked);
10379 state.id = "20260101-000000-act3".to_owned();
10380 state.save_under(&home).unwrap();
10381 let mut t = held_task_with(&state.id);
10382 queue.put(&mut t).unwrap();
10383 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
10384 questions.put(&mut q).unwrap();
10385
10386 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
10387 apply_choice_actions(&queue, &questions, &home);
10388 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10389
10390 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
10391 apply_choice_actions(&queue, &questions, &home);
10392 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10393 }
10394
10395 #[test]
10396 fn action_standing_tells_the_reasons_apart() {
10397 let mut t = held_task_with("r1");
10398 let q = action_question("r1", ask::ChoiceAction::Requeue);
10399 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
10400 let mut plain = q.clone();
10401 plain.actions.clear();
10402 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
10403
10404 t.mark_action_applied(&q.id);
10406 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
10407
10408 let mut t = held_task_with("r1");
10410 t.start("r2".to_owned());
10411 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
10412 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
10414 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10415 t.status = TaskStatus::Blocked;
10416 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
10417
10418 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
10420 c.node = crate::conduct::NODE.to_owned();
10421 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
10422 assert!(!ActionStanding::Busy.daemon_owns());
10423 }
10424
10425 #[test]
10426 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
10427 let dir = tempfile::tempdir().unwrap();
10428 let queue = Queue::at(dir.path().join("queue"));
10429 let questions = Questions::at(dir.path().join("questions"));
10430 let home = dir.path().join("home");
10431 let mut state = run_state(RunStatus::Blocked);
10432 state.id = "20260101-000000-act4".to_owned();
10433 state.save_under(&home).unwrap();
10434 let mut t = held_task_with(&state.id);
10435 t.status = TaskStatus::Running;
10436 queue.put(&mut t).unwrap();
10437 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
10438 q.cwd = Some(dir.path().display().to_string());
10439 questions.put(&mut q).unwrap();
10440
10441 let running = queue.get(&t.id).unwrap();
10444 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
10445 assert!(matches!(
10446 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
10447 crate::waiter::Action::Deliver(_)
10448 ));
10449 apply_choice_actions(&queue, &questions, &home);
10451 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10452 let mut back = queue.get(&t.id).unwrap();
10453 back.hold_machine(Some("later".into()));
10454 queue.put(&mut back).unwrap();
10455 apply_choice_actions(&queue, &questions, &home);
10456 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
10457 let mut again = queue.get(&t.id).unwrap();
10458 again.hold_machine(Some("again".into()));
10459 queue.put(&mut again).unwrap();
10460 apply_choice_actions(&queue, &questions, &home);
10461 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10462 }
10463}