1use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::sync::atomic::{AtomicBool, Ordering};
52use std::sync::{Mutex, MutexGuard};
53use std::time::Duration;
54
55use anyhow::{Context, Result, bail};
56use jiff::Timestamp;
57use serde::{Deserialize, Serialize};
58use tokio::sync::Notify;
59
60use crate::ask::{self, Questions};
61use crate::clean;
62use crate::conduct::Conductor;
63use crate::config::{Config, MergeMode};
64use crate::graph::Runner;
65use crate::land;
66use crate::notices::{self, Link, Notice};
67use crate::queue::{Queue, Task, TaskStatus};
68use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
69use crate::triage;
70
71pub const SCHEMA: u32 = 1;
73
74pub const HEARTBEAT: Duration = Duration::from_secs(5);
78
79pub const STALE_SECS: i64 = 30;
88
89pub const POLL: Duration = Duration::from_secs(5);
91
92pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
96
97pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
116
117#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
119#[serde(default)]
120pub struct Current {
121 pub task: String,
123 pub run: String,
125}
126
127#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct Status {
135 pub schema: u32,
137 pub pid: u32,
139 pub started_at: Timestamp,
141 pub updated_at: Timestamp,
143 pub idle: bool,
145 pub current: Vec<Current>,
151 pub completed: usize,
153 pub polls: u64,
155}
156
157impl Status {
158 #[must_use]
160 pub fn new() -> Self {
161 let now = Timestamp::now();
162 Self {
163 schema: SCHEMA,
164 pid: std::process::id(),
165 started_at: now,
166 updated_at: now,
167 idle: true,
168 current: Vec::new(),
169 completed: 0,
170 polls: 0,
171 }
172 }
173}
174
175impl Default for Status {
176 fn default() -> Self {
177 Self::new()
178 }
179}
180
181#[derive(Debug, Clone)]
183pub struct Opts {
184 pub repo: PathBuf,
186 pub config: Option<PathBuf>,
188 pub poll: Duration,
190 pub max_attempts: usize,
192 pub once: bool,
194 pub merge: Option<String>,
196 pub worktrees_root: Option<PathBuf>,
205}
206
207impl Default for Opts {
208 fn default() -> Self {
209 Self {
210 repo: PathBuf::from("."),
211 config: None,
212 poll: POLL,
213 max_attempts: 2,
214 once: false,
215 merge: None,
216 worktrees_root: None,
217 }
218 }
219}
220
221fn max_concurrent(n: usize) -> usize {
226 n.max(1)
227}
228
229#[must_use]
231pub fn status_path() -> PathBuf {
232 crate::run::home().join("daemon.json")
233}
234
235pub fn write_status(status: &Status) -> Result<()> {
237 write_status_to(&status_path(), status)
238}
239
240pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
245 if let Some(parent) = path.parent() {
246 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
247 }
248 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
249 let tmp = path.with_extension("json.tmp");
250 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
251 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
252 Ok(())
253}
254
255pub fn clear_status() {
258 clear_status_at(&status_path());
259}
260
261fn clear_status_at(path: &Path) {
265 let _ = std::fs::remove_file(path);
266}
267
268#[derive(Debug, Clone, Default)]
281pub struct Stop {
282 stopped: Arc<AtomicBool>,
286 busy: Arc<std::sync::atomic::AtomicUsize>,
292 wake: Arc<Notify>,
296 pause: crate::graph::Pause,
299}
300
301impl Stop {
302 #[must_use]
304 pub fn new() -> Self {
305 Self::default()
306 }
307
308 pub fn stop(&self) {
311 self.stopped.store(true, Ordering::SeqCst);
312 self.wake.notify_one();
316 }
317
318 #[must_use]
320 pub fn stopped(&self) -> bool {
321 self.stopped.load(Ordering::SeqCst)
322 }
323
324 #[must_use]
332 pub fn finishing(&self) -> bool {
333 self.stopped() && self.busy_now()
334 }
335
336 pub fn park(&self) {
347 self.pause.park();
348 self.stop();
349 }
350
351 #[must_use]
353 pub fn parking(&self) -> bool {
354 self.pause.parked()
355 }
356
357 #[must_use]
359 pub fn pause(&self) -> crate::graph::Pause {
360 self.pause.clone()
361 }
362
363 #[must_use]
369 pub fn busy_now(&self) -> bool {
370 self.busy.load(Ordering::SeqCst) > 0
371 }
372
373 fn enter(&self) {
375 self.busy.fetch_add(1, Ordering::SeqCst);
376 }
377
378 fn exit(&self) {
381 self.busy.fetch_sub(1, Ordering::SeqCst);
382 }
383
384 async fn idle(&self, poll: Duration) {
386 tokio::select! {
387 () = tokio::time::sleep(poll) => {}
388 () = self.wake.notified() => {}
389 }
390 }
391}
392
393#[derive(Debug, Clone, Default, Deserialize)]
400#[serde(default)]
401pub struct Reading {
402 pub schema: u32,
404 pub pid: Option<u32>,
406 pub started_at: Option<Timestamp>,
408 pub updated_at: Option<Timestamp>,
410 pub idle: bool,
412 #[serde(deserialize_with = "de_current")]
424 pub current: Vec<Current>,
425 pub completed: u64,
427 pub polls: u64,
429}
430
431fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
434where
435 D: serde::Deserializer<'de>,
436{
437 #[derive(Deserialize)]
438 #[serde(untagged)]
439 enum Shape {
440 Many(Vec<Current>),
441 One(Current),
442 }
443 Ok(
444 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
445 Shape::Many(v) => v,
446 Shape::One(c) => vec![c],
447 }),
448 )
449}
450
451impl Reading {
452 #[must_use]
455 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
456 self.updated_at
457 .map(|at| (now.as_second() - at.as_second()).max(0))
458 }
459
460 #[must_use]
464 pub fn running(&self, now: Timestamp) -> bool {
465 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
466 }
467}
468
469#[must_use]
476pub fn read_status(home: &Path) -> Option<Reading> {
477 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
478 serde_json::from_str(&body).ok()
479}
480
481#[must_use]
493pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
494 read_status(home)
495 .filter(|reading| reading.running(now))
496 .map(|reading| reading.current)
497 .unwrap_or_default()
498}
499
500#[must_use]
502pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
503 current_work(home, now).iter().any(|c| c.run == run)
504}
505
506#[must_use]
516pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
517 current_work(home, now)
518 .iter()
519 .any(|c| crate::run::short_of(&c.run) == short)
520}
521
522#[must_use]
524pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
525 current_work(home, now).iter().any(|c| c.task == task)
526}
527
528pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
572 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
573}
574
575fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
579where
580 F: Fn(u32) -> bool,
581{
582 let this_process = std::process::id();
583 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
584 .into_iter()
585 .flatten()
586 .flatten()
587 .map(|e| e.path())
588 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
589 .filter(|p| {
590 match std::fs::read_to_string(p)
591 .ok()
592 .and_then(|body| body.trim().parse::<u32>().ok())
593 {
594 Some(pid) if pid == this_process => false,
598 Some(pid) => !pid_alive(pid),
599 None => p
600 .metadata()
601 .and_then(|m| m.modified())
602 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
603 .is_ok_and(|age| age >= older_than),
604 }
605 })
606 .filter(|p| std::fs::remove_file(p).is_ok())
607 .filter_map(|p| {
608 p.file_stem()
609 .and_then(|s| s.to_str())
610 .map(std::borrow::ToOwned::to_owned)
611 })
612 .collect();
613 swept.sort_unstable();
614 swept
615}
616
617fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
622 task.status == TaskStatus::Running
623 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
624 && !is_working_on_task(home, &task.id, now)
625}
626
627fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
630 queue
631 .list()
632 .into_iter()
633 .filter(|t| is_stalled(t, home, now))
634 .collect()
635}
636
637fn queued_tasks(queue: &Queue) -> Vec<Task> {
643 queue
644 .list()
645 .into_iter()
646 .filter(|t| t.status == TaskStatus::Queued)
647 .collect()
648}
649
650fn finished_tasks(queue: &Queue) -> Vec<Task> {
653 queue
654 .list()
655 .into_iter()
656 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
657 .collect()
658}
659
660fn resolve_blockers(queue: &Queue, questions: &Questions) {
677 for listed in queue.list() {
678 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
679 continue;
680 }
681 let Ok(_claim) = queue.claim(&listed.id) else {
682 continue;
683 };
684 let Ok(mut task) = queue.get(&listed.id) else {
685 continue;
686 };
687 if task.status != TaskStatus::Blocked {
688 continue;
689 }
690 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
691 if !missing.is_empty() {
692 let language = language_of(&task, Path::new("."));
693 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
694 &task.blocked_by,
695 &missing,
696 &language,
697 )));
698 record(queue, &mut task);
699 continue;
700 }
701 if let Some(q) = task.blocked_by.iter().find_map(|id| {
704 questions.get(id).ok().filter(|q| {
705 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
706 })
707 }) {
708 let language = language_of(&task, Path::new("."));
709 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
710 record(queue, &mut task);
711 continue;
712 }
713 let mut changed = false;
714 for id in task.blocked_by.clone() {
715 if let Ok(dep) = queue.get(&id) {
716 if dep.status == TaskStatus::Done {
717 task.unblock(&id);
718 changed = true;
719 }
720 continue;
721 }
722 if let Ok(q) = questions.get(&id)
723 && q.status == ask::QuestionStatus::Answered
724 {
725 let answer = match &q.answer {
726 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
727 None => String::new(),
728 };
729 task.record_answer(q.summary.clone(), answer);
730 task.unblock(&id);
731 changed = true;
732 }
733 }
734 if changed {
735 record(queue, &mut task);
736 }
737 }
738}
739
740fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
742 if crate::lang::is_japanese(language) {
743 format!(
744 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
745 q.short(),
746 q.summary
747 )
748 } else {
749 format!(
750 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
751 q.short(),
752 q.summary
753 )
754 }
755}
756
757#[derive(Debug, Clone, PartialEq, Eq)]
759enum ActionDecision {
760 Skip,
762 Resume(String),
764 Requeue,
766 Done,
768 Stale,
771 Refuse(String),
774}
775
776fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
784where
785 F: FnOnce(&str) -> Result<RunState>,
786{
787 let Some(action) = q.chosen_action() else {
788 return ActionDecision::Skip;
789 };
790 if task.action_applied(&q.id)
791 || matches!(
792 task.status,
793 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
794 )
795 {
796 return ActionDecision::Skip;
797 }
798 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
801 return ActionDecision::Stale;
802 }
803 match action {
804 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
805 ask::ChoiceAction::Done => ActionDecision::Done,
806 ask::ChoiceAction::Resume { run } => {
807 if task.runs.last() != Some(run) {
808 return ActionDecision::Refuse((p.resume_not_latest)(
809 q.short(),
810 ask::short_id(run),
811 ));
812 }
813 match load(run) {
814 Ok(s) if s.status.resumable() && !s.released() && !exhausted_review_budget(&s) => {
815 ActionDecision::Resume(run.clone())
816 }
817 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
818 q.short(),
819 ask::short_id(run),
820 )),
821 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
822 q.short(),
823 ask::short_id(run),
824 &format!("{e:#}"),
825 )),
826 }
827 }
828 }
829}
830
831struct Phrases {
845 graph_stopped: fn(&str, &str) -> String,
847 quorum_lost: &'static str,
848 quota_took_out: &'static str,
850 run_ended: &'static str,
852 waiting_for_answer: &'static str,
854 recovered_running: &'static str,
855 no_run_to_recover: &'static str,
856 could_not_start: &'static str,
857 resume_not_latest: fn(&str, &str) -> String,
858 resume_cannot_progress: fn(&str, &str) -> String,
859 resume_unreadable: fn(&str, &str, &str) -> String,
860}
861
862const PHRASES_EN: Phrases = Phrases {
863 graph_stopped: |status, detail| {
864 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
865 },
866 quorum_lost: "the judging panel lost its quorum",
867 quota_took_out: "; quota took out ",
868 run_ended: "run ended ",
869 waiting_for_answer: " - waiting for operator answer to question ",
870 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
871 no_run_to_recover: "task was `running` with no live daemon and no readable \
872 run to recover; held for a human to check what happened",
873 could_not_start: "could not start the run: ",
874 resume_not_latest: |q, run| {
875 format!("question {q} asked to resume run {run}, which is not this task's latest run")
876 },
877 resume_cannot_progress: |q, run| {
878 format!("question {q} asked to resume run {run}, which cannot make progress")
879 },
880 resume_unreadable: |q, run, e| {
881 format!("question {q} asked to resume run {run}, which could not be read: {e}")
882 },
883};
884
885const PHRASES_JA: Phrases = Phrases {
886 graph_stopped: |status, detail| {
887 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
888 },
889 quorum_lost: "審査パネルが定足数を失いました",
890 quota_took_out: "。クォータで脱落: ",
891 run_ended: "run 終了: ",
892 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
893 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
894 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
895 何が起きたか人が確認するため保留にしました",
896 could_not_start: "run を開始できませんでした: ",
897 resume_not_latest: |q, run| {
898 format!(
899 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
900 )
901 },
902 resume_cannot_progress: |q, run| {
903 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
904 },
905 resume_unreadable: |q, run, e| {
906 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
907 },
908};
909
910fn phrases(language: &str) -> &'static Phrases {
911 if crate::lang::is_japanese(language) {
912 &PHRASES_JA
913 } else {
914 &PHRASES_EN
915 }
916}
917
918fn language_of(task: &Task, fallback: &Path) -> String {
922 crate::lang::of_repo(&repo_for(task, fallback))
923}
924
925fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
929 if q.node == crate::conduct::NODE {
930 return tasks.iter().find(|t| t.id == q.run);
931 }
932 tasks.iter().find(|t| t.runs.contains(&q.run))
933}
934
935fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
944 let tasks = queue.list();
945 for q in questions.list() {
946 if q.chosen_action().is_none() {
947 continue;
948 }
949 let Some(listed) = task_of_question(&tasks, &q) else {
950 continue;
951 };
952 if listed.action_applied(&q.id) {
953 continue;
954 }
955 let Ok(_claim) = queue.claim(&listed.id) else {
956 continue;
957 };
958 let Ok(mut task) = queue.get(&listed.id) else {
959 continue;
960 };
961 let language = language_of(&task, Path::new("."));
962 let decision = decide_action(&task, &q, phrases(&language), |id| {
963 RunState::load_under(id, home)
964 });
965 let ran = matches!(
966 decision,
967 ActionDecision::Resume(_) | ActionDecision::Requeue | ActionDecision::Done
968 );
969 if ran {
970 if questions
975 .read_lease(&q.id)
976 .is_some_and(|l| l.fresh(Timestamp::now()))
977 {
978 continue;
979 }
980 let taken = questions.update(&q.id, |r| {
981 let free = !r.answer_delivered;
982 r.answer_delivered = true;
983 Ok(free)
984 });
985 if !matches!(taken, Ok((_, true))) {
986 continue;
987 }
988 }
989 match decision {
990 ActionDecision::Skip => continue,
991 ActionDecision::Resume(run) => {
992 task.release();
993 task.resume_override = Some(crate::queue::OperatorResume {
994 question_id: q.id.clone(),
995 at: Timestamp::now(),
996 conductor_rehold: None,
997 forced: true,
998 pinned_run: Some(run),
999 });
1000 }
1001 ActionDecision::Stale => {}
1002 ActionDecision::Requeue => task.requeue(),
1003 ActionDecision::Done => {
1004 task.succeed();
1005 supersede_prior_runs(&task, home);
1006 }
1007 ActionDecision::Refuse(why) => {
1008 task.hold_machine(Some(why));
1009 notices::raise(
1010 Notice::warn(
1011 &format!("action:{}", q.id),
1012 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1013 )
1014 .link(Link::Task {
1015 id: task.id.clone(),
1016 }),
1017 );
1018 }
1019 }
1020 task.mark_action_applied(&q.id);
1021 record(queue, &mut task);
1022 }
1023}
1024
1025fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1036 let tasks = queue.list();
1037 let by_id: std::collections::BTreeMap<&str, &Task> =
1038 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1039 let referenced: std::collections::BTreeSet<&str> = tasks
1040 .iter()
1041 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1042 .collect();
1043
1044 for mut question in questions.list() {
1045 if !question.status.open() || question.node != crate::conduct::NODE {
1046 continue;
1047 }
1048 if referenced.contains(question.id.as_str()) {
1051 continue;
1052 }
1053 let Some(task) = by_id.get(question.run.as_str()) else {
1054 continue;
1055 };
1056 question.abandon(format!(
1057 "task {} no longer waits for this answer",
1058 task.short()
1059 ));
1060 if let Err(e) = questions.put(&mut question) {
1061 tracing::warn!(
1062 "could not retire question {} for task {}: {e:#}",
1063 question.short(),
1064 task.short()
1065 );
1066 }
1067 }
1068}
1069
1070#[derive(Debug, Clone, Copy)]
1077pub struct Verdict {
1078 pub status: RunStatus,
1080 pub left_pr: bool,
1082 pub quota_hit: bool,
1084 pub parked: bool,
1086 pub no_viable_candidates: bool,
1093}
1094
1095pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1152 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1153}
1154
1155fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1157 if verdict.parked {
1163 task.stall(detail);
1164 return;
1165 }
1166 match verdict.status {
1167 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1168 RunStatus::AlreadyInBase => task.already_landed(detail),
1169 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1170 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1171 task.stall(detail)
1172 }
1173 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1174 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1175 RunStatus::Blocked => task.fail(detail, max_attempts),
1176 RunStatus::VerifiedNoop => task.handed_off(detail),
1177 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1178 }
1179}
1180
1181pub fn supersede_prior_runs(task: &Task, home: &Path) {
1230 let now = Timestamp::now();
1231 let last_run_succeeded = task
1238 .runs
1239 .last()
1240 .and_then(|id| RunState::load_under(id, home).ok())
1241 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1242 for id in task.superseded_attempts(last_run_succeeded) {
1243 let mut state = match RunState::load_under(id, home) {
1244 Ok(s) => s,
1245 Err(e) => {
1246 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1247 continue;
1248 }
1249 };
1250 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1251 continue;
1252 }
1253 let daemon_claims = is_working_on(home, id, now);
1254 if state.liveness(daemon_claims) == Liveness::Live {
1255 continue;
1256 }
1257 state.status = RunStatus::Superseded;
1258 if let Err(e) = state.save_under(home) {
1259 tracing::warn!("could not mark run {id} superseded: {e:#}");
1260 }
1261 }
1262}
1263
1264fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1286 for task in queue.list() {
1287 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1288 continue;
1289 }
1290 supersede_prior_runs(&task, home);
1291 }
1292}
1293
1294fn settle_and_diagnose(
1301 task: &mut Task,
1302 verdict: Verdict,
1303 detail: &str,
1304 max_attempts: usize,
1305 state: &RunState,
1306) {
1307 let p = phrases(&state.config.graph.language);
1308 settle_in(task, verdict, detail, max_attempts, p);
1309 if task.status == TaskStatus::Held {
1310 task.diagnostic = diagnostic(state);
1311 note_open_question(task, &state.id, p);
1312 }
1313}
1314
1315fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
1330 let Some(home) = crate::run::try_home() else {
1331 return;
1332 };
1333 let open = Questions::at(home.join("questions")).open_for(run);
1334 let Some(q) = open.first() else {
1335 return;
1336 };
1337 let base = task.hold_reason.clone().unwrap_or_default();
1338 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
1339}
1340
1341fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
1352 match last_run {
1353 Some(state) => {
1354 let verdict = Verdict {
1355 status: state.status,
1356 left_pr: state.pr.is_some(),
1357 quota_hit: !state.quota.is_empty(),
1358 parked: state.parked,
1359 no_viable_candidates: state.viable().is_empty(),
1360 };
1361 let detail = format!(
1362 "{}{}",
1363 phrases(&state.config.graph.language).recovered_running,
1364 describe(&state)
1365 );
1366 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
1367 }
1368 None => {
1369 let why = phrases(language).no_run_to_recover;
1372 task.last_error = Some(why.to_owned());
1373 task.hold_machine(Some(why.to_owned()));
1376 }
1377 }
1378}
1379
1380fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
1402 let mut reclaimed = Vec::new();
1403 for listed in queue.list() {
1404 if listed.status != TaskStatus::Running {
1405 continue;
1406 }
1407 let Ok(_claim) = queue.claim(&listed.id) else {
1408 continue;
1409 };
1410 let Ok(mut task) = queue.get(&listed.id) else {
1414 continue;
1415 };
1416 if task.status != TaskStatus::Running {
1417 continue;
1418 }
1419 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
1420 if let Some(state) = &last_run
1430 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
1431 {
1432 tracing::warn!("abandon questions for {}: {e:#}", state.id);
1433 }
1434 let language = if last_run.is_none() {
1435 language_of(&task, Path::new("."))
1436 } else {
1437 String::new()
1438 };
1439 reclaim(&mut task, last_run, max_attempts, &language);
1440 if task.status == TaskStatus::Done {
1441 supersede_prior_runs(&task, &crate::run::home());
1442 }
1443 record(queue, &mut task);
1444 reclaimed.push(task.id.clone());
1445 }
1446 reclaimed
1447}
1448
1449fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
1481 reclaim_abandoned_runs_with(
1482 home,
1483 now,
1484 crate::proc::pid_status,
1485 crate::proc::process_started_at,
1486 )
1487}
1488
1489fn reclaim_abandoned_runs_with<F, G>(
1495 home: &Path,
1496 now: Timestamp,
1497 query: F,
1498 identity: G,
1499) -> Vec<String>
1500where
1501 F: Fn(u32) -> Option<bool>,
1502 G: Fn(u32) -> Option<String>,
1503{
1504 let mut abandoned = Vec::new();
1505 for entry in std::fs::read_dir(home.join("runs"))
1506 .into_iter()
1507 .flatten()
1508 .flatten()
1509 {
1510 let id = entry.file_name().to_string_lossy().into_owned();
1511 if !crate::run::is_run_id(&id) {
1512 continue;
1513 }
1514 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
1522 continue;
1523 };
1524 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
1525 continue;
1526 };
1527 if state.status.done() || !state.active_all_overrun(now) {
1528 continue;
1529 }
1530 let daemon_claims = is_working_on(home, &id, now);
1542 if state.liveness_with(daemon_claims, &query, &identity) != crate::run::Liveness::Dead {
1543 continue;
1544 }
1545 state.abandon("daemon");
1546 if let Err(e) = state.save_under(home) {
1547 tracing::warn!("could not persist abandoned run {id}: {e:#}");
1548 continue;
1549 }
1550 if let Some(notice) = notices::run_ended(&state) {
1553 notices::raise_in(home, notice);
1554 }
1555 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
1563 tracing::warn!("abandon questions for {id}: {e:#}");
1564 }
1565 abandoned.push(id);
1566 }
1567 abandoned
1568}
1569
1570pub async fn serve(opts: Opts) -> Result<()> {
1576 serve_until(opts, Stop::new()).await
1577}
1578
1579pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
1596 let signal = {
1597 let stop = stop.clone();
1598 tokio::spawn(async move {
1599 if tokio::signal::ctrl_c().await.is_ok() {
1600 stop.stop();
1601 tracing::info!("shutdown requested; a run in flight will be finished first");
1602 }
1603 })
1604 };
1605
1606 let worktrees_root = opts
1607 .worktrees_root
1608 .clone()
1609 .unwrap_or_else(crate::run::default_worktree_root);
1610 let outcome = drive(
1611 &opts,
1612 &Queue::open(),
1613 &status_path(),
1614 &crate::run::home(),
1615 &worktrees_root,
1616 &stop,
1617 )
1618 .await;
1619
1620 signal.abort();
1621 outcome
1622}
1623
1624async fn drive(
1637 opts: &Opts,
1638 queue: &Queue,
1639 status_file: &Path,
1640 home: &Path,
1641 worktrees_root: &Path,
1642 stop: &Stop,
1643) -> Result<()> {
1644 let status = Arc::new(Mutex::new(Status::new()));
1652 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
1653 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
1654
1655 let daemon_cfg = prepare(&opts.repo, opts)
1661 .map(|c| c.daemon)
1662 .unwrap_or_default();
1663 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
1664
1665 let waiter = tokio::spawn(crate::waiter::run(
1670 crate::waiter::Waiter::new(
1671 crate::ask::Questions::at(home.join("questions")),
1672 home.to_path_buf(),
1673 prepare(&opts.repo, opts).ok(),
1674 ),
1675 stop.clone(),
1676 ));
1677
1678 let deputies = tokio::spawn(crate::deputy::run(
1682 crate::deputy::Deputies::new(
1683 crate::ask::Questions::at(home.join("questions")),
1684 home.to_path_buf(),
1685 prepare(&opts.repo, opts).ok(),
1686 opts.repo.clone(),
1687 daemon_cfg.max_deputies,
1688 {
1689 let stop = stop.clone();
1690 Arc::new(move || stop.parking())
1691 },
1692 ),
1693 stop.clone(),
1694 ));
1695
1696 tracing::info!(
1697 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
1698 queue.root().display(),
1699 opts.poll.as_secs(),
1700 opts.max_attempts,
1701 concurrency,
1702 if daemon_cfg.pause_for_interrupts {
1703 ", interrupts enabled"
1704 } else {
1705 ""
1706 }
1707 );
1708
1709 janitor(&opts.repo, opts, home, worktrees_root).await;
1712 resweep_superseded_attempts(queue, home);
1713
1714 let outcome = poll(
1715 opts,
1716 queue,
1717 &status,
1718 home,
1719 worktrees_root,
1720 stop,
1721 DispatchLimits {
1722 max_concurrent: concurrency,
1723 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
1724 },
1725 )
1726 .await;
1727
1728 beat.abort();
1729 waiter.abort();
1730 deputies.abort();
1731 clear_status_at(status_file);
1732 outcome
1733}
1734
1735async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
1741 loop {
1742 tokio::time::sleep(HEARTBEAT).await;
1743 let snapshot = {
1744 let mut guard = lock(&status);
1745 guard.updated_at = Timestamp::now();
1746 guard.clone()
1747 };
1748 if let Err(e) = write_status_to(&path, &snapshot) {
1749 tracing::warn!("could not refresh the daemon status file: {e:#}");
1752 }
1753 }
1754}
1755
1756#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1759enum LandResume {
1760 NotLanding,
1763 StillWaiting,
1768 Ready,
1772}
1773
1774fn land_resume_state(task: &Task) -> LandResume {
1778 let Some(run_id) = task.runs.last() else {
1779 return LandResume::NotLanding;
1780 };
1781 let Ok(state) = RunState::load(run_id) else {
1782 return LandResume::NotLanding;
1783 };
1784 if state.status != RunStatus::Landing || !state.parked {
1785 return LandResume::NotLanding;
1786 }
1787 let store = ask::Questions::open();
1788 let waiting = store
1789 .list()
1790 .into_iter()
1791 .filter(|q| &q.run == run_id && q.node == land::APPROVAL_NODE)
1792 .max_by(|a, b| a.id.cmp(&b.id));
1793 let Some(mut q) = waiting else {
1794 return LandResume::Ready;
1795 };
1796 if !q.status.open() {
1797 return LandResume::Ready;
1798 }
1799 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
1806 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
1807 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
1808 q.abandon(format!(
1809 "no answer within {}s of asking",
1810 timeout.as_secs().max(1)
1811 ));
1812 if store.put(&mut q).is_ok() {
1815 return LandResume::Ready;
1816 }
1817 }
1818 LandResume::StillWaiting
1819}
1820
1821const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
1829
1830const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
1842
1843struct InFlightGuard<'a> {
1856 status: &'a Arc<Mutex<Status>>,
1857 stop: &'a Stop,
1858 task_id: &'a str,
1859}
1860
1861impl Drop for InFlightGuard<'_> {
1862 fn drop(&mut self) {
1863 lock(self.status).current.retain(|c| c.task != self.task_id);
1864 self.stop.exit();
1865 }
1866}
1867
1868#[derive(Debug, Clone, PartialEq, Eq)]
1890enum Interrupt {
1891 Idle,
1893 Parking {
1905 parked: Vec<String>,
1906 interrupt_task: String,
1907 },
1908 Running {
1916 parked: Vec<String>,
1917 interrupt_task: String,
1918 },
1919 Resuming { parked: Vec<String> },
1926}
1927
1928fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
1949 match state {
1950 Interrupt::Idle => {
1951 if in_flight.len() != 1 {
1963 return Interrupt::Idle;
1964 }
1965 match runnable.iter().find(|t| t.interrupt) {
1966 Some(t) => Interrupt::Parking {
1967 parked: in_flight.to_vec(),
1968 interrupt_task: t.id.clone(),
1969 },
1970 None => Interrupt::Idle,
1971 }
1972 }
1973 Interrupt::Parking {
1974 parked,
1975 interrupt_task,
1976 } => {
1977 if in_flight.iter().any(|id| parked.contains(id)) {
1978 Interrupt::Parking {
1980 parked,
1981 interrupt_task,
1982 }
1983 } else if in_flight.contains(&interrupt_task) {
1984 Interrupt::Running {
1985 parked,
1986 interrupt_task,
1987 }
1988 } else if runnable.iter().any(|t| t.id == interrupt_task) {
1989 Interrupt::Parking {
1993 parked,
1994 interrupt_task,
1995 }
1996 } else {
1997 Interrupt::Resuming { parked }
2002 }
2003 }
2004 Interrupt::Running {
2005 parked,
2006 interrupt_task,
2007 } => {
2008 if in_flight.contains(&interrupt_task) {
2009 Interrupt::Running {
2010 parked,
2011 interrupt_task,
2012 }
2013 } else {
2014 Interrupt::Resuming { parked }
2020 }
2021 }
2022 Interrupt::Resuming { parked } => {
2023 if in_flight.iter().any(|id| parked.contains(id)) {
2024 Interrupt::Idle
2030 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
2031 Interrupt::Resuming { parked }
2032 } else {
2033 Interrupt::Idle
2036 }
2037 }
2038 }
2039}
2040
2041fn advance_interrupt_tick(
2047 enabled: bool,
2048 state: Interrupt,
2049 in_flight: &[String],
2050 runnable: &[Task],
2051) -> Interrupt {
2052 if !enabled {
2053 return Interrupt::Idle;
2054 }
2055 advance_interrupt(state, in_flight, runnable)
2056}
2057
2058fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
2063 match state {
2064 Interrupt::Idle => candidates,
2065 Interrupt::Parking {
2066 parked,
2067 interrupt_task,
2068 } => {
2069 if in_flight.iter().any(|id| parked.contains(id)) {
2070 Vec::new()
2071 } else {
2072 candidates
2073 .into_iter()
2074 .filter(|t| &t.id == interrupt_task)
2075 .collect()
2076 }
2077 }
2078 Interrupt::Running { .. } => Vec::new(),
2079 Interrupt::Resuming { parked } => candidates
2087 .into_iter()
2088 .find(|t| parked.contains(&t.id))
2089 .into_iter()
2090 .collect(),
2091 }
2092}
2093
2094struct DispatchLimits {
2098 max_concurrent: usize,
2101 pause_for_interrupts: bool,
2103}
2104
2105#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2114enum PermitKind {
2115 None,
2120 Urgent,
2127 Ordinary,
2130}
2131
2132fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
2135 if priority {
2136 PermitKind::None
2137 } else if urgent {
2138 PermitKind::Urgent
2139 } else {
2140 PermitKind::Ordinary
2141 }
2142}
2143
2144async fn poll(
2163 opts: &Opts,
2164 queue: &Queue,
2165 status: &Arc<Mutex<Status>>,
2166 home: &Path,
2167 worktrees_root: &Path,
2168 stop: &Stop,
2169 limits: DispatchLimits,
2170) -> Result<()> {
2171 let DispatchLimits {
2172 max_concurrent,
2173 pause_for_interrupts,
2174 } = limits;
2175 let mut attempted: Vec<String> = Vec::new();
2180 let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
2181 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(1));
2189 let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
2195 let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
2196 let mut conductor = Conductor::new();
2197 let mut cache_last_checked: Option<Timestamp> = None;
2200 let mut interrupt = Interrupt::Idle;
2202 let mut interrupt_pauses: std::collections::HashMap<String, crate::graph::Pause> =
2208 std::collections::HashMap::new();
2209
2210 while !stop.stopped() {
2211 lock(status).polls += 1;
2212
2213 while let Some(result) = inflight.try_join_next() {
2218 if let Err(e) = result {
2219 tracing::error!("a spawned attempt did not finish cleanly: {e}");
2220 notices::raise(Notice::error(
2221 "loop:attempt",
2222 "A queued attempt ended abnormally; check the task it was running.",
2223 ));
2224 }
2225 }
2226
2227 let swept = sweep_stale_claims(queue, STALE_CLAIM);
2228 if !swept.is_empty() {
2229 tracing::warn!(
2230 "swept {} stale claim(s) left behind by an earlier daemon: {}",
2231 swept.len(),
2232 swept.join(", ")
2233 );
2234 }
2235 let now = Timestamp::now();
2240
2241 if !stop.busy_now() {
2246 maybe_prune_cache_between_runs(
2247 &opts.repo,
2248 opts,
2249 home,
2250 stop,
2251 &mut cache_last_checked,
2252 now,
2253 )
2254 .await;
2255 }
2256
2257 let stalled = stalled_tasks(queue, home, now);
2258 let stalled_ids: std::collections::BTreeSet<_> =
2259 stalled.iter().map(|task| task.id.clone()).collect();
2260 let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
2261 if !reclaimed.is_empty() {
2262 tracing::warn!(
2263 "reclaimed {} task(s) left `running` by a daemon that never \
2264 recorded the outcome: {}",
2265 reclaimed.len(),
2266 reclaimed.join(", ")
2267 );
2268 }
2269 let abandoned_runs = reclaim_abandoned_runs(home, now);
2270 if !abandoned_runs.is_empty() {
2271 tracing::warn!(
2272 "failed {} run(s) left behind by a killed process, past every \
2273 active seat's own timeout: {}",
2274 abandoned_runs.len(),
2275 abandoned_runs.join(", ")
2276 );
2277 }
2278
2279 let questions = Questions::at(home.join("questions"));
2284
2285 resolve_blockers(queue, &questions);
2288 apply_choice_actions(queue, &questions, home);
2289 reconcile_task_questions(queue, &questions);
2290
2291 let finished: Vec<Task> = finished_tasks(queue)
2297 .into_iter()
2298 .filter(|task| !stalled_ids.contains(&task.id))
2299 .filter(|task| !awaiting_resume_with(task, |id| RunState::load_under(id, home)))
2303 .collect();
2304 let queued = queued_tasks(queue);
2305 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
2309 && conductor.worth_a_look(queue, &stalled, &finished)
2310 {
2311 match prepare(&opts.repo, opts) {
2312 Ok(cfg) => {
2313 conductor
2314 .maybe_run(
2315 &cfg,
2316 &opts.repo,
2317 queue,
2318 &questions,
2319 home,
2320 &queued,
2321 &stalled,
2322 &finished,
2323 opts.max_attempts,
2324 )
2325 .await;
2326 }
2327 Err(e) => {
2328 tracing::warn!("conductor: no config: {e:#}");
2329 notices::raise(Notice::warn(
2330 "loop:no-config",
2331 "The loop could not read this repository's config, so held tasks are not being triaged.",
2332 ));
2333 }
2334 }
2335 }
2336
2337 let candidates: Vec<Task> = runnable(queue)
2338 .into_iter()
2339 .filter(|t| !opts.once || !attempted.contains(&t.id))
2340 .collect();
2341
2342 let in_flight: Vec<String> = lock(status)
2347 .current
2348 .iter()
2349 .map(|c| c.task.clone())
2350 .collect();
2351 interrupt_pauses.retain(|id, _| in_flight.contains(id));
2352
2353 interrupt =
2354 advance_interrupt_tick(pause_for_interrupts, interrupt, &in_flight, &candidates);
2355 if let Interrupt::Parking {
2356 parked,
2357 interrupt_task,
2358 } = &interrupt
2359 {
2360 let reason = format!(
2361 "task {} asked to run first",
2362 crate::run::short_of(interrupt_task)
2363 );
2364 for id in parked {
2365 if let Some(pause) = interrupt_pauses.get(id) {
2366 pause.park_because(reason.clone());
2367 }
2368 }
2369 }
2370 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
2371
2372 let cooling_down =
2373 lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
2374
2375 let mut started_any = false;
2376 for candidate in candidates {
2377 if stop.stopped() {
2378 break;
2379 }
2380
2381 let resume = land_resume_state(&candidate);
2382 if resume == LandResume::StillWaiting {
2383 continue;
2384 }
2385 let priority = resume == LandResume::Ready;
2386
2387 if !priority && cooling_down {
2388 continue;
2389 }
2390 let permit = match permit_kind(priority, candidate.urgent) {
2391 PermitKind::None => None,
2392 PermitKind::Urgent => match Arc::clone(&urgent_sem).try_acquire_owned() {
2393 Ok(p) => Some(p),
2394 Err(_) => continue,
2400 },
2401 PermitKind::Ordinary => match Arc::clone(&sem).try_acquire_owned() {
2402 Ok(p) => Some(p),
2403 Err(_) => continue,
2407 },
2408 };
2409
2410 let Ok(claim) = queue.claim(&candidate.id) else {
2415 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
2416 continue;
2417 };
2418 let mut task = match queue.get(&candidate.id) {
2421 Ok(t) if t.status.runnable() => t,
2422 Ok(_) => continue,
2423 Err(e) => {
2424 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
2425 continue;
2426 }
2427 };
2428 let task_id = task.id.clone();
2429 attempted.push(task_id.clone());
2430 lock(status).idle = false;
2431 stop.enter();
2434 started_any = true;
2435
2436 let run_pause = crate::graph::Pause::new();
2440 interrupt_pauses.insert(task_id.clone(), run_pause.clone());
2441
2442 let opts = opts.clone();
2443 let queue = queue.clone();
2444 let status = Arc::clone(status);
2445 let stop = stop.clone();
2446 let quota_cooldown_until = Arc::clone("a_cooldown_until);
2447 inflight.spawn(async move {
2448 let _claim = claim;
2452 let _permit = permit;
2453 let _inflight = InFlightGuard {
2455 status: &status,
2456 stop: &stop,
2457 task_id: &task_id,
2458 };
2459 let quota = attempt(&opts, &queue, &status, &stop, run_pause, &mut task).await;
2460 lock(&status).completed += 1;
2461 let now = Timestamp::now();
2467 if let Some(until) = cooldown_until("a, now) {
2468 let wait = until.as_second() - now.as_second();
2469 *lock("a_cooldown_until) = Some(until);
2470 let hint = quota
2471 .iter()
2472 .find(|q| q.reset.is_some())
2473 .and_then(|q| q.reset.as_deref());
2474 match hint {
2475 Some(h) => tracing::warn!(
2476 "quota hit; waiting {wait}s before taking another ordinary task \
2477 (CLI reported reset: {h})"
2478 ),
2479 None => tracing::warn!(
2480 "quota hit; waiting {wait}s before taking another ordinary task \
2481 (no reset hint reported)"
2482 ),
2483 }
2484 }
2485 });
2486 }
2487
2488 if started_any {
2489 continue;
2490 }
2491
2492 if stop.busy_now() {
2493 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
2498 continue;
2499 }
2500
2501 lock(status).idle = true;
2503 if opts.once {
2504 janitor(&opts.repo, opts, home, worktrees_root).await;
2508 resweep_superseded_attempts(queue, home);
2509 triage_held(queue, home, opts).await;
2510 break;
2511 }
2512 stop.idle(opts.poll).await;
2513 if stop.stopped() {
2514 continue;
2515 }
2516 janitor(&opts.repo, opts, home, worktrees_root).await;
2522 resweep_superseded_attempts(queue, home);
2523 triage_held(queue, home, opts).await;
2524 }
2525
2526 while let Some(result) = inflight.join_next().await {
2531 if let Err(e) = result {
2532 tracing::error!("a spawned attempt did not finish cleanly: {e}");
2533 notices::raise(Notice::error(
2534 "loop:attempt",
2535 "A queued attempt ended abnormally; check the task it was running.",
2536 ));
2537 }
2538 }
2539 Ok(())
2540}
2541
2542async fn attempt(
2548 opts: &Opts,
2549 queue: &Queue,
2550 status: &Arc<Mutex<Status>>,
2551 stop: &Stop,
2552 interrupt_pause: crate::graph::Pause,
2553 task: &mut Task,
2554) -> Vec<QuotaLoss> {
2555 let repo = repo_for(task, &opts.repo);
2556 tracing::info!(
2557 "task {} — {} (repo {})",
2558 task.short(),
2559 task.title,
2560 repo.display()
2561 );
2562
2563 let mut config = match prepare(&repo, opts) {
2564 Ok(c) => c,
2565 Err(e) => {
2566 task.attempts += 1;
2570 task.fail(format!("config: {e:#}"), opts.max_attempts);
2571 record(queue, task);
2572 return Vec::new();
2573 }
2574 };
2575 apply_solo(&mut config, task);
2576 let start_failed = phrases(&config.graph.language).could_not_start;
2577
2578 if let Some(reason) = disk_gate(&repo, &config) {
2586 task.last_error = Some(reason.clone());
2587 task.hold_machine(Some(reason.clone()));
2588 record(queue, task);
2589 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
2590 notices::raise(
2593 Notice::warn(
2594 &format!("disk:{}", repo.display()),
2595 "A task was held for want of free disk space; free some, then release it from the queue.",
2596 )
2597 .link(Link::Task {
2598 id: task.id.clone(),
2599 }),
2600 );
2601 return Vec::new();
2602 }
2603
2604 let unfinished = (!task.fresh_start)
2624 .then(|| unfinished_run(&task.runs, task.short()))
2625 .flatten();
2626 let review_branch = task.review_branch.take();
2632 let branch_exists = match &review_branch {
2633 Some(branch) => crate::git::branch_exists(&repo, branch)
2634 .await
2635 .unwrap_or(false),
2636 None => false,
2637 };
2638 let starter = choose_starter(
2639 review_branch.as_deref(),
2640 branch_exists,
2641 unfinished.as_deref(),
2642 );
2643 let attachments = match task_attachments(queue, task) {
2648 Ok(a) => a,
2649 Err(e) => {
2650 task.attempts += 1;
2651 task.fail(format!("{start_failed}{e:#}"), opts.max_attempts);
2652 record(queue, task);
2653 return Vec::new();
2654 }
2655 };
2656 let started = match &starter {
2657 Starter::Review(branch) => {
2658 tracing::info!(
2659 "task {} reopens `{branch}` as a review-only pass",
2660 task.short()
2661 );
2662 let takeover = crate::handover::Takeover {
2665 earlier: task.earlier_attempts().to_vec(),
2666 home: crate::run::home(),
2667 choice: take_divergence_answer(branch, &config.merge.remote, task),
2668 };
2669 Runner::review_taking_over(
2670 &repo,
2671 branch,
2672 config,
2673 Some(takeover),
2674 crate::run::Origin::queue(&task.id),
2675 )
2676 .await
2677 }
2678 Starter::Resume(id) => {
2679 tracing::info!("resuming run {id} rather than competing again");
2680 Runner::resume(id).map(|mut r| {
2681 if let Some(instruction) =
2682 prepare_instruction(&starter, Some(&r.state.instruction), task)
2683 {
2684 r.state.instruction = instruction;
2685 }
2686 r.state.attachments = attachments.clone();
2687 r
2688 })
2689 }
2690 Starter::Start => {
2691 if let Some(branch) = &review_branch {
2692 tracing::warn!(
2693 "conductor chose review for task {} but branch `{branch}` no longer \
2694 exists; requeuing as a fresh competition instead",
2695 task.short()
2696 );
2697 }
2698 let instruction = prepare_instruction(&starter, None, task)
2699 .unwrap_or_else(|| task.instruction.clone());
2700 Runner::start_naming(
2701 &repo,
2702 instruction,
2703 &task.title,
2704 config,
2705 crate::run::Origin::queue(&task.id),
2706 )
2707 .await
2708 .map(|mut r| {
2709 r.state.attachments = attachments.clone();
2710 r
2711 })
2712 }
2713 };
2714 let mut runner = match started {
2715 Ok(r) => r,
2716 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
2720 let reason = format!("{start_failed}{e:#}");
2721 task.last_error = Some(reason.clone());
2722 let branch = match &starter {
2725 Starter::Review(branch) => Some(branch.clone()),
2726 _ => None,
2727 };
2728 task.hold_for_handover(
2732 branch,
2733 format!(
2734 "{reason} (clean up the other worktree, then release the task from the queue)"
2735 ),
2736 );
2737 record(queue, task);
2738 tracing::warn!(
2739 "holding {} for a branch it cannot take over: {e:#}",
2740 task.short()
2741 );
2742 return Vec::new();
2743 }
2744 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
2749 let d = e
2750 .downcast_ref::<crate::reconcile::Diverged>()
2751 .expect("checked by the guard");
2752 let mut q = ask::Question::new(
2753 task.id.clone(),
2754 "review".to_owned(),
2755 "sync".to_owned(),
2756 d.summary(),
2757 d.detail(),
2758 d.choices(),
2759 );
2760 match Questions::open().put(&mut q) {
2761 Ok(()) => {
2762 task.last_error = Some(format!("{e:#}"));
2763 task.review_branch = Some(d.branch.clone());
2766 task.block(vec![q.id.clone()], Some(d.summary()));
2767 }
2768 Err(put) => {
2769 tracing::warn!("could not file the divergence question: {put:#}");
2770 task.attempts += 1;
2771 task.fail(format!("{start_failed}{e:#}"), opts.max_attempts);
2772 }
2773 }
2774 record(queue, task);
2775 return Vec::new();
2776 }
2777 Err(e) => {
2778 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
2784 && let Starter::Review(branch) = &starter
2785 {
2786 task.review_branch = Some(branch.clone());
2787 task.last_error = Some(format!("{start_failed}{e:#}"));
2788 task.status = crate::queue::TaskStatus::Failed;
2789 record(queue, task);
2790 return Vec::new();
2791 }
2792 task.attempts += 1;
2793 task.fail(format!("{start_failed}{e:#}"), opts.max_attempts);
2794 record(queue, task);
2795 return Vec::new();
2796 }
2797 };
2798 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
2801 runner.on_pause(stop.pause());
2803 runner.watch_interrupt(interrupt_pause);
2807
2808 let run = runner.state.id.clone();
2811 task.start(run.clone());
2812 record(queue, task);
2813 lock(status).current.push(Current {
2814 task: task.id.clone(),
2815 run,
2816 });
2817
2818 let quota_before = runner.state.quota.clone();
2821 let detail = match runner.execute().await {
2822 Ok(()) => describe(&runner.state),
2823 Err(e) => format!("{e:#}"),
2824 };
2825 let fresh = losses_this_attempt("a_before, &runner.state.quota);
2826 let verdict = Verdict {
2827 status: runner.state.status,
2828 left_pr: runner.state.pr.is_some(),
2831 quota_hit: !fresh.is_empty(),
2837 parked: runner.state.parked,
2841 no_viable_candidates: runner.state.viable().is_empty(),
2844 };
2845 settle_and_diagnose(task, verdict, &detail, opts.max_attempts, &runner.state);
2846 if task.status == TaskStatus::Done {
2847 supersede_prior_runs(task, &crate::run::home());
2848 }
2849 record(queue, task);
2850 tracing::info!(
2851 "task {} is {} after run {} ({})",
2852 task.short(),
2853 task.status.as_str(),
2854 runner.state.short(),
2855 label(runner.state.status)
2856 );
2857 fresh
2858}
2859
2860fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
2870 after
2871 .iter()
2872 .filter(|q| !before.contains(q))
2873 .cloned()
2874 .collect()
2875}
2876
2877fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
2880 if quota.is_empty() {
2881 return None;
2882 }
2883 let with_hint = quota.iter().find(|q| q.reset.is_some());
2884 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
2885 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
2886 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
2887 Some(
2888 now.checked_add(jiff::SignedDuration::from_secs(secs))
2889 .unwrap_or(Timestamp::MAX),
2890 )
2891}
2892
2893fn apply_solo(config: &mut Config, task: &Task) {
2903 if task.solo {
2904 config.graph.candidates = 1;
2905 }
2906}
2907
2908fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
2910 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
2911 if let Some(mode) = &opts.merge {
2912 config.merge.mode = merge_mode(mode)?;
2913 }
2914 Ok(config)
2915}
2916
2917async fn maybe_prune_cache_between_runs(
2948 repo: &Path,
2949 opts: &Opts,
2950 home: &Path,
2951 stop: &Stop,
2952 last_checked: &mut Option<Timestamp>,
2953 now: Timestamp,
2954) {
2955 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
2956 return;
2957 }
2958 *last_checked = Some(now);
2959 let cfg = match prepare(repo, opts) {
2960 Ok(cfg) => cfg,
2961 Err(e) => {
2962 tracing::warn!("cache check: no config: {e:#}");
2963 return;
2964 }
2965 };
2966 match clean::prune_cache_if_over_limit(&cfg, home) {
2967 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
2968 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
2969 pruned.files,
2970 pruned.freed
2971 ),
2972 Ok(_) => {}
2973 Err(e) => {
2974 tracing::warn!("housekeep: prune cache: {e:#}");
2975 notices::raise_in(
2976 home,
2977 Notice::warn(
2978 "housekeep:cache",
2979 "Pruning the shared build cache failed; disk usage may keep growing.",
2980 ),
2981 );
2982 }
2983 }
2984}
2985
2986fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
2990 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
2991}
2992
2993async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
3012 let cfg = match prepare(repo, opts) {
3013 Ok(cfg) => cfg,
3014 Err(e) => {
3015 tracing::warn!("housekeep: no config: {e:#}");
3016 return;
3017 }
3018 };
3019 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
3028 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
3029 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
3034 let mut extra = Vec::new();
3035 if out.unreadable > 0 {
3036 extra.push(format!("{} unreadable", out.unreadable));
3037 }
3038 if out.orphaned_worktrees > 0 {
3039 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
3040 }
3041 let detail = if extra.is_empty() {
3042 String::new()
3043 } else {
3044 format!(" ({})", extra.join(", "))
3045 };
3046 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
3047 }
3048 if out.external_merges_recorded > 0 {
3049 tracing::info!(
3050 "housekeep: recorded {} run(s) as merged externally",
3051 out.external_merges_recorded
3052 );
3053 }
3054 if out.stale_pr_states_repaired > 0 {
3055 tracing::info!(
3056 "housekeep: rewrote {} run record(s) whose pull request had already settled",
3057 out.stale_pr_states_repaired
3058 );
3059 }
3060 if out.cache_files > 0 {
3061 tracing::info!(
3062 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
3063 out.cache_files,
3064 out.cache_freed
3065 );
3066 }
3067 if out.questions_abandoned > 0 {
3068 tracing::info!(
3069 "housekeep: abandoned {} question(s) left open by a finished run",
3070 out.questions_abandoned
3071 );
3072 }
3073}
3074
3075async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
3084 let questions = Questions::at(home.join("questions"));
3085 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
3086 if report.is_empty() {
3087 return;
3088 }
3089 if !report.quarantined.is_empty() {
3090 tracing::info!(
3091 "triage: held {} blocked task(s) whose blocked-on task or \
3092 question no longer exists: {}",
3093 report.quarantined.len(),
3094 report.quarantined.join(", ")
3095 );
3096 }
3097 if !report.resumed.is_empty() {
3098 tracing::info!(
3099 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
3100 report.resumed.len(),
3101 report.resumed.join(", ")
3102 );
3103 }
3104 if !report.asked.is_empty() {
3105 tracing::info!(
3106 "triage: asked about {} held task(s): {}",
3107 report.asked.len(),
3108 report.asked.join(", ")
3109 );
3110 }
3111 if !report.answered.is_empty() {
3112 tracing::info!(
3113 "triage: applied {} operator answer(s): {}",
3114 report.answered.len(),
3115 report.answered.join(", ")
3116 );
3117 }
3118}
3119
3120fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
3127 disk_gate_with(repo, config, crate::disk::free_bytes)
3128}
3129
3130fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
3134 repo: &Path,
3135 config: &Config,
3136 free_bytes: F,
3137) -> Option<String> {
3138 let min = config.disk.min_free_bytes;
3139 if min == 0 {
3140 return None;
3141 }
3142 match free_bytes(repo) {
3143 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
3144 Err(e) => Some(crate::disk::unmeasured_in(repo, &e, &config.graph.language)),
3145 }
3146}
3147
3148const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
3155
3156const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
3160
3161fn quota_wait(
3170 reset_at: Option<Timestamp>,
3171 now: Timestamp,
3172 fallback: Duration,
3173 cap: Duration,
3174) -> Duration {
3175 match reset_at {
3176 Some(at) if at > now => {
3177 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
3178 Duration::from_secs(secs).min(cap)
3179 }
3180 _ => fallback,
3181 }
3182}
3183
3184fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
3198 parse_reset_hint_zoned(text, now)
3199 .or_else(|| parse_reset_hint_dated(text))
3200 .or_else(|| parse_reset_hint_relative(text, recorded))
3201}
3202
3203fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
3207 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
3208 let mut rest = rest.trim();
3209 if rest.is_empty() {
3210 return None;
3211 }
3212 let mut total: i64 = 0;
3213 let mut matched = false;
3214 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
3215 if let Some((digits, tail)) = rest.split_once(unit)
3216 && !digits.is_empty()
3217 && digits.bytes().all(|b| b.is_ascii_digit())
3218 {
3219 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
3220 rest = tail;
3221 matched = true;
3222 }
3223 }
3224 if !rest.is_empty() || !matched {
3225 return None;
3226 }
3227 recorded
3228 .checked_add(jiff::SignedDuration::from_secs(total))
3229 .ok()
3230}
3231
3232fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
3236 let clock = clock.trim().to_lowercase();
3237 let (digits, pm) = clock
3238 .strip_suffix("am")
3239 .map(|d| (d, false))
3240 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
3241 let (h, m) = digits.trim().split_once(':')?;
3242 let mut hour: i8 = h.trim().parse().ok()?;
3243 let minute: i8 = m.trim().parse().ok()?;
3244 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
3245 return None;
3246 }
3247 if pm && hour != 12 {
3248 hour += 12;
3249 } else if !pm && hour == 12 {
3250 hour = 0;
3251 }
3252 Some((hour, minute))
3253}
3254
3255fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
3260 let open = text.find('(')?;
3261 let close = text.rfind(')')?;
3262 if close <= open {
3263 return None;
3264 }
3265 let zone = text[open + 1..close].trim();
3266 let (hour, minute) = parse_12h_clock(&text[..open])?;
3267 let tz = jiff::tz::TimeZone::get(zone).ok()?;
3268 let candidate = now
3269 .to_zoned(tz)
3270 .with()
3271 .hour(hour)
3272 .minute(minute)
3273 .second(0)
3274 .millisecond(0)
3275 .microsecond(0)
3276 .nanosecond(0)
3277 .build()
3278 .ok()?;
3279 let mut at = candidate.timestamp();
3280 if at <= now {
3281 at += jiff::SignedDuration::from_hours(24);
3282 }
3283 Some(at)
3284}
3285
3286fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
3295 let words: Vec<&str> = text.split_whitespace().collect();
3296 if words.len() < 5 {
3297 return None;
3298 }
3299 (0..=words.len() - 5)
3300 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
3301}
3302
3303fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
3309 if trailing.is_some_and(|next| next.starts_with('(')) {
3310 return None;
3311 }
3312 let month = month_number(window[0])?;
3313 let day_token = window[1].strip_suffix(',')?.to_lowercase();
3314 let day_digits = ["st", "nd", "rd", "th"]
3315 .iter()
3316 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
3317 let day: i8 = day_digits.parse().ok()?;
3318 let year_token = window[2];
3319 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
3320 return None;
3321 }
3322 let year: i16 = year_token.parse().ok()?;
3323 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
3327 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
3328 let date = jiff::civil::Date::new(year, month, day).ok()?;
3329 let candidate = date
3330 .at(hour, minute, 0, 0)
3331 .to_zoned(jiff::tz::TimeZone::UTC)
3332 .ok()?;
3333 Some(candidate.timestamp())
3334}
3335
3336fn month_number(name: &str) -> Option<i8> {
3339 const NAMES: [&str; 12] = [
3340 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
3341 ];
3342 let lower = name.to_lowercase();
3343 NAMES
3344 .iter()
3345 .position(|n| *n == lower.as_str())
3346 .map(|i| i as i8 + 1)
3347}
3348
3349fn exhausted_review_budget(state: &RunState) -> bool {
3361 state.status == RunStatus::Blocked && state.reviews.len() >= state.config.graph.review_rounds
3362}
3363
3364fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
3401 unfinished_run_with(runs, short, RunState::load)
3402}
3403
3404fn unfinished_run_with<F>(runs: &[String], short: &str, load: F) -> Option<String>
3407where
3408 F: FnOnce(&str) -> Result<RunState>,
3409{
3410 let id = runs.last()?;
3411 match load(id) {
3412 Ok(s)
3417 if s.status.resumable()
3418 && !s.released()
3419 && !exhausted_review_budget(&s)
3420 && s.liveness(false) != crate::run::Liveness::Live =>
3421 {
3422 Some(id.clone())
3423 }
3424 Ok(_) => None,
3425 Err(e) => {
3426 tracing::warn!("could not read run {id} for task {short}: {e:#}");
3427 None
3428 }
3429 }
3430}
3431
3432fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
3440where
3441 F: FnOnce(&str) -> Result<RunState>,
3442{
3443 if task.status != TaskStatus::Failed || task.fresh_start || task.review_branch.is_some() {
3444 return false;
3445 }
3446 let Some(id) = task.runs.last() else {
3447 return false;
3448 };
3449 load(id).is_ok_and(|s| {
3450 s.parked && s.status.resumable() && !s.released() && !exhausted_review_budget(&s)
3451 })
3452}
3453
3454#[derive(Debug, Clone, PartialEq, Eq)]
3457enum Starter {
3458 Review(String),
3461 Resume(String),
3463 Start,
3465}
3466
3467fn take_divergence_answer(
3471 branch: &str,
3472 remote: &str,
3473 task: &mut Task,
3474) -> Option<crate::reconcile::Choice> {
3475 let summary = crate::reconcile::summary_for(branch, remote);
3476 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
3477 (a.question == summary)
3478 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
3479 .flatten()
3480 .map(|c| (i, c))
3481 })?;
3482 task.answers.remove(idx);
3483 Some(choice)
3484}
3485
3486fn choose_starter(
3498 review_branch: Option<&str>,
3499 branch_exists: bool,
3500 unfinished: Option<&str>,
3501) -> Starter {
3502 match review_branch {
3503 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
3504 Some(_) => Starter::Start,
3505 None => match unfinished {
3506 Some(id) => Starter::Resume(id.to_owned()),
3507 None => Starter::Start,
3508 },
3509 }
3510}
3511
3512fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
3515 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
3516 return fallback.to_path_buf();
3517 }
3518 task.repo.clone()
3519}
3520
3521const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
3525
3526fn answers_block(task: &Task, count: usize) -> String {
3528 let mut s = ANSWERS_HEADER.to_owned();
3529 for a in &task.answers[..count] {
3530 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
3531 }
3532 s
3533}
3534
3535fn append_answers(base: &str, task: &Task) -> String {
3538 if task.answers.is_empty() {
3539 return base.to_owned();
3540 }
3541 let mut s = base.to_owned();
3542 s.push_str(&answers_block(task, task.answers.len()));
3543 s
3544}
3545
3546fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
3550 for count in (1..=task.answers.len()).rev() {
3551 let block = answers_block(task, count);
3552 if let Some(base) = instruction.strip_suffix(&block) {
3553 return base;
3554 }
3555 }
3556 instruction
3557}
3558
3559fn instruction_for(task: &Task) -> String {
3567 append_answers(&task.instruction, task)
3568}
3569
3570fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
3582 append_answers(strip_answers_block(old_instruction, task), task)
3583}
3584
3585fn task_attachments(queue: &Queue, task: &Task) -> Result<Vec<PathBuf>> {
3587 let paths = queue.attachment_paths(task);
3588 for (name, path) in task.attachments.iter().zip(&paths) {
3589 if !path.is_file() {
3590 bail!(
3591 "attachment `{name}` is recorded on the task but {} is missing",
3592 path.display()
3593 );
3594 }
3595 }
3596 Ok(paths)
3597}
3598
3599fn prepare_instruction(
3610 starter: &Starter,
3611 old_instruction: Option<&str>,
3612 task: &Task,
3613) -> Option<String> {
3614 match starter {
3615 Starter::Start => Some(instruction_for(task)),
3616 Starter::Resume(_) => Some(resumed_instruction(
3617 old_instruction.expect("a resumed run always has a prior instruction"),
3618 task,
3619 )),
3620 Starter::Review(_) => None,
3621 }
3622}
3623
3624fn record(queue: &Queue, task: &mut Task) {
3628 if let Err(e) = queue.put(task) {
3629 tracing::error!("could not record task {}: {e:#}", task.short());
3630 notices::raise(Notice::error(
3631 "loop:record",
3632 "The loop could not save a task's state; check the disk.",
3633 ));
3634 }
3635}
3636
3637fn runnable(queue: &Queue) -> Vec<Task> {
3643 let mut tasks: Vec<Task> = queue
3644 .list()
3645 .into_iter()
3646 .filter(|t| t.status.runnable())
3647 .collect();
3648 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)));
3649 tasks
3650}
3651
3652fn describe(state: &RunState) -> String {
3666 let p = phrases(&state.config.graph.language);
3667 let mut detail = if state.status == RunStatus::Stalled {
3668 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
3669 seats.sort_unstable();
3670 seats.dedup();
3671 if seats.is_empty() {
3672 p.quorum_lost.to_owned()
3673 } else {
3674 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
3675 }
3676 } else {
3677 format!("{}{}", p.run_ended, state.status.display_label())
3678 };
3679 if let Some(last) = state.events.last() {
3680 detail.push_str(&format!(" ({}: {})", last.node, last.message));
3681 }
3682 detail.push_str(&format!(" [run {}]", state.id));
3683 detail
3684}
3685
3686const DIAGNOSTIC_MAX: usize = 4_000;
3692
3693const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
3698
3699fn diagnostic(state: &RunState) -> Option<String> {
3713 let mut parts: Vec<String> = Vec::new();
3714
3715 for o in state.gate.iter().filter(|o| !o.ok()) {
3717 parts.push(format!(
3718 "gate `{}` failed ({:?}):\n{}",
3719 o.command,
3720 o.code,
3721 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
3722 ));
3723 }
3724
3725 if let Some(last) = state
3728 .events
3729 .iter()
3730 .rev()
3731 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
3732 {
3733 parts.push(last.message.clone());
3734 }
3735
3736 if state.viable().is_empty() {
3743 for c in &state.candidates {
3744 if let Some(evidence) = &c.verified_noop {
3745 parts.push(format!(
3746 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
3747 c.label
3748 ));
3749 } else if !c.summary.trim().is_empty() {
3750 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
3751 } else if let Some(why) = &c.failed {
3752 parts.push(format!("candidate {}: {why}", c.label));
3753 }
3754 }
3755 }
3756
3757 if parts.is_empty() {
3758 return None;
3759 }
3760 Some(crate::run::tail(
3765 &parts.join("\n\n"),
3766 DIAGNOSTIC_MAX.saturating_sub(100),
3767 ))
3768}
3769
3770fn label(status: RunStatus) -> &'static str {
3778 status.as_str()
3779}
3780
3781fn merge_mode(mode: &str) -> Result<MergeMode> {
3783 match mode {
3784 "none" => Ok(MergeMode::None),
3785 "local" => Ok(MergeMode::Local),
3786 "pr" => Ok(MergeMode::Pr),
3787 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
3788 }
3789}
3790
3791fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
3796 mutex
3797 .lock()
3798 .unwrap_or_else(std::sync::PoisonError::into_inner)
3799}
3800
3801#[cfg(test)]
3802mod tests {
3803 use super::*;
3804 use crate::queue::{Source, TaskStatus};
3805 use crate::run::{Candidate, CommandOutcome};
3806 use pretty_assertions::assert_eq;
3807
3808 fn task() -> Task {
3809 Task::new(
3810 "add retries".to_owned(),
3811 "add retries".to_owned(),
3812 PathBuf::from("/repo"),
3813 Source::Human,
3814 )
3815 }
3816
3817 fn interrupt_task(id: &str) -> Task {
3820 let mut t = task();
3821 t.id = id.to_owned();
3822 t.interrupt = true;
3823 t
3824 }
3825
3826 fn task_with_id(id: &str) -> Task {
3828 let mut t = task();
3829 t.id = id.to_owned();
3830 t
3831 }
3832
3833 fn urgent_task(id: &str) -> Task {
3835 let mut t = task();
3836 t.id = id.to_owned();
3837 t.urgent = true;
3838 t
3839 }
3840
3841 #[test]
3846 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
3847 assert_eq!(permit_kind(true, false), PermitKind::None);
3848 assert_eq!(permit_kind(true, true), PermitKind::None);
3849 }
3850
3851 #[test]
3856 fn permit_kind_separates_urgent_from_ordinary() {
3857 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
3858 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
3859 }
3860
3861 #[test]
3868 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
3869 let cfg = Config::default();
3870 let repo = Path::new("/any/repo/path");
3871
3872 let reason =
3873 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
3874 assert!(reason.contains("1024"), "{reason}");
3875 assert!(
3876 reason.contains(&cfg.disk.min_free_bytes.to_string()),
3877 "{reason}"
3878 );
3879
3880 assert_eq!(
3881 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
3882 None,
3883 "exactly at the floor is open"
3884 );
3885 assert_eq!(
3886 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
3887 None,
3888 "comfortably above the floor is open"
3889 );
3890 }
3891
3892 #[test]
3893 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
3894 let mut cfg = Config::default();
3895 cfg.disk.min_free_bytes = 0;
3896 let repo = Path::new("/any/repo/path");
3897 assert_eq!(
3898 disk_gate_with(repo, &cfg, |_| Ok(0)),
3899 None,
3900 "a zero floor never measures at all"
3901 );
3902 }
3903
3904 #[test]
3905 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
3906 let cfg = Config::default();
3907 let repo = Path::new("/any/repo/path");
3908 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
3909 .expect("a measurement failure must close the gate, not open it");
3910 assert!(reason.contains("could not measure"), "{reason}");
3911 }
3912
3913 #[test]
3914 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
3915 let ordinary = task();
3916 let next = advance_interrupt(
3917 Interrupt::Idle,
3918 std::slice::from_ref(&ordinary.id),
3919 std::slice::from_ref(&ordinary),
3920 );
3921 assert_eq!(next, Interrupt::Idle);
3922 }
3923
3924 #[test]
3925 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
3926 let marked = interrupt_task("marked");
3929 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
3930 assert_eq!(next, Interrupt::Idle);
3931 }
3932
3933 #[test]
3934 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
3935 let marked = interrupt_task("marked");
3936 let next = advance_interrupt(
3937 Interrupt::Idle,
3938 &["running".to_owned()],
3939 std::slice::from_ref(&marked),
3940 );
3941 assert_eq!(
3942 next,
3943 Interrupt::Parking {
3944 parked: vec!["running".to_owned()],
3945 interrupt_task: "marked".to_owned(),
3946 }
3947 );
3948 }
3949
3950 #[test]
3959 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
3960 let marked = interrupt_task("marked");
3961
3962 let two = advance_interrupt(
3963 Interrupt::Idle,
3964 &["a".to_owned(), "b".to_owned()],
3965 std::slice::from_ref(&marked),
3966 );
3967 assert_eq!(two, Interrupt::Idle);
3968
3969 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
3970 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
3971 }
3972
3973 #[test]
3974 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
3975 let state = Interrupt::Parking {
3976 parked: vec!["running".to_owned()],
3977 interrupt_task: "marked".to_owned(),
3978 };
3979 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
3981 assert_eq!(still_going, state);
3982
3983 let stopped_but_not_yet_dispatched =
3987 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
3988 assert_eq!(stopped_but_not_yet_dispatched, state);
3989
3990 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
3992 assert_eq!(
3993 dispatched,
3994 Interrupt::Running {
3995 parked: vec!["running".to_owned()],
3996 interrupt_task: "marked".to_owned(),
3997 }
3998 );
3999 }
4000
4001 #[test]
4002 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
4003 let state = Interrupt::Running {
4004 parked: vec!["running".to_owned()],
4005 interrupt_task: "marked".to_owned(),
4006 };
4007 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
4008 assert_eq!(still_running, state);
4009
4010 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
4017 assert_eq!(
4018 ended,
4019 Interrupt::Resuming {
4020 parked: vec!["running".to_owned()]
4021 }
4022 );
4023 }
4024
4025 #[test]
4026 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
4027 let state = Interrupt::Resuming {
4028 parked: vec!["running".to_owned()],
4029 };
4030 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
4031 assert_eq!(still_waiting, state);
4032
4033 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
4034 assert_eq!(dispatched, Interrupt::Idle);
4035 }
4036
4037 #[test]
4043 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
4044 {
4045 let state = Interrupt::Parking {
4046 parked: vec!["running".to_owned()],
4047 interrupt_task: "marked".to_owned(),
4048 };
4049 let next = advance_interrupt(state, &[], &[]);
4052 assert_eq!(
4053 next,
4054 Interrupt::Resuming {
4055 parked: vec!["running".to_owned()]
4056 },
4057 "abandoning the interrupt must not abandon the resume it owes"
4058 );
4059 }
4060
4061 #[test]
4064 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
4065 let state = Interrupt::Resuming {
4066 parked: vec!["running".to_owned()],
4067 };
4068 let next = advance_interrupt(state, &[], &[]);
4069 assert_eq!(
4070 next,
4071 Interrupt::Idle,
4072 "nothing is left to wait for; the loop must not stay wedged"
4073 );
4074 }
4075
4076 #[test]
4077 fn disabled_by_config_the_sequence_can_never_leave_idle() {
4078 let marked = interrupt_task("marked");
4079 let next = advance_interrupt_tick(
4080 false,
4081 Interrupt::Idle,
4082 &["running".to_owned()],
4083 std::slice::from_ref(&marked),
4084 );
4085 assert_eq!(
4086 next,
4087 Interrupt::Idle,
4088 "an unmarked, unconfigured daemon must behave exactly as before"
4089 );
4090 }
4091
4092 #[test]
4093 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
4094 let state = Interrupt::Parking {
4095 parked: vec!["running".to_owned()],
4096 interrupt_task: "marked".to_owned(),
4097 };
4098 let candidates = vec![interrupt_task("marked"), task()];
4099 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
4100 assert!(
4101 allowed.is_empty(),
4102 "nothing may dispatch - not even the interrupt task itself - \
4103 until the parked run has actually stopped"
4104 );
4105 }
4106
4107 #[test]
4121 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
4122 for state in [
4123 Interrupt::Parking {
4124 parked: vec!["running".to_owned()],
4125 interrupt_task: "marked".to_owned(),
4126 },
4127 Interrupt::Running {
4128 parked: vec!["running".to_owned()],
4129 interrupt_task: "marked".to_owned(),
4130 },
4131 Interrupt::Resuming {
4132 parked: vec!["running".to_owned()],
4133 },
4134 ] {
4135 let candidates = vec![
4136 interrupt_task("marked"),
4137 urgent_task("hot"),
4138 task_with_id("ordinary"),
4139 ];
4140 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
4141 assert!(
4142 !allowed.iter().any(|t| t.id == "hot"),
4143 "an urgent candidate must wait out the same gate as anything \
4144 else while the run it would run alongside has not actually \
4145 left flight, for state {state:?}: {allowed:?}"
4146 );
4147 }
4148 }
4149
4150 #[test]
4156 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
4157 let state = Interrupt::Resuming {
4158 parked: vec!["hot".to_owned()],
4159 };
4160 let candidates = vec![urgent_task("hot"), task()];
4161 let allowed = interrupt_gate(&state, &[], candidates);
4162 assert_eq!(
4163 allowed.iter().filter(|t| t.id == "hot").count(),
4164 1,
4165 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
4166 );
4167 }
4168
4169 #[test]
4170 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
4171 let state = Interrupt::Parking {
4172 parked: vec!["running".to_owned()],
4173 interrupt_task: "marked".to_owned(),
4174 };
4175 let other = task();
4176 let candidates = vec![interrupt_task("marked"), other.clone()];
4177 let allowed = interrupt_gate(&state, &[], candidates);
4178 assert_eq!(allowed.len(), 1);
4179 assert_eq!(allowed[0].id, "marked");
4180 }
4181
4182 #[test]
4183 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
4184 let state = Interrupt::Running {
4185 parked: vec!["running".to_owned()],
4186 interrupt_task: "marked".to_owned(),
4187 };
4188 let candidates = vec![task(), task()];
4189 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
4190 assert!(allowed.is_empty());
4191 }
4192
4193 #[test]
4200 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
4201 let state = Interrupt::Resuming {
4202 parked: vec!["a".to_owned(), "c".to_owned()],
4203 };
4204 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
4205 let allowed = interrupt_gate(&state, &[], candidates);
4206 assert_eq!(
4207 allowed.len(),
4208 1,
4209 "at most one candidate may be offered while resuming: {allowed:?}"
4210 );
4211 assert_eq!(allowed[0].id, "a");
4212 }
4213
4214 #[test]
4215 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
4216 let state = Interrupt::Resuming {
4217 parked: vec!["a".to_owned()],
4218 };
4219 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
4220 assert!(allowed.is_empty());
4221 }
4222
4223 #[test]
4229 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
4230 let running = task(); let marked = interrupt_task("marked");
4232
4233 let mut state = Interrupt::Idle;
4234 let in_flight = vec![running.id.clone()];
4236 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
4237 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
4238 assert!(gated.is_empty(), "still waiting on `running` to park");
4239
4240 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
4242 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
4243 assert_eq!(
4244 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
4245 vec!["marked"],
4246 "only the interrupt task may be offered to the dispatcher now"
4247 );
4248
4249 state = advance_interrupt_tick(
4251 true,
4252 state,
4253 &["marked".to_owned()],
4254 std::slice::from_ref(&running),
4255 );
4256 let gated = interrupt_gate(
4257 &state,
4258 &["marked".to_owned()],
4259 vec![marked.clone(), running.clone()],
4260 );
4261 assert!(
4262 gated.is_empty(),
4263 "the parked run must not be offered back while the interrupt \
4264 task is still running"
4265 );
4266
4267 let other = task_with_id("other");
4271 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
4272 assert_eq!(
4273 state,
4274 Interrupt::Resuming {
4275 parked: vec![running.id.clone()]
4276 }
4277 );
4278 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
4279 assert_eq!(
4280 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
4281 vec![running.id.as_str()],
4282 "exactly the parked run resumes - not the unrelated task, even \
4283 though it was offered first"
4284 );
4285
4286 state = advance_interrupt_tick(
4290 true,
4291 state,
4292 std::slice::from_ref(&running.id),
4293 std::slice::from_ref(&other),
4294 );
4295 assert_eq!(state, Interrupt::Idle);
4296 let gated = interrupt_gate(
4297 &state,
4298 std::slice::from_ref(&running.id),
4299 vec![other.clone()],
4300 );
4301 assert_eq!(
4302 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
4303 vec![other.id.as_str()],
4304 "ordinary dispatch is unrestricted again"
4305 );
4306 }
4307
4308 #[test]
4309 fn every_run_status_settles_the_task_it_came_from() {
4310 let table = [
4312 (RunStatus::Merged, TaskStatus::Done, 1),
4313 (RunStatus::Ready, TaskStatus::Done, 1),
4314 (RunStatus::Stalled, TaskStatus::Failed, 0),
4315 (RunStatus::Blocked, TaskStatus::Failed, 1),
4316 (RunStatus::Failed, TaskStatus::Failed, 1),
4317 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
4318 (RunStatus::Prep, TaskStatus::Failed, 1),
4319 (RunStatus::Implementing, TaskStatus::Failed, 1),
4320 (RunStatus::Judging, TaskStatus::Failed, 1),
4321 (RunStatus::Deliberating, TaskStatus::Failed, 1),
4322 (RunStatus::Voting, TaskStatus::Failed, 1),
4323 (RunStatus::Reviewing, TaskStatus::Failed, 1),
4324 (RunStatus::Gating, TaskStatus::Failed, 1),
4325 ];
4326 for (run, want, attempts) in table {
4327 let mut t = task();
4328 t.start("20260902-000000-aaaa".to_owned());
4329 settle(
4330 &mut t,
4331 Verdict {
4332 status: run,
4333 left_pr: false,
4334 parked: false,
4335 quota_hit: matches!(run, RunStatus::Stalled),
4336 no_viable_candidates: false,
4337 },
4338 "why",
4339 2,
4340 );
4341 assert_eq!(t.status, want, "task status after {}", label(run));
4342 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
4343 }
4344 }
4345
4346 #[test]
4347 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
4348 let mut stalled = task();
4349 stalled.start("20260902-000000-aaaa".to_owned());
4350 settle(
4351 &mut stalled,
4352 Verdict {
4353 status: RunStatus::Stalled,
4354 left_pr: false,
4355 parked: false,
4356 quota_hit: true,
4357 no_viable_candidates: false,
4358 },
4359 "quota",
4360 1,
4361 );
4362 assert_eq!(stalled.attempts, 0);
4363 assert!(
4364 stalled.status.runnable(),
4365 "a machine problem must leave the task in line"
4366 );
4367
4368 let mut blocked = task();
4369 blocked.start("20260902-000000-aaaa".to_owned());
4370 settle(
4371 &mut blocked,
4372 Verdict {
4373 status: RunStatus::Blocked,
4374 left_pr: false,
4375 parked: false,
4376 quota_hit: false,
4377 no_viable_candidates: false,
4378 },
4379 "findings open",
4380 1,
4381 );
4382 assert_eq!(blocked.attempts, 1);
4383 assert_eq!(
4384 blocked.status,
4385 TaskStatus::Held,
4386 "the last attempt hands the task to a human"
4387 );
4388 }
4389
4390 #[test]
4391 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
4392 let mut delivered = task();
4395 delivered.start("20260903-080619-01c2".to_owned());
4396 settle(
4397 &mut delivered,
4398 Verdict {
4399 status: RunStatus::Blocked,
4400 left_pr: true,
4401 parked: false,
4402 quota_hit: false,
4403 no_viable_candidates: false,
4404 },
4405 "no check status",
4406 4,
4407 );
4408 assert_eq!(
4409 delivered.status,
4410 TaskStatus::Held,
4411 "a pull request waiting on CI or a person is not a retryable failure"
4412 );
4413 assert!(
4414 !delivered.status.runnable(),
4415 "the loop must not pick this task up again"
4416 );
4417 assert_eq!(
4418 delivered.last_error.as_deref(),
4419 Some("no check status"),
4420 "the operator needs to be told what the gate was waiting for"
4421 );
4422
4423 let mut empty_handed = task();
4426 empty_handed.start("20260903-080619-01c2".to_owned());
4427 settle(
4428 &mut empty_handed,
4429 Verdict {
4430 status: RunStatus::Blocked,
4431 left_pr: false,
4432 parked: false,
4433 quota_hit: false,
4434 no_viable_candidates: false,
4435 },
4436 "findings open",
4437 4,
4438 );
4439 assert_eq!(empty_handed.status, TaskStatus::Failed);
4440 assert!(empty_handed.status.runnable());
4441 }
4442
4443 #[test]
4444 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
4445 let mut noop = task();
4451 noop.start("20260912-131304-391f".to_owned());
4452 settle(
4453 &mut noop,
4454 Verdict {
4455 status: RunStatus::VerifiedNoop,
4456 left_pr: false,
4457 parked: false,
4458 quota_hit: false,
4459 no_viable_candidates: true,
4460 },
4461 "candidate A: already fixed by b32cfc4, on main",
4462 4,
4463 );
4464 assert_eq!(
4465 noop.status,
4466 TaskStatus::Held,
4467 "an unverified claim is a request for a human, not a failure"
4468 );
4469 assert!(
4470 !noop.status.runnable(),
4471 "the loop must not requeue this on the same unverified claim"
4472 );
4473 assert_eq!(noop.attempts, 1);
4478 }
4479
4480 #[test]
4481 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
4482 let mut parked = task();
4487 parked.start("20260903-183634-2d98".to_owned());
4488 settle(
4489 &mut parked,
4490 Verdict {
4491 status: RunStatus::Implementing,
4492 left_pr: false,
4493 quota_hit: false,
4494 parked: true,
4495 no_viable_candidates: false,
4496 },
4497 "parked after `implementing`",
4498 2,
4499 );
4500 assert_eq!(parked.attempts, 0, "a park is refunded");
4501 assert!(
4502 parked.status.runnable(),
4503 "and the task stays in line so the next loop resumes its run"
4504 );
4505 assert_eq!(
4506 parked.last_error.as_deref(),
4507 Some("parked after `implementing`"),
4508 "the card says where it stopped"
4509 );
4510
4511 let mut broken = task();
4515 broken.start("20260903-183634-2d98".to_owned());
4516 settle(
4517 &mut broken,
4518 Verdict {
4519 status: RunStatus::Implementing,
4520 left_pr: false,
4521 quota_hit: false,
4522 parked: false,
4523 no_viable_candidates: false,
4524 },
4525 "returned mid-flight",
4526 2,
4527 );
4528 assert_eq!(broken.attempts, 1);
4529 }
4530
4531 #[test]
4532 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
4533 let mut flaky = task();
4538 flaky.start("20260903-123023-e633".to_owned());
4539 settle(
4540 &mut flaky,
4541 Verdict {
4542 status: RunStatus::Stalled,
4543 left_pr: false,
4544 parked: false,
4545 quota_hit: false,
4546 no_viable_candidates: false,
4547 },
4548 "verdict rests on 1 of 3 judges",
4549 2,
4550 );
4551 assert_eq!(
4552 flaky.attempts, 1,
4553 "flakiness spends an attempt, so `max_attempts` still bounds it"
4554 );
4555 assert!(flaky.status.runnable(), "and it is still worth retrying");
4556
4557 let mut limited = task();
4559 limited.start("20260903-123023-e633".to_owned());
4560 settle(
4561 &mut limited,
4562 Verdict {
4563 status: RunStatus::Stalled,
4564 left_pr: false,
4565 parked: false,
4566 quota_hit: true,
4567 no_viable_candidates: false,
4568 },
4569 "judge-2, judge-3 out of quota",
4570 2,
4571 );
4572 assert_eq!(limited.attempts, 0, "a quota window is refunded");
4573 assert!(limited.status.runnable());
4574
4575 let mut worn = task();
4578 for _ in 0..2 {
4579 worn.release();
4580 }
4581 worn.start("20260903-123023-e633".to_owned());
4582 worn.attempts = 2;
4583 settle(
4584 &mut worn,
4585 Verdict {
4586 status: RunStatus::Stalled,
4587 left_pr: false,
4588 parked: false,
4589 quota_hit: false,
4590 no_viable_candidates: false,
4591 },
4592 "no quorum again",
4593 2,
4594 );
4595 assert_eq!(worn.status, TaskStatus::Held);
4596 assert!(!worn.status.runnable());
4597 }
4598
4599 #[test]
4600 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
4601 let mut wiped_out = task();
4608 wiped_out.start("20260907-025000-a1b2".to_owned());
4609 settle(
4610 &mut wiped_out,
4611 Verdict {
4612 status: RunStatus::Failed,
4613 left_pr: false,
4614 parked: false,
4615 quota_hit: true,
4616 no_viable_candidates: true,
4617 },
4618 "no candidate produced a change; nothing to judge",
4619 2,
4620 );
4621 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
4622 assert!(
4623 wiped_out.status.runnable(),
4624 "a machine problem must leave the task in line"
4625 );
4626
4627 let mut partial_progress = task();
4633 partial_progress.start("20260907-025500-c3d4".to_owned());
4634 settle(
4635 &mut partial_progress,
4636 Verdict {
4637 status: RunStatus::Failed,
4638 left_pr: false,
4639 parked: false,
4640 quota_hit: true,
4641 no_viable_candidates: false,
4642 },
4643 "gate failed on the winning candidate",
4644 2,
4645 );
4646 assert_eq!(
4647 partial_progress.attempts, 1,
4648 "a candidate that actually produced a change spends the attempt \
4649 even though some other seat hit its quota"
4650 );
4651 assert!(partial_progress.status.runnable());
4652 }
4653
4654 #[test]
4655 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
4656 let mut t = task();
4662 t.start("20260907-025000-a1b2".to_owned());
4663 let mut state = run_state(RunStatus::Failed);
4664 state.quota.push(QuotaLoss {
4665 seat: "cand-a".to_owned(),
4666 node: "implement".to_owned(),
4667 at: Timestamp::now(),
4668 reset: None,
4669 });
4670 assert!(
4671 state.viable().is_empty(),
4672 "no candidate was added, so nothing is viable"
4673 );
4674 reclaim(&mut t, Some(state), 2, "en");
4675 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
4676 assert!(t.status.runnable());
4677 }
4678
4679 #[test]
4680 fn a_held_task_is_never_offered_to_the_loop() {
4681 let dir = tempfile::tempdir().unwrap();
4682 let queue = Queue::at(dir.path().to_path_buf());
4683 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
4684 let mut t = task();
4685 t.id = format!("2026090{n}-000000-000{n}");
4686 t.priority = priority;
4687 queue.put(&mut t).unwrap();
4688 }
4689 let mut held = task();
4690 held.id = "20260909-000000-9999".to_owned();
4691 held.priority = 99;
4692 held.hold_machine(None);
4693 queue.put(&mut held).unwrap();
4694
4695 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
4696 assert_eq!(order.len(), 3);
4697 assert!(!order.contains(&held.id));
4698 assert_eq!(
4699 order.first().cloned(),
4700 queue.next_runnable().map(|t| t.id),
4701 "the loop's first candidate is exactly what the queue offers"
4702 );
4703 assert_eq!(
4704 order,
4705 vec![
4706 "20260902-000000-0002".to_owned(),
4707 "20260903-000000-0003".to_owned(),
4708 "20260901-000000-0001".to_owned(),
4709 ],
4710 "priority first, then oldest, so nothing starves"
4711 );
4712 }
4713
4714 #[test]
4715 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
4716 let dir = tempfile::tempdir().unwrap();
4717 let queue = Queue::at(dir.path().to_path_buf());
4718 let mut old = task();
4719 old.id = "20260101-000000-old0".to_owned();
4720 queue.put(&mut old).unwrap();
4721 let mut fresh = task();
4722 fresh.id = "20260101-000000-new0".to_owned();
4723 queue.put(&mut fresh).unwrap();
4724
4725 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
4729 std::thread::sleep(Duration::from_millis(60));
4730 let live = queue.claim(&fresh.id).unwrap();
4731
4732 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
4733 assert_eq!(swept, vec![old.id.clone()]);
4734 assert!(
4735 queue.claim(&old.id).is_ok(),
4736 "an unparseable lock older than the threshold is swept"
4737 );
4738 assert!(
4739 queue.claim(&fresh.id).is_err(),
4740 "a live pid protects its lock regardless of age"
4741 );
4742 drop(live);
4743 }
4744
4745 #[test]
4746 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
4747 let dir = tempfile::tempdir().unwrap();
4757 let queue = Queue::at(dir.path().to_path_buf());
4758 let mut t = task();
4759 t.id = "20260101-000000-live".to_owned();
4760 queue.put(&mut t).unwrap();
4761
4762 let claim = queue.claim(&t.id).unwrap();
4763 std::thread::sleep(Duration::from_millis(60));
4764
4765 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
4766 assert!(
4767 swept.is_empty(),
4768 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
4769 );
4770 assert!(
4771 queue.claim(&t.id).is_err(),
4772 "the lock still protects its task"
4773 );
4774 drop(claim);
4775 }
4776
4777 fn injected_dead_pid() -> u32 {
4780 std::process::id().checked_add(1).unwrap_or(1)
4781 }
4782
4783 #[test]
4784 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
4785 let dir = tempfile::tempdir().unwrap();
4786 let queue = Queue::at(dir.path().to_path_buf());
4787 let mut t = task();
4788 t.id = "20260101-000000-dead".to_owned();
4789 queue.put(&mut t).unwrap();
4790 let dead_pid = injected_dead_pid();
4791
4792 std::fs::write(
4797 dir.path().join(format!("{}.lock", t.id)),
4798 dead_pid.to_string(),
4799 )
4800 .unwrap();
4801
4802 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
4803 pid != dead_pid
4804 });
4805 assert_eq!(
4806 swept,
4807 vec![t.id.clone()],
4808 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
4809 );
4810 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
4811 }
4812
4813 #[test]
4814 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
4815 let dir = tempfile::tempdir().unwrap();
4816 let queue = Queue::at(dir.path().to_path_buf());
4817 let mut t = task();
4818 t.id = "20260101-000000-late".to_owned();
4819 queue.put(&mut t).unwrap();
4820 let dead_pid = injected_dead_pid();
4821
4822 assert!(
4825 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
4826 "nothing has claimed the task yet"
4827 );
4828
4829 std::fs::write(
4832 dir.path().join(format!("{}.lock", t.id)),
4833 dead_pid.to_string(),
4834 )
4835 .unwrap();
4836
4837 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
4841 pid != dead_pid
4842 });
4843 assert_eq!(swept, vec![t.id.clone()]);
4844 }
4845
4846 #[test]
4847 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
4848 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
4853 let dir = tempfile::tempdir().unwrap();
4854 let queue = Queue::at(dir.path().to_path_buf());
4855 let mut t = task();
4856 t.id = "20260101-000000-crsh".to_owned();
4857 t.status = TaskStatus::Running;
4858 t.attempts = 1;
4859 t.runs.push("20260904-000000-4043".to_owned());
4863 queue.put(&mut t).unwrap();
4864 let dead_pid = injected_dead_pid();
4865
4866 std::fs::write(
4869 dir.path().join(format!("{}.lock", t.id)),
4870 dead_pid.to_string(),
4871 )
4872 .unwrap();
4873
4874 assert!(reclaim_orphaned_running(&queue, 2).is_empty());
4880 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
4881
4882 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
4883 pid != dead_pid
4884 });
4885 assert_eq!(swept, vec![t.id.clone()]);
4886
4887 let reclaimed = reclaim_orphaned_running(&queue, 2);
4888 assert_eq!(reclaimed, vec![t.id.clone()]);
4889 let after = queue.get(&t.id).unwrap();
4890 assert_eq!(
4891 after.status,
4892 TaskStatus::Held,
4893 "no run.json to recover from, so a human is asked"
4894 );
4895 assert_eq!(
4896 after.runs,
4897 vec!["20260904-000000-4043".to_owned()],
4898 "the crashed run's id is kept as evidence, not discarded"
4899 );
4900 }
4901
4902 #[test]
4903 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
4904 let dir = tempfile::tempdir().unwrap();
4905 let queue = Queue::at(dir.path().to_path_buf());
4906 let mut t = task();
4907 t.id = "20260101-000000-unknown".to_owned();
4908 queue.put(&mut t).unwrap();
4909 let dead_pid = injected_dead_pid();
4910 std::fs::write(
4911 dir.path().join(format!("{}.lock", t.id)),
4912 dead_pid.to_string(),
4913 )
4914 .unwrap();
4915
4916 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
4917 assert!(swept.is_empty(), "an unknown pid must keep its lock");
4918 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
4919 }
4920
4921 fn run_state_in(status: RunStatus, language: &str) -> RunState {
4922 let mut s = run_state(status);
4923 s.config.graph.language = language.to_owned();
4924 s
4925 }
4926
4927 fn unstarted_verdict(status: RunStatus) -> Verdict {
4928 Verdict {
4929 status,
4930 left_pr: false,
4931 quota_hit: false,
4932 parked: false,
4933 no_viable_candidates: false,
4934 }
4935 }
4936
4937 #[test]
4938 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
4939 let mut t = task();
4940 t.attempts = 1;
4941 settle_in(
4942 &mut t,
4943 unstarted_verdict(RunStatus::AlreadyInBase),
4944 "already in main",
4945 1,
4946 phrases("en"),
4947 );
4948 assert_eq!(t.status, TaskStatus::Done);
4949 assert_eq!(t.attempts, 0);
4950 }
4951
4952 #[test]
4953 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
4954 let reason = |language: &str| {
4955 let mut t = task();
4956 settle_in(
4957 &mut t,
4958 unstarted_verdict(RunStatus::Judging),
4959 "boom",
4960 1,
4961 phrases(language),
4962 );
4963 t.last_error.or(t.hold_reason).unwrap_or_default()
4964 };
4965 assert!(
4966 reason("en").starts_with("the graph stopped at `"),
4967 "{}",
4968 reason("en")
4969 );
4970 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
4971 assert!(reason("日本語").contains("boom"));
4972 assert_eq!(reason("fr"), reason("en"));
4973 }
4974
4975 #[test]
4976 fn describe_follows_the_run_language_and_keeps_the_run_id() {
4977 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
4978 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
4979 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
4980 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
4981 assert!(ja.contains("[run "), "{ja}");
4982 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
4983 assert!(ended.starts_with("run 終了: "), "{ended}");
4984 let mut de = run_state_in(RunStatus::Failed, "de");
4985 let mut en = run_state_in(RunStatus::Failed, "en");
4986 de.id = "same".to_owned();
4987 en.id = "same".to_owned();
4988 assert_eq!(describe(&de), describe(&en));
4989 }
4990
4991 #[test]
4992 fn refusals_and_recovery_prose_follow_the_language() {
4993 let t = held_task_with("r1");
4994 let q = action_question(
4995 "r1",
4996 ask::ChoiceAction::Resume {
4997 run: "r1".to_owned(),
4998 },
4999 );
5000 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
5001 ActionDecision::Refuse(s) => s,
5002 other => panic!("{other:?}"),
5003 };
5004 assert!(refuse(phrases("en")).contains("could not be read: gone"));
5005 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
5006 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
5007
5008 let mut held = task();
5009 reclaim(&mut held, None, 2, "ja");
5010 assert!(held.hold_reason.unwrap().contains("保留にしました"));
5011 let mut held = task();
5012 reclaim(&mut held, None, 2, "xx");
5013 assert!(held.hold_reason.unwrap().contains("held for a human"));
5014 }
5015
5016 fn run_state(status: RunStatus) -> RunState {
5017 let mut state = RunState::new(
5018 PathBuf::from("/repo"),
5019 "main".to_owned(),
5020 "abc1234def".to_owned(),
5021 "add retries".to_owned(),
5022 Config::default(),
5023 );
5024 state.status = status;
5025 state
5026 }
5027
5028 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
5029 Candidate {
5030 index: 0,
5031 label,
5032 agent: "claude".to_owned(),
5033 branch: format!("magi/x/{label}"),
5034 worktree: PathBuf::from("/repo"),
5035 summary: summary.to_owned(),
5036 stat: String::new(),
5037 files: 0,
5038 commits: usize::from(!empty),
5039 empty,
5040 failed: failed.map(str::to_owned),
5041 verified_noop: None,
5042 duration_ms: 0,
5043 folded: false,
5044 }
5045 }
5046
5047 #[test]
5048 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
5049 let mut state = run_state(RunStatus::Blocked);
5050 state.gate = vec![
5051 CommandOutcome {
5052 command: "cargo make check".to_owned(),
5053 code: Some(0),
5054 output_tail: "ok".to_owned(),
5055 duration_ms: 0,
5056 resource_blocked: false,
5057 },
5058 CommandOutcome {
5059 command: "cargo test".to_owned(),
5060 code: Some(101),
5061 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
5062 duration_ms: 0,
5063 resource_blocked: false,
5064 },
5065 ];
5066 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
5067 assert!(d.contains("cargo test"), "{d}");
5068 assert!(
5069 !d.contains("cargo make check"),
5070 "a passing check is not a diagnostic: {d}"
5071 );
5072 assert!(d.contains("assertion failed"), "{d}");
5073 }
5074
5075 #[test]
5076 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
5077 let mut state = run_state(RunStatus::Blocked);
5078 state.event(
5079 "land",
5080 "stopped: the fixer produced no commit while 2 check(s) were failing \
5081 (build, lint); stopping instead of looping on an unchanged tree",
5082 );
5083 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
5084 assert!(d.contains("build"), "{d}");
5085 assert!(d.contains("lint"), "{d}");
5086 assert!(d.contains("fixer produced no commit"), "{d}");
5087 }
5088
5089 #[test]
5090 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
5091 let state = run_state(RunStatus::VerifiedNoop);
5096 let d = describe(&state);
5097 assert!(
5098 d.contains("agent-verified no-op"),
5099 "expected the display label, not the wire spelling: {d}"
5100 );
5101 assert!(!d.contains("verified_noop"), "{d}");
5102 }
5103
5104 #[test]
5105 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
5106 let mut state = run_state(RunStatus::Failed);
5112 state.candidates = vec![candidate(
5113 'A',
5114 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
5115 true,
5116 None,
5117 )];
5118 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
5119 assert!(d.contains("candidate A"), "{d}");
5120 assert!(d.contains("tagged v1.2.3"), "{d}");
5121 }
5122
5123 #[test]
5124 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
5125 let mut state = run_state(RunStatus::Failed);
5126 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
5127 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
5128 assert!(d.contains("candidate A"), "{d}");
5129 assert!(d.contains("agent timed out"), "{d}");
5130 }
5131
5132 #[test]
5133 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
5134 let mut state = run_state(RunStatus::Failed);
5137 state.candidates = vec![candidate('A', "did the work", false, None)];
5138 assert!(diagnostic(&state).is_none());
5139 }
5140
5141 #[test]
5142 fn diagnostic_is_bounded_however_much_a_run_printed() {
5143 let mut state = run_state(RunStatus::Blocked);
5144 state.gate = vec![
5145 CommandOutcome {
5146 command: "cargo test".to_owned(),
5147 code: Some(101),
5148 output_tail: "x".repeat(50_000),
5149 duration_ms: 0,
5150 resource_blocked: false,
5151 },
5152 CommandOutcome {
5153 command: "cargo clippy".to_owned(),
5154 code: Some(1),
5155 output_tail: "y".repeat(50_000),
5156 duration_ms: 0,
5157 resource_blocked: false,
5158 },
5159 ];
5160 state.candidates = vec![
5161 candidate('A', &"z".repeat(50_000), true, None),
5162 candidate('B', &"w".repeat(50_000), true, None),
5163 ];
5164 let d = diagnostic(&state).expect("plenty here to diagnose");
5165 assert!(
5166 d.len() <= DIAGNOSTIC_MAX,
5167 "diagnostic grew to {} bytes, unbounded",
5168 d.len()
5169 );
5170 }
5171
5172 #[test]
5173 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
5174 let mut state = run_state(RunStatus::Blocked);
5175 state.gate = vec![CommandOutcome {
5176 command: "cargo test".to_owned(),
5177 code: Some(101),
5178 output_tail: "assertion failed".to_owned(),
5179 duration_ms: 0,
5180 resource_blocked: false,
5181 }];
5182 let verdict = Verdict {
5183 status: RunStatus::Blocked,
5184 left_pr: false,
5185 quota_hit: false,
5186 parked: false,
5187 no_viable_candidates: false,
5188 };
5189
5190 let mut t = task();
5193 t.start("run-1".to_owned());
5194 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
5195 assert_eq!(t.status, TaskStatus::Failed);
5196 assert!(t.diagnostic.is_none());
5197
5198 t.start("run-2".to_owned());
5201 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
5202 assert_eq!(t.status, TaskStatus::Held);
5203 let d = t.diagnostic.expect("a held task must carry its diagnostic");
5204 assert!(d.contains("cargo test"), "{d}");
5205 }
5206
5207 #[test]
5208 fn a_held_task_names_the_open_question_it_is_waiting_on() {
5209 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5214 let home = crate::run::home();
5215 let state = run_state(RunStatus::VerifiedNoop);
5216 let mut q = ask::Question::new(
5217 state.id.clone(),
5218 "implement".to_owned(),
5219 "impl-A".to_owned(),
5220 "is this really a no-op?".to_owned(),
5221 String::new(),
5222 Vec::new(),
5223 );
5224 Questions::at(home.join("questions")).put(&mut q).unwrap();
5225
5226 let verdict = Verdict {
5227 status: RunStatus::VerifiedNoop,
5228 left_pr: false,
5229 quota_hit: false,
5230 parked: false,
5231 no_viable_candidates: false,
5232 };
5233 let mut t = task();
5234 t.start(state.id.clone());
5235 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
5236
5237 assert_eq!(t.status, TaskStatus::Held);
5238 let reason = t.hold_reason.expect("a held task must record why");
5239 assert!(
5240 reason.starts_with("run ended agent-verified no-op"),
5241 "the original settle reason must survive unchanged: {reason}"
5242 );
5243 assert!(
5244 reason.contains(q.short()),
5245 "the open question's id must be named so the notice is actionable: {reason}"
5246 );
5247 }
5248
5249 #[test]
5250 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
5251 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5252 let state = run_state(RunStatus::VerifiedNoop);
5253
5254 let verdict = Verdict {
5255 status: RunStatus::VerifiedNoop,
5256 left_pr: false,
5257 quota_hit: false,
5258 parked: false,
5259 no_viable_candidates: false,
5260 };
5261 let mut t = task();
5262 t.start(state.id.clone());
5263 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
5264
5265 assert_eq!(t.status, TaskStatus::Held);
5266 assert_eq!(
5267 t.hold_reason.as_deref(),
5268 Some("run ended agent-verified no-op"),
5269 "nothing to append when the question was already answered or never asked"
5270 );
5271 }
5272
5273 #[test]
5274 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
5275 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5276 let mut first = run_state(RunStatus::Blocked);
5277 first.id = "20260101-000000-sup1".to_owned();
5278 first.save().unwrap();
5279 let mut second = run_state(RunStatus::Merged);
5280 second.id = "20260101-000000-sup2".to_owned();
5281 second.save().unwrap();
5282
5283 let mut t = task();
5284 t.runs = vec![first.id.clone(), second.id.clone()];
5285 t.status = TaskStatus::Done;
5286
5287 supersede_prior_runs(&t, &crate::run::home());
5288
5289 assert_eq!(
5290 RunState::load(&first.id).unwrap().status,
5291 RunStatus::Superseded,
5292 "the first attempt's Blocked no longer needs anyone's attention"
5293 );
5294 assert_eq!(
5295 RunState::load(&second.id).unwrap().status,
5296 RunStatus::Merged,
5297 "the run that actually succeeded is left exactly as it was"
5298 );
5299 }
5300
5301 #[test]
5302 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
5303 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5309 let mut first = run_state(RunStatus::Blocked);
5310 first.id = "20260101-000000-sup9".to_owned();
5311 first.driver_pid = Some(std::process::id());
5314 first.driver_started_at = Some(
5315 crate::proc::process_started_at(std::process::id())
5316 .expect("this test process's own start time must be queryable"),
5317 );
5318 first.save().unwrap();
5319 let mut second = run_state(RunStatus::Merged);
5320 second.id = "20260101-000000-supa".to_owned();
5321 second.save().unwrap();
5322
5323 let mut t = task();
5324 t.runs = vec![first.id.clone(), second.id.clone()];
5325 t.status = TaskStatus::Done;
5326
5327 supersede_prior_runs(&t, &crate::run::home());
5328
5329 assert_eq!(
5330 RunState::load(&first.id).unwrap().status,
5331 RunStatus::Blocked,
5332 "a live driver_pid means something is still actually working this run, \
5333 even though no daemon claims it - rewriting under it would just be \
5334 undone the next time that process saves"
5335 );
5336 }
5337
5338 #[test]
5339 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
5340 let dir = tempfile::tempdir().unwrap();
5346 let home = dir.path().to_path_buf();
5347 let queue = Queue::at(dir.path().join("queue"));
5348
5349 let mut first = run_state(RunStatus::Blocked);
5350 first.id = "20260101-000000-supd".to_owned();
5351 first.driver_pid = Some(std::process::id());
5352 first.driver_started_at = Some(
5353 crate::proc::process_started_at(std::process::id())
5354 .expect("this test process's own start time must be queryable"),
5355 );
5356 first.save_under(&home).unwrap();
5357 let mut second = run_state(RunStatus::Merged);
5358 second.id = "20260101-000000-supe".to_owned();
5359 second.save_under(&home).unwrap();
5360
5361 let mut t = task();
5362 t.runs = vec![first.id.clone(), second.id.clone()];
5363 t.status = TaskStatus::Done;
5364 queue.put(&mut t).unwrap();
5365
5366 resweep_superseded_attempts(&queue, &home);
5367 assert_eq!(
5368 RunState::load_under(&first.id, &home).unwrap().status,
5369 RunStatus::Blocked,
5370 "still live on the first pass, so still untouched"
5371 );
5372
5373 let mut stale = RunState::load_under(&first.id, &home).unwrap();
5379 stale.driver_started_at = Some("not-this-processes-real-start-time".to_owned());
5380 stale.save_under(&home).unwrap();
5381
5382 resweep_superseded_attempts(&queue, &home);
5383 assert_eq!(
5384 RunState::load_under(&first.id, &home).unwrap().status,
5385 RunStatus::Superseded,
5386 "the second pass catches up what the first one correctly skipped"
5387 );
5388 }
5389
5390 #[test]
5391 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
5392 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5393 let mut first = run_state(RunStatus::Blocked);
5394 first.id = "20260101-000000-sup3".to_owned();
5395 first.save().unwrap();
5396 let mut second = run_state(RunStatus::Blocked);
5397 second.id = "20260101-000000-sup4".to_owned();
5398 second.save().unwrap();
5399
5400 let mut t = task();
5401 t.runs = vec![first.id.clone(), second.id.clone()];
5402 t.status = TaskStatus::Failed;
5406
5407 supersede_prior_runs(&t, &crate::run::home());
5408
5409 assert_eq!(
5410 RunState::load(&first.id).unwrap().status,
5411 RunStatus::Blocked
5412 );
5413 assert_eq!(
5414 RunState::load(&second.id).unwrap().status,
5415 RunStatus::Blocked
5416 );
5417 }
5418
5419 #[test]
5420 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
5421 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5425 let mut first = run_state(RunStatus::Blocked);
5426 first.id = "20260101-000000-sup5".to_owned();
5427 first.save().unwrap();
5428
5429 let mut t = task();
5430 t.runs = vec![first.id.clone()];
5431 t.status = TaskStatus::Done;
5432
5433 supersede_prior_runs(&t, &crate::run::home());
5434
5435 assert_eq!(
5436 RunState::load(&first.id).unwrap().status,
5437 RunStatus::Blocked,
5438 "a single-attempt task has no earlier run to supersede"
5439 );
5440 }
5441
5442 #[test]
5443 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
5444 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5451 let mut first = run_state(RunStatus::Blocked);
5452 first.id = "20260101-000000-supb".to_owned();
5453 first.save().unwrap();
5454 let mut second = run_state(RunStatus::Failed);
5455 second.id = "20260101-000000-supc".to_owned();
5456 second.save().unwrap();
5457
5458 let mut t = task();
5459 t.runs = vec![first.id.clone(), second.id.clone()];
5460 t.status = TaskStatus::Done;
5461
5462 supersede_prior_runs(&t, &crate::run::home());
5463
5464 assert_eq!(
5465 RunState::load(&first.id).unwrap().status,
5466 RunStatus::Blocked,
5467 "the task's last attempt never landed, so there is nothing here \
5468 actually superseding it"
5469 );
5470 }
5471
5472 #[test]
5473 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
5474 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5478 let mut failed = run_state(RunStatus::Failed);
5479 failed.id = "20260101-000000-sup6".to_owned();
5480 failed.save().unwrap();
5481 let mut noop = run_state(RunStatus::VerifiedNoop);
5482 noop.id = "20260101-000000-sup7".to_owned();
5483 noop.save().unwrap();
5484 let mut winner = run_state(RunStatus::Ready);
5485 winner.id = "20260101-000000-sup8".to_owned();
5486 winner.save().unwrap();
5487
5488 let mut t = task();
5489 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
5490 t.status = TaskStatus::Done;
5491
5492 supersede_prior_runs(&t, &crate::run::home());
5493
5494 assert_eq!(
5495 RunState::load(&failed.id).unwrap().status,
5496 RunStatus::Failed
5497 );
5498 assert_eq!(
5499 RunState::load(&noop.id).unwrap().status,
5500 RunStatus::VerifiedNoop
5501 );
5502 }
5503
5504 fn approval_question(run: &str) -> ask::Question {
5505 ask::Question::new(
5506 run.to_owned(),
5507 land::APPROVAL_NODE.to_owned(),
5508 "land".to_owned(),
5509 "merge?".to_owned(),
5510 String::new(),
5511 vec!["merge".to_owned(), "hold".to_owned()],
5512 )
5513 }
5514
5515 #[test]
5516 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
5517 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5518 let mut state = run_state(RunStatus::Landing);
5519 state.id = "20260101-000000-fre1".to_owned();
5520 state.parked = true;
5521 state.save().unwrap();
5522 ask::Questions::open()
5523 .put(&mut approval_question(&state.id))
5524 .unwrap();
5525
5526 let mut t = task();
5527 t.runs.push(state.id.clone());
5528 assert_eq!(
5529 land_resume_state(&t),
5530 LandResume::StillWaiting,
5531 "nobody has answered and the timeout has not passed"
5532 );
5533 }
5534
5535 #[test]
5536 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
5537 crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
5542 let mut state = run_state(RunStatus::Landing);
5543 state.id = "20260101-000000-exp1".to_owned();
5544 state.parked = true;
5545 state.config.graph.answer_timeout = 60;
5546 state.save().unwrap();
5547
5548 let store = ask::Questions::open();
5549 let mut q = approval_question(&state.id);
5550 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
5551 store.put(&mut q).unwrap();
5552
5553 let mut t = task();
5554 t.runs.push(state.id.clone());
5555 assert_eq!(
5556 land_resume_state(&t),
5557 LandResume::Ready,
5558 "an expired question must not be waited on forever"
5559 );
5560
5561 let after = store.get(&q.id).unwrap();
5562 assert!(
5563 !after.status.open(),
5564 "the question is abandoned, not silently ignored"
5565 );
5566 assert!(
5567 after.resolution().is_none(),
5568 "an abandoned question is not read as a decision"
5569 );
5570 }
5571
5572 #[test]
5573 fn reclaim_settles_a_running_task_against_its_last_run() {
5574 let mut t = task();
5575 t.start("20260904-000000-4043".to_owned());
5576 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
5577 assert_eq!(
5578 t.status,
5579 TaskStatus::Done,
5580 "a run that actually finished must not stay `running` forever"
5581 );
5582 }
5583
5584 #[test]
5585 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
5586 let mut t = task();
5590 t.start("20260904-000000-4043".to_owned());
5591 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
5592 assert_eq!(t.status, TaskStatus::Failed);
5593 assert!(t.status.runnable());
5594 }
5595
5596 #[test]
5597 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
5598 let mut t = task();
5599 t.start("20260904-000000-4043".to_owned());
5600 reclaim(&mut t, None, 2, "en");
5601 assert_eq!(t.status, TaskStatus::Held);
5602 assert!(
5603 t.last_error
5604 .as_deref()
5605 .is_some_and(|e| e.contains("running")),
5606 "the operator needs to know why this task was held"
5607 );
5608 }
5609
5610 #[test]
5611 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
5612 let dir = tempfile::tempdir().unwrap();
5613 let queue = Queue::at(dir.path().to_path_buf());
5614
5615 let mut orphaned = task();
5617 orphaned.id = "20260904-000000-orph".to_owned();
5618 orphaned.status = TaskStatus::Running;
5619 orphaned.attempts = 1;
5620 queue.put(&mut orphaned).unwrap();
5621
5622 let mut alive = task();
5623 alive.id = "20260904-000000-live".to_owned();
5624 alive.status = TaskStatus::Running;
5625 alive.attempts = 1;
5626 queue.put(&mut alive).unwrap();
5627 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
5628
5629 let mut queued = task();
5630 queued.id = "20260904-000000-wait".to_owned();
5631 queue.put(&mut queued).unwrap();
5632
5633 let reclaimed = reclaim_orphaned_running(&queue, 2);
5634 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
5635
5636 assert_eq!(
5637 queue.get(&orphaned.id).unwrap().status,
5638 TaskStatus::Held,
5639 "nothing was driving it and there was no run to recover"
5640 );
5641 assert_eq!(
5642 queue.get(&alive.id).unwrap().status,
5643 TaskStatus::Running,
5644 "a live claim must protect the task it belongs to"
5645 );
5646 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
5647 }
5648
5649 fn read_run_under(home: &Path, id: &str) -> RunState {
5655 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
5656 serde_json::from_str(&body).unwrap()
5657 }
5658
5659 #[test]
5660 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
5661 let dir = tempfile::tempdir().unwrap();
5662 let home = dir.path().to_path_buf();
5663 let now = Timestamp::now();
5664 let overrun_seat = || crate::run::ActiveSeat {
5665 node: "implement".to_owned(),
5666 started_at: now - jiff::SignedDuration::new(21_000, 0),
5667 timeout_secs: 3_600,
5668 attempt: 0,
5669 task: None,
5670 command: None,
5671 index: None,
5672 total: None,
5673 };
5674
5675 let mut dead = run_state(RunStatus::Implementing);
5676 dead.id = "20260101-000000-dead".to_owned();
5677 dead.active.insert("impl-A".to_owned(), overrun_seat());
5678 dead.driver_pid = Some(4242);
5681 dead.save_under(&home).unwrap();
5682
5683 let mut alive = run_state(RunStatus::Implementing);
5686 alive.id = "20260101-000000-aliv".to_owned();
5687 alive.active.insert("impl-A".to_owned(), overrun_seat());
5688 alive.save_under(&home).unwrap();
5689 let mut status = Status::new();
5690 status.current = vec![Current {
5691 task: "20260101-000000-task".to_owned(),
5692 run: alive.id.clone(),
5693 }];
5694 write_status_to(&home.join("daemon.json"), &status).unwrap();
5695
5696 let questions = Questions::at(home.join("questions"));
5700 let mut q = ask::Question::new(
5701 dead.id.clone(),
5702 "implement".to_owned(),
5703 "impl-A".to_owned(),
5704 "Which storage backend?".to_owned(),
5705 String::new(),
5706 vec!["SQLite".to_owned(), "Redis".to_owned()],
5707 );
5708 questions.put(&mut q).unwrap();
5709
5710 let abandoned = reclaim_abandoned_runs_with(
5711 &home,
5712 now,
5713 |pid| if pid == 4242 { Some(false) } else { None },
5714 |_| panic!("a query answering Dead outright needs no identity corroboration"),
5715 );
5716 assert_eq!(abandoned, vec![dead.id.clone()]);
5717
5718 let reloaded = read_run_under(&home, &dead.id);
5719 assert_eq!(reloaded.status, RunStatus::Failed);
5720 assert!(reloaded.active.is_empty());
5721 assert!(
5722 !questions.get(&q.id).unwrap().status.open(),
5723 "the failed run's own open question must be settled in the same pass"
5724 );
5725
5726 let still_alive = read_run_under(&home, &alive.id);
5727 assert_eq!(
5728 still_alive.status,
5729 RunStatus::Implementing,
5730 "a live daemon's claim protects it"
5731 );
5732 assert!(!still_alive.active.is_empty());
5733 }
5734
5735 #[test]
5745 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
5746 let dir = tempfile::tempdir().unwrap();
5747 let home = dir.path().to_path_buf();
5748 let now = Timestamp::now();
5749
5750 let mut manual = run_state(RunStatus::Reviewing);
5751 manual.id = "20260101-000000-manl".to_owned();
5752 manual.active.insert(
5753 "review-1".to_owned(),
5754 crate::run::ActiveSeat {
5755 node: "review".to_owned(),
5756 started_at: now - jiff::SignedDuration::new(21_000, 0),
5757 timeout_secs: 3_600,
5758 attempt: 0,
5759 task: None,
5760 command: None,
5761 index: None,
5762 total: None,
5763 },
5764 );
5765 manual.driver_pid = Some(4242);
5769 manual.driver_started_at = Some("2026-09-22T10:00:00Z".to_owned());
5770 manual.save_under(&home).unwrap();
5771
5772 let abandoned = reclaim_abandoned_runs_with(
5773 &home,
5774 now,
5775 |pid| if pid == 4242 { Some(true) } else { None },
5776 |pid| {
5777 if pid == 4242 {
5778 Some("2026-09-22T10:00:00Z".to_owned())
5779 } else {
5780 None
5781 }
5782 },
5783 );
5784 assert!(
5785 abandoned.is_empty(),
5786 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
5787 );
5788
5789 let reloaded = read_run_under(&home, &manual.id);
5790 assert_eq!(reloaded.status, RunStatus::Reviewing);
5791 assert!(!reloaded.active.is_empty());
5792 }
5793
5794 #[test]
5795 fn an_already_claimed_task_is_skipped_rather_than_failed() {
5796 let dir = tempfile::tempdir().unwrap();
5797 let queue = Queue::at(dir.path().to_path_buf());
5798 let mut only = task();
5799 queue.put(&mut only).unwrap();
5800
5801 let _elsewhere = queue.claim(&only.id).unwrap();
5802 let candidates = runnable(&queue);
5803 assert_eq!(candidates.len(), 1, "the task is still runnable");
5804 assert!(
5805 queue.claim(&candidates[0].id).is_err(),
5806 "the loop cannot take a claim somebody else holds"
5807 );
5808
5809 let after = queue.get(&only.id).unwrap();
5810 assert_eq!(after.status, TaskStatus::Queued);
5811 assert_eq!(
5812 after.attempts, 0,
5813 "losing the race is not an attempt at the task"
5814 );
5815 assert_eq!(after.last_error, None);
5816 }
5817
5818 #[test]
5819 fn the_status_file_round_trips_and_its_heartbeat_advances() {
5820 let dir = tempfile::tempdir().unwrap();
5821 let path = dir.path().join("daemon.json");
5822
5823 let mut status = Status::new();
5824 status.idle = false;
5825 status.completed = 7;
5826 status.current = vec![Current {
5827 task: "20260902-000000-t111".to_owned(),
5828 run: "20260902-000001-r111".to_owned(),
5829 }];
5830 write_status_to(&path, &status).unwrap();
5831 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5832 assert_eq!(first.schema, SCHEMA);
5833 assert_eq!(first.pid, std::process::id());
5834 assert!(!first.idle);
5835 assert_eq!(first.completed, 7);
5836 assert_eq!(first.current, status.current);
5837 assert!(
5838 !path.with_extension("json.tmp").exists(),
5839 "the temp file is renamed, not left behind"
5840 );
5841
5842 std::thread::sleep(Duration::from_millis(5));
5843 status.updated_at = Timestamp::now();
5844 status.polls = 3;
5845 write_status_to(&path, &status).unwrap();
5846 let second: Status =
5847 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5848 assert!(
5849 second.updated_at > first.updated_at,
5850 "a reader can only detect staleness if the heartbeat moves"
5851 );
5852 assert_eq!(
5853 second.started_at, first.started_at,
5854 "the start time is not a heartbeat"
5855 );
5856 assert_eq!(second.polls, 3);
5857 }
5858
5859 #[test]
5860 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
5861 let dir = tempfile::tempdir().unwrap();
5862
5863 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
5864
5865 let mut status = Status::new();
5866 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
5867 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
5868 let stale = read_status(dir.path()).unwrap();
5869 assert!(
5870 !stale.running(Timestamp::now()),
5871 "a minute without a heartbeat is a dead daemon, not a busy one"
5872 );
5873 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
5874
5875 status.updated_at = Timestamp::now();
5876 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
5877 let fresh = read_status(dir.path()).unwrap();
5878 assert!(fresh.running(Timestamp::now()));
5879 }
5880
5881 #[test]
5882 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
5883 let dir = tempfile::tempdir().unwrap();
5884 let now = Timestamp::now();
5885 let mine = "20260903-080619-01c2";
5886
5887 assert!(
5888 !is_working_on(dir.path(), mine, now),
5889 "no status file means nobody is working on anything"
5890 );
5891
5892 let mut status = Status::new();
5893 status.current = vec![Current {
5894 task: "20260903-080340-0167".to_owned(),
5895 run: mine.to_owned(),
5896 }];
5897 status.updated_at = now;
5898 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
5899 assert!(is_working_on(dir.path(), mine, now));
5900 assert!(
5901 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
5902 "a daemon busy with one run is not working on another"
5903 );
5904
5905 status.updated_at = now - jiff::SignedDuration::from_secs(600);
5908 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
5909 assert!(
5910 !is_working_on(dir.path(), mine, now),
5911 "a stale heartbeat is a dead daemon, so its run is a leftover"
5912 );
5913 }
5914
5915 #[test]
5916 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
5917 let dir = tempfile::tempdir().unwrap();
5918 let now = Timestamp::now();
5919
5920 assert!(
5921 !is_working_on_short(dir.path(), "01c2", now),
5922 "no status file means nobody is working on anything"
5923 );
5924
5925 let mut status = Status::new();
5926 status.current = vec![Current {
5927 task: "20260903-080340-0167".to_owned(),
5928 run: "20260903-080619-01c2".to_owned(),
5929 }];
5930 status.updated_at = now;
5931 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
5932 assert!(
5933 is_working_on_short(dir.path(), "01c2", now),
5934 "the run's short id is the last block of its full id"
5935 );
5936 assert!(
5937 !is_working_on_short(dir.path(), "3cbf", now),
5938 "a daemon busy with one worktree bay is not working on another"
5939 );
5940 }
5941
5942 #[test]
5943 fn a_newer_status_file_still_yields_a_reading() {
5944 let dir = tempfile::tempdir().unwrap();
5945 std::fs::write(
5948 dir.path().join("daemon.json"),
5949 serde_json::json!({
5950 "schema": 2,
5951 "updated_at": Timestamp::now().to_string(),
5952 "idle": true,
5953 "surprise": { "nested": [1, 2, 3] },
5954 })
5955 .to_string(),
5956 )
5957 .unwrap();
5958
5959 let reading = read_status(dir.path()).expect("a forward-compatible read");
5960 assert!(reading.running(Timestamp::now()));
5961 assert!(reading.idle);
5962 assert!(reading.current.is_empty());
5963 }
5964
5965 #[test]
5966 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
5967 let dir = tempfile::tempdir().unwrap();
5973 std::fs::write(
5974 dir.path().join("daemon.json"),
5975 serde_json::json!({
5976 "schema": 1,
5977 "pid": 4242,
5978 "updated_at": Timestamp::now().to_string(),
5979 "idle": false,
5980 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
5981 "completed": 3,
5982 "polls": 9,
5983 })
5984 .to_string(),
5985 )
5986 .unwrap();
5987
5988 let reading = read_status(dir.path()).expect("an older shape must still parse");
5989 assert!(reading.running(Timestamp::now()));
5990 assert_eq!(
5991 reading.current,
5992 vec![Current {
5993 task: "20260902-140501-aaaa".to_owned(),
5994 run: "20260902-140502-bbbb".to_owned(),
5995 }]
5996 );
5997 }
5998
5999 #[test]
6000 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
6001 let dir = tempfile::tempdir().unwrap();
6002 std::fs::write(
6003 dir.path().join("daemon.json"),
6004 serde_json::json!({
6005 "schema": 1,
6006 "updated_at": Timestamp::now().to_string(),
6007 "idle": true,
6008 "current": null,
6009 })
6010 .to_string(),
6011 )
6012 .unwrap();
6013 let with_null = read_status(dir.path()).expect("null must still parse");
6014 assert!(with_null.current.is_empty());
6015
6016 std::fs::write(
6017 dir.path().join("daemon.json"),
6018 serde_json::json!({
6019 "schema": 1,
6020 "updated_at": Timestamp::now().to_string(),
6021 "idle": true,
6022 })
6023 .to_string(),
6024 )
6025 .unwrap();
6026 let absent = read_status(dir.path()).expect("a missing field must still parse");
6027 assert!(absent.current.is_empty());
6028 }
6029
6030 #[test]
6031 fn a_task_without_a_repository_runs_in_the_daemons_default() {
6032 let fallback = Path::new("/default");
6033 let mut blank = task();
6034 blank.repo = PathBuf::new();
6035 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
6036 let mut dot = task();
6037 dot.repo = PathBuf::from(".");
6038 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
6039 assert_eq!(
6040 repo_for(&task(), fallback),
6041 PathBuf::from("/repo"),
6042 "a task that names a repository keeps it"
6043 );
6044 }
6045
6046 #[test]
6047 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
6048 let mut solo_cfg = Config::default();
6054 solo_cfg.graph.candidates = 3;
6055 let mut solo_task = task();
6056 solo_task.solo = true;
6057 apply_solo(&mut solo_cfg, &solo_task);
6058 assert_eq!(solo_cfg.graph.candidates, 1);
6059
6060 let mut plain_cfg = Config::default();
6061 plain_cfg.graph.candidates = 3;
6062 let plain_task = task();
6063 assert!(!plain_task.solo);
6064 apply_solo(&mut plain_cfg, &plain_task);
6065 assert_eq!(
6066 plain_cfg.graph.candidates, 3,
6067 "a task that did not ask to run alone keeps the config's candidates"
6068 );
6069 }
6070
6071 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
6072 QuotaLoss {
6073 seat: seat.into(),
6074 node: "judge".into(),
6075 at: at.parse().unwrap(),
6076 reset: reset.map(str::to_string),
6077 }
6078 }
6079
6080 #[test]
6081 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
6082 let old: Vec<QuotaLoss> = (1..=4)
6083 .map(|i| {
6084 loss(
6085 &format!("judge-{i}"),
6086 "2026-09-23T05:23:00Z",
6087 Some("2:40pm (Asia/Tokyo)"),
6088 )
6089 })
6090 .collect();
6091 let fresh = losses_this_attempt(&old, &old);
6092 assert!(fresh.is_empty());
6093 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
6094 }
6096
6097 #[test]
6098 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
6099 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
6100 let now = Timestamp::now();
6101 let mut after = old.clone();
6102 after.push(loss("judge-2", &now.to_string(), None));
6103 let fresh = losses_this_attempt(&old, &after);
6104 assert_eq!(fresh, vec![after[1].clone()]);
6105 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
6106 assert_eq!(
6107 until,
6108 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
6109 );
6110 }
6111
6112 #[test]
6113 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
6114 let before = vec![
6117 loss("judge-1", "2026-09-23T05:23:00Z", None),
6118 loss("judge-2", "2026-09-23T05:24:00Z", None),
6119 ];
6120 let after = vec![
6121 loss("judge-2", "2026-09-23T05:24:00Z", None),
6122 loss("judge-1", "2026-09-24T01:00:00Z", None),
6123 ];
6124 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
6125 }
6126
6127 #[test]
6128 fn merge_overrides_are_parsed_or_refused() {
6129 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
6130 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
6131 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
6132 assert!(merge_mode("squash").is_err());
6133 }
6134
6135 #[test]
6136 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
6137 let now = Timestamp::now();
6138 let fallback = Duration::from_secs(300);
6139 let cap = Duration::from_secs(1800);
6140
6141 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
6143
6144 let soon = now + jiff::SignedDuration::from_secs(600);
6146 assert_eq!(
6147 quota_wait(Some(soon), now, fallback, cap),
6148 Duration::from_secs(600)
6149 );
6150
6151 let past = now - jiff::SignedDuration::from_secs(60);
6154 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
6155
6156 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
6159 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
6160 }
6161
6162 #[test]
6163 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
6164 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
6165
6166 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
6167 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
6168
6169 let already_past =
6173 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
6174 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
6175
6176 assert!(
6177 parse_reset_hint("session limit reached", now, now).is_none(),
6178 "free text with no recognised shape is not guessed at"
6179 );
6180 assert!(
6181 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
6182 "an unresolvable zone name is not guessed at either"
6183 );
6184 }
6185
6186 #[test]
6187 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
6188 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
6189
6190 let at = parse_reset_hint(
6191 "You've hit your usage limit. Visit \
6192 https://chatgpt.com/codex/settings/usage to purchase more \
6193 credits or try again at Sep 19th, 2026 5:10 PM.",
6194 now,
6195 now,
6196 )
6197 .expect("the codex reset wording is a recognised shape");
6198 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
6199
6200 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
6205 .expect("an explicit year needs no rollover");
6206 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
6207
6208 assert!(
6209 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
6210 "a two-digit year is not the documented shape and is not guessed at"
6211 );
6212 assert!(
6213 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
6214 "a four-letter month name is not the documented three-letter abbreviation"
6215 );
6216 assert!(
6217 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
6218 "an explicit zone on the dated shape is a format nobody has \
6219 documented, and is refused rather than guessed at as UTC"
6220 );
6221 }
6222
6223 #[test]
6224 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
6225 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
6226 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
6227
6228 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
6229 assert_eq!(at.as_second() - recorded.as_second(), 3769);
6230
6231 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
6232 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
6233
6234 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
6235 assert!(
6236 parse_reset_hint(bad, now, recorded).is_none(),
6237 "{bad:?} must not be guessed at"
6238 );
6239 }
6240 }
6241
6242 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
6246 let config = dir.join("magi.toml");
6247 std::fs::write(
6248 &config,
6249 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
6250 )
6251 .unwrap();
6252 let opts = Opts {
6253 poll: Duration::from_secs(30),
6254 config: Some(config),
6255 repo: dir.join("repo"),
6259 ..Opts::default()
6260 };
6261 let home = dir.join("home");
6270 let worktrees = dir.join("wt");
6271 (
6272 opts,
6273 Queue::at(dir.join("queue")),
6274 home.join("daemon.json"),
6275 home,
6276 worktrees,
6277 )
6278 }
6279
6280 #[test]
6281 fn a_stop_is_idempotent_and_once_set_stays_set() {
6282 let stop = Stop::new();
6283 assert!(!stop.stopped());
6284
6285 stop.stop();
6286 assert!(stop.stopped());
6287 stop.stop();
6288 assert!(stop.stopped(), "a second stop is not a toggle");
6289
6290 let shared = stop.clone();
6291 assert!(
6292 shared.stopped(),
6293 "a clone is the same stop; that is how the loop and its caller share one"
6294 );
6295 }
6296
6297 #[test]
6298 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
6299 let stop = Stop::new();
6300 stop.enter();
6301 assert!(
6302 !stop.finishing(),
6303 "a busy loop nobody has asked to stop is just running"
6304 );
6305
6306 stop.stop();
6307 assert!(
6308 stop.finishing(),
6309 "a stop asked for mid-run has not landed until the run is settled"
6310 );
6311
6312 stop.exit();
6313 assert!(
6314 !stop.finishing(),
6315 "once the run is settled the stop has landed and there is nothing to finish"
6316 );
6317 }
6318
6319 #[test]
6320 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
6321 let stop = Stop::new();
6322 stop.enter();
6323 stop.enter();
6324 stop.stop();
6325 assert!(stop.finishing(), "two runs still in flight");
6326
6327 stop.exit();
6328 assert!(
6329 stop.finishing(),
6330 "one run finished, but a sibling is still working"
6331 );
6332
6333 stop.exit();
6334 assert!(
6335 !stop.finishing(),
6336 "the last run out is what actually lands the stop"
6337 );
6338 }
6339
6340 #[tokio::test]
6341 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
6342 let dir = tempfile::tempdir().unwrap();
6343 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
6344 let stop = Stop::new();
6345 stop.stop();
6346
6347 let began = std::time::Instant::now();
6348 tokio::time::timeout(
6349 Duration::from_secs(2),
6350 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
6351 )
6352 .await
6353 .expect("a stopped loop must return, not sit out its poll interval")
6354 .expect("the loop's own setup and teardown must not fail");
6355 assert!(
6356 began.elapsed() < opts.poll,
6357 "returned only after {:?}, which is a poll interval, not a stop",
6358 began.elapsed()
6359 );
6360 }
6361
6362 #[tokio::test]
6363 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
6364 let dir = tempfile::tempdir().unwrap();
6365 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
6366 let stop = Stop::new();
6367
6368 let asker = {
6371 let stop = stop.clone();
6372 tokio::spawn(async move {
6373 tokio::time::sleep(Duration::from_millis(20)).await;
6374 stop.stop();
6375 })
6376 };
6377
6378 let began = std::time::Instant::now();
6379 tokio::time::timeout(
6380 Duration::from_secs(2),
6381 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
6382 )
6383 .await
6384 .expect("a stop asked for while idle must wake the wait")
6385 .expect("the loop's own setup and teardown must not fail");
6386 asker.await.unwrap();
6387 assert!(
6388 began.elapsed() < opts.poll,
6389 "returned only after {:?}, so the stop waited on the sleep",
6390 began.elapsed()
6391 );
6392 }
6393
6394 #[tokio::test]
6395 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
6396 let dir = tempfile::tempdir().unwrap();
6397 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
6398 let stop = Stop::new();
6399 stop.stop();
6400
6401 tokio::time::timeout(
6402 Duration::from_secs(2),
6403 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
6404 )
6405 .await
6406 .expect("a stopped loop must return")
6407 .expect("the loop's own setup and teardown must not fail");
6408
6409 assert!(
6410 home.is_dir(),
6411 "the loop did publish a status file, so its removal is the teardown and not an absence"
6412 );
6413 assert!(
6414 !status_file.exists(),
6415 "a stopped loop clears its status file"
6416 );
6417 assert!(
6418 read_status(&home).is_none(),
6419 "a reader must see no daemon at all, not a heartbeat that merely stopped"
6420 );
6421 }
6422
6423 #[tokio::test]
6424 async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
6425 let dir = tempfile::tempdir().unwrap();
6426 let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
6427 opts.once = true;
6428
6429 let mut settled = RunState::new(
6430 dir.path().join("repo"),
6431 "main".to_owned(),
6432 "abc1234".to_owned(),
6433 "fixture".to_owned(),
6434 Config::default(),
6435 );
6436 settled.status = RunStatus::Ready;
6437 let run_dir = home.join("runs").join(&settled.id);
6438 std::fs::create_dir_all(&run_dir).unwrap();
6439 std::fs::write(
6440 run_dir.join("run.json"),
6441 serde_json::to_string_pretty(&settled).unwrap(),
6442 )
6443 .unwrap();
6444 let questions = Questions::at(home.join("questions"));
6445 let mut question = ask::Question::new(
6446 settled.id.clone(),
6447 "review".to_owned(),
6448 "reviewer-1".to_owned(),
6449 "Continue?".to_owned(),
6450 String::new(),
6451 Vec::new(),
6452 );
6453 questions.put(&mut question).unwrap();
6454
6455 drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
6456 .await
6457 .unwrap();
6458
6459 assert_eq!(
6460 questions.get(&question.id).unwrap().status,
6461 ask::QuestionStatus::Abandoned,
6462 "an empty --once drain still performs startup question cleanup"
6463 );
6464 }
6465
6466 #[test]
6467 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
6468 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
6469
6470 assert!(
6471 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
6472 "never checked before: due at once"
6473 );
6474
6475 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
6476 assert!(
6477 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
6478 "well inside the interval: not due yet"
6479 );
6480
6481 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
6482 assert!(
6483 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
6484 "exactly at the edge: not yet due, same convention as `clean::due`"
6485 );
6486
6487 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
6488 assert!(
6489 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
6490 "past the interval: due again"
6491 );
6492 }
6493
6494 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
6499 let config = dir.join("magi.toml");
6500 std::fs::write(
6506 &config,
6507 format!(
6508 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
6509 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
6510 cache_dir.display()
6511 ),
6512 )
6513 .unwrap();
6514 Opts {
6515 config: Some(config),
6516 repo: dir.join("repo"),
6517 ..Opts::default()
6518 }
6519 }
6520
6521 #[tokio::test]
6522 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
6523 let dir = tempfile::tempdir().unwrap();
6524 let home = dir.path().join("home");
6525 let cache_dir = dir.path().join("cache");
6526 std::fs::create_dir_all(&cache_dir).unwrap();
6527 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
6528 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
6529
6530 let running = Stop::new();
6533 let mut last_checked = None;
6534 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
6535 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
6536 .await;
6537 assert_eq!(
6538 crate::disk::dir_size(&cache_dir),
6539 0,
6540 "over the cap on the first check ever: pruned at once, no idle queue required"
6541 );
6542 assert_eq!(last_checked, Some(t0));
6543
6544 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
6546 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
6547 maybe_prune_cache_between_runs(
6548 &opts.repo,
6549 &opts,
6550 &home,
6551 &running,
6552 &mut last_checked,
6553 too_soon,
6554 )
6555 .await;
6556 assert_eq!(
6557 crate::disk::dir_size(&cache_dir),
6558 10,
6559 "too soon since the last check: left alone rather than rescanned every call"
6560 );
6561 assert_eq!(
6562 last_checked,
6563 Some(t0),
6564 "an idle check does not reset the clock"
6565 );
6566
6567 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
6569 maybe_prune_cache_between_runs(
6570 &opts.repo,
6571 &opts,
6572 &home,
6573 &running,
6574 &mut last_checked,
6575 due_again,
6576 )
6577 .await;
6578 assert_eq!(
6579 crate::disk::dir_size(&cache_dir),
6580 0,
6581 "due again: pruned back under the cap"
6582 );
6583 }
6584
6585 #[tokio::test]
6593 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
6594 let dir = tempfile::tempdir().unwrap();
6595 let home = dir.path().join("home");
6596 let cache_dir = dir.path().join("cache");
6597 std::fs::create_dir_all(&cache_dir).unwrap();
6598 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
6599 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
6600
6601 let stop = Stop::new();
6602 stop.stop();
6603 assert!(
6604 !stop.finishing(),
6605 "no run is in flight at a between-runs boundary, so nothing else \
6606 would tell the operator this stop had not taken effect yet"
6607 );
6608
6609 let mut last_checked = None;
6610 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
6611 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
6612 .await;
6613 assert_eq!(
6614 crate::disk::dir_size(&cache_dir),
6615 10,
6616 "over its cap, and due for the first check ever, but a stop outranks \
6617 it: the cap is a standing policy the next start measures again"
6618 );
6619 assert_eq!(
6620 last_checked, None,
6621 "a check that never happened must not claim the interval"
6622 );
6623 }
6624
6625 #[tokio::test]
6639 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
6640 let dir = tempfile::tempdir().unwrap();
6641 let cache_dir = dir.path().join("cache");
6642 std::fs::create_dir_all(&cache_dir).unwrap();
6643 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
6644
6645 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
6646 opts.poll = Duration::from_millis(20);
6647 opts.max_attempts = 1_000;
6648
6649 let queue = Queue::at(dir.path().join("queue"));
6650 let mut t = Task::new(
6651 "x".to_owned(),
6652 "x".to_owned(),
6653 opts.repo.clone(),
6654 Source::Human,
6655 );
6656 queue.put(&mut t).unwrap();
6657
6658 let home = dir.path().join("home");
6659 let worktrees = dir.path().join("wt");
6660 let status_file = home.join("daemon.json");
6661 let stop = Stop::new();
6662 let stopper = {
6663 let stop = stop.clone();
6664 tokio::spawn(async move {
6665 tokio::time::sleep(Duration::from_millis(400)).await;
6666 stop.stop();
6667 })
6668 };
6669
6670 tokio::time::timeout(
6671 Duration::from_secs(10),
6672 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
6673 )
6674 .await
6675 .expect("the loop must not hang on a queue that keeps producing failing work")
6676 .expect("the loop's own setup and teardown must not fail");
6677 stopper.await.unwrap();
6678
6679 let after = queue.get(&t.id).unwrap();
6680 assert!(
6681 after.attempts >= 2,
6682 "the harness must actually have retried more than once, or this is not \
6683 exercising a busy queue at all (got {} attempt(s))",
6684 after.attempts
6685 );
6686 assert!(
6687 after.status.runnable(),
6688 "still under its attempt budget: the queue never reached a natural idle \
6689 on its own, only the external stop ended the test"
6690 );
6691
6692 assert_eq!(
6693 crate::disk::dir_size(&cache_dir),
6694 0,
6695 "an oversized cache must not be left to grow unboundedly just because the \
6696 queue kept the loop busy the whole time"
6697 );
6698 }
6699
6700 #[test]
6701 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
6702 let dir = tempfile::tempdir().unwrap();
6703 let queue = Queue::at(dir.path().join("queue"));
6704 let questions = Questions::at(dir.path().join("questions"));
6705 let mut task = task();
6706 queue.put(&mut task).unwrap();
6707
6708 let mut task_question = ask::Question::new(
6709 task.id.clone(),
6710 crate::conduct::NODE.to_owned(),
6711 "conduct".to_owned(),
6712 "Which backend?".to_owned(),
6713 String::new(),
6714 Vec::new(),
6715 );
6716 questions.put(&mut task_question).unwrap();
6717 task.block(vec![task_question.id.clone()], None);
6718 queue.put(&mut task).unwrap();
6719
6720 let mut run_question = ask::Question::new(
6721 "20260101-000000-run1".to_owned(),
6722 "review".to_owned(),
6723 "reviewer-1".to_owned(),
6724 "Run question".to_owned(),
6725 String::new(),
6726 Vec::new(),
6727 );
6728 questions.put(&mut run_question).unwrap();
6729
6730 let mut coincidental = ask::Question::new(
6735 task.id.clone(),
6736 "review".to_owned(),
6737 "reviewer-1".to_owned(),
6738 "Unrelated review question".to_owned(),
6739 String::new(),
6740 Vec::new(),
6741 );
6742 questions.put(&mut coincidental).unwrap();
6743
6744 reconcile_task_questions(&queue, &questions);
6745 assert!(questions.get(&task_question.id).unwrap().status.open());
6746 assert!(questions.get(&run_question.id).unwrap().status.open());
6747 assert!(questions.get(&coincidental.id).unwrap().status.open());
6748
6749 task.release();
6750 queue.put(&mut task).unwrap();
6751 reconcile_task_questions(&queue, &questions);
6752 assert_eq!(
6753 questions.get(&task_question.id).unwrap().status,
6754 ask::QuestionStatus::Abandoned
6755 );
6756 assert!(
6757 questions.get(&run_question.id).unwrap().status.open(),
6758 "run questions remain the run janitor's responsibility"
6759 );
6760 assert!(
6761 questions.get(&coincidental.id).unwrap().status.open(),
6762 "a non-conductor question must not be abandoned just because its \
6763 run id coincides with a task id"
6764 );
6765 }
6766
6767 #[test]
6768 fn a_freshly_started_running_task_is_never_stalled() {
6769 let dir = tempfile::tempdir().unwrap();
6770 let mut t = task();
6771 t.start("run-1".to_owned());
6772 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
6775 }
6776
6777 #[test]
6778 fn a_long_running_task_with_no_live_daemon_is_stalled() {
6779 let dir = tempfile::tempdir().unwrap();
6780 let mut t = task();
6781 t.start("run-1".to_owned());
6782 t.updated_at = Timestamp::now()
6783 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
6784 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
6785 assert_eq!(
6786 stalled_tasks(
6787 &Queue::at(dir.path().join("q")),
6788 dir.path(),
6789 Timestamp::now()
6790 )
6791 .len(),
6792 0,
6793 "the task was never written to this queue"
6794 );
6795 }
6796
6797 #[test]
6798 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
6799 let dir = tempfile::tempdir().unwrap();
6800 let mut t = task();
6801 t.id = "20260903-080340-0167".to_owned();
6802 t.start("20260903-080619-01c2".to_owned());
6803 t.updated_at = Timestamp::now()
6804 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
6805
6806 let mut status = Status::new();
6807 status.current = vec![Current {
6808 task: t.id.clone(),
6809 run: "20260903-080619-01c2".to_owned(),
6810 }];
6811 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
6812
6813 assert!(
6814 !is_stalled(&t, dir.path(), Timestamp::now()),
6815 "a live daemon's own heartbeat rules out stalled, however long the task has run"
6816 );
6817 }
6818
6819 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
6823 let path = queue.path_of(id);
6824 let body = std::fs::read_to_string(&path).unwrap();
6825 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
6826 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
6827 v["updated_at"] = serde_json::Value::String(old.to_string());
6828 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
6829 }
6830
6831 #[test]
6832 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
6833 let dir = tempfile::tempdir().unwrap();
6846 let queue = Queue::at(dir.path().join("queue"));
6847 let home = dir.path().join("home");
6848
6849 let mut t = task();
6850 t.id = "20260101-000001-lock".to_owned();
6851 t.start("run-1".to_owned());
6852 queue.put(&mut t).unwrap();
6853 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
6854 std::fs::write(
6855 dir.path().join("queue").join(format!("{}.lock", t.id)),
6856 "not a pid",
6857 )
6858 .unwrap();
6859
6860 let now = Timestamp::now();
6861 assert!(
6862 reclaim_orphaned_running(&queue, 2).is_empty(),
6863 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
6864 and reclaim must leave the task alone"
6865 );
6866 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
6867
6868 let stalled = stalled_tasks(&queue, &home, now);
6869 assert_eq!(
6870 stalled.len(),
6871 1,
6872 "reclaim's inability to claim it yet must not hide it from the conductor"
6873 );
6874 assert_eq!(stalled[0].id, t.id);
6875 }
6876
6877 #[test]
6878 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
6879 let dir = tempfile::tempdir().unwrap();
6880 crate::run::set_home(dir.path().join("run-home"));
6881 let queue = Queue::at(dir.path().join("queue"));
6882 let home = dir.path().join("home");
6883 let questions = Questions::at(dir.path().join("questions"));
6884
6885 let mut t = task();
6886 t.id = "20260101-000003-dead".to_owned();
6887 t.start("missing-run".to_owned());
6888 queue.put(&mut t).unwrap();
6889 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
6890
6891 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
6894 assert_eq!(
6895 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
6896 [&t.id]
6897 );
6898 assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
6899 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
6900
6901 crate::conduct::apply(
6904 &queue,
6905 &questions,
6906 &crate::conduct::Verdict {
6907 decisions: vec![crate::conduct::Decision {
6908 id: t.id.clone(),
6909 recovery: Some(crate::conduct::Recovery::Requeue),
6910 ..crate::conduct::Decision::default()
6911 }],
6912 },
6913 )
6914 .unwrap();
6915 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
6916 }
6917
6918 #[test]
6919 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
6920 let dir = tempfile::tempdir().unwrap();
6921 let queue = Queue::at(dir.path().join("queue"));
6922 let home = dir.path().join("home");
6923
6924 let mut fresh = task();
6925 fresh.id = "20260101-000001-aaaa".to_owned();
6926 fresh.start("run-1".to_owned());
6927 queue.put(&mut fresh).unwrap();
6928
6929 let mut old = task();
6930 old.id = "20260101-000002-bbbb".to_owned();
6931 old.start("run-2".to_owned());
6932 queue.put(&mut old).unwrap();
6933 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
6934
6935 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
6936 assert_eq!(stalled.len(), 1);
6937 assert_eq!(stalled[0].id, old.id);
6938 }
6939
6940 #[test]
6941 fn queued_and_finished_task_views_partition_by_status() {
6942 let dir = tempfile::tempdir().unwrap();
6943 let queue = Queue::at(dir.path().join("queue"));
6944
6945 let mut queued = task();
6946 queued.id = "20260101-000001-aaaa".to_owned();
6947 queue.put(&mut queued).unwrap();
6948
6949 let mut failed = task();
6950 failed.id = "20260101-000002-bbbb".to_owned();
6951 failed.start("run-1".to_owned());
6952 failed.fail("gate red", 5);
6953 queue.put(&mut failed).unwrap();
6954
6955 let mut held = task();
6956 held.id = "20260101-000003-cccc".to_owned();
6957 held.hold_machine(None);
6958 queue.put(&mut held).unwrap();
6959
6960 let mut running = task();
6961 running.id = "20260101-000004-dddd".to_owned();
6962 running.start("run-2".to_owned());
6963 queue.put(&mut running).unwrap();
6964
6965 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
6966 assert_eq!(queued_ids, [queued.id.clone()]);
6967
6968 let mut finished_ids: Vec<String> =
6969 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
6970 finished_ids.sort_unstable();
6971 let mut want = vec![failed.id.clone(), held.id.clone()];
6972 want.sort_unstable();
6973 assert_eq!(finished_ids, want);
6974 }
6975
6976 #[test]
6977 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
6978 let dir = tempfile::tempdir().unwrap();
6979 let queue = Queue::at(dir.path().join("queue"));
6980 let questions = ask::Questions::at(dir.path().join("questions"));
6981
6982 let mut dep = task();
6983 dep.id = "20260101-000001-dep0".to_owned();
6984 dep.succeed();
6985 queue.put(&mut dep).unwrap();
6986
6987 let mut still_going = task();
6988 still_going.id = "20260101-000002-dep1".to_owned();
6989 queue.put(&mut still_going).unwrap();
6990
6991 let mut blocked = task();
6992 blocked.id = "20260101-000003-main".to_owned();
6993 blocked.block(
6994 vec![dep.id.clone(), still_going.id.clone()],
6995 Some("waits on both".to_owned()),
6996 );
6997 queue.put(&mut blocked).unwrap();
6998
6999 resolve_blockers(&queue, &questions);
7000
7001 let after = queue.get(&blocked.id).unwrap();
7002 assert_eq!(
7003 after.status,
7004 TaskStatus::Blocked,
7005 "one dependency is still outstanding"
7006 );
7007 assert_eq!(after.blocked_by, [still_going.id.clone()]);
7008 }
7009
7010 #[test]
7011 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
7012 let dir = tempfile::tempdir().unwrap();
7013 let queue = Queue::at(dir.path().join("queue"));
7014 let questions = ask::Questions::at(dir.path().join("questions"));
7015
7016 let mut q = crate::ask::Question::new(
7017 "20260101-000001-main".to_owned(),
7018 crate::conduct::NODE.to_owned(),
7019 "conduct".to_owned(),
7020 "Which backend?".to_owned(),
7021 String::new(),
7022 Vec::new(),
7023 );
7024 questions.put(&mut q).unwrap();
7025 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
7026 .unwrap();
7027 questions.put(&mut q).unwrap();
7028
7029 let mut blocked = task();
7030 blocked.id = "20260101-000001-main".to_owned();
7031 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
7032 queue.put(&mut blocked).unwrap();
7033
7034 resolve_blockers(&queue, &questions);
7035
7036 let after = queue.get(&blocked.id).unwrap();
7037 assert_eq!(
7038 after.status,
7039 TaskStatus::Queued,
7040 "the only blocker resolved"
7041 );
7042 assert_eq!(after.answers.len(), 1);
7043 assert_eq!(after.answers[0].question, "Which backend?");
7044 assert_eq!(after.answers[0].answer, "SQLite");
7045
7046 let instruction = instruction_for(&after);
7048 assert!(instruction.contains("Which backend?"));
7049 assert!(instruction.contains("SQLite"));
7050 }
7051
7052 #[test]
7053 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
7054 let dir = tempfile::tempdir().unwrap();
7055 let queue = Queue::at(dir.path().join("queue"));
7056 let questions = ask::Questions::at(dir.path().join("questions"));
7057
7058 let mut q = crate::ask::Question::new(
7059 "20260101-000001-main".to_owned(),
7060 crate::conduct::NODE.to_owned(),
7061 "conduct".to_owned(),
7062 "Is the setup done?".to_owned(),
7063 String::new(),
7064 Vec::new(),
7065 );
7066 q.abandon("no answer within 60s of asking");
7067 questions.put(&mut q).unwrap();
7068
7069 let mut blocked = task();
7070 blocked.id = "20260101-000001-main".to_owned();
7071 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
7072 queue.put(&mut blocked).unwrap();
7073
7074 resolve_blockers(&queue, &questions);
7075
7076 let after = queue.get(&blocked.id).unwrap();
7077 assert_eq!(
7078 after.status,
7079 TaskStatus::Held,
7080 "never left blocked on nothing"
7081 );
7082 assert!(!after.operator_held(), "a machine hold, for triage");
7083 assert!(
7084 after
7085 .hold_reason
7086 .as_deref()
7087 .unwrap_or_default()
7088 .contains("went unanswered")
7089 );
7090 }
7091
7092 #[test]
7093 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
7094 let dir = tempfile::tempdir().unwrap();
7100 let queue = Queue::at(dir.path().join("queue"));
7101 let questions = ask::Questions::at(dir.path().join("questions"));
7102
7103 let mut q = crate::ask::Question::new(
7104 "20260101-000001-main".to_owned(),
7105 crate::conduct::NODE.to_owned(),
7106 "conduct".to_owned(),
7107 "How should this be handled?".to_owned(),
7108 String::new(),
7109 Vec::new(),
7110 );
7111 questions.put(&mut q).unwrap();
7112 q.answer(crate::ask::Answer::Text(
7113 "leave it held, a human will look at it later".to_owned(),
7114 ))
7115 .unwrap();
7116 questions.put(&mut q).unwrap();
7117
7118 let mut held = task();
7119 held.id = "20260101-000001-main".to_owned();
7120 held.hold_machine(Some("out of attempts".to_owned()));
7121 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
7122 queue.put(&mut held).unwrap();
7123
7124 resolve_blockers(&queue, &questions);
7125
7126 let after = queue.get(&held.id).unwrap();
7127 assert_eq!(after.status, TaskStatus::Held);
7128 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
7129 assert_eq!(
7130 after.answers[0].answer,
7131 "leave it held, a human will look at it later"
7132 );
7133 }
7134
7135 #[test]
7136 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
7137 let dir = tempfile::tempdir().unwrap();
7143 let queue = Queue::at(dir.path().join("queue"));
7144 let questions = ask::Questions::at(dir.path().join("questions"));
7145
7146 let mut still_going = task();
7147 still_going.id = "20260101-000002-dep1".to_owned();
7148 queue.put(&mut still_going).unwrap();
7149
7150 let mut blocked = task();
7151 blocked.id = "20260101-000003-main".to_owned();
7152 blocked.block(
7153 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
7154 Some("waits on both".to_owned()),
7155 );
7156 queue.put(&mut blocked).unwrap();
7157
7158 resolve_blockers(&queue, &questions);
7159
7160 let after = queue.get(&blocked.id).unwrap();
7161 assert_eq!(
7162 after.status,
7163 TaskStatus::Held,
7164 "a missing dependency must not leave the task blocked forever"
7165 );
7166 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
7167 assert!(after.blocked_by.is_empty());
7168 let reason = after.hold_reason.as_deref().unwrap_or_default();
7169 assert!(
7170 reason.contains("20260101-000001-gone"),
7171 "the missing id must be named so an operator can tell what happened: {reason}"
7172 );
7173 assert!(
7174 reason.contains(&still_going.id),
7175 "the still-valid dependency must not silently vanish from the record: {reason}"
7176 );
7177 }
7178
7179 #[test]
7180 fn instruction_for_is_unchanged_without_any_answers() {
7181 let t = task();
7182 assert_eq!(instruction_for(&t), t.instruction);
7183 }
7184
7185 #[test]
7186 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
7187 let dir = tempfile::tempdir().unwrap();
7188 let q = Queue::at(dir.path().join("queue"));
7189 let src = dir.path().join("shot.png");
7190 std::fs::write(&src, "x").unwrap();
7191 let mut t = task();
7192 q.attach(&mut t, &[src]).unwrap();
7193 let paths = task_attachments(&q, &t).unwrap();
7194 assert_eq!(paths.len(), 1);
7195 assert!(paths[0].is_absolute() && paths[0].is_file());
7196 std::fs::remove_file(&paths[0]).unwrap();
7197 let err = task_attachments(&q, &t).unwrap_err().to_string();
7198 assert!(err.contains("shot.png"), "{err}");
7199 }
7200
7201 #[test]
7202 fn resumed_instruction_is_unchanged_without_any_answers() {
7203 let t = task();
7204 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
7205 }
7206
7207 #[test]
7208 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
7209 let mut t = task();
7210 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
7211 let old = t.instruction.clone();
7215
7216 let refreshed = resumed_instruction(&old, &t);
7217 assert!(refreshed.starts_with(&old), "the original text is kept");
7218 assert!(refreshed.contains("Which backend?"));
7219 assert!(refreshed.contains("SQLite"));
7220 }
7221
7222 #[test]
7223 fn resumed_instruction_keeps_an_original_answers_heading() {
7224 let mut t = task();
7225 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
7226 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
7227
7228 let refreshed = resumed_instruction(&t.instruction, &t);
7229
7230 assert!(
7231 refreshed.starts_with(&t.instruction),
7232 "an answers heading in the original instruction is not the appended block"
7233 );
7234 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
7235 assert!(refreshed.contains("Which backend?"));
7236 assert!(refreshed.contains("SQLite"));
7237
7238 let repeated = resumed_instruction(&refreshed, &t);
7239 assert_eq!(
7240 repeated, refreshed,
7241 "only the final appended block is refreshed"
7242 );
7243 }
7244
7245 #[test]
7246 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
7247 let mut t = task();
7248 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
7249
7250 let once = resumed_instruction(&t.instruction, &t);
7254 let twice = resumed_instruction(&once, &t);
7255 assert_eq!(once, twice);
7256 assert_eq!(once.matches("Which backend?").count(), 1);
7257
7258 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
7260 let refreshed = resumed_instruction(&once, &t);
7261 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
7262 assert!(refreshed.contains("Which backend?"));
7263 assert!(refreshed.contains("Which cache?"));
7264 }
7265
7266 #[test]
7267 fn prepare_instruction_covers_all_three_starters() {
7268 let mut t = task();
7269 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
7270
7271 assert_eq!(
7274 prepare_instruction(&Starter::Start, None, &t),
7275 Some(instruction_for(&t))
7276 );
7277
7278 let old = t.instruction.clone();
7281 assert_eq!(
7282 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
7283 Some(resumed_instruction(&old, &t))
7284 );
7285
7286 assert_eq!(
7290 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
7291 None
7292 );
7293 }
7294
7295 #[test]
7296 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
7297 assert_eq!(
7298 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
7299 Starter::Review("magi/eba2/A".to_owned())
7300 );
7301 }
7302
7303 #[test]
7304 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
7305 assert_eq!(
7306 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
7307 Starter::Start,
7308 "a vanished review branch must not fall back to resuming the old run either"
7309 );
7310 }
7311
7312 #[test]
7313 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
7314 let mut t = task();
7315 t.start("old-run".to_owned());
7316 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
7317 t.release();
7318 let branch = t.review_branch.take();
7319 assert_eq!(
7320 choose_starter(branch.as_deref(), true, Some("old-run")),
7321 Starter::Review("magi/eba2/A".to_owned()),
7322 "a review wins over resuming the old run"
7323 );
7324 }
7325
7326 #[test]
7327 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
7328 assert_eq!(
7329 choose_starter(None, false, Some("some-run")),
7330 Starter::Resume("some-run".to_owned())
7331 );
7332 assert_eq!(choose_starter(None, false, None), Starter::Start);
7333 }
7334
7335 #[test]
7336 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
7337 let mut released = task();
7338 released.start("stalled-run".to_owned());
7339 released.requeue();
7340 let unfinished = (!released.fresh_start)
7341 .then(|| Some("stalled-run".to_owned()))
7342 .flatten();
7343 assert_eq!(
7344 choose_starter(None, false, unfinished.as_deref()),
7345 Starter::Start,
7346 "release keeps run history but must not resume it"
7347 );
7348 assert_eq!(released.runs, ["stalled-run"]);
7349 }
7350
7351 #[test]
7352 fn an_ordinary_release_keeps_a_resumable_run_available() {
7353 let mut released = task();
7354 released.start("stalled-run".to_owned());
7355 released.release();
7356 let unfinished = (!released.fresh_start)
7357 .then(|| Some("stalled-run".to_owned()))
7358 .flatten();
7359 assert_eq!(
7360 choose_starter(None, false, unfinished.as_deref()),
7361 Starter::Resume("stalled-run".to_owned()),
7362 "manual release must preserve the normal resume path"
7363 );
7364 }
7365
7366 #[test]
7367 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
7368 let mut state = run_state(RunStatus::Blocked);
7369 state.config.graph.review_rounds = 3;
7370 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
7371 assert!(exhausted_review_budget(&state));
7372
7373 state.reviews.pop();
7375 assert!(!exhausted_review_budget(&state));
7376
7377 let mut stalled = run_state(RunStatus::Stalled);
7380 stalled.config.graph.review_rounds = 1;
7381 stalled.reviews = vec![review_round(1)];
7382 assert!(!exhausted_review_budget(&stalled));
7383 }
7384
7385 fn review_round(round: usize) -> crate::run::ReviewRound {
7386 crate::run::ReviewRound {
7387 round,
7388 head: "deadbeef".to_owned(),
7389 verified_head: None,
7390 verified_at: None,
7391 reviews: Vec::new(),
7392 e2e: Vec::new(),
7393 verify_retried: false,
7394 e2e_deferred: false,
7395 e2e_defer_reason: None,
7396 fix: None,
7397 blocking: 0,
7398 answered: 1,
7399 expected: 1,
7400 clean: false,
7401 progressed: true,
7402 vote_split: false,
7403 reconsideration: Vec::new(),
7404 verdict: None,
7405 }
7406 }
7407
7408 #[test]
7409 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
7410 let mut run = RunState::new(
7411 PathBuf::from("/repo"),
7412 "main".to_owned(),
7413 "abc1234def".to_owned(),
7414 "add retries".to_owned(),
7415 Config::default(),
7416 );
7417 run.status = RunStatus::Judging;
7418 run.parked = true;
7419 let mut task = Task::new(
7420 "add retries".to_owned(),
7421 "add retries".to_owned(),
7422 PathBuf::from("/repo"),
7423 crate::queue::Source::Human,
7424 );
7425 task.status = TaskStatus::Failed;
7426 task.runs = vec![run.id.clone()];
7427 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
7428 assert!(with(&task, &run), "parked after judging is the case");
7429
7430 let mut not_parked = run.clone();
7431 not_parked.parked = false;
7432 not_parked.status = RunStatus::Stalled;
7433 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
7434
7435 let mut fresh = task.clone();
7436 fresh.fresh_start = true;
7437 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
7438
7439 let mut review = task.clone();
7440 review.review_branch = Some("magi/x/A".to_owned());
7441 assert!(!with(&review, &run), "review is ranked before resume");
7442
7443 let mut held = task.clone();
7444 held.status = TaskStatus::Held;
7445 assert!(!with(&held, &run), "a hold stays visible to the conductor");
7446
7447 let mut released = run.clone();
7448 released.released_to = Some("20260901-000000-new1".to_owned());
7449 assert!(!with(&task, &released), "nothing left to resume into");
7450
7451 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
7452 "unreadable"
7453 )));
7454 }
7455
7456 #[test]
7457 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
7458 let mut released = RunState::new(
7459 PathBuf::from("/repo"),
7460 "main".to_owned(),
7461 "abc1234def".to_owned(),
7462 "add retries".to_owned(),
7463 Config::default(),
7464 );
7465 released.status = RunStatus::Blocked;
7466 assert_eq!(
7467 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
7468 Some(released.id.clone())
7469 );
7470 released.released_to = Some("20260901-000000-new1".to_owned());
7471 assert_eq!(
7472 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
7473 None,
7474 "there is nothing left to resume it into"
7475 );
7476 }
7477
7478 #[test]
7479 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
7480 let mut exhausted = RunState::new(
7489 PathBuf::from("/repo"),
7490 "main".to_owned(),
7491 "abc1234def".to_owned(),
7492 "add retries".to_owned(),
7493 Config::default(),
7494 );
7495 exhausted.status = RunStatus::Blocked;
7496 exhausted.config.graph.review_rounds = 1;
7497 exhausted.reviews = vec![review_round(1)];
7498
7499 assert_eq!(
7500 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
7501 None,
7502 "an exhausted `Blocked` run must not be offered as resumable"
7503 );
7504
7505 let mut has_budget_left = RunState::new(
7508 PathBuf::from("/repo"),
7509 "main".to_owned(),
7510 "abc1234def".to_owned(),
7511 "add retries".to_owned(),
7512 Config::default(),
7513 );
7514 has_budget_left.status = RunStatus::Blocked;
7515 has_budget_left.config.graph.review_rounds = 3;
7516 has_budget_left.reviews = vec![review_round(1)];
7517
7518 assert_eq!(
7519 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
7520 Ok(has_budget_left.clone())
7521 }),
7522 Some(has_budget_left.id.clone())
7523 );
7524 }
7525
7526 #[test]
7527 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
7528 let mut older_stalled = RunState::new(
7536 PathBuf::from("/repo"),
7537 "main".to_owned(),
7538 "abc1234def".to_owned(),
7539 "add retries".to_owned(),
7540 Config::default(),
7541 );
7542 older_stalled.status = RunStatus::Stalled;
7543
7544 let mut newest_exhausted = RunState::new(
7545 PathBuf::from("/repo"),
7546 "main".to_owned(),
7547 "abc1234def".to_owned(),
7548 "add retries".to_owned(),
7549 Config::default(),
7550 );
7551 newest_exhausted.status = RunStatus::Blocked;
7552 newest_exhausted.config.graph.review_rounds = 1;
7553 newest_exhausted.reviews = vec![review_round(1)];
7554
7555 assert_eq!(
7556 unfinished_run_with(
7557 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
7558 "t",
7559 |_| Ok(newest_exhausted.clone())
7560 ),
7561 None,
7562 "the newest run is exhausted, so nothing here is worth resuming - \
7563 least of all the older, already-superseded run"
7564 );
7565 }
7566
7567 #[test]
7568 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
7569 assert_eq!(
7570 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
7571 Err(anyhow::anyhow!("fixture is absent"))
7572 }),
7573 None
7574 );
7575 }
7576
7577 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
7578 let mut q = ask::Question::new(
7579 run.to_owned(),
7580 "implement".to_owned(),
7581 "impl-A".to_owned(),
7582 "continue?".to_owned(),
7583 String::new(),
7584 vec!["resume で続行する".to_owned(), "other".to_owned()],
7585 );
7586 q.actions.insert("resume で続行する".to_owned(), action);
7587 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
7588 .unwrap();
7589 q
7590 }
7591
7592 fn held_task_with(run: &str) -> Task {
7593 let mut t = task();
7594 t.runs = vec![run.to_owned()];
7595 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
7596 t
7597 }
7598
7599 fn resume_action(run: &str) -> ask::ChoiceAction {
7600 ask::ChoiceAction::Resume { run: run.into() }
7601 }
7602
7603 #[test]
7604 fn decide_action_resumes_only_the_latest_resumable_run() {
7605 let t = held_task_with("r1");
7606 let q = action_question("r1", resume_action("r1"));
7607 let load = |s: RunState| move |_: &str| Ok(s);
7608 assert_eq!(
7609 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
7610 ActionDecision::Resume("r1".into())
7611 );
7612 let q_other = action_question("r1", resume_action("r0"));
7614 assert!(matches!(
7615 decide_action(
7616 &t,
7617 &q_other,
7618 &PHRASES_EN,
7619 load(run_state(RunStatus::Blocked))
7620 ),
7621 ActionDecision::Refuse(_)
7622 ));
7623 assert!(matches!(
7625 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
7626 ActionDecision::Refuse(_)
7627 ));
7628 let mut released = run_state(RunStatus::Blocked);
7630 released.released_to = Some("elsewhere".into());
7631 assert!(matches!(
7632 decide_action(&t, &q, &PHRASES_EN, load(released)),
7633 ActionDecision::Refuse(_)
7634 ));
7635 assert!(matches!(
7637 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
7638 ActionDecision::Refuse(_)
7639 ));
7640 }
7641
7642 #[test]
7643 fn decide_action_ignores_a_question_about_an_earlier_run() {
7644 let mut t = held_task_with("r1");
7645 t.runs.push("r2".to_owned());
7646 let never = |_: &str| -> Result<RunState> { bail!("not read") };
7647 assert_eq!(
7648 decide_action(
7649 &t,
7650 &action_question("r1", ask::ChoiceAction::Done),
7651 &PHRASES_EN,
7652 never
7653 ),
7654 ActionDecision::Stale
7655 );
7656 }
7657
7658 #[test]
7659 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
7660 let mut t = held_task_with("r1");
7661 let never = |_: &str| -> Result<RunState> { bail!("not read") };
7662 assert_eq!(
7663 decide_action(
7664 &t,
7665 &action_question("r1", ask::ChoiceAction::Requeue),
7666 &PHRASES_EN,
7667 never
7668 ),
7669 ActionDecision::Requeue
7670 );
7671 let done_q = action_question("r1", ask::ChoiceAction::Done);
7672 assert_eq!(
7673 decide_action(&t, &done_q, &PHRASES_EN, never),
7674 ActionDecision::Done
7675 );
7676 t.mark_action_applied(&done_q.id);
7677 assert_eq!(
7678 decide_action(&t, &done_q, &PHRASES_EN, never),
7679 ActionDecision::Skip
7680 );
7681
7682 let mut plain = action_question("r1", ask::ChoiceAction::Done);
7684 plain.actions.clear();
7685 assert_eq!(
7686 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
7687 ActionDecision::Skip
7688 );
7689 let mut running = held_task_with("r1");
7691 running.status = TaskStatus::Running;
7692 assert_eq!(
7693 decide_action(
7694 &running,
7695 &action_question("r1", ask::ChoiceAction::Done),
7696 &PHRASES_EN,
7697 never
7698 ),
7699 ActionDecision::Skip
7700 );
7701 }
7702
7703 #[test]
7704 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
7705 let dir = tempfile::tempdir().unwrap();
7706 let queue = Queue::at(dir.path().join("queue"));
7707 let questions = Questions::at(dir.path().join("questions"));
7708 let home = dir.path().join("home");
7709 let mut state = run_state(RunStatus::Blocked);
7710 state.id = "20260101-000000-act1".to_owned();
7711 state.save_under(&home).unwrap();
7712
7713 let mut t = held_task_with(&state.id);
7714 queue.put(&mut t).unwrap();
7715 let mut q = action_question(&state.id, resume_action(&state.id));
7716 questions.put(&mut q).unwrap();
7717
7718 apply_choice_actions(&queue, &questions, &home);
7719 let after = queue.get(&t.id).unwrap();
7720 assert_eq!(after.status, TaskStatus::Queued);
7721 assert!(!after.fresh_start);
7722 assert!(after.action_applied(&q.id));
7723 let pin = after.resume_override.clone().unwrap();
7724 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
7725 assert!(pin.forced);
7726
7727 let mut again = queue.get(&t.id).unwrap();
7729 again.hold_machine(Some("later".into()));
7730 queue.put(&mut again).unwrap();
7731 apply_choice_actions(&queue, &questions, &home);
7732 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
7733 }
7734
7735 #[test]
7736 fn an_answer_the_waiter_already_delivered_is_not_acted_on_again() {
7737 let dir = tempfile::tempdir().unwrap();
7738 let queue = Queue::at(dir.path().join("queue"));
7739 let questions = Questions::at(dir.path().join("questions"));
7740 let home = dir.path().join("home");
7741 let mut state = run_state(RunStatus::Blocked);
7742 state.id = "20260101-000000-act2".to_owned();
7743 state.save_under(&home).unwrap();
7744
7745 let mut t = held_task_with(&state.id);
7746 queue.put(&mut t).unwrap();
7747 let mut q = action_question(&state.id, resume_action(&state.id));
7748 q.answer_delivered = true;
7749 questions.put(&mut q).unwrap();
7750
7751 apply_choice_actions(&queue, &questions, &home);
7752 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
7753
7754 let mut q2 = action_question(&state.id, resume_action(&state.id));
7756 questions.put(&mut q2).unwrap();
7757 apply_choice_actions(&queue, &questions, &home);
7758 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
7759 assert!(questions.get(&q2.id).unwrap().answer_delivered);
7760 }
7761}