1use std::collections::HashMap;
34use std::num::NonZeroUsize;
35use std::path::{Path, PathBuf};
36
37use anyhow::{Context, Result, bail};
38use jiff::Timestamp;
39use serde::{Deserialize, Serialize};
40
41use crate::ask::Questions;
42
43pub const SCHEMA: u32 = 19;
133
134pub(crate) fn dispatch_order(a: &Task, b: &Task) -> std::cmp::Ordering {
140 match (a.upgrade_resume_key(), b.upgrade_resume_key()) {
141 (Some(a), Some(b)) => a.cmp(&b),
142 (Some(_), None) => std::cmp::Ordering::Less,
143 (None, Some(_)) => std::cmp::Ordering::Greater,
144 (None, None) => b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)),
145 }
146}
147
148#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
150#[serde(rename_all = "lowercase")]
151pub enum HoldSource {
152 Manual,
154 Machine,
156}
157
158impl HoldSource {
159 pub fn label(self) -> &'static str {
161 match self {
162 Self::Manual => "manual",
163 Self::Machine => "machine",
164 }
165 }
166}
167
168#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
171#[serde(tag = "kind", rename_all = "lowercase")]
172pub enum Source {
173 Human,
175 Agent {
178 run: String,
180 node: String,
182 },
183 Issue {
185 number: u64,
187 repo: String,
189 },
190}
191
192pub const CHAT_NODE: &str = "chat";
195
196impl Source {
197 pub fn label(&self) -> String {
199 match self {
200 Self::Human => "human".to_owned(),
201 Self::Agent { run, node } => format!("{node}@{}", short(run)),
202 Self::Issue { number, .. } => format!("issue #{number}"),
203 }
204 }
205}
206
207#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
209#[serde(rename_all = "lowercase")]
210pub enum TaskStatus {
211 Queued,
213 Running,
215 Done,
217 Failed,
219 Held,
221 Blocked,
225 Parked,
230}
231
232impl TaskStatus {
233 pub fn runnable(self) -> bool {
235 matches!(self, Self::Queued | Self::Failed | Self::Parked)
236 }
237
238 pub fn as_str(self) -> &'static str {
240 match self {
241 Self::Queued => "queued",
242 Self::Running => "running",
243 Self::Done => "done",
244 Self::Failed => "failed",
245 Self::Held => "held",
246 Self::Blocked => "blocked",
247 Self::Parked => "parked",
248 }
249 }
250}
251
252#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
255pub struct TaskCounts {
256 pub queued: usize,
258 pub running: usize,
260 pub done: usize,
262 pub failed: usize,
264 pub held: usize,
266 pub blocked: usize,
268 pub parked: usize,
270}
271
272impl TaskCounts {
273 pub fn of(tasks: &[Task]) -> Self {
277 Self::of_statuses(tasks.iter().map(|task| task.status))
278 }
279
280 pub(crate) fn of_statuses(statuses: impl IntoIterator<Item = TaskStatus>) -> Self {
282 let mut counts = Self::default();
283 for status in statuses {
284 match status {
285 TaskStatus::Queued => counts.queued += 1,
286 TaskStatus::Running => counts.running += 1,
287 TaskStatus::Done => counts.done += 1,
288 TaskStatus::Failed => counts.failed += 1,
289 TaskStatus::Held => counts.held += 1,
290 TaskStatus::Blocked => counts.blocked += 1,
291 TaskStatus::Parked => counts.parked += 1,
292 }
293 }
294 counts
295 }
296}
297
298#[derive(Debug, Clone, Serialize, Deserialize)]
300#[serde(deny_unknown_fields)]
301pub struct Task {
302 pub schema: u32,
304 pub id: String,
306 pub title: String,
308 pub instruction: String,
310 #[serde(default, skip_serializing_if = "crate::refs::Sources::is_empty")]
312 pub sources: crate::refs::Sources,
313 pub repo: PathBuf,
315 pub source: Source,
317 #[serde(default)]
319 pub priority: i32,
320 #[serde(default)]
331 pub solo: bool,
332 pub status: TaskStatus,
334 #[serde(default)]
336 pub attempts: usize,
337 #[serde(default)]
339 pub attempt_limit: Option<NonZeroUsize>,
340 #[serde(default)]
343 pub runs: Vec<String>,
344 #[serde(default)]
346 pub last_error: Option<String>,
347 #[serde(default)]
360 pub hold_reason: Option<String>,
361 #[serde(default)]
364 pub hold_source: Option<HoldSource>,
365 #[serde(default)]
377 pub diagnostic: Option<String>,
378 #[serde(default)]
388 pub blocked_by: Vec<String>,
389 #[serde(default)]
392 pub block_reason: Option<String>,
393 #[serde(default)]
406 pub blocked_from: Option<TaskStatus>,
407 #[serde(default)]
409 pub blocked_at: Option<Timestamp>,
410 #[serde(default)]
421 pub answers: Vec<AnsweredQuestion>,
422 #[serde(default)]
428 pub triage_applied: Vec<String>,
429 #[serde(default)]
435 pub actions_applied: Vec<String>,
436 #[serde(default)]
441 pub resume_override: Option<OperatorResume>,
442 #[serde(default)]
445 pub resumption: Option<Resumption>,
446 #[serde(default)]
448 pub resumption_history: Vec<Resumption>,
449 #[serde(default)]
451 pub operator_decided_at: Option<Timestamp>,
452 #[serde(default)]
464 pub review_branch: Option<String>,
465 #[serde(default)]
467 pub review_instruction: Option<String>,
468 #[serde(default)]
470 pub pr_recovery: Option<crate::pr_recovery::Selection>,
471 #[serde(default)]
474 pub fresh_start: bool,
475 #[serde(default)]
477 pub replacement: Option<crate::merge_decision::Replacement>,
478 #[serde(default)]
489 pub interrupt: bool,
490 #[serde(default)]
513 pub urgent: bool,
514 #[serde(skip)]
519 pub(crate) unfiled: bool,
520 #[serde(default)]
526 pub attachments: Vec<String>,
527 #[serde(default)]
531 pub followup: Option<FollowUp>,
532 #[serde(default)]
538 pub origin_chat: Option<String>,
539 #[serde(default)]
546 pub overrides: Option<RunOverrides>,
547 #[serde(default)]
554 pub review_of: Option<String>,
555 #[serde(default)]
560 pub held_at: Option<Timestamp>,
561 #[serde(default)]
565 pub park_reason: Option<String>,
566 #[serde(default)]
569 pub chat_notice: Option<ChatNotice>,
570 #[serde(default)]
572 pub chat_notices: Vec<String>,
573 #[serde(default)]
576 pub hold_notice_cause: Option<NoticeCause>,
577 #[serde(default)]
582 pub passes: Vec<TaskPass>,
583 #[serde(default)]
587 pub dispatched_at: Option<Timestamp>,
588 #[serde(default)]
590 pub recovery_budget: Option<RecoveryBudget>,
591 #[serde(default)]
594 pub recovery_admission: Option<RecoveryAdmission>,
595 #[serde(default)]
598 pub recovery_generations: Vec<RecoveryBudget>,
599 pub created_at: Timestamp,
601 pub updated_at: Timestamp,
603}
604
605#[derive(Debug, Clone, Serialize, Deserialize)]
607pub struct RecoveryAdmission {
608 pub base: String,
610 pub inputs: String,
612}
613
614#[derive(Debug, Clone, Serialize, Deserialize)]
617pub struct RecoveryBudget {
618 pub instruction: String,
620 pub base: String,
622 pub inputs: String,
624 pub stage: String,
626 pub failures: usize,
628 #[serde(default)]
631 pub progress: Option<String>,
632}
633
634fn recovery_progress(state: &crate::run::RunState) -> Option<String> {
635 use crate::proc::Quiet as _;
636 use std::hash::{Hash, Hasher};
637 let mut patches = Vec::new();
638 for candidate in state.candidates.iter().filter(|c| c.commits > 0) {
639 let output = std::process::Command::new("git")
640 .current_dir(&state.repo)
641 .args(["diff", "--binary", &state.base_commit, &candidate.branch])
642 .quiet()
643 .output()
644 .ok()?;
645 if !output.status.success() {
646 return None;
647 }
648 patches.push(output.stdout);
649 }
650 patches.sort();
651 patches.dedup();
652 let mut hash = std::collections::hash_map::DefaultHasher::new();
653 patches.hash(&mut hash);
654 Some(format!("{:x}", hash.finish()))
655}
656
657pub(crate) fn recovery_inputs(sources: &crate::refs::Sources) -> String {
661 serde_json::to_string(
662 &sources
663 .start_from
664 .iter()
665 .map(|i| ("start", &i.token, &i.sha))
666 .chain(
667 sources
668 .pick_commits
669 .iter()
670 .map(|i| ("pick", &i.token, &i.sha)),
671 )
672 .collect::<Vec<_>>(),
673 )
674 .unwrap_or_default()
675}
676
677#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
679pub struct NoticeCause {
680 pub stage: String,
682 #[serde(default, skip_serializing_if = "Option::is_none")]
685 pub target: Option<String>,
686 pub run: Option<String>,
688 pub pass: Option<usize>,
690}
691
692pub(crate) fn is_done_notice_key(key: &str) -> bool {
694 key == "done"
695 || key
696 .strip_prefix("done#")
697 .is_some_and(|count| !count.is_empty() && count.bytes().all(|b| b.is_ascii_digit()))
698}
699
700#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
702#[serde(rename_all = "snake_case")]
703pub enum PassCost {
704 Spent,
706 Refunded,
708}
709
710#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
712pub struct TaskPass {
713 pub run: String,
715 pub pass_index: usize,
717 pub cost: PassCost,
719 pub attempts_after: usize,
721 #[serde(default)]
723 pub started_at: Option<Timestamp>,
724 pub settled_at: Timestamp,
726 #[serde(default)]
728 pub parked: Option<crate::run::ParkCause>,
729}
730
731#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
733#[serde(rename_all = "lowercase")]
734pub enum ChatNoticeOutcome {
735 Posted,
737 Skipped,
739}
740
741#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
743pub struct ChatNotice {
744 pub key: String,
746 pub at: Timestamp,
748 pub outcome: ChatNoticeOutcome,
750}
751
752#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
754pub struct RunOverrides {
755 #[serde(default)]
758 pub inherit_daemon_options: bool,
759 #[serde(default)]
761 pub merge: Option<String>,
762 #[serde(default)]
764 pub candidates: Option<usize>,
765 #[serde(default)]
767 pub judges: Option<usize>,
768 #[serde(default)]
770 pub reviewers: Option<usize>,
771 #[serde(default)]
773 pub review_rounds: Option<usize>,
774 #[serde(default)]
776 pub seed: Option<u64>,
777 #[serde(default)]
779 pub config: Option<PathBuf>,
780}
781
782impl RunOverrides {
783 pub fn merge_over(&mut self, other: &RunOverrides) {
785 macro_rules! take {
786 ($($f:ident),*) => {$(
787 if other.$f.is_some() {
788 self.$f.clone_from(&other.$f);
789 }
790 )*};
791 }
792 take!(
793 merge,
794 candidates,
795 judges,
796 reviewers,
797 review_rounds,
798 seed,
799 config
800 );
801 }
802
803 pub fn apply(&self, config: &mut crate::config::Config) {
805 if let Some(n) = self.candidates {
806 config.graph.implementers = n;
807 }
808 if let Some(n) = self.judges {
809 config.graph.judges = n;
810 }
811 if let Some(n) = self.reviewers {
812 config.graph.reviewers = n;
813 }
814 if let Some(n) = self.review_rounds {
815 config.graph.review_rounds = n;
816 }
817 if let Some(m) = &self.merge
818 && let Ok(mode) = crate::daemon::merge_mode(m)
819 {
820 config.merge.mode = mode;
821 }
822 if let Some(s) = self.seed {
823 config.blind.seed = Some(s);
824 }
825 }
826}
827
828#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
830pub struct FollowUp {
831 pub run: String,
833 #[serde(default)]
835 pub origin_task: Option<String>,
836 pub pr: String,
838 pub findings: Vec<String>,
840 pub generation: u32,
843}
844
845#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
847pub struct Resumption {
848 pub generation: u64,
850 pub source: String,
852 pub question_id: Option<String>,
854 pub at: Timestamp,
856 pub instruction: String,
858 pub answers_through: usize,
860 pub runs_through: usize,
863 pub cleared_reason: Option<String>,
865 #[serde(default)]
867 pub blocking_evidence: Vec<String>,
868}
869
870#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
873pub struct OperatorResume {
874 pub question_id: String,
876 pub at: Timestamp,
878 #[serde(default)]
881 pub conductor_rehold: Option<String>,
882 #[serde(default)]
885 pub forced: bool,
886 #[serde(default)]
893 pub pinned_run: Option<String>,
894}
895
896#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
899pub struct AnsweredQuestion {
900 pub question: String,
902 pub answer: String,
904}
905
906impl Task {
907 pub fn new(title: String, instruction: String, repo: PathBuf, source: Source) -> Self {
909 let now = Timestamp::now();
910 let origin_chat = match &source {
911 Source::Agent { run, node } if node == CHAT_NODE => Some(run.clone()),
912 _ => None,
913 };
914 Self {
915 schema: SCHEMA,
916 id: new_id(),
917 title,
918 instruction,
919 sources: crate::refs::Sources::default(),
920 repo,
921 source,
922 priority: 0,
923 solo: false,
924 status: TaskStatus::Queued,
925 attempts: 0,
926 attempt_limit: None,
927 runs: Vec::new(),
928 last_error: None,
929 hold_reason: None,
930 hold_source: None,
931 diagnostic: None,
932 blocked_by: Vec::new(),
933 block_reason: None,
934 blocked_from: None,
935 blocked_at: None,
936 answers: Vec::new(),
937 triage_applied: Vec::new(),
938 actions_applied: Vec::new(),
939 resume_override: None,
940 resumption: None,
941 resumption_history: Vec::new(),
942 operator_decided_at: None,
943 review_branch: None,
944 review_instruction: None,
945 pr_recovery: None,
946 fresh_start: false,
947 replacement: None,
948 interrupt: false,
949 urgent: false,
950 unfiled: true,
951 attachments: Vec::new(),
952 followup: None,
953 origin_chat,
954 overrides: None,
955 review_of: None,
956 held_at: None,
957 park_reason: None,
958 chat_notice: None,
959 chat_notices: Vec::new(),
960 hold_notice_cause: None,
961 passes: Vec::new(),
962 dispatched_at: None,
963 recovery_budget: None,
964 recovery_admission: None,
965 recovery_generations: Vec::new(),
966 created_at: now,
967 updated_at: now,
968 }
969 }
970
971 pub fn chat_talk(&self) -> Option<&str> {
974 if let Some(id) = &self.origin_chat {
975 return Some(id);
976 }
977 match &self.source {
978 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
979 _ => None,
980 }
981 }
982
983 pub fn filed_by_chat(&self) -> Option<&str> {
987 match &self.source {
988 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
989 _ => None,
990 }
991 }
992
993 pub fn chat_notice_key(&self) -> Option<String> {
996 match self.status {
997 TaskStatus::Done => Some("done".to_owned()),
998 TaskStatus::Held => {
999 let cause = self
1000 .hold_notice_cause
1001 .clone()
1002 .unwrap_or_else(|| self.unresolved_hold_cause());
1003 Some(format!(
1004 "cause:{}",
1005 serde_json::to_string(&cause).expect("cause")
1006 ))
1007 }
1008 TaskStatus::Blocked => {
1009 let ids: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1010 let evidence = self
1013 .hold_notice_cause
1014 .as_ref()
1015 .filter(|c| c.stage.starts_with("conductor-evidence:"));
1016 Some(format!(
1017 "dependencies:{}",
1018 serde_json::to_string(&(ids, evidence)).expect("ids")
1019 ))
1020 }
1021 _ => None,
1022 }
1023 }
1024
1025 fn unresolved_hold_cause(&self) -> NoticeCause {
1026 NoticeCause {
1027 stage: "hold".into(),
1028 target: None,
1029 run: self.runs.last().cloned(),
1030 pass: None,
1031 }
1032 }
1033
1034 pub(crate) fn legacy_chat_notice_key(&self) -> Option<String> {
1036 let key = match self.status {
1037 TaskStatus::Held if self.hold_notice_cause.is_none() => Some(format!(
1038 "held#{}",
1039 self.held_at.map(|t| t.to_string()).unwrap_or_default()
1040 )),
1041 TaskStatus::Blocked => Some(format!(
1042 "blocked#{}#{}",
1043 self.blocked_by.join(","),
1044 self.block_reason.as_deref().unwrap_or_default()
1045 )),
1046 _ => None,
1047 };
1048 key.filter(|key| self.legacy_notice_matches(key))
1049 }
1050
1051 pub(crate) fn legacy_notice_matches(&self, key: &str) -> bool {
1052 match self.status {
1053 TaskStatus::Held if self.hold_notice_cause.is_none() => key
1054 .strip_prefix("held#")
1055 .is_some_and(|time| time.is_empty() || time.parse::<Timestamp>().is_ok()),
1056 TaskStatus::Blocked
1057 if self
1058 .hold_notice_cause
1059 .as_ref()
1060 .is_none_or(|c| !c.stage.starts_with("conductor-evidence:")) =>
1061 {
1062 key.strip_prefix("blocked#").is_some_and(|tail| {
1063 let Some((ids, _)) = tail.split_once('#') else {
1064 return false;
1065 };
1066 let old: std::collections::BTreeSet<_> =
1067 ids.split(',').filter(|id| !id.is_empty()).collect();
1068 let current: std::collections::BTreeSet<_> =
1069 self.blocked_by.iter().map(String::as_str).collect();
1070 old == current
1071 })
1072 }
1073 _ => false,
1074 }
1075 }
1076
1077 fn import_chat_notice(&mut self) {
1078 if let Some(n) = &self.chat_notice {
1079 let key = if is_done_notice_key(&n.key) {
1080 "done".to_owned()
1081 } else if self.legacy_notice_matches(&n.key) {
1082 self.chat_notice_key().unwrap_or_else(|| n.key.clone())
1083 } else {
1084 n.key.clone()
1085 };
1086 if !self.chat_notices.contains(&key) {
1087 self.chat_notices.push(key);
1088 }
1089 }
1090 }
1091
1092 pub(crate) fn notice_cause(&mut self, stage: &str, run: Option<&str>, pass: Option<usize>) {
1095 self.import_chat_notice();
1096 self.hold_notice_cause = Some(NoticeCause {
1097 stage: stage.to_owned(),
1098 target: None,
1099 run: run.map(str::to_owned),
1100 pass,
1101 });
1102 }
1103
1104 pub(crate) fn startup_notice_cause(&mut self, stage: &str) {
1107 let recovery_only = matches!(
1108 stage,
1109 "startup-source" | "startup-candidates" | "startup-public-head"
1110 );
1111 match self.hold_notice_cause.as_ref() {
1112 Some(cause) if recovery_only && !cause.stage.starts_with("startup-") => return,
1113 None if recovery_only
1114 && self
1115 .chat_notice
1116 .as_ref()
1117 .is_some_and(|n| !is_done_notice_key(&n.key)) =>
1118 {
1119 return;
1120 }
1121 _ => {}
1122 }
1123 self.notice_cause(stage, None, None);
1124 }
1125
1126 pub(crate) fn fail_startup(
1129 &mut self,
1130 stage: &str,
1131 why: impl Into<String>,
1132 max_attempts: usize,
1133 ) {
1134 let unresolved = self.hold_notice_cause.clone();
1135 self.fail(why, max_attempts);
1136 self.hold_notice_cause = unresolved;
1137 self.startup_notice_cause(stage);
1138 }
1139
1140 pub(crate) fn fail_startup_target(
1142 &mut self,
1143 stage: &str,
1144 target: String,
1145 why: impl Into<String>,
1146 max_attempts: usize,
1147 ) {
1148 self.fail_startup(stage, why, max_attempts);
1149 if let Some(cause) = self.hold_notice_cause.as_mut() {
1150 cause.target = Some(target);
1151 }
1152 }
1153
1154 pub fn chat_notice_due(&self) -> Option<String> {
1156 self.filed_by_chat()?;
1157 if self.schema < 13 {
1158 return None;
1159 }
1160 let key = self.chat_notice_key()?;
1161 if self.chat_notices.contains(&key) {
1162 return None;
1163 }
1164 match &self.chat_notice {
1165 Some(n)
1166 if n.key == key
1167 || (key == "done" && is_done_notice_key(&n.key))
1168 || self.legacy_notice_matches(&n.key) =>
1169 {
1170 None
1171 }
1172 _ => Some(key),
1173 }
1174 }
1175
1176 pub fn short(&self) -> &str {
1178 short(&self.id)
1179 }
1180
1181 pub fn mark_triage_applied(&mut self, question_id: &str) {
1183 if !self.triage_applied(question_id) {
1184 self.triage_applied.push(question_id.to_owned());
1185 }
1186 }
1187
1188 pub fn action_applied(&self, question_id: &str) -> bool {
1191 self.actions_applied.iter().any(|id| id == question_id)
1192 }
1193
1194 pub fn mark_action_applied(&mut self, question_id: &str) {
1196 if !self.action_applied(question_id) {
1197 self.actions_applied.push(question_id.to_owned());
1198 }
1199 }
1200
1201 pub fn triage_applied(&self, question_id: &str) -> bool {
1203 self.triage_applied.iter().any(|id| id == question_id)
1204 }
1205
1206 pub fn start(&mut self, run: String) {
1219 self.stamp_schema();
1220 self.status = TaskStatus::Running;
1221 self.attempts += 1;
1222 self.runs.retain(|id| id != &run);
1223 self.runs.push(run);
1224 self.last_error = None;
1225 self.park_reason = None;
1226 self.fresh_start = false;
1227 self.interrupt = false;
1228 self.resume_override = None;
1230 }
1231
1232 pub fn link_run(&mut self, run: &str) -> bool {
1237 if self.runs.iter().any(|r| r == run) {
1238 return false;
1239 }
1240 self.runs.push(run.to_owned());
1241 true
1242 }
1243
1244 pub fn succeed(&mut self) {
1254 self.stamp_schema();
1255 self.status = TaskStatus::Done;
1256 self.resume_override = None;
1257 self.last_error = None;
1258 self.park_reason = None;
1259 self.hold_reason = None;
1260 self.hold_source = None;
1261 self.diagnostic = None;
1262 self.blocked_by.clear();
1263 self.block_reason = None;
1264 self.blocked_from = None;
1265 }
1266
1267 pub fn already_landed(&mut self, note: impl Into<String>) {
1275 self.succeed();
1276 self.attempts = self.attempts.saturating_sub(1);
1277 self.last_error = Some(note.into());
1278 }
1279
1280 pub fn superseded_attempts(&self, last_run_succeeded: bool) -> &[String] {
1310 if self.status != TaskStatus::Done || !last_run_succeeded || self.runs.len() < 2 {
1311 return &[];
1312 }
1313 &self.runs[..self.runs.len() - 1]
1314 }
1315
1316 pub fn successor_of(&self, run: &str) -> Option<&String> {
1324 let pos = self.runs.iter().rposition(|r| r == run)?;
1325 self.runs.get(pos + 1)
1326 }
1327
1328 pub fn earlier_attempts(&self) -> &[String] {
1335 &self.runs
1336 }
1337
1338 fn stamp_schema(&mut self) {
1345 self.schema = self.schema.max(SCHEMA);
1346 }
1347
1348 fn note_held(&mut self) {
1349 self.stamp_schema();
1350 let continuing = self.status == TaskStatus::Held
1351 || (self.status == TaskStatus::Blocked
1352 && (self.blocked_from == Some(TaskStatus::Held)
1353 || (self.blocked_from.is_none()
1354 && (self.hold_reason.is_some() || self.hold_source.is_some()))));
1355 if !continuing || self.held_at.is_none() {
1356 self.held_at = Some(Timestamp::now());
1357 }
1358 }
1359
1360 pub fn fail(&mut self, why: impl Into<String>, max_attempts: usize) {
1376 let why = why.into();
1377 if self.status == TaskStatus::Running {
1378 let run = self.runs.last().cloned();
1379 self.notice_cause("run-failed", run.as_deref(), Some(self.attempts));
1380 }
1381 self.diagnostic = None;
1382 self.park_reason = None;
1383 self.status = if self.attempts >= self.effective_attempt_limit(max_attempts) {
1384 self.note_held();
1385 self.hold_source = Some(HoldSource::Machine);
1386 self.hold_reason = Some(why.clone());
1387 TaskStatus::Held
1388 } else {
1389 TaskStatus::Failed
1390 };
1391 self.last_error = Some(why);
1392 }
1393
1394 pub fn stall(&mut self, why: impl Into<String>) {
1404 self.last_error = Some(why.into());
1405 self.diagnostic = None;
1406 self.attempts = self.attempts.saturating_sub(1);
1407 self.status = TaskStatus::Failed;
1408 }
1409
1410 pub fn park(&mut self, why: impl Into<String>) {
1416 self.diagnostic = None;
1417 self.attempts = self.attempts.saturating_sub(1);
1418 self.status = TaskStatus::Parked;
1419 self.park_reason = Some(why.into());
1420 self.schema = self.schema.max(SCHEMA);
1423 }
1424
1425 pub fn record_pass(
1427 &mut self,
1428 run: &str,
1429 pass_index: usize,
1430 attempts_before: usize,
1431 parked: Option<crate::run::ParkCause>,
1432 ) {
1433 self.record_pass_at(run, pass_index, None, attempts_before, parked);
1434 }
1435
1436 pub fn record_pass_at(
1439 &mut self,
1440 run: &str,
1441 pass_index: usize,
1442 started_at: Option<Timestamp>,
1443 attempts_before: usize,
1444 parked: Option<crate::run::ParkCause>,
1445 ) {
1446 if self
1447 .passes
1448 .iter()
1449 .any(|p| p.run == run && p.pass_index == pass_index)
1450 {
1451 return;
1452 }
1453 let cost = if self.attempts < attempts_before {
1454 PassCost::Refunded
1455 } else {
1456 PassCost::Spent
1457 };
1458 self.passes.push(TaskPass {
1459 run: run.to_owned(),
1460 pass_index,
1461 started_at: self.dispatched_at.or(started_at),
1462 cost,
1463 attempts_after: self.attempts,
1464 settled_at: Timestamp::now(),
1465 parked,
1466 });
1467 self.schema = self.schema.max(SCHEMA);
1468 }
1469
1470 pub(crate) fn upgrade_resume_key(&self) -> Option<(Timestamp, &str)> {
1473 if self.status != TaskStatus::Parked || self.fresh_start {
1474 return None;
1475 }
1476 let pass = self.passes.last()?;
1477 if self.runs.last() != Some(&pass.run)
1478 || !matches!(pass.parked, Some(crate::run::ParkCause::Upgrade { .. }))
1479 {
1480 return None;
1481 }
1482 Some((pass.started_at.unwrap_or(pass.settled_at), &self.id))
1485 }
1486
1487 pub fn operator_held(&self) -> bool {
1493 self.status == TaskStatus::Held && !matches!(self.hold_source, Some(HoldSource::Machine))
1494 }
1495
1496 pub fn hold_manual(&mut self, reason: Option<String>) {
1506 self.operator_decided_at = Some(Timestamp::now());
1508 self.hold_unrequested_retry(reason);
1509 }
1510
1511 pub(crate) fn hold_unrequested_retry(&mut self, reason: Option<String>) {
1514 self.park_reason = None;
1515 self.note_held();
1516 self.status = TaskStatus::Held;
1517 if reason.is_some() {
1518 self.hold_reason = reason;
1519 }
1520 self.hold_source = Some(HoldSource::Manual);
1521 self.blocked_by.clear();
1522 self.block_reason = None;
1523 self.blocked_from = None;
1524 }
1525
1526 pub fn hold_machine(&mut self, reason: Option<String>) {
1531 self.park_reason = None;
1532 self.note_held();
1533 self.status = TaskStatus::Held;
1534 if reason.is_some() {
1535 self.hold_reason = reason;
1536 }
1537 self.hold_source = Some(HoldSource::Machine);
1538 self.blocked_by.clear();
1539 self.block_reason = None;
1540 self.blocked_from = None;
1541 }
1542
1543 pub fn block(&mut self, blocked_by: Vec<String>, reason: Option<String>) {
1553 self.stamp_schema();
1554 let before: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1555 let after: std::collections::BTreeSet<_> = blocked_by.iter().collect();
1556 if self.status != TaskStatus::Blocked || before != after {
1557 self.blocked_at = Some(Timestamp::now());
1558 }
1559 if self.status != TaskStatus::Blocked {
1560 self.blocked_from = Some(self.status);
1561 }
1562 self.status = TaskStatus::Blocked;
1563 self.blocked_by = blocked_by;
1564 self.block_reason = reason;
1565 }
1566
1567 pub fn unblock(&mut self, resolved_id: &str) {
1590 if self.status != TaskStatus::Blocked {
1591 return;
1592 }
1593 self.blocked_by.retain(|id| id != resolved_id);
1594 self.restore_if_unblocked();
1595 }
1596
1597 pub fn dependency_deleted(&mut self, deleted_id: &str) -> bool {
1608 if self.status != TaskStatus::Blocked || !self.blocked_by.iter().any(|b| b == deleted_id) {
1609 return false;
1610 }
1611 self.blocked_by.retain(|id| id != deleted_id);
1612 self.restore_if_unblocked();
1613 true
1614 }
1615
1616 fn restore_if_unblocked(&mut self) {
1620 if self.blocked_by.is_empty() {
1621 self.status = match self.blocked_from {
1622 Some(TaskStatus::Running) => TaskStatus::Queued,
1623 Some(other) => other,
1624 None if self.hold_reason.is_some() || self.hold_source.is_some() => {
1625 TaskStatus::Held
1626 }
1627 None => TaskStatus::Queued,
1628 };
1629 self.block_reason = None;
1630 self.blocked_from = None;
1631 }
1632 }
1633
1634 pub fn record_answer(&mut self, question: String, answer: String) {
1639 self.answers.push(AnsweredQuestion { question, answer });
1640 }
1641
1642 pub fn request_review(&mut self, branch: String) {
1646 self.release();
1647 self.review_branch = Some(branch);
1648 self.review_instruction = Some(self.instruction.clone());
1649 }
1650
1651 pub fn requeue(&mut self) {
1654 self.release();
1655 self.review_branch = None;
1657 self.review_instruction = None;
1658 self.fresh_start = true;
1659 }
1660
1661 pub fn hold_for_handover(&mut self, branch: Option<String>, reason: String) {
1670 if branch.is_some() {
1671 self.review_branch = branch;
1672 self.review_instruction = Some(self.instruction.clone());
1673 }
1674 self.hold_machine(Some(reason));
1675 }
1676
1677 pub fn set_urgent(&mut self, urgent: bool) -> Result<()> {
1679 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1680 bail!(
1681 "task {} is {}; urgent can only change while queued or held; \
1682 wait for the active attempt to finish and requeue or hold the task first",
1683 self.short(),
1684 self.status.as_str()
1685 );
1686 }
1687 self.urgent = urgent;
1688 Ok(())
1689 }
1690
1691 pub fn set_priority(&mut self, priority: i32) -> Result<()> {
1700 if self.status == TaskStatus::Running {
1701 bail!(
1702 "task {} is running; its priority cannot be changed until \
1703 this attempt finishes",
1704 self.short()
1705 );
1706 }
1707 self.priority = priority;
1708 Ok(())
1709 }
1710
1711 pub fn set_interrupt(&mut self, interrupt: bool) -> Result<()> {
1726 if interrupt && !self.status.runnable() {
1727 bail!(
1728 "task {} is {}; only a queued or failed task can be marked \
1729 to interrupt",
1730 self.short(),
1731 self.status.as_str()
1732 );
1733 }
1734 self.interrupt = interrupt;
1735 Ok(())
1736 }
1737
1738 pub fn effective_attempt_limit(&self, daemon_limit: usize) -> usize {
1740 self.attempt_limit.map_or(daemon_limit, NonZeroUsize::get)
1741 }
1742
1743 pub fn change_attempt_limit(&mut self, limit: Option<NonZeroUsize>) -> Result<()> {
1746 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1747 bail!("only queued or held tasks can change the attempt limit");
1748 }
1749 self.stamp_schema();
1750 self.attempt_limit = limit;
1751 Ok(())
1752 }
1753
1754 pub fn implementers(&self) -> Option<usize> {
1756 if self.solo {
1757 Some(1)
1758 } else {
1759 self.overrides.as_ref().and_then(|o| o.candidates)
1760 }
1761 }
1762
1763 pub fn change_implementers(&mut self, count: u8) -> Result<()> {
1766 self.check_implementers_editable()?;
1767 let explicit = self.overrides.as_ref().and_then(|o| o.config.as_deref());
1768 let (mut config, _) = crate::config::Config::discover(&self.repo, explicit)?;
1769 if let Some(o) = &self.overrides {
1770 o.apply(&mut config);
1771 }
1772 self.set_implementers(count, &config)
1773 }
1774
1775 fn check_implementers_editable(&self) -> Result<()> {
1776 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1777 bail!(
1778 "task {} is {}; only queued or held tasks can change implementers",
1779 self.short(),
1780 self.status.as_str()
1781 );
1782 }
1783 Ok(())
1784 }
1785
1786 pub fn set_implementers(&mut self, count: u8, config: &crate::config::Config) -> Result<()> {
1788 self.check_implementers_editable()?;
1789 crate::talk::check_implementers(count, config).map_err(anyhow::Error::msg)?;
1790 self.stamp_schema();
1791 self.solo = count == 1;
1792 let mut overrides = self.overrides.clone().unwrap_or_else(|| RunOverrides {
1793 inherit_daemon_options: true,
1794 ..Default::default()
1795 });
1796 overrides.candidates = (count > 1).then_some(usize::from(count));
1797 let mut settings = overrides.clone();
1798 settings.inherit_daemon_options = false;
1799 self.overrides = if overrides.inherit_daemon_options && settings == RunOverrides::default()
1801 {
1802 None
1803 } else {
1804 Some(overrides)
1805 };
1806 Ok(())
1807 }
1808
1809 pub fn edit(&mut self, title: String, instruction: String) -> Result<()> {
1821 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1822 bail!(
1823 "task {} is {}; only a queued or held task's instruction can \
1824 be edited",
1825 self.short(),
1826 self.status.as_str()
1827 );
1828 }
1829 if self.review_branch.is_some() && self.review_instruction.is_none() {
1832 self.review_instruction = Some(self.instruction.clone());
1833 }
1834 if self.instruction != instruction {
1835 self.resume_override = None;
1838 }
1839 self.title = title;
1840 self.instruction = instruction;
1841 Ok(())
1842 }
1843
1844 pub fn handed_off(&mut self, why: impl Into<String>) {
1864 let why = why.into();
1865 self.diagnostic = None;
1866 self.note_held();
1867 self.status = TaskStatus::Held;
1868 self.hold_source = Some(HoldSource::Machine);
1869 self.hold_reason = Some(why.clone());
1870 self.last_error = Some(why);
1871 }
1872
1873 pub fn release_operator(&mut self, source: &str, question_id: Option<&str>) {
1875 self.recovery_budget = None;
1876 self.recovery_generations.clear();
1877 self.recovery_admission = None;
1878 if self
1879 .resumption
1880 .as_ref()
1881 .is_some_and(|r| r.instruction != self.instruction)
1882 {
1883 self.resume_override = None;
1886 }
1887 let approval = Resumption {
1888 generation: self.resumption.as_ref().map_or(1, |r| r.generation + 1),
1889 source: source.to_owned(),
1890 question_id: question_id.map(str::to_owned),
1891 at: Timestamp::now(),
1892 instruction: self.instruction.clone(),
1893 answers_through: self.answers.len(),
1894 runs_through: self.runs.len(),
1895 blocking_evidence: Vec::new(),
1896 cleared_reason: self
1897 .hold_reason
1898 .clone()
1899 .or_else(|| self.block_reason.clone()),
1900 };
1901 self.release();
1902 self.operator_decided_at = Some(approval.at);
1903 if let Some(previous) = self.resumption.replace(approval) {
1904 self.resumption_history.push(previous);
1905 }
1906 }
1907
1908 pub(crate) fn recovery_failed(&mut self, state: &crate::run::RunState) {
1910 let (base, inputs) = self.recovery_admission.as_ref().map_or_else(
1911 || (state.base_commit.clone(), recovery_inputs(&state.sources)),
1912 |a| (a.base.clone(), a.inputs.clone()),
1913 );
1914 let progress = recovery_progress(state);
1915 let stage = if state.prep_failure.is_some() {
1916 "preparation"
1917 } else {
1918 "execution"
1919 };
1920 let matches = |b: &RecoveryBudget| {
1921 b.instruction == self.instruction
1922 && b.base == base
1923 && b.inputs == inputs
1924 && (progress.is_none() || b.progress == progress)
1925 };
1926 if !self.recovery_budget.as_ref().is_some_and(matches) {
1927 let previous = self
1928 .recovery_generations
1929 .iter()
1930 .position(matches)
1931 .map(|index| self.recovery_generations.remove(index));
1932 if let Some(old) = self.recovery_budget.take() {
1933 self.recovery_generations.push(old);
1934 }
1935 self.recovery_budget = previous;
1936 }
1937 if let Some(budget) = &mut self.recovery_budget {
1938 budget.failures = budget.failures.saturating_add(1);
1939 budget.stage = stage.to_owned();
1940 } else {
1941 self.recovery_budget = Some(RecoveryBudget {
1942 instruction: self.instruction.clone(),
1943 base,
1944 inputs,
1945 stage: stage.to_owned(),
1946 failures: 1,
1947 progress,
1948 });
1949 }
1950 }
1951
1952 pub(crate) fn recovery_exhausted(
1953 &self,
1954 base: &str,
1955 inputs: &str,
1956 limit: usize,
1957 ) -> Option<String> {
1958 let same_input = |b: &&RecoveryBudget| {
1959 b.instruction == self.instruction && b.base == base && b.inputs == inputs
1960 };
1961 let budget = self
1962 .recovery_budget
1963 .as_ref()
1964 .filter(same_input)
1965 .or_else(|| self.recovery_generations.iter().rev().find(same_input))?;
1966 (budget.instruction == self.instruction && budget.base == base && budget.inputs == inputs
1967 && budget.failures >= limit.max(1)).then(|| format!(
1968 "[recovery-budget] {} failed passes against unchanged inputs ({}). Inspect the retained recovery sources and correct the cause, change the inputs, or explicitly release the task to approve another generation. Automatic requeue does not reset this budget.",
1969 budget.failures, budget.stage))
1970 }
1971
1972 pub fn release(&mut self) {
1976 self.import_chat_notice();
1979 if self.hold_notice_cause.is_none()
1980 && matches!(self.status, TaskStatus::Held | TaskStatus::Blocked)
1981 {
1982 self.hold_notice_cause = Some(self.unresolved_hold_cause());
1983 }
1984 let refused_handover = self.status == TaskStatus::Held
1985 && self.hold_source == Some(HoldSource::Machine)
1986 && self.review_branch.is_some()
1987 && self
1988 .review_instruction
1989 .as_ref()
1990 .is_none_or(|i| *i == self.instruction);
1991 self.status = TaskStatus::Queued;
1992 self.held_at = None;
1993 self.attempts = 0;
1994 self.last_error = None;
1995 self.park_reason = None;
1996 self.hold_reason = None;
1999 self.hold_source = None;
2000 self.diagnostic = None;
2001 self.blocked_by.clear();
2006 self.block_reason = None;
2007 self.blocked_from = None;
2008 if !refused_handover {
2012 self.review_branch = None;
2013 self.review_instruction = None;
2014 }
2015 self.fresh_start = false;
2016 }
2017}
2018
2019fn copy_new(dir: &Path, src: &Path, name: &str) -> Result<(String, PathBuf)> {
2023 use std::io::ErrorKind;
2024 let (stem, ext) = match name.rfind('.') {
2025 Some(i) if i > 0 => (&name[..i], &name[i..]),
2026 _ => (name, ""),
2027 };
2028 for n in 1u32.. {
2029 let candidate = if n == 1 {
2030 name.to_owned()
2031 } else {
2032 let suffix = format!("-{n}");
2033 let room = 64usize.saturating_sub(suffix.len() + ext.len());
2034 let stem: String = stem.chars().take(room).collect();
2035 format!("{stem}{suffix}{ext}")
2036 };
2037 if !crate::ask::valid_asset_name(&candidate) {
2038 bail!("no valid attachment name is left for `{name}`");
2039 }
2040 let path = dir.join(&candidate);
2041 match std::fs::OpenOptions::new()
2042 .write(true)
2043 .create_new(true)
2044 .open(&path)
2045 {
2046 Ok(mut out) => {
2047 let copied = std::fs::File::open(src)
2048 .and_then(|mut input| std::io::copy(&mut input, &mut out));
2049 if let Err(e) = copied {
2050 drop(out);
2051 let _ = std::fs::remove_file(&path);
2052 return Err(e).with_context(|| format!("copy {}", src.display()));
2053 }
2054 return Ok((candidate, path));
2055 }
2056 Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
2057 Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
2058 }
2059 }
2060 unreachable!("the counter never runs out")
2061}
2062
2063const TASK_LOCK_STALE: std::time::Duration = std::time::Duration::from_secs(10);
2066
2067pub(crate) struct TaskLock {
2070 path: PathBuf,
2071 token: String,
2072}
2073
2074fn owner_token() -> String {
2077 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
2078 let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2079 let mut r = crate::rng::SplitMix64::new(crate::rng::entropy() ^ n.rotate_left(32));
2080 format!("{}-{:016x}", std::process::id(), r.next_u64())
2081}
2082
2083fn break_marker(path: &Path, token: &str) -> PathBuf {
2086 let mut h: u64 = 0xcbf29ce484222325;
2087 for b in token.bytes() {
2088 h ^= u64::from(b);
2089 h = h.wrapping_mul(0x100000001b3);
2090 }
2091 let mut name = path.as_os_str().to_owned();
2092 name.push(format!(".break-{h:016x}"));
2093 PathBuf::from(name)
2094}
2095
2096fn older_than_stale(path: &Path) -> bool {
2097 std::fs::metadata(path)
2098 .and_then(|m| m.modified())
2099 .ok()
2100 .and_then(|t| t.elapsed().ok())
2101 .is_some_and(|age| age > TASK_LOCK_STALE)
2102}
2103
2104fn writer_may_be_gone(token: &str) -> bool {
2105 let Some(pid) = token
2106 .split_once('-')
2107 .and_then(|(pid, _)| pid.parse::<u32>().ok())
2108 else {
2109 return true;
2112 };
2113 if pid == std::process::id() {
2114 return false;
2115 }
2116 crate::proc::pid_status(pid) == Some(false)
2118}
2119
2120const MAX_MARKER_DEPTH: u8 = 3;
2124
2125fn take_marker(
2131 path: &Path,
2132 token: &str,
2133 depth: u8,
2134 retry: crate::lock_file::Retry,
2135) -> Option<(PathBuf, String)> {
2136 use std::io::Write;
2137 let marker = break_marker(path, token);
2138 for _ in 0..2 {
2139 match retry.create(&marker) {
2140 Ok(mut f) => {
2141 let mine = owner_token();
2142 if f.write_all(mine.as_bytes()).is_err() {
2143 drop(f);
2144 let _ = std::fs::remove_file(&marker);
2145 return None;
2146 }
2147 return Some((marker, mine));
2148 }
2149 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2150 let judged = std::fs::read_to_string(&marker).ok();
2151 match judged.filter(|_| older_than_stale(&marker)) {
2152 Some(judged) if depth < MAX_MARKER_DEPTH => {
2153 remove_lock_if(&marker, &judged, true, depth + 1, retry)?;
2154 }
2155 _ => return None,
2156 }
2157 }
2158 Err(_) => return None,
2159 }
2160 }
2161 None
2162}
2163
2164fn remove_lock_if(
2169 path: &Path,
2170 token: &str,
2171 require_stale: bool,
2172 depth: u8,
2173 retry: crate::lock_file::Retry,
2174) -> Option<bool> {
2175 let (marker, mine) = take_marker(path, token, depth, retry)?;
2176 let still = std::fs::read_to_string(path).is_ok_and(|c| c == token)
2177 && (!require_stale || older_than_stale(path) && writer_may_be_gone(token));
2178 if still {
2179 let _ = std::fs::remove_file(path);
2180 }
2181 if depth >= MAX_MARKER_DEPTH {
2183 if std::fs::read_to_string(&marker).is_ok_and(|c| c == mine) {
2184 let _ = std::fs::remove_file(&marker);
2185 }
2186 } else {
2187 let _ = remove_lock_if(&marker, &mine, false, depth + 1, retry);
2188 }
2189 Some(still)
2190}
2191
2192#[cfg(test)]
2195fn break_stale(path: &Path, judged: &str) -> bool {
2196 break_stale_with(path, judged, crate::lock_file::Retry::new(None))
2197}
2198
2199fn break_stale_with(path: &Path, judged: &str, retry: crate::lock_file::Retry) -> bool {
2200 remove_lock_if(path, judged, true, 0, retry).unwrap_or(false)
2201}
2202
2203impl Drop for TaskLock {
2204 fn drop(&mut self) {
2205 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
2206 let retry = crate::lock_file::Retry::new(Some(deadline));
2207 loop {
2208 match remove_lock_if(&self.path, &self.token, false, 0, retry) {
2209 Some(_) => return,
2210 None if std::time::Instant::now() > deadline => return,
2211 None => std::thread::sleep(std::time::Duration::from_millis(5)),
2212 }
2213 }
2214 }
2215}
2216
2217#[derive(Debug, Clone)]
2219pub struct Queue {
2220 root: PathBuf,
2221 ids: IdSource,
2222}
2223
2224#[derive(Clone)]
2227struct IdSource(Option<std::sync::Arc<dyn Fn() -> String + Send + Sync>>);
2228
2229impl std::fmt::Debug for IdSource {
2230 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2231 f.write_str(if self.0.is_some() {
2232 "IdSource(custom)"
2233 } else {
2234 "IdSource"
2235 })
2236 }
2237}
2238
2239fn path_exists(path: &Path) -> Result<bool> {
2242 match std::fs::symlink_metadata(path) {
2243 Ok(_) => Ok(true),
2244 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
2245 Err(e) => Err(e).with_context(|| format!("stat {}", path.display())),
2246 }
2247}
2248
2249fn validate_attachments(sources: &[PathBuf]) -> Result<Vec<(&PathBuf, &str)>> {
2250 let mut wanted = Vec::new();
2251 for src in sources {
2252 let name = src
2253 .file_name()
2254 .and_then(|n| n.to_str())
2255 .with_context(|| format!("`{}` has no usable file name", src.display()))?;
2256 if !crate::ask::valid_asset_name(name) {
2257 bail!(
2258 "attachment name `{name}` must match ^[A-Za-z0-9][A-Za-z0-9._-]{{0,63}}$ \
2259 with no `..`; rename the file and try again"
2260 );
2261 }
2262 if !src.is_file() {
2263 bail!("attachment `{}` is not a file", src.display());
2264 }
2265 wanted.push((src, name));
2266 }
2267 Ok(wanted)
2268}
2269
2270const FILE_ATTEMPTS: usize = 256;
2272
2273impl Queue {
2274 pub fn open() -> Self {
2276 Self::at(crate::run::home().join("queue"))
2277 }
2278
2279 pub fn at(root: PathBuf) -> Self {
2282 Self {
2283 root,
2284 ids: IdSource(None),
2285 }
2286 }
2287
2288 #[doc(hidden)]
2291 pub fn with_id_source(mut self, f: impl Fn() -> String + Send + Sync + 'static) -> Self {
2292 self.ids = IdSource(Some(std::sync::Arc::new(f)));
2293 self
2294 }
2295
2296 fn mint_id(&self) -> String {
2297 match &self.ids.0 {
2298 Some(f) => f(),
2299 None => new_id(),
2300 }
2301 }
2302
2303 fn file_with(
2314 &self,
2315 task: &mut Task,
2316 hook: impl FnMut(&mut Task) -> Result<bool>,
2317 undo: impl FnMut(&mut Task),
2318 ) -> Result<()> {
2319 self.reserve_id(task, hook, undo, true)
2320 }
2321
2322 fn reserve_id(
2330 &self,
2331 task: &mut Task,
2332 mut hook: impl FnMut(&mut Task) -> Result<bool>,
2333 mut undo: impl FnMut(&mut Task),
2334 store: bool,
2335 ) -> Result<()> {
2336 let redraw_first = self.ids.0.is_some() && task.attachments.is_empty();
2337 for attempt in 0..FILE_ATTEMPTS {
2338 if attempt > 0 || redraw_first {
2339 task.id = self.mint_id();
2340 }
2341 let _lock = self.lock_task(&task.id)?;
2342 if path_exists(&self.path_of(&task.id))? {
2343 continue;
2344 }
2345 if !hook(task)? {
2346 continue;
2347 }
2348 if store {
2349 if let Err(e) = self.write_record(task) {
2350 undo(task);
2351 return Err(e);
2352 }
2353 task.unfiled = false;
2354 }
2355 return Ok(());
2356 }
2357 bail!("no free task id after {FILE_ATTEMPTS} attempts")
2358 }
2359
2360 fn id_in_use(&self, id: &str, check_attachments: bool) -> Result<bool> {
2365 if path_exists(&self.lock_path(id))? {
2366 return Ok(true);
2367 }
2368 Ok(check_attachments && path_exists(&self.attachments_dir(id))?)
2369 }
2370
2371 fn plain_filing_hook(&self, task: &Task) -> Result<bool> {
2375 Ok(!self.id_in_use(&task.id, task.attachments.is_empty())?)
2376 }
2377
2378 pub fn file_claimed(&self, task: &mut Task) -> Result<Claim> {
2381 let mut claim = None;
2382 self.file_with(
2383 task,
2384 |t| {
2385 if self.id_in_use(&t.id, true)? {
2386 return Ok(false);
2387 }
2388 claim = Some(self.claim(&t.id)?);
2389 Ok(true)
2390 },
2391 |_| {},
2392 )?;
2393 claim.context("claim missing after filing")
2394 }
2395
2396 pub fn root(&self) -> &Path {
2398 &self.root
2399 }
2400
2401 pub fn path_of(&self, id: &str) -> PathBuf {
2403 self.root.join(format!("{id}.json"))
2404 }
2405
2406 pub fn attachments_dir(&self, id: &str) -> PathBuf {
2409 self.root.join(format!("{id}.attachments"))
2410 }
2411
2412 pub fn attach(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2423 let wanted = validate_attachments(sources)?;
2424 if task.unfiled && task.attachments.is_empty() {
2425 let names = std::cell::RefCell::new(Vec::new());
2428 self.reserve_id(
2429 task,
2430 |t| {
2431 if self.id_in_use(&t.id, true)? {
2432 return Ok(false);
2433 }
2434 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2435 Ok(true)
2436 },
2437 |_| {},
2438 false,
2439 )?;
2440 return Ok(names.into_inner());
2441 }
2442 self.copy_attachments(task, &wanted)
2443 }
2444
2445 fn copy_attachments(
2447 &self,
2448 task: &mut Task,
2449 wanted: &[(&PathBuf, &str)],
2450 ) -> Result<Vec<String>> {
2451 let dir = self.attachments_dir(&task.id);
2452 let existed = dir.is_dir();
2453 std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
2454 let mut created: Vec<PathBuf> = Vec::new();
2455 let mut names = Vec::new();
2456 let mut copy_all = || -> Result<()> {
2457 for (src, name) in wanted {
2458 let (stored, path) = copy_new(&dir, src, name)?;
2459 created.push(path);
2460 names.push(stored);
2461 }
2462 Ok(())
2463 };
2464 if let Err(e) = copy_all() {
2465 for path in &created {
2466 let _ = std::fs::remove_file(path);
2467 }
2468 if !existed {
2469 let _ = std::fs::remove_dir(&dir);
2470 }
2471 return Err(e);
2472 }
2473 task.attachments.extend(names.iter().cloned());
2474 Ok(names)
2475 }
2476
2477 pub fn attach_and_put(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2488 if task.unfiled && task.attachments.is_empty() {
2489 return self.attach_and_file(task, sources);
2490 }
2491 let dir = self.attachments_dir(&task.id);
2492 let existed = dir.is_dir();
2493 let before = task.attachments.len();
2494 let names = self.attach(task, sources)?;
2495 if let Err(e) = self.put(task) {
2496 for name in &names {
2497 let _ = std::fs::remove_file(dir.join(name));
2498 }
2499 if !existed {
2500 let _ = std::fs::remove_dir(&dir);
2501 }
2502 task.attachments.truncate(before);
2503 return Err(e);
2504 }
2505 Ok(names)
2506 }
2507
2508 fn attach_and_file(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2512 let wanted = validate_attachments(sources)?;
2513 let before = task.attachments.len();
2514 let names = std::cell::RefCell::new(Vec::new());
2515 let result = self.file_with(
2516 task,
2517 |t| {
2518 if self.id_in_use(&t.id, true)? {
2519 return Ok(false);
2520 }
2521 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2522 Ok(true)
2523 },
2524 |t| {
2525 let dir = self.attachments_dir(&t.id);
2526 for name in names.borrow().iter() {
2527 let _ = std::fs::remove_file(dir.join(name));
2528 }
2529 let _ = std::fs::remove_dir(&dir);
2531 t.attachments.truncate(before);
2532 },
2533 );
2534 result.map(|()| names.into_inner())
2535 }
2536
2537 pub fn attachment_paths(&self, task: &Task) -> Vec<PathBuf> {
2541 let dir = self.attachments_dir(&task.id);
2542 task.attachments
2543 .iter()
2544 .map(|n| {
2545 let p = dir.join(n);
2546 std::path::absolute(&p).unwrap_or(p)
2547 })
2548 .collect()
2549 }
2550
2551 pub fn put(&self, task: &mut Task) -> Result<()> {
2560 if task.unfiled {
2561 return self.file_with(task, |t| self.plain_filing_hook(t), |_| {});
2563 }
2564 let _lock = self.lock_task(&task.id)?;
2565 self.put_unlocked(task)
2566 }
2567
2568 pub fn create_new(&self, task: &mut Task) -> Result<bool> {
2573 let _lock = self.lock_task(&task.id)?;
2574 if path_exists(&self.path_of(&task.id))? {
2575 return Ok(false);
2576 }
2577 self.write_record(task)?;
2578 task.unfiled = false;
2579 Ok(true)
2580 }
2581
2582 fn put_unlocked(&self, task: &mut Task) -> Result<()> {
2584 self.merge_stored(task);
2585 self.write_record(task)
2586 }
2587
2588 fn merge_stored(&self, task: &mut Task) {
2590 if let Ok(mut stored) = read_path(&self.path_of(&task.id)) {
2591 stored.import_chat_notice();
2592 for key in stored.chat_notices {
2593 if !task.chat_notices.contains(&key) {
2594 task.chat_notices.push(key);
2595 }
2596 }
2597 for run in stored.runs {
2598 if !task.runs.contains(&run) {
2599 task.runs.push(run);
2600 }
2601 }
2602 for p in stored.passes {
2603 if !task
2604 .passes
2605 .iter()
2606 .any(|q| q.run == p.run && q.pass_index == p.pass_index)
2607 {
2608 task.passes.push(p);
2609 }
2610 }
2611 if let Some(n) = stored.chat_notice
2614 && task.chat_notice.as_ref().is_none_or(|mine| mine.at < n.at)
2615 {
2616 task.chat_notice = Some(n);
2617 }
2618 }
2619 }
2620
2621 fn write_record(&self, task: &mut Task) -> Result<()> {
2623 task.import_chat_notice();
2624 task.updated_at = Timestamp::now();
2625 std::fs::create_dir_all(&self.root)
2626 .with_context(|| format!("create {}", self.root.display()))?;
2627 let body = serde_json::to_string_pretty(task).context("serialize task")?;
2628 let path = self.path_of(&task.id);
2629 let tmp = path.with_extension("json.tmp");
2630 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
2631 std::fs::rename(&tmp, &path).with_context(|| format!("replace {}", path.display()))?;
2632 if let (Some(notice), Some(home)) = (
2637 crate::notices::task_held(task),
2638 self.root.parent().filter(|p| !p.as_os_str().is_empty()),
2639 ) {
2640 crate::notices::raise_in(home, notice);
2641 }
2642 Ok(())
2643 }
2644
2645 pub fn modify(&self, id: &str, f: impl FnOnce(&mut Task) -> bool) -> Result<bool> {
2650 let id = self.resolve_id(id)?;
2651 let _lock = self.lock_task(&id)?;
2652 let mut task = self.get(&id)?;
2653 if !f(&mut task) {
2654 return Ok(false);
2655 }
2656 self.put_unlocked(&mut task)?;
2657 Ok(true)
2658 }
2659
2660 pub fn link_run(&self, id: &str, run: &str) -> Result<Task> {
2665 let id = self.resolve_id(id)?;
2666 let _lock = self.lock_task(&id)?;
2669 let mut task = self.get(&id)?;
2670 if task.link_run(run) {
2671 self.put_unlocked(&mut task)?;
2672 }
2673 Ok(task)
2674 }
2675
2676 pub(crate) fn lock_task(&self, id: &str) -> Result<TaskLock> {
2684 std::fs::create_dir_all(&self.root)
2685 .with_context(|| format!("create {}", self.root.display()))?;
2686 let path = self.root.join(format!("{id}.write-lock"));
2687 let started = std::time::Instant::now();
2688 let retry = crate::lock_file::Retry::task_write(started + TASK_LOCK_STALE);
2689 let marker_retry = crate::lock_file::Retry::new(Some(started + TASK_LOCK_STALE));
2690 loop {
2691 match retry.create(&path) {
2692 Ok(mut file) => {
2693 use std::io::Write;
2694 let token = owner_token();
2695 if let Err(e) = file.write_all(token.as_bytes()) {
2696 drop(file);
2697 let _ = std::fs::remove_file(&path);
2698 return Err(e).with_context(|| format!("lock {}", path.display()));
2699 }
2700 return Ok(TaskLock { path, token });
2701 }
2702 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2703 let judged = std::fs::read_to_string(&path).ok();
2706 let broken = judged
2707 .filter(|_| older_than_stale(&path))
2708 .is_some_and(|judged| break_stale_with(&path, &judged, marker_retry));
2709 if !broken {
2713 if started.elapsed() > TASK_LOCK_STALE {
2714 bail!("could not lock task {id}");
2715 }
2716 std::thread::sleep(std::time::Duration::from_millis(15));
2717 }
2718 }
2719 Err(e) => return Err(e).with_context(|| format!("lock {}", path.display())),
2720 }
2721 }
2722 }
2723
2724 pub fn get(&self, id: &str) -> Result<Task> {
2726 let resolved = self.resolve_id(id)?;
2727 read_path(&self.path_of(&resolved))
2728 }
2729
2730 pub fn remove(&self, id: &str, in_flight: bool, questions: &Questions) -> Result<Removal> {
2762 let resolved = self.resolve_id(id)?;
2763 if in_flight {
2764 bail!("task {resolved} is being run by a live daemon right now");
2765 }
2766 self.write_tombstone(&resolved)?;
2767 if let Err(e) = self.remove_record_with_attachments(&resolved, |p| std::fs::remove_file(p))
2768 {
2769 if self.path_of(&resolved).exists() {
2773 let _ = std::fs::remove_file(self.tombstone_path(&resolved));
2774 }
2775 return Err(e);
2776 }
2777 let (released, still_blocked) = self.release_dependents_of(&resolved, questions);
2778 Ok(Removal {
2779 id: resolved,
2780 released,
2781 still_blocked,
2782 })
2783 }
2784
2785 fn tombstone_path(&self, id: &str) -> PathBuf {
2788 self.root.join(format!("{id}.removed"))
2789 }
2790
2791 fn write_tombstone(&self, id: &str) -> Result<()> {
2792 let path = self.tombstone_path(id);
2793 let tmp = self.root.join(format!("{id}.removed.tmp"));
2794 std::fs::write(&tmp, b"")
2795 .and_then(|()| std::fs::rename(&tmp, &path))
2796 .with_context(|| format!("write {}", path.display()))
2797 }
2798
2799 fn was_deleted(&self, id: &str) -> bool {
2803 self.tombstone_path(id).is_file() && !self.path_of(id).exists()
2804 }
2805
2806 pub fn apply_deleted_blockers(&self, task: &mut Task, questions: &Questions) -> Vec<String> {
2810 let deleted = deleted_blockers(self, questions, &task.blocked_by);
2811 deleted
2812 .into_iter()
2813 .filter(|id| task.dependency_deleted(id))
2814 .collect()
2815 }
2816
2817 pub fn note_dependency_deleted(&self, dependent: &Task, deleted: &str) {
2821 let Some(home) = self.root.parent().filter(|p| !p.as_os_str().is_empty()) else {
2822 return;
2823 };
2824 let ja = crate::lang::is_japanese(&crate::lang::of_repo(&dependent.repo));
2825 let waiting = dependent.status == TaskStatus::Blocked;
2826 let message = match (ja, waiting) {
2827 (true, false) => format!(
2828 "タスク {} は、待っていた {} が削除されたため待機を解除し、元の状態に戻しました",
2829 dependent.short(),
2830 short(deleted)
2831 ),
2832 (true, true) => format!(
2833 "タスク {} は、待っていた {} が削除されたため、残りの依存を待っています",
2834 dependent.short(),
2835 short(deleted)
2836 ),
2837 (false, false) => format!(
2838 "Task {} stopped waiting on {} because it was deleted, and returned to its previous state",
2839 dependent.short(),
2840 short(deleted)
2841 ),
2842 (false, true) => format!(
2843 "Task {} stopped waiting on {} because it was deleted, and is still waiting on its other dependencies",
2844 dependent.short(),
2845 short(deleted)
2846 ),
2847 };
2848 crate::notices::raise_in(
2849 home,
2850 crate::notices::Notice::info(
2851 &format!("unblocked:{}:{}", dependent.id, deleted),
2852 message,
2853 )
2854 .link(crate::notices::Link::Task {
2855 id: dependent.id.clone(),
2856 }),
2857 );
2858 }
2859
2860 fn release_dependents_of(
2865 &self,
2866 dependency: &str,
2867 questions: &Questions,
2868 ) -> (Vec<String>, Vec<String>) {
2869 let (mut released, mut still_blocked) = (Vec::new(), Vec::new());
2870 for listed in self.list() {
2871 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| {
2872 matches!(crate::queue::dependency(self, questions, b), Dependency::Deleted(id) if id == dependency)
2873 }) {
2874 continue;
2875 }
2876 let Ok(_claim) = self.claim(&listed.id) else {
2877 continue;
2878 };
2879 let Ok(mut task) = self.get(&listed.id) else {
2880 continue;
2881 };
2882 let references: Vec<String> = task.blocked_by.iter().filter(|id| {
2883 matches!(crate::queue::dependency(self, questions, id), Dependency::Deleted(full) if full == dependency)
2884 }).cloned().collect();
2885 if references.is_empty() {
2886 continue;
2887 }
2888 for reference in references {
2889 task.dependency_deleted(&reference);
2890 }
2891 if self.put(&mut task).is_err() {
2892 continue;
2893 }
2894 self.note_dependency_deleted(&task, dependency);
2895 if task.status == TaskStatus::Blocked {
2896 still_blocked.push(task.id.clone());
2897 } else {
2898 released.push(task.id.clone());
2899 }
2900 }
2901 (released, still_blocked)
2902 }
2903
2904 fn remove_record_with_attachments(
2908 &self,
2909 resolved: &str,
2910 remove_record: impl FnOnce(&Path) -> std::io::Result<()>,
2911 ) -> Result<()> {
2912 self.sweep_removed_attachments();
2919 let attachments = self.attachments_dir(resolved);
2920 let aside = self.root.join(format!("{resolved}.attachments.removing"));
2921 let moved = match std::fs::rename(&attachments, &aside) {
2922 Ok(()) => true,
2923 Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
2924 Err(e) => {
2925 return Err(e).with_context(|| format!("remove {}", attachments.display()));
2926 }
2927 };
2928 let path = self.path_of(resolved);
2929 if let Err(e) = remove_record(&path) {
2930 if moved {
2931 let _ = std::fs::rename(&aside, &attachments);
2932 }
2933 return Err(e).with_context(|| format!("remove {}", path.display()));
2934 }
2935 if moved {
2936 if let Err(e) = std::fs::remove_dir_all(&aside) {
2937 tracing::warn!("leftover attachments {}: {e}", aside.display());
2938 }
2939 }
2940 let lock = self.lock_path(resolved);
2941 if let Err(e) = std::fs::remove_file(&lock) {
2942 if e.kind() != std::io::ErrorKind::NotFound {
2943 return Err(e).with_context(|| format!("remove {}", lock.display()));
2944 }
2945 }
2946 Ok(())
2947 }
2948
2949 fn sweep_removed_attachments(&self) {
2955 let Ok(entries) = std::fs::read_dir(&self.root) else {
2956 return;
2957 };
2958 for entry in entries.flatten() {
2959 let name = entry.file_name();
2960 let name = name.to_string_lossy();
2961 let Some(id) = name.strip_suffix(".attachments.removing") else {
2962 continue;
2963 };
2964 if !self.path_of(id).exists() {
2967 if let Err(e) = std::fs::remove_dir_all(entry.path()) {
2968 tracing::warn!("leftover attachments {}: {e}", entry.path().display());
2969 }
2970 }
2971 }
2972 self.sweep_tombstones();
2973 }
2974
2975 fn sweep_tombstones(&self) {
2984 let Ok(entries) = std::fs::read_dir(&self.root) else {
2985 return;
2986 };
2987 let tasks = self.list();
2988 for entry in entries.flatten() {
2989 let name = entry.file_name();
2990 let name = name.to_string_lossy();
2991 let Some(id) = name.strip_suffix(".removed") else {
2992 continue;
2993 };
2994 if self.path_of(id).exists()
2995 || tasks.iter().any(|t| {
2996 t.blocked_by
2997 .iter()
2998 .any(|b| id.starts_with(b) || id.ends_with(b))
2999 })
3000 {
3001 continue;
3002 }
3003 let old = entry
3004 .metadata()
3005 .and_then(|m| m.modified())
3006 .ok()
3007 .and_then(|m| m.elapsed().ok())
3008 .is_some_and(|age| age >= TOMBSTONE_GRACE);
3009 if old {
3010 let _ = std::fs::remove_file(entry.path());
3011 }
3012 }
3013 }
3014
3015 fn lock_path(&self, id: &str) -> PathBuf {
3018 self.root.join(format!("{id}.lock"))
3019 }
3020
3021 pub fn list(&self) -> Vec<Task> {
3034 let mut tasks: Vec<Task> = std::fs::read_dir(&self.root)
3035 .into_iter()
3036 .flatten()
3037 .flatten()
3038 .map(|e| e.path())
3039 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3040 .filter_map(|p| read_path(&p).ok())
3041 .collect();
3042 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then_with(|| b.id.cmp(&a.id)));
3043 tasks
3044 }
3045
3046 pub fn unreadable(&self) -> Vec<PathBuf> {
3050 std::fs::read_dir(&self.root)
3051 .into_iter()
3052 .flatten()
3053 .flatten()
3054 .map(|e| e.path())
3055 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3056 .filter(|p| read_path(p).is_err())
3057 .collect()
3058 }
3059
3060 pub fn superseded(&self) -> HashMap<String, String> {
3073 let mut by = HashMap::new();
3074 for task in self.list() {
3075 for earlier in &task.runs {
3076 if let Some(later) = task.successor_of(earlier) {
3077 by.insert(earlier.clone(), later.clone());
3078 }
3079 }
3080 }
3081 by
3082 }
3083
3084 pub fn superseded_by(&self, run: &str) -> Option<String> {
3092 for task in self.list() {
3093 if task.runs.iter().any(|r| r == run) {
3094 return task.successor_of(run).cloned();
3095 }
3096 }
3097 None
3098 }
3099
3100 pub fn latest_attempt(&self, run: &str) -> Option<String> {
3111 for task in self.list() {
3112 if task.runs.iter().any(|r| r == run) {
3113 return task.runs.last().filter(|last| **last != run).cloned();
3114 }
3115 }
3116 None
3117 }
3118
3119 pub fn next_runnable(&self) -> Option<Task> {
3124 let mut runnable: Vec<Task> = self
3125 .list()
3126 .into_iter()
3127 .filter(|t| t.status.runnable())
3128 .collect();
3129 runnable.sort_unstable_by(dispatch_order);
3130 runnable.into_iter().next()
3131 }
3132
3133 pub fn claim(&self, id: &str) -> Result<Claim> {
3140 std::fs::create_dir_all(&self.root)
3141 .with_context(|| format!("create {}", self.root.display()))?;
3142 let path = self.lock_path(id);
3143 let retry = crate::lock_file::Retry::new(None);
3144 match retry.create(&path) {
3145 Ok(mut f) => {
3146 use std::io::Write as _;
3147 let _ = writeln!(f, "{}", std::process::id());
3149 Ok(Claim { path })
3150 }
3151 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
3152 bail!("task {id} is already claimed ({} exists)", path.display())
3153 }
3154 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
3155 }
3156 }
3157
3158 pub fn resolve_id(&self, prefix: &str) -> Result<String> {
3160 if !prefix.is_empty()
3161 && prefix != "."
3162 && prefix != ".."
3163 && !prefix.contains(['/', '\\'])
3164 && self.path_of(prefix).is_file()
3165 {
3166 return Ok(prefix.to_owned());
3167 }
3168 let ids = record_ids(&self.root, "json")?;
3169 match resolve_reference(prefix, ids.iter().map(String::as_str)) {
3170 Reference::Found(id) => Ok(id),
3171 Reference::Missing => bail!("no task matches `{prefix}`"),
3172 Reference::Ambiguous => bail!("`{prefix}` matches multiple tasks"),
3173 }
3174 }
3175
3176 pub fn revision(&self) -> u64 {
3183 self.revision_excluding(&std::collections::BTreeSet::new())
3184 }
3185
3186 pub fn revision_excluding(&self, skip: &std::collections::BTreeSet<String>) -> u64 {
3189 use std::hash::{Hash as _, Hasher as _};
3190
3191 let mut entries: Vec<(String, u64)> = std::fs::read_dir(&self.root)
3192 .into_iter()
3193 .flatten()
3194 .flatten()
3195 .filter(|e| e.path().extension().is_some_and(|ext| ext == "json"))
3196 .filter(|e| {
3197 let path = e.path();
3198 !path
3199 .file_stem()
3200 .is_some_and(|stem| skip.contains(stem.to_string_lossy().as_ref()))
3201 })
3202 .filter_map(|e| {
3203 let name = e.file_name().to_string_lossy().into_owned();
3204 let mtime = e
3205 .metadata()
3206 .ok()?
3207 .modified()
3208 .ok()?
3209 .duration_since(std::time::UNIX_EPOCH)
3210 .ok()?
3211 .as_millis() as u64;
3212 Some((name, mtime))
3213 })
3214 .collect();
3215
3216 if entries.is_empty() {
3217 return 0;
3218 }
3219
3220 entries.sort_unstable();
3221 let mut hasher = std::hash::DefaultHasher::new();
3222 for (name, mtime) in &entries {
3223 name.hash(&mut hasher);
3224 mtime.hash(&mut hasher);
3225 }
3226 let h = hasher.finish();
3227 if h == 0 { 1 } else { h }
3228 }
3229}
3230
3231const TOMBSTONE_GRACE: std::time::Duration = std::time::Duration::from_secs(3600);
3234
3235#[derive(Debug, Clone)]
3237pub struct Removal {
3238 pub id: String,
3240 pub released: Vec<String>,
3244 pub still_blocked: Vec<String>,
3247}
3248
3249#[derive(Debug)]
3251pub struct Claim {
3252 path: PathBuf,
3253}
3254
3255impl Drop for Claim {
3256 fn drop(&mut self) {
3257 let _ = std::fs::remove_file(&self.path);
3258 }
3259}
3260
3261pub fn title_from(instruction: &str, max: usize) -> String {
3264 let Some(line) = first_line(instruction) else {
3265 return "(empty task)".to_owned();
3266 };
3267 if line.chars().count() <= max {
3268 return line.to_owned();
3269 }
3270 let head: String = line.chars().take(max.saturating_sub(1)).collect();
3271 format!("{head}…")
3272}
3273
3274pub(crate) fn first_line(instruction: &str) -> Option<&str> {
3280 let line = instruction
3281 .lines()
3282 .map(str::trim)
3283 .find(|l| !l.is_empty())?
3284 .trim_start_matches(['#', '-', '*', '>', ' '])
3285 .trim();
3286 (!line.is_empty()).then_some(line)
3287}
3288
3289fn read_path(path: &Path) -> Result<Task> {
3290 let body = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
3291 let task: Task =
3292 serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
3293 if task.schema > SCHEMA {
3299 bail!(
3300 "task {} was written by a different magi (schema {}, this build \
3301 speaks {SCHEMA})",
3302 task.id,
3303 task.schema
3304 );
3305 }
3306 Ok(task)
3307}
3308
3309#[derive(Debug, PartialEq, Eq)]
3312pub(crate) enum Reference {
3313 Found(String),
3314 Missing,
3315 Ambiguous,
3316}
3317
3318pub(crate) fn resolve_reference<'a>(id: &str, ids: impl Iterator<Item = &'a str>) -> Reference {
3319 let mut hits = Vec::new();
3320 for candidate in ids {
3321 if candidate == id {
3322 return Reference::Found(candidate.to_owned());
3323 }
3324 if candidate.starts_with(id) || candidate.ends_with(id) {
3325 hits.push(candidate);
3326 }
3327 }
3328 match hits.as_slice() {
3329 [] => Reference::Missing,
3330 [one] => Reference::Found((*one).to_owned()),
3331 _ => Reference::Ambiguous,
3332 }
3333}
3334
3335pub(crate) fn record_ids(root: &Path, extension: &str) -> Result<Vec<String>> {
3338 let entries = match std::fs::read_dir(root) {
3339 Ok(entries) => entries,
3340 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
3341 Err(e) => return Err(e.into()),
3342 };
3343 let mut ids = Vec::new();
3344 for entry in entries {
3345 let path = entry?.path();
3346 if path.extension().is_some_and(|e| e == extension)
3347 && let Some(id) = path.file_stem().and_then(|s| s.to_str())
3348 {
3349 ids.push(id.to_owned());
3350 }
3351 }
3352 Ok(ids)
3353}
3354
3355#[derive(Debug, PartialEq, Eq)]
3358pub enum Dependency {
3359 Task(String),
3361 Question(String),
3363 Deleted(String),
3365 Missing,
3367 Ambiguous,
3369 Unknown,
3371}
3372
3373pub fn dependency(queue: &Queue, questions: &Questions, id: &str) -> Dependency {
3375 let (Ok(tasks), Ok(qs), Ok(removed)) = (
3376 record_ids(queue.root(), "json"),
3377 record_ids(questions.root(), "json"),
3378 record_ids(queue.root(), "removed"),
3379 ) else {
3380 return Dependency::Unknown;
3381 };
3382 let mut ids = tasks
3385 .iter()
3386 .chain(qs.iter())
3387 .chain(removed.iter())
3388 .cloned()
3389 .collect::<Vec<_>>();
3390 ids.sort();
3391 ids.dedup();
3392 match resolve_reference(id, ids.iter().map(String::as_str)) {
3393 Reference::Found(id) if tasks.contains(&id) => Dependency::Task(id),
3394 Reference::Found(id) if qs.contains(&id) => Dependency::Question(id),
3395 Reference::Found(id) if queue.was_deleted(&id) => Dependency::Deleted(id),
3396 Reference::Found(_) => Dependency::Unknown,
3397 Reference::Missing => Dependency::Missing,
3398 Reference::Ambiguous => Dependency::Ambiguous,
3399 }
3400}
3401
3402pub fn normalize_blockers(queue: &Queue, questions: &Questions, ids: &mut [String]) {
3404 for id in ids {
3405 if let Dependency::Task(full) | Dependency::Question(full) | Dependency::Deleted(full) =
3406 dependency(queue, questions, id)
3407 {
3408 *id = full;
3409 }
3410 }
3411}
3412
3413pub fn dependency_facts(queue: &Queue, questions: &Questions) -> String {
3416 let inventory = crate::blockers::Inventory::load(queue, questions);
3417 let mut text = String::from(
3418 "Current authoritative queue (historical missing-task claims may be stale; unreadable or ambiguous references must keep waiting):\n",
3419 );
3420 match record_ids(queue.root(), "json") {
3421 Ok(ids) => {
3422 for id in ids {
3423 match queue.get(&id) {
3424 Ok(task) => text.push_str(&format!(
3425 "{}: {}; blocked_by={:?}; chain={:?}\n",
3426 task.id,
3427 task.status.as_str(),
3428 task.blocked_by,
3429 inventory.waits_on(&task)
3430 )),
3431 Err(_) => text.push_str(&format!("{id}: exists, state unreadable\n")),
3432 }
3433 }
3434 }
3435 Err(_) => text.push_str("Queue inventory unreadable: existence is unknown.\n"),
3436 }
3437 text
3438}
3439
3440pub fn missing_blockers(
3442 queue: &Queue,
3443 questions: &Questions,
3444 blocked_by: &[String],
3445) -> Vec<String> {
3446 blocked_by
3447 .iter()
3448 .filter(|id| dependency(queue, questions, id) == Dependency::Missing)
3449 .cloned()
3450 .collect()
3451}
3452
3453pub fn deleted_blockers(
3456 queue: &Queue,
3457 questions: &Questions,
3458 blocked_by: &[String],
3459) -> Vec<String> {
3460 blocked_by
3461 .iter()
3462 .filter(|id| matches!(dependency(queue, questions, id), Dependency::Deleted(_)))
3463 .cloned()
3464 .collect()
3465}
3466
3467pub fn missing_blocker_hold_reason(blocked_by: &[String], missing: &[String]) -> String {
3479 missing_blocker_hold_reason_in(blocked_by, missing, "en")
3480}
3481
3482pub fn missing_blocker_hold_reason_in(
3484 blocked_by: &[String],
3485 missing: &[String],
3486 language: &str,
3487) -> String {
3488 if crate::lang::is_japanese(language) {
3489 format!(
3490 "{} を待っていましたが、{} はディスク上に存在しません - `magi task triage` を参照",
3491 blocked_by.join(", "),
3492 missing.join(", "),
3493 )
3494 } else {
3495 format!(
3496 "blocked on {} but {} no longer exist(s) on disk - see `magi task triage`",
3497 blocked_by.join(", "),
3498 missing.join(", "),
3499 )
3500 }
3501}
3502
3503pub fn short(id: &str) -> &str {
3505 id.split('-').next_back().unwrap_or(id)
3506}
3507
3508fn new_id() -> String {
3509 let stamp = jiff::Zoned::now().strftime("%Y%m%d-%H%M%S");
3510 let seed = crate::rng::entropy();
3511 format!("{stamp}-{:04x}", (seed ^ (seed >> 32)) & 0xffff)
3512}
3513
3514#[cfg(test)]
3515mod tests {
3516 #[test]
3517 fn explicit_resumption_is_not_invented_for_legacy_or_automatic_release() {
3518 let t = Task::new(
3519 "retry".into(),
3520 "do it".into(),
3521 PathBuf::from("."),
3522 Source::Human,
3523 );
3524 let mut value = serde_json::to_value(&t).unwrap();
3525 for field in ["resumption", "resumption_history", "operator_decided_at"] {
3526 value.as_object_mut().unwrap().remove(field);
3527 }
3528 value["schema"] = serde_json::json!(15);
3529 let mut legacy: Task = serde_json::from_value(value).unwrap();
3530 legacy.hold_machine(Some("automatic retry".into()));
3531 legacy.release();
3532 assert!(legacy.resumption.is_none());
3533 assert!(legacy.resumption_history.is_empty());
3534 assert!(legacy.operator_decided_at.is_none());
3535 legacy.release_operator("cli", None);
3536 legacy.start("run".into());
3537 let reloaded: Task =
3538 serde_json::from_str(&serde_json::to_string(&legacy).unwrap()).unwrap();
3539 assert!(reloaded.resume_override.is_none());
3540 assert_eq!(reloaded.resumption.as_ref().unwrap().generation, 1);
3541 }
3542
3543 #[test]
3544 fn merge_over_lets_the_later_choice_win_and_keeps_the_rest() {
3545 let mut base = RunOverrides {
3546 merge: Some("pr".to_owned()),
3547 candidates: Some(3),
3548 ..RunOverrides::default()
3549 };
3550 base.merge_over(&RunOverrides {
3551 merge: Some("none".to_owned()),
3552 seed: Some(7),
3553 ..RunOverrides::default()
3554 });
3555 assert_eq!(base.merge.as_deref(), Some("none"));
3556 assert_eq!(base.candidates, Some(3));
3557 assert_eq!(base.seed, Some(7));
3558 }
3559
3560 #[test]
3561 fn an_already_landed_task_is_done_with_its_attempt_refunded() {
3562 let mut t = task("relanded");
3563 t.attempts = 1;
3564 t.status = TaskStatus::Running;
3565 t.already_landed("already in main as 0e368de");
3566 assert_eq!(t.status, TaskStatus::Done);
3567 assert_eq!(t.attempts, 0);
3568 assert!(t.hold_reason.is_none());
3569 assert_eq!(t.last_error.as_deref(), Some("already in main as 0e368de"));
3570 }
3571
3572 #[test]
3573 fn missing_blocker_reason_follows_the_language() {
3574 let b = vec!["a".to_owned()];
3575 let en = missing_blocker_hold_reason_in(&b, &b, "en");
3576 assert_eq!(en, missing_blocker_hold_reason(&b, &b));
3577 assert!(en.starts_with("blocked on a"));
3578 assert!(missing_blocker_hold_reason_in(&b, &b, "ja").contains("存在しません"));
3579 assert_eq!(missing_blocker_hold_reason_in(&b, &b, "de"), en);
3580 }
3581
3582 use super::*;
3583
3584 #[test]
3585 fn triage_applied_survives_release_and_old_records_read_as_empty() {
3586 let mut t = Task::new(
3587 "t".to_owned(),
3588 "i".to_owned(),
3589 PathBuf::from("r"),
3590 Source::Human,
3591 );
3592 t.mark_triage_applied("q1");
3593 t.mark_triage_applied("q1");
3594 t.hold_machine(Some("x".to_owned()));
3595 t.release();
3596 assert_eq!(t.triage_applied, ["q1"]);
3597 assert!(t.triage_applied("q1") && !t.triage_applied("q2"));
3598
3599 let mut v = serde_json::to_value(&t).unwrap();
3600 v.as_object_mut().unwrap().remove("triage_applied");
3601 let old: Task = serde_json::from_value(v).unwrap();
3602 assert!(old.triage_applied.is_empty());
3603 }
3604
3605 #[test]
3606 fn task_counts_of_empty_is_all_zero() {
3607 assert_eq!(TaskCounts::of(&[]), TaskCounts::default());
3608 }
3609
3610 #[test]
3611 fn task_counts_of_tallies_every_status() {
3612 let mut queued = Task::new(
3613 "q".to_owned(),
3614 "i".to_owned(),
3615 PathBuf::from("."),
3616 Source::Human,
3617 );
3618 queued.status = TaskStatus::Queued;
3619 let mut running = queued.clone();
3620 running.status = TaskStatus::Running;
3621 let mut done = queued.clone();
3622 done.status = TaskStatus::Done;
3623 let mut failed = queued.clone();
3624 failed.status = TaskStatus::Failed;
3625 let mut held = queued.clone();
3626 held.status = TaskStatus::Held;
3627 let mut blocked = queued.clone();
3628 blocked.status = TaskStatus::Blocked;
3629
3630 let mut parked = queued.clone();
3631 parked.status = TaskStatus::Parked;
3632
3633 let counts = TaskCounts::of(&[
3634 queued,
3635 running,
3636 done.clone(),
3637 done,
3638 failed,
3639 held,
3640 blocked,
3641 parked,
3642 ]);
3643 assert_eq!(
3644 counts,
3645 TaskCounts {
3646 queued: 1,
3647 running: 1,
3648 done: 2,
3649 failed: 1,
3650 held: 1,
3651 blocked: 1,
3652 parked: 1,
3653 }
3654 );
3655 }
3656
3657 #[test]
3658 fn task_implementers_persist_normalized_without_losing_history() {
3659 let (_dir, q) = queue();
3660 let mut t = task("keep me");
3661 t.hold_manual(Some("waiting".into()));
3662 t.runs.push("earlier-run".into());
3663 t.attachments.push("reference.txt".into());
3664 let cfg: crate::config::Config =
3665 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
3666 .unwrap();
3667 q.put(&mut t).unwrap();
3668 let original = serde_json::to_value(&t).unwrap();
3669 for n in [2, 1, 2] {
3670 t.set_implementers(n, &cfg).unwrap();
3671 q.put(&mut t).unwrap();
3672 t = q.get(&t.id).unwrap();
3673 assert_eq!(t.implementers(), Some(usize::from(n)));
3674 assert_eq!(t.solo, n == 1);
3675 if n == 1 {
3676 assert!(t.overrides.is_none());
3677 }
3678 let mut actual = serde_json::to_value(&t).unwrap();
3679 for field in ["solo", "overrides", "updated_at"] {
3680 actual[field] = original[field].clone();
3681 }
3682 assert_eq!(actual, original);
3683 }
3684 t.overrides.as_mut().unwrap().judges = Some(2);
3685 t.set_implementers(1, &cfg).unwrap();
3686 assert_eq!(t.overrides.as_ref().unwrap().judges, Some(2));
3687 assert_eq!(t.overrides.as_ref().unwrap().candidates, None);
3688 }
3689
3690 #[test]
3691 fn task_implementers_old_records_keep_inheritance_until_changed() {
3692 let mut value = serde_json::to_value(task("old task")).unwrap();
3693 value.as_object_mut().unwrap().remove("solo");
3694 value.as_object_mut().unwrap().remove("overrides");
3695 let mut old: Task = serde_json::from_value(value).unwrap();
3696 assert_eq!(old.implementers(), None);
3697 old.set_implementers(1, &crate::config::Config::default())
3698 .unwrap();
3699 assert!(old.solo);
3700 assert!(old.overrides.is_none());
3701 }
3702
3703 #[test]
3704 fn task_implementers_reject_invalid_counts_and_states() {
3705 let cfg = crate::config::Config::default();
3706 let mut t = task("unchanged");
3707 for n in [0, 4, 255] {
3708 assert!(t.set_implementers(n, &cfg).is_err());
3709 }
3710 for status in [
3711 TaskStatus::Running,
3712 TaskStatus::Done,
3713 TaskStatus::Failed,
3714 TaskStatus::Blocked,
3715 TaskStatus::Parked,
3716 ] {
3717 t.status = status;
3718 let before = serde_json::to_value(&t).unwrap();
3719 assert!(t.set_implementers(1, &cfg).is_err());
3720 assert_eq!(serde_json::to_value(&t).unwrap(), before);
3721 }
3722 }
3723
3724 fn queue() -> (tempfile::TempDir, Queue) {
3727 let dir = tempfile::tempdir().unwrap();
3728 let q = Queue::at(dir.path().join("queue"));
3729 (dir, q)
3730 }
3731
3732 #[test]
3733 fn putting_a_machine_held_task_files_a_notification_beside_the_queue() {
3734 let dir = tempfile::tempdir().unwrap();
3735 let q = Queue::at(dir.path().join("queue"));
3736 let mut t = task("held");
3737 q.put(&mut t).unwrap();
3738 assert_eq!(
3739 crate::notices::Notices::at(dir.path().join("notifications"))
3740 .list()
3741 .len(),
3742 0
3743 );
3744 t.hold_machine(Some("out of attempts".to_owned()));
3745 q.put(&mut t).unwrap();
3746 let listed = crate::notices::Notices::at(dir.path().join("notifications")).list();
3747 assert_eq!(listed.len(), 1);
3748 assert!(listed[0].message.contains("out of attempts"));
3749 }
3750
3751 fn task(title: &str) -> Task {
3752 Task::new(
3753 title.to_owned(),
3754 format!("do {title}"),
3755 PathBuf::from("."),
3756 Source::Human,
3757 )
3758 }
3759
3760 #[test]
3761 fn attempt_limit_inherits_old_records_and_controls_failure() {
3762 let mut t = task("budget");
3763 let mut old = serde_json::to_value(&t).unwrap();
3764 old.as_object_mut().unwrap().remove("attempt_limit");
3765 old["schema"] = serde_json::json!(15);
3766 let legacy: Task = serde_json::from_value(old).unwrap();
3767 assert_eq!(legacy.attempt_limit, None);
3768 assert_eq!(legacy.effective_attempt_limit(2), 2);
3769 t.attempts = 2;
3770 t.change_attempt_limit(NonZeroUsize::new(3)).unwrap();
3771 t.fail("retry", 1);
3772 assert_eq!(t.status, TaskStatus::Failed);
3773 t.attempts = 3;
3774 t.fail("stop", 10);
3775 assert_eq!(t.status, TaskStatus::Held);
3776 t.change_attempt_limit(None).unwrap();
3777 assert_eq!(t.status, TaskStatus::Held);
3778 assert_eq!(t.attempts, 3);
3779 t.fail("inherit", 4);
3780 assert_eq!(t.status, TaskStatus::Failed);
3781 for status in [
3782 TaskStatus::Running,
3783 TaskStatus::Done,
3784 TaskStatus::Failed,
3785 TaskStatus::Blocked,
3786 TaskStatus::Parked,
3787 ] {
3788 t.status = status;
3789 assert!(t.change_attempt_limit(NonZeroUsize::new(1)).is_err());
3790 assert_eq!(t.attempt_limit, None);
3791 }
3792 }
3793
3794 #[test]
3795 fn earlier_attempts_is_every_recorded_run_and_agrees_with_the_display() {
3796 let mut t = task("retried");
3797 assert!(
3798 t.earlier_attempts().is_empty(),
3799 "a first attempt takes nothing over"
3800 );
3801 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
3802 assert_eq!(t.earlier_attempts(), ["aaaa", "bbbb"]);
3803 assert_eq!(t.successor_of("aaaa"), Some(&"bbbb".to_owned()));
3805 assert_eq!(t.successor_of("bbbb"), None);
3806 assert_eq!(t.successor_of("zzzz"), None);
3807 }
3808
3809 #[test]
3810 fn resumed_runs_are_superseded_only_after_their_last_occurrence() {
3811 let (_dir, q) = queue();
3812 let mut t = task("resumed");
3813 for (runs, successor) in [
3814 (vec!["aaaa", "aaaa"], None),
3815 (vec!["aaaa", "bbbb", "aaaa"], None),
3816 (vec!["aaaa", "aaaa", "bbbb"], Some("bbbb")),
3817 (vec!["aaaa", "bbbb", "aaaa", "cccc"], Some("cccc")),
3818 ] {
3819 t.runs = runs.into_iter().map(str::to_owned).collect();
3820 q.put(&mut t).unwrap();
3821
3822 assert_eq!(t.successor_of("aaaa").map(String::as_str), successor);
3823 assert_eq!(q.superseded_by("aaaa").as_deref(), successor);
3824 assert_eq!(q.superseded().get("aaaa").map(String::as_str), successor);
3825 }
3826 }
3827
3828 #[test]
3829 fn superseded_by_names_the_next_attempt_and_none_for_the_last() {
3830 let (_dir, q) = queue();
3831 let mut t = task("retried");
3832 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
3833 q.put(&mut t).unwrap();
3834
3835 assert_eq!(q.superseded_by("aaaa"), Some("bbbb".to_owned()));
3836 assert_eq!(q.superseded_by("bbbb"), Some("cccc".to_owned()));
3837 assert_eq!(
3838 q.superseded_by("cccc"),
3839 None,
3840 "the latest attempt replaces nothing"
3841 );
3842 assert_eq!(
3843 q.superseded_by("never-heard-of-it"),
3844 None,
3845 "a run belonging to no task on this queue is not superseded"
3846 );
3847
3848 let mut by = HashMap::new();
3849 by.insert("aaaa".to_owned(), "bbbb".to_owned());
3850 by.insert("bbbb".to_owned(), "cccc".to_owned());
3851 assert_eq!(
3852 q.superseded(),
3853 by,
3854 "the whole-map and single-run forms must agree"
3855 );
3856 }
3857
3858 #[test]
3859 fn superseded_attempts_is_empty_until_the_task_is_done() {
3860 let mut t = task("retried");
3861 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
3862 t.status = TaskStatus::Failed;
3863 assert_eq!(
3864 t.superseded_attempts(true),
3865 &[] as &[String],
3866 "a task still retrying has no attempt yet that a later one made moot"
3867 );
3868
3869 t.status = TaskStatus::Running;
3870 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
3871 }
3872
3873 #[test]
3874 fn superseded_attempts_names_every_run_before_the_one_that_succeeded() {
3875 let mut t = task("retried");
3876 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
3877 t.status = TaskStatus::Done;
3878 assert_eq!(
3879 t.superseded_attempts(true),
3880 &["aaaa".to_owned(), "bbbb".to_owned()],
3881 "cccc is the attempt whose success made the task done, and stays out"
3882 );
3883 }
3884
3885 #[test]
3886 fn superseded_attempts_is_empty_for_a_done_task_with_only_one_attempt() {
3887 let mut t = task("first try landed");
3888 t.runs = vec!["aaaa".to_owned()];
3889 t.status = TaskStatus::Done;
3890 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
3891 }
3892
3893 #[test]
3894 fn superseded_attempts_is_empty_when_the_last_run_never_actually_succeeded() {
3895 let mut t = task("closed by hand after a manual merge");
3902 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
3903 t.status = TaskStatus::Done;
3904 assert_eq!(
3905 t.superseded_attempts(false),
3906 &[] as &[String],
3907 "nothing here is provably why the task is done, so nothing is superseded"
3908 );
3909 }
3910
3911 #[test]
3912 fn latest_attempt_names_the_chain_s_current_head_not_just_the_next_one() {
3913 let (_dir, q) = queue();
3914 let mut t = task("retried twice");
3915 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
3916 q.put(&mut t).unwrap();
3917
3918 assert_eq!(
3919 q.latest_attempt("aaaa"),
3920 Some("cccc".to_owned()),
3921 "an old attempt points straight at the chain's current head, not the \
3922 next attempt in the middle of it"
3923 );
3924 assert_eq!(q.latest_attempt("bbbb"), Some("cccc".to_owned()));
3925 assert_eq!(
3926 q.latest_attempt("cccc"),
3927 None,
3928 "the latest attempt is not superseded by anything"
3929 );
3930 assert_eq!(
3931 q.latest_attempt("never-heard-of-it"),
3932 None,
3933 "a run belonging to no task on this queue is not superseded"
3934 );
3935 }
3936
3937 #[test]
3938 fn a_markdown_heading_is_the_title_not_decoration() {
3939 assert_eq!(
3944 title_from("# Rework the config loader\n\nIt re-reads it.\n", 40),
3945 "Rework the config loader"
3946 );
3947 assert_eq!(title_from("- fix the thing", 40), "fix the thing");
3948 assert_eq!(title_from("> quoted task", 40), "quoted task");
3949 assert_eq!(title_from(" \n\n", 40), "(empty task)");
3951 assert_eq!(title_from("###\n", 40), "(empty task)");
3952 }
3953
3954 #[test]
3955 fn a_long_title_is_elided_by_characters_not_bytes() {
3956 let long = "課題".repeat(30);
3958 let title = title_from(&long, 10);
3959 assert_eq!(title.chars().count(), 10);
3960 assert!(title.ends_with('…'));
3961 }
3962
3963 #[test]
3964 fn priority_wins_and_ties_break_oldest_first() {
3965 let (_dir, q) = queue();
3966 let mut a = task("first");
3967 let mut b = task("second");
3968 let mut c = task("urgent");
3969 a.id = "20260101-000001-aaaa".to_owned();
3971 b.id = "20260101-000002-bbbb".to_owned();
3972 c.id = "20260101-000003-cccc".to_owned();
3973 c.priority = 5;
3974 for t in [&mut a, &mut b, &mut c] {
3975 q.put(t).unwrap();
3976 }
3977
3978 assert_eq!(q.next_runnable().unwrap().id, c.id);
3980 c.hold_machine(None);
3981 q.put(&mut c).unwrap();
3982 assert_eq!(q.next_runnable().unwrap().id, a.id);
3984 assert_eq!(q.list().len(), 3, "b is still waiting its turn");
3985 }
3986
3987 #[test]
3988 fn a_blocked_task_never_starves_another_runnable_one() {
3989 let (_dir, q) = queue();
3990 let mut blocked = task("blocked");
3991 blocked.block(vec!["something".to_owned()], None);
3992 q.put(&mut blocked).unwrap();
3993
3994 let mut runnable = task("free to go");
3995 q.put(&mut runnable).unwrap();
3996
3997 let next = q.next_runnable().expect("a runnable task is still offered");
3998 assert_eq!(next.id, runnable.id);
3999 }
4000
4001 #[test]
4002 fn a_held_task_is_never_offered_to_the_loop() {
4003 let (_dir, q) = queue();
4004 let mut t = task("held");
4005 q.put(&mut t).unwrap();
4006 assert!(q.next_runnable().is_some());
4007
4008 t.hold_machine(None);
4009 q.put(&mut t).unwrap();
4010 assert!(
4011 q.next_runnable().is_none(),
4012 "a held task must wait for a human"
4013 );
4014
4015 t.status = TaskStatus::Failed;
4017 q.put(&mut t).unwrap();
4018 assert!(q.next_runnable().is_some());
4019 }
4020
4021 #[test]
4022 fn attempts_are_capped_and_then_the_task_is_held() {
4023 let mut t = task("doomed");
4024
4025 t.start("run-1".to_owned());
4026 t.fail("gate red", 2);
4027 assert_eq!(t.status, TaskStatus::Failed, "one attempt of two: retry");
4028
4029 t.start("run-2".to_owned());
4030 t.fail("gate red", 2);
4031 assert_eq!(
4032 t.status,
4033 TaskStatus::Held,
4034 "out of attempts: stop spending money on it"
4035 );
4036 assert_eq!(t.runs, ["run-1", "run-2"]);
4037 assert_eq!(t.last_error.as_deref(), Some("gate red"));
4038 assert_eq!(
4039 t.hold_reason.as_deref(),
4040 Some("gate red"),
4041 "the hold must say why, not leave hold_reason null next to a \
4042 populated last_error"
4043 );
4044 }
4045
4046 #[test]
4047 fn handing_off_a_task_records_a_hold_reason_too() {
4048 let mut t = task("left a pull request");
4049 t.start("run-1".to_owned());
4050 t.handed_off("run ended with a pull request open [run run-1]");
4051 assert_eq!(t.status, TaskStatus::Held);
4052 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4053 assert_eq!(
4054 t.hold_reason.as_deref(),
4055 Some("run ended with a pull request open [run run-1]")
4056 );
4057 assert_eq!(t.hold_reason, t.last_error);
4058 }
4059
4060 #[test]
4061 fn a_quota_stall_is_refunded_so_the_backlog_survives_the_night() {
4062 let mut t = task("stalled by quota");
4063
4064 t.start("run-1".to_owned());
4065 assert_eq!(t.attempts, 1);
4066 t.stall("judge-1, judge-2 out of quota");
4067 assert_eq!(
4068 t.attempts, 0,
4069 "a closed quota window must not spend the task's retry budget"
4070 );
4071 assert_eq!(t.status, TaskStatus::Failed, "the loop should retry it");
4072 assert_eq!(
4073 t.last_error.as_deref(),
4074 Some("judge-1, judge-2 out of quota")
4075 );
4076
4077 for _ in 0..20 {
4080 t.start("run-n".to_owned());
4081 t.stall("still out of quota");
4082 }
4083 t.start("run-real".to_owned());
4084 t.fail("gate red", 2);
4085 assert_eq!(
4086 t.status,
4087 TaskStatus::Failed,
4088 "the first attempt that was really judged is attempt one"
4089 );
4090 }
4091
4092 #[test]
4093 fn releasing_a_held_task_gives_it_a_real_second_chance() {
4094 let mut t = task("retry me");
4095 t.start("run-1".to_owned());
4096 t.fail("gate red", 1);
4097 assert_eq!(t.status, TaskStatus::Held);
4098
4099 t.release();
4100 assert_eq!(t.status, TaskStatus::Queued);
4101 assert_eq!(t.attempts, 0);
4104 assert!(t.last_error.is_none());
4105 assert_eq!(
4106 t.runs.len(),
4107 1,
4108 "history is kept: attempts reset, evidence does not"
4109 );
4110 }
4111
4112 #[test]
4113 fn a_hold_reason_survives_and_a_release_clears_it() {
4114 let mut t = task("waiting on something else");
4115 t.hold_manual(Some(
4116 "waiting for 20260101-000000-aaaa to land first".to_owned(),
4117 ));
4118 assert_eq!(t.status, TaskStatus::Held);
4119 assert_eq!(
4120 t.hold_reason.as_deref(),
4121 Some("waiting for 20260101-000000-aaaa to land first")
4122 );
4123
4124 t.hold_manual(None);
4126 assert_eq!(
4127 t.hold_reason.as_deref(),
4128 Some("waiting for 20260101-000000-aaaa to land first"),
4129 "a bare re-hold keeps whatever a human already wrote down"
4130 );
4131
4132 let mut plain = task("no reason given");
4134 plain.hold_manual(None);
4135 assert_eq!(plain.status, TaskStatus::Held);
4136 assert!(plain.hold_reason.is_none());
4137
4138 t.release();
4139 assert_eq!(t.status, TaskStatus::Queued);
4140 assert!(
4141 t.hold_reason.is_none(),
4142 "a stale reason must not greet the next person who holds this task"
4143 );
4144 }
4145
4146 #[test]
4147 fn closing_a_held_task_as_done_clears_its_hold_reason_too() {
4148 let mut t = task("landed by hand while held");
4153 t.hold_manual(Some("waiting on 3ed9".to_owned()));
4154 assert_eq!(t.hold_reason.as_deref(), Some("waiting on 3ed9"));
4155
4156 t.succeed();
4157 assert_eq!(t.status, TaskStatus::Done);
4158 assert!(
4159 t.hold_reason.is_none(),
4160 "a done task cannot still be waiting on something"
4161 );
4162 }
4163
4164 #[test]
4165 fn holding_or_closing_a_blocked_task_clears_its_dependency_too() {
4166 let mut held = task("held straight out of blocked");
4173 held.block(
4174 vec!["20260101-000000-dead".to_owned()],
4175 Some("waiting on the migration script".to_owned()),
4176 );
4177 assert_eq!(held.status, TaskStatus::Blocked);
4178
4179 held.hold_manual(None);
4180 assert_eq!(held.status, TaskStatus::Held);
4181 assert!(
4182 held.blocked_by.is_empty(),
4183 "hold overrides the wait, same as release"
4184 );
4185 assert!(held.block_reason.is_none());
4186
4187 let mut done = task("closed straight out of blocked");
4188 done.block(
4189 vec!["20260101-000000-dead".to_owned()],
4190 Some("waiting on the migration script".to_owned()),
4191 );
4192 done.succeed();
4193 assert_eq!(done.status, TaskStatus::Done);
4194 assert!(
4195 done.blocked_by.is_empty(),
4196 "a done task cannot still be waiting on a dependency"
4197 );
4198 assert!(done.block_reason.is_none());
4199 }
4200
4201 #[test]
4202 fn a_blocked_task_is_never_offered_to_the_loop() {
4203 let mut t = task("blocked");
4204 assert!(t.status.runnable());
4205 t.block(
4206 vec!["dep-id".to_owned()],
4207 Some("waits on dep-id".to_owned()),
4208 );
4209 assert_eq!(t.status, TaskStatus::Blocked);
4210 assert!(!t.status.runnable());
4211 assert_eq!(TaskStatus::Blocked.as_str(), "blocked");
4212 }
4213
4214 #[test]
4215 fn unblocking_the_last_dependency_returns_the_task_to_queued() {
4216 let mut t = task("blocked on two");
4217 t.block(
4218 vec!["a".to_owned(), "b".to_owned()],
4219 Some("waits on a and b".to_owned()),
4220 );
4221
4222 t.unblock("a");
4223 assert_eq!(t.status, TaskStatus::Blocked, "b is still outstanding");
4224 assert_eq!(t.blocked_by, ["b"]);
4225
4226 t.unblock("b");
4227 assert_eq!(t.status, TaskStatus::Queued);
4228 assert!(t.blocked_by.is_empty());
4229 assert!(t.block_reason.is_none());
4230 }
4231
4232 #[test]
4233 fn unblocking_an_id_on_a_task_that_is_not_blocked_is_a_no_op() {
4234 let mut t = task("never blocked");
4235 t.unblock("whatever");
4236 assert_eq!(t.status, TaskStatus::Queued);
4237 }
4238
4239 #[test]
4240 fn a_held_task_blocked_on_a_question_returns_to_held_not_queued() {
4241 let mut t = task("held, then asked about");
4246 t.hold_machine(Some("out of attempts".to_owned()));
4247 assert_eq!(t.status, TaskStatus::Held);
4248
4249 t.block(vec!["q1".to_owned()], Some("what now?".to_owned()));
4250 assert_eq!(t.status, TaskStatus::Blocked);
4251
4252 t.record_answer("what now?".to_owned(), "leave it held".to_owned());
4253 t.unblock("q1");
4254 assert_eq!(t.status, TaskStatus::Held, "must restore, not requeue");
4255 assert_eq!(t.hold_reason.as_deref(), Some("out of attempts"));
4256 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4257 assert!(t.blocked_from.is_none(), "consumed once restored");
4258 }
4259
4260 #[test]
4261 fn a_manually_held_task_blocked_on_a_question_returns_to_held() {
4262 let mut t = task("manually held, then asked about");
4263 t.hold_manual(Some("waiting on a dependency".to_owned()));
4264
4265 t.block(vec!["q1".to_owned()], None);
4266 t.unblock("q1");
4267
4268 assert_eq!(t.status, TaskStatus::Held);
4269 assert_eq!(t.hold_source, Some(HoldSource::Manual));
4270 }
4271
4272 #[test]
4273 fn re_blocking_an_already_blocked_task_keeps_the_original_blocked_from() {
4274 let mut t = task("held, blocked twice");
4278 t.hold_machine(None);
4279 t.block(vec!["q1".to_owned()], Some("first".to_owned()));
4280 t.block(
4281 vec!["q1".to_owned(), "q2".to_owned()],
4282 Some("second".to_owned()),
4283 );
4284
4285 t.unblock("q1");
4286 assert_eq!(t.status, TaskStatus::Blocked, "q2 still outstanding");
4287 t.unblock("q2");
4288 assert_eq!(t.status, TaskStatus::Held);
4289 }
4290
4291 #[test]
4292 fn unblocking_a_task_blocked_while_running_lands_on_queued_not_running() {
4293 let mut t = task("blocked mid-run");
4297 t.start("run-1".to_owned());
4298 assert_eq!(t.status, TaskStatus::Running);
4299
4300 t.block(vec!["q1".to_owned()], None);
4301 t.unblock("q1");
4302 assert_eq!(t.status, TaskStatus::Queued);
4303 }
4304
4305 #[test]
4306 fn a_pre_schema_4_blocked_record_with_hold_evidence_restores_to_held() {
4307 let mut t = task("legacy record, held before it was blocked");
4313 t.hold_source = Some(HoldSource::Machine);
4314 t.hold_reason = Some("legacy hold reason".to_owned());
4315 t.status = TaskStatus::Blocked;
4316 t.blocked_by = vec!["q1".to_owned()];
4317 t.blocked_from = None;
4318
4319 t.unblock("q1");
4320 assert_eq!(t.status, TaskStatus::Held);
4321 }
4322
4323 #[test]
4324 fn a_pre_schema_4_blocked_record_with_no_hold_evidence_restores_to_queued() {
4325 let mut t = task("legacy record, ordinary dependency block");
4326 t.status = TaskStatus::Blocked;
4327 t.blocked_by = vec!["dep".to_owned()];
4328 t.blocked_from = None;
4329
4330 t.unblock("dep");
4331 assert_eq!(t.status, TaskStatus::Queued);
4332 }
4333
4334 #[test]
4335 fn answering_a_question_is_recorded_and_survives_a_release() {
4336 let mut t = task("asked something");
4337 t.block(vec!["q1".to_owned()], Some("which backend?".to_owned()));
4338 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
4339 t.unblock("q1");
4340 assert_eq!(t.status, TaskStatus::Queued);
4341 assert_eq!(t.answers.len(), 1);
4342 assert_eq!(t.answers[0].answer, "SQLite");
4343
4344 t.release();
4348 assert_eq!(t.answers.len(), 1, "the answer is not lost on release");
4349 }
4350
4351 #[test]
4352 fn resumption_review_branch_requires_unchanged_authored_criteria() {
4353 for source in ["cli", "web"] {
4354 let mut t = task("review old candidate");
4355 t.request_review("candidate-A".into());
4356 t.hold_for_handover(Some("candidate-A".into()), "handover refused".into());
4357 let mut unchanged = t.clone();
4358 unchanged.release_operator(source, None);
4359 assert_eq!(unchanged.review_branch.as_deref(), Some("candidate-A"));
4360 t.edit(
4361 "review approved head".into(),
4362 "Reuse measured PR head B; fix Clippy; review".into(),
4363 )
4364 .unwrap();
4365 t.release_operator(source, None);
4366 assert!(t.review_branch.is_none());
4367 assert!(t.review_instruction.is_none());
4368 let mut legacy = unchanged;
4370 legacy.review_instruction = None;
4371 legacy.hold_for_handover(None, "still checked out".into());
4372 legacy
4373 .edit("new plan".into(), "Reuse measured PR head B".into())
4374 .unwrap();
4375 legacy.release_operator(source, None);
4376 assert!(legacy.review_branch.is_none());
4377 }
4378 }
4379
4380 #[test]
4381 fn a_refused_handover_keeps_the_review_branch_across_release() {
4382 let mut t = task("refused takeover");
4383 t.start("run-1".to_owned());
4384 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
4385 assert_eq!(t.status, TaskStatus::Held);
4386 assert_eq!(t.attempts, 1);
4387 t.release();
4388 assert_eq!(t.status, TaskStatus::Queued);
4389 assert_eq!(t.attempts, 0);
4390 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4391
4392 t.hold_for_handover(None, "again".to_owned());
4394 t.requeue();
4395 assert!(t.review_branch.is_none());
4396
4397 let mut m = task("manual");
4399 m.review_branch = Some("magi/x/A".to_owned());
4400 m.hold_manual(None);
4401 m.release();
4402 assert!(m.review_branch.is_none());
4403 }
4404
4405 #[test]
4406 fn requesting_review_requeues_the_task_and_remembers_the_branch() {
4407 let mut t = task("blocked run with a surviving branch");
4408 t.start("run-1".to_owned());
4409 t.fail("blocked with major findings", 5);
4410 assert_eq!(t.status, TaskStatus::Failed);
4411
4412 t.request_review("magi/eba2/A".to_owned());
4413 assert_eq!(t.status, TaskStatus::Queued);
4414 assert_eq!(t.attempts, 0);
4415 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4416
4417 t.release();
4419 assert!(t.review_branch.is_none());
4420 }
4421
4422 #[test]
4423 fn conductor_requeue_but_not_an_ordinary_release_forces_a_fresh_start() {
4424 let mut t = task("retry");
4425 t.start("run-1".to_owned());
4426 t.requeue();
4427 assert!(t.fresh_start);
4428
4429 t.release();
4430 assert!(!t.fresh_start);
4431 }
4432
4433 #[test]
4434 fn urgent_only_changes_queued_or_held_and_preserves_settings() {
4435 for status in [
4436 TaskStatus::Queued,
4437 TaskStatus::Held,
4438 TaskStatus::Running,
4439 TaskStatus::Done,
4440 TaskStatus::Failed,
4441 TaskStatus::Blocked,
4442 TaskStatus::Parked,
4443 ] {
4444 let mut t = task("urgent settings");
4445 t.status = status;
4446 t.attempts = 2;
4447 t.hold_reason = Some("waiting".into());
4448 t.priority = 7;
4449 t.interrupt = true;
4450 let before = serde_json::to_value(&t).unwrap();
4451 for urgent in [true, true, false, false] {
4452 let result = t.set_urgent(urgent);
4453 let mut expected = before.clone();
4454 if matches!(status, TaskStatus::Queued | TaskStatus::Held) {
4455 result.unwrap();
4456 expected["urgent"] = urgent.into();
4457 } else {
4458 let error = result.unwrap_err().to_string();
4459 assert!(error.contains(status.as_str()), "{error}");
4460 assert!(error.contains("queued or held"), "{error}");
4461 }
4462 assert_eq!(serde_json::to_value(&t).unwrap(), expected);
4463 }
4464 }
4465 }
4466
4467 #[test]
4468 fn priority_can_be_changed_while_queued_but_not_while_running() {
4469 let mut t = task("reprioritise me");
4470 t.set_priority(5).unwrap();
4471 assert_eq!(t.priority, 5);
4472
4473 t.start("run-1".to_owned());
4474 let err = t.set_priority(9).unwrap_err().to_string();
4475 assert!(err.contains("running"), "{err}");
4476 assert_eq!(t.priority, 5, "the rejected write must not partially apply");
4477 }
4478
4479 #[test]
4480 fn interrupt_can_be_marked_while_queued_but_not_while_running() {
4481 let mut t = task("interrupt me");
4482 assert!(!t.interrupt, "off unless asked, same as any other task");
4483
4484 t.set_interrupt(true).unwrap();
4485 assert!(t.interrupt);
4486
4487 t.start("run-1".to_owned());
4488 assert!(
4489 !t.interrupt,
4490 "the mark is one-shot: dispatching the task fulfils it, \
4491 whatever the run that follows ends up doing"
4492 );
4493 let err = t.set_interrupt(true).unwrap_err().to_string();
4494 assert!(err.contains("running"), "{err}");
4495 t.set_interrupt(false).unwrap();
4498 assert!(!t.interrupt);
4499 }
4500
4501 #[test]
4505 fn a_failed_run_does_not_leave_the_task_still_marked_to_interrupt() {
4506 let mut t = task("interrupt me");
4507 t.set_interrupt(true).unwrap();
4508 t.start("run-1".to_owned());
4509 t.fail("mock failure", 5);
4510 assert_eq!(t.status, TaskStatus::Failed);
4511 assert!(
4512 !t.interrupt,
4513 "one attempt already spent the mark; a retry is an ordinary \
4514 requeue, not a fresh interrupt request"
4515 );
4516 }
4517
4518 #[test]
4519 fn changing_priority_moves_a_task_ahead_in_the_real_queue_order() {
4520 let (_dir, q) = queue();
4521 let mut a = task("first filed");
4522 let mut b = task("second filed");
4523 a.id = "20260101-000001-aaaa".to_owned();
4524 b.id = "20260101-000002-bbbb".to_owned();
4525 q.put(&mut a).unwrap();
4526 q.put(&mut b).unwrap();
4527
4528 assert_eq!(
4529 q.next_runnable().unwrap().id,
4530 a.id,
4531 "with equal priority the older task goes first, so a burst of \
4532 new work cannot starve it"
4533 );
4534 assert_eq!(
4535 q.list()[0].id,
4536 b.id,
4537 "but the list an operator reads is newest first, the same as \
4538 before priority existed - a's turn to run does not make it the \
4539 newest task"
4540 );
4541
4542 let mut a = q.get(&a.id).unwrap();
4543 a.set_priority(10).unwrap();
4544 q.put(&mut a).unwrap();
4545
4546 assert_eq!(
4547 q.next_runnable().unwrap().id,
4548 a.id,
4549 "a raised priority must be reflected the moment it is saved"
4550 );
4551 assert_eq!(
4555 q.list()[0].id,
4556 a.id,
4557 "the raised task must sort first in the list an operator reads, \
4558 not only in next_runnable's own ordering"
4559 );
4560 }
4561
4562 #[test]
4563 fn editing_replaces_title_and_instruction_but_keeps_identity_and_history() {
4564 let mut t = Task::new(
4565 "old title".to_owned(),
4566 "old instruction".to_owned(),
4567 PathBuf::from("/repo"),
4568 Source::Agent {
4569 run: "20260101-000000-beef".to_owned(),
4570 node: "implement".to_owned(),
4571 },
4572 );
4573 let id = t.id.clone();
4574 let created_at = t.created_at;
4575 t.runs.push("20260101-000000-beef".to_owned());
4576
4577 t.edit("new title".to_owned(), "new instruction".to_owned())
4578 .unwrap();
4579
4580 assert_eq!(t.title, "new title");
4581 assert_eq!(t.instruction, "new instruction");
4582 assert_eq!(t.id, id, "editing must not mint a new id");
4583 assert_eq!(t.created_at, created_at);
4584 assert_eq!(
4585 t.source,
4586 Source::Agent {
4587 run: "20260101-000000-beef".to_owned(),
4588 node: "implement".to_owned(),
4589 },
4590 "editing must not turn agent attribution into human"
4591 );
4592 assert_eq!(t.runs, ["20260101-000000-beef"]);
4593 }
4594
4595 #[test]
4596 fn editing_is_refused_once_a_task_is_running_or_finished() {
4597 let mut running = task("in flight");
4598 running.start("run-1".to_owned());
4599 let err = running
4600 .edit("x".to_owned(), "y".to_owned())
4601 .unwrap_err()
4602 .to_string();
4603 assert!(err.contains("running"), "{err}");
4604
4605 let mut done = task("finished");
4606 done.succeed();
4607 let err = done
4608 .edit("x".to_owned(), "y".to_owned())
4609 .unwrap_err()
4610 .to_string();
4611 assert!(err.contains("done"), "{err}");
4612
4613 let mut queued = task("waiting");
4615 queued.edit("x".to_owned(), "y".to_owned()).unwrap();
4616 let mut held = task("parked");
4617 held.hold_machine(None);
4618 held.edit("x".to_owned(), "y".to_owned()).unwrap();
4619 }
4620
4621 #[test]
4622 fn a_task_recorded_without_a_hold_reason_still_reads_as_none() {
4623 let (_dir, q) = queue();
4624 let path = q.path_of("20260101-000000-aaaa");
4625 std::fs::create_dir_all(q.root()).unwrap();
4626 std::fs::write(
4627 &path,
4628 serde_json::json!({
4629 "schema": SCHEMA,
4630 "id": "20260101-000000-aaaa",
4631 "title": "from before hold reasons existed",
4632 "instruction": "from before hold reasons existed",
4633 "repo": ".",
4634 "source": { "kind": "human" },
4635 "status": "held",
4636 "created_at": Timestamp::now().to_string(),
4637 "updated_at": Timestamp::now().to_string(),
4638 })
4639 .to_string(),
4640 )
4641 .unwrap();
4642
4643 let task = q.get("20260101-000000-aaaa").expect("must still read");
4644 assert!(task.hold_reason.is_none());
4645 assert!(task.operator_held());
4646 }
4647
4648 #[test]
4649 fn a_legacy_reasoned_hold_defaults_to_operator_protection() {
4650 let (_dir, q) = queue();
4651 let path = q.path_of("20260101-000000-bbbb");
4652 std::fs::create_dir_all(q.root()).unwrap();
4653 std::fs::write(
4654 &path,
4655 serde_json::json!({
4656 "schema": 2,
4657 "id": "20260101-000000-bbbb",
4658 "title": "old manual recovery",
4659 "instruction": "old manual recovery",
4660 "repo": ".",
4661 "source": { "kind": "human" },
4662 "status": "held",
4663 "hold_reason": "active manual recovery run20260912-224242-daf5",
4664 "created_at": Timestamp::now().to_string(),
4665 "updated_at": Timestamp::now().to_string(),
4666 })
4667 .to_string(),
4668 )
4669 .unwrap();
4670
4671 let task = q.get("20260101-000000-bbbb").expect("must still read");
4672 assert_eq!(task.hold_source, None);
4673 assert!(task.operator_held());
4674 }
4675
4676 #[test]
4677 fn a_task_recorded_without_a_diagnostic_still_reads_as_none() {
4678 let (_dir, q) = queue();
4679 let path = q.path_of("20260101-000000-aaaa");
4680 std::fs::create_dir_all(q.root()).unwrap();
4681 std::fs::write(
4682 &path,
4683 serde_json::json!({
4684 "schema": SCHEMA,
4685 "id": "20260101-000000-aaaa",
4686 "title": "from before diagnostics existed",
4687 "instruction": "from before diagnostics existed",
4688 "repo": ".",
4689 "source": { "kind": "human" },
4690 "status": "held",
4691 "created_at": Timestamp::now().to_string(),
4692 "updated_at": Timestamp::now().to_string(),
4693 })
4694 .to_string(),
4695 )
4696 .unwrap();
4697
4698 let task = q.get("20260101-000000-aaaa").expect("must still read");
4699 assert!(task.diagnostic.is_none());
4700 }
4701
4702 #[test]
4703 fn a_schema_1_task_with_no_blocking_fields_still_reads() {
4704 let (_dir, q) = queue();
4708 let path = q.path_of("20260101-000000-aaaa");
4709 std::fs::create_dir_all(q.root()).unwrap();
4710 std::fs::write(
4711 &path,
4712 serde_json::json!({
4713 "schema": 1,
4714 "id": "20260101-000000-aaaa",
4715 "title": "from before blocking existed",
4716 "instruction": "from before blocking existed",
4717 "repo": ".",
4718 "source": { "kind": "human" },
4719 "status": "queued",
4720 "created_at": Timestamp::now().to_string(),
4721 "updated_at": Timestamp::now().to_string(),
4722 })
4723 .to_string(),
4724 )
4725 .unwrap();
4726
4727 let task = q.get("20260101-000000-aaaa").expect("must still read");
4728 assert!(task.blocked_by.is_empty());
4729 assert!(task.block_reason.is_none());
4730 assert!(task.answers.is_empty());
4731 assert!(task.review_branch.is_none());
4732 }
4733
4734 #[test]
4735 fn releasing_or_finishing_a_task_clears_its_stale_diagnostic() {
4736 let mut held = task("diagnosed");
4741 held.start("run-1".to_owned());
4742 held.fail("gate red", 1);
4743 held.diagnostic = Some("cargo test failed: ...".to_owned());
4744 assert_eq!(held.status, TaskStatus::Held);
4745
4746 held.release();
4747 assert!(held.diagnostic.is_none());
4748
4749 held.diagnostic = Some("cargo test failed: ...".to_owned());
4750 held.succeed();
4751 assert!(held.diagnostic.is_none());
4752 }
4753
4754 #[test]
4755 fn failing_a_task_always_clears_whatever_diagnostic_it_carried() {
4756 let mut t = task("retried");
4757 t.start("run-1".to_owned());
4758 t.diagnostic = Some("stale evidence from a previous hold".to_owned());
4759 t.fail("unrelated config error", 5);
4760 assert_eq!(t.status, TaskStatus::Failed);
4761 assert!(
4762 t.diagnostic.is_none(),
4763 "fail() must not let an old diagnostic outlive the run that produced it"
4764 );
4765 }
4766
4767 #[test]
4768 fn a_claim_is_exclusive_and_releases_on_drop() {
4769 let (_dir, q) = queue();
4770 let mut t = task("contended");
4771 q.put(&mut t).unwrap();
4772
4773 let held = q.claim(&t.id).unwrap();
4774 assert!(
4775 q.claim(&t.id).is_err(),
4776 "two daemons must not drive one task into two runs"
4777 );
4778 drop(held);
4779 assert!(q.claim(&t.id).is_ok(), "a released claim is reclaimable");
4780 }
4781
4782 #[test]
4783 fn a_round_trip_survives_disk() {
4784 let (_dir, q) = queue();
4785 let mut t = Task::new(
4786 "titled".to_owned(),
4787 "body".to_owned(),
4788 PathBuf::from("/repo"),
4789 Source::Agent {
4790 run: "20260101-000000-beef".to_owned(),
4791 node: "implement".to_owned(),
4792 },
4793 );
4794 t.priority = 3;
4795 q.put(&mut t).unwrap();
4796
4797 let back = q.get(&t.id).unwrap();
4798 assert_eq!(back.id, t.id);
4799 assert_eq!(back.priority, 3);
4800 assert_eq!(back.source.label(), "implement@beef");
4801 assert_eq!(q.get(t.short()).unwrap().id, t.id);
4803 }
4804
4805 #[test]
4806 fn an_unreadable_task_does_not_take_the_queue_down() {
4807 let (_dir, q) = queue();
4808 let mut t = task("fine");
4809 q.put(&mut t).unwrap();
4810 std::fs::write(q.root().join("broken.json"), "{ not json").unwrap();
4811
4812 let listed = q.list();
4813 assert_eq!(listed.len(), 1, "the readable task still lists");
4814 assert_eq!(listed[0].id, t.id);
4815 }
4816
4817 #[test]
4818 fn a_task_recorded_without_a_solo_field_still_reads_as_not_solo() {
4819 let (_dir, q) = queue();
4820 let path = q.path_of("20260101-000000-aaaa");
4821 std::fs::create_dir_all(q.root()).unwrap();
4822 std::fs::write(
4823 &path,
4824 serde_json::json!({
4825 "schema": SCHEMA,
4826 "id": "20260101-000000-aaaa",
4827 "title": "from before solo existed",
4828 "instruction": "from before solo existed",
4829 "repo": ".",
4830 "source": { "kind": "human" },
4831 "status": "queued",
4832 "created_at": Timestamp::now().to_string(),
4833 "updated_at": Timestamp::now().to_string(),
4834 })
4835 .to_string(),
4836 )
4837 .unwrap();
4838
4839 let task = q.get("20260101-000000-aaaa").expect("must still read");
4840 assert!(!task.solo, "a queue file with no `solo` field means false");
4841 }
4842
4843 #[test]
4844 fn a_task_recorded_without_an_urgent_field_still_reads_as_not_urgent() {
4845 let (_dir, q) = queue();
4846 let path = q.path_of("20260101-000000-bbbb");
4847 std::fs::create_dir_all(q.root()).unwrap();
4848 std::fs::write(
4849 &path,
4850 serde_json::json!({
4851 "schema": SCHEMA,
4852 "id": "20260101-000000-bbbb",
4853 "title": "from before urgent existed",
4854 "instruction": "from before urgent existed",
4855 "repo": ".",
4856 "source": { "kind": "human" },
4857 "status": "queued",
4858 "created_at": Timestamp::now().to_string(),
4859 "updated_at": Timestamp::now().to_string(),
4860 })
4861 .to_string(),
4862 )
4863 .unwrap();
4864
4865 let task = q.get("20260101-000000-bbbb").expect("must still read");
4866 assert!(
4867 !task.urgent,
4868 "a queue file with no `urgent` field means false, same as `solo`"
4869 );
4870 }
4871
4872 #[test]
4873 fn a_task_from_a_future_schema_is_refused_rather_than_guessed_at() {
4874 let (_dir, q) = queue();
4875 let mut t = task("from the future");
4876 q.put(&mut t).unwrap();
4877 let path = q.path_of(&t.id);
4878 let body = std::fs::read_to_string(&path)
4879 .unwrap()
4880 .replace(&format!("\"schema\": {SCHEMA}"), "\"schema\": 99");
4881 std::fs::write(&path, body).unwrap();
4882
4883 let err = q.get(&t.id).unwrap_err().to_string();
4884 assert!(err.contains("schema 99"), "{err}");
4885 }
4886
4887 #[test]
4888 fn revision_moves_when_the_queue_changes() {
4889 let (_dir, q) = queue();
4890 assert_eq!(q.revision(), 0, "an empty queue has no revision");
4891 let mut t = task("first");
4892 q.put(&mut t).unwrap();
4893 assert!(q.revision() > 0, "a written task moves the revision");
4894 }
4895
4896 #[test]
4897 fn revision_moves_when_deleting_an_older_task() {
4898 let (dir, q) = queue();
4899 let questions = Questions::at(dir.path().join("questions"));
4900 let mut t1 = task("older");
4901 q.put(&mut t1).unwrap();
4902 std::thread::sleep(std::time::Duration::from_millis(10));
4904 let mut t2 = task("newer");
4905 q.put(&mut t2).unwrap();
4906
4907 let rev_before = q.revision();
4908 q.remove(&t1.id, false, &questions).unwrap();
4909 let rev_after = q.revision();
4910
4911 assert_ne!(
4912 rev_before, rev_after,
4913 "deleting an older task must change the revision so other clients see the deletion"
4914 );
4915 }
4916
4917 #[test]
4918 fn removing_a_task_takes_it_out_of_the_listing() {
4919 let (dir, q) = queue();
4920 let questions = Questions::at(dir.path().join("questions"));
4921 let mut t = task("delete me");
4922 q.put(&mut t).unwrap();
4923 let removed = q.remove(t.short(), false, &questions).unwrap();
4924 assert_eq!(removed.id, t.id, "a prefix resolves before deleting");
4925 assert!(removed.released.is_empty() && removed.still_blocked.is_empty());
4926 assert!(q.list().is_empty());
4927 assert!(
4928 q.remove(&t.id, false, &questions).is_err(),
4929 "removing twice is an error"
4930 );
4931 }
4932
4933 #[test]
4934 fn removing_a_task_takes_its_stale_lock_with_it() {
4935 let (dir, q) = queue();
4936 let questions = Questions::at(dir.path().join("questions"));
4937 let mut t = task("interrupted");
4938 q.put(&mut t).unwrap();
4939
4940 let claim = q.claim(&t.id).unwrap();
4943 std::mem::forget(claim);
4944 assert!(
4945 q.claim(&t.id).is_err(),
4946 "the orphaned lock is what makes the task look claimed"
4947 );
4948
4949 let err = q.remove(&t.id, true, &questions).unwrap_err().to_string();
4951 assert!(err.contains("live daemon"), "{err}");
4952 assert!(q.get(&t.id).is_ok(), "a refused delete keeps the task");
4953
4954 q.remove(&t.id, false, &questions).unwrap();
4956 assert!(q.list().is_empty());
4957 let mut again = task("interrupted");
4958 again.id = t.id.clone();
4959 q.put(&mut again).unwrap();
4960 assert!(
4961 q.claim(&t.id).is_ok(),
4962 "a task that comes back must be claimable, which a left-behind lock would prevent"
4963 );
4964 }
4965
4966 fn notices_of(dir: &Path) -> Vec<crate::notices::Notice> {
4967 crate::notices::Notices::at(dir.join("notifications")).list()
4968 }
4969
4970 #[test]
4971 fn removing_a_sole_dependency_releases_the_dependent_without_a_hold() {
4972 let (dir, q) = queue();
4973 let questions = Questions::at(dir.path().join("questions"));
4974 let mut dep = task("dependency");
4975 q.put(&mut dep).unwrap();
4976 let mut blocked = task("waiting");
4977 blocked.block(vec![dep.id.clone()], Some("waits".to_owned()));
4978 q.put(&mut blocked).unwrap();
4979
4980 let removed = q.remove(&dep.id, false, &questions).unwrap();
4981 assert_eq!(removed.released, [blocked.id.clone()]);
4982 assert!(removed.still_blocked.is_empty());
4983
4984 let after = q.get(&blocked.id).unwrap();
4985 assert_eq!(after.status, TaskStatus::Queued);
4986 assert!(after.blocked_by.is_empty());
4987 assert!(after.block_reason.is_none());
4988 let notes = notices_of(dir.path());
4989 assert_eq!(notes.len(), 1, "{notes:?}");
4990 assert_eq!(notes[0].severity, crate::notices::Severity::Info);
4991 assert_eq!(
4992 notes[0].link,
4993 Some(crate::notices::Link::Task {
4994 id: blocked.id.clone(),
4995 })
4996 );
4997 }
4998
4999 #[test]
5000 fn removing_one_of_two_dependencies_keeps_the_other() {
5001 let (dir, q) = queue();
5002 let questions = Questions::at(dir.path().join("questions"));
5003 let mut dep = task("dependency");
5004 q.put(&mut dep).unwrap();
5005 let mut other = task("other");
5006 q.put(&mut other).unwrap();
5007 let mut blocked = task("waiting");
5008 blocked.block(
5009 vec![dep.id.clone(), other.id.clone()],
5010 Some("waits on both".to_owned()),
5011 );
5012 q.put(&mut blocked).unwrap();
5013
5014 let removed = q.remove(&dep.id, false, &questions).unwrap();
5015 assert!(removed.released.is_empty());
5016 assert_eq!(removed.still_blocked, [blocked.id.clone()]);
5017
5018 let after = q.get(&blocked.id).unwrap();
5019 assert_eq!(after.status, TaskStatus::Blocked);
5020 assert_eq!(after.blocked_by, [other.id.clone()]);
5021 assert!(missing_blockers(&q, &questions, &after.blocked_by).is_empty());
5024 }
5025
5026 #[test]
5027 fn a_dependency_deleted_before_its_dependents_were_rewritten_is_released_later() {
5028 let (dir, q) = queue();
5029 let questions = Questions::at(dir.path().join("questions"));
5030 let mut dep = task("dependency");
5031 q.put(&mut dep).unwrap();
5032 let mut blocked = task("waiting");
5033 blocked.block(vec![dep.id.clone()], None);
5034 q.put(&mut blocked).unwrap();
5035
5036 let claim = q.claim(&blocked.id).unwrap();
5038 let removed = q.remove(&dep.id, false, &questions).unwrap();
5039 assert!(removed.released.is_empty());
5040 drop(claim);
5041
5042 let mut task = q.get(&blocked.id).unwrap();
5043 assert_eq!(task.status, TaskStatus::Blocked);
5044 assert!(missing_blockers(&q, &questions, &task.blocked_by).is_empty());
5045 assert_eq!(
5046 q.apply_deleted_blockers(&mut task, &questions),
5047 [dep.id.clone()]
5048 );
5049 assert_eq!(task.status, TaskStatus::Queued);
5050 }
5051
5052 #[test]
5053 fn dependency_resolution_short_deleted_reference_releases_only_its_waiters() {
5054 let (dir, q) = queue();
5055 let questions = Questions::at(dir.path().join("questions"));
5056 let mut dep = task("dependency");
5057 dep.id = "20261013-193611-ca5b".into();
5058 q.put(&mut dep).unwrap();
5059 let mut waiting = task("waiting");
5060 waiting.block(vec!["ca5b".into()], None);
5061 q.put(&mut waiting).unwrap();
5062 q.remove(&dep.id, false, &questions).unwrap();
5063 assert_eq!(q.get(&waiting.id).unwrap().status, TaskStatus::Queued);
5064 assert_eq!(
5065 dependency(&q, &questions, "ca5b"),
5066 Dependency::Deleted(dep.id)
5067 );
5068 }
5069
5070 #[test]
5071 fn dependency_resolution_preserves_cross_store_ambiguity_and_exact_ids() {
5072 let (dir, queue) = queue();
5073 let questions = Questions::at(dir.path().join("questions"));
5074 let mut dep = task("dependency");
5075 dep.id = "20261013-193611-ca5b".into();
5076 queue.put(&mut dep).unwrap();
5077 std::fs::create_dir_all(questions.root()).unwrap();
5078 let question_id = "20261014-193611-ca5b";
5079 std::fs::write(questions.path_of(question_id), "unreadable question").unwrap();
5080 assert_eq!(questions.resolve_id("ca5b").unwrap(), question_id);
5081 assert_eq!(
5082 dependency(&queue, &questions, "ca5b"),
5083 Dependency::Ambiguous
5084 );
5085 assert_eq!(
5086 dependency(&queue, &questions, &dep.id),
5087 Dependency::Task(dep.id)
5088 );
5089 assert_eq!(
5090 dependency(&queue, &questions, question_id),
5091 Dependency::Question(question_id.into())
5092 );
5093 assert_eq!(
5094 dependency(&queue, &questions, "missing"),
5095 Dependency::Missing
5096 );
5097 }
5098
5099 #[test]
5100 fn dependency_resolution_directory_errors_are_unknown_not_missing() {
5101 let dir = tempfile::tempdir().unwrap();
5102 let q = Queue::at(dir.path().join("not-a-directory"));
5103 std::fs::write(q.root(), "file").unwrap();
5104 let questions = Questions::at(dir.path().join("questions"));
5105 assert_eq!(dependency(&q, &questions, "ca5b"), Dependency::Unknown);
5106 assert!(missing_blockers(&q, &questions, &["ca5b".into()]).is_empty());
5107 }
5108
5109 #[test]
5110 fn a_dependency_without_a_tombstone_is_still_missing() {
5111 let (dir, q) = queue();
5112 let questions = Questions::at(dir.path().join("questions"));
5113 let mut dep = task("dependency");
5114 q.put(&mut dep).unwrap();
5115 std::fs::remove_file(q.path_of(&dep.id)).unwrap();
5116 assert_eq!(
5117 missing_blockers(&q, &questions, std::slice::from_ref(&dep.id)),
5118 [dep.id.clone()]
5119 );
5120 assert!(deleted_blockers(&q, &questions, std::slice::from_ref(&dep.id)).is_empty());
5121 }
5122
5123 #[test]
5124 fn a_held_dependent_is_left_alone_by_a_dependency_deletion() {
5125 let mut t = task("held");
5126 t.hold_machine(Some("because".to_owned()));
5127 t.blocked_by = vec!["gone".to_owned()];
5128 assert!(!t.dependency_deleted("gone"));
5129 assert_eq!(t.status, TaskStatus::Held);
5130 }
5131
5132 #[test]
5133 fn a_failed_record_removal_takes_the_tombstone_back() {
5134 let (_dir, q) = queue();
5135 let mut t = task("stays");
5136 q.put(&mut t).unwrap();
5137 q.write_tombstone(&t.id).unwrap();
5138 let err = q.remove_record_with_attachments(&t.id, |_| Err(std::io::Error::other("nope")));
5139 assert!(err.is_err());
5140 assert!(!q.was_deleted(&t.id));
5143 }
5144
5145 fn source_file(dir: &Path, name: &str, body: &str) -> PathBuf {
5146 let p = dir.join(name);
5147 std::fs::write(&p, body).unwrap();
5148 p
5149 }
5150
5151 #[test]
5152 fn an_attachment_copy_survives_deleting_its_source() {
5153 let (dir, q) = queue();
5154 let src = source_file(dir.path(), "shot.png", "pixels");
5155 let mut t = task("with a picture");
5156 let names = q.attach(&mut t, std::slice::from_ref(&src)).unwrap();
5157 q.put(&mut t).unwrap();
5158 std::fs::remove_file(&src).unwrap();
5159 assert_eq!(names, ["shot.png"]);
5160 let loaded = q.get(&t.id).unwrap();
5161 let paths = q.attachment_paths(&loaded);
5162 assert_eq!(paths.len(), 1);
5163 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "pixels");
5164 }
5165
5166 #[test]
5167 fn attachment_names_that_could_traverse_or_are_odd_are_refused() {
5168 let (dir, q) = queue();
5169 let mut t = task("bad names");
5170 for name in ["a..b.png", ".hidden", "with space.png", "-x.png"] {
5171 let src = source_file(dir.path(), name, "x");
5172 assert!(
5173 q.attach(&mut t, &[src]).is_err(),
5174 "`{name}` must be refused"
5175 );
5176 }
5177 assert!(!crate::ask::valid_asset_name("C:foo.png"));
5181 #[cfg(not(windows))]
5182 {
5183 let src = source_file(dir.path(), "C:foo.png", "x");
5184 assert!(q.attach(&mut t, &[src]).is_err());
5185 }
5186 let long = format!("{}.png", "a".repeat(70));
5187 let src = source_file(dir.path(), &long, "x");
5188 assert!(q.attach(&mut t, &[src]).is_err());
5189 assert!(t.attachments.is_empty());
5190 assert!(!q.attachments_dir(&t.id).exists());
5191 }
5192
5193 #[test]
5194 fn a_taken_attachment_name_is_numbered_not_overwritten() {
5195 let (dir, q) = queue();
5196 let a = source_file(dir.path(), "shot.png", "one");
5197 let sub = dir.path().join("other");
5198 std::fs::create_dir_all(&sub).unwrap();
5199 let b = source_file(&sub, "shot.png", "two");
5200 let mut t = task("collision");
5201 q.attach(&mut t, &[a]).unwrap();
5202 q.attach(&mut t, &[b]).unwrap();
5203 assert_eq!(t.attachments, ["shot.png", "shot-2.png"]);
5204 let paths = q.attachment_paths(&t);
5205 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "one");
5206 assert_eq!(std::fs::read_to_string(&paths[1]).unwrap(), "two");
5207 }
5208
5209 #[test]
5210 fn a_renumbered_name_stays_inside_the_length_bound() {
5211 let (dir, q) = queue();
5212 let name = format!("{}.png", "a".repeat(60));
5213 assert_eq!(name.len(), 64);
5214 let a = source_file(dir.path(), &name, "one");
5215 let sub = dir.path().join("other");
5216 std::fs::create_dir_all(&sub).unwrap();
5217 let b = source_file(&sub, &name, "two");
5218 let mut t = task("long");
5219 q.attach(&mut t, &[a, b]).unwrap();
5220 assert_eq!(t.attachments.len(), 2);
5221 assert!(
5222 t.attachments
5223 .iter()
5224 .all(|n| crate::ask::valid_asset_name(n))
5225 );
5226 assert!(t.attachments[1].ends_with("-2.png"));
5227 }
5228
5229 #[test]
5230 fn a_failed_attach_keeps_existing_attachments_and_leaves_no_partial_copy() {
5231 let (dir, q) = queue();
5232 let good = source_file(dir.path(), "good.png", "ok");
5233 let mut t = task("partial");
5234 q.attach(&mut t, &[good]).unwrap();
5235 let more = source_file(dir.path(), "more.png", "ok");
5236 let missing = dir.path().join("missing.png");
5237 assert!(q.attach(&mut t, &[more, missing]).is_err());
5238 assert_eq!(t.attachments, ["good.png"]);
5239 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5240 .unwrap()
5241 .flatten()
5242 .collect();
5243 assert_eq!(on_disk.len(), 1);
5244 }
5245
5246 fn block_put(q: &Queue, t: &Task) -> PathBuf {
5248 let tmp = q.path_of(&t.id).with_extension("json.tmp");
5249 std::fs::create_dir_all(&tmp).unwrap();
5250 tmp
5251 }
5252
5253 #[test]
5254 fn a_failed_put_leaves_no_new_attachment_directory() {
5255 let (dir, q) = queue();
5256 let mut t = task("fresh");
5257 let tmp = block_put(&q, &t);
5258 let src = source_file(dir.path(), "shot.png", "x");
5259 assert!(q.attach_and_put(&mut t, &[src]).is_err());
5260 assert!(t.attachments.is_empty());
5261 assert!(!q.attachments_dir(&t.id).exists());
5262 assert!(!q.path_of(&t.id).exists());
5263 std::fs::remove_dir(tmp).unwrap();
5264 }
5265
5266 #[test]
5267 fn a_failed_put_removes_only_the_copy_it_just_made() {
5268 let (dir, q) = queue();
5269 let mut t = task("edited");
5270 let first = source_file(dir.path(), "first.png", "1");
5271 q.attach_and_put(&mut t, &[first]).unwrap();
5272 block_put(&q, &t);
5273 let second = source_file(dir.path(), "second.png", "2");
5274 assert!(q.attach_and_put(&mut t, &[second]).is_err());
5275 assert_eq!(t.attachments, ["first.png"]);
5276 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5277 .unwrap()
5278 .flatten()
5279 .map(|e| e.file_name().to_string_lossy().into_owned())
5280 .collect();
5281 assert_eq!(on_disk, ["first.png"]);
5282 assert_eq!(q.get(&t.id).unwrap().attachments, ["first.png"]);
5283 }
5284
5285 #[test]
5286 fn a_leftover_removing_directory_is_swept_by_the_next_removal() {
5287 let (dir, q) = queue();
5288 let questions = Questions::at(dir.path().join("questions"));
5289 let gone = task("gone");
5290 let mut other = task("other");
5291 let mut live = task("live");
5292 q.put(&mut other).unwrap();
5293 q.put(&mut live).unwrap();
5294 let orphan = q.root.join(format!("{}.attachments.removing", gone.id));
5297 std::fs::create_dir_all(&orphan).unwrap();
5298 std::fs::write(orphan.join("shot.png"), "x").unwrap();
5299 let busy = q.root.join(format!("{}.attachments.removing", live.id));
5302 std::fs::create_dir_all(&busy).unwrap();
5303
5304 q.remove(&other.id, false, &questions).unwrap();
5305 assert!(!orphan.exists(), "an orphan is swept");
5306 assert!(busy.exists(), "a removal in progress is left alone");
5307 }
5308
5309 #[test]
5310 fn a_blocked_aside_rename_fails_the_removal_and_loses_nothing() {
5311 let (dir, q) = queue();
5312 let questions = Questions::at(dir.path().join("questions"));
5313 let mut t = task("stuck");
5314 let src = source_file(dir.path(), "shot.png", "x");
5315 q.attach_and_put(&mut t, &[src]).unwrap();
5316 let aside = q.root.join(format!("{}.attachments.removing", t.id));
5319 std::fs::create_dir_all(&aside).unwrap();
5320 std::fs::write(aside.join("old.png"), "o").unwrap();
5321 assert!(q.remove(&t.id, false, &questions).is_err());
5322 assert!(q.path_of(&t.id).exists());
5323 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5324 }
5325
5326 #[test]
5327 fn a_failed_record_removal_puts_the_attachments_back() {
5328 let (dir, q) = queue();
5329 let mut t = task("rollback");
5330 let src = source_file(dir.path(), "shot.png", "x");
5331 q.attach_and_put(&mut t, &[src]).unwrap();
5332 let err = q
5333 .remove_record_with_attachments(&t.id, |_| {
5334 Err(std::io::Error::other("injected failure"))
5335 })
5336 .unwrap_err();
5337 assert!(format!("{err:#}").contains("injected failure"));
5338 assert!(q.path_of(&t.id).exists());
5339 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5340 assert!(
5341 !q.root
5342 .join(format!("{}.attachments.removing", t.id))
5343 .exists()
5344 );
5345 }
5346
5347 #[test]
5348 fn editing_a_task_keeps_its_attachments() {
5349 let (dir, q) = queue();
5350 let src = source_file(dir.path(), "shot.png", "x");
5351 let mut t = task("editable");
5352 q.attach(&mut t, &[src]).unwrap();
5353 t.edit("new".to_owned(), "new text".to_owned()).unwrap();
5354 q.put(&mut t).unwrap();
5355 assert_eq!(q.get(&t.id).unwrap().attachments, ["shot.png"]);
5356 }
5357
5358 #[test]
5359 fn removing_a_task_deletes_its_attachments() {
5360 let (dir, q) = queue();
5361 let questions = Questions::at(dir.path().join("questions"));
5362 let src = source_file(dir.path(), "shot.png", "x");
5363 let mut t = task("doomed");
5364 q.attach(&mut t, &[src]).unwrap();
5365 q.put(&mut t).unwrap();
5366 assert!(q.attachments_dir(&t.id).is_dir());
5367 q.remove(&t.id, false, &questions).unwrap();
5368 assert!(!q.attachments_dir(&t.id).exists());
5369 assert!(q.list().is_empty());
5370 }
5371
5372 #[test]
5373 fn a_task_written_before_attachments_still_reads() {
5374 let (_dir, q) = queue();
5375 let mut t = task("old");
5376 q.put(&mut t).unwrap();
5377 let path = q.path_of(&t.id);
5378 let mut v: serde_json::Value =
5379 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5380 v.as_object_mut().unwrap().remove("attachments");
5381 std::fs::write(&path, v.to_string()).unwrap();
5382 assert!(q.get(&t.id).unwrap().attachments.is_empty());
5383 }
5384
5385 #[test]
5386 fn attachment_paths_are_absolute_even_when_the_root_is_relative() {
5387 let q = Queue::at(PathBuf::from("relative-queue"));
5388 let mut t = task("rel");
5389 t.attachments.push("shot.png".to_owned());
5390 let paths = q.attachment_paths(&t);
5391 assert!(paths[0].is_absolute(), "{}", paths[0].display());
5392 assert!(paths[0].ends_with(format!("{}.attachments/shot.png", t.id)));
5393 }
5394
5395 fn upgrade_parked(id: &str, started: &str) -> Task {
5396 let mut t = task(id);
5397 t.id = id.into();
5398 t.start(format!("run-{id}"));
5399 t.park("upgrade");
5400 t.record_pass_at(
5401 &format!("run-{id}"),
5402 0,
5403 Some(started.parse().unwrap()),
5404 1,
5405 Some(crate::run::ParkCause::Upgrade {
5406 from: "old".into(),
5407 to: None,
5408 }),
5409 );
5410 t
5411 }
5412
5413 #[test]
5414 fn upgrade_resume_order_follows_dispatch_claim_not_graph_start() {
5415 let stamp = |id: &str, dispatched: &str, started: &str| {
5417 let mut t = task(id);
5418 t.id = id.into();
5419 t.start(format!("run-{id}"));
5420 t.park("upgrade");
5421 t.dispatched_at = Some(dispatched.parse().unwrap());
5422 t.record_pass_at(
5423 &format!("run-{id}"),
5424 0,
5425 Some(started.parse().unwrap()),
5426 1,
5427 Some(crate::run::ParkCause::Upgrade {
5428 from: "old".into(),
5429 to: None,
5430 }),
5431 );
5432 t
5433 };
5434 let a = stamp("a", "2026-10-22T00:00:01Z", "2026-10-22T00:00:09Z");
5435 let b = stamp("b", "2026-10-22T00:00:02Z", "2026-10-22T00:00:03Z");
5436 assert!(dispatch_order(&a, &b).is_lt());
5437 }
5438
5439 #[test]
5440 fn upgrade_resume_order_survives_queue_reload_and_reversed_settlement() {
5441 let (_dir, q) = queue();
5442 let mut later = upgrade_parked("later", "2026-10-22T00:00:02Z");
5443 let mut earlier = upgrade_parked("earlier", "2026-10-22T00:00:01Z");
5444 earlier.priority = -10;
5445 let mut fresh = task("fresh");
5446 fresh.id = "newer".into();
5447 fresh.priority = 100;
5448 for t in [&mut fresh, &mut later, &mut earlier] {
5449 q.put(t).unwrap();
5450 }
5451 let reloaded = Queue::at(q.root().to_owned());
5452 let mut tasks = reloaded.list();
5453 tasks.sort_unstable_by(dispatch_order);
5454 assert_eq!(
5455 tasks.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5456 vec!["earlier", "later", "newer"]
5457 );
5458 assert_eq!(reloaded.next_runnable().unwrap().id, "earlier");
5459 assert_eq!(reloaded.get("earlier").unwrap().attempts, 0);
5460 }
5461
5462 #[test]
5463 fn upgrade_resume_legacy_order_uses_settlement_and_id() {
5464 let mut a = upgrade_parked("a", "2026-10-22T00:00:02Z");
5465 let mut b = upgrade_parked("b", "2026-10-22T00:00:01Z");
5466 for t in [&mut a, &mut b] {
5467 let mut value = serde_json::to_value(&*t).unwrap();
5468 value["passes"][0]
5469 .as_object_mut()
5470 .unwrap()
5471 .remove("started_at");
5472 *t = serde_json::from_value(value).unwrap();
5473 }
5474 assert!(dispatch_order(&a, &b).is_lt());
5475 b.passes[0].settled_at = a.passes[0].settled_at;
5476 assert!(dispatch_order(&a, &b).is_lt());
5477 }
5478
5479 #[test]
5480 fn upgrade_resume_key_excludes_other_parks_and_explicit_task_changes() {
5481 let original = upgrade_parked("a", "2026-10-22T00:00:01Z");
5482 assert!(original.upgrade_resume_key().is_some());
5483 for cause in [
5484 crate::run::ParkCause::Operator,
5485 crate::run::ParkCause::Question,
5486 crate::run::ParkCause::Interrupt {
5487 reason: "interrupt".into(),
5488 },
5489 crate::run::ParkCause::Unspecified,
5490 ] {
5491 let mut t = original.clone();
5492 t.passes[0].parked = Some(cause);
5493 assert!(t.upgrade_resume_key().is_none());
5494 let mut fresh = task("fresh");
5495 fresh.priority = 1;
5496 assert!(dispatch_order(&fresh, &t).is_lt());
5497 }
5498 let mut t = original.clone();
5499 t.release();
5500 assert!(t.upgrade_resume_key().is_none());
5501 t.edit("edited".into(), "new instruction".into()).unwrap();
5502 assert!(t.upgrade_resume_key().is_none());
5503 let mut t = original.clone();
5504 t.requeue();
5505 t.status = TaskStatus::Parked;
5506 assert!(t.upgrade_resume_key().is_none());
5507 let mut t = original.clone();
5508 t.start("different-run".into());
5509 t.park("no upgrade");
5510 assert!(t.upgrade_resume_key().is_none());
5511 }
5512
5513 #[test]
5514 fn explicitly_resumed_older_run_becomes_current_without_duplicate_history() {
5515 let dir = tempfile::tempdir().unwrap();
5516 let q = Queue::at(dir.path().join("queue"));
5517 let mut t = task("resume");
5518 t.start("A".into());
5519 t.record_pass("A", 0, 1, None);
5520 t.start("B".into());
5521 q.put(&mut t).unwrap();
5522 assert_eq!(t.runs, vec!["A", "B"]);
5523 t.start("A".into());
5524 t.record_pass("A", 1, 3, None);
5525 q.put(&mut t).unwrap();
5526 let saved = q.get(&t.id).unwrap();
5527 assert_eq!(saved.runs, vec!["B", "A"]);
5528 assert_eq!(saved.runs.last().map(String::as_str), Some("A"));
5529 assert_eq!(saved.attempts, 3);
5530 assert_eq!(saved.passes.len(), 2);
5531 assert_eq!(saved.passes[0].run, "A");
5532 assert_eq!(saved.passes[1].run, "A");
5533 }
5534
5535 #[test]
5536 fn repeated_start_records_one_run_and_separate_passes() {
5537 let mut t = task("resume");
5538 t.start("same-run".into());
5539 t.record_pass("same-run", 0, 1, None);
5540 t.start("same-run".into());
5541 t.record_pass("same-run", 1, 2, None);
5542 assert_eq!(t.runs, vec!["same-run"]);
5543 assert_eq!(t.attempts, 2);
5544 assert_eq!(t.passes.len(), 2);
5545 }
5546
5547 #[test]
5548 fn link_run_adds_a_run_once_and_touches_nothing_else() {
5549 let dir = tempfile::tempdir().unwrap();
5550 let queue = Queue::at(dir.path().join("queue"));
5551 let mut t = Task::new(
5552 "t".to_owned(),
5553 "do it".to_owned(),
5554 PathBuf::from("."),
5555 Source::Human,
5556 );
5557 queue.put(&mut t).unwrap();
5558 let before = queue.get(&t.id).unwrap();
5559
5560 let linked = queue.link_run(&t.id[..4], "20260930-092817-ec34").unwrap();
5561 assert_eq!(linked.runs, vec!["20260930-092817-ec34".to_owned()]);
5562 assert_eq!(linked.status, before.status);
5563 assert_eq!(linked.attempts, before.attempts);
5564 assert_eq!(linked.interrupt, before.interrupt);
5565
5566 let again = queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
5567 assert_eq!(again.runs.len(), 1, "linking twice must not duplicate");
5568 assert_eq!(queue.get(&t.id).unwrap().runs.len(), 1);
5569 assert!(queue.link_run("no-such-task", "r").is_err());
5570 }
5571
5572 #[test]
5573 fn put_keeps_a_run_linked_after_the_writer_took_its_snapshot() {
5574 let dir = tempfile::tempdir().unwrap();
5575 let queue = Queue::at(dir.path().join("queue"));
5576 let mut t = Task::new(
5577 "t".to_owned(),
5578 "do it".to_owned(),
5579 PathBuf::from("."),
5580 Source::Human,
5581 );
5582 queue.put(&mut t).unwrap();
5583 let mut snapshot = queue.get(&t.id).unwrap();
5585 queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
5586
5587 snapshot.start("20260930-000000-aaaa".to_owned());
5588 queue.put(&mut snapshot).unwrap();
5589
5590 let stored = queue.get(&t.id).unwrap();
5591 assert!(stored.runs.contains(&"20260930-092817-ec34".to_owned()));
5592 assert!(stored.runs.contains(&"20260930-000000-aaaa".to_owned()));
5593 assert_eq!(stored.attempts, 1);
5594 }
5595
5596 #[test]
5597 fn concurrent_links_and_daemon_saves_lose_nothing() {
5598 let dir = tempfile::tempdir().unwrap();
5599 let queue = Queue::at(dir.path().join("queue"));
5600 let mut t = Task::new(
5601 "t".to_owned(),
5602 "do it".to_owned(),
5603 PathBuf::from("."),
5604 Source::Human,
5605 );
5606 queue.put(&mut t).unwrap();
5607 let id = t.id.clone();
5608
5609 let linkers: Vec<_> = (0..4)
5610 .map(|n| {
5611 let (queue, id) = (queue.clone(), id.clone());
5612 std::thread::spawn(move || {
5613 for k in 0..10 {
5614 queue
5615 .link_run(&id, &format!("20260930-00000{n}-l{k:03}"))
5616 .unwrap();
5617 }
5618 })
5619 })
5620 .collect();
5621 let mut mine = queue.get(&id).unwrap();
5624 for k in 0..10 {
5625 mine.start(format!("20260930-000009-d{k:03}"));
5626 queue.put(&mut mine).unwrap();
5627 }
5628 for l in linkers {
5629 l.join().unwrap();
5630 }
5631
5632 let stored = queue.get(&id).unwrap();
5633 assert_eq!(stored.runs.len(), 50, "{:?}", stored.runs);
5634 assert_eq!(
5635 stored.attempts, 10,
5636 "linking never rewinds the daemon's work"
5637 );
5638 }
5639
5640 fn age_lock(path: &Path) {
5641 let f = std::fs::OpenOptions::new().write(true).open(path).unwrap();
5642 f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(60))
5643 .unwrap();
5644 }
5645
5646 #[test]
5647 fn concurrent_stale_takeover_yields_one_holder() {
5648 use std::sync::atomic::{AtomicUsize, Ordering};
5649 use std::sync::{Arc, Barrier};
5650 for _ in 0..5 {
5651 let dir = tempfile::tempdir().unwrap();
5652 let q = Queue::at(dir.path().to_path_buf());
5653 let lock = dir.path().join("t.write-lock");
5654 std::fs::write(&lock, "dead-0000").unwrap();
5655 age_lock(&lock);
5656 let n = 6;
5657 let barrier = Arc::new(Barrier::new(n));
5658 let (now, max) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
5659 let handles: Vec<_> = (0..n)
5660 .map(|_| {
5661 let (q, b, now, max) = (q.clone(), barrier.clone(), now.clone(), max.clone());
5662 std::thread::spawn(move || {
5663 b.wait();
5664 let g = q.lock_task("t").unwrap();
5665 let held = now.fetch_add(1, Ordering::SeqCst) + 1;
5666 max.fetch_max(held, Ordering::SeqCst);
5667 std::thread::sleep(std::time::Duration::from_millis(20));
5668 now.fetch_sub(1, Ordering::SeqCst);
5669 drop(g);
5670 })
5671 })
5672 .collect();
5673 for h in handles {
5674 h.join().unwrap();
5675 }
5676 assert_eq!(max.load(Ordering::SeqCst), 1);
5677 assert!(!lock.exists());
5678 }
5679 }
5680
5681 #[test]
5682 fn dropping_a_replaced_lock_leaves_the_new_holders_token() {
5683 let dir = tempfile::tempdir().unwrap();
5684 let q = Queue::at(dir.path().to_path_buf());
5685 let lock = dir.path().join("t.write-lock");
5686 let a = q.lock_task("t").unwrap();
5687 let replacement = owner_token();
5690 std::fs::write(&lock, &replacement).unwrap();
5691 drop(a);
5692 assert_eq!(std::fs::read_to_string(&lock).unwrap(), replacement);
5693 }
5694
5695 #[test]
5696 fn aged_live_task_lock_excludes_updates_until_retirement_releases_it() {
5697 let dir = tempfile::tempdir().unwrap();
5698 let q = Queue::at(dir.path().to_path_buf());
5699 let held = q.lock_task("t").unwrap();
5700 let path = dir.path().join("t.write-lock");
5701 age_lock(&path);
5702 assert!(!break_stale(&path, &held.token));
5703 let (started_tx, started_rx) = std::sync::mpsc::channel();
5704 let (acquired_tx, acquired_rx) = std::sync::mpsc::channel();
5705 let updater = std::thread::spawn(move || {
5706 started_tx.send(()).unwrap();
5707 let _write = q.lock_task("t").unwrap();
5708 acquired_tx.send(()).unwrap();
5709 });
5710 started_rx.recv().unwrap();
5711 assert!(
5712 acquired_rx
5713 .recv_timeout(std::time::Duration::from_millis(100))
5714 .is_err()
5715 );
5716 assert_eq!(std::fs::read_to_string(&path).unwrap(), held.token);
5717 drop(held);
5718 acquired_rx
5719 .recv_timeout(std::time::Duration::from_secs(2))
5720 .unwrap();
5721 updater.join().unwrap();
5722 assert!(!path.exists());
5723 }
5724
5725 #[test]
5726 fn break_stale_leaves_a_lock_that_replaced_the_one_judged() {
5727 let dir = tempfile::tempdir().unwrap();
5728 let lock = dir.path().join("t.write-lock");
5729 std::fs::write(&lock, "new-token").unwrap();
5730 assert!(!break_stale(&lock, "old-token"));
5731 assert_eq!(std::fs::read_to_string(&lock).unwrap(), "new-token");
5732 assert!(!break_stale(&lock, "new-token"));
5734 assert!(lock.exists());
5735 age_lock(&lock);
5736 assert!(break_stale(&lock, "new-token"));
5737 assert!(!lock.exists());
5738 }
5739
5740 #[test]
5741 fn a_stale_break_marker_is_recovered_and_a_fresh_one_is_respected() {
5742 let dir = tempfile::tempdir().unwrap();
5743 let lock = dir.path().join("t.write-lock");
5744 std::fs::write(&lock, "dead-1").unwrap();
5745 age_lock(&lock);
5746 let marker = break_marker(&lock, "dead-1");
5747 std::fs::write(&marker, "crashed-remover").unwrap();
5748 assert!(!break_stale(&lock, "dead-1"));
5750 assert!(lock.exists());
5751 age_lock(&marker);
5753 assert!(break_stale(&lock, "dead-1"));
5754 assert!(!lock.exists());
5755 assert!(!marker.exists());
5756 }
5757
5758 fn chat_task() -> Task {
5759 Task::new(
5760 "t".to_owned(),
5761 "do t".to_owned(),
5762 PathBuf::from("."),
5763 Source::Agent {
5764 run: "20260902-000000-beef".to_owned(),
5765 node: CHAT_NODE.to_owned(),
5766 },
5767 )
5768 }
5769
5770 #[test]
5771 fn a_blocked_hold_keeps_its_anchor_but_a_new_run_starts_a_new_hold() {
5772 let mut t = chat_task();
5773 t.hold_machine(Some("waiting".into()));
5774 let old: Timestamp = "2020-01-01T00:00:00Z".parse().unwrap();
5775 t.held_at = Some(old);
5776 t.block(vec!["decision".into()], None);
5777 t.hold_machine(Some("same waiting decision".into()));
5778 assert_eq!(t.held_at, Some(old));
5779 t.start("real-new-run".into());
5780 t.fail("new failure", 1);
5781 assert_ne!(
5782 t.held_at,
5783 Some(old),
5784 "old answers cannot approve a new hold"
5785 );
5786 }
5787
5788 #[test]
5789 fn legacy_delivery_survives_upgrade_and_a_no_progress_run_link() {
5790 let (_dir, queue) = queue();
5791 let mut t = chat_task();
5792 t.start("old-run".into());
5793 t.hold_machine(Some("operator is considering recovery".into()));
5794 t.chat_notice = Some(ChatNotice {
5795 key: format!("held#{}", t.held_at.unwrap()),
5796 at: Timestamp::now(),
5797 outcome: ChatNoticeOutcome::Posted,
5798 });
5799 let mut raw = serde_json::to_value(&t).unwrap();
5800 raw.as_object_mut().unwrap().remove("chat_notices");
5801 raw.as_object_mut().unwrap().remove("hold_notice_cause");
5802 std::fs::create_dir_all(queue.path_of(&t.id).parent().unwrap()).unwrap();
5803 std::fs::write(queue.path_of(&t.id), serde_json::to_string(&raw).unwrap()).unwrap();
5804 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
5805 queue
5806 .modify(&t.id, |task| {
5807 task.requeue();
5808 task.link_run("unprepared-run");
5809 task.startup_notice_cause("startup-source");
5810 task.fail("same unresolved preflight", 0);
5811 true
5812 })
5813 .unwrap();
5814 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
5815 queue
5816 .modify(&t.id, |task| {
5817 task.succeed();
5818 true
5819 })
5820 .unwrap();
5821 assert_eq!(
5822 queue.get(&t.id).unwrap().chat_notice_due().as_deref(),
5823 Some("done")
5824 );
5825 }
5826
5827 #[test]
5828 fn a_dependency_wait_does_not_change_when_released_and_blocked_again() {
5829 let mut t = chat_task();
5830 t.block(vec!["a".into(), "b".into()], Some("waiting".into()));
5831 let key = t.chat_notice_key().unwrap();
5832 t.chat_notices.push(key);
5833 t.release();
5834 t.block(
5835 vec!["b".into(), "a".into(), "a".into()],
5836 Some("still waiting on prerequisites".into()),
5837 );
5838 assert_eq!(t.chat_notice_due(), None);
5839 t.block(
5840 vec!["new-question".into()],
5841 Some("different decision".into()),
5842 );
5843 assert!(t.chat_notice_due().is_some());
5844 }
5845
5846 #[test]
5847 fn running_startup_failures_preserve_stage_and_report_distinct_blockers() {
5848 let mut t = chat_task();
5849 t.start("existing-run".into());
5850 t.fail("implementation failed", 1);
5851 let failed = t.chat_notice_key().unwrap();
5852 t.chat_notices.push(failed.clone());
5853 t.release_operator("test", None);
5854 t.start("existing-run".into());
5855 t.fail_startup("startup-source", "same unresolved recovery source", 1);
5856 assert_eq!(
5857 t.chat_notice_due(),
5858 None,
5859 "no-progress recovery stays quiet"
5860 );
5861 t.release_operator("test", None);
5862 t.start("existing-run".into());
5863 t.fail_startup("startup-attachments", "attachment missing", 1);
5864 let attachments = t.chat_notice_due().expect("new attachment blocker");
5865 assert_ne!(attachments, failed);
5866 assert_eq!(
5867 t.hold_notice_cause.as_ref().unwrap().stage,
5868 "startup-attachments"
5869 );
5870 t.chat_notices.push(attachments);
5871 t.release_operator("test", None);
5872 t.start("existing-run".into());
5873 t.fail_startup(
5874 "startup-attachments",
5875 "same missing attachment, reworded",
5876 1,
5877 );
5878 assert_eq!(
5879 t.chat_notice_due(),
5880 None,
5881 "same stage after attempt counter reset"
5882 );
5883 t.release_operator("test", None);
5884 t.start("existing-run".into());
5885 t.fail_startup("startup-config", "different preflight stage", 1);
5886 assert!(t.chat_notice_due().is_some());
5887 assert_eq!(
5888 t.hold_notice_cause.as_ref().unwrap().stage,
5889 "startup-config"
5890 );
5891 }
5892
5893 #[test]
5894 fn structured_causes_preserve_no_progress_retries_and_report_new_passes() {
5895 let mut t = chat_task();
5896 t.hold_machine(Some("startup source missing".into()));
5897 t.startup_notice_cause("startup-source");
5898 let first = t.chat_notice_key().unwrap();
5899 t.chat_notices.push(first.clone());
5900 t.release_operator("test", None);
5901 t.startup_notice_cause("startup-source");
5902 t.hold_machine(Some("same source cannot be read".into()));
5903 assert_eq!(t.chat_notice_due(), None);
5904 t.startup_notice_cause("startup-config");
5905 assert!(t.chat_notice_due().is_some(), "different startup stage");
5906 t.notice_cause("failed", Some("existing-run"), Some(0));
5907 let run_failure = t.chat_notice_key().unwrap();
5908 t.chat_notices.push(run_failure.clone());
5909 t.release_operator("test", None);
5910 t.startup_notice_cause("startup-source");
5911 t.hold_machine(Some("no new work happened".into()));
5912 assert_eq!(t.chat_notice_due(), None);
5913 t.notice_cause("failed", Some("existing-run"), Some(1));
5914 assert!(t.chat_notice_due().is_some(), "a real resumed pass failed");
5915 assert_ne!(t.chat_notice_key().unwrap(), run_failure);
5916 }
5917
5918 #[test]
5919 fn legacy_hold_baseline_does_not_replay_or_hide_new_failures_and_completion() {
5920 let mut t = chat_task();
5921 t.hold_machine(Some("still waiting".into()));
5922 t.chat_notice = Some(ChatNotice {
5923 key: "held#2026-10-14T00:00:00Z".into(),
5924 at: Timestamp::now(),
5925 outcome: ChatNoticeOutcome::Posted,
5926 });
5927 assert_eq!(t.chat_notice_due(), None, "old hold baseline");
5928 t.import_chat_notice();
5929 t.notice_cause("failed", Some("new-run"), Some(0));
5930 assert!(t.chat_notice_due().is_some(), "new producer evidence");
5931 t.succeed();
5932 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
5933 }
5934
5935 #[test]
5936 fn a_chat_notice_is_due_only_for_a_final_state_and_only_once() {
5937 let mut t = chat_task();
5938 assert_eq!(t.chat_notice_due(), None, "queued");
5939 t.start("r1".to_owned());
5940 assert_eq!(t.chat_notice_due(), None, "running");
5941 t.fail("boom", 3);
5942 assert_eq!(t.status, TaskStatus::Failed);
5943 assert_eq!(t.chat_notice_due(), None, "a retried failure is not final");
5944 t.start("r2".to_owned());
5945 t.park("upgrade");
5946 assert_eq!(t.chat_notice_due(), None, "parked");
5947 t.fail("boom", 1);
5948 assert_eq!(t.status, TaskStatus::Held);
5949 let key = t
5950 .chat_notice_due()
5951 .expect("held on the last attempt is final");
5952 t.chat_notice = Some(ChatNotice {
5953 key,
5954 at: Timestamp::now(),
5955 outcome: ChatNoticeOutcome::Posted,
5956 });
5957 assert_eq!(t.chat_notice_due(), None, "same hold, no second notice");
5958 t.release();
5959 assert_eq!(t.chat_notice_due(), None, "requeued after a release");
5960 t.start("r3".to_owned());
5961 t.succeed();
5962 assert!(t.chat_notice_due().is_some(), "done is a new final state");
5963 }
5964
5965 #[test]
5966 fn completed_chat_notices_keep_current_and_legacy_keys_after_requeue() {
5967 for key in ["done", "done#0", "done#1", "done#12"] {
5968 let mut t = chat_task();
5969 t.start("r1".to_owned());
5970 t.succeed();
5971 t.chat_notice = Some(ChatNotice {
5972 key: key.to_owned(),
5973 at: Timestamp::now(),
5974 outcome: ChatNoticeOutcome::Posted,
5975 });
5976 t.requeue();
5977 t.start("r2".to_owned());
5978 t.succeed();
5979 assert_eq!(t.chat_notice_key().as_deref(), Some("done"));
5980 assert_eq!(t.chat_notice_due(), None, "recorded key: {key}");
5981 }
5982 }
5983
5984 #[test]
5985 fn a_release_without_progress_keeps_the_unresolved_notice_cause() {
5986 let mut t = chat_task();
5987 t.hold_manual(Some("wait".to_owned()));
5988 let first = t.chat_notice_due().expect("held");
5989 t.chat_notice = Some(ChatNotice {
5990 key: first.clone(),
5991 at: Timestamp::now(),
5992 outcome: ChatNoticeOutcome::Posted,
5993 });
5994 t.hold_manual(Some("still waiting".to_owned()));
5995 assert_eq!(t.chat_notice_due(), None, "a re-hold keeps its start");
5996 t.release();
5997 std::thread::sleep(std::time::Duration::from_millis(5));
5998 t.hold_manual(None);
5999 assert_eq!(t.chat_notice_due(), None, "no progress, same cause");
6000 }
6001
6002 #[test]
6003 fn a_chat_notice_needs_a_chat_source_and_a_current_schema() {
6004 let mut human = task("h");
6005 human.succeed();
6006 assert_eq!(human.chat_notice_due(), None);
6007 let mut inherited = task("i");
6009 inherited.origin_chat = Some("20260902-000000-beef".to_owned());
6010 inherited.succeed();
6011 assert_eq!(inherited.chat_notice_due(), None);
6012 let mut old = chat_task();
6013 old.schema = 12;
6014 old.status = TaskStatus::Done;
6015 assert_eq!(old.chat_notice_due(), None, "history is not replayed");
6016 let mut live = chat_task();
6017 live.schema = 12;
6018 live.start("r1".to_owned());
6019 live.succeed();
6020 assert!(live.chat_notice_due().is_some(), "started by this build");
6021 let mut held = chat_task();
6022 held.schema = 12;
6023 held.hold_machine(Some("config".to_owned()));
6024 assert!(held.chat_notice_due().is_some(), "held before any start");
6025 }
6026
6027 #[test]
6028 fn a_stale_snapshot_does_not_rewind_a_recorded_chat_notice() {
6029 let tmp = tempfile::tempdir().expect("tempdir");
6030 let q = Queue::at(tmp.path().join("queue"));
6031 let mut t = chat_task();
6032 t.succeed();
6033 q.put(&mut t).expect("put");
6034 let mut stale = q.get(&t.id).expect("get");
6035 let key = t.chat_notice_due().expect("due");
6036 q.modify(&t.id, |x| {
6037 x.chat_notice = Some(ChatNotice {
6038 key: key.clone(),
6039 at: Timestamp::now(),
6040 outcome: ChatNoticeOutcome::Posted,
6041 });
6042 true
6043 })
6044 .expect("modify");
6045 q.put(&mut stale).expect("stale put");
6046 assert_eq!(q.get(&t.id).expect("get").chat_notice_due(), None);
6047 }
6048
6049 #[test]
6050 fn empty_sources_are_omitted_so_older_builds_still_read_the_task() {
6051 let mut t = task("plain");
6054 let value = serde_json::to_value(&t).unwrap();
6055 assert!(value.get("sources").is_none(), "{value}");
6056 t.sources.pick_commits.push(crate::refs::Input {
6057 token: "abc1234".into(),
6058 sha: "a".repeat(40),
6059 provenance: crate::refs::Provenance::ExplicitCli,
6060 });
6061 let value = serde_json::to_value(&t).unwrap();
6062 assert_eq!(value["sources"]["pick_commits"][0]["sha"], "a".repeat(40));
6063 let restored: Task = serde_json::from_value(value).unwrap();
6064 assert_eq!(restored.sources, t.sources);
6065 }
6066
6067 fn queue_with_ids(dir: &Path, ids: &[&str]) -> Queue {
6069 let ids: Vec<String> = ids.iter().map(|s| (*s).to_owned()).collect();
6070 let n = std::sync::atomic::AtomicUsize::new(0);
6071 Queue::at(dir.join("queue")).with_id_source(move || {
6072 let i = n.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6073 ids.get(i)
6074 .cloned()
6075 .unwrap_or_else(|| format!("20260101-000000-f{i:03x}"))
6076 })
6077 }
6078
6079 const TAKEN: &str = "20260101-000000-aaaa";
6080
6081 #[test]
6082 fn a_new_task_never_merges_into_a_record_with_its_candidate_id() {
6083 let dir = tempfile::tempdir().unwrap();
6084 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6085 let mut first = task("first");
6086 first.start("run-1".into());
6087 first.succeed();
6088 q.put(&mut first).unwrap();
6089 assert_eq!(first.id, TAKEN);
6090 let stored_before = q.get(&first.id).unwrap();
6091 assert!(!stored_before.runs.is_empty());
6092
6093 let mut second = task("second");
6094 q.put(&mut second).unwrap();
6095
6096 assert_ne!(second.id, first.id);
6097 assert!(second.runs.is_empty());
6098 assert!(second.passes.is_empty());
6099 assert!(second.chat_notices.is_empty());
6100 assert!(second.chat_notice.is_none());
6101 assert_eq!(second.chat_notice_due(), task("fresh").chat_notice_due());
6102 let a = q.get(&first.id).unwrap();
6103 let b = q.get(&second.id).unwrap();
6104 assert_eq!(a.title, "first");
6105 assert_eq!(a.runs, stored_before.runs);
6106 assert_eq!(a.passes, stored_before.passes);
6107 assert_eq!(a.chat_notices, stored_before.chat_notices);
6108 assert_eq!(a.chat_notice, stored_before.chat_notice);
6109 assert_eq!(a.chat_notice_due(), stored_before.chat_notice_due());
6110 assert_eq!(b.title, "second");
6111 assert!(b.runs.is_empty());
6112 assert!(b.passes.is_empty());
6113 assert!(b.chat_notice.is_none());
6114
6115 let mut again = q.get(&second.id).unwrap();
6117 again.title = "second, edited".into();
6118 q.put(&mut again).unwrap();
6119 assert_eq!(q.get(&second.id).unwrap().title, "second, edited");
6120 assert_eq!(q.list().len(), 2);
6121 }
6122
6123 #[test]
6124 fn a_new_task_skips_an_id_with_an_orphaned_claim_or_attachments() {
6125 let dir = tempfile::tempdir().unwrap();
6126 let q = queue_with_ids(dir.path(), &["orphan-lock", "orphan-dir", "free"]);
6127 std::fs::create_dir_all(q.root()).unwrap();
6128 std::fs::write(q.root().join("orphan-lock.lock"), "x").unwrap();
6129 std::fs::create_dir_all(q.attachments_dir("orphan-dir")).unwrap();
6130 let mut t = task("t");
6131 q.put(&mut t).unwrap();
6132 assert_eq!(t.id, "free");
6133 }
6134
6135 #[test]
6136 fn a_claimed_new_task_holds_the_claim_on_the_reserved_id() {
6137 let dir = tempfile::tempdir().unwrap();
6138 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6139 let mut first = task("first");
6140 q.put(&mut first).unwrap();
6141 let mut second = task("second");
6142 let claim = q.file_claimed(&mut second).unwrap();
6143 assert_ne!(second.id, first.id);
6144 assert!(q.claim(&second.id).is_err());
6145 assert!(q.claim(&first.id).is_ok(), "the first task was not claimed");
6146 drop(claim);
6147 }
6148
6149 #[test]
6150 fn an_attached_new_task_skips_a_taken_id() {
6151 let dir = tempfile::tempdir().unwrap();
6152 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6153 let mut first = task("first");
6154 q.put(&mut first).unwrap();
6155 let src = source_file(dir.path(), "shot.png", "x");
6156 let mut second = task("second");
6157 q.attach_and_put(&mut second, &[src]).unwrap();
6158 assert_ne!(second.id, first.id);
6159 assert!(!q.attachments_dir(&first.id).exists());
6160 assert_eq!(q.attachment_paths(&second).len(), 1);
6161 assert!(q.attachment_paths(&second)[0].is_file());
6162 }
6163
6164 #[test]
6165 fn attaching_before_putting_a_new_task_reserves_the_id() {
6166 let dir = tempfile::tempdir().unwrap();
6167 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6168 let mut first = task("first");
6169 q.put(&mut first).unwrap();
6170 let src = source_file(dir.path(), "shot.png", "x");
6171 let mut second = task("second");
6172 q.attach(&mut second, &[src]).unwrap();
6173 assert_ne!(second.id, first.id);
6174 assert!(q.attachments_dir(&second.id).is_dir());
6175 q.put(&mut second).unwrap();
6176 assert_eq!(q.get(&second.id).unwrap().attachments.len(), 1);
6177 assert_eq!(q.get(&first.id).unwrap().title, "first");
6178 assert!(q.get(&first.id).unwrap().attachments.is_empty());
6179 }
6180
6181 #[test]
6182 fn create_new_keeps_its_already_exists_meaning() {
6183 let dir = tempfile::tempdir().unwrap();
6184 let q = Queue::at(dir.path().join("queue"));
6185 let mut a = task("a");
6186 a.id = "fixed-id".into();
6187 assert!(q.create_new(&mut a).unwrap());
6188 let mut b = task("b");
6189 b.id = "fixed-id".into();
6190 assert!(!q.create_new(&mut b).unwrap());
6191 assert_eq!(q.get("fixed-id").unwrap().title, "a");
6192 }
6193
6194 #[test]
6195 fn filing_gives_up_when_no_id_is_ever_free() {
6196 let dir = tempfile::tempdir().unwrap();
6197 let q = Queue::at(dir.path().join("queue")).with_id_source(|| TAKEN.to_owned());
6198 let mut first = task("first");
6199 q.put(&mut first).unwrap();
6200 let mut second = task("second");
6201 assert!(q.put(&mut second).is_err());
6202 assert_eq!(q.get(&first.id).unwrap().title, "first");
6203 }
6204}