1use std::collections::HashMap;
34use std::num::NonZeroUsize;
35use std::path::{Path, PathBuf};
36
37use anyhow::{Context, Result, bail};
38use jiff::Timestamp;
39use serde::{Deserialize, Serialize};
40
41use crate::ask::Questions;
42
43pub const SCHEMA: u32 = 18;
130
131#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
133#[serde(rename_all = "lowercase")]
134pub enum HoldSource {
135 Manual,
137 Machine,
139}
140
141impl HoldSource {
142 pub fn label(self) -> &'static str {
144 match self {
145 Self::Manual => "manual",
146 Self::Machine => "machine",
147 }
148 }
149}
150
151#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
154#[serde(tag = "kind", rename_all = "lowercase")]
155pub enum Source {
156 Human,
158 Agent {
161 run: String,
163 node: String,
165 },
166 Issue {
168 number: u64,
170 repo: String,
172 },
173}
174
175pub const CHAT_NODE: &str = "chat";
178
179impl Source {
180 pub fn label(&self) -> String {
182 match self {
183 Self::Human => "human".to_owned(),
184 Self::Agent { run, node } => format!("{node}@{}", short(run)),
185 Self::Issue { number, .. } => format!("issue #{number}"),
186 }
187 }
188}
189
190#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
192#[serde(rename_all = "lowercase")]
193pub enum TaskStatus {
194 Queued,
196 Running,
198 Done,
200 Failed,
202 Held,
204 Blocked,
208 Parked,
213}
214
215impl TaskStatus {
216 pub fn runnable(self) -> bool {
218 matches!(self, Self::Queued | Self::Failed | Self::Parked)
219 }
220
221 pub fn as_str(self) -> &'static str {
223 match self {
224 Self::Queued => "queued",
225 Self::Running => "running",
226 Self::Done => "done",
227 Self::Failed => "failed",
228 Self::Held => "held",
229 Self::Blocked => "blocked",
230 Self::Parked => "parked",
231 }
232 }
233}
234
235#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
238pub struct TaskCounts {
239 pub queued: usize,
241 pub running: usize,
243 pub done: usize,
245 pub failed: usize,
247 pub held: usize,
249 pub blocked: usize,
251 pub parked: usize,
253}
254
255impl TaskCounts {
256 pub fn of(tasks: &[Task]) -> Self {
260 Self::of_statuses(tasks.iter().map(|task| task.status))
261 }
262
263 pub(crate) fn of_statuses(statuses: impl IntoIterator<Item = TaskStatus>) -> Self {
265 let mut counts = Self::default();
266 for status in statuses {
267 match status {
268 TaskStatus::Queued => counts.queued += 1,
269 TaskStatus::Running => counts.running += 1,
270 TaskStatus::Done => counts.done += 1,
271 TaskStatus::Failed => counts.failed += 1,
272 TaskStatus::Held => counts.held += 1,
273 TaskStatus::Blocked => counts.blocked += 1,
274 TaskStatus::Parked => counts.parked += 1,
275 }
276 }
277 counts
278 }
279}
280
281#[derive(Debug, Clone, Serialize, Deserialize)]
283#[serde(deny_unknown_fields)]
284pub struct Task {
285 pub schema: u32,
287 pub id: String,
289 pub title: String,
291 pub instruction: String,
293 #[serde(default, skip_serializing_if = "crate::refs::Sources::is_empty")]
295 pub sources: crate::refs::Sources,
296 pub repo: PathBuf,
298 pub source: Source,
300 #[serde(default)]
302 pub priority: i32,
303 #[serde(default)]
314 pub solo: bool,
315 pub status: TaskStatus,
317 #[serde(default)]
319 pub attempts: usize,
320 #[serde(default)]
322 pub attempt_limit: Option<NonZeroUsize>,
323 #[serde(default)]
326 pub runs: Vec<String>,
327 #[serde(default)]
329 pub last_error: Option<String>,
330 #[serde(default)]
343 pub hold_reason: Option<String>,
344 #[serde(default)]
347 pub hold_source: Option<HoldSource>,
348 #[serde(default)]
360 pub diagnostic: Option<String>,
361 #[serde(default)]
371 pub blocked_by: Vec<String>,
372 #[serde(default)]
375 pub block_reason: Option<String>,
376 #[serde(default)]
389 pub blocked_from: Option<TaskStatus>,
390 #[serde(default)]
392 pub blocked_at: Option<Timestamp>,
393 #[serde(default)]
404 pub answers: Vec<AnsweredQuestion>,
405 #[serde(default)]
411 pub triage_applied: Vec<String>,
412 #[serde(default)]
418 pub actions_applied: Vec<String>,
419 #[serde(default)]
424 pub resume_override: Option<OperatorResume>,
425 #[serde(default)]
428 pub resumption: Option<Resumption>,
429 #[serde(default)]
431 pub resumption_history: Vec<Resumption>,
432 #[serde(default)]
434 pub operator_decided_at: Option<Timestamp>,
435 #[serde(default)]
447 pub review_branch: Option<String>,
448 #[serde(default)]
450 pub review_instruction: Option<String>,
451 #[serde(default)]
453 pub pr_recovery: Option<crate::pr_recovery::Selection>,
454 #[serde(default)]
457 pub fresh_start: bool,
458 #[serde(default)]
460 pub replacement: Option<crate::merge_decision::Replacement>,
461 #[serde(default)]
472 pub interrupt: bool,
473 #[serde(default)]
496 pub urgent: bool,
497 #[serde(default)]
503 pub attachments: Vec<String>,
504 #[serde(default)]
508 pub followup: Option<FollowUp>,
509 #[serde(default)]
515 pub origin_chat: Option<String>,
516 #[serde(default)]
523 pub overrides: Option<RunOverrides>,
524 #[serde(default)]
531 pub review_of: Option<String>,
532 #[serde(default)]
537 pub held_at: Option<Timestamp>,
538 #[serde(default)]
542 pub park_reason: Option<String>,
543 #[serde(default)]
546 pub chat_notice: Option<ChatNotice>,
547 #[serde(default)]
549 pub chat_notices: Vec<String>,
550 #[serde(default)]
553 pub hold_notice_cause: Option<NoticeCause>,
554 #[serde(default)]
559 pub passes: Vec<TaskPass>,
560 #[serde(default)]
562 pub recovery_budget: Option<RecoveryBudget>,
563 #[serde(default)]
566 pub recovery_admission: Option<RecoveryAdmission>,
567 #[serde(default)]
570 pub recovery_generations: Vec<RecoveryBudget>,
571 pub created_at: Timestamp,
573 pub updated_at: Timestamp,
575}
576
577#[derive(Debug, Clone, Serialize, Deserialize)]
579pub struct RecoveryAdmission {
580 pub base: String,
582 pub inputs: String,
584}
585
586#[derive(Debug, Clone, Serialize, Deserialize)]
589pub struct RecoveryBudget {
590 pub instruction: String,
592 pub base: String,
594 pub inputs: String,
596 pub stage: String,
598 pub failures: usize,
600 #[serde(default)]
603 pub progress: Option<String>,
604}
605
606fn recovery_progress(state: &crate::run::RunState) -> Option<String> {
607 use crate::proc::Quiet as _;
608 use std::hash::{Hash, Hasher};
609 let mut patches = Vec::new();
610 for candidate in state.candidates.iter().filter(|c| c.commits > 0) {
611 let output = std::process::Command::new("git")
612 .current_dir(&state.repo)
613 .args(["diff", "--binary", &state.base_commit, &candidate.branch])
614 .quiet()
615 .output()
616 .ok()?;
617 if !output.status.success() {
618 return None;
619 }
620 patches.push(output.stdout);
621 }
622 patches.sort();
623 patches.dedup();
624 let mut hash = std::collections::hash_map::DefaultHasher::new();
625 patches.hash(&mut hash);
626 Some(format!("{:x}", hash.finish()))
627}
628
629pub(crate) fn recovery_inputs(sources: &crate::refs::Sources) -> String {
633 serde_json::to_string(
634 &sources
635 .start_from
636 .iter()
637 .map(|i| ("start", &i.token, &i.sha))
638 .chain(
639 sources
640 .pick_commits
641 .iter()
642 .map(|i| ("pick", &i.token, &i.sha)),
643 )
644 .collect::<Vec<_>>(),
645 )
646 .unwrap_or_default()
647}
648
649#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
651pub struct NoticeCause {
652 pub stage: String,
654 #[serde(default, skip_serializing_if = "Option::is_none")]
657 pub target: Option<String>,
658 pub run: Option<String>,
660 pub pass: Option<usize>,
662}
663
664pub(crate) fn is_done_notice_key(key: &str) -> bool {
666 key == "done"
667 || key
668 .strip_prefix("done#")
669 .is_some_and(|count| !count.is_empty() && count.bytes().all(|b| b.is_ascii_digit()))
670}
671
672#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
674#[serde(rename_all = "snake_case")]
675pub enum PassCost {
676 Spent,
678 Refunded,
680}
681
682#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
684pub struct TaskPass {
685 pub run: String,
687 pub pass_index: usize,
689 pub cost: PassCost,
691 pub attempts_after: usize,
693 pub settled_at: Timestamp,
695 #[serde(default)]
697 pub parked: Option<crate::run::ParkCause>,
698}
699
700#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
702#[serde(rename_all = "lowercase")]
703pub enum ChatNoticeOutcome {
704 Posted,
706 Skipped,
708}
709
710#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
712pub struct ChatNotice {
713 pub key: String,
715 pub at: Timestamp,
717 pub outcome: ChatNoticeOutcome,
719}
720
721#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
723pub struct RunOverrides {
724 #[serde(default)]
727 pub inherit_daemon_options: bool,
728 #[serde(default)]
730 pub merge: Option<String>,
731 #[serde(default)]
733 pub candidates: Option<usize>,
734 #[serde(default)]
736 pub judges: Option<usize>,
737 #[serde(default)]
739 pub reviewers: Option<usize>,
740 #[serde(default)]
742 pub review_rounds: Option<usize>,
743 #[serde(default)]
745 pub seed: Option<u64>,
746 #[serde(default)]
748 pub config: Option<PathBuf>,
749}
750
751impl RunOverrides {
752 pub fn merge_over(&mut self, other: &RunOverrides) {
754 macro_rules! take {
755 ($($f:ident),*) => {$(
756 if other.$f.is_some() {
757 self.$f.clone_from(&other.$f);
758 }
759 )*};
760 }
761 take!(
762 merge,
763 candidates,
764 judges,
765 reviewers,
766 review_rounds,
767 seed,
768 config
769 );
770 }
771
772 pub fn apply(&self, config: &mut crate::config::Config) {
774 if let Some(n) = self.candidates {
775 config.graph.implementers = n;
776 }
777 if let Some(n) = self.judges {
778 config.graph.judges = n;
779 }
780 if let Some(n) = self.reviewers {
781 config.graph.reviewers = n;
782 }
783 if let Some(n) = self.review_rounds {
784 config.graph.review_rounds = n;
785 }
786 if let Some(m) = &self.merge
787 && let Ok(mode) = crate::daemon::merge_mode(m)
788 {
789 config.merge.mode = mode;
790 }
791 if let Some(s) = self.seed {
792 config.blind.seed = Some(s);
793 }
794 }
795}
796
797#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
799pub struct FollowUp {
800 pub run: String,
802 #[serde(default)]
804 pub origin_task: Option<String>,
805 pub pr: String,
807 pub findings: Vec<String>,
809 pub generation: u32,
812}
813
814#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
816pub struct Resumption {
817 pub generation: u64,
819 pub source: String,
821 pub question_id: Option<String>,
823 pub at: Timestamp,
825 pub instruction: String,
827 pub answers_through: usize,
829 pub runs_through: usize,
832 pub cleared_reason: Option<String>,
834 #[serde(default)]
836 pub blocking_evidence: Vec<String>,
837}
838
839#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
842pub struct OperatorResume {
843 pub question_id: String,
845 pub at: Timestamp,
847 #[serde(default)]
850 pub conductor_rehold: Option<String>,
851 #[serde(default)]
854 pub forced: bool,
855 #[serde(default)]
862 pub pinned_run: Option<String>,
863}
864
865#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
868pub struct AnsweredQuestion {
869 pub question: String,
871 pub answer: String,
873}
874
875impl Task {
876 pub fn new(title: String, instruction: String, repo: PathBuf, source: Source) -> Self {
878 let now = Timestamp::now();
879 let origin_chat = match &source {
880 Source::Agent { run, node } if node == CHAT_NODE => Some(run.clone()),
881 _ => None,
882 };
883 Self {
884 schema: SCHEMA,
885 id: new_id(),
886 title,
887 instruction,
888 sources: crate::refs::Sources::default(),
889 repo,
890 source,
891 priority: 0,
892 solo: false,
893 status: TaskStatus::Queued,
894 attempts: 0,
895 attempt_limit: None,
896 runs: Vec::new(),
897 last_error: None,
898 hold_reason: None,
899 hold_source: None,
900 diagnostic: None,
901 blocked_by: Vec::new(),
902 block_reason: None,
903 blocked_from: None,
904 blocked_at: None,
905 answers: Vec::new(),
906 triage_applied: Vec::new(),
907 actions_applied: Vec::new(),
908 resume_override: None,
909 resumption: None,
910 resumption_history: Vec::new(),
911 operator_decided_at: None,
912 review_branch: None,
913 review_instruction: None,
914 pr_recovery: None,
915 fresh_start: false,
916 replacement: None,
917 interrupt: false,
918 urgent: false,
919 attachments: Vec::new(),
920 followup: None,
921 origin_chat,
922 overrides: None,
923 review_of: None,
924 held_at: None,
925 park_reason: None,
926 chat_notice: None,
927 chat_notices: Vec::new(),
928 hold_notice_cause: None,
929 passes: Vec::new(),
930 recovery_budget: None,
931 recovery_admission: None,
932 recovery_generations: Vec::new(),
933 created_at: now,
934 updated_at: now,
935 }
936 }
937
938 pub fn chat_talk(&self) -> Option<&str> {
941 if let Some(id) = &self.origin_chat {
942 return Some(id);
943 }
944 match &self.source {
945 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
946 _ => None,
947 }
948 }
949
950 pub fn filed_by_chat(&self) -> Option<&str> {
954 match &self.source {
955 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
956 _ => None,
957 }
958 }
959
960 pub fn chat_notice_key(&self) -> Option<String> {
963 match self.status {
964 TaskStatus::Done => Some("done".to_owned()),
965 TaskStatus::Held => {
966 let cause = self
967 .hold_notice_cause
968 .clone()
969 .unwrap_or_else(|| self.unresolved_hold_cause());
970 Some(format!(
971 "cause:{}",
972 serde_json::to_string(&cause).expect("cause")
973 ))
974 }
975 TaskStatus::Blocked => {
976 let ids: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
977 let evidence = self
980 .hold_notice_cause
981 .as_ref()
982 .filter(|c| c.stage.starts_with("conductor-evidence:"));
983 Some(format!(
984 "dependencies:{}",
985 serde_json::to_string(&(ids, evidence)).expect("ids")
986 ))
987 }
988 _ => None,
989 }
990 }
991
992 fn unresolved_hold_cause(&self) -> NoticeCause {
993 NoticeCause {
994 stage: "hold".into(),
995 target: None,
996 run: self.runs.last().cloned(),
997 pass: None,
998 }
999 }
1000
1001 pub(crate) fn legacy_chat_notice_key(&self) -> Option<String> {
1003 let key = match self.status {
1004 TaskStatus::Held if self.hold_notice_cause.is_none() => Some(format!(
1005 "held#{}",
1006 self.held_at.map(|t| t.to_string()).unwrap_or_default()
1007 )),
1008 TaskStatus::Blocked => Some(format!(
1009 "blocked#{}#{}",
1010 self.blocked_by.join(","),
1011 self.block_reason.as_deref().unwrap_or_default()
1012 )),
1013 _ => None,
1014 };
1015 key.filter(|key| self.legacy_notice_matches(key))
1016 }
1017
1018 pub(crate) fn legacy_notice_matches(&self, key: &str) -> bool {
1019 match self.status {
1020 TaskStatus::Held if self.hold_notice_cause.is_none() => key
1021 .strip_prefix("held#")
1022 .is_some_and(|time| time.is_empty() || time.parse::<Timestamp>().is_ok()),
1023 TaskStatus::Blocked
1024 if self
1025 .hold_notice_cause
1026 .as_ref()
1027 .is_none_or(|c| !c.stage.starts_with("conductor-evidence:")) =>
1028 {
1029 key.strip_prefix("blocked#").is_some_and(|tail| {
1030 let Some((ids, _)) = tail.split_once('#') else {
1031 return false;
1032 };
1033 let old: std::collections::BTreeSet<_> =
1034 ids.split(',').filter(|id| !id.is_empty()).collect();
1035 let current: std::collections::BTreeSet<_> =
1036 self.blocked_by.iter().map(String::as_str).collect();
1037 old == current
1038 })
1039 }
1040 _ => false,
1041 }
1042 }
1043
1044 fn import_chat_notice(&mut self) {
1045 if let Some(n) = &self.chat_notice {
1046 let key = if is_done_notice_key(&n.key) {
1047 "done".to_owned()
1048 } else if self.legacy_notice_matches(&n.key) {
1049 self.chat_notice_key().unwrap_or_else(|| n.key.clone())
1050 } else {
1051 n.key.clone()
1052 };
1053 if !self.chat_notices.contains(&key) {
1054 self.chat_notices.push(key);
1055 }
1056 }
1057 }
1058
1059 pub(crate) fn notice_cause(&mut self, stage: &str, run: Option<&str>, pass: Option<usize>) {
1062 self.import_chat_notice();
1063 self.hold_notice_cause = Some(NoticeCause {
1064 stage: stage.to_owned(),
1065 target: None,
1066 run: run.map(str::to_owned),
1067 pass,
1068 });
1069 }
1070
1071 pub(crate) fn startup_notice_cause(&mut self, stage: &str) {
1074 let recovery_only = matches!(
1075 stage,
1076 "startup-source" | "startup-candidates" | "startup-public-head"
1077 );
1078 match self.hold_notice_cause.as_ref() {
1079 Some(cause) if recovery_only && !cause.stage.starts_with("startup-") => return,
1080 None if recovery_only
1081 && self
1082 .chat_notice
1083 .as_ref()
1084 .is_some_and(|n| !is_done_notice_key(&n.key)) =>
1085 {
1086 return;
1087 }
1088 _ => {}
1089 }
1090 self.notice_cause(stage, None, None);
1091 }
1092
1093 pub(crate) fn fail_startup(
1096 &mut self,
1097 stage: &str,
1098 why: impl Into<String>,
1099 max_attempts: usize,
1100 ) {
1101 let unresolved = self.hold_notice_cause.clone();
1102 self.fail(why, max_attempts);
1103 self.hold_notice_cause = unresolved;
1104 self.startup_notice_cause(stage);
1105 }
1106
1107 pub(crate) fn fail_startup_target(
1109 &mut self,
1110 stage: &str,
1111 target: String,
1112 why: impl Into<String>,
1113 max_attempts: usize,
1114 ) {
1115 self.fail_startup(stage, why, max_attempts);
1116 if let Some(cause) = self.hold_notice_cause.as_mut() {
1117 cause.target = Some(target);
1118 }
1119 }
1120
1121 pub fn chat_notice_due(&self) -> Option<String> {
1123 self.filed_by_chat()?;
1124 if self.schema < 13 {
1125 return None;
1126 }
1127 let key = self.chat_notice_key()?;
1128 if self.chat_notices.contains(&key) {
1129 return None;
1130 }
1131 match &self.chat_notice {
1132 Some(n)
1133 if n.key == key
1134 || (key == "done" && is_done_notice_key(&n.key))
1135 || self.legacy_notice_matches(&n.key) =>
1136 {
1137 None
1138 }
1139 _ => Some(key),
1140 }
1141 }
1142
1143 pub fn short(&self) -> &str {
1145 short(&self.id)
1146 }
1147
1148 pub fn mark_triage_applied(&mut self, question_id: &str) {
1150 if !self.triage_applied(question_id) {
1151 self.triage_applied.push(question_id.to_owned());
1152 }
1153 }
1154
1155 pub fn action_applied(&self, question_id: &str) -> bool {
1158 self.actions_applied.iter().any(|id| id == question_id)
1159 }
1160
1161 pub fn mark_action_applied(&mut self, question_id: &str) {
1163 if !self.action_applied(question_id) {
1164 self.actions_applied.push(question_id.to_owned());
1165 }
1166 }
1167
1168 pub fn triage_applied(&self, question_id: &str) -> bool {
1170 self.triage_applied.iter().any(|id| id == question_id)
1171 }
1172
1173 pub fn start(&mut self, run: String) {
1186 self.stamp_schema();
1187 self.status = TaskStatus::Running;
1188 self.attempts += 1;
1189 self.runs.retain(|id| id != &run);
1190 self.runs.push(run);
1191 self.last_error = None;
1192 self.park_reason = None;
1193 self.fresh_start = false;
1194 self.interrupt = false;
1195 self.resume_override = None;
1197 }
1198
1199 pub fn link_run(&mut self, run: &str) -> bool {
1204 if self.runs.iter().any(|r| r == run) {
1205 return false;
1206 }
1207 self.runs.push(run.to_owned());
1208 true
1209 }
1210
1211 pub fn succeed(&mut self) {
1221 self.stamp_schema();
1222 self.status = TaskStatus::Done;
1223 self.resume_override = None;
1224 self.last_error = None;
1225 self.park_reason = None;
1226 self.hold_reason = None;
1227 self.hold_source = None;
1228 self.diagnostic = None;
1229 self.blocked_by.clear();
1230 self.block_reason = None;
1231 self.blocked_from = None;
1232 }
1233
1234 pub fn already_landed(&mut self, note: impl Into<String>) {
1242 self.succeed();
1243 self.attempts = self.attempts.saturating_sub(1);
1244 self.last_error = Some(note.into());
1245 }
1246
1247 pub fn superseded_attempts(&self, last_run_succeeded: bool) -> &[String] {
1277 if self.status != TaskStatus::Done || !last_run_succeeded || self.runs.len() < 2 {
1278 return &[];
1279 }
1280 &self.runs[..self.runs.len() - 1]
1281 }
1282
1283 pub fn successor_of(&self, run: &str) -> Option<&String> {
1291 let pos = self.runs.iter().rposition(|r| r == run)?;
1292 self.runs.get(pos + 1)
1293 }
1294
1295 pub fn earlier_attempts(&self) -> &[String] {
1302 &self.runs
1303 }
1304
1305 fn stamp_schema(&mut self) {
1312 self.schema = self.schema.max(SCHEMA);
1313 }
1314
1315 fn note_held(&mut self) {
1316 self.stamp_schema();
1317 let continuing = self.status == TaskStatus::Held
1318 || (self.status == TaskStatus::Blocked
1319 && (self.blocked_from == Some(TaskStatus::Held)
1320 || (self.blocked_from.is_none()
1321 && (self.hold_reason.is_some() || self.hold_source.is_some()))));
1322 if !continuing || self.held_at.is_none() {
1323 self.held_at = Some(Timestamp::now());
1324 }
1325 }
1326
1327 pub fn fail(&mut self, why: impl Into<String>, max_attempts: usize) {
1343 let why = why.into();
1344 if self.status == TaskStatus::Running {
1345 let run = self.runs.last().cloned();
1346 self.notice_cause("run-failed", run.as_deref(), Some(self.attempts));
1347 }
1348 self.diagnostic = None;
1349 self.park_reason = None;
1350 self.status = if self.attempts >= self.effective_attempt_limit(max_attempts) {
1351 self.note_held();
1352 self.hold_source = Some(HoldSource::Machine);
1353 self.hold_reason = Some(why.clone());
1354 TaskStatus::Held
1355 } else {
1356 TaskStatus::Failed
1357 };
1358 self.last_error = Some(why);
1359 }
1360
1361 pub fn stall(&mut self, why: impl Into<String>) {
1371 self.last_error = Some(why.into());
1372 self.diagnostic = None;
1373 self.attempts = self.attempts.saturating_sub(1);
1374 self.status = TaskStatus::Failed;
1375 }
1376
1377 pub fn park(&mut self, why: impl Into<String>) {
1383 self.diagnostic = None;
1384 self.attempts = self.attempts.saturating_sub(1);
1385 self.status = TaskStatus::Parked;
1386 self.park_reason = Some(why.into());
1387 self.schema = self.schema.max(SCHEMA);
1390 }
1391
1392 pub fn record_pass(
1395 &mut self,
1396 run: &str,
1397 pass_index: usize,
1398 attempts_before: usize,
1399 parked: Option<crate::run::ParkCause>,
1400 ) {
1401 if self
1402 .passes
1403 .iter()
1404 .any(|p| p.run == run && p.pass_index == pass_index)
1405 {
1406 return;
1407 }
1408 let cost = if self.attempts < attempts_before {
1409 PassCost::Refunded
1410 } else {
1411 PassCost::Spent
1412 };
1413 self.passes.push(TaskPass {
1414 run: run.to_owned(),
1415 pass_index,
1416 cost,
1417 attempts_after: self.attempts,
1418 settled_at: Timestamp::now(),
1419 parked,
1420 });
1421 self.schema = self.schema.max(SCHEMA);
1422 }
1423
1424 pub fn operator_held(&self) -> bool {
1430 self.status == TaskStatus::Held && !matches!(self.hold_source, Some(HoldSource::Machine))
1431 }
1432
1433 pub fn hold_manual(&mut self, reason: Option<String>) {
1443 self.operator_decided_at = Some(Timestamp::now());
1445 self.hold_unrequested_retry(reason);
1446 }
1447
1448 pub(crate) fn hold_unrequested_retry(&mut self, reason: Option<String>) {
1451 self.park_reason = None;
1452 self.note_held();
1453 self.status = TaskStatus::Held;
1454 if reason.is_some() {
1455 self.hold_reason = reason;
1456 }
1457 self.hold_source = Some(HoldSource::Manual);
1458 self.blocked_by.clear();
1459 self.block_reason = None;
1460 self.blocked_from = None;
1461 }
1462
1463 pub fn hold_machine(&mut self, reason: Option<String>) {
1468 self.park_reason = None;
1469 self.note_held();
1470 self.status = TaskStatus::Held;
1471 if reason.is_some() {
1472 self.hold_reason = reason;
1473 }
1474 self.hold_source = Some(HoldSource::Machine);
1475 self.blocked_by.clear();
1476 self.block_reason = None;
1477 self.blocked_from = None;
1478 }
1479
1480 pub fn block(&mut self, blocked_by: Vec<String>, reason: Option<String>) {
1490 self.stamp_schema();
1491 let before: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1492 let after: std::collections::BTreeSet<_> = blocked_by.iter().collect();
1493 if self.status != TaskStatus::Blocked || before != after {
1494 self.blocked_at = Some(Timestamp::now());
1495 }
1496 if self.status != TaskStatus::Blocked {
1497 self.blocked_from = Some(self.status);
1498 }
1499 self.status = TaskStatus::Blocked;
1500 self.blocked_by = blocked_by;
1501 self.block_reason = reason;
1502 }
1503
1504 pub fn unblock(&mut self, resolved_id: &str) {
1527 if self.status != TaskStatus::Blocked {
1528 return;
1529 }
1530 self.blocked_by.retain(|id| id != resolved_id);
1531 self.restore_if_unblocked();
1532 }
1533
1534 pub fn dependency_deleted(&mut self, deleted_id: &str) -> bool {
1545 if self.status != TaskStatus::Blocked || !self.blocked_by.iter().any(|b| b == deleted_id) {
1546 return false;
1547 }
1548 self.blocked_by.retain(|id| id != deleted_id);
1549 self.restore_if_unblocked();
1550 true
1551 }
1552
1553 fn restore_if_unblocked(&mut self) {
1557 if self.blocked_by.is_empty() {
1558 self.status = match self.blocked_from {
1559 Some(TaskStatus::Running) => TaskStatus::Queued,
1560 Some(other) => other,
1561 None if self.hold_reason.is_some() || self.hold_source.is_some() => {
1562 TaskStatus::Held
1563 }
1564 None => TaskStatus::Queued,
1565 };
1566 self.block_reason = None;
1567 self.blocked_from = None;
1568 }
1569 }
1570
1571 pub fn record_answer(&mut self, question: String, answer: String) {
1576 self.answers.push(AnsweredQuestion { question, answer });
1577 }
1578
1579 pub fn request_review(&mut self, branch: String) {
1583 self.release();
1584 self.review_branch = Some(branch);
1585 self.review_instruction = Some(self.instruction.clone());
1586 }
1587
1588 pub fn requeue(&mut self) {
1591 self.release();
1592 self.review_branch = None;
1594 self.review_instruction = None;
1595 self.fresh_start = true;
1596 }
1597
1598 pub fn hold_for_handover(&mut self, branch: Option<String>, reason: String) {
1607 if branch.is_some() {
1608 self.review_branch = branch;
1609 self.review_instruction = Some(self.instruction.clone());
1610 }
1611 self.hold_machine(Some(reason));
1612 }
1613
1614 pub fn set_urgent(&mut self, urgent: bool) -> Result<()> {
1616 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1617 bail!(
1618 "task {} is {}; urgent can only change while queued or held; \
1619 wait for the active attempt to finish and requeue or hold the task first",
1620 self.short(),
1621 self.status.as_str()
1622 );
1623 }
1624 self.urgent = urgent;
1625 Ok(())
1626 }
1627
1628 pub fn set_priority(&mut self, priority: i32) -> Result<()> {
1637 if self.status == TaskStatus::Running {
1638 bail!(
1639 "task {} is running; its priority cannot be changed until \
1640 this attempt finishes",
1641 self.short()
1642 );
1643 }
1644 self.priority = priority;
1645 Ok(())
1646 }
1647
1648 pub fn set_interrupt(&mut self, interrupt: bool) -> Result<()> {
1663 if interrupt && !self.status.runnable() {
1664 bail!(
1665 "task {} is {}; only a queued or failed task can be marked \
1666 to interrupt",
1667 self.short(),
1668 self.status.as_str()
1669 );
1670 }
1671 self.interrupt = interrupt;
1672 Ok(())
1673 }
1674
1675 pub fn effective_attempt_limit(&self, daemon_limit: usize) -> usize {
1677 self.attempt_limit.map_or(daemon_limit, NonZeroUsize::get)
1678 }
1679
1680 pub fn change_attempt_limit(&mut self, limit: Option<NonZeroUsize>) -> Result<()> {
1683 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1684 bail!("only queued or held tasks can change the attempt limit");
1685 }
1686 self.stamp_schema();
1687 self.attempt_limit = limit;
1688 Ok(())
1689 }
1690
1691 pub fn implementers(&self) -> Option<usize> {
1693 if self.solo {
1694 Some(1)
1695 } else {
1696 self.overrides.as_ref().and_then(|o| o.candidates)
1697 }
1698 }
1699
1700 pub fn change_implementers(&mut self, count: u8) -> Result<()> {
1703 self.check_implementers_editable()?;
1704 let explicit = self.overrides.as_ref().and_then(|o| o.config.as_deref());
1705 let (mut config, _) = crate::config::Config::discover(&self.repo, explicit)?;
1706 if let Some(o) = &self.overrides {
1707 o.apply(&mut config);
1708 }
1709 self.set_implementers(count, &config)
1710 }
1711
1712 fn check_implementers_editable(&self) -> Result<()> {
1713 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1714 bail!(
1715 "task {} is {}; only queued or held tasks can change implementers",
1716 self.short(),
1717 self.status.as_str()
1718 );
1719 }
1720 Ok(())
1721 }
1722
1723 pub fn set_implementers(&mut self, count: u8, config: &crate::config::Config) -> Result<()> {
1725 self.check_implementers_editable()?;
1726 crate::talk::check_implementers(count, config).map_err(anyhow::Error::msg)?;
1727 self.stamp_schema();
1728 self.solo = count == 1;
1729 let mut overrides = self.overrides.clone().unwrap_or_else(|| RunOverrides {
1730 inherit_daemon_options: true,
1731 ..Default::default()
1732 });
1733 overrides.candidates = (count > 1).then_some(usize::from(count));
1734 let mut settings = overrides.clone();
1735 settings.inherit_daemon_options = false;
1736 self.overrides = if overrides.inherit_daemon_options && settings == RunOverrides::default()
1738 {
1739 None
1740 } else {
1741 Some(overrides)
1742 };
1743 Ok(())
1744 }
1745
1746 pub fn edit(&mut self, title: String, instruction: String) -> Result<()> {
1758 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1759 bail!(
1760 "task {} is {}; only a queued or held task's instruction can \
1761 be edited",
1762 self.short(),
1763 self.status.as_str()
1764 );
1765 }
1766 if self.review_branch.is_some() && self.review_instruction.is_none() {
1769 self.review_instruction = Some(self.instruction.clone());
1770 }
1771 if self.instruction != instruction {
1772 self.resume_override = None;
1775 }
1776 self.title = title;
1777 self.instruction = instruction;
1778 Ok(())
1779 }
1780
1781 pub fn handed_off(&mut self, why: impl Into<String>) {
1801 let why = why.into();
1802 self.diagnostic = None;
1803 self.note_held();
1804 self.status = TaskStatus::Held;
1805 self.hold_source = Some(HoldSource::Machine);
1806 self.hold_reason = Some(why.clone());
1807 self.last_error = Some(why);
1808 }
1809
1810 pub fn release_operator(&mut self, source: &str, question_id: Option<&str>) {
1812 self.recovery_budget = None;
1813 self.recovery_generations.clear();
1814 self.recovery_admission = None;
1815 if self
1816 .resumption
1817 .as_ref()
1818 .is_some_and(|r| r.instruction != self.instruction)
1819 {
1820 self.resume_override = None;
1823 }
1824 let approval = Resumption {
1825 generation: self.resumption.as_ref().map_or(1, |r| r.generation + 1),
1826 source: source.to_owned(),
1827 question_id: question_id.map(str::to_owned),
1828 at: Timestamp::now(),
1829 instruction: self.instruction.clone(),
1830 answers_through: self.answers.len(),
1831 runs_through: self.runs.len(),
1832 blocking_evidence: Vec::new(),
1833 cleared_reason: self
1834 .hold_reason
1835 .clone()
1836 .or_else(|| self.block_reason.clone()),
1837 };
1838 self.release();
1839 self.operator_decided_at = Some(approval.at);
1840 if let Some(previous) = self.resumption.replace(approval) {
1841 self.resumption_history.push(previous);
1842 }
1843 }
1844
1845 pub(crate) fn recovery_failed(&mut self, state: &crate::run::RunState) {
1847 let (base, inputs) = self.recovery_admission.as_ref().map_or_else(
1848 || (state.base_commit.clone(), recovery_inputs(&state.sources)),
1849 |a| (a.base.clone(), a.inputs.clone()),
1850 );
1851 let progress = recovery_progress(state);
1852 let stage = if state.prep_failure.is_some() {
1853 "preparation"
1854 } else {
1855 "execution"
1856 };
1857 let matches = |b: &RecoveryBudget| {
1858 b.instruction == self.instruction
1859 && b.base == base
1860 && b.inputs == inputs
1861 && (progress.is_none() || b.progress == progress)
1862 };
1863 if !self.recovery_budget.as_ref().is_some_and(matches) {
1864 let previous = self
1865 .recovery_generations
1866 .iter()
1867 .position(matches)
1868 .map(|index| self.recovery_generations.remove(index));
1869 if let Some(old) = self.recovery_budget.take() {
1870 self.recovery_generations.push(old);
1871 }
1872 self.recovery_budget = previous;
1873 }
1874 if let Some(budget) = &mut self.recovery_budget {
1875 budget.failures = budget.failures.saturating_add(1);
1876 budget.stage = stage.to_owned();
1877 } else {
1878 self.recovery_budget = Some(RecoveryBudget {
1879 instruction: self.instruction.clone(),
1880 base,
1881 inputs,
1882 stage: stage.to_owned(),
1883 failures: 1,
1884 progress,
1885 });
1886 }
1887 }
1888
1889 pub(crate) fn recovery_exhausted(
1890 &self,
1891 base: &str,
1892 inputs: &str,
1893 limit: usize,
1894 ) -> Option<String> {
1895 let same_input = |b: &&RecoveryBudget| {
1896 b.instruction == self.instruction && b.base == base && b.inputs == inputs
1897 };
1898 let budget = self
1899 .recovery_budget
1900 .as_ref()
1901 .filter(same_input)
1902 .or_else(|| self.recovery_generations.iter().rev().find(same_input))?;
1903 (budget.instruction == self.instruction && budget.base == base && budget.inputs == inputs
1904 && budget.failures >= limit.max(1)).then(|| format!(
1905 "[recovery-budget] {} failed passes against unchanged inputs ({}). Inspect the retained recovery sources and correct the cause, change the inputs, or explicitly release the task to approve another generation. Automatic requeue does not reset this budget.",
1906 budget.failures, budget.stage))
1907 }
1908
1909 pub fn release(&mut self) {
1913 self.import_chat_notice();
1916 if self.hold_notice_cause.is_none()
1917 && matches!(self.status, TaskStatus::Held | TaskStatus::Blocked)
1918 {
1919 self.hold_notice_cause = Some(self.unresolved_hold_cause());
1920 }
1921 let refused_handover = self.status == TaskStatus::Held
1922 && self.hold_source == Some(HoldSource::Machine)
1923 && self.review_branch.is_some()
1924 && self
1925 .review_instruction
1926 .as_ref()
1927 .is_none_or(|i| *i == self.instruction);
1928 self.status = TaskStatus::Queued;
1929 self.held_at = None;
1930 self.attempts = 0;
1931 self.last_error = None;
1932 self.park_reason = None;
1933 self.hold_reason = None;
1936 self.hold_source = None;
1937 self.diagnostic = None;
1938 self.blocked_by.clear();
1943 self.block_reason = None;
1944 self.blocked_from = None;
1945 if !refused_handover {
1949 self.review_branch = None;
1950 self.review_instruction = None;
1951 }
1952 self.fresh_start = false;
1953 }
1954}
1955
1956fn copy_new(dir: &Path, src: &Path, name: &str) -> Result<(String, PathBuf)> {
1960 use std::io::ErrorKind;
1961 let (stem, ext) = match name.rfind('.') {
1962 Some(i) if i > 0 => (&name[..i], &name[i..]),
1963 _ => (name, ""),
1964 };
1965 for n in 1u32.. {
1966 let candidate = if n == 1 {
1967 name.to_owned()
1968 } else {
1969 let suffix = format!("-{n}");
1970 let room = 64usize.saturating_sub(suffix.len() + ext.len());
1971 let stem: String = stem.chars().take(room).collect();
1972 format!("{stem}{suffix}{ext}")
1973 };
1974 if !crate::ask::valid_asset_name(&candidate) {
1975 bail!("no valid attachment name is left for `{name}`");
1976 }
1977 let path = dir.join(&candidate);
1978 match std::fs::OpenOptions::new()
1979 .write(true)
1980 .create_new(true)
1981 .open(&path)
1982 {
1983 Ok(mut out) => {
1984 let copied = std::fs::File::open(src)
1985 .and_then(|mut input| std::io::copy(&mut input, &mut out));
1986 if let Err(e) = copied {
1987 drop(out);
1988 let _ = std::fs::remove_file(&path);
1989 return Err(e).with_context(|| format!("copy {}", src.display()));
1990 }
1991 return Ok((candidate, path));
1992 }
1993 Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
1994 Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
1995 }
1996 }
1997 unreachable!("the counter never runs out")
1998}
1999
2000const TASK_LOCK_STALE: std::time::Duration = std::time::Duration::from_secs(10);
2003
2004pub(crate) struct TaskLock {
2007 path: PathBuf,
2008 token: String,
2009}
2010
2011fn owner_token() -> String {
2014 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
2015 let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2016 let mut r = crate::rng::SplitMix64::new(crate::rng::entropy() ^ n.rotate_left(32));
2017 format!("{}-{:016x}", std::process::id(), r.next_u64())
2018}
2019
2020fn break_marker(path: &Path, token: &str) -> PathBuf {
2023 let mut h: u64 = 0xcbf29ce484222325;
2024 for b in token.bytes() {
2025 h ^= u64::from(b);
2026 h = h.wrapping_mul(0x100000001b3);
2027 }
2028 let mut name = path.as_os_str().to_owned();
2029 name.push(format!(".break-{h:016x}"));
2030 PathBuf::from(name)
2031}
2032
2033fn older_than_stale(path: &Path) -> bool {
2034 std::fs::metadata(path)
2035 .and_then(|m| m.modified())
2036 .ok()
2037 .and_then(|t| t.elapsed().ok())
2038 .is_some_and(|age| age > TASK_LOCK_STALE)
2039}
2040
2041fn writer_may_be_gone(token: &str) -> bool {
2042 let Some(pid) = token
2043 .split_once('-')
2044 .and_then(|(pid, _)| pid.parse::<u32>().ok())
2045 else {
2046 return true;
2049 };
2050 if pid == std::process::id() {
2051 return false;
2052 }
2053 crate::proc::pid_status(pid) == Some(false)
2055}
2056
2057const MAX_MARKER_DEPTH: u8 = 3;
2061
2062fn take_marker(
2068 path: &Path,
2069 token: &str,
2070 depth: u8,
2071 retry: crate::lock_file::Retry,
2072) -> Option<(PathBuf, String)> {
2073 use std::io::Write;
2074 let marker = break_marker(path, token);
2075 for _ in 0..2 {
2076 match retry.create(&marker) {
2077 Ok(mut f) => {
2078 let mine = owner_token();
2079 if f.write_all(mine.as_bytes()).is_err() {
2080 drop(f);
2081 let _ = std::fs::remove_file(&marker);
2082 return None;
2083 }
2084 return Some((marker, mine));
2085 }
2086 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2087 let judged = std::fs::read_to_string(&marker).ok();
2088 match judged.filter(|_| older_than_stale(&marker)) {
2089 Some(judged) if depth < MAX_MARKER_DEPTH => {
2090 remove_lock_if(&marker, &judged, true, depth + 1, retry)?;
2091 }
2092 _ => return None,
2093 }
2094 }
2095 Err(_) => return None,
2096 }
2097 }
2098 None
2099}
2100
2101fn remove_lock_if(
2106 path: &Path,
2107 token: &str,
2108 require_stale: bool,
2109 depth: u8,
2110 retry: crate::lock_file::Retry,
2111) -> Option<bool> {
2112 let (marker, mine) = take_marker(path, token, depth, retry)?;
2113 let still = std::fs::read_to_string(path).is_ok_and(|c| c == token)
2114 && (!require_stale || older_than_stale(path) && writer_may_be_gone(token));
2115 if still {
2116 let _ = std::fs::remove_file(path);
2117 }
2118 if depth >= MAX_MARKER_DEPTH {
2120 if std::fs::read_to_string(&marker).is_ok_and(|c| c == mine) {
2121 let _ = std::fs::remove_file(&marker);
2122 }
2123 } else {
2124 let _ = remove_lock_if(&marker, &mine, false, depth + 1, retry);
2125 }
2126 Some(still)
2127}
2128
2129#[cfg(test)]
2132fn break_stale(path: &Path, judged: &str) -> bool {
2133 break_stale_with(path, judged, crate::lock_file::Retry::new(None))
2134}
2135
2136fn break_stale_with(path: &Path, judged: &str, retry: crate::lock_file::Retry) -> bool {
2137 remove_lock_if(path, judged, true, 0, retry).unwrap_or(false)
2138}
2139
2140impl Drop for TaskLock {
2141 fn drop(&mut self) {
2142 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
2143 let retry = crate::lock_file::Retry::new(Some(deadline));
2144 loop {
2145 match remove_lock_if(&self.path, &self.token, false, 0, retry) {
2146 Some(_) => return,
2147 None if std::time::Instant::now() > deadline => return,
2148 None => std::thread::sleep(std::time::Duration::from_millis(5)),
2149 }
2150 }
2151 }
2152}
2153
2154#[derive(Debug, Clone)]
2156pub struct Queue {
2157 root: PathBuf,
2158}
2159
2160impl Queue {
2161 pub fn open() -> Self {
2163 Self::at(crate::run::home().join("queue"))
2164 }
2165
2166 pub fn at(root: PathBuf) -> Self {
2169 Self { root }
2170 }
2171
2172 pub fn root(&self) -> &Path {
2174 &self.root
2175 }
2176
2177 pub fn path_of(&self, id: &str) -> PathBuf {
2179 self.root.join(format!("{id}.json"))
2180 }
2181
2182 pub fn attachments_dir(&self, id: &str) -> PathBuf {
2185 self.root.join(format!("{id}.attachments"))
2186 }
2187
2188 pub fn attach(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2199 let mut wanted = Vec::new();
2200 for src in sources {
2201 let name = src
2202 .file_name()
2203 .and_then(|n| n.to_str())
2204 .with_context(|| format!("`{}` has no usable file name", src.display()))?;
2205 if !crate::ask::valid_asset_name(name) {
2206 bail!(
2207 "attachment name `{name}` must match ^[A-Za-z0-9][A-Za-z0-9._-]{{0,63}}$ \
2208 with no `..`; rename the file and try again"
2209 );
2210 }
2211 if !src.is_file() {
2212 bail!("attachment `{}` is not a file", src.display());
2213 }
2214 wanted.push((src, name));
2215 }
2216 let dir = self.attachments_dir(&task.id);
2217 let existed = dir.is_dir();
2218 std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
2219 let mut created: Vec<PathBuf> = Vec::new();
2220 let mut names = Vec::new();
2221 let mut copy_all = || -> Result<()> {
2222 for (src, name) in &wanted {
2223 let (stored, path) = copy_new(&dir, src, name)?;
2224 created.push(path);
2225 names.push(stored);
2226 }
2227 Ok(())
2228 };
2229 if let Err(e) = copy_all() {
2230 for path in &created {
2231 let _ = std::fs::remove_file(path);
2232 }
2233 if !existed {
2234 let _ = std::fs::remove_dir(&dir);
2235 }
2236 return Err(e);
2237 }
2238 task.attachments.extend(names.iter().cloned());
2239 Ok(names)
2240 }
2241
2242 pub fn attach_and_put(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2253 let dir = self.attachments_dir(&task.id);
2254 let existed = dir.is_dir();
2255 let before = task.attachments.len();
2256 let names = self.attach(task, sources)?;
2257 if let Err(e) = self.put(task) {
2258 for name in &names {
2259 let _ = std::fs::remove_file(dir.join(name));
2260 }
2261 if !existed {
2262 let _ = std::fs::remove_dir(&dir);
2263 }
2264 task.attachments.truncate(before);
2265 return Err(e);
2266 }
2267 Ok(names)
2268 }
2269
2270 pub fn attachment_paths(&self, task: &Task) -> Vec<PathBuf> {
2274 let dir = self.attachments_dir(&task.id);
2275 task.attachments
2276 .iter()
2277 .map(|n| {
2278 let p = dir.join(n);
2279 std::path::absolute(&p).unwrap_or(p)
2280 })
2281 .collect()
2282 }
2283
2284 pub fn put(&self, task: &mut Task) -> Result<()> {
2293 let _lock = self.lock_task(&task.id)?;
2294 self.put_unlocked(task)
2295 }
2296
2297 pub fn create_new(&self, task: &mut Task) -> Result<bool> {
2302 let _lock = self.lock_task(&task.id)?;
2303 if self.path_of(&task.id).exists() {
2304 return Ok(false);
2305 }
2306 self.put_unlocked(task)?;
2307 Ok(true)
2308 }
2309
2310 fn put_unlocked(&self, task: &mut Task) -> Result<()> {
2312 if let Ok(mut stored) = read_path(&self.path_of(&task.id)) {
2313 stored.import_chat_notice();
2314 for key in stored.chat_notices {
2315 if !task.chat_notices.contains(&key) {
2316 task.chat_notices.push(key);
2317 }
2318 }
2319 for run in stored.runs {
2320 if !task.runs.contains(&run) {
2321 task.runs.push(run);
2322 }
2323 }
2324 for p in stored.passes {
2325 if !task
2326 .passes
2327 .iter()
2328 .any(|q| q.run == p.run && q.pass_index == p.pass_index)
2329 {
2330 task.passes.push(p);
2331 }
2332 }
2333 if let Some(n) = stored.chat_notice
2336 && task.chat_notice.as_ref().is_none_or(|mine| mine.at < n.at)
2337 {
2338 task.chat_notice = Some(n);
2339 }
2340 }
2341 task.import_chat_notice();
2342 task.updated_at = Timestamp::now();
2343 std::fs::create_dir_all(&self.root)
2344 .with_context(|| format!("create {}", self.root.display()))?;
2345 let body = serde_json::to_string_pretty(task).context("serialize task")?;
2346 let path = self.path_of(&task.id);
2347 let tmp = path.with_extension("json.tmp");
2348 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
2349 std::fs::rename(&tmp, &path).with_context(|| format!("replace {}", path.display()))?;
2350 if let (Some(notice), Some(home)) = (
2355 crate::notices::task_held(task),
2356 self.root.parent().filter(|p| !p.as_os_str().is_empty()),
2357 ) {
2358 crate::notices::raise_in(home, notice);
2359 }
2360 Ok(())
2361 }
2362
2363 pub fn modify(&self, id: &str, f: impl FnOnce(&mut Task) -> bool) -> Result<bool> {
2368 let id = self.resolve_id(id)?;
2369 let _lock = self.lock_task(&id)?;
2370 let mut task = self.get(&id)?;
2371 if !f(&mut task) {
2372 return Ok(false);
2373 }
2374 self.put_unlocked(&mut task)?;
2375 Ok(true)
2376 }
2377
2378 pub fn link_run(&self, id: &str, run: &str) -> Result<Task> {
2383 let id = self.resolve_id(id)?;
2384 let _lock = self.lock_task(&id)?;
2387 let mut task = self.get(&id)?;
2388 if task.link_run(run) {
2389 self.put_unlocked(&mut task)?;
2390 }
2391 Ok(task)
2392 }
2393
2394 pub(crate) fn lock_task(&self, id: &str) -> Result<TaskLock> {
2402 std::fs::create_dir_all(&self.root)
2403 .with_context(|| format!("create {}", self.root.display()))?;
2404 let path = self.root.join(format!("{id}.write-lock"));
2405 let started = std::time::Instant::now();
2406 let retry = crate::lock_file::Retry::task_write(started + TASK_LOCK_STALE);
2407 let marker_retry = crate::lock_file::Retry::new(Some(started + TASK_LOCK_STALE));
2408 loop {
2409 match retry.create(&path) {
2410 Ok(mut file) => {
2411 use std::io::Write;
2412 let token = owner_token();
2413 if let Err(e) = file.write_all(token.as_bytes()) {
2414 drop(file);
2415 let _ = std::fs::remove_file(&path);
2416 return Err(e).with_context(|| format!("lock {}", path.display()));
2417 }
2418 return Ok(TaskLock { path, token });
2419 }
2420 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2421 let judged = std::fs::read_to_string(&path).ok();
2424 let broken = judged
2425 .filter(|_| older_than_stale(&path))
2426 .is_some_and(|judged| break_stale_with(&path, &judged, marker_retry));
2427 if !broken {
2431 if started.elapsed() > TASK_LOCK_STALE {
2432 bail!("could not lock task {id}");
2433 }
2434 std::thread::sleep(std::time::Duration::from_millis(15));
2435 }
2436 }
2437 Err(e) => return Err(e).with_context(|| format!("lock {}", path.display())),
2438 }
2439 }
2440 }
2441
2442 pub fn get(&self, id: &str) -> Result<Task> {
2444 let resolved = self.resolve_id(id)?;
2445 read_path(&self.path_of(&resolved))
2446 }
2447
2448 pub fn remove(&self, id: &str, in_flight: bool, questions: &Questions) -> Result<Removal> {
2480 let resolved = self.resolve_id(id)?;
2481 if in_flight {
2482 bail!("task {resolved} is being run by a live daemon right now");
2483 }
2484 self.write_tombstone(&resolved)?;
2485 if let Err(e) = self.remove_record_with_attachments(&resolved, |p| std::fs::remove_file(p))
2486 {
2487 if self.path_of(&resolved).exists() {
2491 let _ = std::fs::remove_file(self.tombstone_path(&resolved));
2492 }
2493 return Err(e);
2494 }
2495 let (released, still_blocked) = self.release_dependents_of(&resolved, questions);
2496 Ok(Removal {
2497 id: resolved,
2498 released,
2499 still_blocked,
2500 })
2501 }
2502
2503 fn tombstone_path(&self, id: &str) -> PathBuf {
2506 self.root.join(format!("{id}.removed"))
2507 }
2508
2509 fn write_tombstone(&self, id: &str) -> Result<()> {
2510 let path = self.tombstone_path(id);
2511 let tmp = self.root.join(format!("{id}.removed.tmp"));
2512 std::fs::write(&tmp, b"")
2513 .and_then(|()| std::fs::rename(&tmp, &path))
2514 .with_context(|| format!("write {}", path.display()))
2515 }
2516
2517 fn was_deleted(&self, id: &str) -> bool {
2521 self.tombstone_path(id).is_file() && !self.path_of(id).exists()
2522 }
2523
2524 pub fn apply_deleted_blockers(&self, task: &mut Task, questions: &Questions) -> Vec<String> {
2528 let deleted = deleted_blockers(self, questions, &task.blocked_by);
2529 deleted
2530 .into_iter()
2531 .filter(|id| task.dependency_deleted(id))
2532 .collect()
2533 }
2534
2535 pub fn note_dependency_deleted(&self, dependent: &Task, deleted: &str) {
2539 let Some(home) = self.root.parent().filter(|p| !p.as_os_str().is_empty()) else {
2540 return;
2541 };
2542 let ja = crate::lang::is_japanese(&crate::lang::of_repo(&dependent.repo));
2543 let waiting = dependent.status == TaskStatus::Blocked;
2544 let message = match (ja, waiting) {
2545 (true, false) => format!(
2546 "タスク {} は、待っていた {} が削除されたため待機を解除し、元の状態に戻しました",
2547 dependent.short(),
2548 short(deleted)
2549 ),
2550 (true, true) => format!(
2551 "タスク {} は、待っていた {} が削除されたため、残りの依存を待っています",
2552 dependent.short(),
2553 short(deleted)
2554 ),
2555 (false, false) => format!(
2556 "Task {} stopped waiting on {} because it was deleted, and returned to its previous state",
2557 dependent.short(),
2558 short(deleted)
2559 ),
2560 (false, true) => format!(
2561 "Task {} stopped waiting on {} because it was deleted, and is still waiting on its other dependencies",
2562 dependent.short(),
2563 short(deleted)
2564 ),
2565 };
2566 crate::notices::raise_in(
2567 home,
2568 crate::notices::Notice::info(
2569 &format!("unblocked:{}:{}", dependent.id, deleted),
2570 message,
2571 )
2572 .link(crate::notices::Link::Task {
2573 id: dependent.id.clone(),
2574 }),
2575 );
2576 }
2577
2578 fn release_dependents_of(
2583 &self,
2584 dependency: &str,
2585 questions: &Questions,
2586 ) -> (Vec<String>, Vec<String>) {
2587 let (mut released, mut still_blocked) = (Vec::new(), Vec::new());
2588 for listed in self.list() {
2589 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| {
2590 matches!(crate::queue::dependency(self, questions, b), Dependency::Deleted(id) if id == dependency)
2591 }) {
2592 continue;
2593 }
2594 let Ok(_claim) = self.claim(&listed.id) else {
2595 continue;
2596 };
2597 let Ok(mut task) = self.get(&listed.id) else {
2598 continue;
2599 };
2600 let references: Vec<String> = task.blocked_by.iter().filter(|id| {
2601 matches!(crate::queue::dependency(self, questions, id), Dependency::Deleted(full) if full == dependency)
2602 }).cloned().collect();
2603 if references.is_empty() {
2604 continue;
2605 }
2606 for reference in references {
2607 task.dependency_deleted(&reference);
2608 }
2609 if self.put(&mut task).is_err() {
2610 continue;
2611 }
2612 self.note_dependency_deleted(&task, dependency);
2613 if task.status == TaskStatus::Blocked {
2614 still_blocked.push(task.id.clone());
2615 } else {
2616 released.push(task.id.clone());
2617 }
2618 }
2619 (released, still_blocked)
2620 }
2621
2622 fn remove_record_with_attachments(
2626 &self,
2627 resolved: &str,
2628 remove_record: impl FnOnce(&Path) -> std::io::Result<()>,
2629 ) -> Result<()> {
2630 self.sweep_removed_attachments();
2637 let attachments = self.attachments_dir(resolved);
2638 let aside = self.root.join(format!("{resolved}.attachments.removing"));
2639 let moved = match std::fs::rename(&attachments, &aside) {
2640 Ok(()) => true,
2641 Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
2642 Err(e) => {
2643 return Err(e).with_context(|| format!("remove {}", attachments.display()));
2644 }
2645 };
2646 let path = self.path_of(resolved);
2647 if let Err(e) = remove_record(&path) {
2648 if moved {
2649 let _ = std::fs::rename(&aside, &attachments);
2650 }
2651 return Err(e).with_context(|| format!("remove {}", path.display()));
2652 }
2653 if moved {
2654 if let Err(e) = std::fs::remove_dir_all(&aside) {
2655 tracing::warn!("leftover attachments {}: {e}", aside.display());
2656 }
2657 }
2658 let lock = self.lock_path(resolved);
2659 if let Err(e) = std::fs::remove_file(&lock) {
2660 if e.kind() != std::io::ErrorKind::NotFound {
2661 return Err(e).with_context(|| format!("remove {}", lock.display()));
2662 }
2663 }
2664 Ok(())
2665 }
2666
2667 fn sweep_removed_attachments(&self) {
2673 let Ok(entries) = std::fs::read_dir(&self.root) else {
2674 return;
2675 };
2676 for entry in entries.flatten() {
2677 let name = entry.file_name();
2678 let name = name.to_string_lossy();
2679 let Some(id) = name.strip_suffix(".attachments.removing") else {
2680 continue;
2681 };
2682 if !self.path_of(id).exists() {
2685 if let Err(e) = std::fs::remove_dir_all(entry.path()) {
2686 tracing::warn!("leftover attachments {}: {e}", entry.path().display());
2687 }
2688 }
2689 }
2690 self.sweep_tombstones();
2691 }
2692
2693 fn sweep_tombstones(&self) {
2702 let Ok(entries) = std::fs::read_dir(&self.root) else {
2703 return;
2704 };
2705 let tasks = self.list();
2706 for entry in entries.flatten() {
2707 let name = entry.file_name();
2708 let name = name.to_string_lossy();
2709 let Some(id) = name.strip_suffix(".removed") else {
2710 continue;
2711 };
2712 if self.path_of(id).exists()
2713 || tasks.iter().any(|t| {
2714 t.blocked_by
2715 .iter()
2716 .any(|b| id.starts_with(b) || id.ends_with(b))
2717 })
2718 {
2719 continue;
2720 }
2721 let old = entry
2722 .metadata()
2723 .and_then(|m| m.modified())
2724 .ok()
2725 .and_then(|m| m.elapsed().ok())
2726 .is_some_and(|age| age >= TOMBSTONE_GRACE);
2727 if old {
2728 let _ = std::fs::remove_file(entry.path());
2729 }
2730 }
2731 }
2732
2733 fn lock_path(&self, id: &str) -> PathBuf {
2736 self.root.join(format!("{id}.lock"))
2737 }
2738
2739 pub fn list(&self) -> Vec<Task> {
2752 let mut tasks: Vec<Task> = std::fs::read_dir(&self.root)
2753 .into_iter()
2754 .flatten()
2755 .flatten()
2756 .map(|e| e.path())
2757 .filter(|p| p.extension().is_some_and(|x| x == "json"))
2758 .filter_map(|p| read_path(&p).ok())
2759 .collect();
2760 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then_with(|| b.id.cmp(&a.id)));
2761 tasks
2762 }
2763
2764 pub fn unreadable(&self) -> Vec<PathBuf> {
2768 std::fs::read_dir(&self.root)
2769 .into_iter()
2770 .flatten()
2771 .flatten()
2772 .map(|e| e.path())
2773 .filter(|p| p.extension().is_some_and(|x| x == "json"))
2774 .filter(|p| read_path(p).is_err())
2775 .collect()
2776 }
2777
2778 pub fn superseded(&self) -> HashMap<String, String> {
2791 let mut by = HashMap::new();
2792 for task in self.list() {
2793 for earlier in &task.runs {
2794 if let Some(later) = task.successor_of(earlier) {
2795 by.insert(earlier.clone(), later.clone());
2796 }
2797 }
2798 }
2799 by
2800 }
2801
2802 pub fn superseded_by(&self, run: &str) -> Option<String> {
2810 for task in self.list() {
2811 if task.runs.iter().any(|r| r == run) {
2812 return task.successor_of(run).cloned();
2813 }
2814 }
2815 None
2816 }
2817
2818 pub fn latest_attempt(&self, run: &str) -> Option<String> {
2829 for task in self.list() {
2830 if task.runs.iter().any(|r| r == run) {
2831 return task.runs.last().filter(|last| **last != run).cloned();
2832 }
2833 }
2834 None
2835 }
2836
2837 pub fn next_runnable(&self) -> Option<Task> {
2842 let mut runnable: Vec<Task> = self
2843 .list()
2844 .into_iter()
2845 .filter(|t| t.status.runnable())
2846 .collect();
2847 runnable.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)));
2848 runnable.into_iter().next()
2849 }
2850
2851 pub fn claim(&self, id: &str) -> Result<Claim> {
2858 std::fs::create_dir_all(&self.root)
2859 .with_context(|| format!("create {}", self.root.display()))?;
2860 let path = self.lock_path(id);
2861 let retry = crate::lock_file::Retry::new(None);
2862 match retry.create(&path) {
2863 Ok(mut f) => {
2864 use std::io::Write as _;
2865 let _ = writeln!(f, "{}", std::process::id());
2867 Ok(Claim { path })
2868 }
2869 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2870 bail!("task {id} is already claimed ({} exists)", path.display())
2871 }
2872 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
2873 }
2874 }
2875
2876 pub fn resolve_id(&self, prefix: &str) -> Result<String> {
2878 if !prefix.is_empty()
2879 && prefix != "."
2880 && prefix != ".."
2881 && !prefix.contains(['/', '\\'])
2882 && self.path_of(prefix).is_file()
2883 {
2884 return Ok(prefix.to_owned());
2885 }
2886 let ids = record_ids(&self.root, "json")?;
2887 match resolve_reference(prefix, ids.iter().map(String::as_str)) {
2888 Reference::Found(id) => Ok(id),
2889 Reference::Missing => bail!("no task matches `{prefix}`"),
2890 Reference::Ambiguous => bail!("`{prefix}` matches multiple tasks"),
2891 }
2892 }
2893
2894 pub fn revision(&self) -> u64 {
2901 self.revision_excluding(&std::collections::BTreeSet::new())
2902 }
2903
2904 pub fn revision_excluding(&self, skip: &std::collections::BTreeSet<String>) -> u64 {
2907 use std::hash::{Hash as _, Hasher as _};
2908
2909 let mut entries: Vec<(String, u64)> = std::fs::read_dir(&self.root)
2910 .into_iter()
2911 .flatten()
2912 .flatten()
2913 .filter(|e| e.path().extension().is_some_and(|ext| ext == "json"))
2914 .filter(|e| {
2915 let path = e.path();
2916 !path
2917 .file_stem()
2918 .is_some_and(|stem| skip.contains(stem.to_string_lossy().as_ref()))
2919 })
2920 .filter_map(|e| {
2921 let name = e.file_name().to_string_lossy().into_owned();
2922 let mtime = e
2923 .metadata()
2924 .ok()?
2925 .modified()
2926 .ok()?
2927 .duration_since(std::time::UNIX_EPOCH)
2928 .ok()?
2929 .as_millis() as u64;
2930 Some((name, mtime))
2931 })
2932 .collect();
2933
2934 if entries.is_empty() {
2935 return 0;
2936 }
2937
2938 entries.sort_unstable();
2939 let mut hasher = std::hash::DefaultHasher::new();
2940 for (name, mtime) in &entries {
2941 name.hash(&mut hasher);
2942 mtime.hash(&mut hasher);
2943 }
2944 let h = hasher.finish();
2945 if h == 0 { 1 } else { h }
2946 }
2947}
2948
2949const TOMBSTONE_GRACE: std::time::Duration = std::time::Duration::from_secs(3600);
2952
2953#[derive(Debug, Clone)]
2955pub struct Removal {
2956 pub id: String,
2958 pub released: Vec<String>,
2962 pub still_blocked: Vec<String>,
2965}
2966
2967#[derive(Debug)]
2969pub struct Claim {
2970 path: PathBuf,
2971}
2972
2973impl Drop for Claim {
2974 fn drop(&mut self) {
2975 let _ = std::fs::remove_file(&self.path);
2976 }
2977}
2978
2979pub fn title_from(instruction: &str, max: usize) -> String {
2982 let Some(line) = first_line(instruction) else {
2983 return "(empty task)".to_owned();
2984 };
2985 if line.chars().count() <= max {
2986 return line.to_owned();
2987 }
2988 let head: String = line.chars().take(max.saturating_sub(1)).collect();
2989 format!("{head}…")
2990}
2991
2992pub(crate) fn first_line(instruction: &str) -> Option<&str> {
2998 let line = instruction
2999 .lines()
3000 .map(str::trim)
3001 .find(|l| !l.is_empty())?
3002 .trim_start_matches(['#', '-', '*', '>', ' '])
3003 .trim();
3004 (!line.is_empty()).then_some(line)
3005}
3006
3007fn read_path(path: &Path) -> Result<Task> {
3008 let body = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
3009 let task: Task =
3010 serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
3011 if task.schema > SCHEMA {
3017 bail!(
3018 "task {} was written by a different magi (schema {}, this build \
3019 speaks {SCHEMA})",
3020 task.id,
3021 task.schema
3022 );
3023 }
3024 Ok(task)
3025}
3026
3027#[derive(Debug, PartialEq, Eq)]
3030pub(crate) enum Reference {
3031 Found(String),
3032 Missing,
3033 Ambiguous,
3034}
3035
3036pub(crate) fn resolve_reference<'a>(id: &str, ids: impl Iterator<Item = &'a str>) -> Reference {
3037 let mut hits = Vec::new();
3038 for candidate in ids {
3039 if candidate == id {
3040 return Reference::Found(candidate.to_owned());
3041 }
3042 if candidate.starts_with(id) || candidate.ends_with(id) {
3043 hits.push(candidate);
3044 }
3045 }
3046 match hits.as_slice() {
3047 [] => Reference::Missing,
3048 [one] => Reference::Found((*one).to_owned()),
3049 _ => Reference::Ambiguous,
3050 }
3051}
3052
3053pub(crate) fn record_ids(root: &Path, extension: &str) -> Result<Vec<String>> {
3056 let entries = match std::fs::read_dir(root) {
3057 Ok(entries) => entries,
3058 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
3059 Err(e) => return Err(e.into()),
3060 };
3061 let mut ids = Vec::new();
3062 for entry in entries {
3063 let path = entry?.path();
3064 if path.extension().is_some_and(|e| e == extension)
3065 && let Some(id) = path.file_stem().and_then(|s| s.to_str())
3066 {
3067 ids.push(id.to_owned());
3068 }
3069 }
3070 Ok(ids)
3071}
3072
3073#[derive(Debug, PartialEq, Eq)]
3076pub enum Dependency {
3077 Task(String),
3079 Question(String),
3081 Deleted(String),
3083 Missing,
3085 Ambiguous,
3087 Unknown,
3089}
3090
3091pub fn dependency(queue: &Queue, questions: &Questions, id: &str) -> Dependency {
3093 let (Ok(tasks), Ok(qs), Ok(removed)) = (
3094 record_ids(queue.root(), "json"),
3095 record_ids(questions.root(), "json"),
3096 record_ids(queue.root(), "removed"),
3097 ) else {
3098 return Dependency::Unknown;
3099 };
3100 let mut ids = tasks
3103 .iter()
3104 .chain(qs.iter())
3105 .chain(removed.iter())
3106 .cloned()
3107 .collect::<Vec<_>>();
3108 ids.sort();
3109 ids.dedup();
3110 match resolve_reference(id, ids.iter().map(String::as_str)) {
3111 Reference::Found(id) if tasks.contains(&id) => Dependency::Task(id),
3112 Reference::Found(id) if qs.contains(&id) => Dependency::Question(id),
3113 Reference::Found(id) if queue.was_deleted(&id) => Dependency::Deleted(id),
3114 Reference::Found(_) => Dependency::Unknown,
3115 Reference::Missing => Dependency::Missing,
3116 Reference::Ambiguous => Dependency::Ambiguous,
3117 }
3118}
3119
3120pub fn normalize_blockers(queue: &Queue, questions: &Questions, ids: &mut [String]) {
3122 for id in ids {
3123 if let Dependency::Task(full) | Dependency::Question(full) | Dependency::Deleted(full) =
3124 dependency(queue, questions, id)
3125 {
3126 *id = full;
3127 }
3128 }
3129}
3130
3131pub fn dependency_facts(queue: &Queue, questions: &Questions) -> String {
3134 let inventory = crate::blockers::Inventory::load(queue, questions);
3135 let mut text = String::from(
3136 "Current authoritative queue (historical missing-task claims may be stale; unreadable or ambiguous references must keep waiting):\n",
3137 );
3138 match record_ids(queue.root(), "json") {
3139 Ok(ids) => {
3140 for id in ids {
3141 match queue.get(&id) {
3142 Ok(task) => text.push_str(&format!(
3143 "{}: {}; blocked_by={:?}; chain={:?}\n",
3144 task.id,
3145 task.status.as_str(),
3146 task.blocked_by,
3147 inventory.waits_on(&task)
3148 )),
3149 Err(_) => text.push_str(&format!("{id}: exists, state unreadable\n")),
3150 }
3151 }
3152 }
3153 Err(_) => text.push_str("Queue inventory unreadable: existence is unknown.\n"),
3154 }
3155 text
3156}
3157
3158pub fn missing_blockers(
3160 queue: &Queue,
3161 questions: &Questions,
3162 blocked_by: &[String],
3163) -> Vec<String> {
3164 blocked_by
3165 .iter()
3166 .filter(|id| dependency(queue, questions, id) == Dependency::Missing)
3167 .cloned()
3168 .collect()
3169}
3170
3171pub fn deleted_blockers(
3174 queue: &Queue,
3175 questions: &Questions,
3176 blocked_by: &[String],
3177) -> Vec<String> {
3178 blocked_by
3179 .iter()
3180 .filter(|id| matches!(dependency(queue, questions, id), Dependency::Deleted(_)))
3181 .cloned()
3182 .collect()
3183}
3184
3185pub fn missing_blocker_hold_reason(blocked_by: &[String], missing: &[String]) -> String {
3197 missing_blocker_hold_reason_in(blocked_by, missing, "en")
3198}
3199
3200pub fn missing_blocker_hold_reason_in(
3202 blocked_by: &[String],
3203 missing: &[String],
3204 language: &str,
3205) -> String {
3206 if crate::lang::is_japanese(language) {
3207 format!(
3208 "{} を待っていましたが、{} はディスク上に存在しません - `magi task triage` を参照",
3209 blocked_by.join(", "),
3210 missing.join(", "),
3211 )
3212 } else {
3213 format!(
3214 "blocked on {} but {} no longer exist(s) on disk - see `magi task triage`",
3215 blocked_by.join(", "),
3216 missing.join(", "),
3217 )
3218 }
3219}
3220
3221pub fn short(id: &str) -> &str {
3223 id.split('-').next_back().unwrap_or(id)
3224}
3225
3226fn new_id() -> String {
3227 let stamp = jiff::Zoned::now().strftime("%Y%m%d-%H%M%S");
3228 let seed = crate::rng::entropy();
3229 format!("{stamp}-{:04x}", (seed ^ (seed >> 32)) & 0xffff)
3230}
3231
3232#[cfg(test)]
3233mod tests {
3234 #[test]
3235 fn explicit_resumption_is_not_invented_for_legacy_or_automatic_release() {
3236 let t = Task::new(
3237 "retry".into(),
3238 "do it".into(),
3239 PathBuf::from("."),
3240 Source::Human,
3241 );
3242 let mut value = serde_json::to_value(&t).unwrap();
3243 for field in ["resumption", "resumption_history", "operator_decided_at"] {
3244 value.as_object_mut().unwrap().remove(field);
3245 }
3246 value["schema"] = serde_json::json!(15);
3247 let mut legacy: Task = serde_json::from_value(value).unwrap();
3248 legacy.hold_machine(Some("automatic retry".into()));
3249 legacy.release();
3250 assert!(legacy.resumption.is_none());
3251 assert!(legacy.resumption_history.is_empty());
3252 assert!(legacy.operator_decided_at.is_none());
3253 legacy.release_operator("cli", None);
3254 legacy.start("run".into());
3255 let reloaded: Task =
3256 serde_json::from_str(&serde_json::to_string(&legacy).unwrap()).unwrap();
3257 assert!(reloaded.resume_override.is_none());
3258 assert_eq!(reloaded.resumption.as_ref().unwrap().generation, 1);
3259 }
3260
3261 #[test]
3262 fn merge_over_lets_the_later_choice_win_and_keeps_the_rest() {
3263 let mut base = RunOverrides {
3264 merge: Some("pr".to_owned()),
3265 candidates: Some(3),
3266 ..RunOverrides::default()
3267 };
3268 base.merge_over(&RunOverrides {
3269 merge: Some("none".to_owned()),
3270 seed: Some(7),
3271 ..RunOverrides::default()
3272 });
3273 assert_eq!(base.merge.as_deref(), Some("none"));
3274 assert_eq!(base.candidates, Some(3));
3275 assert_eq!(base.seed, Some(7));
3276 }
3277
3278 #[test]
3279 fn an_already_landed_task_is_done_with_its_attempt_refunded() {
3280 let mut t = task("relanded");
3281 t.attempts = 1;
3282 t.status = TaskStatus::Running;
3283 t.already_landed("already in main as 0e368de");
3284 assert_eq!(t.status, TaskStatus::Done);
3285 assert_eq!(t.attempts, 0);
3286 assert!(t.hold_reason.is_none());
3287 assert_eq!(t.last_error.as_deref(), Some("already in main as 0e368de"));
3288 }
3289
3290 #[test]
3291 fn missing_blocker_reason_follows_the_language() {
3292 let b = vec!["a".to_owned()];
3293 let en = missing_blocker_hold_reason_in(&b, &b, "en");
3294 assert_eq!(en, missing_blocker_hold_reason(&b, &b));
3295 assert!(en.starts_with("blocked on a"));
3296 assert!(missing_blocker_hold_reason_in(&b, &b, "ja").contains("存在しません"));
3297 assert_eq!(missing_blocker_hold_reason_in(&b, &b, "de"), en);
3298 }
3299
3300 use super::*;
3301
3302 #[test]
3303 fn triage_applied_survives_release_and_old_records_read_as_empty() {
3304 let mut t = Task::new(
3305 "t".to_owned(),
3306 "i".to_owned(),
3307 PathBuf::from("r"),
3308 Source::Human,
3309 );
3310 t.mark_triage_applied("q1");
3311 t.mark_triage_applied("q1");
3312 t.hold_machine(Some("x".to_owned()));
3313 t.release();
3314 assert_eq!(t.triage_applied, ["q1"]);
3315 assert!(t.triage_applied("q1") && !t.triage_applied("q2"));
3316
3317 let mut v = serde_json::to_value(&t).unwrap();
3318 v.as_object_mut().unwrap().remove("triage_applied");
3319 let old: Task = serde_json::from_value(v).unwrap();
3320 assert!(old.triage_applied.is_empty());
3321 }
3322
3323 #[test]
3324 fn task_counts_of_empty_is_all_zero() {
3325 assert_eq!(TaskCounts::of(&[]), TaskCounts::default());
3326 }
3327
3328 #[test]
3329 fn task_counts_of_tallies_every_status() {
3330 let mut queued = Task::new(
3331 "q".to_owned(),
3332 "i".to_owned(),
3333 PathBuf::from("."),
3334 Source::Human,
3335 );
3336 queued.status = TaskStatus::Queued;
3337 let mut running = queued.clone();
3338 running.status = TaskStatus::Running;
3339 let mut done = queued.clone();
3340 done.status = TaskStatus::Done;
3341 let mut failed = queued.clone();
3342 failed.status = TaskStatus::Failed;
3343 let mut held = queued.clone();
3344 held.status = TaskStatus::Held;
3345 let mut blocked = queued.clone();
3346 blocked.status = TaskStatus::Blocked;
3347
3348 let mut parked = queued.clone();
3349 parked.status = TaskStatus::Parked;
3350
3351 let counts = TaskCounts::of(&[
3352 queued,
3353 running,
3354 done.clone(),
3355 done,
3356 failed,
3357 held,
3358 blocked,
3359 parked,
3360 ]);
3361 assert_eq!(
3362 counts,
3363 TaskCounts {
3364 queued: 1,
3365 running: 1,
3366 done: 2,
3367 failed: 1,
3368 held: 1,
3369 blocked: 1,
3370 parked: 1,
3371 }
3372 );
3373 }
3374
3375 #[test]
3376 fn task_implementers_persist_normalized_without_losing_history() {
3377 let (_dir, q) = queue();
3378 let mut t = task("keep me");
3379 t.hold_manual(Some("waiting".into()));
3380 t.runs.push("earlier-run".into());
3381 t.attachments.push("reference.txt".into());
3382 let cfg: crate::config::Config =
3383 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
3384 .unwrap();
3385 q.put(&mut t).unwrap();
3386 let original = serde_json::to_value(&t).unwrap();
3387 for n in [2, 1, 2] {
3388 t.set_implementers(n, &cfg).unwrap();
3389 q.put(&mut t).unwrap();
3390 t = q.get(&t.id).unwrap();
3391 assert_eq!(t.implementers(), Some(usize::from(n)));
3392 assert_eq!(t.solo, n == 1);
3393 if n == 1 {
3394 assert!(t.overrides.is_none());
3395 }
3396 let mut actual = serde_json::to_value(&t).unwrap();
3397 for field in ["solo", "overrides", "updated_at"] {
3398 actual[field] = original[field].clone();
3399 }
3400 assert_eq!(actual, original);
3401 }
3402 t.overrides.as_mut().unwrap().judges = Some(2);
3403 t.set_implementers(1, &cfg).unwrap();
3404 assert_eq!(t.overrides.as_ref().unwrap().judges, Some(2));
3405 assert_eq!(t.overrides.as_ref().unwrap().candidates, None);
3406 }
3407
3408 #[test]
3409 fn task_implementers_old_records_keep_inheritance_until_changed() {
3410 let mut value = serde_json::to_value(task("old task")).unwrap();
3411 value.as_object_mut().unwrap().remove("solo");
3412 value.as_object_mut().unwrap().remove("overrides");
3413 let mut old: Task = serde_json::from_value(value).unwrap();
3414 assert_eq!(old.implementers(), None);
3415 old.set_implementers(1, &crate::config::Config::default())
3416 .unwrap();
3417 assert!(old.solo);
3418 assert!(old.overrides.is_none());
3419 }
3420
3421 #[test]
3422 fn task_implementers_reject_invalid_counts_and_states() {
3423 let cfg = crate::config::Config::default();
3424 let mut t = task("unchanged");
3425 for n in [0, 4, 255] {
3426 assert!(t.set_implementers(n, &cfg).is_err());
3427 }
3428 for status in [
3429 TaskStatus::Running,
3430 TaskStatus::Done,
3431 TaskStatus::Failed,
3432 TaskStatus::Blocked,
3433 TaskStatus::Parked,
3434 ] {
3435 t.status = status;
3436 let before = serde_json::to_value(&t).unwrap();
3437 assert!(t.set_implementers(1, &cfg).is_err());
3438 assert_eq!(serde_json::to_value(&t).unwrap(), before);
3439 }
3440 }
3441
3442 fn queue() -> (tempfile::TempDir, Queue) {
3445 let dir = tempfile::tempdir().unwrap();
3446 let q = Queue::at(dir.path().join("queue"));
3447 (dir, q)
3448 }
3449
3450 #[test]
3451 fn putting_a_machine_held_task_files_a_notification_beside_the_queue() {
3452 let dir = tempfile::tempdir().unwrap();
3453 let q = Queue::at(dir.path().join("queue"));
3454 let mut t = task("held");
3455 q.put(&mut t).unwrap();
3456 assert_eq!(
3457 crate::notices::Notices::at(dir.path().join("notifications"))
3458 .list()
3459 .len(),
3460 0
3461 );
3462 t.hold_machine(Some("out of attempts".to_owned()));
3463 q.put(&mut t).unwrap();
3464 let listed = crate::notices::Notices::at(dir.path().join("notifications")).list();
3465 assert_eq!(listed.len(), 1);
3466 assert!(listed[0].message.contains("out of attempts"));
3467 }
3468
3469 fn task(title: &str) -> Task {
3470 Task::new(
3471 title.to_owned(),
3472 format!("do {title}"),
3473 PathBuf::from("."),
3474 Source::Human,
3475 )
3476 }
3477
3478 #[test]
3479 fn attempt_limit_inherits_old_records_and_controls_failure() {
3480 let mut t = task("budget");
3481 let mut old = serde_json::to_value(&t).unwrap();
3482 old.as_object_mut().unwrap().remove("attempt_limit");
3483 old["schema"] = serde_json::json!(15);
3484 let legacy: Task = serde_json::from_value(old).unwrap();
3485 assert_eq!(legacy.attempt_limit, None);
3486 assert_eq!(legacy.effective_attempt_limit(2), 2);
3487 t.attempts = 2;
3488 t.change_attempt_limit(NonZeroUsize::new(3)).unwrap();
3489 t.fail("retry", 1);
3490 assert_eq!(t.status, TaskStatus::Failed);
3491 t.attempts = 3;
3492 t.fail("stop", 10);
3493 assert_eq!(t.status, TaskStatus::Held);
3494 t.change_attempt_limit(None).unwrap();
3495 assert_eq!(t.status, TaskStatus::Held);
3496 assert_eq!(t.attempts, 3);
3497 t.fail("inherit", 4);
3498 assert_eq!(t.status, TaskStatus::Failed);
3499 for status in [
3500 TaskStatus::Running,
3501 TaskStatus::Done,
3502 TaskStatus::Failed,
3503 TaskStatus::Blocked,
3504 TaskStatus::Parked,
3505 ] {
3506 t.status = status;
3507 assert!(t.change_attempt_limit(NonZeroUsize::new(1)).is_err());
3508 assert_eq!(t.attempt_limit, None);
3509 }
3510 }
3511
3512 #[test]
3513 fn earlier_attempts_is_every_recorded_run_and_agrees_with_the_display() {
3514 let mut t = task("retried");
3515 assert!(
3516 t.earlier_attempts().is_empty(),
3517 "a first attempt takes nothing over"
3518 );
3519 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
3520 assert_eq!(t.earlier_attempts(), ["aaaa", "bbbb"]);
3521 assert_eq!(t.successor_of("aaaa"), Some(&"bbbb".to_owned()));
3523 assert_eq!(t.successor_of("bbbb"), None);
3524 assert_eq!(t.successor_of("zzzz"), None);
3525 }
3526
3527 #[test]
3528 fn resumed_runs_are_superseded_only_after_their_last_occurrence() {
3529 let (_dir, q) = queue();
3530 let mut t = task("resumed");
3531 for (runs, successor) in [
3532 (vec!["aaaa", "aaaa"], None),
3533 (vec!["aaaa", "bbbb", "aaaa"], None),
3534 (vec!["aaaa", "aaaa", "bbbb"], Some("bbbb")),
3535 (vec!["aaaa", "bbbb", "aaaa", "cccc"], Some("cccc")),
3536 ] {
3537 t.runs = runs.into_iter().map(str::to_owned).collect();
3538 q.put(&mut t).unwrap();
3539
3540 assert_eq!(t.successor_of("aaaa").map(String::as_str), successor);
3541 assert_eq!(q.superseded_by("aaaa").as_deref(), successor);
3542 assert_eq!(q.superseded().get("aaaa").map(String::as_str), successor);
3543 }
3544 }
3545
3546 #[test]
3547 fn superseded_by_names_the_next_attempt_and_none_for_the_last() {
3548 let (_dir, q) = queue();
3549 let mut t = task("retried");
3550 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
3551 q.put(&mut t).unwrap();
3552
3553 assert_eq!(q.superseded_by("aaaa"), Some("bbbb".to_owned()));
3554 assert_eq!(q.superseded_by("bbbb"), Some("cccc".to_owned()));
3555 assert_eq!(
3556 q.superseded_by("cccc"),
3557 None,
3558 "the latest attempt replaces nothing"
3559 );
3560 assert_eq!(
3561 q.superseded_by("never-heard-of-it"),
3562 None,
3563 "a run belonging to no task on this queue is not superseded"
3564 );
3565
3566 let mut by = HashMap::new();
3567 by.insert("aaaa".to_owned(), "bbbb".to_owned());
3568 by.insert("bbbb".to_owned(), "cccc".to_owned());
3569 assert_eq!(
3570 q.superseded(),
3571 by,
3572 "the whole-map and single-run forms must agree"
3573 );
3574 }
3575
3576 #[test]
3577 fn superseded_attempts_is_empty_until_the_task_is_done() {
3578 let mut t = task("retried");
3579 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
3580 t.status = TaskStatus::Failed;
3581 assert_eq!(
3582 t.superseded_attempts(true),
3583 &[] as &[String],
3584 "a task still retrying has no attempt yet that a later one made moot"
3585 );
3586
3587 t.status = TaskStatus::Running;
3588 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
3589 }
3590
3591 #[test]
3592 fn superseded_attempts_names_every_run_before_the_one_that_succeeded() {
3593 let mut t = task("retried");
3594 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
3595 t.status = TaskStatus::Done;
3596 assert_eq!(
3597 t.superseded_attempts(true),
3598 &["aaaa".to_owned(), "bbbb".to_owned()],
3599 "cccc is the attempt whose success made the task done, and stays out"
3600 );
3601 }
3602
3603 #[test]
3604 fn superseded_attempts_is_empty_for_a_done_task_with_only_one_attempt() {
3605 let mut t = task("first try landed");
3606 t.runs = vec!["aaaa".to_owned()];
3607 t.status = TaskStatus::Done;
3608 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
3609 }
3610
3611 #[test]
3612 fn superseded_attempts_is_empty_when_the_last_run_never_actually_succeeded() {
3613 let mut t = task("closed by hand after a manual merge");
3620 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
3621 t.status = TaskStatus::Done;
3622 assert_eq!(
3623 t.superseded_attempts(false),
3624 &[] as &[String],
3625 "nothing here is provably why the task is done, so nothing is superseded"
3626 );
3627 }
3628
3629 #[test]
3630 fn latest_attempt_names_the_chain_s_current_head_not_just_the_next_one() {
3631 let (_dir, q) = queue();
3632 let mut t = task("retried twice");
3633 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
3634 q.put(&mut t).unwrap();
3635
3636 assert_eq!(
3637 q.latest_attempt("aaaa"),
3638 Some("cccc".to_owned()),
3639 "an old attempt points straight at the chain's current head, not the \
3640 next attempt in the middle of it"
3641 );
3642 assert_eq!(q.latest_attempt("bbbb"), Some("cccc".to_owned()));
3643 assert_eq!(
3644 q.latest_attempt("cccc"),
3645 None,
3646 "the latest attempt is not superseded by anything"
3647 );
3648 assert_eq!(
3649 q.latest_attempt("never-heard-of-it"),
3650 None,
3651 "a run belonging to no task on this queue is not superseded"
3652 );
3653 }
3654
3655 #[test]
3656 fn a_markdown_heading_is_the_title_not_decoration() {
3657 assert_eq!(
3662 title_from("# Rework the config loader\n\nIt re-reads it.\n", 40),
3663 "Rework the config loader"
3664 );
3665 assert_eq!(title_from("- fix the thing", 40), "fix the thing");
3666 assert_eq!(title_from("> quoted task", 40), "quoted task");
3667 assert_eq!(title_from(" \n\n", 40), "(empty task)");
3669 assert_eq!(title_from("###\n", 40), "(empty task)");
3670 }
3671
3672 #[test]
3673 fn a_long_title_is_elided_by_characters_not_bytes() {
3674 let long = "課題".repeat(30);
3676 let title = title_from(&long, 10);
3677 assert_eq!(title.chars().count(), 10);
3678 assert!(title.ends_with('…'));
3679 }
3680
3681 #[test]
3682 fn priority_wins_and_ties_break_oldest_first() {
3683 let (_dir, q) = queue();
3684 let mut a = task("first");
3685 let mut b = task("second");
3686 let mut c = task("urgent");
3687 a.id = "20260101-000001-aaaa".to_owned();
3689 b.id = "20260101-000002-bbbb".to_owned();
3690 c.id = "20260101-000003-cccc".to_owned();
3691 c.priority = 5;
3692 for t in [&mut a, &mut b, &mut c] {
3693 q.put(t).unwrap();
3694 }
3695
3696 assert_eq!(q.next_runnable().unwrap().id, c.id);
3698 c.hold_machine(None);
3699 q.put(&mut c).unwrap();
3700 assert_eq!(q.next_runnable().unwrap().id, a.id);
3702 assert_eq!(q.list().len(), 3, "b is still waiting its turn");
3703 }
3704
3705 #[test]
3706 fn a_blocked_task_never_starves_another_runnable_one() {
3707 let (_dir, q) = queue();
3708 let mut blocked = task("blocked");
3709 blocked.block(vec!["something".to_owned()], None);
3710 q.put(&mut blocked).unwrap();
3711
3712 let mut runnable = task("free to go");
3713 q.put(&mut runnable).unwrap();
3714
3715 let next = q.next_runnable().expect("a runnable task is still offered");
3716 assert_eq!(next.id, runnable.id);
3717 }
3718
3719 #[test]
3720 fn a_held_task_is_never_offered_to_the_loop() {
3721 let (_dir, q) = queue();
3722 let mut t = task("held");
3723 q.put(&mut t).unwrap();
3724 assert!(q.next_runnable().is_some());
3725
3726 t.hold_machine(None);
3727 q.put(&mut t).unwrap();
3728 assert!(
3729 q.next_runnable().is_none(),
3730 "a held task must wait for a human"
3731 );
3732
3733 t.status = TaskStatus::Failed;
3735 q.put(&mut t).unwrap();
3736 assert!(q.next_runnable().is_some());
3737 }
3738
3739 #[test]
3740 fn attempts_are_capped_and_then_the_task_is_held() {
3741 let mut t = task("doomed");
3742
3743 t.start("run-1".to_owned());
3744 t.fail("gate red", 2);
3745 assert_eq!(t.status, TaskStatus::Failed, "one attempt of two: retry");
3746
3747 t.start("run-2".to_owned());
3748 t.fail("gate red", 2);
3749 assert_eq!(
3750 t.status,
3751 TaskStatus::Held,
3752 "out of attempts: stop spending money on it"
3753 );
3754 assert_eq!(t.runs, ["run-1", "run-2"]);
3755 assert_eq!(t.last_error.as_deref(), Some("gate red"));
3756 assert_eq!(
3757 t.hold_reason.as_deref(),
3758 Some("gate red"),
3759 "the hold must say why, not leave hold_reason null next to a \
3760 populated last_error"
3761 );
3762 }
3763
3764 #[test]
3765 fn handing_off_a_task_records_a_hold_reason_too() {
3766 let mut t = task("left a pull request");
3767 t.start("run-1".to_owned());
3768 t.handed_off("run ended with a pull request open [run run-1]");
3769 assert_eq!(t.status, TaskStatus::Held);
3770 assert_eq!(t.hold_source, Some(HoldSource::Machine));
3771 assert_eq!(
3772 t.hold_reason.as_deref(),
3773 Some("run ended with a pull request open [run run-1]")
3774 );
3775 assert_eq!(t.hold_reason, t.last_error);
3776 }
3777
3778 #[test]
3779 fn a_quota_stall_is_refunded_so_the_backlog_survives_the_night() {
3780 let mut t = task("stalled by quota");
3781
3782 t.start("run-1".to_owned());
3783 assert_eq!(t.attempts, 1);
3784 t.stall("judge-1, judge-2 out of quota");
3785 assert_eq!(
3786 t.attempts, 0,
3787 "a closed quota window must not spend the task's retry budget"
3788 );
3789 assert_eq!(t.status, TaskStatus::Failed, "the loop should retry it");
3790 assert_eq!(
3791 t.last_error.as_deref(),
3792 Some("judge-1, judge-2 out of quota")
3793 );
3794
3795 for _ in 0..20 {
3798 t.start("run-n".to_owned());
3799 t.stall("still out of quota");
3800 }
3801 t.start("run-real".to_owned());
3802 t.fail("gate red", 2);
3803 assert_eq!(
3804 t.status,
3805 TaskStatus::Failed,
3806 "the first attempt that was really judged is attempt one"
3807 );
3808 }
3809
3810 #[test]
3811 fn releasing_a_held_task_gives_it_a_real_second_chance() {
3812 let mut t = task("retry me");
3813 t.start("run-1".to_owned());
3814 t.fail("gate red", 1);
3815 assert_eq!(t.status, TaskStatus::Held);
3816
3817 t.release();
3818 assert_eq!(t.status, TaskStatus::Queued);
3819 assert_eq!(t.attempts, 0);
3822 assert!(t.last_error.is_none());
3823 assert_eq!(
3824 t.runs.len(),
3825 1,
3826 "history is kept: attempts reset, evidence does not"
3827 );
3828 }
3829
3830 #[test]
3831 fn a_hold_reason_survives_and_a_release_clears_it() {
3832 let mut t = task("waiting on something else");
3833 t.hold_manual(Some(
3834 "waiting for 20260101-000000-aaaa to land first".to_owned(),
3835 ));
3836 assert_eq!(t.status, TaskStatus::Held);
3837 assert_eq!(
3838 t.hold_reason.as_deref(),
3839 Some("waiting for 20260101-000000-aaaa to land first")
3840 );
3841
3842 t.hold_manual(None);
3844 assert_eq!(
3845 t.hold_reason.as_deref(),
3846 Some("waiting for 20260101-000000-aaaa to land first"),
3847 "a bare re-hold keeps whatever a human already wrote down"
3848 );
3849
3850 let mut plain = task("no reason given");
3852 plain.hold_manual(None);
3853 assert_eq!(plain.status, TaskStatus::Held);
3854 assert!(plain.hold_reason.is_none());
3855
3856 t.release();
3857 assert_eq!(t.status, TaskStatus::Queued);
3858 assert!(
3859 t.hold_reason.is_none(),
3860 "a stale reason must not greet the next person who holds this task"
3861 );
3862 }
3863
3864 #[test]
3865 fn closing_a_held_task_as_done_clears_its_hold_reason_too() {
3866 let mut t = task("landed by hand while held");
3871 t.hold_manual(Some("waiting on 3ed9".to_owned()));
3872 assert_eq!(t.hold_reason.as_deref(), Some("waiting on 3ed9"));
3873
3874 t.succeed();
3875 assert_eq!(t.status, TaskStatus::Done);
3876 assert!(
3877 t.hold_reason.is_none(),
3878 "a done task cannot still be waiting on something"
3879 );
3880 }
3881
3882 #[test]
3883 fn holding_or_closing_a_blocked_task_clears_its_dependency_too() {
3884 let mut held = task("held straight out of blocked");
3891 held.block(
3892 vec!["20260101-000000-dead".to_owned()],
3893 Some("waiting on the migration script".to_owned()),
3894 );
3895 assert_eq!(held.status, TaskStatus::Blocked);
3896
3897 held.hold_manual(None);
3898 assert_eq!(held.status, TaskStatus::Held);
3899 assert!(
3900 held.blocked_by.is_empty(),
3901 "hold overrides the wait, same as release"
3902 );
3903 assert!(held.block_reason.is_none());
3904
3905 let mut done = task("closed straight out of blocked");
3906 done.block(
3907 vec!["20260101-000000-dead".to_owned()],
3908 Some("waiting on the migration script".to_owned()),
3909 );
3910 done.succeed();
3911 assert_eq!(done.status, TaskStatus::Done);
3912 assert!(
3913 done.blocked_by.is_empty(),
3914 "a done task cannot still be waiting on a dependency"
3915 );
3916 assert!(done.block_reason.is_none());
3917 }
3918
3919 #[test]
3920 fn a_blocked_task_is_never_offered_to_the_loop() {
3921 let mut t = task("blocked");
3922 assert!(t.status.runnable());
3923 t.block(
3924 vec!["dep-id".to_owned()],
3925 Some("waits on dep-id".to_owned()),
3926 );
3927 assert_eq!(t.status, TaskStatus::Blocked);
3928 assert!(!t.status.runnable());
3929 assert_eq!(TaskStatus::Blocked.as_str(), "blocked");
3930 }
3931
3932 #[test]
3933 fn unblocking_the_last_dependency_returns_the_task_to_queued() {
3934 let mut t = task("blocked on two");
3935 t.block(
3936 vec!["a".to_owned(), "b".to_owned()],
3937 Some("waits on a and b".to_owned()),
3938 );
3939
3940 t.unblock("a");
3941 assert_eq!(t.status, TaskStatus::Blocked, "b is still outstanding");
3942 assert_eq!(t.blocked_by, ["b"]);
3943
3944 t.unblock("b");
3945 assert_eq!(t.status, TaskStatus::Queued);
3946 assert!(t.blocked_by.is_empty());
3947 assert!(t.block_reason.is_none());
3948 }
3949
3950 #[test]
3951 fn unblocking_an_id_on_a_task_that_is_not_blocked_is_a_no_op() {
3952 let mut t = task("never blocked");
3953 t.unblock("whatever");
3954 assert_eq!(t.status, TaskStatus::Queued);
3955 }
3956
3957 #[test]
3958 fn a_held_task_blocked_on_a_question_returns_to_held_not_queued() {
3959 let mut t = task("held, then asked about");
3964 t.hold_machine(Some("out of attempts".to_owned()));
3965 assert_eq!(t.status, TaskStatus::Held);
3966
3967 t.block(vec!["q1".to_owned()], Some("what now?".to_owned()));
3968 assert_eq!(t.status, TaskStatus::Blocked);
3969
3970 t.record_answer("what now?".to_owned(), "leave it held".to_owned());
3971 t.unblock("q1");
3972 assert_eq!(t.status, TaskStatus::Held, "must restore, not requeue");
3973 assert_eq!(t.hold_reason.as_deref(), Some("out of attempts"));
3974 assert_eq!(t.hold_source, Some(HoldSource::Machine));
3975 assert!(t.blocked_from.is_none(), "consumed once restored");
3976 }
3977
3978 #[test]
3979 fn a_manually_held_task_blocked_on_a_question_returns_to_held() {
3980 let mut t = task("manually held, then asked about");
3981 t.hold_manual(Some("waiting on a dependency".to_owned()));
3982
3983 t.block(vec!["q1".to_owned()], None);
3984 t.unblock("q1");
3985
3986 assert_eq!(t.status, TaskStatus::Held);
3987 assert_eq!(t.hold_source, Some(HoldSource::Manual));
3988 }
3989
3990 #[test]
3991 fn re_blocking_an_already_blocked_task_keeps_the_original_blocked_from() {
3992 let mut t = task("held, blocked twice");
3996 t.hold_machine(None);
3997 t.block(vec!["q1".to_owned()], Some("first".to_owned()));
3998 t.block(
3999 vec!["q1".to_owned(), "q2".to_owned()],
4000 Some("second".to_owned()),
4001 );
4002
4003 t.unblock("q1");
4004 assert_eq!(t.status, TaskStatus::Blocked, "q2 still outstanding");
4005 t.unblock("q2");
4006 assert_eq!(t.status, TaskStatus::Held);
4007 }
4008
4009 #[test]
4010 fn unblocking_a_task_blocked_while_running_lands_on_queued_not_running() {
4011 let mut t = task("blocked mid-run");
4015 t.start("run-1".to_owned());
4016 assert_eq!(t.status, TaskStatus::Running);
4017
4018 t.block(vec!["q1".to_owned()], None);
4019 t.unblock("q1");
4020 assert_eq!(t.status, TaskStatus::Queued);
4021 }
4022
4023 #[test]
4024 fn a_pre_schema_4_blocked_record_with_hold_evidence_restores_to_held() {
4025 let mut t = task("legacy record, held before it was blocked");
4031 t.hold_source = Some(HoldSource::Machine);
4032 t.hold_reason = Some("legacy hold reason".to_owned());
4033 t.status = TaskStatus::Blocked;
4034 t.blocked_by = vec!["q1".to_owned()];
4035 t.blocked_from = None;
4036
4037 t.unblock("q1");
4038 assert_eq!(t.status, TaskStatus::Held);
4039 }
4040
4041 #[test]
4042 fn a_pre_schema_4_blocked_record_with_no_hold_evidence_restores_to_queued() {
4043 let mut t = task("legacy record, ordinary dependency block");
4044 t.status = TaskStatus::Blocked;
4045 t.blocked_by = vec!["dep".to_owned()];
4046 t.blocked_from = None;
4047
4048 t.unblock("dep");
4049 assert_eq!(t.status, TaskStatus::Queued);
4050 }
4051
4052 #[test]
4053 fn answering_a_question_is_recorded_and_survives_a_release() {
4054 let mut t = task("asked something");
4055 t.block(vec!["q1".to_owned()], Some("which backend?".to_owned()));
4056 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
4057 t.unblock("q1");
4058 assert_eq!(t.status, TaskStatus::Queued);
4059 assert_eq!(t.answers.len(), 1);
4060 assert_eq!(t.answers[0].answer, "SQLite");
4061
4062 t.release();
4066 assert_eq!(t.answers.len(), 1, "the answer is not lost on release");
4067 }
4068
4069 #[test]
4070 fn resumption_review_branch_requires_unchanged_authored_criteria() {
4071 for source in ["cli", "web"] {
4072 let mut t = task("review old candidate");
4073 t.request_review("candidate-A".into());
4074 t.hold_for_handover(Some("candidate-A".into()), "handover refused".into());
4075 let mut unchanged = t.clone();
4076 unchanged.release_operator(source, None);
4077 assert_eq!(unchanged.review_branch.as_deref(), Some("candidate-A"));
4078 t.edit(
4079 "review approved head".into(),
4080 "Reuse measured PR head B; fix Clippy; review".into(),
4081 )
4082 .unwrap();
4083 t.release_operator(source, None);
4084 assert!(t.review_branch.is_none());
4085 assert!(t.review_instruction.is_none());
4086 let mut legacy = unchanged;
4088 legacy.review_instruction = None;
4089 legacy.hold_for_handover(None, "still checked out".into());
4090 legacy
4091 .edit("new plan".into(), "Reuse measured PR head B".into())
4092 .unwrap();
4093 legacy.release_operator(source, None);
4094 assert!(legacy.review_branch.is_none());
4095 }
4096 }
4097
4098 #[test]
4099 fn a_refused_handover_keeps_the_review_branch_across_release() {
4100 let mut t = task("refused takeover");
4101 t.start("run-1".to_owned());
4102 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
4103 assert_eq!(t.status, TaskStatus::Held);
4104 assert_eq!(t.attempts, 1);
4105 t.release();
4106 assert_eq!(t.status, TaskStatus::Queued);
4107 assert_eq!(t.attempts, 0);
4108 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4109
4110 t.hold_for_handover(None, "again".to_owned());
4112 t.requeue();
4113 assert!(t.review_branch.is_none());
4114
4115 let mut m = task("manual");
4117 m.review_branch = Some("magi/x/A".to_owned());
4118 m.hold_manual(None);
4119 m.release();
4120 assert!(m.review_branch.is_none());
4121 }
4122
4123 #[test]
4124 fn requesting_review_requeues_the_task_and_remembers_the_branch() {
4125 let mut t = task("blocked run with a surviving branch");
4126 t.start("run-1".to_owned());
4127 t.fail("blocked with major findings", 5);
4128 assert_eq!(t.status, TaskStatus::Failed);
4129
4130 t.request_review("magi/eba2/A".to_owned());
4131 assert_eq!(t.status, TaskStatus::Queued);
4132 assert_eq!(t.attempts, 0);
4133 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4134
4135 t.release();
4137 assert!(t.review_branch.is_none());
4138 }
4139
4140 #[test]
4141 fn conductor_requeue_but_not_an_ordinary_release_forces_a_fresh_start() {
4142 let mut t = task("retry");
4143 t.start("run-1".to_owned());
4144 t.requeue();
4145 assert!(t.fresh_start);
4146
4147 t.release();
4148 assert!(!t.fresh_start);
4149 }
4150
4151 #[test]
4152 fn urgent_only_changes_queued_or_held_and_preserves_settings() {
4153 for status in [
4154 TaskStatus::Queued,
4155 TaskStatus::Held,
4156 TaskStatus::Running,
4157 TaskStatus::Done,
4158 TaskStatus::Failed,
4159 TaskStatus::Blocked,
4160 TaskStatus::Parked,
4161 ] {
4162 let mut t = task("urgent settings");
4163 t.status = status;
4164 t.attempts = 2;
4165 t.hold_reason = Some("waiting".into());
4166 t.priority = 7;
4167 t.interrupt = true;
4168 let before = serde_json::to_value(&t).unwrap();
4169 for urgent in [true, true, false, false] {
4170 let result = t.set_urgent(urgent);
4171 let mut expected = before.clone();
4172 if matches!(status, TaskStatus::Queued | TaskStatus::Held) {
4173 result.unwrap();
4174 expected["urgent"] = urgent.into();
4175 } else {
4176 let error = result.unwrap_err().to_string();
4177 assert!(error.contains(status.as_str()), "{error}");
4178 assert!(error.contains("queued or held"), "{error}");
4179 }
4180 assert_eq!(serde_json::to_value(&t).unwrap(), expected);
4181 }
4182 }
4183 }
4184
4185 #[test]
4186 fn priority_can_be_changed_while_queued_but_not_while_running() {
4187 let mut t = task("reprioritise me");
4188 t.set_priority(5).unwrap();
4189 assert_eq!(t.priority, 5);
4190
4191 t.start("run-1".to_owned());
4192 let err = t.set_priority(9).unwrap_err().to_string();
4193 assert!(err.contains("running"), "{err}");
4194 assert_eq!(t.priority, 5, "the rejected write must not partially apply");
4195 }
4196
4197 #[test]
4198 fn interrupt_can_be_marked_while_queued_but_not_while_running() {
4199 let mut t = task("interrupt me");
4200 assert!(!t.interrupt, "off unless asked, same as any other task");
4201
4202 t.set_interrupt(true).unwrap();
4203 assert!(t.interrupt);
4204
4205 t.start("run-1".to_owned());
4206 assert!(
4207 !t.interrupt,
4208 "the mark is one-shot: dispatching the task fulfils it, \
4209 whatever the run that follows ends up doing"
4210 );
4211 let err = t.set_interrupt(true).unwrap_err().to_string();
4212 assert!(err.contains("running"), "{err}");
4213 t.set_interrupt(false).unwrap();
4216 assert!(!t.interrupt);
4217 }
4218
4219 #[test]
4223 fn a_failed_run_does_not_leave_the_task_still_marked_to_interrupt() {
4224 let mut t = task("interrupt me");
4225 t.set_interrupt(true).unwrap();
4226 t.start("run-1".to_owned());
4227 t.fail("mock failure", 5);
4228 assert_eq!(t.status, TaskStatus::Failed);
4229 assert!(
4230 !t.interrupt,
4231 "one attempt already spent the mark; a retry is an ordinary \
4232 requeue, not a fresh interrupt request"
4233 );
4234 }
4235
4236 #[test]
4237 fn changing_priority_moves_a_task_ahead_in_the_real_queue_order() {
4238 let (_dir, q) = queue();
4239 let mut a = task("first filed");
4240 let mut b = task("second filed");
4241 a.id = "20260101-000001-aaaa".to_owned();
4242 b.id = "20260101-000002-bbbb".to_owned();
4243 q.put(&mut a).unwrap();
4244 q.put(&mut b).unwrap();
4245
4246 assert_eq!(
4247 q.next_runnable().unwrap().id,
4248 a.id,
4249 "with equal priority the older task goes first, so a burst of \
4250 new work cannot starve it"
4251 );
4252 assert_eq!(
4253 q.list()[0].id,
4254 b.id,
4255 "but the list an operator reads is newest first, the same as \
4256 before priority existed - a's turn to run does not make it the \
4257 newest task"
4258 );
4259
4260 let mut a = q.get(&a.id).unwrap();
4261 a.set_priority(10).unwrap();
4262 q.put(&mut a).unwrap();
4263
4264 assert_eq!(
4265 q.next_runnable().unwrap().id,
4266 a.id,
4267 "a raised priority must be reflected the moment it is saved"
4268 );
4269 assert_eq!(
4273 q.list()[0].id,
4274 a.id,
4275 "the raised task must sort first in the list an operator reads, \
4276 not only in next_runnable's own ordering"
4277 );
4278 }
4279
4280 #[test]
4281 fn editing_replaces_title_and_instruction_but_keeps_identity_and_history() {
4282 let mut t = Task::new(
4283 "old title".to_owned(),
4284 "old instruction".to_owned(),
4285 PathBuf::from("/repo"),
4286 Source::Agent {
4287 run: "20260101-000000-beef".to_owned(),
4288 node: "implement".to_owned(),
4289 },
4290 );
4291 let id = t.id.clone();
4292 let created_at = t.created_at;
4293 t.runs.push("20260101-000000-beef".to_owned());
4294
4295 t.edit("new title".to_owned(), "new instruction".to_owned())
4296 .unwrap();
4297
4298 assert_eq!(t.title, "new title");
4299 assert_eq!(t.instruction, "new instruction");
4300 assert_eq!(t.id, id, "editing must not mint a new id");
4301 assert_eq!(t.created_at, created_at);
4302 assert_eq!(
4303 t.source,
4304 Source::Agent {
4305 run: "20260101-000000-beef".to_owned(),
4306 node: "implement".to_owned(),
4307 },
4308 "editing must not turn agent attribution into human"
4309 );
4310 assert_eq!(t.runs, ["20260101-000000-beef"]);
4311 }
4312
4313 #[test]
4314 fn editing_is_refused_once_a_task_is_running_or_finished() {
4315 let mut running = task("in flight");
4316 running.start("run-1".to_owned());
4317 let err = running
4318 .edit("x".to_owned(), "y".to_owned())
4319 .unwrap_err()
4320 .to_string();
4321 assert!(err.contains("running"), "{err}");
4322
4323 let mut done = task("finished");
4324 done.succeed();
4325 let err = done
4326 .edit("x".to_owned(), "y".to_owned())
4327 .unwrap_err()
4328 .to_string();
4329 assert!(err.contains("done"), "{err}");
4330
4331 let mut queued = task("waiting");
4333 queued.edit("x".to_owned(), "y".to_owned()).unwrap();
4334 let mut held = task("parked");
4335 held.hold_machine(None);
4336 held.edit("x".to_owned(), "y".to_owned()).unwrap();
4337 }
4338
4339 #[test]
4340 fn a_task_recorded_without_a_hold_reason_still_reads_as_none() {
4341 let (_dir, q) = queue();
4342 let path = q.path_of("20260101-000000-aaaa");
4343 std::fs::create_dir_all(q.root()).unwrap();
4344 std::fs::write(
4345 &path,
4346 serde_json::json!({
4347 "schema": SCHEMA,
4348 "id": "20260101-000000-aaaa",
4349 "title": "from before hold reasons existed",
4350 "instruction": "from before hold reasons existed",
4351 "repo": ".",
4352 "source": { "kind": "human" },
4353 "status": "held",
4354 "created_at": Timestamp::now().to_string(),
4355 "updated_at": Timestamp::now().to_string(),
4356 })
4357 .to_string(),
4358 )
4359 .unwrap();
4360
4361 let task = q.get("20260101-000000-aaaa").expect("must still read");
4362 assert!(task.hold_reason.is_none());
4363 assert!(task.operator_held());
4364 }
4365
4366 #[test]
4367 fn a_legacy_reasoned_hold_defaults_to_operator_protection() {
4368 let (_dir, q) = queue();
4369 let path = q.path_of("20260101-000000-bbbb");
4370 std::fs::create_dir_all(q.root()).unwrap();
4371 std::fs::write(
4372 &path,
4373 serde_json::json!({
4374 "schema": 2,
4375 "id": "20260101-000000-bbbb",
4376 "title": "old manual recovery",
4377 "instruction": "old manual recovery",
4378 "repo": ".",
4379 "source": { "kind": "human" },
4380 "status": "held",
4381 "hold_reason": "active manual recovery run20260912-224242-daf5",
4382 "created_at": Timestamp::now().to_string(),
4383 "updated_at": Timestamp::now().to_string(),
4384 })
4385 .to_string(),
4386 )
4387 .unwrap();
4388
4389 let task = q.get("20260101-000000-bbbb").expect("must still read");
4390 assert_eq!(task.hold_source, None);
4391 assert!(task.operator_held());
4392 }
4393
4394 #[test]
4395 fn a_task_recorded_without_a_diagnostic_still_reads_as_none() {
4396 let (_dir, q) = queue();
4397 let path = q.path_of("20260101-000000-aaaa");
4398 std::fs::create_dir_all(q.root()).unwrap();
4399 std::fs::write(
4400 &path,
4401 serde_json::json!({
4402 "schema": SCHEMA,
4403 "id": "20260101-000000-aaaa",
4404 "title": "from before diagnostics existed",
4405 "instruction": "from before diagnostics existed",
4406 "repo": ".",
4407 "source": { "kind": "human" },
4408 "status": "held",
4409 "created_at": Timestamp::now().to_string(),
4410 "updated_at": Timestamp::now().to_string(),
4411 })
4412 .to_string(),
4413 )
4414 .unwrap();
4415
4416 let task = q.get("20260101-000000-aaaa").expect("must still read");
4417 assert!(task.diagnostic.is_none());
4418 }
4419
4420 #[test]
4421 fn a_schema_1_task_with_no_blocking_fields_still_reads() {
4422 let (_dir, q) = queue();
4426 let path = q.path_of("20260101-000000-aaaa");
4427 std::fs::create_dir_all(q.root()).unwrap();
4428 std::fs::write(
4429 &path,
4430 serde_json::json!({
4431 "schema": 1,
4432 "id": "20260101-000000-aaaa",
4433 "title": "from before blocking existed",
4434 "instruction": "from before blocking existed",
4435 "repo": ".",
4436 "source": { "kind": "human" },
4437 "status": "queued",
4438 "created_at": Timestamp::now().to_string(),
4439 "updated_at": Timestamp::now().to_string(),
4440 })
4441 .to_string(),
4442 )
4443 .unwrap();
4444
4445 let task = q.get("20260101-000000-aaaa").expect("must still read");
4446 assert!(task.blocked_by.is_empty());
4447 assert!(task.block_reason.is_none());
4448 assert!(task.answers.is_empty());
4449 assert!(task.review_branch.is_none());
4450 }
4451
4452 #[test]
4453 fn releasing_or_finishing_a_task_clears_its_stale_diagnostic() {
4454 let mut held = task("diagnosed");
4459 held.start("run-1".to_owned());
4460 held.fail("gate red", 1);
4461 held.diagnostic = Some("cargo test failed: ...".to_owned());
4462 assert_eq!(held.status, TaskStatus::Held);
4463
4464 held.release();
4465 assert!(held.diagnostic.is_none());
4466
4467 held.diagnostic = Some("cargo test failed: ...".to_owned());
4468 held.succeed();
4469 assert!(held.diagnostic.is_none());
4470 }
4471
4472 #[test]
4473 fn failing_a_task_always_clears_whatever_diagnostic_it_carried() {
4474 let mut t = task("retried");
4475 t.start("run-1".to_owned());
4476 t.diagnostic = Some("stale evidence from a previous hold".to_owned());
4477 t.fail("unrelated config error", 5);
4478 assert_eq!(t.status, TaskStatus::Failed);
4479 assert!(
4480 t.diagnostic.is_none(),
4481 "fail() must not let an old diagnostic outlive the run that produced it"
4482 );
4483 }
4484
4485 #[test]
4486 fn a_claim_is_exclusive_and_releases_on_drop() {
4487 let (_dir, q) = queue();
4488 let mut t = task("contended");
4489 q.put(&mut t).unwrap();
4490
4491 let held = q.claim(&t.id).unwrap();
4492 assert!(
4493 q.claim(&t.id).is_err(),
4494 "two daemons must not drive one task into two runs"
4495 );
4496 drop(held);
4497 assert!(q.claim(&t.id).is_ok(), "a released claim is reclaimable");
4498 }
4499
4500 #[test]
4501 fn a_round_trip_survives_disk() {
4502 let (_dir, q) = queue();
4503 let mut t = Task::new(
4504 "titled".to_owned(),
4505 "body".to_owned(),
4506 PathBuf::from("/repo"),
4507 Source::Agent {
4508 run: "20260101-000000-beef".to_owned(),
4509 node: "implement".to_owned(),
4510 },
4511 );
4512 t.priority = 3;
4513 q.put(&mut t).unwrap();
4514
4515 let back = q.get(&t.id).unwrap();
4516 assert_eq!(back.id, t.id);
4517 assert_eq!(back.priority, 3);
4518 assert_eq!(back.source.label(), "implement@beef");
4519 assert_eq!(q.get(t.short()).unwrap().id, t.id);
4521 }
4522
4523 #[test]
4524 fn an_unreadable_task_does_not_take_the_queue_down() {
4525 let (_dir, q) = queue();
4526 let mut t = task("fine");
4527 q.put(&mut t).unwrap();
4528 std::fs::write(q.root().join("broken.json"), "{ not json").unwrap();
4529
4530 let listed = q.list();
4531 assert_eq!(listed.len(), 1, "the readable task still lists");
4532 assert_eq!(listed[0].id, t.id);
4533 }
4534
4535 #[test]
4536 fn a_task_recorded_without_a_solo_field_still_reads_as_not_solo() {
4537 let (_dir, q) = queue();
4538 let path = q.path_of("20260101-000000-aaaa");
4539 std::fs::create_dir_all(q.root()).unwrap();
4540 std::fs::write(
4541 &path,
4542 serde_json::json!({
4543 "schema": SCHEMA,
4544 "id": "20260101-000000-aaaa",
4545 "title": "from before solo existed",
4546 "instruction": "from before solo existed",
4547 "repo": ".",
4548 "source": { "kind": "human" },
4549 "status": "queued",
4550 "created_at": Timestamp::now().to_string(),
4551 "updated_at": Timestamp::now().to_string(),
4552 })
4553 .to_string(),
4554 )
4555 .unwrap();
4556
4557 let task = q.get("20260101-000000-aaaa").expect("must still read");
4558 assert!(!task.solo, "a queue file with no `solo` field means false");
4559 }
4560
4561 #[test]
4562 fn a_task_recorded_without_an_urgent_field_still_reads_as_not_urgent() {
4563 let (_dir, q) = queue();
4564 let path = q.path_of("20260101-000000-bbbb");
4565 std::fs::create_dir_all(q.root()).unwrap();
4566 std::fs::write(
4567 &path,
4568 serde_json::json!({
4569 "schema": SCHEMA,
4570 "id": "20260101-000000-bbbb",
4571 "title": "from before urgent existed",
4572 "instruction": "from before urgent existed",
4573 "repo": ".",
4574 "source": { "kind": "human" },
4575 "status": "queued",
4576 "created_at": Timestamp::now().to_string(),
4577 "updated_at": Timestamp::now().to_string(),
4578 })
4579 .to_string(),
4580 )
4581 .unwrap();
4582
4583 let task = q.get("20260101-000000-bbbb").expect("must still read");
4584 assert!(
4585 !task.urgent,
4586 "a queue file with no `urgent` field means false, same as `solo`"
4587 );
4588 }
4589
4590 #[test]
4591 fn a_task_from_a_future_schema_is_refused_rather_than_guessed_at() {
4592 let (_dir, q) = queue();
4593 let mut t = task("from the future");
4594 q.put(&mut t).unwrap();
4595 let path = q.path_of(&t.id);
4596 let body = std::fs::read_to_string(&path)
4597 .unwrap()
4598 .replace(&format!("\"schema\": {SCHEMA}"), "\"schema\": 99");
4599 std::fs::write(&path, body).unwrap();
4600
4601 let err = q.get(&t.id).unwrap_err().to_string();
4602 assert!(err.contains("schema 99"), "{err}");
4603 }
4604
4605 #[test]
4606 fn revision_moves_when_the_queue_changes() {
4607 let (_dir, q) = queue();
4608 assert_eq!(q.revision(), 0, "an empty queue has no revision");
4609 let mut t = task("first");
4610 q.put(&mut t).unwrap();
4611 assert!(q.revision() > 0, "a written task moves the revision");
4612 }
4613
4614 #[test]
4615 fn revision_moves_when_deleting_an_older_task() {
4616 let (dir, q) = queue();
4617 let questions = Questions::at(dir.path().join("questions"));
4618 let mut t1 = task("older");
4619 q.put(&mut t1).unwrap();
4620 std::thread::sleep(std::time::Duration::from_millis(10));
4622 let mut t2 = task("newer");
4623 q.put(&mut t2).unwrap();
4624
4625 let rev_before = q.revision();
4626 q.remove(&t1.id, false, &questions).unwrap();
4627 let rev_after = q.revision();
4628
4629 assert_ne!(
4630 rev_before, rev_after,
4631 "deleting an older task must change the revision so other clients see the deletion"
4632 );
4633 }
4634
4635 #[test]
4636 fn removing_a_task_takes_it_out_of_the_listing() {
4637 let (dir, q) = queue();
4638 let questions = Questions::at(dir.path().join("questions"));
4639 let mut t = task("delete me");
4640 q.put(&mut t).unwrap();
4641 let removed = q.remove(t.short(), false, &questions).unwrap();
4642 assert_eq!(removed.id, t.id, "a prefix resolves before deleting");
4643 assert!(removed.released.is_empty() && removed.still_blocked.is_empty());
4644 assert!(q.list().is_empty());
4645 assert!(
4646 q.remove(&t.id, false, &questions).is_err(),
4647 "removing twice is an error"
4648 );
4649 }
4650
4651 #[test]
4652 fn removing_a_task_takes_its_stale_lock_with_it() {
4653 let (dir, q) = queue();
4654 let questions = Questions::at(dir.path().join("questions"));
4655 let mut t = task("interrupted");
4656 q.put(&mut t).unwrap();
4657
4658 let claim = q.claim(&t.id).unwrap();
4661 std::mem::forget(claim);
4662 assert!(
4663 q.claim(&t.id).is_err(),
4664 "the orphaned lock is what makes the task look claimed"
4665 );
4666
4667 let err = q.remove(&t.id, true, &questions).unwrap_err().to_string();
4669 assert!(err.contains("live daemon"), "{err}");
4670 assert!(q.get(&t.id).is_ok(), "a refused delete keeps the task");
4671
4672 q.remove(&t.id, false, &questions).unwrap();
4674 assert!(q.list().is_empty());
4675 let mut again = task("interrupted");
4676 again.id = t.id.clone();
4677 q.put(&mut again).unwrap();
4678 assert!(
4679 q.claim(&t.id).is_ok(),
4680 "a task that comes back must be claimable, which a left-behind lock would prevent"
4681 );
4682 }
4683
4684 fn notices_of(dir: &Path) -> Vec<crate::notices::Notice> {
4685 crate::notices::Notices::at(dir.join("notifications")).list()
4686 }
4687
4688 #[test]
4689 fn removing_a_sole_dependency_releases_the_dependent_without_a_hold() {
4690 let (dir, q) = queue();
4691 let questions = Questions::at(dir.path().join("questions"));
4692 let mut dep = task("dependency");
4693 q.put(&mut dep).unwrap();
4694 let mut blocked = task("waiting");
4695 blocked.block(vec![dep.id.clone()], Some("waits".to_owned()));
4696 q.put(&mut blocked).unwrap();
4697
4698 let removed = q.remove(&dep.id, false, &questions).unwrap();
4699 assert_eq!(removed.released, [blocked.id.clone()]);
4700 assert!(removed.still_blocked.is_empty());
4701
4702 let after = q.get(&blocked.id).unwrap();
4703 assert_eq!(after.status, TaskStatus::Queued);
4704 assert!(after.blocked_by.is_empty());
4705 assert!(after.block_reason.is_none());
4706 let notes = notices_of(dir.path());
4707 assert_eq!(notes.len(), 1, "{notes:?}");
4708 assert_eq!(notes[0].severity, crate::notices::Severity::Info);
4709 assert_eq!(
4710 notes[0].link,
4711 Some(crate::notices::Link::Task {
4712 id: blocked.id.clone(),
4713 })
4714 );
4715 }
4716
4717 #[test]
4718 fn removing_one_of_two_dependencies_keeps_the_other() {
4719 let (dir, q) = queue();
4720 let questions = Questions::at(dir.path().join("questions"));
4721 let mut dep = task("dependency");
4722 q.put(&mut dep).unwrap();
4723 let mut other = task("other");
4724 q.put(&mut other).unwrap();
4725 let mut blocked = task("waiting");
4726 blocked.block(
4727 vec![dep.id.clone(), other.id.clone()],
4728 Some("waits on both".to_owned()),
4729 );
4730 q.put(&mut blocked).unwrap();
4731
4732 let removed = q.remove(&dep.id, false, &questions).unwrap();
4733 assert!(removed.released.is_empty());
4734 assert_eq!(removed.still_blocked, [blocked.id.clone()]);
4735
4736 let after = q.get(&blocked.id).unwrap();
4737 assert_eq!(after.status, TaskStatus::Blocked);
4738 assert_eq!(after.blocked_by, [other.id.clone()]);
4739 assert!(missing_blockers(&q, &questions, &after.blocked_by).is_empty());
4742 }
4743
4744 #[test]
4745 fn a_dependency_deleted_before_its_dependents_were_rewritten_is_released_later() {
4746 let (dir, q) = queue();
4747 let questions = Questions::at(dir.path().join("questions"));
4748 let mut dep = task("dependency");
4749 q.put(&mut dep).unwrap();
4750 let mut blocked = task("waiting");
4751 blocked.block(vec![dep.id.clone()], None);
4752 q.put(&mut blocked).unwrap();
4753
4754 let claim = q.claim(&blocked.id).unwrap();
4756 let removed = q.remove(&dep.id, false, &questions).unwrap();
4757 assert!(removed.released.is_empty());
4758 drop(claim);
4759
4760 let mut task = q.get(&blocked.id).unwrap();
4761 assert_eq!(task.status, TaskStatus::Blocked);
4762 assert!(missing_blockers(&q, &questions, &task.blocked_by).is_empty());
4763 assert_eq!(
4764 q.apply_deleted_blockers(&mut task, &questions),
4765 [dep.id.clone()]
4766 );
4767 assert_eq!(task.status, TaskStatus::Queued);
4768 }
4769
4770 #[test]
4771 fn dependency_resolution_short_deleted_reference_releases_only_its_waiters() {
4772 let (dir, q) = queue();
4773 let questions = Questions::at(dir.path().join("questions"));
4774 let mut dep = task("dependency");
4775 dep.id = "20261013-193611-ca5b".into();
4776 q.put(&mut dep).unwrap();
4777 let mut waiting = task("waiting");
4778 waiting.block(vec!["ca5b".into()], None);
4779 q.put(&mut waiting).unwrap();
4780 q.remove(&dep.id, false, &questions).unwrap();
4781 assert_eq!(q.get(&waiting.id).unwrap().status, TaskStatus::Queued);
4782 assert_eq!(
4783 dependency(&q, &questions, "ca5b"),
4784 Dependency::Deleted(dep.id)
4785 );
4786 }
4787
4788 #[test]
4789 fn dependency_resolution_preserves_cross_store_ambiguity_and_exact_ids() {
4790 let (dir, queue) = queue();
4791 let questions = Questions::at(dir.path().join("questions"));
4792 let mut dep = task("dependency");
4793 dep.id = "20261013-193611-ca5b".into();
4794 queue.put(&mut dep).unwrap();
4795 std::fs::create_dir_all(questions.root()).unwrap();
4796 let question_id = "20261014-193611-ca5b";
4797 std::fs::write(questions.path_of(question_id), "unreadable question").unwrap();
4798 assert_eq!(questions.resolve_id("ca5b").unwrap(), question_id);
4799 assert_eq!(
4800 dependency(&queue, &questions, "ca5b"),
4801 Dependency::Ambiguous
4802 );
4803 assert_eq!(
4804 dependency(&queue, &questions, &dep.id),
4805 Dependency::Task(dep.id)
4806 );
4807 assert_eq!(
4808 dependency(&queue, &questions, question_id),
4809 Dependency::Question(question_id.into())
4810 );
4811 assert_eq!(
4812 dependency(&queue, &questions, "missing"),
4813 Dependency::Missing
4814 );
4815 }
4816
4817 #[test]
4818 fn dependency_resolution_directory_errors_are_unknown_not_missing() {
4819 let dir = tempfile::tempdir().unwrap();
4820 let q = Queue::at(dir.path().join("not-a-directory"));
4821 std::fs::write(q.root(), "file").unwrap();
4822 let questions = Questions::at(dir.path().join("questions"));
4823 assert_eq!(dependency(&q, &questions, "ca5b"), Dependency::Unknown);
4824 assert!(missing_blockers(&q, &questions, &["ca5b".into()]).is_empty());
4825 }
4826
4827 #[test]
4828 fn a_dependency_without_a_tombstone_is_still_missing() {
4829 let (dir, q) = queue();
4830 let questions = Questions::at(dir.path().join("questions"));
4831 let mut dep = task("dependency");
4832 q.put(&mut dep).unwrap();
4833 std::fs::remove_file(q.path_of(&dep.id)).unwrap();
4834 assert_eq!(
4835 missing_blockers(&q, &questions, std::slice::from_ref(&dep.id)),
4836 [dep.id.clone()]
4837 );
4838 assert!(deleted_blockers(&q, &questions, std::slice::from_ref(&dep.id)).is_empty());
4839 }
4840
4841 #[test]
4842 fn a_held_dependent_is_left_alone_by_a_dependency_deletion() {
4843 let mut t = task("held");
4844 t.hold_machine(Some("because".to_owned()));
4845 t.blocked_by = vec!["gone".to_owned()];
4846 assert!(!t.dependency_deleted("gone"));
4847 assert_eq!(t.status, TaskStatus::Held);
4848 }
4849
4850 #[test]
4851 fn a_failed_record_removal_takes_the_tombstone_back() {
4852 let (_dir, q) = queue();
4853 let mut t = task("stays");
4854 q.put(&mut t).unwrap();
4855 q.write_tombstone(&t.id).unwrap();
4856 let err = q.remove_record_with_attachments(&t.id, |_| Err(std::io::Error::other("nope")));
4857 assert!(err.is_err());
4858 assert!(!q.was_deleted(&t.id));
4861 }
4862
4863 fn source_file(dir: &Path, name: &str, body: &str) -> PathBuf {
4864 let p = dir.join(name);
4865 std::fs::write(&p, body).unwrap();
4866 p
4867 }
4868
4869 #[test]
4870 fn an_attachment_copy_survives_deleting_its_source() {
4871 let (dir, q) = queue();
4872 let src = source_file(dir.path(), "shot.png", "pixels");
4873 let mut t = task("with a picture");
4874 let names = q.attach(&mut t, std::slice::from_ref(&src)).unwrap();
4875 q.put(&mut t).unwrap();
4876 std::fs::remove_file(&src).unwrap();
4877 assert_eq!(names, ["shot.png"]);
4878 let loaded = q.get(&t.id).unwrap();
4879 let paths = q.attachment_paths(&loaded);
4880 assert_eq!(paths.len(), 1);
4881 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "pixels");
4882 }
4883
4884 #[test]
4885 fn attachment_names_that_could_traverse_or_are_odd_are_refused() {
4886 let (dir, q) = queue();
4887 let mut t = task("bad names");
4888 for name in ["a..b.png", ".hidden", "with space.png", "-x.png"] {
4889 let src = source_file(dir.path(), name, "x");
4890 assert!(
4891 q.attach(&mut t, &[src]).is_err(),
4892 "`{name}` must be refused"
4893 );
4894 }
4895 assert!(!crate::ask::valid_asset_name("C:foo.png"));
4899 #[cfg(not(windows))]
4900 {
4901 let src = source_file(dir.path(), "C:foo.png", "x");
4902 assert!(q.attach(&mut t, &[src]).is_err());
4903 }
4904 let long = format!("{}.png", "a".repeat(70));
4905 let src = source_file(dir.path(), &long, "x");
4906 assert!(q.attach(&mut t, &[src]).is_err());
4907 assert!(t.attachments.is_empty());
4908 assert!(!q.attachments_dir(&t.id).exists());
4909 }
4910
4911 #[test]
4912 fn a_taken_attachment_name_is_numbered_not_overwritten() {
4913 let (dir, q) = queue();
4914 let a = source_file(dir.path(), "shot.png", "one");
4915 let sub = dir.path().join("other");
4916 std::fs::create_dir_all(&sub).unwrap();
4917 let b = source_file(&sub, "shot.png", "two");
4918 let mut t = task("collision");
4919 q.attach(&mut t, &[a]).unwrap();
4920 q.attach(&mut t, &[b]).unwrap();
4921 assert_eq!(t.attachments, ["shot.png", "shot-2.png"]);
4922 let paths = q.attachment_paths(&t);
4923 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "one");
4924 assert_eq!(std::fs::read_to_string(&paths[1]).unwrap(), "two");
4925 }
4926
4927 #[test]
4928 fn a_renumbered_name_stays_inside_the_length_bound() {
4929 let (dir, q) = queue();
4930 let name = format!("{}.png", "a".repeat(60));
4931 assert_eq!(name.len(), 64);
4932 let a = source_file(dir.path(), &name, "one");
4933 let sub = dir.path().join("other");
4934 std::fs::create_dir_all(&sub).unwrap();
4935 let b = source_file(&sub, &name, "two");
4936 let mut t = task("long");
4937 q.attach(&mut t, &[a, b]).unwrap();
4938 assert_eq!(t.attachments.len(), 2);
4939 assert!(
4940 t.attachments
4941 .iter()
4942 .all(|n| crate::ask::valid_asset_name(n))
4943 );
4944 assert!(t.attachments[1].ends_with("-2.png"));
4945 }
4946
4947 #[test]
4948 fn a_failed_attach_keeps_existing_attachments_and_leaves_no_partial_copy() {
4949 let (dir, q) = queue();
4950 let good = source_file(dir.path(), "good.png", "ok");
4951 let mut t = task("partial");
4952 q.attach(&mut t, &[good]).unwrap();
4953 let more = source_file(dir.path(), "more.png", "ok");
4954 let missing = dir.path().join("missing.png");
4955 assert!(q.attach(&mut t, &[more, missing]).is_err());
4956 assert_eq!(t.attachments, ["good.png"]);
4957 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
4958 .unwrap()
4959 .flatten()
4960 .collect();
4961 assert_eq!(on_disk.len(), 1);
4962 }
4963
4964 fn block_put(q: &Queue, t: &Task) -> PathBuf {
4966 let tmp = q.path_of(&t.id).with_extension("json.tmp");
4967 std::fs::create_dir_all(&tmp).unwrap();
4968 tmp
4969 }
4970
4971 #[test]
4972 fn a_failed_put_leaves_no_new_attachment_directory() {
4973 let (dir, q) = queue();
4974 let mut t = task("fresh");
4975 let tmp = block_put(&q, &t);
4976 let src = source_file(dir.path(), "shot.png", "x");
4977 assert!(q.attach_and_put(&mut t, &[src]).is_err());
4978 assert!(t.attachments.is_empty());
4979 assert!(!q.attachments_dir(&t.id).exists());
4980 assert!(!q.path_of(&t.id).exists());
4981 std::fs::remove_dir(tmp).unwrap();
4982 }
4983
4984 #[test]
4985 fn a_failed_put_removes_only_the_copy_it_just_made() {
4986 let (dir, q) = queue();
4987 let mut t = task("edited");
4988 let first = source_file(dir.path(), "first.png", "1");
4989 q.attach_and_put(&mut t, &[first]).unwrap();
4990 block_put(&q, &t);
4991 let second = source_file(dir.path(), "second.png", "2");
4992 assert!(q.attach_and_put(&mut t, &[second]).is_err());
4993 assert_eq!(t.attachments, ["first.png"]);
4994 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
4995 .unwrap()
4996 .flatten()
4997 .map(|e| e.file_name().to_string_lossy().into_owned())
4998 .collect();
4999 assert_eq!(on_disk, ["first.png"]);
5000 assert_eq!(q.get(&t.id).unwrap().attachments, ["first.png"]);
5001 }
5002
5003 #[test]
5004 fn a_leftover_removing_directory_is_swept_by_the_next_removal() {
5005 let (dir, q) = queue();
5006 let questions = Questions::at(dir.path().join("questions"));
5007 let gone = task("gone");
5008 let mut other = task("other");
5009 let mut live = task("live");
5010 q.put(&mut other).unwrap();
5011 q.put(&mut live).unwrap();
5012 let orphan = q.root.join(format!("{}.attachments.removing", gone.id));
5015 std::fs::create_dir_all(&orphan).unwrap();
5016 std::fs::write(orphan.join("shot.png"), "x").unwrap();
5017 let busy = q.root.join(format!("{}.attachments.removing", live.id));
5020 std::fs::create_dir_all(&busy).unwrap();
5021
5022 q.remove(&other.id, false, &questions).unwrap();
5023 assert!(!orphan.exists(), "an orphan is swept");
5024 assert!(busy.exists(), "a removal in progress is left alone");
5025 }
5026
5027 #[test]
5028 fn a_blocked_aside_rename_fails_the_removal_and_loses_nothing() {
5029 let (dir, q) = queue();
5030 let questions = Questions::at(dir.path().join("questions"));
5031 let mut t = task("stuck");
5032 let src = source_file(dir.path(), "shot.png", "x");
5033 q.attach_and_put(&mut t, &[src]).unwrap();
5034 let aside = q.root.join(format!("{}.attachments.removing", t.id));
5037 std::fs::create_dir_all(&aside).unwrap();
5038 std::fs::write(aside.join("old.png"), "o").unwrap();
5039 assert!(q.remove(&t.id, false, &questions).is_err());
5040 assert!(q.path_of(&t.id).exists());
5041 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5042 }
5043
5044 #[test]
5045 fn a_failed_record_removal_puts_the_attachments_back() {
5046 let (dir, q) = queue();
5047 let mut t = task("rollback");
5048 let src = source_file(dir.path(), "shot.png", "x");
5049 q.attach_and_put(&mut t, &[src]).unwrap();
5050 let err = q
5051 .remove_record_with_attachments(&t.id, |_| {
5052 Err(std::io::Error::other("injected failure"))
5053 })
5054 .unwrap_err();
5055 assert!(format!("{err:#}").contains("injected failure"));
5056 assert!(q.path_of(&t.id).exists());
5057 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5058 assert!(
5059 !q.root
5060 .join(format!("{}.attachments.removing", t.id))
5061 .exists()
5062 );
5063 }
5064
5065 #[test]
5066 fn editing_a_task_keeps_its_attachments() {
5067 let (dir, q) = queue();
5068 let src = source_file(dir.path(), "shot.png", "x");
5069 let mut t = task("editable");
5070 q.attach(&mut t, &[src]).unwrap();
5071 t.edit("new".to_owned(), "new text".to_owned()).unwrap();
5072 q.put(&mut t).unwrap();
5073 assert_eq!(q.get(&t.id).unwrap().attachments, ["shot.png"]);
5074 }
5075
5076 #[test]
5077 fn removing_a_task_deletes_its_attachments() {
5078 let (dir, q) = queue();
5079 let questions = Questions::at(dir.path().join("questions"));
5080 let src = source_file(dir.path(), "shot.png", "x");
5081 let mut t = task("doomed");
5082 q.attach(&mut t, &[src]).unwrap();
5083 q.put(&mut t).unwrap();
5084 assert!(q.attachments_dir(&t.id).is_dir());
5085 q.remove(&t.id, false, &questions).unwrap();
5086 assert!(!q.attachments_dir(&t.id).exists());
5087 assert!(q.list().is_empty());
5088 }
5089
5090 #[test]
5091 fn a_task_written_before_attachments_still_reads() {
5092 let (_dir, q) = queue();
5093 let mut t = task("old");
5094 q.put(&mut t).unwrap();
5095 let path = q.path_of(&t.id);
5096 let mut v: serde_json::Value =
5097 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5098 v.as_object_mut().unwrap().remove("attachments");
5099 std::fs::write(&path, v.to_string()).unwrap();
5100 assert!(q.get(&t.id).unwrap().attachments.is_empty());
5101 }
5102
5103 #[test]
5104 fn attachment_paths_are_absolute_even_when_the_root_is_relative() {
5105 let q = Queue::at(PathBuf::from("relative-queue"));
5106 let mut t = task("rel");
5107 t.attachments.push("shot.png".to_owned());
5108 let paths = q.attachment_paths(&t);
5109 assert!(paths[0].is_absolute(), "{}", paths[0].display());
5110 assert!(paths[0].ends_with(format!("{}.attachments/shot.png", t.id)));
5111 }
5112
5113 #[test]
5114 fn explicitly_resumed_older_run_becomes_current_without_duplicate_history() {
5115 let dir = tempfile::tempdir().unwrap();
5116 let q = Queue::at(dir.path().join("queue"));
5117 let mut t = task("resume");
5118 t.start("A".into());
5119 t.record_pass("A", 0, 1, None);
5120 t.start("B".into());
5121 q.put(&mut t).unwrap();
5122 assert_eq!(t.runs, vec!["A", "B"]);
5123 t.start("A".into());
5124 t.record_pass("A", 1, 3, None);
5125 q.put(&mut t).unwrap();
5126 let saved = q.get(&t.id).unwrap();
5127 assert_eq!(saved.runs, vec!["B", "A"]);
5128 assert_eq!(saved.runs.last().map(String::as_str), Some("A"));
5129 assert_eq!(saved.attempts, 3);
5130 assert_eq!(saved.passes.len(), 2);
5131 assert_eq!(saved.passes[0].run, "A");
5132 assert_eq!(saved.passes[1].run, "A");
5133 }
5134
5135 #[test]
5136 fn repeated_start_records_one_run_and_separate_passes() {
5137 let mut t = task("resume");
5138 t.start("same-run".into());
5139 t.record_pass("same-run", 0, 1, None);
5140 t.start("same-run".into());
5141 t.record_pass("same-run", 1, 2, None);
5142 assert_eq!(t.runs, vec!["same-run"]);
5143 assert_eq!(t.attempts, 2);
5144 assert_eq!(t.passes.len(), 2);
5145 }
5146
5147 #[test]
5148 fn link_run_adds_a_run_once_and_touches_nothing_else() {
5149 let dir = tempfile::tempdir().unwrap();
5150 let queue = Queue::at(dir.path().join("queue"));
5151 let mut t = Task::new(
5152 "t".to_owned(),
5153 "do it".to_owned(),
5154 PathBuf::from("."),
5155 Source::Human,
5156 );
5157 queue.put(&mut t).unwrap();
5158 let before = queue.get(&t.id).unwrap();
5159
5160 let linked = queue.link_run(&t.id[..4], "20260930-092817-ec34").unwrap();
5161 assert_eq!(linked.runs, vec!["20260930-092817-ec34".to_owned()]);
5162 assert_eq!(linked.status, before.status);
5163 assert_eq!(linked.attempts, before.attempts);
5164 assert_eq!(linked.interrupt, before.interrupt);
5165
5166 let again = queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
5167 assert_eq!(again.runs.len(), 1, "linking twice must not duplicate");
5168 assert_eq!(queue.get(&t.id).unwrap().runs.len(), 1);
5169 assert!(queue.link_run("no-such-task", "r").is_err());
5170 }
5171
5172 #[test]
5173 fn put_keeps_a_run_linked_after_the_writer_took_its_snapshot() {
5174 let dir = tempfile::tempdir().unwrap();
5175 let queue = Queue::at(dir.path().join("queue"));
5176 let mut t = Task::new(
5177 "t".to_owned(),
5178 "do it".to_owned(),
5179 PathBuf::from("."),
5180 Source::Human,
5181 );
5182 queue.put(&mut t).unwrap();
5183 let mut snapshot = queue.get(&t.id).unwrap();
5185 queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
5186
5187 snapshot.start("20260930-000000-aaaa".to_owned());
5188 queue.put(&mut snapshot).unwrap();
5189
5190 let stored = queue.get(&t.id).unwrap();
5191 assert!(stored.runs.contains(&"20260930-092817-ec34".to_owned()));
5192 assert!(stored.runs.contains(&"20260930-000000-aaaa".to_owned()));
5193 assert_eq!(stored.attempts, 1);
5194 }
5195
5196 #[test]
5197 fn concurrent_links_and_daemon_saves_lose_nothing() {
5198 let dir = tempfile::tempdir().unwrap();
5199 let queue = Queue::at(dir.path().join("queue"));
5200 let mut t = Task::new(
5201 "t".to_owned(),
5202 "do it".to_owned(),
5203 PathBuf::from("."),
5204 Source::Human,
5205 );
5206 queue.put(&mut t).unwrap();
5207 let id = t.id.clone();
5208
5209 let linkers: Vec<_> = (0..4)
5210 .map(|n| {
5211 let (queue, id) = (queue.clone(), id.clone());
5212 std::thread::spawn(move || {
5213 for k in 0..10 {
5214 queue
5215 .link_run(&id, &format!("20260930-00000{n}-l{k:03}"))
5216 .unwrap();
5217 }
5218 })
5219 })
5220 .collect();
5221 let mut mine = queue.get(&id).unwrap();
5224 for k in 0..10 {
5225 mine.start(format!("20260930-000009-d{k:03}"));
5226 queue.put(&mut mine).unwrap();
5227 }
5228 for l in linkers {
5229 l.join().unwrap();
5230 }
5231
5232 let stored = queue.get(&id).unwrap();
5233 assert_eq!(stored.runs.len(), 50, "{:?}", stored.runs);
5234 assert_eq!(
5235 stored.attempts, 10,
5236 "linking never rewinds the daemon's work"
5237 );
5238 }
5239
5240 fn age_lock(path: &Path) {
5241 let f = std::fs::OpenOptions::new().write(true).open(path).unwrap();
5242 f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(60))
5243 .unwrap();
5244 }
5245
5246 #[test]
5247 fn concurrent_stale_takeover_yields_one_holder() {
5248 use std::sync::atomic::{AtomicUsize, Ordering};
5249 use std::sync::{Arc, Barrier};
5250 for _ in 0..5 {
5251 let dir = tempfile::tempdir().unwrap();
5252 let q = Queue::at(dir.path().to_path_buf());
5253 let lock = dir.path().join("t.write-lock");
5254 std::fs::write(&lock, "dead-0000").unwrap();
5255 age_lock(&lock);
5256 let n = 6;
5257 let barrier = Arc::new(Barrier::new(n));
5258 let (now, max) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
5259 let handles: Vec<_> = (0..n)
5260 .map(|_| {
5261 let (q, b, now, max) = (q.clone(), barrier.clone(), now.clone(), max.clone());
5262 std::thread::spawn(move || {
5263 b.wait();
5264 let g = q.lock_task("t").unwrap();
5265 let held = now.fetch_add(1, Ordering::SeqCst) + 1;
5266 max.fetch_max(held, Ordering::SeqCst);
5267 std::thread::sleep(std::time::Duration::from_millis(20));
5268 now.fetch_sub(1, Ordering::SeqCst);
5269 drop(g);
5270 })
5271 })
5272 .collect();
5273 for h in handles {
5274 h.join().unwrap();
5275 }
5276 assert_eq!(max.load(Ordering::SeqCst), 1);
5277 assert!(!lock.exists());
5278 }
5279 }
5280
5281 #[test]
5282 fn dropping_a_replaced_lock_leaves_the_new_holders_token() {
5283 let dir = tempfile::tempdir().unwrap();
5284 let q = Queue::at(dir.path().to_path_buf());
5285 let lock = dir.path().join("t.write-lock");
5286 let a = q.lock_task("t").unwrap();
5287 let replacement = owner_token();
5290 std::fs::write(&lock, &replacement).unwrap();
5291 drop(a);
5292 assert_eq!(std::fs::read_to_string(&lock).unwrap(), replacement);
5293 }
5294
5295 #[test]
5296 fn aged_live_task_lock_excludes_updates_until_retirement_releases_it() {
5297 let dir = tempfile::tempdir().unwrap();
5298 let q = Queue::at(dir.path().to_path_buf());
5299 let held = q.lock_task("t").unwrap();
5300 let path = dir.path().join("t.write-lock");
5301 age_lock(&path);
5302 assert!(!break_stale(&path, &held.token));
5303 let (started_tx, started_rx) = std::sync::mpsc::channel();
5304 let (acquired_tx, acquired_rx) = std::sync::mpsc::channel();
5305 let updater = std::thread::spawn(move || {
5306 started_tx.send(()).unwrap();
5307 let _write = q.lock_task("t").unwrap();
5308 acquired_tx.send(()).unwrap();
5309 });
5310 started_rx.recv().unwrap();
5311 assert!(
5312 acquired_rx
5313 .recv_timeout(std::time::Duration::from_millis(100))
5314 .is_err()
5315 );
5316 assert_eq!(std::fs::read_to_string(&path).unwrap(), held.token);
5317 drop(held);
5318 acquired_rx
5319 .recv_timeout(std::time::Duration::from_secs(2))
5320 .unwrap();
5321 updater.join().unwrap();
5322 assert!(!path.exists());
5323 }
5324
5325 #[test]
5326 fn break_stale_leaves_a_lock_that_replaced_the_one_judged() {
5327 let dir = tempfile::tempdir().unwrap();
5328 let lock = dir.path().join("t.write-lock");
5329 std::fs::write(&lock, "new-token").unwrap();
5330 assert!(!break_stale(&lock, "old-token"));
5331 assert_eq!(std::fs::read_to_string(&lock).unwrap(), "new-token");
5332 assert!(!break_stale(&lock, "new-token"));
5334 assert!(lock.exists());
5335 age_lock(&lock);
5336 assert!(break_stale(&lock, "new-token"));
5337 assert!(!lock.exists());
5338 }
5339
5340 #[test]
5341 fn a_stale_break_marker_is_recovered_and_a_fresh_one_is_respected() {
5342 let dir = tempfile::tempdir().unwrap();
5343 let lock = dir.path().join("t.write-lock");
5344 std::fs::write(&lock, "dead-1").unwrap();
5345 age_lock(&lock);
5346 let marker = break_marker(&lock, "dead-1");
5347 std::fs::write(&marker, "crashed-remover").unwrap();
5348 assert!(!break_stale(&lock, "dead-1"));
5350 assert!(lock.exists());
5351 age_lock(&marker);
5353 assert!(break_stale(&lock, "dead-1"));
5354 assert!(!lock.exists());
5355 assert!(!marker.exists());
5356 }
5357
5358 fn chat_task() -> Task {
5359 Task::new(
5360 "t".to_owned(),
5361 "do t".to_owned(),
5362 PathBuf::from("."),
5363 Source::Agent {
5364 run: "20260902-000000-beef".to_owned(),
5365 node: CHAT_NODE.to_owned(),
5366 },
5367 )
5368 }
5369
5370 #[test]
5371 fn a_blocked_hold_keeps_its_anchor_but_a_new_run_starts_a_new_hold() {
5372 let mut t = chat_task();
5373 t.hold_machine(Some("waiting".into()));
5374 let old: Timestamp = "2020-01-01T00:00:00Z".parse().unwrap();
5375 t.held_at = Some(old);
5376 t.block(vec!["decision".into()], None);
5377 t.hold_machine(Some("same waiting decision".into()));
5378 assert_eq!(t.held_at, Some(old));
5379 t.start("real-new-run".into());
5380 t.fail("new failure", 1);
5381 assert_ne!(
5382 t.held_at,
5383 Some(old),
5384 "old answers cannot approve a new hold"
5385 );
5386 }
5387
5388 #[test]
5389 fn legacy_delivery_survives_upgrade_and_a_no_progress_run_link() {
5390 let (_dir, queue) = queue();
5391 let mut t = chat_task();
5392 t.start("old-run".into());
5393 t.hold_machine(Some("operator is considering recovery".into()));
5394 t.chat_notice = Some(ChatNotice {
5395 key: format!("held#{}", t.held_at.unwrap()),
5396 at: Timestamp::now(),
5397 outcome: ChatNoticeOutcome::Posted,
5398 });
5399 let mut raw = serde_json::to_value(&t).unwrap();
5400 raw.as_object_mut().unwrap().remove("chat_notices");
5401 raw.as_object_mut().unwrap().remove("hold_notice_cause");
5402 std::fs::create_dir_all(queue.path_of(&t.id).parent().unwrap()).unwrap();
5403 std::fs::write(queue.path_of(&t.id), serde_json::to_string(&raw).unwrap()).unwrap();
5404 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
5405 queue
5406 .modify(&t.id, |task| {
5407 task.requeue();
5408 task.link_run("unprepared-run");
5409 task.startup_notice_cause("startup-source");
5410 task.fail("same unresolved preflight", 0);
5411 true
5412 })
5413 .unwrap();
5414 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
5415 queue
5416 .modify(&t.id, |task| {
5417 task.succeed();
5418 true
5419 })
5420 .unwrap();
5421 assert_eq!(
5422 queue.get(&t.id).unwrap().chat_notice_due().as_deref(),
5423 Some("done")
5424 );
5425 }
5426
5427 #[test]
5428 fn a_dependency_wait_does_not_change_when_released_and_blocked_again() {
5429 let mut t = chat_task();
5430 t.block(vec!["a".into(), "b".into()], Some("waiting".into()));
5431 let key = t.chat_notice_key().unwrap();
5432 t.chat_notices.push(key);
5433 t.release();
5434 t.block(
5435 vec!["b".into(), "a".into(), "a".into()],
5436 Some("still waiting on prerequisites".into()),
5437 );
5438 assert_eq!(t.chat_notice_due(), None);
5439 t.block(
5440 vec!["new-question".into()],
5441 Some("different decision".into()),
5442 );
5443 assert!(t.chat_notice_due().is_some());
5444 }
5445
5446 #[test]
5447 fn running_startup_failures_preserve_stage_and_report_distinct_blockers() {
5448 let mut t = chat_task();
5449 t.start("existing-run".into());
5450 t.fail("implementation failed", 1);
5451 let failed = t.chat_notice_key().unwrap();
5452 t.chat_notices.push(failed.clone());
5453 t.release_operator("test", None);
5454 t.start("existing-run".into());
5455 t.fail_startup("startup-source", "same unresolved recovery source", 1);
5456 assert_eq!(
5457 t.chat_notice_due(),
5458 None,
5459 "no-progress recovery stays quiet"
5460 );
5461 t.release_operator("test", None);
5462 t.start("existing-run".into());
5463 t.fail_startup("startup-attachments", "attachment missing", 1);
5464 let attachments = t.chat_notice_due().expect("new attachment blocker");
5465 assert_ne!(attachments, failed);
5466 assert_eq!(
5467 t.hold_notice_cause.as_ref().unwrap().stage,
5468 "startup-attachments"
5469 );
5470 t.chat_notices.push(attachments);
5471 t.release_operator("test", None);
5472 t.start("existing-run".into());
5473 t.fail_startup(
5474 "startup-attachments",
5475 "same missing attachment, reworded",
5476 1,
5477 );
5478 assert_eq!(
5479 t.chat_notice_due(),
5480 None,
5481 "same stage after attempt counter reset"
5482 );
5483 t.release_operator("test", None);
5484 t.start("existing-run".into());
5485 t.fail_startup("startup-config", "different preflight stage", 1);
5486 assert!(t.chat_notice_due().is_some());
5487 assert_eq!(
5488 t.hold_notice_cause.as_ref().unwrap().stage,
5489 "startup-config"
5490 );
5491 }
5492
5493 #[test]
5494 fn structured_causes_preserve_no_progress_retries_and_report_new_passes() {
5495 let mut t = chat_task();
5496 t.hold_machine(Some("startup source missing".into()));
5497 t.startup_notice_cause("startup-source");
5498 let first = t.chat_notice_key().unwrap();
5499 t.chat_notices.push(first.clone());
5500 t.release_operator("test", None);
5501 t.startup_notice_cause("startup-source");
5502 t.hold_machine(Some("same source cannot be read".into()));
5503 assert_eq!(t.chat_notice_due(), None);
5504 t.startup_notice_cause("startup-config");
5505 assert!(t.chat_notice_due().is_some(), "different startup stage");
5506 t.notice_cause("failed", Some("existing-run"), Some(0));
5507 let run_failure = t.chat_notice_key().unwrap();
5508 t.chat_notices.push(run_failure.clone());
5509 t.release_operator("test", None);
5510 t.startup_notice_cause("startup-source");
5511 t.hold_machine(Some("no new work happened".into()));
5512 assert_eq!(t.chat_notice_due(), None);
5513 t.notice_cause("failed", Some("existing-run"), Some(1));
5514 assert!(t.chat_notice_due().is_some(), "a real resumed pass failed");
5515 assert_ne!(t.chat_notice_key().unwrap(), run_failure);
5516 }
5517
5518 #[test]
5519 fn legacy_hold_baseline_does_not_replay_or_hide_new_failures_and_completion() {
5520 let mut t = chat_task();
5521 t.hold_machine(Some("still waiting".into()));
5522 t.chat_notice = Some(ChatNotice {
5523 key: "held#2026-10-14T00:00:00Z".into(),
5524 at: Timestamp::now(),
5525 outcome: ChatNoticeOutcome::Posted,
5526 });
5527 assert_eq!(t.chat_notice_due(), None, "old hold baseline");
5528 t.import_chat_notice();
5529 t.notice_cause("failed", Some("new-run"), Some(0));
5530 assert!(t.chat_notice_due().is_some(), "new producer evidence");
5531 t.succeed();
5532 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
5533 }
5534
5535 #[test]
5536 fn a_chat_notice_is_due_only_for_a_final_state_and_only_once() {
5537 let mut t = chat_task();
5538 assert_eq!(t.chat_notice_due(), None, "queued");
5539 t.start("r1".to_owned());
5540 assert_eq!(t.chat_notice_due(), None, "running");
5541 t.fail("boom", 3);
5542 assert_eq!(t.status, TaskStatus::Failed);
5543 assert_eq!(t.chat_notice_due(), None, "a retried failure is not final");
5544 t.start("r2".to_owned());
5545 t.park("upgrade");
5546 assert_eq!(t.chat_notice_due(), None, "parked");
5547 t.fail("boom", 1);
5548 assert_eq!(t.status, TaskStatus::Held);
5549 let key = t
5550 .chat_notice_due()
5551 .expect("held on the last attempt is final");
5552 t.chat_notice = Some(ChatNotice {
5553 key,
5554 at: Timestamp::now(),
5555 outcome: ChatNoticeOutcome::Posted,
5556 });
5557 assert_eq!(t.chat_notice_due(), None, "same hold, no second notice");
5558 t.release();
5559 assert_eq!(t.chat_notice_due(), None, "requeued after a release");
5560 t.start("r3".to_owned());
5561 t.succeed();
5562 assert!(t.chat_notice_due().is_some(), "done is a new final state");
5563 }
5564
5565 #[test]
5566 fn completed_chat_notices_keep_current_and_legacy_keys_after_requeue() {
5567 for key in ["done", "done#0", "done#1", "done#12"] {
5568 let mut t = chat_task();
5569 t.start("r1".to_owned());
5570 t.succeed();
5571 t.chat_notice = Some(ChatNotice {
5572 key: key.to_owned(),
5573 at: Timestamp::now(),
5574 outcome: ChatNoticeOutcome::Posted,
5575 });
5576 t.requeue();
5577 t.start("r2".to_owned());
5578 t.succeed();
5579 assert_eq!(t.chat_notice_key().as_deref(), Some("done"));
5580 assert_eq!(t.chat_notice_due(), None, "recorded key: {key}");
5581 }
5582 }
5583
5584 #[test]
5585 fn a_release_without_progress_keeps_the_unresolved_notice_cause() {
5586 let mut t = chat_task();
5587 t.hold_manual(Some("wait".to_owned()));
5588 let first = t.chat_notice_due().expect("held");
5589 t.chat_notice = Some(ChatNotice {
5590 key: first.clone(),
5591 at: Timestamp::now(),
5592 outcome: ChatNoticeOutcome::Posted,
5593 });
5594 t.hold_manual(Some("still waiting".to_owned()));
5595 assert_eq!(t.chat_notice_due(), None, "a re-hold keeps its start");
5596 t.release();
5597 std::thread::sleep(std::time::Duration::from_millis(5));
5598 t.hold_manual(None);
5599 assert_eq!(t.chat_notice_due(), None, "no progress, same cause");
5600 }
5601
5602 #[test]
5603 fn a_chat_notice_needs_a_chat_source_and_a_current_schema() {
5604 let mut human = task("h");
5605 human.succeed();
5606 assert_eq!(human.chat_notice_due(), None);
5607 let mut inherited = task("i");
5609 inherited.origin_chat = Some("20260902-000000-beef".to_owned());
5610 inherited.succeed();
5611 assert_eq!(inherited.chat_notice_due(), None);
5612 let mut old = chat_task();
5613 old.schema = 12;
5614 old.status = TaskStatus::Done;
5615 assert_eq!(old.chat_notice_due(), None, "history is not replayed");
5616 let mut live = chat_task();
5617 live.schema = 12;
5618 live.start("r1".to_owned());
5619 live.succeed();
5620 assert!(live.chat_notice_due().is_some(), "started by this build");
5621 let mut held = chat_task();
5622 held.schema = 12;
5623 held.hold_machine(Some("config".to_owned()));
5624 assert!(held.chat_notice_due().is_some(), "held before any start");
5625 }
5626
5627 #[test]
5628 fn a_stale_snapshot_does_not_rewind_a_recorded_chat_notice() {
5629 let tmp = tempfile::tempdir().expect("tempdir");
5630 let q = Queue::at(tmp.path().join("queue"));
5631 let mut t = chat_task();
5632 t.succeed();
5633 q.put(&mut t).expect("put");
5634 let mut stale = q.get(&t.id).expect("get");
5635 let key = t.chat_notice_due().expect("due");
5636 q.modify(&t.id, |x| {
5637 x.chat_notice = Some(ChatNotice {
5638 key: key.clone(),
5639 at: Timestamp::now(),
5640 outcome: ChatNoticeOutcome::Posted,
5641 });
5642 true
5643 })
5644 .expect("modify");
5645 q.put(&mut stale).expect("stale put");
5646 assert_eq!(q.get(&t.id).expect("get").chat_notice_due(), None);
5647 }
5648
5649 #[test]
5650 fn empty_sources_are_omitted_so_older_builds_still_read_the_task() {
5651 let mut t = task("plain");
5654 let value = serde_json::to_value(&t).unwrap();
5655 assert!(value.get("sources").is_none(), "{value}");
5656 t.sources.pick_commits.push(crate::refs::Input {
5657 token: "abc1234".into(),
5658 sha: "a".repeat(40),
5659 provenance: crate::refs::Provenance::ExplicitCli,
5660 });
5661 let value = serde_json::to_value(&t).unwrap();
5662 assert_eq!(value["sources"]["pick_commits"][0]["sha"], "a".repeat(40));
5663 let restored: Task = serde_json::from_value(value).unwrap();
5664 assert_eq!(restored.sources, t.sources);
5665 }
5666}