1#[cfg(test)]
34thread_local! {
35 pub(crate) static LIST_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
36}
37
38use std::collections::HashMap;
39use std::num::NonZeroUsize;
40use std::path::{Path, PathBuf};
41
42use anyhow::{Context, Result, bail};
43use jiff::Timestamp;
44use serde::{Deserialize, Serialize};
45
46use crate::ask::Questions;
47use crate::id_reserve::{IdSource, path_exists};
48
49pub const SCHEMA: u32 = 22;
144
145pub(crate) fn dispatch_order(a: &Task, b: &Task) -> std::cmp::Ordering {
151 match (a.upgrade_resume_key(), b.upgrade_resume_key()) {
152 (Some(a), Some(b)) => a.cmp(&b),
153 (Some(_), None) => std::cmp::Ordering::Less,
154 (None, Some(_)) => std::cmp::Ordering::Greater,
155 (None, None) => b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)),
156 }
157}
158
159#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
166#[serde(tag = "kind", rename_all = "snake_case")]
167pub enum MachineHoldCause {
168 DiskCapacity {
170 reason: String,
172 },
173}
174
175#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
177#[serde(rename_all = "lowercase")]
178pub enum HoldSource {
179 Manual,
181 Machine,
183}
184
185impl HoldSource {
186 pub fn label(self) -> &'static str {
188 match self {
189 Self::Manual => "manual",
190 Self::Machine => "machine",
191 }
192 }
193}
194
195#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
198#[serde(tag = "kind", rename_all = "lowercase")]
199pub enum Source {
200 Human,
202 Agent {
205 run: String,
207 node: String,
209 },
210 Issue {
212 number: u64,
214 repo: String,
216 },
217}
218
219pub const CHAT_NODE: &str = "chat";
222
223impl Source {
224 pub fn label(&self) -> String {
226 match self {
227 Self::Human => "human".to_owned(),
228 Self::Agent { run, node } => format!("{node}@{}", short(run)),
229 Self::Issue { number, .. } => format!("issue #{number}"),
230 }
231 }
232}
233
234#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
236#[serde(rename_all = "lowercase")]
237pub enum TaskStatus {
238 Queued,
240 Running,
242 Done,
244 Failed,
246 Held,
248 Blocked,
252 Parked,
257}
258
259impl TaskStatus {
260 pub fn runnable(self) -> bool {
262 matches!(self, Self::Queued | Self::Failed | Self::Parked)
263 }
264
265 pub fn as_str(self) -> &'static str {
267 match self {
268 Self::Queued => "queued",
269 Self::Running => "running",
270 Self::Done => "done",
271 Self::Failed => "failed",
272 Self::Held => "held",
273 Self::Blocked => "blocked",
274 Self::Parked => "parked",
275 }
276 }
277}
278
279#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
282pub struct TaskCounts {
283 pub queued: usize,
285 pub running: usize,
287 pub done: usize,
289 pub failed: usize,
291 pub held: usize,
293 pub blocked: usize,
295 pub parked: usize,
297}
298
299impl TaskCounts {
300 pub fn of(tasks: &[Task]) -> Self {
304 Self::of_statuses(tasks.iter().map(|task| task.status))
305 }
306
307 pub(crate) fn of_statuses(statuses: impl IntoIterator<Item = TaskStatus>) -> Self {
309 let mut counts = Self::default();
310 for status in statuses {
311 match status {
312 TaskStatus::Queued => counts.queued += 1,
313 TaskStatus::Running => counts.running += 1,
314 TaskStatus::Done => counts.done += 1,
315 TaskStatus::Failed => counts.failed += 1,
316 TaskStatus::Held => counts.held += 1,
317 TaskStatus::Blocked => counts.blocked += 1,
318 TaskStatus::Parked => counts.parked += 1,
319 }
320 }
321 counts
322 }
323}
324
325#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
327pub struct Reassignment {
328 pub from: PathBuf,
330 pub to: PathBuf,
332 pub actor: String,
334 pub at: Timestamp,
336 pub run_boundary: usize,
338}
339
340#[derive(Debug, Clone, Serialize, Deserialize)]
342#[serde(deny_unknown_fields)]
343pub struct Task {
344 pub schema: u32,
346 pub id: String,
348 pub title: String,
350 pub instruction: String,
352 #[serde(default, skip_serializing_if = "crate::refs::Sources::is_empty")]
354 pub sources: crate::refs::Sources,
355 pub repo: PathBuf,
357 #[serde(default)]
359 pub reassignments: Vec<Reassignment>,
360 pub source: Source,
362 #[serde(default)]
364 pub chat: Option<String>,
365 #[serde(default)]
367 pub priority: i32,
368 #[serde(default)]
379 pub solo: bool,
380 pub status: TaskStatus,
382 #[serde(default)]
384 pub attempts: usize,
385 #[serde(default)]
387 pub attempt_limit: Option<NonZeroUsize>,
388 #[serde(default)]
391 pub runs: Vec<String>,
392 #[serde(default)]
394 pub startup_failures: Vec<crate::run::StartupFailure>,
395 #[serde(default)]
397 pub last_error: Option<String>,
398 #[serde(default)]
411 pub hold_reason: Option<String>,
412 #[serde(default)]
415 pub hold_source: Option<HoldSource>,
416 #[serde(default)]
418 pub machine_hold_cause: Option<MachineHoldCause>,
419 #[serde(default)]
421 pub disk_resume_run: Option<String>,
422 #[serde(default)]
424 pub disk_checked_at: Option<Timestamp>,
425 #[serde(default)]
437 pub diagnostic: Option<String>,
438 #[serde(default)]
448 pub blocked_by: Vec<String>,
449 #[serde(default)]
452 pub block_reason: Option<String>,
453 #[serde(default)]
466 pub blocked_from: Option<TaskStatus>,
467 #[serde(default)]
469 pub blocked_at: Option<Timestamp>,
470 #[serde(default)]
481 pub answers: Vec<AnsweredQuestion>,
482 #[serde(default)]
488 pub triage_applied: Vec<String>,
489 #[serde(default)]
495 pub actions_applied: Vec<String>,
496 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
499 pub decision_effects: std::collections::BTreeMap<String, String>,
500 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
503 pub superseded_resumes: std::collections::BTreeMap<String, String>,
504 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
508 pub dispatched_resumes: std::collections::BTreeMap<String, String>,
509 #[serde(default)]
511 pub base_recovery_requests: Vec<String>,
512 #[serde(default)]
517 pub resume_override: Option<OperatorResume>,
518 #[serde(default)]
520 pub repair_grants: Vec<crate::repair_budget::Receipt>,
521 #[serde(default)]
523 pub repair_reconciliations: Vec<crate::repair_budget::reconcile::Receipt>,
524 #[serde(default)]
527 pub resumption: Option<Resumption>,
528 #[serde(default)]
530 pub resumption_history: Vec<Resumption>,
531 #[serde(default)]
533 pub operator_decided_at: Option<Timestamp>,
534 #[serde(default)]
546 pub review_branch: Option<String>,
547 #[serde(default)]
549 pub review_instruction: Option<String>,
550 #[serde(default)]
552 pub pr_recovery: Option<crate::pr_recovery::Selection>,
553 #[serde(default)]
556 pub fresh_start: bool,
557 #[serde(default)]
559 pub replacement: Option<crate::merge_decision::Replacement>,
560 #[serde(default)]
571 pub interrupt: bool,
572 #[serde(default)]
595 pub urgent: bool,
596 #[serde(skip)]
601 pub(crate) unfiled: bool,
602 #[serde(default)]
608 pub attachments: Vec<String>,
609 #[serde(default)]
613 pub followup: Option<FollowUp>,
614 #[serde(default)]
620 pub origin_chat: Option<String>,
621 #[serde(default)]
628 pub overrides: Option<RunOverrides>,
629 #[serde(default)]
636 pub review_of: Option<String>,
637 #[serde(default)]
642 pub held_at: Option<Timestamp>,
643 #[serde(default)]
647 pub park_reason: Option<String>,
648 #[serde(default)]
651 pub chat_notice: Option<ChatNotice>,
652 #[serde(default)]
654 pub chat_notices: Vec<String>,
655 #[serde(default)]
658 pub hold_notice_cause: Option<NoticeCause>,
659 #[serde(default)]
664 pub passes: Vec<TaskPass>,
665 #[serde(default)]
669 pub dispatched_at: Option<Timestamp>,
670 #[serde(default)]
672 pub recovery_budget: Option<RecoveryBudget>,
673 #[serde(default)]
676 pub recovery_admission: Option<RecoveryAdmission>,
677 #[serde(default)]
680 pub recovery_generations: Vec<RecoveryBudget>,
681 pub created_at: Timestamp,
683 pub updated_at: Timestamp,
685}
686
687#[derive(Debug, Clone, Serialize, Deserialize)]
689pub struct RecoveryAdmission {
690 pub base: String,
692 pub inputs: String,
694}
695
696#[derive(Debug, Clone, Serialize, Deserialize)]
699pub struct RecoveryBudget {
700 pub instruction: String,
702 pub base: String,
704 pub inputs: String,
706 pub stage: String,
708 pub failures: usize,
710 #[serde(default)]
713 pub progress: Option<String>,
714}
715
716fn recovery_progress(state: &crate::run::RunState) -> Option<String> {
717 use crate::proc::Quiet as _;
718 use std::hash::{Hash, Hasher};
719 let mut patches = Vec::new();
720 for candidate in state.candidates.iter().filter(|c| c.commits > 0) {
721 let output = std::process::Command::new("git")
722 .current_dir(&state.repo)
723 .args(["diff", "--binary", &state.base_commit, &candidate.branch])
724 .quiet()
725 .output()
726 .ok()?;
727 if !output.status.success() {
728 return None;
729 }
730 patches.push(output.stdout);
731 }
732 patches.sort();
733 patches.dedup();
734 let mut hash = std::collections::hash_map::DefaultHasher::new();
735 patches.hash(&mut hash);
736 Some(format!("{:x}", hash.finish()))
737}
738
739pub(crate) fn recovery_inputs(sources: &crate::refs::Sources) -> String {
743 serde_json::to_string(
744 &sources
745 .start_from
746 .iter()
747 .map(|i| ("start", &i.token, &i.sha))
748 .chain(
749 sources
750 .pick_commits
751 .iter()
752 .map(|i| ("pick", &i.token, &i.sha)),
753 )
754 .collect::<Vec<_>>(),
755 )
756 .unwrap_or_default()
757}
758
759#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
761pub struct NoticeCause {
762 pub stage: String,
764 #[serde(default, skip_serializing_if = "Option::is_none")]
767 pub target: Option<String>,
768 pub run: Option<String>,
770 pub pass: Option<usize>,
772}
773
774pub(crate) fn is_done_notice_key(key: &str) -> bool {
776 key == "done"
777 || key
778 .strip_prefix("done#")
779 .is_some_and(|count| !count.is_empty() && count.bytes().all(|b| b.is_ascii_digit()))
780}
781
782#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
784#[serde(rename_all = "snake_case")]
785pub enum PassCost {
786 Spent,
788 Refunded,
790}
791
792#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
794pub struct TaskPass {
795 pub run: String,
797 pub pass_index: usize,
799 pub cost: PassCost,
801 pub attempts_after: usize,
803 #[serde(default)]
805 pub started_at: Option<Timestamp>,
806 pub settled_at: Timestamp,
808 #[serde(default)]
810 pub parked: Option<crate::run::ParkCause>,
811}
812
813#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
815#[serde(rename_all = "lowercase")]
816pub enum ChatNoticeOutcome {
817 Posted,
819 Skipped,
821}
822
823#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
825pub struct ChatNotice {
826 pub key: String,
828 pub at: Timestamp,
830 pub outcome: ChatNoticeOutcome,
832}
833
834#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
836pub struct RunOverrides {
837 #[serde(default)]
840 pub inherit_daemon_options: bool,
841 #[serde(default)]
843 pub merge: Option<String>,
844 #[serde(default)]
846 pub candidates: Option<usize>,
847 #[serde(default)]
849 pub judges: Option<usize>,
850 #[serde(default)]
852 pub reviewers: Option<usize>,
853 #[serde(default)]
855 pub review_rounds: Option<usize>,
856 #[serde(default)]
858 pub seed: Option<u64>,
859 #[serde(default)]
861 pub config: Option<PathBuf>,
862}
863
864impl RunOverrides {
865 pub fn merge_over(&mut self, other: &RunOverrides) {
867 macro_rules! take {
868 ($($f:ident),*) => {$(
869 if other.$f.is_some() {
870 self.$f.clone_from(&other.$f);
871 }
872 )*};
873 }
874 take!(
875 merge,
876 candidates,
877 judges,
878 reviewers,
879 review_rounds,
880 seed,
881 config
882 );
883 }
884
885 pub fn apply(&self, config: &mut crate::config::Config) {
887 if let Some(n) = self.candidates {
888 config.graph.implementers = n;
889 }
890 if let Some(n) = self.judges {
891 config.graph.judges = n;
892 }
893 if let Some(n) = self.reviewers {
894 config.graph.reviewers = n;
895 }
896 if let Some(n) = self.review_rounds {
897 config.graph.review_rounds = n;
898 }
899 if let Some(m) = &self.merge
900 && let Ok(mode) = crate::daemon::merge_mode(m)
901 {
902 config.merge.mode = mode;
903 }
904 if let Some(s) = self.seed {
905 config.blind.seed = Some(s);
906 }
907 }
908}
909
910#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
912pub struct FollowUp {
913 pub run: String,
915 #[serde(default)]
917 pub origin_task: Option<String>,
918 pub pr: String,
920 pub findings: Vec<String>,
922 pub generation: u32,
925}
926
927#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
929pub struct Resumption {
930 pub generation: u64,
932 pub source: String,
934 pub question_id: Option<String>,
936 pub at: Timestamp,
938 pub instruction: String,
940 pub answers_through: usize,
942 pub runs_through: usize,
945 pub cleared_reason: Option<String>,
947 #[serde(default)]
949 pub blocking_evidence: Vec<String>,
950}
951
952#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
955pub struct OperatorResume {
956 pub question_id: String,
958 pub at: Timestamp,
960 #[serde(default)]
963 pub conductor_rehold: Option<String>,
964 #[serde(default)]
967 pub forced: bool,
968 #[serde(default)]
975 pub pinned_run: Option<String>,
976}
977
978#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
981pub struct AnsweredQuestion {
982 pub question: String,
984 pub answer: String,
986}
987
988impl Task {
989 pub fn new(title: String, instruction: String, repo: PathBuf, source: Source) -> Self {
991 let now = Timestamp::now();
992 let origin_chat = match &source {
993 Source::Agent { run, node } if node == CHAT_NODE => Some(run.clone()),
994 _ => None,
995 };
996 Self {
997 schema: SCHEMA,
998 id: new_id(),
999 title,
1000 instruction,
1001 sources: crate::refs::Sources::default(),
1002 repo,
1003 reassignments: Vec::new(),
1004 source,
1005 chat: None,
1006 priority: 0,
1007 solo: false,
1008 status: TaskStatus::Queued,
1009 attempts: 0,
1010 attempt_limit: None,
1011 runs: Vec::new(),
1012 startup_failures: Vec::new(),
1013 last_error: None,
1014 hold_reason: None,
1015 hold_source: None,
1016 machine_hold_cause: None,
1017 disk_resume_run: None,
1018 disk_checked_at: None,
1019 diagnostic: None,
1020 blocked_by: Vec::new(),
1021 block_reason: None,
1022 blocked_from: None,
1023 blocked_at: None,
1024 answers: Vec::new(),
1025 triage_applied: Vec::new(),
1026 actions_applied: Vec::new(),
1027 decision_effects: Default::default(),
1028 superseded_resumes: Default::default(),
1029 dispatched_resumes: Default::default(),
1030 base_recovery_requests: Vec::new(),
1031 resume_override: None,
1032 repair_grants: Vec::new(),
1033 repair_reconciliations: Vec::new(),
1034 resumption: None,
1035 resumption_history: Vec::new(),
1036 operator_decided_at: None,
1037 review_branch: None,
1038 review_instruction: None,
1039 pr_recovery: None,
1040 fresh_start: false,
1041 replacement: None,
1042 interrupt: false,
1043 urgent: false,
1044 unfiled: true,
1045 attachments: Vec::new(),
1046 followup: None,
1047 origin_chat,
1048 overrides: None,
1049 review_of: None,
1050 held_at: None,
1051 park_reason: None,
1052 chat_notice: None,
1053 chat_notices: Vec::new(),
1054 hold_notice_cause: None,
1055 passes: Vec::new(),
1056 dispatched_at: None,
1057 recovery_budget: None,
1058 recovery_admission: None,
1059 recovery_generations: Vec::new(),
1060 created_at: now,
1061 updated_at: now,
1062 }
1063 }
1064
1065 pub(crate) fn reassign_repository(&mut self, repo: PathBuf, actor: &str) {
1067 self.stamp_schema();
1068 if matches!(self.status, TaskStatus::Failed | TaskStatus::Parked) {
1069 self.hold_manual(Some(
1070 "Repository reassigned; inspect history and release to start fresh.".into(),
1071 ));
1072 }
1073 let at = Timestamp::now();
1074 self.reassignments.push(Reassignment {
1075 from: self.repo.clone(),
1076 to: repo.clone(),
1077 actor: actor.to_owned(),
1078 at,
1079 run_boundary: self.runs.len(),
1080 });
1081 self.repo = repo;
1082 self.review_branch = None;
1083 self.review_instruction = None;
1084 self.pr_recovery = None;
1085 self.replacement = None;
1086 self.interrupt = false;
1087 self.resume_override = None;
1088 self.fresh_start = true;
1089 self.operator_decided_at = Some(at);
1090 }
1091
1092 pub fn publication_branch(&self) -> String {
1094 crate::run::task_pr_branch_name(&self.id, self.reassignments.len())
1095 }
1096
1097 pub fn execution_runs(&self) -> &[String] {
1099 let boundary = self.reassignments.last().map_or(0, |r| r.run_boundary);
1100 self.runs.get(boundary..).unwrap_or_default()
1101 }
1102
1103 pub fn execution_run(&self, run: &str) -> bool {
1105 self.execution_runs().iter().any(|id| id == run)
1106 }
1107
1108 pub(crate) fn assignment_accepts(&self, run: &crate::run::RunState) -> bool {
1110 self.reassignments.last().is_none_or(|boundary| {
1111 self.execution_run(&run.id)
1112 && run.task_assignment == self.reassignments.len()
1113 && run.created_at > boundary.at
1114 && run
1115 .repo
1116 .canonicalize()
1117 .ok()
1118 .zip(self.repo.canonicalize().ok())
1119 .is_some_and(|(run_repo, task_repo)| run_repo == task_repo)
1120 })
1121 }
1122
1123 pub(crate) fn stale_question(&self, q: &crate::ask::Question) -> bool {
1125 self.reassignments
1126 .last()
1127 .is_some_and(|r| q.asked_at <= r.at)
1128 || (self.runs.contains(&q.run) && !self.execution_run(&q.run))
1129 }
1130
1131 pub fn chat_talk(&self) -> Option<&str> {
1133 if let Some(id) = &self.origin_chat {
1134 return Some(id);
1135 }
1136 match &self.source {
1137 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
1138 _ => None,
1139 }
1140 }
1141
1142 pub fn filed_by_chat(&self) -> Option<&str> {
1146 match &self.source {
1147 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
1148 _ => None,
1149 }
1150 }
1151
1152 pub fn chat_notice_key(&self) -> Option<String> {
1155 match self.status {
1156 TaskStatus::Done => Some("done".to_owned()),
1157 TaskStatus::Held => {
1158 let cause = self
1159 .hold_notice_cause
1160 .clone()
1161 .unwrap_or_else(|| self.unresolved_hold_cause());
1162 Some(format!(
1163 "cause:{}",
1164 serde_json::to_string(&cause).expect("cause")
1165 ))
1166 }
1167 TaskStatus::Blocked => {
1168 let ids: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1169 let evidence = self
1172 .hold_notice_cause
1173 .as_ref()
1174 .filter(|c| c.stage.starts_with("conductor-evidence:"));
1175 Some(format!(
1176 "dependencies:{}",
1177 serde_json::to_string(&(ids, evidence)).expect("ids")
1178 ))
1179 }
1180 _ => None,
1181 }
1182 }
1183
1184 fn unresolved_hold_cause(&self) -> NoticeCause {
1185 NoticeCause {
1186 stage: "hold".into(),
1187 target: None,
1188 run: self.runs.last().cloned(),
1189 pass: None,
1190 }
1191 }
1192
1193 pub(crate) fn legacy_chat_notice_key(&self) -> Option<String> {
1195 let key = match self.status {
1196 TaskStatus::Held if self.hold_notice_cause.is_none() => Some(format!(
1197 "held#{}",
1198 self.held_at.map(|t| t.to_string()).unwrap_or_default()
1199 )),
1200 TaskStatus::Blocked => Some(format!(
1201 "blocked#{}#{}",
1202 self.blocked_by.join(","),
1203 self.block_reason.as_deref().unwrap_or_default()
1204 )),
1205 _ => None,
1206 };
1207 key.filter(|key| self.legacy_notice_matches(key))
1208 }
1209
1210 pub(crate) fn legacy_notice_matches(&self, key: &str) -> bool {
1211 match self.status {
1212 TaskStatus::Held if self.hold_notice_cause.is_none() => key
1213 .strip_prefix("held#")
1214 .is_some_and(|time| time.is_empty() || time.parse::<Timestamp>().is_ok()),
1215 TaskStatus::Blocked
1216 if self
1217 .hold_notice_cause
1218 .as_ref()
1219 .is_none_or(|c| !c.stage.starts_with("conductor-evidence:")) =>
1220 {
1221 key.strip_prefix("blocked#").is_some_and(|tail| {
1222 let Some((ids, _)) = tail.split_once('#') else {
1223 return false;
1224 };
1225 let old: std::collections::BTreeSet<_> =
1226 ids.split(',').filter(|id| !id.is_empty()).collect();
1227 let current: std::collections::BTreeSet<_> =
1228 self.blocked_by.iter().map(String::as_str).collect();
1229 old == current
1230 })
1231 }
1232 _ => false,
1233 }
1234 }
1235
1236 fn import_chat_notice(&mut self) {
1237 if let Some(n) = &self.chat_notice {
1238 let key = if is_done_notice_key(&n.key) {
1239 "done".to_owned()
1240 } else if self.legacy_notice_matches(&n.key) {
1241 self.chat_notice_key().unwrap_or_else(|| n.key.clone())
1242 } else {
1243 n.key.clone()
1244 };
1245 if !self.chat_notices.contains(&key) {
1246 self.chat_notices.push(key);
1247 }
1248 }
1249 }
1250
1251 pub(crate) fn notice_cause(&mut self, stage: &str, run: Option<&str>, pass: Option<usize>) {
1254 self.import_chat_notice();
1255 self.hold_notice_cause = Some(NoticeCause {
1256 stage: stage.to_owned(),
1257 target: None,
1258 run: run.map(str::to_owned),
1259 pass,
1260 });
1261 }
1262
1263 pub(crate) fn startup_notice_cause(&mut self, stage: &str) {
1266 let recovery_only = matches!(
1267 stage,
1268 "startup-source" | "startup-candidates" | "startup-public-head"
1269 );
1270 match self.hold_notice_cause.as_ref() {
1271 Some(cause) if recovery_only && !cause.stage.starts_with("startup-") => return,
1272 None if recovery_only
1273 && self
1274 .chat_notice
1275 .as_ref()
1276 .is_some_and(|n| !is_done_notice_key(&n.key)) =>
1277 {
1278 return;
1279 }
1280 _ => {}
1281 }
1282 self.notice_cause(stage, None, None);
1283 }
1284
1285 pub(crate) fn fail_startup(
1288 &mut self,
1289 stage: &str,
1290 why: impl Into<String>,
1291 max_attempts: usize,
1292 ) {
1293 let unresolved = self.hold_notice_cause.clone();
1294 self.fail(why, max_attempts);
1295 self.hold_notice_cause = unresolved;
1296 self.startup_notice_cause(stage);
1297 }
1298
1299 pub(crate) fn fail_startup_target(
1301 &mut self,
1302 stage: &str,
1303 target: String,
1304 why: impl Into<String>,
1305 max_attempts: usize,
1306 ) {
1307 self.fail_startup(stage, why, max_attempts);
1308 if let Some(cause) = self.hold_notice_cause.as_mut() {
1309 cause.target = Some(target);
1310 }
1311 }
1312
1313 pub fn chat_notice_due(&self) -> Option<String> {
1315 self.filed_by_chat()?;
1316 if self.schema < 13 {
1317 return None;
1318 }
1319 let key = self.chat_notice_key()?;
1320 if self.chat_notices.contains(&key) {
1321 return None;
1322 }
1323 match &self.chat_notice {
1324 Some(n)
1325 if n.key == key
1326 || (key == "done" && is_done_notice_key(&n.key))
1327 || self.legacy_notice_matches(&n.key) =>
1328 {
1329 None
1330 }
1331 _ => Some(key),
1332 }
1333 }
1334
1335 pub fn short(&self) -> &str {
1337 short(&self.id)
1338 }
1339
1340 pub fn mark_triage_applied(&mut self, question_id: &str) {
1342 if !self.triage_applied(question_id) {
1343 self.triage_applied.push(question_id.to_owned());
1344 }
1345 }
1346
1347 pub fn action_applied(&self, question_id: &str) -> bool {
1350 self.actions_applied.iter().any(|id| id == question_id)
1351 }
1352
1353 pub fn mark_action_applied(&mut self, question_id: &str) {
1355 if !self.action_applied(question_id) {
1356 self.actions_applied.push(question_id.to_owned());
1357 }
1358 }
1359
1360 pub fn triage_applied(&self, question_id: &str) -> bool {
1362 self.triage_applied.iter().any(|id| id == question_id)
1363 }
1364
1365 pub fn start(&mut self, run: String) {
1378 self.stamp_schema();
1379 self.status = TaskStatus::Running;
1380 self.attempts += 1;
1381 self.runs.retain(|id| id != &run);
1382 if let Some(pin) = &self.resume_override
1383 && pin.pinned_run.as_ref() == Some(&run)
1384 && !pin.question_id.is_empty()
1385 {
1386 self.dispatched_resumes
1387 .insert(pin.question_id.clone(), run.clone());
1388 }
1389 self.runs.push(run);
1390 self.last_error = None;
1391 self.park_reason = None;
1392 self.fresh_start = false;
1393 self.interrupt = false;
1394 self.resume_override = None;
1396 self.disk_resume_run = None;
1399 }
1400
1401 pub fn record_startup_failure(&mut self, failure: crate::run::StartupFailure) {
1403 if !self.startup_failures.iter().any(|f| f.run == failure.run) {
1404 self.startup_failures.push(failure);
1405 }
1406 }
1407
1408 pub fn link_run(&mut self, run: &str) -> bool {
1413 if self.runs.iter().any(|r| r == run) {
1414 return false;
1415 }
1416 self.runs.push(run.to_owned());
1417 true
1418 }
1419
1420 pub fn succeed(&mut self) {
1430 self.stamp_schema();
1431 self.status = TaskStatus::Done;
1432 self.resume_override = None;
1433 self.last_error = None;
1434 self.park_reason = None;
1435 self.hold_reason = None;
1436 self.hold_source = None;
1437 self.machine_hold_cause = None;
1438 self.disk_resume_run = None;
1439 self.disk_checked_at = None;
1440 self.diagnostic = None;
1441 self.blocked_by.clear();
1442 self.block_reason = None;
1443 self.blocked_from = None;
1444 }
1445
1446 pub fn already_landed(&mut self, note: impl Into<String>) {
1454 self.succeed();
1455 self.attempts = self.attempts.saturating_sub(1);
1456 self.last_error = Some(note.into());
1457 }
1458
1459 pub fn superseded_attempts(&self, last_run_succeeded: bool) -> &[String] {
1489 if self.status != TaskStatus::Done || !last_run_succeeded || self.execution_runs().len() < 2
1490 {
1491 return &[];
1492 }
1493 &self.execution_runs()[..self.execution_runs().len() - 1]
1494 }
1495
1496 pub fn successor_of(&self, run: &str) -> Option<&String> {
1504 let runs = self.execution_runs();
1505 let pos = runs.iter().rposition(|r| r == run)?;
1506 runs.get(pos + 1)
1507 }
1508
1509 pub fn earlier_attempts(&self) -> &[String] {
1516 self.execution_runs()
1517 }
1518
1519 fn stamp_schema(&mut self) {
1526 self.schema = self.schema.max(SCHEMA);
1527 }
1528
1529 fn note_held(&mut self) {
1530 self.stamp_schema();
1531 let continuing = self.status == TaskStatus::Held
1532 || (self.status == TaskStatus::Blocked
1533 && (self.blocked_from == Some(TaskStatus::Held)
1534 || (self.blocked_from.is_none()
1535 && (self.hold_reason.is_some() || self.hold_source.is_some()))));
1536 if !continuing || self.held_at.is_none() {
1537 self.held_at = Some(Timestamp::now());
1538 }
1539 }
1540
1541 pub fn fail(&mut self, why: impl Into<String>, max_attempts: usize) {
1557 let why = why.into();
1558 if self.status == TaskStatus::Running {
1559 let run = self.runs.last().cloned();
1560 self.notice_cause("run-failed", run.as_deref(), Some(self.attempts));
1561 }
1562 self.diagnostic = None;
1563 self.park_reason = None;
1564 self.status = if self.attempts >= self.effective_attempt_limit(max_attempts) {
1565 self.note_held();
1566 self.hold_source = Some(HoldSource::Machine);
1567 self.machine_hold_cause = None;
1568 self.disk_resume_run = None;
1569 self.disk_checked_at = None;
1570 self.hold_reason = Some(why.clone());
1571 TaskStatus::Held
1572 } else {
1573 TaskStatus::Failed
1574 };
1575 self.last_error = Some(why);
1576 }
1577
1578 pub fn stall(&mut self, why: impl Into<String>) {
1588 self.last_error = Some(why.into());
1589 self.diagnostic = None;
1590 self.attempts = self.attempts.saturating_sub(1);
1591 self.status = TaskStatus::Failed;
1592 }
1593
1594 pub fn park(&mut self, why: impl Into<String>) {
1600 self.diagnostic = None;
1601 self.attempts = self.attempts.saturating_sub(1);
1602 self.status = TaskStatus::Parked;
1603 self.park_reason = Some(why.into());
1604 self.schema = self.schema.max(SCHEMA);
1607 }
1608
1609 pub fn record_pass(
1611 &mut self,
1612 run: &str,
1613 pass_index: usize,
1614 attempts_before: usize,
1615 parked: Option<crate::run::ParkCause>,
1616 ) {
1617 self.record_pass_at(run, pass_index, None, attempts_before, parked);
1618 }
1619
1620 pub fn record_pass_at(
1623 &mut self,
1624 run: &str,
1625 pass_index: usize,
1626 started_at: Option<Timestamp>,
1627 attempts_before: usize,
1628 parked: Option<crate::run::ParkCause>,
1629 ) {
1630 if self
1631 .passes
1632 .iter()
1633 .any(|p| p.run == run && p.pass_index == pass_index)
1634 {
1635 return;
1636 }
1637 let cost = if self.attempts < attempts_before {
1638 PassCost::Refunded
1639 } else {
1640 PassCost::Spent
1641 };
1642 self.passes.push(TaskPass {
1643 run: run.to_owned(),
1644 pass_index,
1645 started_at: self.dispatched_at.or(started_at),
1646 cost,
1647 attempts_after: self.attempts,
1648 settled_at: Timestamp::now(),
1649 parked,
1650 });
1651 self.schema = self.schema.max(SCHEMA);
1652 }
1653
1654 pub(crate) fn upgrade_resume_key(&self) -> Option<(Timestamp, &str)> {
1657 if self.status != TaskStatus::Parked || self.fresh_start {
1658 return None;
1659 }
1660 let pass = self.passes.last()?;
1661 if self.execution_runs().last() != Some(&pass.run)
1662 || !matches!(pass.parked, Some(crate::run::ParkCause::Upgrade { .. }))
1663 {
1664 return None;
1665 }
1666 Some((pass.started_at.unwrap_or(pass.settled_at), &self.id))
1669 }
1670
1671 pub fn operator_held(&self) -> bool {
1677 self.status == TaskStatus::Held && !matches!(self.hold_source, Some(HoldSource::Machine))
1678 }
1679
1680 pub fn hold_manual(&mut self, reason: Option<String>) {
1690 self.operator_decided_at = Some(Timestamp::now());
1692 self.hold_unrequested_retry(reason);
1693 }
1694
1695 pub(crate) fn hold_unrequested_retry(&mut self, reason: Option<String>) {
1698 self.park_reason = None;
1699 self.note_held();
1700 self.status = TaskStatus::Held;
1701 if reason.is_some() {
1702 self.hold_reason = reason;
1703 }
1704 self.hold_source = Some(HoldSource::Manual);
1705 self.machine_hold_cause = None;
1706 self.disk_resume_run = None;
1707 self.disk_checked_at = None;
1708 self.blocked_by.clear();
1709 self.block_reason = None;
1710 self.blocked_from = None;
1711 }
1712
1713 pub fn hold_machine(&mut self, reason: Option<String>) {
1718 self.park_reason = None;
1719 self.note_held();
1720 self.status = TaskStatus::Held;
1721 if reason.is_some() {
1722 self.hold_reason = reason;
1723 }
1724 self.hold_source = Some(HoldSource::Machine);
1725 self.machine_hold_cause = None;
1726 self.disk_resume_run = None;
1727 self.disk_checked_at = None;
1728 self.blocked_by.clear();
1729 self.block_reason = None;
1730 self.blocked_from = None;
1731 }
1732
1733 pub(crate) fn mark_disk_capacity(&mut self) {
1735 self.machine_hold_cause = Some(MachineHoldCause::DiskCapacity {
1736 reason: self.hold_reason.clone().unwrap_or_default(),
1737 });
1738 }
1739
1740 pub fn has_disk_capacity_cause(&self) -> bool {
1742 matches!(
1743 (&self.machine_hold_cause, self.hold_reason.as_deref()),
1744 (Some(MachineHoldCause::DiskCapacity { reason }), Some(now)) if reason == now
1745 )
1746 }
1747
1748 pub(crate) fn resume_disk_wait(&mut self) {
1750 let retained = self.execution_runs().last().cloned();
1751 self.status = TaskStatus::Queued;
1752 self.held_at = None;
1753 self.hold_source = None;
1754 self.hold_reason = None;
1755 self.machine_hold_cause = None;
1756 self.last_error = None;
1757 self.disk_resume_run = retained;
1758 }
1759
1760 pub fn block(&mut self, blocked_by: Vec<String>, reason: Option<String>) {
1770 self.stamp_schema();
1771 let before: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1772 let after: std::collections::BTreeSet<_> = blocked_by.iter().collect();
1773 if self.status != TaskStatus::Blocked || before != after {
1774 self.blocked_at = Some(Timestamp::now());
1775 }
1776 if self.status != TaskStatus::Blocked {
1777 self.blocked_from = Some(self.status);
1778 }
1779 self.status = TaskStatus::Blocked;
1780 self.blocked_by = blocked_by;
1781 self.block_reason = reason;
1782 }
1783
1784 pub fn unblock(&mut self, resolved_id: &str) {
1807 if self.status != TaskStatus::Blocked {
1808 return;
1809 }
1810 self.blocked_by.retain(|id| id != resolved_id);
1811 self.restore_if_unblocked();
1812 }
1813
1814 pub fn dependency_deleted(&mut self, deleted_id: &str) -> bool {
1825 if self.status != TaskStatus::Blocked || !self.blocked_by.iter().any(|b| b == deleted_id) {
1826 return false;
1827 }
1828 self.blocked_by.retain(|id| id != deleted_id);
1829 self.restore_if_unblocked();
1830 true
1831 }
1832
1833 fn restore_if_unblocked(&mut self) {
1837 if self.blocked_by.is_empty() {
1838 self.status = match self.blocked_from {
1839 Some(TaskStatus::Running) => TaskStatus::Queued,
1840 Some(other) => other,
1841 None if self.hold_reason.is_some() || self.hold_source.is_some() => {
1842 TaskStatus::Held
1843 }
1844 None => TaskStatus::Queued,
1845 };
1846 self.block_reason = None;
1847 self.blocked_from = None;
1848 }
1849 }
1850
1851 pub fn record_answer(&mut self, question: String, answer: String) {
1856 self.answers.push(AnsweredQuestion { question, answer });
1857 }
1858
1859 pub fn request_review(&mut self, branch: String) {
1863 self.release();
1864 self.review_branch = Some(branch);
1865 self.review_instruction = Some(self.instruction.clone());
1866 }
1867
1868 pub fn requeue(&mut self) {
1871 self.release();
1872 self.review_branch = None;
1874 self.review_instruction = None;
1875 self.fresh_start = true;
1876 }
1877
1878 pub fn hold_for_handover(&mut self, branch: Option<String>, reason: String) {
1887 if branch.is_some() {
1888 self.review_branch = branch;
1889 self.review_instruction = Some(self.instruction.clone());
1890 }
1891 self.hold_machine(Some(reason));
1892 }
1893
1894 pub fn set_urgent(&mut self, urgent: bool) -> Result<()> {
1896 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1897 bail!(
1898 "task {} is {}; urgent can only change while queued or held; \
1899 wait for the active attempt to finish and requeue or hold the task first",
1900 self.short(),
1901 self.status.as_str()
1902 );
1903 }
1904 self.urgent = urgent;
1905 Ok(())
1906 }
1907
1908 pub fn set_priority(&mut self, priority: i32) -> Result<()> {
1917 if self.status == TaskStatus::Running {
1918 bail!(
1919 "task {} is running; its priority cannot be changed until \
1920 this attempt finishes",
1921 self.short()
1922 );
1923 }
1924 self.priority = priority;
1925 Ok(())
1926 }
1927
1928 pub fn set_interrupt(&mut self, interrupt: bool) -> Result<()> {
1943 if interrupt && !self.status.runnable() {
1944 bail!(
1945 "task {} is {}; only a queued or failed task can be marked \
1946 to interrupt",
1947 self.short(),
1948 self.status.as_str()
1949 );
1950 }
1951 self.interrupt = interrupt;
1952 Ok(())
1953 }
1954
1955 pub fn effective_attempt_limit(&self, daemon_limit: usize) -> usize {
1957 self.attempt_limit.map_or(daemon_limit, NonZeroUsize::get)
1958 }
1959
1960 pub fn change_attempt_limit(&mut self, limit: Option<NonZeroUsize>) -> Result<()> {
1963 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1964 bail!("only queued or held tasks can change the attempt limit");
1965 }
1966 self.stamp_schema();
1967 self.attempt_limit = limit;
1968 Ok(())
1969 }
1970
1971 pub fn implementers(&self) -> Option<usize> {
1973 if self.solo {
1974 Some(1)
1975 } else {
1976 self.overrides.as_ref().and_then(|o| o.candidates)
1977 }
1978 }
1979
1980 pub fn change_implementers(&mut self, count: u8) -> Result<()> {
1983 self.check_implementers_editable()?;
1984 let explicit = self.overrides.as_ref().and_then(|o| o.config.as_deref());
1985 let (mut config, _) = crate::config::Config::discover(&self.repo, explicit)?;
1986 if let Some(o) = &self.overrides {
1987 o.apply(&mut config);
1988 }
1989 self.set_implementers(count, &config)
1990 }
1991
1992 fn check_implementers_editable(&self) -> Result<()> {
1993 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1994 bail!(
1995 "task {} is {}; only queued or held tasks can change implementers",
1996 self.short(),
1997 self.status.as_str()
1998 );
1999 }
2000 Ok(())
2001 }
2002
2003 pub fn set_implementers(&mut self, count: u8, config: &crate::config::Config) -> Result<()> {
2005 self.check_implementers_editable()?;
2006 crate::talk::check_implementers(count, config).map_err(anyhow::Error::msg)?;
2007 self.stamp_schema();
2008 self.solo = count == 1;
2009 let mut overrides = self.overrides.clone().unwrap_or_else(|| RunOverrides {
2010 inherit_daemon_options: true,
2011 ..Default::default()
2012 });
2013 overrides.candidates = (count > 1).then_some(usize::from(count));
2014 let mut settings = overrides.clone();
2015 settings.inherit_daemon_options = false;
2016 self.overrides = if overrides.inherit_daemon_options && settings == RunOverrides::default()
2018 {
2019 None
2020 } else {
2021 Some(overrides)
2022 };
2023 Ok(())
2024 }
2025
2026 pub fn edit(&mut self, title: String, instruction: String) -> Result<()> {
2038 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
2039 bail!(
2040 "task {} is {}; only a queued or held task's instruction can \
2041 be edited",
2042 self.short(),
2043 self.status.as_str()
2044 );
2045 }
2046 if self.review_branch.is_some() && self.review_instruction.is_none() {
2049 self.review_instruction = Some(self.instruction.clone());
2050 }
2051 if self.instruction != instruction {
2052 self.resume_override = None;
2055 }
2056 if self.status == TaskStatus::Held && self.machine_hold_cause.is_some() {
2057 self.hold_manual(self.hold_reason.clone());
2058 }
2059 self.machine_hold_cause = None;
2060 self.disk_resume_run = None;
2061 self.disk_checked_at = None;
2062 self.title = title;
2063 self.instruction = instruction;
2064 Ok(())
2065 }
2066
2067 pub fn handed_off(&mut self, why: impl Into<String>) {
2087 let why = why.into();
2088 self.diagnostic = None;
2089 self.note_held();
2090 self.status = TaskStatus::Held;
2091 self.hold_source = Some(HoldSource::Machine);
2092 self.machine_hold_cause = None;
2093 self.disk_resume_run = None;
2094 self.disk_checked_at = None;
2095 self.hold_reason = Some(why.clone());
2096 self.last_error = Some(why);
2097 }
2098
2099 pub fn release_operator(&mut self, source: &str, question_id: Option<&str>) {
2101 self.recovery_budget = None;
2102 self.recovery_generations.clear();
2103 self.recovery_admission = None;
2104 if self
2105 .resumption
2106 .as_ref()
2107 .is_some_and(|r| r.instruction != self.instruction)
2108 {
2109 self.resume_override = None;
2112 }
2113 let approval = Resumption {
2114 generation: self.resumption.as_ref().map_or(1, |r| r.generation + 1),
2115 source: source.to_owned(),
2116 question_id: question_id.map(str::to_owned),
2117 at: Timestamp::now(),
2118 instruction: self.instruction.clone(),
2119 answers_through: self.answers.len(),
2120 runs_through: self.runs.len(),
2121 blocking_evidence: Vec::new(),
2122 cleared_reason: self
2123 .hold_reason
2124 .clone()
2125 .or_else(|| self.block_reason.clone()),
2126 };
2127 self.release();
2128 self.operator_decided_at = Some(approval.at);
2129 if let Some(previous) = self.resumption.replace(approval) {
2130 self.resumption_history.push(previous);
2131 }
2132 }
2133
2134 pub(crate) fn recovery_failed(&mut self, state: &crate::run::RunState) {
2136 let (base, inputs) = self.recovery_admission.as_ref().map_or_else(
2137 || (state.base_commit.clone(), recovery_inputs(&state.sources)),
2138 |a| (a.base.clone(), a.inputs.clone()),
2139 );
2140 let progress = recovery_progress(state);
2141 let stage = if state.prep_failure.is_some() {
2142 "preparation"
2143 } else {
2144 "execution"
2145 };
2146 let matches = |b: &RecoveryBudget| {
2147 b.instruction == self.instruction
2148 && b.base == base
2149 && b.inputs == inputs
2150 && (progress.is_none() || b.progress == progress)
2151 };
2152 if !self.recovery_budget.as_ref().is_some_and(matches) {
2153 let previous = self
2154 .recovery_generations
2155 .iter()
2156 .position(matches)
2157 .map(|index| self.recovery_generations.remove(index));
2158 if let Some(old) = self.recovery_budget.take() {
2159 self.recovery_generations.push(old);
2160 }
2161 self.recovery_budget = previous;
2162 }
2163 if let Some(budget) = &mut self.recovery_budget {
2164 budget.failures = budget.failures.saturating_add(1);
2165 budget.stage = stage.to_owned();
2166 } else {
2167 self.recovery_budget = Some(RecoveryBudget {
2168 instruction: self.instruction.clone(),
2169 base,
2170 inputs,
2171 stage: stage.to_owned(),
2172 failures: 1,
2173 progress,
2174 });
2175 }
2176 }
2177
2178 pub(crate) fn recovery_exhausted(
2179 &self,
2180 base: &str,
2181 inputs: &str,
2182 limit: usize,
2183 ) -> Option<String> {
2184 let same_input = |b: &&RecoveryBudget| {
2185 b.instruction == self.instruction && b.base == base && b.inputs == inputs
2186 };
2187 let budget = self
2188 .recovery_budget
2189 .as_ref()
2190 .filter(same_input)
2191 .or_else(|| self.recovery_generations.iter().rev().find(same_input))?;
2192 (budget.instruction == self.instruction && budget.base == base && budget.inputs == inputs
2193 && budget.failures >= limit.max(1)).then(|| format!(
2194 "[recovery-budget] {} failed passes against unchanged inputs ({}). Inspect the retained recovery sources and correct the cause, change the inputs, or explicitly release the task to approve another generation. Automatic requeue does not reset this budget.",
2195 budget.failures, budget.stage))
2196 }
2197
2198 pub fn release(&mut self) {
2202 self.import_chat_notice();
2205 if self.hold_notice_cause.is_none()
2206 && matches!(self.status, TaskStatus::Held | TaskStatus::Blocked)
2207 {
2208 self.hold_notice_cause = Some(self.unresolved_hold_cause());
2209 }
2210 let refused_handover = self.status == TaskStatus::Held
2211 && self.hold_source == Some(HoldSource::Machine)
2212 && self.review_branch.is_some()
2213 && self
2214 .review_instruction
2215 .as_ref()
2216 .is_none_or(|i| *i == self.instruction);
2217 self.status = TaskStatus::Queued;
2218 self.held_at = None;
2219 self.attempts = 0;
2220 self.last_error = None;
2221 self.park_reason = None;
2222 self.hold_reason = None;
2225 self.hold_source = None;
2226 self.machine_hold_cause = None;
2227 self.disk_resume_run = None;
2228 self.disk_checked_at = None;
2229 self.diagnostic = None;
2230 self.blocked_by.clear();
2235 self.block_reason = None;
2236 self.blocked_from = None;
2237 if !refused_handover {
2241 self.review_branch = None;
2242 self.review_instruction = None;
2243 }
2244 self.fresh_start = false;
2245 }
2246}
2247
2248fn copy_new(dir: &Path, src: &Path, name: &str) -> Result<(String, PathBuf)> {
2252 use std::io::ErrorKind;
2253 let (stem, ext) = match name.rfind('.') {
2254 Some(i) if i > 0 => (&name[..i], &name[i..]),
2255 _ => (name, ""),
2256 };
2257 for n in 1u32.. {
2258 let candidate = if n == 1 {
2259 name.to_owned()
2260 } else {
2261 let suffix = format!("-{n}");
2262 let room = 64usize.saturating_sub(suffix.len() + ext.len());
2263 let stem: String = stem.chars().take(room).collect();
2264 format!("{stem}{suffix}{ext}")
2265 };
2266 if !crate::ask::valid_asset_name(&candidate) {
2267 bail!("no valid attachment name is left for `{name}`");
2268 }
2269 let path = dir.join(&candidate);
2270 match std::fs::OpenOptions::new()
2271 .write(true)
2272 .create_new(true)
2273 .open(&path)
2274 {
2275 Ok(mut out) => {
2276 let copied = std::fs::File::open(src)
2277 .and_then(|mut input| std::io::copy(&mut input, &mut out));
2278 if let Err(e) = copied {
2279 drop(out);
2280 let _ = std::fs::remove_file(&path);
2281 return Err(e).with_context(|| format!("copy {}", src.display()));
2282 }
2283 return Ok((candidate, path));
2284 }
2285 Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
2286 Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
2287 }
2288 }
2289 unreachable!("the counter never runs out")
2290}
2291
2292const TASK_LOCK_STALE: std::time::Duration = std::time::Duration::from_secs(10);
2295
2296pub(crate) struct TaskLock {
2299 path: PathBuf,
2300 token: String,
2301}
2302
2303fn owner_token() -> String {
2306 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
2307 let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2308 let mut r = crate::rng::SplitMix64::new(crate::rng::entropy() ^ n.rotate_left(32));
2309 format!("{}-{:016x}", std::process::id(), r.next_u64())
2310}
2311
2312fn break_marker(path: &Path, token: &str) -> PathBuf {
2315 let mut h: u64 = 0xcbf29ce484222325;
2316 for b in token.bytes() {
2317 h ^= u64::from(b);
2318 h = h.wrapping_mul(0x100000001b3);
2319 }
2320 let mut name = path.as_os_str().to_owned();
2321 name.push(format!(".break-{h:016x}"));
2322 PathBuf::from(name)
2323}
2324
2325fn older_than_stale(path: &Path) -> bool {
2326 std::fs::metadata(path)
2327 .and_then(|m| m.modified())
2328 .ok()
2329 .and_then(|t| t.elapsed().ok())
2330 .is_some_and(|age| age > TASK_LOCK_STALE)
2331}
2332
2333fn writer_may_be_gone(token: &str) -> bool {
2334 let Some(pid) = token
2335 .split_once('-')
2336 .and_then(|(pid, _)| pid.parse::<u32>().ok())
2337 else {
2338 return true;
2341 };
2342 if pid == std::process::id() {
2343 return false;
2344 }
2345 crate::proc::pid_status(pid) == Some(false)
2347}
2348
2349const MAX_MARKER_DEPTH: u8 = 3;
2353
2354fn take_marker(
2360 path: &Path,
2361 token: &str,
2362 depth: u8,
2363 retry: crate::lock_file::Retry,
2364) -> Option<(PathBuf, String)> {
2365 use std::io::Write;
2366 let marker = break_marker(path, token);
2367 for _ in 0..2 {
2368 match retry.create(&marker) {
2369 Ok(mut f) => {
2370 let mine = owner_token();
2371 if f.write_all(mine.as_bytes()).is_err() {
2372 drop(f);
2373 let _ = std::fs::remove_file(&marker);
2374 return None;
2375 }
2376 return Some((marker, mine));
2377 }
2378 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2379 let judged = std::fs::read_to_string(&marker).ok();
2380 match judged.filter(|_| older_than_stale(&marker)) {
2381 Some(judged) if depth < MAX_MARKER_DEPTH => {
2382 remove_lock_if(&marker, &judged, true, depth + 1, retry)?;
2383 }
2384 _ => return None,
2385 }
2386 }
2387 Err(_) => return None,
2388 }
2389 }
2390 None
2391}
2392
2393fn remove_lock_if(
2398 path: &Path,
2399 token: &str,
2400 require_stale: bool,
2401 depth: u8,
2402 retry: crate::lock_file::Retry,
2403) -> Option<bool> {
2404 let (marker, mine) = take_marker(path, token, depth, retry)?;
2405 let still = std::fs::read_to_string(path).is_ok_and(|c| c == token)
2406 && (!require_stale || older_than_stale(path) && writer_may_be_gone(token));
2407 if still {
2408 let _ = std::fs::remove_file(path);
2409 }
2410 if depth >= MAX_MARKER_DEPTH {
2412 if std::fs::read_to_string(&marker).is_ok_and(|c| c == mine) {
2413 let _ = std::fs::remove_file(&marker);
2414 }
2415 } else {
2416 let _ = remove_lock_if(&marker, &mine, false, depth + 1, retry);
2417 }
2418 Some(still)
2419}
2420
2421#[cfg(test)]
2424fn break_stale(path: &Path, judged: &str) -> bool {
2425 break_stale_with(path, judged, crate::lock_file::Retry::new(None))
2426}
2427
2428fn break_stale_with(path: &Path, judged: &str, retry: crate::lock_file::Retry) -> bool {
2429 remove_lock_if(path, judged, true, 0, retry).unwrap_or(false)
2430}
2431
2432impl Drop for TaskLock {
2433 fn drop(&mut self) {
2434 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
2435 let retry = crate::lock_file::Retry::new(Some(deadline));
2436 loop {
2437 match remove_lock_if(&self.path, &self.token, false, 0, retry) {
2438 Some(_) => return,
2439 None if std::time::Instant::now() > deadline => return,
2440 None => std::thread::sleep(std::time::Duration::from_millis(5)),
2441 }
2442 }
2443 }
2444}
2445
2446#[derive(Debug, Clone)]
2448pub struct Queue {
2449 root: PathBuf,
2450 ids: IdSource,
2451}
2452
2453fn validate_attachments(sources: &[PathBuf]) -> Result<Vec<(&PathBuf, &str)>> {
2454 let mut wanted = Vec::new();
2455 for src in sources {
2456 let name = src
2457 .file_name()
2458 .and_then(|n| n.to_str())
2459 .with_context(|| format!("`{}` has no usable file name", src.display()))?;
2460 if !crate::ask::valid_asset_name(name) {
2461 bail!(
2462 "attachment name `{name}` must match ^[A-Za-z0-9][A-Za-z0-9._-]{{0,63}}$ \
2463 with no `..`; rename the file and try again"
2464 );
2465 }
2466 if !src.is_file() {
2467 bail!("attachment `{}` is not a file", src.display());
2468 }
2469 wanted.push((src, name));
2470 }
2471 Ok(wanted)
2472}
2473
2474impl Queue {
2475 pub fn open() -> Self {
2477 Self::at(crate::run::home().join("queue"))
2478 }
2479
2480 pub fn at(root: PathBuf) -> Self {
2483 Self {
2484 root,
2485 ids: IdSource::new(new_id),
2486 }
2487 }
2488
2489 #[doc(hidden)]
2492 pub fn with_id_source(mut self, f: impl Fn() -> String + Send + Sync + 'static) -> Self {
2493 self.ids = IdSource::custom(f);
2494 self
2495 }
2496
2497 fn file_with(
2508 &self,
2509 task: &mut Task,
2510 hook: impl FnMut(&mut Task) -> Result<bool>,
2511 undo: impl FnMut(&mut Task),
2512 ) -> Result<()> {
2513 self.reserve_id(task, hook, undo, true)
2514 }
2515
2516 fn reserve_id(
2524 &self,
2525 task: &mut Task,
2526 mut hook: impl FnMut(&mut Task) -> Result<bool>,
2527 mut undo: impl FnMut(&mut Task),
2528 store: bool,
2529 ) -> Result<()> {
2530 let redraw_first = self.ids.is_custom() && task.attachments.is_empty();
2531 let initial = task.id.clone();
2532 self.ids
2533 .reserve((!redraw_first).then_some(initial.as_str()), "task", |id| {
2534 task.id = id.to_owned();
2535 let _lock = self.lock_task(&task.id)?;
2536 if path_exists(&self.path_of(&task.id))? || !hook(task)? {
2537 return Ok(false);
2538 }
2539 if store {
2540 if let Err(e) = self.write_record(task) {
2541 undo(task);
2542 return Err(e);
2543 }
2544 task.unfiled = false;
2545 }
2546 Ok(true)
2547 })?;
2548 Ok(())
2549 }
2550
2551 fn id_in_use(&self, id: &str, check_attachments: bool) -> Result<bool> {
2556 if path_exists(&self.lock_path(id))? {
2557 return Ok(true);
2558 }
2559 Ok(check_attachments && path_exists(&self.attachments_dir(id))?)
2560 }
2561
2562 fn plain_filing_hook(&self, task: &Task) -> Result<bool> {
2566 Ok(!self.id_in_use(&task.id, task.attachments.is_empty())?)
2567 }
2568
2569 pub fn file_claimed(&self, task: &mut Task) -> Result<Claim> {
2572 let mut claim = None;
2573 self.file_with(
2574 task,
2575 |t| {
2576 if self.id_in_use(&t.id, true)? {
2577 return Ok(false);
2578 }
2579 claim = Some(self.claim(&t.id)?);
2580 Ok(true)
2581 },
2582 |_| {},
2583 )?;
2584 claim.context("claim missing after filing")
2585 }
2586
2587 pub fn root(&self) -> &Path {
2589 &self.root
2590 }
2591
2592 pub fn path_of(&self, id: &str) -> PathBuf {
2594 self.root.join(format!("{id}.json"))
2595 }
2596
2597 pub fn attachments_dir(&self, id: &str) -> PathBuf {
2600 self.root.join(format!("{id}.attachments"))
2601 }
2602
2603 pub fn attach(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2614 let wanted = validate_attachments(sources)?;
2615 if task.unfiled && task.attachments.is_empty() {
2616 let names = std::cell::RefCell::new(Vec::new());
2619 self.reserve_id(
2620 task,
2621 |t| {
2622 if self.id_in_use(&t.id, true)? {
2623 return Ok(false);
2624 }
2625 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2626 Ok(true)
2627 },
2628 |_| {},
2629 false,
2630 )?;
2631 return Ok(names.into_inner());
2632 }
2633 self.copy_attachments(task, &wanted)
2634 }
2635
2636 fn copy_attachments(
2638 &self,
2639 task: &mut Task,
2640 wanted: &[(&PathBuf, &str)],
2641 ) -> Result<Vec<String>> {
2642 let dir = self.attachments_dir(&task.id);
2643 let existed = dir.is_dir();
2644 std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
2645 let mut created: Vec<PathBuf> = Vec::new();
2646 let mut names = Vec::new();
2647 let mut copy_all = || -> Result<()> {
2648 for (src, name) in wanted {
2649 let (stored, path) = copy_new(&dir, src, name)?;
2650 created.push(path);
2651 names.push(stored);
2652 }
2653 Ok(())
2654 };
2655 if let Err(e) = copy_all() {
2656 for path in &created {
2657 let _ = std::fs::remove_file(path);
2658 }
2659 if !existed {
2660 let _ = std::fs::remove_dir(&dir);
2661 }
2662 return Err(e);
2663 }
2664 task.attachments.extend(names.iter().cloned());
2665 Ok(names)
2666 }
2667
2668 pub fn attach_and_put(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2679 if task.unfiled && task.attachments.is_empty() {
2680 return self.attach_and_file(task, sources);
2681 }
2682 let dir = self.attachments_dir(&task.id);
2683 let existed = dir.is_dir();
2684 let before = task.attachments.len();
2685 let names = self.attach(task, sources)?;
2686 if let Err(e) = self.put(task) {
2687 for name in &names {
2688 let _ = std::fs::remove_file(dir.join(name));
2689 }
2690 if !existed {
2691 let _ = std::fs::remove_dir(&dir);
2692 }
2693 task.attachments.truncate(before);
2694 return Err(e);
2695 }
2696 Ok(names)
2697 }
2698
2699 fn attach_and_file(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2703 let wanted = validate_attachments(sources)?;
2704 let before = task.attachments.len();
2705 let names = std::cell::RefCell::new(Vec::new());
2706 let result = self.file_with(
2707 task,
2708 |t| {
2709 if self.id_in_use(&t.id, true)? {
2710 return Ok(false);
2711 }
2712 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2713 Ok(true)
2714 },
2715 |t| {
2716 let dir = self.attachments_dir(&t.id);
2717 for name in names.borrow().iter() {
2718 let _ = std::fs::remove_file(dir.join(name));
2719 }
2720 let _ = std::fs::remove_dir(&dir);
2722 t.attachments.truncate(before);
2723 },
2724 );
2725 result.map(|()| names.into_inner())
2726 }
2727
2728 pub fn attachment_paths(&self, task: &Task) -> Vec<PathBuf> {
2732 let dir = self.attachments_dir(&task.id);
2733 task.attachments
2734 .iter()
2735 .map(|n| {
2736 let p = dir.join(n);
2737 std::path::absolute(&p).unwrap_or(p)
2738 })
2739 .collect()
2740 }
2741
2742 pub fn put(&self, task: &mut Task) -> Result<()> {
2751 if task.unfiled {
2752 return self.file_with(task, |t| self.plain_filing_hook(t), |_| {});
2754 }
2755 let _lock = self.lock_task(&task.id)?;
2756 self.put_unlocked(task)
2757 }
2758
2759 pub fn create_new(&self, task: &mut Task) -> Result<bool> {
2764 let _lock = self.lock_task(&task.id)?;
2765 if path_exists(&self.path_of(&task.id))? {
2766 return Ok(false);
2767 }
2768 self.write_record(task)?;
2769 task.unfiled = false;
2770 Ok(true)
2771 }
2772
2773 fn put_unlocked(&self, task: &mut Task) -> Result<()> {
2775 self.merge_stored(task);
2776 if let Some(boundary) = task.reassignments.last() {
2777 let stored = self.get(&task.id)?;
2778 anyhow::ensure!(
2779 task.reassignments == stored.reassignments
2780 && task.repo == stored.repo
2781 && task.runs.get(..boundary.run_boundary)
2782 == stored.runs.get(..boundary.run_boundary),
2783 "historical repository execution cannot be restarted; reload the task"
2784 );
2785 }
2786 self.write_record(task)
2787 }
2788
2789 pub(crate) fn commit_base_recovery(
2791 &self,
2792 task: &mut Task,
2793 expected: &serde_json::Value,
2794 ) -> Result<()> {
2795 let _lock = self.lock_task(&task.id)?;
2796 anyhow::ensure!(
2797 serde_json::to_value(self.get(&task.id)?)? == *expected,
2798 "task changed during base recovery; inspect the pending authorization"
2799 );
2800 self.put_unlocked(task)
2802 }
2803
2804 pub(crate) fn commit_disk_wait(&self, task: &mut Task, expected: &Task) -> Result<()> {
2806 let _lock = self.lock_task(&task.id)?;
2807 anyhow::ensure!(
2808 serde_json::to_value(self.get(&task.id)?)? == serde_json::to_value(expected)?,
2809 "task changed during disk monitoring"
2810 );
2811 task.schema = task.schema.max(SCHEMA);
2812 self.write_record_inner(task, false)
2813 }
2814
2815 pub(crate) fn commit_reassignment(&self, task: &mut Task, expected: &Task) -> Result<()> {
2817 let _lock = self.lock_task(&task.id)?;
2818 let stored = self.get(&task.id)?;
2819 anyhow::ensure!(
2820 serde_json::to_value(&stored)? == serde_json::to_value(expected)?
2821 && stored.reassignments.len() + 1 == task.reassignments.len()
2822 && stored.repo == task.reassignments.last().expect("new boundary").from
2823 && stored.runs == task.runs,
2824 "task changed during reassignment; retry after inspecting it"
2825 );
2826 self.write_record(task)
2827 }
2828
2829 fn merge_stored(&self, task: &mut Task) {
2831 if let Ok(mut stored) = read_path(&self.path_of(&task.id)) {
2832 stored.import_chat_notice();
2833 if stored.reassignments.len() > task.reassignments.len() {
2834 let stale = std::mem::replace(task, stored.clone());
2837 for answer in stale.answers {
2838 if !task.answers.contains(&answer) {
2839 task.answers.push(answer);
2840 }
2841 }
2842 }
2843 for key in &stored.base_recovery_requests {
2844 if !task.base_recovery_requests.contains(key) {
2845 task.base_recovery_requests.push(key.clone());
2846 }
2847 }
2848 for receipt in &stored.repair_grants {
2849 if !task
2850 .repair_grants
2851 .iter()
2852 .any(|r| r.request.request == receipt.request.request)
2853 {
2854 task.repair_grants.push(receipt.clone());
2856 task.resume_override = stored.resume_override.clone();
2857 task.status = stored.status;
2858 task.hold_source = stored.hold_source;
2859 task.machine_hold_cause = stored.machine_hold_cause.clone();
2860 task.disk_resume_run = stored.disk_resume_run.clone();
2861 task.disk_checked_at = stored.disk_checked_at;
2862 task.hold_reason = stored.hold_reason.clone();
2863 task.held_at = stored.held_at;
2864 }
2865 }
2866 for receipt in &stored.repair_reconciliations {
2867 if !task.repair_reconciliations.contains(receipt) {
2868 task.repair_reconciliations.push(receipt.clone());
2871 task.resume_override = stored.resume_override.clone();
2872 task.status = stored.status;
2873 task.hold_source = stored.hold_source;
2874 task.machine_hold_cause = stored.machine_hold_cause.clone();
2875 task.disk_resume_run = stored.disk_resume_run.clone();
2876 task.disk_checked_at = stored.disk_checked_at;
2877 task.hold_reason = stored.hold_reason.clone();
2878 task.held_at = stored.held_at;
2879 }
2880 }
2881 task.superseded_resumes.extend(stored.superseded_resumes);
2882 task.dispatched_resumes.extend(stored.dispatched_resumes);
2883 if task.status != TaskStatus::Running
2884 && let Some(pin) = &stored.resume_override
2885 && !pin.question_id.is_empty()
2886 && let Some(run) = &pin.pinned_run
2887 && task.resume_override.as_ref().is_none_or(|next| {
2888 next.question_id != pin.question_id || next.pinned_run != pin.pinned_run
2889 })
2890 {
2891 task.superseded_resumes
2892 .insert(pin.question_id.clone(), run.clone());
2893 }
2894 for (id, receipt) in stored.decision_effects {
2897 task.decision_effects.insert(id, receipt);
2898 }
2899 if !task.decision_effects.is_empty() {
2900 task.schema = task.schema.max(SCHEMA);
2901 }
2902 if task.chat.is_none() {
2905 task.chat = stored.chat.clone();
2906 }
2907 for key in stored.chat_notices {
2908 if !task.chat_notices.contains(&key) {
2909 task.chat_notices.push(key);
2910 }
2911 }
2912 for failure in stored.startup_failures {
2913 task.record_startup_failure(failure);
2914 }
2915 for run in stored.runs {
2916 if !task.runs.contains(&run) {
2917 task.runs.push(run);
2918 }
2919 }
2920 for p in stored.passes {
2921 if !task
2922 .passes
2923 .iter()
2924 .any(|q| q.run == p.run && q.pass_index == p.pass_index)
2925 {
2926 task.passes.push(p);
2927 }
2928 }
2929 if let Some(n) = stored.chat_notice
2932 && task.chat_notice.as_ref().is_none_or(|mine| mine.at < n.at)
2933 {
2934 task.chat_notice = Some(n);
2935 }
2936 }
2937 }
2938
2939 pub(crate) fn write_startup_history(&self, task: &Task) -> Result<()> {
2942 let path = self.path_of(&task.id);
2943 let tmp = path.with_extension("json.tmp");
2944 std::fs::write(&tmp, serde_json::to_vec_pretty(task)?)?;
2945 std::fs::rename(tmp, path)?;
2946 Ok(())
2947 }
2948
2949 fn write_record(&self, task: &mut Task) -> Result<()> {
2951 self.write_record_inner(task, true)
2952 }
2953
2954 fn write_record_inner(&self, task: &mut Task, notify: bool) -> Result<()> {
2955 task.import_chat_notice();
2956 task.updated_at = Timestamp::now();
2957 std::fs::create_dir_all(&self.root)
2958 .with_context(|| format!("create {}", self.root.display()))?;
2959 let body = serde_json::to_string_pretty(task).context("serialize task")?;
2960 let path = self.path_of(&task.id);
2961 let tmp = path.with_extension("json.tmp");
2962 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
2963 std::fs::rename(&tmp, &path).with_context(|| format!("replace {}", path.display()))?;
2964 if let (Some(notice), Some(home)) = (
2969 notify.then(|| crate::notices::task_held(task)).flatten(),
2970 self.root.parent().filter(|p| !p.as_os_str().is_empty()),
2971 ) {
2972 crate::notices::raise_in(home, notice);
2973 }
2974 Ok(())
2975 }
2976
2977 pub fn modify(&self, id: &str, f: impl FnOnce(&mut Task) -> bool) -> Result<bool> {
2982 let id = self.resolve_id(id)?;
2983 let _lock = self.lock_task(&id)?;
2984 let mut task = self.get(&id)?;
2985 if !f(&mut task) {
2986 return Ok(false);
2987 }
2988 self.put_unlocked(&mut task)?;
2989 Ok(true)
2990 }
2991
2992 pub fn link_run(&self, id: &str, run: &str) -> Result<Task> {
2997 let id = self.resolve_id(id)?;
2998 let _lock = self.lock_task(&id)?;
3001 let mut task = self.get(&id)?;
3002 if task.link_run(run) {
3003 self.put_unlocked(&mut task)?;
3004 }
3005 Ok(task)
3006 }
3007
3008 pub(crate) fn lock_task(&self, id: &str) -> Result<TaskLock> {
3016 std::fs::create_dir_all(&self.root)
3017 .with_context(|| format!("create {}", self.root.display()))?;
3018 let path = self.root.join(format!("{id}.write-lock"));
3019 let started = std::time::Instant::now();
3020 let retry = crate::lock_file::Retry::task_write(started + TASK_LOCK_STALE);
3021 let marker_retry = crate::lock_file::Retry::new(Some(started + TASK_LOCK_STALE));
3022 loop {
3023 match retry.create(&path) {
3024 Ok(mut file) => {
3025 use std::io::Write;
3026 let token = owner_token();
3027 if let Err(e) = file.write_all(token.as_bytes()) {
3028 drop(file);
3029 let _ = std::fs::remove_file(&path);
3030 return Err(e).with_context(|| format!("lock {}", path.display()));
3031 }
3032 return Ok(TaskLock { path, token });
3033 }
3034 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
3035 let judged = std::fs::read_to_string(&path).ok();
3038 let broken = judged
3039 .filter(|_| older_than_stale(&path))
3040 .is_some_and(|judged| break_stale_with(&path, &judged, marker_retry));
3041 if !broken {
3045 if started.elapsed() > TASK_LOCK_STALE {
3046 bail!("could not lock task {id}");
3047 }
3048 std::thread::sleep(std::time::Duration::from_millis(15));
3049 }
3050 }
3051 Err(e) => return Err(e).with_context(|| format!("lock {}", path.display())),
3052 }
3053 }
3054 }
3055
3056 pub fn get(&self, id: &str) -> Result<Task> {
3058 let resolved = self.resolve_id(id)?;
3059 read_path(&self.path_of(&resolved))
3060 }
3061
3062 pub fn remove(&self, id: &str, in_flight: bool, questions: &Questions) -> Result<Removal> {
3094 self.remove_with_receipt(id, in_flight, questions, None)
3095 }
3096
3097 pub(crate) fn discard_triage(
3099 &self,
3100 id: &str,
3101 questions: &Questions,
3102 inquiry: &str,
3103 ) -> Result<Removal> {
3104 self.remove_with_receipt(id, false, questions, Some(inquiry))
3105 }
3106
3107 pub(crate) fn triage_discard_applied(&self, id: &str, inquiry: &str) -> Result<bool> {
3108 let path = self
3109 .root
3110 .join("triage-discard")
3111 .join(format!("{inquiry}.json"));
3112 let bytes = match std::fs::read(path) {
3113 Ok(bytes) => bytes,
3114 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
3115 Err(e) => return Err(e.into()),
3116 };
3117 let receipt: Task = serde_json::from_slice(&bytes)?;
3118 Ok(receipt.id == id && !self.path_of(id).exists())
3119 }
3120
3121 fn remove_with_receipt(
3122 &self,
3123 id: &str,
3124 in_flight: bool,
3125 questions: &Questions,
3126 inquiry: Option<&str>,
3127 ) -> Result<Removal> {
3128 let resolved = self.resolve_id(id)?;
3129 if in_flight {
3130 bail!("task {resolved} is being run by a live daemon right now");
3131 }
3132 let receipt = inquiry.map(|q| self.root.join("triage-discard").join(format!("{q}.json")));
3133 if let Some(path) = &receipt {
3134 std::fs::create_dir_all(path.parent().expect("receipt directory"))?;
3135 }
3136 self.write_tombstone(&resolved)?;
3137 if let Err(e) = self.remove_record_with_attachments(&resolved, |p| match &receipt {
3138 Some(path) => std::fs::rename(p, path),
3139 None => std::fs::remove_file(p),
3140 }) {
3141 if self.path_of(&resolved).exists() {
3145 let _ = std::fs::remove_file(self.tombstone_path(&resolved));
3146 }
3147 return Err(e);
3148 }
3149 let (released, still_blocked) = self.release_dependents_of(&resolved, questions);
3150 Ok(Removal {
3151 id: resolved,
3152 released,
3153 still_blocked,
3154 })
3155 }
3156
3157 fn tombstone_path(&self, id: &str) -> PathBuf {
3160 self.root.join(format!("{id}.removed"))
3161 }
3162
3163 fn write_tombstone(&self, id: &str) -> Result<()> {
3164 let path = self.tombstone_path(id);
3165 let tmp = self.root.join(format!("{id}.removed.tmp"));
3166 std::fs::write(&tmp, b"")
3167 .and_then(|()| std::fs::rename(&tmp, &path))
3168 .with_context(|| format!("write {}", path.display()))
3169 }
3170
3171 fn was_deleted(&self, id: &str) -> bool {
3175 self.tombstone_path(id).is_file() && !self.path_of(id).exists()
3176 }
3177
3178 pub fn apply_deleted_blockers(&self, task: &mut Task, questions: &Questions) -> Vec<String> {
3182 let deleted = deleted_blockers(self, questions, &task.blocked_by);
3183 deleted
3184 .into_iter()
3185 .filter(|id| task.dependency_deleted(id))
3186 .collect()
3187 }
3188
3189 pub fn note_dependency_deleted(&self, dependent: &Task, deleted: &str) {
3193 let Some(home) = self.root.parent().filter(|p| !p.as_os_str().is_empty()) else {
3194 return;
3195 };
3196 let ja = crate::lang::is_japanese(&crate::lang::of_repo(&dependent.repo));
3197 let waiting = dependent.status == TaskStatus::Blocked;
3198 let message = match (ja, waiting) {
3199 (true, false) => format!(
3200 "タスク {} は、待っていた {} が削除されたため待機を解除し、元の状態に戻しました",
3201 dependent.short(),
3202 short(deleted)
3203 ),
3204 (true, true) => format!(
3205 "タスク {} は、待っていた {} が削除されたため、残りの依存を待っています",
3206 dependent.short(),
3207 short(deleted)
3208 ),
3209 (false, false) => format!(
3210 "Task {} stopped waiting on {} because it was deleted, and returned to its previous state",
3211 dependent.short(),
3212 short(deleted)
3213 ),
3214 (false, true) => format!(
3215 "Task {} stopped waiting on {} because it was deleted, and is still waiting on its other dependencies",
3216 dependent.short(),
3217 short(deleted)
3218 ),
3219 };
3220 crate::notices::raise_in(
3221 home,
3222 crate::notices::Notice::info(
3223 &format!("unblocked:{}:{}", dependent.id, deleted),
3224 message,
3225 )
3226 .link(crate::notices::Link::Task {
3227 id: dependent.id.clone(),
3228 }),
3229 );
3230 }
3231
3232 fn release_dependents_of(
3237 &self,
3238 dependency: &str,
3239 questions: &Questions,
3240 ) -> (Vec<String>, Vec<String>) {
3241 let (mut released, mut still_blocked) = (Vec::new(), Vec::new());
3242 for listed in self.list() {
3243 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| {
3244 matches!(crate::queue::dependency(self, questions, b), Dependency::Deleted(id) if id == dependency)
3245 }) {
3246 continue;
3247 }
3248 let Ok(_claim) = self.claim(&listed.id) else {
3249 continue;
3250 };
3251 let Ok(mut task) = self.get(&listed.id) else {
3252 continue;
3253 };
3254 let references: Vec<String> = task.blocked_by.iter().filter(|id| {
3255 matches!(crate::queue::dependency(self, questions, id), Dependency::Deleted(full) if full == dependency)
3256 }).cloned().collect();
3257 if references.is_empty() {
3258 continue;
3259 }
3260 for reference in references {
3261 task.dependency_deleted(&reference);
3262 }
3263 if self.put(&mut task).is_err() {
3264 continue;
3265 }
3266 self.note_dependency_deleted(&task, dependency);
3267 if task.status == TaskStatus::Blocked {
3268 still_blocked.push(task.id.clone());
3269 } else {
3270 released.push(task.id.clone());
3271 }
3272 }
3273 (released, still_blocked)
3274 }
3275
3276 fn remove_record_with_attachments(
3280 &self,
3281 resolved: &str,
3282 remove_record: impl FnOnce(&Path) -> std::io::Result<()>,
3283 ) -> Result<()> {
3284 self.sweep_removed_attachments();
3291 let attachments = self.attachments_dir(resolved);
3292 let aside = self.root.join(format!("{resolved}.attachments.removing"));
3293 let moved = match std::fs::rename(&attachments, &aside) {
3294 Ok(()) => true,
3295 Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
3296 Err(e) => {
3297 return Err(e).with_context(|| format!("remove {}", attachments.display()));
3298 }
3299 };
3300 let path = self.path_of(resolved);
3301 if let Err(e) = remove_record(&path) {
3302 if moved {
3303 let _ = std::fs::rename(&aside, &attachments);
3304 }
3305 return Err(e).with_context(|| format!("remove {}", path.display()));
3306 }
3307 if moved {
3308 if let Err(e) = std::fs::remove_dir_all(&aside) {
3309 tracing::warn!("leftover attachments {}: {e}", aside.display());
3310 }
3311 }
3312 let lock = self.lock_path(resolved);
3313 if let Err(e) = std::fs::remove_file(&lock) {
3314 if e.kind() != std::io::ErrorKind::NotFound {
3315 return Err(e).with_context(|| format!("remove {}", lock.display()));
3316 }
3317 }
3318 Ok(())
3319 }
3320
3321 fn sweep_removed_attachments(&self) {
3327 let Ok(entries) = std::fs::read_dir(&self.root) else {
3328 return;
3329 };
3330 for entry in entries.flatten() {
3331 let name = entry.file_name();
3332 let name = name.to_string_lossy();
3333 let Some(id) = name.strip_suffix(".attachments.removing") else {
3334 continue;
3335 };
3336 if !self.path_of(id).exists() {
3339 if let Err(e) = std::fs::remove_dir_all(entry.path()) {
3340 tracing::warn!("leftover attachments {}: {e}", entry.path().display());
3341 }
3342 }
3343 }
3344 self.sweep_tombstones();
3345 }
3346
3347 fn sweep_tombstones(&self) {
3356 let Ok(entries) = std::fs::read_dir(&self.root) else {
3357 return;
3358 };
3359 let tasks = self.list();
3360 for entry in entries.flatten() {
3361 let name = entry.file_name();
3362 let name = name.to_string_lossy();
3363 let Some(id) = name.strip_suffix(".removed") else {
3364 continue;
3365 };
3366 if self.path_of(id).exists()
3367 || tasks.iter().any(|t| {
3368 t.blocked_by
3369 .iter()
3370 .any(|b| id.starts_with(b) || id.ends_with(b))
3371 })
3372 {
3373 continue;
3374 }
3375 let old = entry
3376 .metadata()
3377 .and_then(|m| m.modified())
3378 .ok()
3379 .and_then(|m| m.elapsed().ok())
3380 .is_some_and(|age| age >= TOMBSTONE_GRACE);
3381 if old {
3382 let _ = std::fs::remove_file(entry.path());
3383 }
3384 }
3385 }
3386
3387 fn lock_path(&self, id: &str) -> PathBuf {
3390 self.root.join(format!("{id}.lock"))
3391 }
3392
3393 pub fn list(&self) -> Vec<Task> {
3406 #[cfg(test)]
3407 LIST_READS.with(|count| count.set(count.get() + 1));
3408 let mut tasks: Vec<Task> = std::fs::read_dir(&self.root)
3409 .into_iter()
3410 .flatten()
3411 .flatten()
3412 .map(|e| e.path())
3413 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3414 .filter_map(|p| read_path(&p).ok())
3415 .collect();
3416 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then_with(|| b.id.cmp(&a.id)));
3417 tasks
3418 }
3419
3420 pub fn unreadable(&self) -> Vec<PathBuf> {
3424 std::fs::read_dir(&self.root)
3425 .into_iter()
3426 .flatten()
3427 .flatten()
3428 .map(|e| e.path())
3429 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3430 .filter(|p| read_path(p).is_err())
3431 .collect()
3432 }
3433
3434 pub fn superseded(&self) -> HashMap<String, String> {
3447 let mut by = HashMap::new();
3448 for task in self.list() {
3449 for earlier in &task.runs {
3450 if let Some(later) = task.successor_of(earlier) {
3451 by.insert(earlier.clone(), later.clone());
3452 }
3453 }
3454 }
3455 by
3456 }
3457
3458 pub fn superseded_by(&self, run: &str) -> Option<String> {
3466 for task in self.list() {
3467 if task.runs.iter().any(|r| r == run) {
3468 return task.successor_of(run).cloned();
3469 }
3470 }
3471 None
3472 }
3473
3474 pub fn latest_attempt(&self, run: &str) -> Option<String> {
3485 for task in self.list() {
3486 if task.runs.iter().any(|r| r == run) {
3487 return task.runs.last().filter(|last| **last != run).cloned();
3488 }
3489 }
3490 None
3491 }
3492
3493 pub fn next_runnable(&self) -> Option<Task> {
3498 let mut runnable: Vec<Task> = self
3499 .list()
3500 .into_iter()
3501 .filter(|t| t.status.runnable())
3502 .collect();
3503 runnable.sort_unstable_by(dispatch_order);
3504 runnable.into_iter().next()
3505 }
3506
3507 pub fn claim(&self, id: &str) -> Result<Claim> {
3514 std::fs::create_dir_all(&self.root)
3515 .with_context(|| format!("create {}", self.root.display()))?;
3516 let path = self.lock_path(id);
3517 let retry = crate::lock_file::Retry::new(None);
3518 match retry.create(&path) {
3519 Ok(mut f) => {
3520 use std::io::Write as _;
3521 let _ = writeln!(f, "{}", std::process::id());
3523 Ok(Claim { path })
3524 }
3525 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
3526 bail!("task {id} is already claimed ({} exists)", path.display())
3527 }
3528 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
3529 }
3530 }
3531
3532 pub fn resolve_id(&self, prefix: &str) -> Result<String> {
3534 if !prefix.is_empty()
3535 && prefix != "."
3536 && prefix != ".."
3537 && !prefix.contains(['/', '\\'])
3538 && self.path_of(prefix).is_file()
3539 {
3540 return Ok(prefix.to_owned());
3541 }
3542 let ids = record_ids(&self.root, "json")?;
3543 match resolve_reference(prefix, ids.iter().map(String::as_str)) {
3544 Reference::Found(id) => Ok(id),
3545 Reference::Missing => bail!("no task matches `{prefix}`"),
3546 Reference::Ambiguous => bail!("`{prefix}` matches multiple tasks"),
3547 }
3548 }
3549
3550 pub fn revision(&self) -> u64 {
3557 self.revision_excluding(&std::collections::BTreeSet::new())
3558 }
3559
3560 pub fn revision_excluding(&self, skip: &std::collections::BTreeSet<String>) -> u64 {
3563 use std::hash::{Hash as _, Hasher as _};
3564
3565 let mut entries: Vec<(String, u64)> = std::fs::read_dir(&self.root)
3566 .into_iter()
3567 .flatten()
3568 .flatten()
3569 .filter(|e| e.path().extension().is_some_and(|ext| ext == "json"))
3570 .filter(|e| {
3571 let path = e.path();
3572 !path
3573 .file_stem()
3574 .is_some_and(|stem| skip.contains(stem.to_string_lossy().as_ref()))
3575 })
3576 .filter_map(|e| {
3577 let name = e.file_name().to_string_lossy().into_owned();
3578 let mtime = e
3579 .metadata()
3580 .ok()?
3581 .modified()
3582 .ok()?
3583 .duration_since(std::time::UNIX_EPOCH)
3584 .ok()?
3585 .as_millis() as u64;
3586 Some((name, mtime))
3587 })
3588 .collect();
3589
3590 if entries.is_empty() {
3591 return 0;
3592 }
3593
3594 entries.sort_unstable();
3595 let mut hasher = std::hash::DefaultHasher::new();
3596 for (name, mtime) in &entries {
3597 name.hash(&mut hasher);
3598 mtime.hash(&mut hasher);
3599 }
3600 let h = hasher.finish();
3601 if h == 0 { 1 } else { h }
3602 }
3603}
3604
3605const TOMBSTONE_GRACE: std::time::Duration = std::time::Duration::from_secs(3600);
3608
3609#[derive(Debug, Clone)]
3611pub struct Removal {
3612 pub id: String,
3614 pub released: Vec<String>,
3618 pub still_blocked: Vec<String>,
3621}
3622
3623#[derive(Debug)]
3625pub struct Claim {
3626 path: PathBuf,
3627}
3628
3629impl Drop for Claim {
3630 fn drop(&mut self) {
3631 let _ = std::fs::remove_file(&self.path);
3632 }
3633}
3634
3635pub fn title_from(instruction: &str, max: usize) -> String {
3638 let Some(line) = first_line(instruction) else {
3639 return "(empty task)".to_owned();
3640 };
3641 if line.chars().count() <= max {
3642 return line.to_owned();
3643 }
3644 let head: String = line.chars().take(max.saturating_sub(1)).collect();
3645 format!("{head}…")
3646}
3647
3648pub(crate) fn first_line(instruction: &str) -> Option<&str> {
3654 let line = instruction
3655 .lines()
3656 .map(str::trim)
3657 .find(|l| !l.is_empty())?
3658 .trim_start_matches(['#', '-', '*', '>', ' '])
3659 .trim();
3660 (!line.is_empty()).then_some(line)
3661}
3662
3663#[cfg(test)]
3664thread_local! {
3665 pub(crate) static TASK_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
3666}
3667
3668fn read_path(path: &Path) -> Result<Task> {
3669 #[cfg(test)]
3670 TASK_READS.with(|count| count.set(count.get() + 1));
3671 let body = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
3672 let mut task: Task =
3673 serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
3674 if task.schema > SCHEMA {
3680 bail!(
3681 "task {} was written by a different magi (schema {}, this build \
3682 speaks {SCHEMA})",
3683 task.id,
3684 task.schema
3685 );
3686 }
3687 if task.schema < 22
3688 && task.status == TaskStatus::Held
3689 && task.hold_source == Some(HoldSource::Machine)
3690 && task.resume_override.is_none()
3691 && task
3692 .hold_reason
3693 .as_deref()
3694 .is_some_and(crate::disk::is_gate_reason)
3695 {
3696 task.mark_disk_capacity();
3697 }
3698 Ok(task)
3699}
3700
3701#[derive(Debug, PartialEq, Eq)]
3704pub(crate) enum Reference {
3705 Found(String),
3706 Missing,
3707 Ambiguous,
3708}
3709
3710pub(crate) fn resolve_reference<'a>(id: &str, ids: impl Iterator<Item = &'a str>) -> Reference {
3711 let mut hits = Vec::new();
3712 for candidate in ids {
3713 if candidate == id {
3714 return Reference::Found(candidate.to_owned());
3715 }
3716 if candidate.starts_with(id) || candidate.ends_with(id) {
3717 hits.push(candidate);
3718 }
3719 }
3720 match hits.as_slice() {
3721 [] => Reference::Missing,
3722 [one] => Reference::Found((*one).to_owned()),
3723 _ => Reference::Ambiguous,
3724 }
3725}
3726
3727pub(crate) fn record_ids(root: &Path, extension: &str) -> Result<Vec<String>> {
3730 let entries = match std::fs::read_dir(root) {
3731 Ok(entries) => entries,
3732 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
3733 Err(e) => return Err(e.into()),
3734 };
3735 let mut ids = Vec::new();
3736 for entry in entries {
3737 let path = entry?.path();
3738 if path.extension().is_some_and(|e| e == extension)
3739 && let Some(id) = path.file_stem().and_then(|s| s.to_str())
3740 {
3741 ids.push(id.to_owned());
3742 }
3743 }
3744 Ok(ids)
3745}
3746
3747#[derive(Debug, PartialEq, Eq)]
3750pub enum Dependency {
3751 Task(String),
3753 Question(String),
3755 Deleted(String),
3757 Missing,
3759 Ambiguous,
3761 Unknown,
3763}
3764
3765pub fn dependency(queue: &Queue, questions: &Questions, id: &str) -> Dependency {
3767 let (Ok(tasks), Ok(qs), Ok(removed)) = (
3768 record_ids(queue.root(), "json"),
3769 record_ids(questions.root(), "json"),
3770 record_ids(queue.root(), "removed"),
3771 ) else {
3772 return Dependency::Unknown;
3773 };
3774 let mut ids = tasks
3777 .iter()
3778 .chain(qs.iter())
3779 .chain(removed.iter())
3780 .cloned()
3781 .collect::<Vec<_>>();
3782 ids.sort();
3783 ids.dedup();
3784 match resolve_reference(id, ids.iter().map(String::as_str)) {
3785 Reference::Found(id) if tasks.contains(&id) => Dependency::Task(id),
3786 Reference::Found(id) if qs.contains(&id) => Dependency::Question(id),
3787 Reference::Found(id) if queue.was_deleted(&id) => Dependency::Deleted(id),
3788 Reference::Found(_) => Dependency::Unknown,
3789 Reference::Missing => Dependency::Missing,
3790 Reference::Ambiguous => Dependency::Ambiguous,
3791 }
3792}
3793
3794pub fn normalize_blockers(queue: &Queue, questions: &Questions, ids: &mut [String]) {
3796 for id in ids {
3797 if let Dependency::Task(full) | Dependency::Question(full) | Dependency::Deleted(full) =
3798 dependency(queue, questions, id)
3799 {
3800 *id = full;
3801 }
3802 }
3803}
3804
3805pub fn dependency_facts(queue: &Queue, questions: &Questions) -> String {
3808 let inventory = crate::blockers::Inventory::load(queue, questions);
3809 let mut text = String::from(
3810 "Current authoritative queue (historical missing-task claims may be stale; unreadable or ambiguous references must keep waiting):\n",
3811 );
3812 match record_ids(queue.root(), "json") {
3813 Ok(ids) => {
3814 for id in ids {
3815 match queue.get(&id) {
3816 Ok(task) => text.push_str(&format!(
3817 "{}: {}; blocked_by={:?}; chain={:?}\n",
3818 task.id,
3819 task.status.as_str(),
3820 task.blocked_by,
3821 inventory.waits_on(&task)
3822 )),
3823 Err(_) => text.push_str(&format!("{id}: exists, state unreadable\n")),
3824 }
3825 }
3826 }
3827 Err(_) => text.push_str("Queue inventory unreadable: existence is unknown.\n"),
3828 }
3829 text
3830}
3831
3832pub fn missing_blockers(
3834 queue: &Queue,
3835 questions: &Questions,
3836 blocked_by: &[String],
3837) -> Vec<String> {
3838 blocked_by
3839 .iter()
3840 .filter(|id| dependency(queue, questions, id) == Dependency::Missing)
3841 .cloned()
3842 .collect()
3843}
3844
3845pub fn deleted_blockers(
3848 queue: &Queue,
3849 questions: &Questions,
3850 blocked_by: &[String],
3851) -> Vec<String> {
3852 blocked_by
3853 .iter()
3854 .filter(|id| matches!(dependency(queue, questions, id), Dependency::Deleted(_)))
3855 .cloned()
3856 .collect()
3857}
3858
3859pub fn missing_blocker_hold_reason(blocked_by: &[String], missing: &[String]) -> String {
3871 missing_blocker_hold_reason_in(blocked_by, missing, "en")
3872}
3873
3874pub fn missing_blocker_hold_reason_in(
3876 blocked_by: &[String],
3877 missing: &[String],
3878 language: &str,
3879) -> String {
3880 if crate::lang::is_japanese(language) {
3881 format!(
3882 "{} を待っていましたが、{} はディスク上に存在しません - `magi task triage` を参照",
3883 blocked_by.join(", "),
3884 missing.join(", "),
3885 )
3886 } else {
3887 format!(
3888 "blocked on {} but {} no longer exist(s) on disk - see `magi task triage`",
3889 blocked_by.join(", "),
3890 missing.join(", "),
3891 )
3892 }
3893}
3894
3895pub fn short(id: &str) -> &str {
3897 id.split('-').next_back().unwrap_or(id)
3898}
3899
3900fn new_id() -> String {
3901 let stamp = jiff::Zoned::now().strftime("%Y%m%d-%H%M%S");
3902 let seed = crate::rng::entropy();
3903 format!("{stamp}-{:04x}", (seed ^ (seed >> 32)) & 0xffff)
3904}
3905
3906#[cfg(test)]
3907mod tests {
3908 #[test]
3909 fn reassignment_clears_repository_bound_execution_selections() {
3910 let mut task = Task::new(
3911 "move".into(),
3912 "evidence".into(),
3913 "old".into(),
3914 Source::Human,
3915 );
3916 task.review_branch = Some("old-review".into());
3917 task.review_instruction = Some("old criteria".into());
3918 task.pr_recovery = Some(crate::pr_recovery::Selection {
3919 url: "https://example.invalid/pull/1".into(),
3920 branch: "old-pr".into(),
3921 head: "head".into(),
3922 source_head: None,
3923 source_run: None,
3924 source_proof: None,
3925 disposition: None,
3926 instruction: "old criteria".into(),
3927 approved_at: Timestamp::now(),
3928 });
3929 task.replacement = Some(crate::merge_decision::Replacement {
3930 question: "owner".into(),
3931 run: "old-run".into(),
3932 pr: "https://example.invalid/pull/1".into(),
3933 agent: "worker".into(),
3934 context: "evidence".into(),
3935 });
3936 task.resume_override = Some(OperatorResume {
3937 question_id: "old-question".into(),
3938 at: Timestamp::now(),
3939 conductor_rehold: None,
3940 forced: true,
3941 pinned_run: Some("old-run".into()),
3942 });
3943 task.interrupt = true;
3944 task.reassign_repository("new".into(), "test");
3945 assert!(task.review_branch.is_none());
3946 assert!(task.review_instruction.is_none());
3947 assert!(task.pr_recovery.is_none());
3948 assert!(task.replacement.is_none());
3949 assert!(task.resume_override.is_none());
3950 assert!(!task.interrupt);
3951 assert!(task.fresh_start);
3952 assert_eq!(task.instruction, "evidence");
3953 }
3954
3955 #[test]
3956 fn explicit_resumption_is_not_invented_for_legacy_or_automatic_release() {
3957 let t = Task::new(
3958 "retry".into(),
3959 "do it".into(),
3960 PathBuf::from("."),
3961 Source::Human,
3962 );
3963 let mut value = serde_json::to_value(&t).unwrap();
3964 for field in ["resumption", "resumption_history", "operator_decided_at"] {
3965 value.as_object_mut().unwrap().remove(field);
3966 }
3967 value["schema"] = serde_json::json!(15);
3968 let mut legacy: Task = serde_json::from_value(value).unwrap();
3969 legacy.hold_machine(Some("automatic retry".into()));
3970 legacy.release();
3971 assert!(legacy.resumption.is_none());
3972 assert!(legacy.resumption_history.is_empty());
3973 assert!(legacy.operator_decided_at.is_none());
3974 legacy.release_operator("cli", None);
3975 legacy.start("run".into());
3976 let reloaded: Task =
3977 serde_json::from_str(&serde_json::to_string(&legacy).unwrap()).unwrap();
3978 assert!(reloaded.resume_override.is_none());
3979 assert_eq!(reloaded.resumption.as_ref().unwrap().generation, 1);
3980 }
3981
3982 #[test]
3983 fn merge_over_lets_the_later_choice_win_and_keeps_the_rest() {
3984 let mut base = RunOverrides {
3985 merge: Some("pr".to_owned()),
3986 candidates: Some(3),
3987 ..RunOverrides::default()
3988 };
3989 base.merge_over(&RunOverrides {
3990 merge: Some("none".to_owned()),
3991 seed: Some(7),
3992 ..RunOverrides::default()
3993 });
3994 assert_eq!(base.merge.as_deref(), Some("none"));
3995 assert_eq!(base.candidates, Some(3));
3996 assert_eq!(base.seed, Some(7));
3997 }
3998
3999 #[test]
4000 fn an_already_landed_task_is_done_with_its_attempt_refunded() {
4001 let mut t = task("relanded");
4002 t.attempts = 1;
4003 t.status = TaskStatus::Running;
4004 t.already_landed("already in main as 0e368de");
4005 assert_eq!(t.status, TaskStatus::Done);
4006 assert_eq!(t.attempts, 0);
4007 assert!(t.hold_reason.is_none());
4008 assert_eq!(t.last_error.as_deref(), Some("already in main as 0e368de"));
4009 }
4010
4011 #[test]
4012 fn missing_blocker_reason_follows_the_language() {
4013 let b = vec!["a".to_owned()];
4014 let en = missing_blocker_hold_reason_in(&b, &b, "en");
4015 assert_eq!(en, missing_blocker_hold_reason(&b, &b));
4016 assert!(en.starts_with("blocked on a"));
4017 assert!(missing_blocker_hold_reason_in(&b, &b, "ja").contains("存在しません"));
4018 assert_eq!(missing_blocker_hold_reason_in(&b, &b, "de"), en);
4019 }
4020
4021 use super::*;
4022
4023 #[test]
4024 fn triage_applied_survives_release_and_old_records_read_as_empty() {
4025 let mut t = Task::new(
4026 "t".to_owned(),
4027 "i".to_owned(),
4028 PathBuf::from("r"),
4029 Source::Human,
4030 );
4031 t.mark_triage_applied("q1");
4032 t.mark_triage_applied("q1");
4033 t.hold_machine(Some("x".to_owned()));
4034 t.release();
4035 assert_eq!(t.triage_applied, ["q1"]);
4036 assert!(t.triage_applied("q1") && !t.triage_applied("q2"));
4037
4038 let mut v = serde_json::to_value(&t).unwrap();
4039 v.as_object_mut().unwrap().remove("triage_applied");
4040 let old: Task = serde_json::from_value(v).unwrap();
4041 assert!(old.triage_applied.is_empty());
4042 }
4043
4044 #[test]
4045 fn task_counts_of_empty_is_all_zero() {
4046 assert_eq!(TaskCounts::of(&[]), TaskCounts::default());
4047 }
4048
4049 #[test]
4050 fn task_counts_of_tallies_every_status() {
4051 let mut queued = Task::new(
4052 "q".to_owned(),
4053 "i".to_owned(),
4054 PathBuf::from("."),
4055 Source::Human,
4056 );
4057 queued.status = TaskStatus::Queued;
4058 let mut running = queued.clone();
4059 running.status = TaskStatus::Running;
4060 let mut done = queued.clone();
4061 done.status = TaskStatus::Done;
4062 let mut failed = queued.clone();
4063 failed.status = TaskStatus::Failed;
4064 let mut held = queued.clone();
4065 held.status = TaskStatus::Held;
4066 let mut blocked = queued.clone();
4067 blocked.status = TaskStatus::Blocked;
4068
4069 let mut parked = queued.clone();
4070 parked.status = TaskStatus::Parked;
4071
4072 let counts = TaskCounts::of(&[
4073 queued,
4074 running,
4075 done.clone(),
4076 done,
4077 failed,
4078 held,
4079 blocked,
4080 parked,
4081 ]);
4082 assert_eq!(
4083 counts,
4084 TaskCounts {
4085 queued: 1,
4086 running: 1,
4087 done: 2,
4088 failed: 1,
4089 held: 1,
4090 blocked: 1,
4091 parked: 1,
4092 }
4093 );
4094 }
4095
4096 #[test]
4097 fn task_implementers_persist_normalized_without_losing_history() {
4098 let (_dir, q) = queue();
4099 let mut t = task("keep me");
4100 t.hold_manual(Some("waiting".into()));
4101 t.runs.push("earlier-run".into());
4102 t.attachments.push("reference.txt".into());
4103 let cfg: crate::config::Config =
4104 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
4105 .unwrap();
4106 q.put(&mut t).unwrap();
4107 let original = serde_json::to_value(&t).unwrap();
4108 for n in [2, 1, 2] {
4109 t.set_implementers(n, &cfg).unwrap();
4110 q.put(&mut t).unwrap();
4111 t = q.get(&t.id).unwrap();
4112 assert_eq!(t.implementers(), Some(usize::from(n)));
4113 assert_eq!(t.solo, n == 1);
4114 if n == 1 {
4115 assert!(t.overrides.is_none());
4116 }
4117 let mut actual = serde_json::to_value(&t).unwrap();
4118 for field in ["solo", "overrides", "updated_at"] {
4119 actual[field] = original[field].clone();
4120 }
4121 assert_eq!(actual, original);
4122 }
4123 t.overrides.as_mut().unwrap().judges = Some(2);
4124 t.set_implementers(1, &cfg).unwrap();
4125 assert_eq!(t.overrides.as_ref().unwrap().judges, Some(2));
4126 assert_eq!(t.overrides.as_ref().unwrap().candidates, None);
4127 }
4128
4129 #[test]
4130 fn task_implementers_old_records_keep_inheritance_until_changed() {
4131 let mut value = serde_json::to_value(task("old task")).unwrap();
4132 value.as_object_mut().unwrap().remove("solo");
4133 value.as_object_mut().unwrap().remove("overrides");
4134 let mut old: Task = serde_json::from_value(value).unwrap();
4135 assert_eq!(old.implementers(), None);
4136 old.set_implementers(1, &crate::config::Config::default())
4137 .unwrap();
4138 assert!(old.solo);
4139 assert!(old.overrides.is_none());
4140 }
4141
4142 #[test]
4143 fn task_implementers_reject_invalid_counts_and_states() {
4144 let cfg = crate::config::Config::default();
4145 let mut t = task("unchanged");
4146 for n in [0, 4, 255] {
4147 assert!(t.set_implementers(n, &cfg).is_err());
4148 }
4149 for status in [
4150 TaskStatus::Running,
4151 TaskStatus::Done,
4152 TaskStatus::Failed,
4153 TaskStatus::Blocked,
4154 TaskStatus::Parked,
4155 ] {
4156 t.status = status;
4157 let before = serde_json::to_value(&t).unwrap();
4158 assert!(t.set_implementers(1, &cfg).is_err());
4159 assert_eq!(serde_json::to_value(&t).unwrap(), before);
4160 }
4161 }
4162
4163 fn queue() -> (tempfile::TempDir, Queue) {
4166 let dir = tempfile::tempdir().unwrap();
4167 let q = Queue::at(dir.path().join("queue"));
4168 (dir, q)
4169 }
4170
4171 #[test]
4172 fn base_recovery_commit_retains_decision_and_superseded_resume_evidence() {
4173 let (_dir, q) = queue();
4174 let mut original = task("base recovery");
4175 original
4176 .decision_effects
4177 .insert("old-effect".into(), "receipt".into());
4178 original
4179 .dispatched_resumes
4180 .insert("dispatched".into(), "prior-run".into());
4181 original.resume_override = Some(OperatorResume {
4182 question_id: "prior-decision".into(),
4183 at: Timestamp::now(),
4184 conductor_rehold: None,
4185 forced: true,
4186 pinned_run: Some("retained-run".into()),
4187 });
4188 q.put(&mut original).unwrap();
4189 let expected = serde_json::to_value(&original).unwrap();
4190 let mut recovered = original.clone();
4191 recovered.base_recovery_requests.push("chat/request".into());
4192 recovered
4193 .resume_override
4194 .as_mut()
4195 .unwrap()
4196 .question_id
4197 .clear();
4198 q.commit_base_recovery(&mut recovered, &expected).unwrap();
4199 let saved = q.get(&original.id).unwrap();
4200 assert_eq!(saved.decision_effects, original.decision_effects);
4201 assert_eq!(saved.dispatched_resumes, original.dispatched_resumes);
4202 assert_eq!(
4203 saved
4204 .superseded_resumes
4205 .get("prior-decision")
4206 .map(String::as_str),
4207 Some("retained-run")
4208 );
4209 assert_eq!(saved.base_recovery_requests, ["chat/request"]);
4210 assert!(saved.resume_override.unwrap().question_id.is_empty());
4211 assert!(q.commit_base_recovery(&mut recovered, &expected).is_err());
4212 }
4213
4214 #[test]
4215 fn a_stale_write_does_not_drop_the_recorded_task_chat() {
4216 let (_dir, q) = queue();
4217 let mut stale = task("chat");
4218 q.put(&mut stale).unwrap();
4219 let mut fresh = q.get(&stale.id).unwrap();
4220 fresh.chat = Some("talk-1".to_owned());
4221 q.put(&mut fresh).unwrap();
4222 q.put(&mut stale).unwrap();
4223 assert_eq!(q.get(&stale.id).unwrap().chat.as_deref(), Some("talk-1"));
4224 }
4225
4226 #[test]
4227 fn putting_a_machine_held_task_files_a_notification_beside_the_queue() {
4228 let dir = tempfile::tempdir().unwrap();
4229 let q = Queue::at(dir.path().join("queue"));
4230 let mut t = task("held");
4231 q.put(&mut t).unwrap();
4232 assert_eq!(
4233 crate::notices::Notices::at(dir.path().join("notifications"))
4234 .list()
4235 .len(),
4236 0
4237 );
4238 t.hold_machine(Some("out of attempts".to_owned()));
4239 q.put(&mut t).unwrap();
4240 let listed = crate::notices::Notices::at(dir.path().join("notifications")).list();
4241 assert_eq!(listed.len(), 1);
4242 assert!(listed[0].message.contains("out of attempts"));
4243 }
4244
4245 #[test]
4246 fn start_consumes_the_disk_resume_pin() {
4247 let mut t = task("pin");
4248 t.disk_resume_run = Some("run-1".into());
4249 t.start("run-1".into());
4250 assert!(t.disk_resume_run.is_none());
4251 }
4252
4253 fn task(title: &str) -> Task {
4254 Task::new(
4255 title.to_owned(),
4256 format!("do {title}"),
4257 PathBuf::from("."),
4258 Source::Human,
4259 )
4260 }
4261
4262 #[test]
4263 fn attempt_limit_inherits_old_records_and_controls_failure() {
4264 let mut t = task("budget");
4265 let mut old = serde_json::to_value(&t).unwrap();
4266 old.as_object_mut().unwrap().remove("attempt_limit");
4267 old["schema"] = serde_json::json!(15);
4268 let legacy: Task = serde_json::from_value(old).unwrap();
4269 assert_eq!(legacy.attempt_limit, None);
4270 assert_eq!(legacy.effective_attempt_limit(2), 2);
4271 t.attempts = 2;
4272 t.change_attempt_limit(NonZeroUsize::new(3)).unwrap();
4273 t.fail("retry", 1);
4274 assert_eq!(t.status, TaskStatus::Failed);
4275 t.attempts = 3;
4276 t.fail("stop", 10);
4277 assert_eq!(t.status, TaskStatus::Held);
4278 t.change_attempt_limit(None).unwrap();
4279 assert_eq!(t.status, TaskStatus::Held);
4280 assert_eq!(t.attempts, 3);
4281 t.fail("inherit", 4);
4282 assert_eq!(t.status, TaskStatus::Failed);
4283 for status in [
4284 TaskStatus::Running,
4285 TaskStatus::Done,
4286 TaskStatus::Failed,
4287 TaskStatus::Blocked,
4288 TaskStatus::Parked,
4289 ] {
4290 t.status = status;
4291 assert!(t.change_attempt_limit(NonZeroUsize::new(1)).is_err());
4292 assert_eq!(t.attempt_limit, None);
4293 }
4294 }
4295
4296 #[test]
4297 fn earlier_attempts_is_every_recorded_run_and_agrees_with_the_display() {
4298 let mut t = task("retried");
4299 assert!(
4300 t.earlier_attempts().is_empty(),
4301 "a first attempt takes nothing over"
4302 );
4303 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4304 assert_eq!(t.earlier_attempts(), ["aaaa", "bbbb"]);
4305 assert_eq!(t.successor_of("aaaa"), Some(&"bbbb".to_owned()));
4307 assert_eq!(t.successor_of("bbbb"), None);
4308 assert_eq!(t.successor_of("zzzz"), None);
4309 }
4310
4311 #[test]
4312 fn resumed_runs_are_superseded_only_after_their_last_occurrence() {
4313 let (_dir, q) = queue();
4314 let mut t = task("resumed");
4315 for (runs, successor) in [
4316 (vec!["aaaa", "aaaa"], None),
4317 (vec!["aaaa", "bbbb", "aaaa"], None),
4318 (vec!["aaaa", "aaaa", "bbbb"], Some("bbbb")),
4319 (vec!["aaaa", "bbbb", "aaaa", "cccc"], Some("cccc")),
4320 ] {
4321 t.runs = runs.into_iter().map(str::to_owned).collect();
4322 q.put(&mut t).unwrap();
4323
4324 assert_eq!(t.successor_of("aaaa").map(String::as_str), successor);
4325 assert_eq!(q.superseded_by("aaaa").as_deref(), successor);
4326 assert_eq!(q.superseded().get("aaaa").map(String::as_str), successor);
4327 }
4328 }
4329
4330 #[test]
4331 fn superseded_by_names_the_next_attempt_and_none_for_the_last() {
4332 let (_dir, q) = queue();
4333 let mut t = task("retried");
4334 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4335 q.put(&mut t).unwrap();
4336
4337 assert_eq!(q.superseded_by("aaaa"), Some("bbbb".to_owned()));
4338 assert_eq!(q.superseded_by("bbbb"), Some("cccc".to_owned()));
4339 assert_eq!(
4340 q.superseded_by("cccc"),
4341 None,
4342 "the latest attempt replaces nothing"
4343 );
4344 assert_eq!(
4345 q.superseded_by("never-heard-of-it"),
4346 None,
4347 "a run belonging to no task on this queue is not superseded"
4348 );
4349
4350 let mut by = HashMap::new();
4351 by.insert("aaaa".to_owned(), "bbbb".to_owned());
4352 by.insert("bbbb".to_owned(), "cccc".to_owned());
4353 assert_eq!(
4354 q.superseded(),
4355 by,
4356 "the whole-map and single-run forms must agree"
4357 );
4358 }
4359
4360 #[test]
4361 fn superseded_attempts_is_empty_until_the_task_is_done() {
4362 let mut t = task("retried");
4363 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4364 t.status = TaskStatus::Failed;
4365 assert_eq!(
4366 t.superseded_attempts(true),
4367 &[] as &[String],
4368 "a task still retrying has no attempt yet that a later one made moot"
4369 );
4370
4371 t.status = TaskStatus::Running;
4372 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
4373 }
4374
4375 #[test]
4376 fn superseded_attempts_names_every_run_before_the_one_that_succeeded() {
4377 let mut t = task("retried");
4378 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4379 t.status = TaskStatus::Done;
4380 assert_eq!(
4381 t.superseded_attempts(true),
4382 &["aaaa".to_owned(), "bbbb".to_owned()],
4383 "cccc is the attempt whose success made the task done, and stays out"
4384 );
4385 }
4386
4387 #[test]
4388 fn superseded_attempts_is_empty_for_a_done_task_with_only_one_attempt() {
4389 let mut t = task("first try landed");
4390 t.runs = vec!["aaaa".to_owned()];
4391 t.status = TaskStatus::Done;
4392 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
4393 }
4394
4395 #[test]
4396 fn superseded_attempts_is_empty_when_the_last_run_never_actually_succeeded() {
4397 let mut t = task("closed by hand after a manual merge");
4404 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4405 t.status = TaskStatus::Done;
4406 assert_eq!(
4407 t.superseded_attempts(false),
4408 &[] as &[String],
4409 "nothing here is provably why the task is done, so nothing is superseded"
4410 );
4411 }
4412
4413 #[test]
4414 fn latest_attempt_names_the_chain_s_current_head_not_just_the_next_one() {
4415 let (_dir, q) = queue();
4416 let mut t = task("retried twice");
4417 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4418 q.put(&mut t).unwrap();
4419
4420 assert_eq!(
4421 q.latest_attempt("aaaa"),
4422 Some("cccc".to_owned()),
4423 "an old attempt points straight at the chain's current head, not the \
4424 next attempt in the middle of it"
4425 );
4426 assert_eq!(q.latest_attempt("bbbb"), Some("cccc".to_owned()));
4427 assert_eq!(
4428 q.latest_attempt("cccc"),
4429 None,
4430 "the latest attempt is not superseded by anything"
4431 );
4432 assert_eq!(
4433 q.latest_attempt("never-heard-of-it"),
4434 None,
4435 "a run belonging to no task on this queue is not superseded"
4436 );
4437 }
4438
4439 #[test]
4440 fn a_markdown_heading_is_the_title_not_decoration() {
4441 assert_eq!(
4446 title_from("# Rework the config loader\n\nIt re-reads it.\n", 40),
4447 "Rework the config loader"
4448 );
4449 assert_eq!(title_from("- fix the thing", 40), "fix the thing");
4450 assert_eq!(title_from("> quoted task", 40), "quoted task");
4451 assert_eq!(title_from(" \n\n", 40), "(empty task)");
4453 assert_eq!(title_from("###\n", 40), "(empty task)");
4454 }
4455
4456 #[test]
4457 fn a_long_title_is_elided_by_characters_not_bytes() {
4458 let long = "課題".repeat(30);
4460 let title = title_from(&long, 10);
4461 assert_eq!(title.chars().count(), 10);
4462 assert!(title.ends_with('…'));
4463 }
4464
4465 #[test]
4466 fn priority_wins_and_ties_break_oldest_first() {
4467 let (_dir, q) = queue();
4468 let mut a = task("first");
4469 let mut b = task("second");
4470 let mut c = task("urgent");
4471 a.id = "20260101-000001-aaaa".to_owned();
4473 b.id = "20260101-000002-bbbb".to_owned();
4474 c.id = "20260101-000003-cccc".to_owned();
4475 c.priority = 5;
4476 for t in [&mut a, &mut b, &mut c] {
4477 q.put(t).unwrap();
4478 }
4479
4480 assert_eq!(q.next_runnable().unwrap().id, c.id);
4482 c.hold_machine(None);
4483 q.put(&mut c).unwrap();
4484 assert_eq!(q.next_runnable().unwrap().id, a.id);
4486 assert_eq!(q.list().len(), 3, "b is still waiting its turn");
4487 }
4488
4489 #[test]
4490 fn a_blocked_task_never_starves_another_runnable_one() {
4491 let (_dir, q) = queue();
4492 let mut blocked = task("blocked");
4493 blocked.block(vec!["something".to_owned()], None);
4494 q.put(&mut blocked).unwrap();
4495
4496 let mut runnable = task("free to go");
4497 q.put(&mut runnable).unwrap();
4498
4499 let next = q.next_runnable().expect("a runnable task is still offered");
4500 assert_eq!(next.id, runnable.id);
4501 }
4502
4503 #[test]
4504 fn a_held_task_is_never_offered_to_the_loop() {
4505 let (_dir, q) = queue();
4506 let mut t = task("held");
4507 q.put(&mut t).unwrap();
4508 assert!(q.next_runnable().is_some());
4509
4510 t.hold_machine(None);
4511 q.put(&mut t).unwrap();
4512 assert!(
4513 q.next_runnable().is_none(),
4514 "a held task must wait for a human"
4515 );
4516
4517 t.status = TaskStatus::Failed;
4519 q.put(&mut t).unwrap();
4520 assert!(q.next_runnable().is_some());
4521 }
4522
4523 #[test]
4524 fn attempts_are_capped_and_then_the_task_is_held() {
4525 let mut t = task("doomed");
4526
4527 t.start("run-1".to_owned());
4528 t.fail("gate red", 2);
4529 assert_eq!(t.status, TaskStatus::Failed, "one attempt of two: retry");
4530
4531 t.start("run-2".to_owned());
4532 t.fail("gate red", 2);
4533 assert_eq!(
4534 t.status,
4535 TaskStatus::Held,
4536 "out of attempts: stop spending money on it"
4537 );
4538 assert_eq!(t.runs, ["run-1", "run-2"]);
4539 assert_eq!(t.last_error.as_deref(), Some("gate red"));
4540 assert_eq!(
4541 t.hold_reason.as_deref(),
4542 Some("gate red"),
4543 "the hold must say why, not leave hold_reason null next to a \
4544 populated last_error"
4545 );
4546 }
4547
4548 #[test]
4549 fn handing_off_a_task_records_a_hold_reason_too() {
4550 let mut t = task("left a pull request");
4551 t.start("run-1".to_owned());
4552 t.handed_off("run ended with a pull request open [run run-1]");
4553 assert_eq!(t.status, TaskStatus::Held);
4554 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4555 assert_eq!(
4556 t.hold_reason.as_deref(),
4557 Some("run ended with a pull request open [run run-1]")
4558 );
4559 assert_eq!(t.hold_reason, t.last_error);
4560 }
4561
4562 #[test]
4563 fn a_quota_stall_is_refunded_so_the_backlog_survives_the_night() {
4564 let mut t = task("stalled by quota");
4565
4566 t.start("run-1".to_owned());
4567 assert_eq!(t.attempts, 1);
4568 t.stall("judge-1, judge-2 out of quota");
4569 assert_eq!(
4570 t.attempts, 0,
4571 "a closed quota window must not spend the task's retry budget"
4572 );
4573 assert_eq!(t.status, TaskStatus::Failed, "the loop should retry it");
4574 assert_eq!(
4575 t.last_error.as_deref(),
4576 Some("judge-1, judge-2 out of quota")
4577 );
4578
4579 for _ in 0..20 {
4582 t.start("run-n".to_owned());
4583 t.stall("still out of quota");
4584 }
4585 t.start("run-real".to_owned());
4586 t.fail("gate red", 2);
4587 assert_eq!(
4588 t.status,
4589 TaskStatus::Failed,
4590 "the first attempt that was really judged is attempt one"
4591 );
4592 }
4593
4594 #[test]
4595 fn releasing_a_held_task_gives_it_a_real_second_chance() {
4596 let mut t = task("retry me");
4597 t.start("run-1".to_owned());
4598 t.fail("gate red", 1);
4599 assert_eq!(t.status, TaskStatus::Held);
4600
4601 t.release();
4602 assert_eq!(t.status, TaskStatus::Queued);
4603 assert_eq!(t.attempts, 0);
4606 assert!(t.last_error.is_none());
4607 assert_eq!(
4608 t.runs.len(),
4609 1,
4610 "history is kept: attempts reset, evidence does not"
4611 );
4612 }
4613
4614 #[test]
4615 fn a_hold_reason_survives_and_a_release_clears_it() {
4616 let mut t = task("waiting on something else");
4617 t.hold_manual(Some(
4618 "waiting for 20260101-000000-aaaa to land first".to_owned(),
4619 ));
4620 assert_eq!(t.status, TaskStatus::Held);
4621 assert_eq!(
4622 t.hold_reason.as_deref(),
4623 Some("waiting for 20260101-000000-aaaa to land first")
4624 );
4625
4626 t.hold_manual(None);
4628 assert_eq!(
4629 t.hold_reason.as_deref(),
4630 Some("waiting for 20260101-000000-aaaa to land first"),
4631 "a bare re-hold keeps whatever a human already wrote down"
4632 );
4633
4634 let mut plain = task("no reason given");
4636 plain.hold_manual(None);
4637 assert_eq!(plain.status, TaskStatus::Held);
4638 assert!(plain.hold_reason.is_none());
4639
4640 t.release();
4641 assert_eq!(t.status, TaskStatus::Queued);
4642 assert!(
4643 t.hold_reason.is_none(),
4644 "a stale reason must not greet the next person who holds this task"
4645 );
4646 }
4647
4648 #[test]
4649 fn closing_a_held_task_as_done_clears_its_hold_reason_too() {
4650 let mut t = task("landed by hand while held");
4655 t.hold_manual(Some("waiting on 3ed9".to_owned()));
4656 assert_eq!(t.hold_reason.as_deref(), Some("waiting on 3ed9"));
4657
4658 t.succeed();
4659 assert_eq!(t.status, TaskStatus::Done);
4660 assert!(
4661 t.hold_reason.is_none(),
4662 "a done task cannot still be waiting on something"
4663 );
4664 }
4665
4666 #[test]
4667 fn holding_or_closing_a_blocked_task_clears_its_dependency_too() {
4668 let mut held = task("held straight out of blocked");
4675 held.block(
4676 vec!["20260101-000000-dead".to_owned()],
4677 Some("waiting on the migration script".to_owned()),
4678 );
4679 assert_eq!(held.status, TaskStatus::Blocked);
4680
4681 held.hold_manual(None);
4682 assert_eq!(held.status, TaskStatus::Held);
4683 assert!(
4684 held.blocked_by.is_empty(),
4685 "hold overrides the wait, same as release"
4686 );
4687 assert!(held.block_reason.is_none());
4688
4689 let mut done = task("closed straight out of blocked");
4690 done.block(
4691 vec!["20260101-000000-dead".to_owned()],
4692 Some("waiting on the migration script".to_owned()),
4693 );
4694 done.succeed();
4695 assert_eq!(done.status, TaskStatus::Done);
4696 assert!(
4697 done.blocked_by.is_empty(),
4698 "a done task cannot still be waiting on a dependency"
4699 );
4700 assert!(done.block_reason.is_none());
4701 }
4702
4703 #[test]
4704 fn a_blocked_task_is_never_offered_to_the_loop() {
4705 let mut t = task("blocked");
4706 assert!(t.status.runnable());
4707 t.block(
4708 vec!["dep-id".to_owned()],
4709 Some("waits on dep-id".to_owned()),
4710 );
4711 assert_eq!(t.status, TaskStatus::Blocked);
4712 assert!(!t.status.runnable());
4713 assert_eq!(TaskStatus::Blocked.as_str(), "blocked");
4714 }
4715
4716 #[test]
4717 fn unblocking_the_last_dependency_returns_the_task_to_queued() {
4718 let mut t = task("blocked on two");
4719 t.block(
4720 vec!["a".to_owned(), "b".to_owned()],
4721 Some("waits on a and b".to_owned()),
4722 );
4723
4724 t.unblock("a");
4725 assert_eq!(t.status, TaskStatus::Blocked, "b is still outstanding");
4726 assert_eq!(t.blocked_by, ["b"]);
4727
4728 t.unblock("b");
4729 assert_eq!(t.status, TaskStatus::Queued);
4730 assert!(t.blocked_by.is_empty());
4731 assert!(t.block_reason.is_none());
4732 }
4733
4734 #[test]
4735 fn unblocking_an_id_on_a_task_that_is_not_blocked_is_a_no_op() {
4736 let mut t = task("never blocked");
4737 t.unblock("whatever");
4738 assert_eq!(t.status, TaskStatus::Queued);
4739 }
4740
4741 #[test]
4742 fn a_held_task_blocked_on_a_question_returns_to_held_not_queued() {
4743 let mut t = task("held, then asked about");
4748 t.hold_machine(Some("out of attempts".to_owned()));
4749 assert_eq!(t.status, TaskStatus::Held);
4750
4751 t.block(vec!["q1".to_owned()], Some("what now?".to_owned()));
4752 assert_eq!(t.status, TaskStatus::Blocked);
4753
4754 t.record_answer("what now?".to_owned(), "leave it held".to_owned());
4755 t.unblock("q1");
4756 assert_eq!(t.status, TaskStatus::Held, "must restore, not requeue");
4757 assert_eq!(t.hold_reason.as_deref(), Some("out of attempts"));
4758 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4759 assert!(t.blocked_from.is_none(), "consumed once restored");
4760 }
4761
4762 #[test]
4763 fn a_manually_held_task_blocked_on_a_question_returns_to_held() {
4764 let mut t = task("manually held, then asked about");
4765 t.hold_manual(Some("waiting on a dependency".to_owned()));
4766
4767 t.block(vec!["q1".to_owned()], None);
4768 t.unblock("q1");
4769
4770 assert_eq!(t.status, TaskStatus::Held);
4771 assert_eq!(t.hold_source, Some(HoldSource::Manual));
4772 }
4773
4774 #[test]
4775 fn re_blocking_an_already_blocked_task_keeps_the_original_blocked_from() {
4776 let mut t = task("held, blocked twice");
4780 t.hold_machine(None);
4781 t.block(vec!["q1".to_owned()], Some("first".to_owned()));
4782 t.block(
4783 vec!["q1".to_owned(), "q2".to_owned()],
4784 Some("second".to_owned()),
4785 );
4786
4787 t.unblock("q1");
4788 assert_eq!(t.status, TaskStatus::Blocked, "q2 still outstanding");
4789 t.unblock("q2");
4790 assert_eq!(t.status, TaskStatus::Held);
4791 }
4792
4793 #[test]
4794 fn unblocking_a_task_blocked_while_running_lands_on_queued_not_running() {
4795 let mut t = task("blocked mid-run");
4799 t.start("run-1".to_owned());
4800 assert_eq!(t.status, TaskStatus::Running);
4801
4802 t.block(vec!["q1".to_owned()], None);
4803 t.unblock("q1");
4804 assert_eq!(t.status, TaskStatus::Queued);
4805 }
4806
4807 #[test]
4808 fn a_pre_schema_4_blocked_record_with_hold_evidence_restores_to_held() {
4809 let mut t = task("legacy record, held before it was blocked");
4815 t.hold_source = Some(HoldSource::Machine);
4816 t.hold_reason = Some("legacy hold reason".to_owned());
4817 t.status = TaskStatus::Blocked;
4818 t.blocked_by = vec!["q1".to_owned()];
4819 t.blocked_from = None;
4820
4821 t.unblock("q1");
4822 assert_eq!(t.status, TaskStatus::Held);
4823 }
4824
4825 #[test]
4826 fn a_pre_schema_4_blocked_record_with_no_hold_evidence_restores_to_queued() {
4827 let mut t = task("legacy record, ordinary dependency block");
4828 t.status = TaskStatus::Blocked;
4829 t.blocked_by = vec!["dep".to_owned()];
4830 t.blocked_from = None;
4831
4832 t.unblock("dep");
4833 assert_eq!(t.status, TaskStatus::Queued);
4834 }
4835
4836 #[test]
4837 fn answering_a_question_is_recorded_and_survives_a_release() {
4838 let mut t = task("asked something");
4839 t.block(vec!["q1".to_owned()], Some("which backend?".to_owned()));
4840 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
4841 t.unblock("q1");
4842 assert_eq!(t.status, TaskStatus::Queued);
4843 assert_eq!(t.answers.len(), 1);
4844 assert_eq!(t.answers[0].answer, "SQLite");
4845
4846 t.release();
4850 assert_eq!(t.answers.len(), 1, "the answer is not lost on release");
4851 }
4852
4853 #[test]
4854 fn resumption_review_branch_requires_unchanged_authored_criteria() {
4855 for source in ["cli", "web"] {
4856 let mut t = task("review old candidate");
4857 t.request_review("candidate-A".into());
4858 t.hold_for_handover(Some("candidate-A".into()), "handover refused".into());
4859 let mut unchanged = t.clone();
4860 unchanged.release_operator(source, None);
4861 assert_eq!(unchanged.review_branch.as_deref(), Some("candidate-A"));
4862 t.edit(
4863 "review approved head".into(),
4864 "Reuse measured PR head B; fix Clippy; review".into(),
4865 )
4866 .unwrap();
4867 t.release_operator(source, None);
4868 assert!(t.review_branch.is_none());
4869 assert!(t.review_instruction.is_none());
4870 let mut legacy = unchanged;
4872 legacy.review_instruction = None;
4873 legacy.hold_for_handover(None, "still checked out".into());
4874 legacy
4875 .edit("new plan".into(), "Reuse measured PR head B".into())
4876 .unwrap();
4877 legacy.release_operator(source, None);
4878 assert!(legacy.review_branch.is_none());
4879 }
4880 }
4881
4882 #[test]
4883 fn a_refused_handover_keeps_the_review_branch_across_release() {
4884 let mut t = task("refused takeover");
4885 t.start("run-1".to_owned());
4886 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
4887 assert_eq!(t.status, TaskStatus::Held);
4888 assert_eq!(t.attempts, 1);
4889 t.release();
4890 assert_eq!(t.status, TaskStatus::Queued);
4891 assert_eq!(t.attempts, 0);
4892 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4893
4894 t.hold_for_handover(None, "again".to_owned());
4896 t.requeue();
4897 assert!(t.review_branch.is_none());
4898
4899 let mut m = task("manual");
4901 m.review_branch = Some("magi/x/A".to_owned());
4902 m.hold_manual(None);
4903 m.release();
4904 assert!(m.review_branch.is_none());
4905 }
4906
4907 #[test]
4908 fn requesting_review_requeues_the_task_and_remembers_the_branch() {
4909 let mut t = task("blocked run with a surviving branch");
4910 t.start("run-1".to_owned());
4911 t.fail("blocked with major findings", 5);
4912 assert_eq!(t.status, TaskStatus::Failed);
4913
4914 t.request_review("magi/eba2/A".to_owned());
4915 assert_eq!(t.status, TaskStatus::Queued);
4916 assert_eq!(t.attempts, 0);
4917 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4918
4919 t.release();
4921 assert!(t.review_branch.is_none());
4922 }
4923
4924 #[test]
4925 fn conductor_requeue_but_not_an_ordinary_release_forces_a_fresh_start() {
4926 let mut t = task("retry");
4927 t.start("run-1".to_owned());
4928 t.requeue();
4929 assert!(t.fresh_start);
4930
4931 t.release();
4932 assert!(!t.fresh_start);
4933 }
4934
4935 #[test]
4936 fn urgent_only_changes_queued_or_held_and_preserves_settings() {
4937 for status in [
4938 TaskStatus::Queued,
4939 TaskStatus::Held,
4940 TaskStatus::Running,
4941 TaskStatus::Done,
4942 TaskStatus::Failed,
4943 TaskStatus::Blocked,
4944 TaskStatus::Parked,
4945 ] {
4946 let mut t = task("urgent settings");
4947 t.status = status;
4948 t.attempts = 2;
4949 t.hold_reason = Some("waiting".into());
4950 t.priority = 7;
4951 t.interrupt = true;
4952 let before = serde_json::to_value(&t).unwrap();
4953 for urgent in [true, true, false, false] {
4954 let result = t.set_urgent(urgent);
4955 let mut expected = before.clone();
4956 if matches!(status, TaskStatus::Queued | TaskStatus::Held) {
4957 result.unwrap();
4958 expected["urgent"] = urgent.into();
4959 } else {
4960 let error = result.unwrap_err().to_string();
4961 assert!(error.contains(status.as_str()), "{error}");
4962 assert!(error.contains("queued or held"), "{error}");
4963 }
4964 assert_eq!(serde_json::to_value(&t).unwrap(), expected);
4965 }
4966 }
4967 }
4968
4969 #[test]
4970 fn priority_can_be_changed_while_queued_but_not_while_running() {
4971 let mut t = task("reprioritise me");
4972 t.set_priority(5).unwrap();
4973 assert_eq!(t.priority, 5);
4974
4975 t.start("run-1".to_owned());
4976 let err = t.set_priority(9).unwrap_err().to_string();
4977 assert!(err.contains("running"), "{err}");
4978 assert_eq!(t.priority, 5, "the rejected write must not partially apply");
4979 }
4980
4981 #[test]
4982 fn interrupt_can_be_marked_while_queued_but_not_while_running() {
4983 let mut t = task("interrupt me");
4984 assert!(!t.interrupt, "off unless asked, same as any other task");
4985
4986 t.set_interrupt(true).unwrap();
4987 assert!(t.interrupt);
4988
4989 t.start("run-1".to_owned());
4990 assert!(
4991 !t.interrupt,
4992 "the mark is one-shot: dispatching the task fulfils it, \
4993 whatever the run that follows ends up doing"
4994 );
4995 let err = t.set_interrupt(true).unwrap_err().to_string();
4996 assert!(err.contains("running"), "{err}");
4997 t.set_interrupt(false).unwrap();
5000 assert!(!t.interrupt);
5001 }
5002
5003 #[test]
5007 fn a_failed_run_does_not_leave_the_task_still_marked_to_interrupt() {
5008 let mut t = task("interrupt me");
5009 t.set_interrupt(true).unwrap();
5010 t.start("run-1".to_owned());
5011 t.fail("mock failure", 5);
5012 assert_eq!(t.status, TaskStatus::Failed);
5013 assert!(
5014 !t.interrupt,
5015 "one attempt already spent the mark; a retry is an ordinary \
5016 requeue, not a fresh interrupt request"
5017 );
5018 }
5019
5020 #[test]
5021 fn changing_priority_moves_a_task_ahead_in_the_real_queue_order() {
5022 let (_dir, q) = queue();
5023 let mut a = task("first filed");
5024 let mut b = task("second filed");
5025 a.id = "20260101-000001-aaaa".to_owned();
5026 b.id = "20260101-000002-bbbb".to_owned();
5027 q.put(&mut a).unwrap();
5028 q.put(&mut b).unwrap();
5029
5030 assert_eq!(
5031 q.next_runnable().unwrap().id,
5032 a.id,
5033 "with equal priority the older task goes first, so a burst of \
5034 new work cannot starve it"
5035 );
5036 assert_eq!(
5037 q.list()[0].id,
5038 b.id,
5039 "but the list an operator reads is newest first, the same as \
5040 before priority existed - a's turn to run does not make it the \
5041 newest task"
5042 );
5043
5044 let mut a = q.get(&a.id).unwrap();
5045 a.set_priority(10).unwrap();
5046 q.put(&mut a).unwrap();
5047
5048 assert_eq!(
5049 q.next_runnable().unwrap().id,
5050 a.id,
5051 "a raised priority must be reflected the moment it is saved"
5052 );
5053 assert_eq!(
5057 q.list()[0].id,
5058 a.id,
5059 "the raised task must sort first in the list an operator reads, \
5060 not only in next_runnable's own ordering"
5061 );
5062 }
5063
5064 #[test]
5065 fn editing_replaces_title_and_instruction_but_keeps_identity_and_history() {
5066 let mut t = Task::new(
5067 "old title".to_owned(),
5068 "old instruction".to_owned(),
5069 PathBuf::from("/repo"),
5070 Source::Agent {
5071 run: "20260101-000000-beef".to_owned(),
5072 node: "implement".to_owned(),
5073 },
5074 );
5075 let id = t.id.clone();
5076 let created_at = t.created_at;
5077 t.runs.push("20260101-000000-beef".to_owned());
5078
5079 t.edit("new title".to_owned(), "new instruction".to_owned())
5080 .unwrap();
5081
5082 assert_eq!(t.title, "new title");
5083 assert_eq!(t.instruction, "new instruction");
5084 assert_eq!(t.id, id, "editing must not mint a new id");
5085 assert_eq!(t.created_at, created_at);
5086 assert_eq!(
5087 t.source,
5088 Source::Agent {
5089 run: "20260101-000000-beef".to_owned(),
5090 node: "implement".to_owned(),
5091 },
5092 "editing must not turn agent attribution into human"
5093 );
5094 assert_eq!(t.runs, ["20260101-000000-beef"]);
5095 }
5096
5097 #[test]
5098 fn editing_is_refused_once_a_task_is_running_or_finished() {
5099 let mut running = task("in flight");
5100 running.start("run-1".to_owned());
5101 let err = running
5102 .edit("x".to_owned(), "y".to_owned())
5103 .unwrap_err()
5104 .to_string();
5105 assert!(err.contains("running"), "{err}");
5106
5107 let mut done = task("finished");
5108 done.succeed();
5109 let err = done
5110 .edit("x".to_owned(), "y".to_owned())
5111 .unwrap_err()
5112 .to_string();
5113 assert!(err.contains("done"), "{err}");
5114
5115 let mut queued = task("waiting");
5117 queued.edit("x".to_owned(), "y".to_owned()).unwrap();
5118 let mut held = task("parked");
5119 held.hold_machine(None);
5120 held.edit("x".to_owned(), "y".to_owned()).unwrap();
5121 }
5122
5123 #[test]
5124 fn a_task_recorded_without_a_hold_reason_still_reads_as_none() {
5125 let (_dir, q) = queue();
5126 let path = q.path_of("20260101-000000-aaaa");
5127 std::fs::create_dir_all(q.root()).unwrap();
5128 std::fs::write(
5129 &path,
5130 serde_json::json!({
5131 "schema": SCHEMA,
5132 "id": "20260101-000000-aaaa",
5133 "title": "from before hold reasons existed",
5134 "instruction": "from before hold reasons existed",
5135 "repo": ".",
5136 "source": { "kind": "human" },
5137 "status": "held",
5138 "created_at": Timestamp::now().to_string(),
5139 "updated_at": Timestamp::now().to_string(),
5140 })
5141 .to_string(),
5142 )
5143 .unwrap();
5144
5145 let task = q.get("20260101-000000-aaaa").expect("must still read");
5146 assert!(task.hold_reason.is_none());
5147 assert!(task.operator_held());
5148 }
5149
5150 #[test]
5151 fn a_legacy_reasoned_hold_defaults_to_operator_protection() {
5152 let (_dir, q) = queue();
5153 let path = q.path_of("20260101-000000-bbbb");
5154 std::fs::create_dir_all(q.root()).unwrap();
5155 std::fs::write(
5156 &path,
5157 serde_json::json!({
5158 "schema": 2,
5159 "id": "20260101-000000-bbbb",
5160 "title": "old manual recovery",
5161 "instruction": "old manual recovery",
5162 "repo": ".",
5163 "source": { "kind": "human" },
5164 "status": "held",
5165 "hold_reason": "active manual recovery run20260912-224242-daf5",
5166 "created_at": Timestamp::now().to_string(),
5167 "updated_at": Timestamp::now().to_string(),
5168 })
5169 .to_string(),
5170 )
5171 .unwrap();
5172
5173 let task = q.get("20260101-000000-bbbb").expect("must still read");
5174 assert_eq!(task.hold_source, None);
5175 assert!(task.operator_held());
5176 }
5177
5178 #[test]
5179 fn a_task_recorded_without_a_diagnostic_still_reads_as_none() {
5180 let (_dir, q) = queue();
5181 let path = q.path_of("20260101-000000-aaaa");
5182 std::fs::create_dir_all(q.root()).unwrap();
5183 std::fs::write(
5184 &path,
5185 serde_json::json!({
5186 "schema": SCHEMA,
5187 "id": "20260101-000000-aaaa",
5188 "title": "from before diagnostics existed",
5189 "instruction": "from before diagnostics existed",
5190 "repo": ".",
5191 "source": { "kind": "human" },
5192 "status": "held",
5193 "created_at": Timestamp::now().to_string(),
5194 "updated_at": Timestamp::now().to_string(),
5195 })
5196 .to_string(),
5197 )
5198 .unwrap();
5199
5200 let task = q.get("20260101-000000-aaaa").expect("must still read");
5201 assert!(task.diagnostic.is_none());
5202 }
5203
5204 #[test]
5205 fn a_schema_1_task_with_no_blocking_fields_still_reads() {
5206 let (_dir, q) = queue();
5210 let path = q.path_of("20260101-000000-aaaa");
5211 std::fs::create_dir_all(q.root()).unwrap();
5212 std::fs::write(
5213 &path,
5214 serde_json::json!({
5215 "schema": 1,
5216 "id": "20260101-000000-aaaa",
5217 "title": "from before blocking existed",
5218 "instruction": "from before blocking existed",
5219 "repo": ".",
5220 "source": { "kind": "human" },
5221 "status": "queued",
5222 "created_at": Timestamp::now().to_string(),
5223 "updated_at": Timestamp::now().to_string(),
5224 })
5225 .to_string(),
5226 )
5227 .unwrap();
5228
5229 let task = q.get("20260101-000000-aaaa").expect("must still read");
5230 assert!(task.blocked_by.is_empty());
5231 assert!(task.block_reason.is_none());
5232 assert!(task.answers.is_empty());
5233 assert!(task.review_branch.is_none());
5234 }
5235
5236 #[test]
5237 fn releasing_or_finishing_a_task_clears_its_stale_diagnostic() {
5238 let mut held = task("diagnosed");
5243 held.start("run-1".to_owned());
5244 held.fail("gate red", 1);
5245 held.diagnostic = Some("cargo test failed: ...".to_owned());
5246 assert_eq!(held.status, TaskStatus::Held);
5247
5248 held.release();
5249 assert!(held.diagnostic.is_none());
5250
5251 held.diagnostic = Some("cargo test failed: ...".to_owned());
5252 held.succeed();
5253 assert!(held.diagnostic.is_none());
5254 }
5255
5256 #[test]
5257 fn failing_a_task_always_clears_whatever_diagnostic_it_carried() {
5258 let mut t = task("retried");
5259 t.start("run-1".to_owned());
5260 t.diagnostic = Some("stale evidence from a previous hold".to_owned());
5261 t.fail("unrelated config error", 5);
5262 assert_eq!(t.status, TaskStatus::Failed);
5263 assert!(
5264 t.diagnostic.is_none(),
5265 "fail() must not let an old diagnostic outlive the run that produced it"
5266 );
5267 }
5268
5269 #[test]
5270 fn a_claim_is_exclusive_and_releases_on_drop() {
5271 let (_dir, q) = queue();
5272 let mut t = task("contended");
5273 q.put(&mut t).unwrap();
5274
5275 let held = q.claim(&t.id).unwrap();
5276 assert!(
5277 q.claim(&t.id).is_err(),
5278 "two daemons must not drive one task into two runs"
5279 );
5280 drop(held);
5281 assert!(q.claim(&t.id).is_ok(), "a released claim is reclaimable");
5282 }
5283
5284 #[test]
5285 fn a_round_trip_survives_disk() {
5286 let (_dir, q) = queue();
5287 let mut t = Task::new(
5288 "titled".to_owned(),
5289 "body".to_owned(),
5290 PathBuf::from("/repo"),
5291 Source::Agent {
5292 run: "20260101-000000-beef".to_owned(),
5293 node: "implement".to_owned(),
5294 },
5295 );
5296 t.priority = 3;
5297 q.put(&mut t).unwrap();
5298
5299 let back = q.get(&t.id).unwrap();
5300 assert_eq!(back.id, t.id);
5301 assert_eq!(back.priority, 3);
5302 assert_eq!(back.source.label(), "implement@beef");
5303 assert_eq!(q.get(t.short()).unwrap().id, t.id);
5305 }
5306
5307 #[test]
5308 fn an_unreadable_task_does_not_take_the_queue_down() {
5309 let (_dir, q) = queue();
5310 let mut t = task("fine");
5311 q.put(&mut t).unwrap();
5312 std::fs::write(q.root().join("broken.json"), "{ not json").unwrap();
5313
5314 let listed = q.list();
5315 assert_eq!(listed.len(), 1, "the readable task still lists");
5316 assert_eq!(listed[0].id, t.id);
5317 }
5318
5319 #[test]
5320 fn a_task_recorded_without_a_solo_field_still_reads_as_not_solo() {
5321 let (_dir, q) = queue();
5322 let path = q.path_of("20260101-000000-aaaa");
5323 std::fs::create_dir_all(q.root()).unwrap();
5324 std::fs::write(
5325 &path,
5326 serde_json::json!({
5327 "schema": SCHEMA,
5328 "id": "20260101-000000-aaaa",
5329 "title": "from before solo existed",
5330 "instruction": "from before solo existed",
5331 "repo": ".",
5332 "source": { "kind": "human" },
5333 "status": "queued",
5334 "created_at": Timestamp::now().to_string(),
5335 "updated_at": Timestamp::now().to_string(),
5336 })
5337 .to_string(),
5338 )
5339 .unwrap();
5340
5341 let task = q.get("20260101-000000-aaaa").expect("must still read");
5342 assert!(!task.solo, "a queue file with no `solo` field means false");
5343 }
5344
5345 #[test]
5346 fn a_task_recorded_without_an_urgent_field_still_reads_as_not_urgent() {
5347 let (_dir, q) = queue();
5348 let path = q.path_of("20260101-000000-bbbb");
5349 std::fs::create_dir_all(q.root()).unwrap();
5350 std::fs::write(
5351 &path,
5352 serde_json::json!({
5353 "schema": SCHEMA,
5354 "id": "20260101-000000-bbbb",
5355 "title": "from before urgent existed",
5356 "instruction": "from before urgent existed",
5357 "repo": ".",
5358 "source": { "kind": "human" },
5359 "status": "queued",
5360 "created_at": Timestamp::now().to_string(),
5361 "updated_at": Timestamp::now().to_string(),
5362 })
5363 .to_string(),
5364 )
5365 .unwrap();
5366
5367 let task = q.get("20260101-000000-bbbb").expect("must still read");
5368 assert!(
5369 !task.urgent,
5370 "a queue file with no `urgent` field means false, same as `solo`"
5371 );
5372 }
5373
5374 #[test]
5375 fn a_task_from_a_future_schema_is_refused_rather_than_guessed_at() {
5376 let (_dir, q) = queue();
5377 let mut t = task("from the future");
5378 q.put(&mut t).unwrap();
5379 let path = q.path_of(&t.id);
5380 let body = std::fs::read_to_string(&path)
5381 .unwrap()
5382 .replace(&format!("\"schema\": {SCHEMA}"), "\"schema\": 99");
5383 std::fs::write(&path, body).unwrap();
5384
5385 let err = q.get(&t.id).unwrap_err().to_string();
5386 assert!(err.contains("schema 99"), "{err}");
5387 }
5388
5389 #[test]
5390 fn revision_moves_when_the_queue_changes() {
5391 let (_dir, q) = queue();
5392 assert_eq!(q.revision(), 0, "an empty queue has no revision");
5393 let mut t = task("first");
5394 q.put(&mut t).unwrap();
5395 assert!(q.revision() > 0, "a written task moves the revision");
5396 }
5397
5398 #[test]
5399 fn revision_moves_when_deleting_an_older_task() {
5400 let (dir, q) = queue();
5401 let questions = Questions::at(dir.path().join("questions"));
5402 let mut t1 = task("older");
5403 q.put(&mut t1).unwrap();
5404 std::thread::sleep(std::time::Duration::from_millis(10));
5406 let mut t2 = task("newer");
5407 q.put(&mut t2).unwrap();
5408
5409 let rev_before = q.revision();
5410 q.remove(&t1.id, false, &questions).unwrap();
5411 let rev_after = q.revision();
5412
5413 assert_ne!(
5414 rev_before, rev_after,
5415 "deleting an older task must change the revision so other clients see the deletion"
5416 );
5417 }
5418
5419 #[test]
5420 fn removing_a_task_takes_it_out_of_the_listing() {
5421 let (dir, q) = queue();
5422 let questions = Questions::at(dir.path().join("questions"));
5423 let mut t = task("delete me");
5424 q.put(&mut t).unwrap();
5425 let removed = q.remove(t.short(), false, &questions).unwrap();
5426 assert_eq!(removed.id, t.id, "a prefix resolves before deleting");
5427 assert!(removed.released.is_empty() && removed.still_blocked.is_empty());
5428 assert!(q.list().is_empty());
5429 assert!(
5430 q.remove(&t.id, false, &questions).is_err(),
5431 "removing twice is an error"
5432 );
5433 }
5434
5435 #[test]
5436 fn removing_a_task_takes_its_stale_lock_with_it() {
5437 let (dir, q) = queue();
5438 let questions = Questions::at(dir.path().join("questions"));
5439 let mut t = task("interrupted");
5440 q.put(&mut t).unwrap();
5441
5442 let claim = q.claim(&t.id).unwrap();
5445 std::mem::forget(claim);
5446 assert!(
5447 q.claim(&t.id).is_err(),
5448 "the orphaned lock is what makes the task look claimed"
5449 );
5450
5451 let err = q.remove(&t.id, true, &questions).unwrap_err().to_string();
5453 assert!(err.contains("live daemon"), "{err}");
5454 assert!(q.get(&t.id).is_ok(), "a refused delete keeps the task");
5455
5456 q.remove(&t.id, false, &questions).unwrap();
5458 assert!(q.list().is_empty());
5459 let mut again = task("interrupted");
5460 again.id = t.id.clone();
5461 q.put(&mut again).unwrap();
5462 assert!(
5463 q.claim(&t.id).is_ok(),
5464 "a task that comes back must be claimable, which a left-behind lock would prevent"
5465 );
5466 }
5467
5468 fn notices_of(dir: &Path) -> Vec<crate::notices::Notice> {
5469 crate::notices::Notices::at(dir.join("notifications")).list()
5470 }
5471
5472 #[test]
5473 fn removing_a_sole_dependency_releases_the_dependent_without_a_hold() {
5474 let (dir, q) = queue();
5475 let questions = Questions::at(dir.path().join("questions"));
5476 let mut dep = task("dependency");
5477 q.put(&mut dep).unwrap();
5478 let mut blocked = task("waiting");
5479 blocked.block(vec![dep.id.clone()], Some("waits".to_owned()));
5480 q.put(&mut blocked).unwrap();
5481
5482 let removed = q.remove(&dep.id, false, &questions).unwrap();
5483 assert_eq!(removed.released, [blocked.id.clone()]);
5484 assert!(removed.still_blocked.is_empty());
5485
5486 let after = q.get(&blocked.id).unwrap();
5487 assert_eq!(after.status, TaskStatus::Queued);
5488 assert!(after.blocked_by.is_empty());
5489 assert!(after.block_reason.is_none());
5490 let notes = notices_of(dir.path());
5491 assert_eq!(notes.len(), 1, "{notes:?}");
5492 assert_eq!(notes[0].severity, crate::notices::Severity::Info);
5493 assert_eq!(
5494 notes[0].link,
5495 Some(crate::notices::Link::Task {
5496 id: blocked.id.clone(),
5497 })
5498 );
5499 }
5500
5501 #[test]
5502 fn removing_one_of_two_dependencies_keeps_the_other() {
5503 let (dir, q) = queue();
5504 let questions = Questions::at(dir.path().join("questions"));
5505 let mut dep = task("dependency");
5506 q.put(&mut dep).unwrap();
5507 let mut other = task("other");
5508 q.put(&mut other).unwrap();
5509 let mut blocked = task("waiting");
5510 blocked.block(
5511 vec![dep.id.clone(), other.id.clone()],
5512 Some("waits on both".to_owned()),
5513 );
5514 q.put(&mut blocked).unwrap();
5515
5516 let removed = q.remove(&dep.id, false, &questions).unwrap();
5517 assert!(removed.released.is_empty());
5518 assert_eq!(removed.still_blocked, [blocked.id.clone()]);
5519
5520 let after = q.get(&blocked.id).unwrap();
5521 assert_eq!(after.status, TaskStatus::Blocked);
5522 assert_eq!(after.blocked_by, [other.id.clone()]);
5523 assert!(missing_blockers(&q, &questions, &after.blocked_by).is_empty());
5526 }
5527
5528 #[test]
5529 fn a_dependency_deleted_before_its_dependents_were_rewritten_is_released_later() {
5530 let (dir, q) = queue();
5531 let questions = Questions::at(dir.path().join("questions"));
5532 let mut dep = task("dependency");
5533 q.put(&mut dep).unwrap();
5534 let mut blocked = task("waiting");
5535 blocked.block(vec![dep.id.clone()], None);
5536 q.put(&mut blocked).unwrap();
5537
5538 let claim = q.claim(&blocked.id).unwrap();
5540 let removed = q.remove(&dep.id, false, &questions).unwrap();
5541 assert!(removed.released.is_empty());
5542 drop(claim);
5543
5544 let mut task = q.get(&blocked.id).unwrap();
5545 assert_eq!(task.status, TaskStatus::Blocked);
5546 assert!(missing_blockers(&q, &questions, &task.blocked_by).is_empty());
5547 assert_eq!(
5548 q.apply_deleted_blockers(&mut task, &questions),
5549 [dep.id.clone()]
5550 );
5551 assert_eq!(task.status, TaskStatus::Queued);
5552 }
5553
5554 #[test]
5555 fn dependency_resolution_short_deleted_reference_releases_only_its_waiters() {
5556 let (dir, q) = queue();
5557 let questions = Questions::at(dir.path().join("questions"));
5558 let mut dep = task("dependency");
5559 dep.id = "20261013-193611-ca5b".into();
5560 q.put(&mut dep).unwrap();
5561 let mut waiting = task("waiting");
5562 waiting.block(vec!["ca5b".into()], None);
5563 q.put(&mut waiting).unwrap();
5564 q.remove(&dep.id, false, &questions).unwrap();
5565 assert_eq!(q.get(&waiting.id).unwrap().status, TaskStatus::Queued);
5566 assert_eq!(
5567 dependency(&q, &questions, "ca5b"),
5568 Dependency::Deleted(dep.id)
5569 );
5570 }
5571
5572 #[test]
5573 fn dependency_resolution_preserves_cross_store_ambiguity_and_exact_ids() {
5574 let (dir, queue) = queue();
5575 let questions = Questions::at(dir.path().join("questions"));
5576 let mut dep = task("dependency");
5577 dep.id = "20261013-193611-ca5b".into();
5578 queue.put(&mut dep).unwrap();
5579 std::fs::create_dir_all(questions.root()).unwrap();
5580 let question_id = "20261014-193611-ca5b";
5581 std::fs::write(questions.path_of(question_id), "unreadable question").unwrap();
5582 assert_eq!(questions.resolve_id("ca5b").unwrap(), question_id);
5583 assert_eq!(
5584 dependency(&queue, &questions, "ca5b"),
5585 Dependency::Ambiguous
5586 );
5587 assert_eq!(
5588 dependency(&queue, &questions, &dep.id),
5589 Dependency::Task(dep.id)
5590 );
5591 assert_eq!(
5592 dependency(&queue, &questions, question_id),
5593 Dependency::Question(question_id.into())
5594 );
5595 assert_eq!(
5596 dependency(&queue, &questions, "missing"),
5597 Dependency::Missing
5598 );
5599 }
5600
5601 #[test]
5602 fn dependency_resolution_directory_errors_are_unknown_not_missing() {
5603 let dir = tempfile::tempdir().unwrap();
5604 let q = Queue::at(dir.path().join("not-a-directory"));
5605 std::fs::write(q.root(), "file").unwrap();
5606 let questions = Questions::at(dir.path().join("questions"));
5607 assert_eq!(dependency(&q, &questions, "ca5b"), Dependency::Unknown);
5608 assert!(missing_blockers(&q, &questions, &["ca5b".into()]).is_empty());
5609 }
5610
5611 #[test]
5612 fn a_dependency_without_a_tombstone_is_still_missing() {
5613 let (dir, q) = queue();
5614 let questions = Questions::at(dir.path().join("questions"));
5615 let mut dep = task("dependency");
5616 q.put(&mut dep).unwrap();
5617 std::fs::remove_file(q.path_of(&dep.id)).unwrap();
5618 assert_eq!(
5619 missing_blockers(&q, &questions, std::slice::from_ref(&dep.id)),
5620 [dep.id.clone()]
5621 );
5622 assert!(deleted_blockers(&q, &questions, std::slice::from_ref(&dep.id)).is_empty());
5623 }
5624
5625 #[test]
5626 fn a_held_dependent_is_left_alone_by_a_dependency_deletion() {
5627 let mut t = task("held");
5628 t.hold_machine(Some("because".to_owned()));
5629 t.blocked_by = vec!["gone".to_owned()];
5630 assert!(!t.dependency_deleted("gone"));
5631 assert_eq!(t.status, TaskStatus::Held);
5632 }
5633
5634 #[test]
5635 fn a_failed_record_removal_takes_the_tombstone_back() {
5636 let (_dir, q) = queue();
5637 let mut t = task("stays");
5638 q.put(&mut t).unwrap();
5639 q.write_tombstone(&t.id).unwrap();
5640 let err = q.remove_record_with_attachments(&t.id, |_| Err(std::io::Error::other("nope")));
5641 assert!(err.is_err());
5642 assert!(!q.was_deleted(&t.id));
5645 }
5646
5647 fn source_file(dir: &Path, name: &str, body: &str) -> PathBuf {
5648 let p = dir.join(name);
5649 std::fs::write(&p, body).unwrap();
5650 p
5651 }
5652
5653 #[test]
5654 fn an_attachment_copy_survives_deleting_its_source() {
5655 let (dir, q) = queue();
5656 let src = source_file(dir.path(), "shot.png", "pixels");
5657 let mut t = task("with a picture");
5658 let names = q.attach(&mut t, std::slice::from_ref(&src)).unwrap();
5659 q.put(&mut t).unwrap();
5660 std::fs::remove_file(&src).unwrap();
5661 assert_eq!(names, ["shot.png"]);
5662 let loaded = q.get(&t.id).unwrap();
5663 let paths = q.attachment_paths(&loaded);
5664 assert_eq!(paths.len(), 1);
5665 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "pixels");
5666 }
5667
5668 #[test]
5669 fn attachment_names_that_could_traverse_or_are_odd_are_refused() {
5670 let (dir, q) = queue();
5671 let mut t = task("bad names");
5672 for name in ["a..b.png", ".hidden", "with space.png", "-x.png"] {
5673 let src = source_file(dir.path(), name, "x");
5674 assert!(
5675 q.attach(&mut t, &[src]).is_err(),
5676 "`{name}` must be refused"
5677 );
5678 }
5679 assert!(!crate::ask::valid_asset_name("C:foo.png"));
5683 #[cfg(not(windows))]
5684 {
5685 let src = source_file(dir.path(), "C:foo.png", "x");
5686 assert!(q.attach(&mut t, &[src]).is_err());
5687 }
5688 let long = format!("{}.png", "a".repeat(70));
5689 let src = source_file(dir.path(), &long, "x");
5690 assert!(q.attach(&mut t, &[src]).is_err());
5691 assert!(t.attachments.is_empty());
5692 assert!(!q.attachments_dir(&t.id).exists());
5693 }
5694
5695 #[test]
5696 fn a_taken_attachment_name_is_numbered_not_overwritten() {
5697 let (dir, q) = queue();
5698 let a = source_file(dir.path(), "shot.png", "one");
5699 let sub = dir.path().join("other");
5700 std::fs::create_dir_all(&sub).unwrap();
5701 let b = source_file(&sub, "shot.png", "two");
5702 let mut t = task("collision");
5703 q.attach(&mut t, &[a]).unwrap();
5704 q.attach(&mut t, &[b]).unwrap();
5705 assert_eq!(t.attachments, ["shot.png", "shot-2.png"]);
5706 let paths = q.attachment_paths(&t);
5707 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "one");
5708 assert_eq!(std::fs::read_to_string(&paths[1]).unwrap(), "two");
5709 }
5710
5711 #[test]
5712 fn a_renumbered_name_stays_inside_the_length_bound() {
5713 let (dir, q) = queue();
5714 let name = format!("{}.png", "a".repeat(60));
5715 assert_eq!(name.len(), 64);
5716 let a = source_file(dir.path(), &name, "one");
5717 let sub = dir.path().join("other");
5718 std::fs::create_dir_all(&sub).unwrap();
5719 let b = source_file(&sub, &name, "two");
5720 let mut t = task("long");
5721 q.attach(&mut t, &[a, b]).unwrap();
5722 assert_eq!(t.attachments.len(), 2);
5723 assert!(
5724 t.attachments
5725 .iter()
5726 .all(|n| crate::ask::valid_asset_name(n))
5727 );
5728 assert!(t.attachments[1].ends_with("-2.png"));
5729 }
5730
5731 #[test]
5732 fn a_failed_attach_keeps_existing_attachments_and_leaves_no_partial_copy() {
5733 let (dir, q) = queue();
5734 let good = source_file(dir.path(), "good.png", "ok");
5735 let mut t = task("partial");
5736 q.attach(&mut t, &[good]).unwrap();
5737 let more = source_file(dir.path(), "more.png", "ok");
5738 let missing = dir.path().join("missing.png");
5739 assert!(q.attach(&mut t, &[more, missing]).is_err());
5740 assert_eq!(t.attachments, ["good.png"]);
5741 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5742 .unwrap()
5743 .flatten()
5744 .collect();
5745 assert_eq!(on_disk.len(), 1);
5746 }
5747
5748 fn block_put(q: &Queue, t: &Task) -> PathBuf {
5750 let tmp = q.path_of(&t.id).with_extension("json.tmp");
5751 std::fs::create_dir_all(&tmp).unwrap();
5752 tmp
5753 }
5754
5755 #[test]
5756 fn a_failed_put_leaves_no_new_attachment_directory() {
5757 let (dir, q) = queue();
5758 let mut t = task("fresh");
5759 let tmp = block_put(&q, &t);
5760 let src = source_file(dir.path(), "shot.png", "x");
5761 assert!(q.attach_and_put(&mut t, &[src]).is_err());
5762 assert!(t.attachments.is_empty());
5763 assert!(!q.attachments_dir(&t.id).exists());
5764 assert!(!q.path_of(&t.id).exists());
5765 std::fs::remove_dir(tmp).unwrap();
5766 }
5767
5768 #[test]
5769 fn a_failed_put_removes_only_the_copy_it_just_made() {
5770 let (dir, q) = queue();
5771 let mut t = task("edited");
5772 let first = source_file(dir.path(), "first.png", "1");
5773 q.attach_and_put(&mut t, &[first]).unwrap();
5774 block_put(&q, &t);
5775 let second = source_file(dir.path(), "second.png", "2");
5776 assert!(q.attach_and_put(&mut t, &[second]).is_err());
5777 assert_eq!(t.attachments, ["first.png"]);
5778 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5779 .unwrap()
5780 .flatten()
5781 .map(|e| e.file_name().to_string_lossy().into_owned())
5782 .collect();
5783 assert_eq!(on_disk, ["first.png"]);
5784 assert_eq!(q.get(&t.id).unwrap().attachments, ["first.png"]);
5785 }
5786
5787 #[test]
5788 fn a_leftover_removing_directory_is_swept_by_the_next_removal() {
5789 let (dir, q) = queue();
5790 let questions = Questions::at(dir.path().join("questions"));
5791 let gone = task("gone");
5792 let mut other = task("other");
5793 let mut live = task("live");
5794 q.put(&mut other).unwrap();
5795 q.put(&mut live).unwrap();
5796 let orphan = q.root.join(format!("{}.attachments.removing", gone.id));
5799 std::fs::create_dir_all(&orphan).unwrap();
5800 std::fs::write(orphan.join("shot.png"), "x").unwrap();
5801 let busy = q.root.join(format!("{}.attachments.removing", live.id));
5804 std::fs::create_dir_all(&busy).unwrap();
5805
5806 q.remove(&other.id, false, &questions).unwrap();
5807 assert!(!orphan.exists(), "an orphan is swept");
5808 assert!(busy.exists(), "a removal in progress is left alone");
5809 }
5810
5811 #[test]
5812 fn a_blocked_aside_rename_fails_the_removal_and_loses_nothing() {
5813 let (dir, q) = queue();
5814 let questions = Questions::at(dir.path().join("questions"));
5815 let mut t = task("stuck");
5816 let src = source_file(dir.path(), "shot.png", "x");
5817 q.attach_and_put(&mut t, &[src]).unwrap();
5818 let aside = q.root.join(format!("{}.attachments.removing", t.id));
5821 std::fs::create_dir_all(&aside).unwrap();
5822 std::fs::write(aside.join("old.png"), "o").unwrap();
5823 assert!(q.remove(&t.id, false, &questions).is_err());
5824 assert!(q.path_of(&t.id).exists());
5825 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5826 }
5827
5828 #[test]
5829 fn a_failed_record_removal_puts_the_attachments_back() {
5830 let (dir, q) = queue();
5831 let mut t = task("rollback");
5832 let src = source_file(dir.path(), "shot.png", "x");
5833 q.attach_and_put(&mut t, &[src]).unwrap();
5834 let err = q
5835 .remove_record_with_attachments(&t.id, |_| {
5836 Err(std::io::Error::other("injected failure"))
5837 })
5838 .unwrap_err();
5839 assert!(format!("{err:#}").contains("injected failure"));
5840 assert!(q.path_of(&t.id).exists());
5841 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5842 assert!(
5843 !q.root
5844 .join(format!("{}.attachments.removing", t.id))
5845 .exists()
5846 );
5847 }
5848
5849 #[test]
5850 fn editing_a_task_keeps_its_attachments() {
5851 let (dir, q) = queue();
5852 let src = source_file(dir.path(), "shot.png", "x");
5853 let mut t = task("editable");
5854 q.attach(&mut t, &[src]).unwrap();
5855 t.edit("new".to_owned(), "new text".to_owned()).unwrap();
5856 q.put(&mut t).unwrap();
5857 assert_eq!(q.get(&t.id).unwrap().attachments, ["shot.png"]);
5858 }
5859
5860 #[test]
5861 fn removing_a_task_deletes_its_attachments() {
5862 let (dir, q) = queue();
5863 let questions = Questions::at(dir.path().join("questions"));
5864 let src = source_file(dir.path(), "shot.png", "x");
5865 let mut t = task("doomed");
5866 q.attach(&mut t, &[src]).unwrap();
5867 q.put(&mut t).unwrap();
5868 assert!(q.attachments_dir(&t.id).is_dir());
5869 q.remove(&t.id, false, &questions).unwrap();
5870 assert!(!q.attachments_dir(&t.id).exists());
5871 assert!(q.list().is_empty());
5872 }
5873
5874 #[test]
5875 fn a_task_written_before_attachments_still_reads() {
5876 let (_dir, q) = queue();
5877 let mut t = task("old");
5878 q.put(&mut t).unwrap();
5879 let path = q.path_of(&t.id);
5880 let mut v: serde_json::Value =
5881 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5882 v.as_object_mut().unwrap().remove("attachments");
5883 std::fs::write(&path, v.to_string()).unwrap();
5884 assert!(q.get(&t.id).unwrap().attachments.is_empty());
5885 }
5886
5887 #[test]
5888 fn attachment_paths_are_absolute_even_when_the_root_is_relative() {
5889 let q = Queue::at(PathBuf::from("relative-queue"));
5890 let mut t = task("rel");
5891 t.attachments.push("shot.png".to_owned());
5892 let paths = q.attachment_paths(&t);
5893 assert!(paths[0].is_absolute(), "{}", paths[0].display());
5894 assert!(paths[0].ends_with(format!("{}.attachments/shot.png", t.id)));
5895 }
5896
5897 fn upgrade_parked(id: &str, started: &str) -> Task {
5898 let mut t = task(id);
5899 t.id = id.into();
5900 t.start(format!("run-{id}"));
5901 t.park("upgrade");
5902 t.record_pass_at(
5903 &format!("run-{id}"),
5904 0,
5905 Some(started.parse().unwrap()),
5906 1,
5907 Some(crate::run::ParkCause::Upgrade {
5908 from: "old".into(),
5909 to: None,
5910 }),
5911 );
5912 t
5913 }
5914
5915 #[test]
5916 fn upgrade_resume_order_follows_dispatch_claim_not_graph_start() {
5917 let stamp = |id: &str, dispatched: &str, started: &str| {
5919 let mut t = task(id);
5920 t.id = id.into();
5921 t.start(format!("run-{id}"));
5922 t.park("upgrade");
5923 t.dispatched_at = Some(dispatched.parse().unwrap());
5924 t.record_pass_at(
5925 &format!("run-{id}"),
5926 0,
5927 Some(started.parse().unwrap()),
5928 1,
5929 Some(crate::run::ParkCause::Upgrade {
5930 from: "old".into(),
5931 to: None,
5932 }),
5933 );
5934 t
5935 };
5936 let a = stamp("a", "2026-10-22T00:00:01Z", "2026-10-22T00:00:09Z");
5937 let b = stamp("b", "2026-10-22T00:00:02Z", "2026-10-22T00:00:03Z");
5938 assert!(dispatch_order(&a, &b).is_lt());
5939 }
5940
5941 #[test]
5942 fn upgrade_resume_order_survives_queue_reload_and_reversed_settlement() {
5943 let (_dir, q) = queue();
5944 let mut later = upgrade_parked("later", "2026-10-22T00:00:02Z");
5945 let mut earlier = upgrade_parked("earlier", "2026-10-22T00:00:01Z");
5946 earlier.priority = -10;
5947 let mut fresh = task("fresh");
5948 fresh.id = "newer".into();
5949 fresh.priority = 100;
5950 for t in [&mut fresh, &mut later, &mut earlier] {
5951 q.put(t).unwrap();
5952 }
5953 let reloaded = Queue::at(q.root().to_owned());
5954 let mut tasks = reloaded.list();
5955 tasks.sort_unstable_by(dispatch_order);
5956 assert_eq!(
5957 tasks.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5958 vec!["earlier", "later", "newer"]
5959 );
5960 assert_eq!(reloaded.next_runnable().unwrap().id, "earlier");
5961 assert_eq!(reloaded.get("earlier").unwrap().attempts, 0);
5962 }
5963
5964 #[test]
5965 fn upgrade_resume_legacy_order_uses_settlement_and_id() {
5966 let mut a = upgrade_parked("a", "2026-10-22T00:00:02Z");
5967 let mut b = upgrade_parked("b", "2026-10-22T00:00:01Z");
5968 for t in [&mut a, &mut b] {
5969 let mut value = serde_json::to_value(&*t).unwrap();
5970 value["passes"][0]
5971 .as_object_mut()
5972 .unwrap()
5973 .remove("started_at");
5974 *t = serde_json::from_value(value).unwrap();
5975 }
5976 assert!(dispatch_order(&a, &b).is_lt());
5977 b.passes[0].settled_at = a.passes[0].settled_at;
5978 assert!(dispatch_order(&a, &b).is_lt());
5979 }
5980
5981 #[test]
5982 fn upgrade_resume_key_excludes_other_parks_and_explicit_task_changes() {
5983 let original = upgrade_parked("a", "2026-10-22T00:00:01Z");
5984 assert!(original.upgrade_resume_key().is_some());
5985 for cause in [
5986 crate::run::ParkCause::Operator,
5987 crate::run::ParkCause::Question,
5988 crate::run::ParkCause::Interrupt {
5989 reason: "interrupt".into(),
5990 },
5991 crate::run::ParkCause::Unspecified,
5992 ] {
5993 let mut t = original.clone();
5994 t.passes[0].parked = Some(cause);
5995 assert!(t.upgrade_resume_key().is_none());
5996 let mut fresh = task("fresh");
5997 fresh.priority = 1;
5998 assert!(dispatch_order(&fresh, &t).is_lt());
5999 }
6000 let mut t = original.clone();
6001 t.release();
6002 assert!(t.upgrade_resume_key().is_none());
6003 t.edit("edited".into(), "new instruction".into()).unwrap();
6004 assert!(t.upgrade_resume_key().is_none());
6005 let mut t = original.clone();
6006 t.requeue();
6007 t.status = TaskStatus::Parked;
6008 assert!(t.upgrade_resume_key().is_none());
6009 let mut t = original.clone();
6010 t.start("different-run".into());
6011 t.park("no upgrade");
6012 assert!(t.upgrade_resume_key().is_none());
6013 }
6014
6015 #[test]
6016 fn explicitly_resumed_older_run_becomes_current_without_duplicate_history() {
6017 let dir = tempfile::tempdir().unwrap();
6018 let q = Queue::at(dir.path().join("queue"));
6019 let mut t = task("resume");
6020 t.start("A".into());
6021 t.record_pass("A", 0, 1, None);
6022 t.start("B".into());
6023 q.put(&mut t).unwrap();
6024 assert_eq!(t.runs, vec!["A", "B"]);
6025 t.start("A".into());
6026 t.record_pass("A", 1, 3, None);
6027 q.put(&mut t).unwrap();
6028 let saved = q.get(&t.id).unwrap();
6029 assert_eq!(saved.runs, vec!["B", "A"]);
6030 assert_eq!(saved.runs.last().map(String::as_str), Some("A"));
6031 assert_eq!(saved.attempts, 3);
6032 assert_eq!(saved.passes.len(), 2);
6033 assert_eq!(saved.passes[0].run, "A");
6034 assert_eq!(saved.passes[1].run, "A");
6035 }
6036
6037 #[test]
6038 fn repeated_start_records_one_run_and_separate_passes() {
6039 let mut t = task("resume");
6040 t.start("same-run".into());
6041 t.record_pass("same-run", 0, 1, None);
6042 t.start("same-run".into());
6043 t.record_pass("same-run", 1, 2, None);
6044 assert_eq!(t.runs, vec!["same-run"]);
6045 assert_eq!(t.attempts, 2);
6046 assert_eq!(t.passes.len(), 2);
6047 }
6048
6049 #[test]
6050 fn link_run_adds_a_run_once_and_touches_nothing_else() {
6051 let dir = tempfile::tempdir().unwrap();
6052 let queue = Queue::at(dir.path().join("queue"));
6053 let mut t = Task::new(
6054 "t".to_owned(),
6055 "do it".to_owned(),
6056 PathBuf::from("."),
6057 Source::Human,
6058 );
6059 queue.put(&mut t).unwrap();
6060 let before = queue.get(&t.id).unwrap();
6061
6062 let linked = queue.link_run(&t.id[..4], "20260930-092817-ec34").unwrap();
6063 assert_eq!(linked.runs, vec!["20260930-092817-ec34".to_owned()]);
6064 assert_eq!(linked.status, before.status);
6065 assert_eq!(linked.attempts, before.attempts);
6066 assert_eq!(linked.interrupt, before.interrupt);
6067
6068 let again = queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
6069 assert_eq!(again.runs.len(), 1, "linking twice must not duplicate");
6070 assert_eq!(queue.get(&t.id).unwrap().runs.len(), 1);
6071 assert!(queue.link_run("no-such-task", "r").is_err());
6072 }
6073
6074 #[test]
6075 fn put_keeps_a_run_linked_after_the_writer_took_its_snapshot() {
6076 let dir = tempfile::tempdir().unwrap();
6077 let queue = Queue::at(dir.path().join("queue"));
6078 let mut t = Task::new(
6079 "t".to_owned(),
6080 "do it".to_owned(),
6081 PathBuf::from("."),
6082 Source::Human,
6083 );
6084 queue.put(&mut t).unwrap();
6085 let mut snapshot = queue.get(&t.id).unwrap();
6087 queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
6088
6089 snapshot.start("20260930-000000-aaaa".to_owned());
6090 queue.put(&mut snapshot).unwrap();
6091
6092 let stored = queue.get(&t.id).unwrap();
6093 assert!(stored.runs.contains(&"20260930-092817-ec34".to_owned()));
6094 assert!(stored.runs.contains(&"20260930-000000-aaaa".to_owned()));
6095 assert_eq!(stored.attempts, 1);
6096 }
6097
6098 #[test]
6099 fn concurrent_links_and_daemon_saves_lose_nothing() {
6100 let dir = tempfile::tempdir().unwrap();
6101 let queue = Queue::at(dir.path().join("queue"));
6102 let mut t = Task::new(
6103 "t".to_owned(),
6104 "do it".to_owned(),
6105 PathBuf::from("."),
6106 Source::Human,
6107 );
6108 queue.put(&mut t).unwrap();
6109 let id = t.id.clone();
6110
6111 let linkers: Vec<_> = (0..4)
6112 .map(|n| {
6113 let (queue, id) = (queue.clone(), id.clone());
6114 std::thread::spawn(move || {
6115 for k in 0..10 {
6116 queue
6117 .link_run(&id, &format!("20260930-00000{n}-l{k:03}"))
6118 .unwrap();
6119 }
6120 })
6121 })
6122 .collect();
6123 let mut mine = queue.get(&id).unwrap();
6126 for k in 0..10 {
6127 mine.start(format!("20260930-000009-d{k:03}"));
6128 queue.put(&mut mine).unwrap();
6129 }
6130 for l in linkers {
6131 l.join().unwrap();
6132 }
6133
6134 let stored = queue.get(&id).unwrap();
6135 assert_eq!(stored.runs.len(), 50, "{:?}", stored.runs);
6136 assert_eq!(
6137 stored.attempts, 10,
6138 "linking never rewinds the daemon's work"
6139 );
6140 }
6141
6142 fn age_lock(path: &Path) {
6143 let f = std::fs::OpenOptions::new().write(true).open(path).unwrap();
6144 f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(60))
6145 .unwrap();
6146 }
6147
6148 #[test]
6149 fn concurrent_stale_takeover_yields_one_holder() {
6150 use std::sync::atomic::{AtomicUsize, Ordering};
6151 use std::sync::{Arc, Barrier};
6152 for _ in 0..5 {
6153 let dir = tempfile::tempdir().unwrap();
6154 let q = Queue::at(dir.path().to_path_buf());
6155 let lock = dir.path().join("t.write-lock");
6156 std::fs::write(&lock, "dead-0000").unwrap();
6157 age_lock(&lock);
6158 let n = 6;
6159 let barrier = Arc::new(Barrier::new(n));
6160 let (now, max) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
6161 let handles: Vec<_> = (0..n)
6162 .map(|_| {
6163 let (q, b, now, max) = (q.clone(), barrier.clone(), now.clone(), max.clone());
6164 std::thread::spawn(move || {
6165 b.wait();
6166 let g = q.lock_task("t").unwrap();
6167 let held = now.fetch_add(1, Ordering::SeqCst) + 1;
6168 max.fetch_max(held, Ordering::SeqCst);
6169 std::thread::sleep(std::time::Duration::from_millis(20));
6170 now.fetch_sub(1, Ordering::SeqCst);
6171 drop(g);
6172 })
6173 })
6174 .collect();
6175 for h in handles {
6176 h.join().unwrap();
6177 }
6178 assert_eq!(max.load(Ordering::SeqCst), 1);
6179 assert!(!lock.exists());
6180 }
6181 }
6182
6183 #[test]
6184 fn dropping_a_replaced_lock_leaves_the_new_holders_token() {
6185 let dir = tempfile::tempdir().unwrap();
6186 let q = Queue::at(dir.path().to_path_buf());
6187 let lock = dir.path().join("t.write-lock");
6188 let a = q.lock_task("t").unwrap();
6189 let replacement = owner_token();
6192 std::fs::write(&lock, &replacement).unwrap();
6193 drop(a);
6194 assert_eq!(std::fs::read_to_string(&lock).unwrap(), replacement);
6195 }
6196
6197 #[test]
6198 fn aged_live_task_lock_excludes_updates_until_retirement_releases_it() {
6199 let dir = tempfile::tempdir().unwrap();
6200 let q = Queue::at(dir.path().to_path_buf());
6201 let held = q.lock_task("t").unwrap();
6202 let path = dir.path().join("t.write-lock");
6203 age_lock(&path);
6204 assert!(!break_stale(&path, &held.token));
6205 let (started_tx, started_rx) = std::sync::mpsc::channel();
6206 let (acquired_tx, acquired_rx) = std::sync::mpsc::channel();
6207 let updater = std::thread::spawn(move || {
6208 started_tx.send(()).unwrap();
6209 let _write = q.lock_task("t").unwrap();
6210 acquired_tx.send(()).unwrap();
6211 });
6212 started_rx.recv().unwrap();
6213 assert!(
6214 acquired_rx
6215 .recv_timeout(std::time::Duration::from_millis(100))
6216 .is_err()
6217 );
6218 assert_eq!(std::fs::read_to_string(&path).unwrap(), held.token);
6219 drop(held);
6220 acquired_rx
6221 .recv_timeout(std::time::Duration::from_secs(2))
6222 .unwrap();
6223 updater.join().unwrap();
6224 assert!(!path.exists());
6225 }
6226
6227 #[test]
6228 fn break_stale_leaves_a_lock_that_replaced_the_one_judged() {
6229 let dir = tempfile::tempdir().unwrap();
6230 let lock = dir.path().join("t.write-lock");
6231 std::fs::write(&lock, "new-token").unwrap();
6232 assert!(!break_stale(&lock, "old-token"));
6233 assert_eq!(std::fs::read_to_string(&lock).unwrap(), "new-token");
6234 assert!(!break_stale(&lock, "new-token"));
6236 assert!(lock.exists());
6237 age_lock(&lock);
6238 assert!(break_stale(&lock, "new-token"));
6239 assert!(!lock.exists());
6240 }
6241
6242 #[test]
6243 fn a_stale_break_marker_is_recovered_and_a_fresh_one_is_respected() {
6244 let dir = tempfile::tempdir().unwrap();
6245 let lock = dir.path().join("t.write-lock");
6246 std::fs::write(&lock, "dead-1").unwrap();
6247 age_lock(&lock);
6248 let marker = break_marker(&lock, "dead-1");
6249 std::fs::write(&marker, "crashed-remover").unwrap();
6250 assert!(!break_stale(&lock, "dead-1"));
6252 assert!(lock.exists());
6253 age_lock(&marker);
6255 assert!(break_stale(&lock, "dead-1"));
6256 assert!(!lock.exists());
6257 assert!(!marker.exists());
6258 }
6259
6260 fn chat_task() -> Task {
6261 Task::new(
6262 "t".to_owned(),
6263 "do t".to_owned(),
6264 PathBuf::from("."),
6265 Source::Agent {
6266 run: "20260902-000000-beef".to_owned(),
6267 node: CHAT_NODE.to_owned(),
6268 },
6269 )
6270 }
6271
6272 #[test]
6273 fn a_blocked_hold_keeps_its_anchor_but_a_new_run_starts_a_new_hold() {
6274 let mut t = chat_task();
6275 t.hold_machine(Some("waiting".into()));
6276 let old: Timestamp = "2020-01-01T00:00:00Z".parse().unwrap();
6277 t.held_at = Some(old);
6278 t.block(vec!["decision".into()], None);
6279 t.hold_machine(Some("same waiting decision".into()));
6280 assert_eq!(t.held_at, Some(old));
6281 t.start("real-new-run".into());
6282 t.fail("new failure", 1);
6283 assert_ne!(
6284 t.held_at,
6285 Some(old),
6286 "old answers cannot approve a new hold"
6287 );
6288 }
6289
6290 #[test]
6291 fn legacy_delivery_survives_upgrade_and_a_no_progress_run_link() {
6292 let (_dir, queue) = queue();
6293 let mut t = chat_task();
6294 t.start("old-run".into());
6295 t.hold_machine(Some("operator is considering recovery".into()));
6296 t.chat_notice = Some(ChatNotice {
6297 key: format!("held#{}", t.held_at.unwrap()),
6298 at: Timestamp::now(),
6299 outcome: ChatNoticeOutcome::Posted,
6300 });
6301 let mut raw = serde_json::to_value(&t).unwrap();
6302 raw.as_object_mut().unwrap().remove("chat_notices");
6303 raw.as_object_mut().unwrap().remove("hold_notice_cause");
6304 std::fs::create_dir_all(queue.path_of(&t.id).parent().unwrap()).unwrap();
6305 std::fs::write(queue.path_of(&t.id), serde_json::to_string(&raw).unwrap()).unwrap();
6306 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
6307 queue
6308 .modify(&t.id, |task| {
6309 task.requeue();
6310 task.link_run("unprepared-run");
6311 task.startup_notice_cause("startup-source");
6312 task.fail("same unresolved preflight", 0);
6313 true
6314 })
6315 .unwrap();
6316 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
6317 queue
6318 .modify(&t.id, |task| {
6319 task.succeed();
6320 true
6321 })
6322 .unwrap();
6323 assert_eq!(
6324 queue.get(&t.id).unwrap().chat_notice_due().as_deref(),
6325 Some("done")
6326 );
6327 }
6328
6329 #[test]
6330 fn a_dependency_wait_does_not_change_when_released_and_blocked_again() {
6331 let mut t = chat_task();
6332 t.block(vec!["a".into(), "b".into()], Some("waiting".into()));
6333 let key = t.chat_notice_key().unwrap();
6334 t.chat_notices.push(key);
6335 t.release();
6336 t.block(
6337 vec!["b".into(), "a".into(), "a".into()],
6338 Some("still waiting on prerequisites".into()),
6339 );
6340 assert_eq!(t.chat_notice_due(), None);
6341 t.block(
6342 vec!["new-question".into()],
6343 Some("different decision".into()),
6344 );
6345 assert!(t.chat_notice_due().is_some());
6346 }
6347
6348 #[test]
6349 fn running_startup_failures_preserve_stage_and_report_distinct_blockers() {
6350 let mut t = chat_task();
6351 t.start("existing-run".into());
6352 t.fail("implementation failed", 1);
6353 let failed = t.chat_notice_key().unwrap();
6354 t.chat_notices.push(failed.clone());
6355 t.release_operator("test", None);
6356 t.start("existing-run".into());
6357 t.fail_startup("startup-source", "same unresolved recovery source", 1);
6358 assert_eq!(
6359 t.chat_notice_due(),
6360 None,
6361 "no-progress recovery stays quiet"
6362 );
6363 t.release_operator("test", None);
6364 t.start("existing-run".into());
6365 t.fail_startup("startup-attachments", "attachment missing", 1);
6366 let attachments = t.chat_notice_due().expect("new attachment blocker");
6367 assert_ne!(attachments, failed);
6368 assert_eq!(
6369 t.hold_notice_cause.as_ref().unwrap().stage,
6370 "startup-attachments"
6371 );
6372 t.chat_notices.push(attachments);
6373 t.release_operator("test", None);
6374 t.start("existing-run".into());
6375 t.fail_startup(
6376 "startup-attachments",
6377 "same missing attachment, reworded",
6378 1,
6379 );
6380 assert_eq!(
6381 t.chat_notice_due(),
6382 None,
6383 "same stage after attempt counter reset"
6384 );
6385 t.release_operator("test", None);
6386 t.start("existing-run".into());
6387 t.fail_startup("startup-config", "different preflight stage", 1);
6388 assert!(t.chat_notice_due().is_some());
6389 assert_eq!(
6390 t.hold_notice_cause.as_ref().unwrap().stage,
6391 "startup-config"
6392 );
6393 }
6394
6395 #[test]
6396 fn structured_causes_preserve_no_progress_retries_and_report_new_passes() {
6397 let mut t = chat_task();
6398 t.hold_machine(Some("startup source missing".into()));
6399 t.startup_notice_cause("startup-source");
6400 let first = t.chat_notice_key().unwrap();
6401 t.chat_notices.push(first.clone());
6402 t.release_operator("test", None);
6403 t.startup_notice_cause("startup-source");
6404 t.hold_machine(Some("same source cannot be read".into()));
6405 assert_eq!(t.chat_notice_due(), None);
6406 t.startup_notice_cause("startup-config");
6407 assert!(t.chat_notice_due().is_some(), "different startup stage");
6408 t.notice_cause("failed", Some("existing-run"), Some(0));
6409 let run_failure = t.chat_notice_key().unwrap();
6410 t.chat_notices.push(run_failure.clone());
6411 t.release_operator("test", None);
6412 t.startup_notice_cause("startup-source");
6413 t.hold_machine(Some("no new work happened".into()));
6414 assert_eq!(t.chat_notice_due(), None);
6415 t.notice_cause("failed", Some("existing-run"), Some(1));
6416 assert!(t.chat_notice_due().is_some(), "a real resumed pass failed");
6417 assert_ne!(t.chat_notice_key().unwrap(), run_failure);
6418 }
6419
6420 #[test]
6421 fn legacy_hold_baseline_does_not_replay_or_hide_new_failures_and_completion() {
6422 let mut t = chat_task();
6423 t.hold_machine(Some("still waiting".into()));
6424 t.chat_notice = Some(ChatNotice {
6425 key: "held#2026-10-14T00:00:00Z".into(),
6426 at: Timestamp::now(),
6427 outcome: ChatNoticeOutcome::Posted,
6428 });
6429 assert_eq!(t.chat_notice_due(), None, "old hold baseline");
6430 t.import_chat_notice();
6431 t.notice_cause("failed", Some("new-run"), Some(0));
6432 assert!(t.chat_notice_due().is_some(), "new producer evidence");
6433 t.succeed();
6434 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
6435 }
6436
6437 #[test]
6438 fn a_chat_notice_is_due_only_for_a_final_state_and_only_once() {
6439 let mut t = chat_task();
6440 assert_eq!(t.chat_notice_due(), None, "queued");
6441 t.start("r1".to_owned());
6442 assert_eq!(t.chat_notice_due(), None, "running");
6443 t.fail("boom", 3);
6444 assert_eq!(t.status, TaskStatus::Failed);
6445 assert_eq!(t.chat_notice_due(), None, "a retried failure is not final");
6446 t.start("r2".to_owned());
6447 t.park("upgrade");
6448 assert_eq!(t.chat_notice_due(), None, "parked");
6449 t.fail("boom", 1);
6450 assert_eq!(t.status, TaskStatus::Held);
6451 let key = t
6452 .chat_notice_due()
6453 .expect("held on the last attempt is final");
6454 t.chat_notice = Some(ChatNotice {
6455 key,
6456 at: Timestamp::now(),
6457 outcome: ChatNoticeOutcome::Posted,
6458 });
6459 assert_eq!(t.chat_notice_due(), None, "same hold, no second notice");
6460 t.release();
6461 assert_eq!(t.chat_notice_due(), None, "requeued after a release");
6462 t.start("r3".to_owned());
6463 t.succeed();
6464 assert!(t.chat_notice_due().is_some(), "done is a new final state");
6465 }
6466
6467 #[test]
6468 fn completed_chat_notices_keep_current_and_legacy_keys_after_requeue() {
6469 for key in ["done", "done#0", "done#1", "done#12"] {
6470 let mut t = chat_task();
6471 t.start("r1".to_owned());
6472 t.succeed();
6473 t.chat_notice = Some(ChatNotice {
6474 key: key.to_owned(),
6475 at: Timestamp::now(),
6476 outcome: ChatNoticeOutcome::Posted,
6477 });
6478 t.requeue();
6479 t.start("r2".to_owned());
6480 t.succeed();
6481 assert_eq!(t.chat_notice_key().as_deref(), Some("done"));
6482 assert_eq!(t.chat_notice_due(), None, "recorded key: {key}");
6483 }
6484 }
6485
6486 #[test]
6487 fn a_release_without_progress_keeps_the_unresolved_notice_cause() {
6488 let mut t = chat_task();
6489 t.hold_manual(Some("wait".to_owned()));
6490 let first = t.chat_notice_due().expect("held");
6491 t.chat_notice = Some(ChatNotice {
6492 key: first.clone(),
6493 at: Timestamp::now(),
6494 outcome: ChatNoticeOutcome::Posted,
6495 });
6496 t.hold_manual(Some("still waiting".to_owned()));
6497 assert_eq!(t.chat_notice_due(), None, "a re-hold keeps its start");
6498 t.release();
6499 std::thread::sleep(std::time::Duration::from_millis(5));
6500 t.hold_manual(None);
6501 assert_eq!(t.chat_notice_due(), None, "no progress, same cause");
6502 }
6503
6504 #[test]
6505 fn a_chat_notice_needs_a_chat_source_and_a_current_schema() {
6506 let mut human = task("h");
6507 human.succeed();
6508 assert_eq!(human.chat_notice_due(), None);
6509 let mut inherited = task("i");
6511 inherited.origin_chat = Some("20260902-000000-beef".to_owned());
6512 inherited.succeed();
6513 assert_eq!(inherited.chat_notice_due(), None);
6514 let mut old = chat_task();
6515 old.schema = 12;
6516 old.status = TaskStatus::Done;
6517 assert_eq!(old.chat_notice_due(), None, "history is not replayed");
6518 let mut live = chat_task();
6519 live.schema = 12;
6520 live.start("r1".to_owned());
6521 live.succeed();
6522 assert!(live.chat_notice_due().is_some(), "started by this build");
6523 let mut held = chat_task();
6524 held.schema = 12;
6525 held.hold_machine(Some("config".to_owned()));
6526 assert!(held.chat_notice_due().is_some(), "held before any start");
6527 }
6528
6529 #[test]
6530 fn a_stale_snapshot_does_not_rewind_a_recorded_chat_notice() {
6531 let tmp = tempfile::tempdir().expect("tempdir");
6532 let q = Queue::at(tmp.path().join("queue"));
6533 let mut t = chat_task();
6534 t.succeed();
6535 q.put(&mut t).expect("put");
6536 let mut stale = q.get(&t.id).expect("get");
6537 let key = t.chat_notice_due().expect("due");
6538 q.modify(&t.id, |x| {
6539 x.chat_notice = Some(ChatNotice {
6540 key: key.clone(),
6541 at: Timestamp::now(),
6542 outcome: ChatNoticeOutcome::Posted,
6543 });
6544 true
6545 })
6546 .expect("modify");
6547 q.put(&mut stale).expect("stale put");
6548 assert_eq!(q.get(&t.id).expect("get").chat_notice_due(), None);
6549 }
6550
6551 #[test]
6552 fn empty_sources_are_omitted_so_older_builds_still_read_the_task() {
6553 let mut t = task("plain");
6556 let value = serde_json::to_value(&t).unwrap();
6557 assert!(value.get("sources").is_none(), "{value}");
6558 t.sources.pick_commits.push(crate::refs::Input {
6559 token: "abc1234".into(),
6560 sha: "a".repeat(40),
6561 provenance: crate::refs::Provenance::ExplicitCli,
6562 });
6563 let value = serde_json::to_value(&t).unwrap();
6564 assert_eq!(value["sources"]["pick_commits"][0]["sha"], "a".repeat(40));
6565 let restored: Task = serde_json::from_value(value).unwrap();
6566 assert_eq!(restored.sources, t.sources);
6567 }
6568
6569 fn queue_with_ids(dir: &Path, ids: &[&str]) -> Queue {
6571 let ids: Vec<String> = ids.iter().map(|s| (*s).to_owned()).collect();
6572 let n = std::sync::atomic::AtomicUsize::new(0);
6573 Queue::at(dir.join("queue")).with_id_source(move || {
6574 let i = n.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6575 ids.get(i)
6576 .cloned()
6577 .unwrap_or_else(|| format!("20260101-000000-f{i:03x}"))
6578 })
6579 }
6580
6581 const TAKEN: &str = "20260101-000000-aaaa";
6582
6583 #[test]
6584 fn a_new_task_never_merges_into_a_record_with_its_candidate_id() {
6585 let dir = tempfile::tempdir().unwrap();
6586 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6587 let mut first = task("first");
6588 first.start("run-1".into());
6589 first.succeed();
6590 q.put(&mut first).unwrap();
6591 assert_eq!(first.id, TAKEN);
6592 let stored_before = q.get(&first.id).unwrap();
6593 assert!(!stored_before.runs.is_empty());
6594
6595 let mut second = task("second");
6596 q.put(&mut second).unwrap();
6597
6598 assert_ne!(second.id, first.id);
6599 assert!(second.runs.is_empty());
6600 assert!(second.passes.is_empty());
6601 assert!(second.chat_notices.is_empty());
6602 assert!(second.chat_notice.is_none());
6603 assert_eq!(second.chat_notice_due(), task("fresh").chat_notice_due());
6604 let a = q.get(&first.id).unwrap();
6605 let b = q.get(&second.id).unwrap();
6606 assert_eq!(a.title, "first");
6607 assert_eq!(a.runs, stored_before.runs);
6608 assert_eq!(a.passes, stored_before.passes);
6609 assert_eq!(a.chat_notices, stored_before.chat_notices);
6610 assert_eq!(a.chat_notice, stored_before.chat_notice);
6611 assert_eq!(a.chat_notice_due(), stored_before.chat_notice_due());
6612 assert_eq!(b.title, "second");
6613 assert!(b.runs.is_empty());
6614 assert!(b.passes.is_empty());
6615 assert!(b.chat_notice.is_none());
6616
6617 let mut again = q.get(&second.id).unwrap();
6619 again.title = "second, edited".into();
6620 q.put(&mut again).unwrap();
6621 assert_eq!(q.get(&second.id).unwrap().title, "second, edited");
6622 assert_eq!(q.list().len(), 2);
6623 }
6624
6625 #[test]
6626 fn a_new_task_skips_an_id_with_an_orphaned_claim_or_attachments() {
6627 let dir = tempfile::tempdir().unwrap();
6628 let q = queue_with_ids(dir.path(), &["orphan-lock", "orphan-dir", "free"]);
6629 std::fs::create_dir_all(q.root()).unwrap();
6630 std::fs::write(q.root().join("orphan-lock.lock"), "x").unwrap();
6631 std::fs::create_dir_all(q.attachments_dir("orphan-dir")).unwrap();
6632 let mut t = task("t");
6633 q.put(&mut t).unwrap();
6634 assert_eq!(t.id, "free");
6635 }
6636
6637 #[test]
6638 fn a_claimed_new_task_holds_the_claim_on_the_reserved_id() {
6639 let dir = tempfile::tempdir().unwrap();
6640 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6641 let mut first = task("first");
6642 q.put(&mut first).unwrap();
6643 let mut second = task("second");
6644 let claim = q.file_claimed(&mut second).unwrap();
6645 assert_ne!(second.id, first.id);
6646 assert!(q.claim(&second.id).is_err());
6647 assert!(q.claim(&first.id).is_ok(), "the first task was not claimed");
6648 drop(claim);
6649 }
6650
6651 #[test]
6652 fn an_attached_new_task_skips_a_taken_id() {
6653 let dir = tempfile::tempdir().unwrap();
6654 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6655 let mut first = task("first");
6656 q.put(&mut first).unwrap();
6657 let src = source_file(dir.path(), "shot.png", "x");
6658 let mut second = task("second");
6659 q.attach_and_put(&mut second, &[src]).unwrap();
6660 assert_ne!(second.id, first.id);
6661 assert!(!q.attachments_dir(&first.id).exists());
6662 assert_eq!(q.attachment_paths(&second).len(), 1);
6663 assert!(q.attachment_paths(&second)[0].is_file());
6664 }
6665
6666 #[test]
6667 fn attaching_before_putting_a_new_task_reserves_the_id() {
6668 let dir = tempfile::tempdir().unwrap();
6669 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6670 let mut first = task("first");
6671 q.put(&mut first).unwrap();
6672 let src = source_file(dir.path(), "shot.png", "x");
6673 let mut second = task("second");
6674 q.attach(&mut second, &[src]).unwrap();
6675 assert_ne!(second.id, first.id);
6676 assert!(q.attachments_dir(&second.id).is_dir());
6677 q.put(&mut second).unwrap();
6678 assert_eq!(q.get(&second.id).unwrap().attachments.len(), 1);
6679 assert_eq!(q.get(&first.id).unwrap().title, "first");
6680 assert!(q.get(&first.id).unwrap().attachments.is_empty());
6681 }
6682
6683 #[test]
6684 fn create_new_keeps_its_already_exists_meaning() {
6685 let dir = tempfile::tempdir().unwrap();
6686 let q = Queue::at(dir.path().join("queue"));
6687 let mut a = task("a");
6688 a.id = "fixed-id".into();
6689 assert!(q.create_new(&mut a).unwrap());
6690 let mut b = task("b");
6691 b.id = "fixed-id".into();
6692 assert!(!q.create_new(&mut b).unwrap());
6693 assert_eq!(q.get("fixed-id").unwrap().title, "a");
6694 }
6695
6696 #[test]
6697 fn filing_gives_up_when_no_id_is_ever_free() {
6698 let dir = tempfile::tempdir().unwrap();
6699 let q = Queue::at(dir.path().join("queue")).with_id_source(|| TAKEN.to_owned());
6700 let mut first = task("first");
6701 q.put(&mut first).unwrap();
6702 let mut second = task("second");
6703 assert!(q.put(&mut second).is_err());
6704 assert_eq!(q.get(&first.id).unwrap().title, "first");
6705 }
6706}