1#[cfg(test)]
34thread_local! {
35 pub(crate) static LIST_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
36}
37
38use std::collections::HashMap;
39use std::num::NonZeroUsize;
40use std::path::{Path, PathBuf};
41
42use anyhow::{Context, Result, bail};
43use jiff::Timestamp;
44use serde::{Deserialize, Serialize};
45
46use crate::ask::Questions;
47use crate::config::BuildCache;
48use crate::id_reserve::{IdSource, path_exists};
49
50pub const SCHEMA: u32 = 23;
147
148pub(crate) fn dispatch_order(a: &Task, b: &Task) -> std::cmp::Ordering {
154 match (a.upgrade_resume_key(), b.upgrade_resume_key()) {
155 (Some(a), Some(b)) => a.cmp(&b),
156 (Some(_), None) => std::cmp::Ordering::Less,
157 (None, Some(_)) => std::cmp::Ordering::Greater,
158 (None, None) => b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)),
159 }
160}
161
162#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
169#[serde(tag = "kind", rename_all = "snake_case")]
170pub enum MachineHoldCause {
171 DiskCapacity {
173 reason: String,
175 #[serde(default)]
177 before_start: bool,
178 },
179}
180
181#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
183#[serde(rename_all = "lowercase")]
184pub enum HoldSource {
185 Manual,
187 Machine,
189}
190
191impl HoldSource {
192 pub fn label(self) -> &'static str {
194 match self {
195 Self::Manual => "manual",
196 Self::Machine => "machine",
197 }
198 }
199}
200
201#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
204#[serde(tag = "kind", rename_all = "lowercase")]
205pub enum Source {
206 Human,
208 Agent {
211 run: String,
213 node: String,
215 },
216 Issue {
218 number: u64,
220 repo: String,
222 },
223}
224
225pub const CHAT_NODE: &str = "chat";
228
229impl Source {
230 pub fn label(&self) -> String {
232 match self {
233 Self::Human => "human".to_owned(),
234 Self::Agent { run, node } => format!("{node}@{}", short(run)),
235 Self::Issue { number, .. } => format!("issue #{number}"),
236 }
237 }
238}
239
240#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
242#[serde(rename_all = "lowercase")]
243pub enum TaskStatus {
244 Queued,
246 Running,
248 Done,
250 Failed,
252 Held,
254 Blocked,
258 Parked,
263}
264
265impl TaskStatus {
266 pub fn runnable(self) -> bool {
268 matches!(self, Self::Queued | Self::Failed | Self::Parked)
269 }
270
271 pub fn as_str(self) -> &'static str {
273 match self {
274 Self::Queued => "queued",
275 Self::Running => "running",
276 Self::Done => "done",
277 Self::Failed => "failed",
278 Self::Held => "held",
279 Self::Blocked => "blocked",
280 Self::Parked => "parked",
281 }
282 }
283}
284
285#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
288pub struct TaskCounts {
289 pub queued: usize,
291 pub running: usize,
293 pub done: usize,
295 pub failed: usize,
297 pub held: usize,
299 pub blocked: usize,
301 pub parked: usize,
303}
304
305impl TaskCounts {
306 pub fn of(tasks: &[Task]) -> Self {
310 Self::of_statuses(tasks.iter().map(|task| task.status))
311 }
312
313 pub(crate) fn of_statuses(statuses: impl IntoIterator<Item = TaskStatus>) -> Self {
315 let mut counts = Self::default();
316 for status in statuses {
317 match status {
318 TaskStatus::Queued => counts.queued += 1,
319 TaskStatus::Running => counts.running += 1,
320 TaskStatus::Done => counts.done += 1,
321 TaskStatus::Failed => counts.failed += 1,
322 TaskStatus::Held => counts.held += 1,
323 TaskStatus::Blocked => counts.blocked += 1,
324 TaskStatus::Parked => counts.parked += 1,
325 }
326 }
327 counts
328 }
329}
330
331#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
333pub struct Reassignment {
334 pub from: PathBuf,
336 pub to: PathBuf,
338 pub actor: String,
340 pub at: Timestamp,
342 pub run_boundary: usize,
344}
345
346#[derive(Debug, Clone, Serialize, Deserialize)]
348#[serde(deny_unknown_fields)]
349pub struct Task {
350 pub schema: u32,
352 pub id: String,
354 pub title: String,
356 pub instruction: String,
358 #[serde(default, skip_serializing_if = "crate::refs::Sources::is_empty")]
360 pub sources: crate::refs::Sources,
361 pub repo: PathBuf,
363 #[serde(default)]
365 pub reassignments: Vec<Reassignment>,
366 pub source: Source,
368 #[serde(default)]
370 pub chat: Option<String>,
371 #[serde(default)]
373 pub priority: i32,
374 #[serde(default)]
385 pub solo: bool,
386 pub status: TaskStatus,
388 #[serde(default)]
390 pub attempts: usize,
391 #[serde(default)]
393 pub attempt_limit: Option<NonZeroUsize>,
394 #[serde(default)]
397 pub runs: Vec<String>,
398 #[serde(default)]
400 pub startup_failures: Vec<crate::run::StartupFailure>,
401 #[serde(default)]
403 pub last_error: Option<String>,
404 #[serde(default)]
417 pub hold_reason: Option<String>,
418 #[serde(default)]
421 pub hold_source: Option<HoldSource>,
422 #[serde(default)]
424 pub machine_hold_cause: Option<MachineHoldCause>,
425 #[serde(default)]
427 pub disk_resume_run: Option<String>,
428 #[serde(default)]
430 pub disk_checked_at: Option<Timestamp>,
431 #[serde(default)]
443 pub diagnostic: Option<String>,
444 #[serde(default)]
454 pub blocked_by: Vec<String>,
455 #[serde(default)]
458 pub block_reason: Option<String>,
459 #[serde(default)]
472 pub blocked_from: Option<TaskStatus>,
473 #[serde(default)]
475 pub blocked_at: Option<Timestamp>,
476 #[serde(default)]
487 pub answers: Vec<AnsweredQuestion>,
488 #[serde(default)]
494 pub triage_applied: Vec<String>,
495 #[serde(default)]
501 pub actions_applied: Vec<String>,
502 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
505 pub decision_effects: std::collections::BTreeMap<String, String>,
506 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
509 pub superseded_resumes: std::collections::BTreeMap<String, String>,
510 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
514 pub dispatched_resumes: std::collections::BTreeMap<String, String>,
515 #[serde(default)]
517 pub base_recovery_requests: Vec<String>,
518 #[serde(default)]
523 pub resume_override: Option<OperatorResume>,
524 #[serde(default)]
526 pub repair_grants: Vec<crate::repair_budget::Receipt>,
527 #[serde(default)]
529 pub repair_reconciliations: Vec<crate::repair_budget::reconcile::Receipt>,
530 #[serde(default)]
533 pub resumption: Option<Resumption>,
534 #[serde(default)]
536 pub resumption_history: Vec<Resumption>,
537 #[serde(default)]
539 pub operator_decided_at: Option<Timestamp>,
540 #[serde(default)]
552 pub review_branch: Option<String>,
553 #[serde(default)]
555 pub review_instruction: Option<String>,
556 #[serde(default)]
558 pub pr_recovery: Option<crate::pr_recovery::Selection>,
559 #[serde(default)]
562 pub fresh_start: bool,
563 #[serde(default)]
565 pub replacement: Option<crate::merge_decision::Replacement>,
566 #[serde(default)]
577 pub interrupt: bool,
578 #[serde(default)]
601 pub urgent: bool,
602 #[serde(skip)]
607 pub(crate) unfiled: bool,
608 #[serde(default)]
614 pub attachments: Vec<String>,
615 #[serde(default)]
619 pub followup: Option<FollowUp>,
620 #[serde(default)]
626 pub origin_chat: Option<String>,
627 #[serde(default)]
634 pub overrides: Option<RunOverrides>,
635 #[serde(default)]
642 pub review_of: Option<String>,
643 #[serde(default)]
648 pub held_at: Option<Timestamp>,
649 #[serde(default)]
653 pub park_reason: Option<String>,
654 #[serde(default)]
657 pub chat_notice: Option<ChatNotice>,
658 #[serde(default)]
660 pub chat_notices: Vec<String>,
661 #[serde(default)]
664 pub hold_notice_cause: Option<NoticeCause>,
665 #[serde(default)]
670 pub passes: Vec<TaskPass>,
671 #[serde(default)]
675 pub dispatched_at: Option<Timestamp>,
676 #[serde(default)]
678 pub recovery_budget: Option<RecoveryBudget>,
679 #[serde(default)]
682 pub recovery_admission: Option<RecoveryAdmission>,
683 #[serde(default)]
686 pub recovery_generations: Vec<RecoveryBudget>,
687 pub created_at: Timestamp,
689 pub updated_at: Timestamp,
691}
692
693#[derive(Debug, Clone, Serialize, Deserialize)]
695pub struct RecoveryAdmission {
696 pub base: String,
698 pub inputs: String,
700}
701
702#[derive(Debug, Clone, Serialize, Deserialize)]
705pub struct RecoveryBudget {
706 pub instruction: String,
708 pub base: String,
710 pub inputs: String,
712 pub stage: String,
714 pub failures: usize,
716 #[serde(default)]
719 pub progress: Option<String>,
720}
721
722fn recovery_progress(state: &crate::run::RunState) -> Option<String> {
723 use crate::proc::Quiet as _;
724 use std::hash::{Hash, Hasher};
725 let mut patches = Vec::new();
726 for candidate in state.candidates.iter().filter(|c| c.commits > 0) {
727 let output = std::process::Command::new("git")
728 .current_dir(&state.repo)
729 .args(["diff", "--binary", &state.base_commit, &candidate.branch])
730 .quiet()
731 .output()
732 .ok()?;
733 if !output.status.success() {
734 return None;
735 }
736 patches.push(output.stdout);
737 }
738 patches.sort();
739 patches.dedup();
740 let mut hash = std::collections::hash_map::DefaultHasher::new();
741 patches.hash(&mut hash);
742 Some(format!("{:x}", hash.finish()))
743}
744
745pub(crate) fn recovery_inputs(sources: &crate::refs::Sources) -> String {
749 serde_json::to_string(
750 &sources
751 .start_from
752 .iter()
753 .map(|i| ("start", &i.token, &i.sha))
754 .chain(
755 sources
756 .pick_commits
757 .iter()
758 .map(|i| ("pick", &i.token, &i.sha)),
759 )
760 .collect::<Vec<_>>(),
761 )
762 .unwrap_or_default()
763}
764
765#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
767pub struct NoticeCause {
768 pub stage: String,
770 #[serde(default, skip_serializing_if = "Option::is_none")]
773 pub target: Option<String>,
774 pub run: Option<String>,
776 pub pass: Option<usize>,
778}
779
780pub(crate) fn is_done_notice_key(key: &str) -> bool {
782 key == "done"
783 || key
784 .strip_prefix("done#")
785 .is_some_and(|count| !count.is_empty() && count.bytes().all(|b| b.is_ascii_digit()))
786}
787
788#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
790#[serde(rename_all = "snake_case")]
791pub enum PassCost {
792 Spent,
794 Refunded,
796}
797
798#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
800pub struct TaskPass {
801 pub run: String,
803 pub pass_index: usize,
805 pub cost: PassCost,
807 pub attempts_after: usize,
809 #[serde(default)]
811 pub started_at: Option<Timestamp>,
812 pub settled_at: Timestamp,
814 #[serde(default)]
816 pub parked: Option<crate::run::ParkCause>,
817}
818
819#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
821#[serde(rename_all = "lowercase")]
822pub enum ChatNoticeOutcome {
823 Posted,
825 Skipped,
827}
828
829#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
831pub struct ChatNotice {
832 pub key: String,
834 pub at: Timestamp,
836 pub outcome: ChatNoticeOutcome,
838}
839
840#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
842pub struct RunOverrides {
843 #[serde(default)]
846 pub inherit_daemon_options: bool,
847 #[serde(default)]
849 pub merge: Option<String>,
850 #[serde(default)]
852 pub candidates: Option<usize>,
853 #[serde(default)]
855 pub judges: Option<usize>,
856 #[serde(default)]
858 pub reviewers: Option<usize>,
859 #[serde(default)]
861 pub review_rounds: Option<usize>,
862 #[serde(default)]
864 pub seed: Option<u64>,
865 #[serde(default)]
867 pub config: Option<PathBuf>,
868 #[serde(default, skip_serializing_if = "Option::is_none")]
870 pub build_cache: Option<BuildCache>,
871}
872
873impl RunOverrides {
874 pub fn merge_over(&mut self, other: &RunOverrides) {
876 macro_rules! take {
877 ($($f:ident),*) => {$(
878 if other.$f.is_some() {
879 self.$f.clone_from(&other.$f);
880 }
881 )*};
882 }
883 take!(
884 merge,
885 candidates,
886 judges,
887 reviewers,
888 review_rounds,
889 seed,
890 config,
891 build_cache
892 );
893 }
894
895 pub fn apply(&self, config: &mut crate::config::Config) {
897 if let Some(n) = self.candidates {
898 config.graph.implementers = n;
899 }
900 if let Some(n) = self.judges {
901 config.graph.judges = n;
902 }
903 if let Some(n) = self.reviewers {
904 config.graph.reviewers = n;
905 }
906 if let Some(n) = self.review_rounds {
907 config.graph.review_rounds = n;
908 }
909 if let Some(m) = &self.merge
910 && let Ok(mode) = crate::daemon::merge_mode(m)
911 {
912 config.merge.mode = mode;
913 }
914 if let Some(s) = self.seed {
915 config.blind.seed = Some(s);
916 }
917 if let Some(mode) = self.build_cache {
918 config.verify.build_cache = mode;
919 }
920 }
921}
922
923#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
925pub struct FollowUp {
926 pub run: String,
928 #[serde(default)]
930 pub origin_task: Option<String>,
931 pub pr: String,
933 pub findings: Vec<String>,
935 pub generation: u32,
938}
939
940#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
942pub struct Resumption {
943 pub generation: u64,
945 pub source: String,
947 pub question_id: Option<String>,
949 pub at: Timestamp,
951 pub instruction: String,
953 pub answers_through: usize,
955 pub runs_through: usize,
958 pub cleared_reason: Option<String>,
960 #[serde(default)]
962 pub blocking_evidence: Vec<String>,
963}
964
965#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
968pub struct OperatorResume {
969 pub question_id: String,
971 pub at: Timestamp,
973 #[serde(default)]
976 pub conductor_rehold: Option<String>,
977 #[serde(default)]
980 pub forced: bool,
981 #[serde(default)]
988 pub pinned_run: Option<String>,
989}
990
991#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
994pub struct AnsweredQuestion {
995 pub question: String,
997 pub answer: String,
999}
1000
1001impl Task {
1002 pub fn new(title: String, instruction: String, repo: PathBuf, source: Source) -> Self {
1004 let now = Timestamp::now();
1005 let origin_chat = match &source {
1006 Source::Agent { run, node } if node == CHAT_NODE => Some(run.clone()),
1007 _ => None,
1008 };
1009 Self {
1010 schema: SCHEMA,
1011 id: new_id(),
1012 title,
1013 instruction,
1014 sources: crate::refs::Sources::default(),
1015 repo,
1016 reassignments: Vec::new(),
1017 source,
1018 chat: None,
1019 priority: 0,
1020 solo: false,
1021 status: TaskStatus::Queued,
1022 attempts: 0,
1023 attempt_limit: None,
1024 runs: Vec::new(),
1025 startup_failures: Vec::new(),
1026 last_error: None,
1027 hold_reason: None,
1028 hold_source: None,
1029 machine_hold_cause: None,
1030 disk_resume_run: None,
1031 disk_checked_at: None,
1032 diagnostic: None,
1033 blocked_by: Vec::new(),
1034 block_reason: None,
1035 blocked_from: None,
1036 blocked_at: None,
1037 answers: Vec::new(),
1038 triage_applied: Vec::new(),
1039 actions_applied: Vec::new(),
1040 decision_effects: Default::default(),
1041 superseded_resumes: Default::default(),
1042 dispatched_resumes: Default::default(),
1043 base_recovery_requests: Vec::new(),
1044 resume_override: None,
1045 repair_grants: Vec::new(),
1046 repair_reconciliations: Vec::new(),
1047 resumption: None,
1048 resumption_history: Vec::new(),
1049 operator_decided_at: None,
1050 review_branch: None,
1051 review_instruction: None,
1052 pr_recovery: None,
1053 fresh_start: false,
1054 replacement: None,
1055 interrupt: false,
1056 urgent: false,
1057 unfiled: true,
1058 attachments: Vec::new(),
1059 followup: None,
1060 origin_chat,
1061 overrides: None,
1062 review_of: None,
1063 held_at: None,
1064 park_reason: None,
1065 chat_notice: None,
1066 chat_notices: Vec::new(),
1067 hold_notice_cause: None,
1068 passes: Vec::new(),
1069 dispatched_at: None,
1070 recovery_budget: None,
1071 recovery_admission: None,
1072 recovery_generations: Vec::new(),
1073 created_at: now,
1074 updated_at: now,
1075 }
1076 }
1077
1078 pub(crate) fn reassign_repository(&mut self, repo: PathBuf, actor: &str) {
1080 self.stamp_schema();
1081 if matches!(self.status, TaskStatus::Failed | TaskStatus::Parked) {
1082 self.hold_manual(Some(
1083 "Repository reassigned; inspect history and release to start fresh.".into(),
1084 ));
1085 }
1086 let at = Timestamp::now();
1087 self.reassignments.push(Reassignment {
1088 from: self.repo.clone(),
1089 to: repo.clone(),
1090 actor: actor.to_owned(),
1091 at,
1092 run_boundary: self.runs.len(),
1093 });
1094 self.repo = repo;
1095 self.review_branch = None;
1096 self.review_instruction = None;
1097 self.pr_recovery = None;
1098 self.replacement = None;
1099 self.interrupt = false;
1100 self.resume_override = None;
1101 self.fresh_start = true;
1102 self.operator_decided_at = Some(at);
1103 }
1104
1105 pub fn publication_branch(&self) -> String {
1107 crate::run::task_pr_branch_name(&self.id, self.reassignments.len())
1108 }
1109
1110 pub fn execution_runs(&self) -> &[String] {
1112 let boundary = self.reassignments.last().map_or(0, |r| r.run_boundary);
1113 self.runs.get(boundary..).unwrap_or_default()
1114 }
1115
1116 pub fn execution_run(&self, run: &str) -> bool {
1118 self.execution_runs().iter().any(|id| id == run)
1119 }
1120
1121 pub(crate) fn assignment_accepts(&self, run: &crate::run::RunState) -> bool {
1123 self.reassignments.last().is_none_or(|boundary| {
1124 self.execution_run(&run.id)
1125 && run.task_assignment == self.reassignments.len()
1126 && run.created_at > boundary.at
1127 && run
1128 .repo
1129 .canonicalize()
1130 .ok()
1131 .zip(self.repo.canonicalize().ok())
1132 .is_some_and(|(run_repo, task_repo)| run_repo == task_repo)
1133 })
1134 }
1135
1136 pub(crate) fn stale_question(&self, q: &crate::ask::Question) -> bool {
1138 self.reassignments
1139 .last()
1140 .is_some_and(|r| q.asked_at <= r.at)
1141 || (self.runs.contains(&q.run) && !self.execution_run(&q.run))
1142 }
1143
1144 pub fn chat_talk(&self) -> Option<&str> {
1146 if let Some(id) = &self.origin_chat {
1147 return Some(id);
1148 }
1149 match &self.source {
1150 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
1151 _ => None,
1152 }
1153 }
1154
1155 pub fn filed_by_chat(&self) -> Option<&str> {
1159 match &self.source {
1160 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
1161 _ => None,
1162 }
1163 }
1164
1165 pub fn chat_notice_key(&self) -> Option<String> {
1168 match self.status {
1169 TaskStatus::Done => Some("done".to_owned()),
1170 TaskStatus::Held => {
1171 let cause = self
1172 .hold_notice_cause
1173 .clone()
1174 .unwrap_or_else(|| self.unresolved_hold_cause());
1175 Some(format!(
1176 "cause:{}",
1177 serde_json::to_string(&cause).expect("cause")
1178 ))
1179 }
1180 TaskStatus::Blocked => {
1181 let ids: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1182 let evidence = self
1185 .hold_notice_cause
1186 .as_ref()
1187 .filter(|c| c.stage.starts_with("conductor-evidence:"));
1188 Some(format!(
1189 "dependencies:{}",
1190 serde_json::to_string(&(ids, evidence)).expect("ids")
1191 ))
1192 }
1193 _ => None,
1194 }
1195 }
1196
1197 fn unresolved_hold_cause(&self) -> NoticeCause {
1198 NoticeCause {
1199 stage: "hold".into(),
1200 target: None,
1201 run: self.runs.last().cloned(),
1202 pass: None,
1203 }
1204 }
1205
1206 pub(crate) fn legacy_chat_notice_key(&self) -> Option<String> {
1208 let key = match self.status {
1209 TaskStatus::Held if self.hold_notice_cause.is_none() => Some(format!(
1210 "held#{}",
1211 self.held_at.map(|t| t.to_string()).unwrap_or_default()
1212 )),
1213 TaskStatus::Blocked => Some(format!(
1214 "blocked#{}#{}",
1215 self.blocked_by.join(","),
1216 self.block_reason.as_deref().unwrap_or_default()
1217 )),
1218 _ => None,
1219 };
1220 key.filter(|key| self.legacy_notice_matches(key))
1221 }
1222
1223 pub(crate) fn legacy_notice_matches(&self, key: &str) -> bool {
1224 match self.status {
1225 TaskStatus::Held if self.hold_notice_cause.is_none() => key
1226 .strip_prefix("held#")
1227 .is_some_and(|time| time.is_empty() || time.parse::<Timestamp>().is_ok()),
1228 TaskStatus::Blocked
1229 if self
1230 .hold_notice_cause
1231 .as_ref()
1232 .is_none_or(|c| !c.stage.starts_with("conductor-evidence:")) =>
1233 {
1234 key.strip_prefix("blocked#").is_some_and(|tail| {
1235 let Some((ids, _)) = tail.split_once('#') else {
1236 return false;
1237 };
1238 let old: std::collections::BTreeSet<_> =
1239 ids.split(',').filter(|id| !id.is_empty()).collect();
1240 let current: std::collections::BTreeSet<_> =
1241 self.blocked_by.iter().map(String::as_str).collect();
1242 old == current
1243 })
1244 }
1245 _ => false,
1246 }
1247 }
1248
1249 fn import_chat_notice(&mut self) {
1250 if let Some(n) = &self.chat_notice {
1251 let key = if is_done_notice_key(&n.key) {
1252 "done".to_owned()
1253 } else if self.legacy_notice_matches(&n.key) {
1254 self.chat_notice_key().unwrap_or_else(|| n.key.clone())
1255 } else {
1256 n.key.clone()
1257 };
1258 if !self.chat_notices.contains(&key) {
1259 self.chat_notices.push(key);
1260 }
1261 }
1262 }
1263
1264 pub(crate) fn notice_cause(&mut self, stage: &str, run: Option<&str>, pass: Option<usize>) {
1267 self.import_chat_notice();
1268 self.hold_notice_cause = Some(NoticeCause {
1269 stage: stage.to_owned(),
1270 target: None,
1271 run: run.map(str::to_owned),
1272 pass,
1273 });
1274 }
1275
1276 pub(crate) fn startup_notice_cause(&mut self, stage: &str) {
1279 let recovery_only = matches!(
1280 stage,
1281 "startup-source" | "startup-candidates" | "startup-public-head"
1282 );
1283 match self.hold_notice_cause.as_ref() {
1284 Some(cause) if recovery_only && !cause.stage.starts_with("startup-") => return,
1285 None if recovery_only
1286 && self
1287 .chat_notice
1288 .as_ref()
1289 .is_some_and(|n| !is_done_notice_key(&n.key)) =>
1290 {
1291 return;
1292 }
1293 _ => {}
1294 }
1295 self.notice_cause(stage, None, None);
1296 }
1297
1298 pub(crate) fn fail_startup(
1301 &mut self,
1302 stage: &str,
1303 why: impl Into<String>,
1304 max_attempts: usize,
1305 ) {
1306 let unresolved = self.hold_notice_cause.clone();
1307 self.fail(why, max_attempts);
1308 self.hold_notice_cause = unresolved;
1309 self.startup_notice_cause(stage);
1310 }
1311
1312 pub(crate) fn fail_startup_target(
1314 &mut self,
1315 stage: &str,
1316 target: String,
1317 why: impl Into<String>,
1318 max_attempts: usize,
1319 ) {
1320 self.fail_startup(stage, why, max_attempts);
1321 if let Some(cause) = self.hold_notice_cause.as_mut() {
1322 cause.target = Some(target);
1323 }
1324 }
1325
1326 pub fn chat_notice_due(&self) -> Option<String> {
1328 self.filed_by_chat()?;
1329 if self.schema < 13 {
1330 return None;
1331 }
1332 let key = self.chat_notice_key()?;
1333 if self.chat_notices.contains(&key) {
1334 return None;
1335 }
1336 match &self.chat_notice {
1337 Some(n)
1338 if n.key == key
1339 || (key == "done" && is_done_notice_key(&n.key))
1340 || self.legacy_notice_matches(&n.key) =>
1341 {
1342 None
1343 }
1344 _ => Some(key),
1345 }
1346 }
1347
1348 pub fn short(&self) -> &str {
1350 short(&self.id)
1351 }
1352
1353 pub fn mark_triage_applied(&mut self, question_id: &str) {
1355 if !self.triage_applied(question_id) {
1356 self.triage_applied.push(question_id.to_owned());
1357 }
1358 }
1359
1360 pub fn action_applied(&self, question_id: &str) -> bool {
1363 self.actions_applied.iter().any(|id| id == question_id)
1364 }
1365
1366 pub fn mark_action_applied(&mut self, question_id: &str) {
1368 if !self.action_applied(question_id) {
1369 self.actions_applied.push(question_id.to_owned());
1370 }
1371 }
1372
1373 pub fn triage_applied(&self, question_id: &str) -> bool {
1375 self.triage_applied.iter().any(|id| id == question_id)
1376 }
1377
1378 pub fn start(&mut self, run: String) {
1391 self.stamp_schema();
1392 self.status = TaskStatus::Running;
1393 self.attempts += 1;
1394 self.runs.retain(|id| id != &run);
1395 if let Some(pin) = &self.resume_override
1396 && pin.pinned_run.as_ref() == Some(&run)
1397 && !pin.question_id.is_empty()
1398 {
1399 self.dispatched_resumes
1400 .insert(pin.question_id.clone(), run.clone());
1401 }
1402 self.runs.push(run);
1403 self.last_error = None;
1404 self.park_reason = None;
1405 self.fresh_start = false;
1406 self.interrupt = false;
1407 self.resume_override = None;
1409 self.disk_resume_run = None;
1412 }
1413
1414 pub fn record_startup_failure(&mut self, failure: crate::run::StartupFailure) {
1416 if !self.startup_failures.iter().any(|f| f.run == failure.run) {
1417 self.startup_failures.push(failure);
1418 }
1419 }
1420
1421 pub fn link_run(&mut self, run: &str) -> bool {
1426 if self.runs.iter().any(|r| r == run) {
1427 return false;
1428 }
1429 self.runs.push(run.to_owned());
1430 true
1431 }
1432
1433 pub fn succeed(&mut self) {
1443 self.stamp_schema();
1444 self.status = TaskStatus::Done;
1445 self.resume_override = None;
1446 self.last_error = None;
1447 self.park_reason = None;
1448 self.hold_reason = None;
1449 self.hold_source = None;
1450 self.machine_hold_cause = None;
1451 self.disk_resume_run = None;
1452 self.disk_checked_at = None;
1453 self.diagnostic = None;
1454 self.blocked_by.clear();
1455 self.block_reason = None;
1456 self.blocked_from = None;
1457 }
1458
1459 pub fn already_landed(&mut self, note: impl Into<String>) {
1467 self.succeed();
1468 self.attempts = self.attempts.saturating_sub(1);
1469 self.last_error = Some(note.into());
1470 }
1471
1472 pub fn superseded_attempts(&self, last_run_succeeded: bool) -> &[String] {
1502 if self.status != TaskStatus::Done || !last_run_succeeded || self.execution_runs().len() < 2
1503 {
1504 return &[];
1505 }
1506 &self.execution_runs()[..self.execution_runs().len() - 1]
1507 }
1508
1509 pub fn successor_of(&self, run: &str) -> Option<&String> {
1517 let runs = self.execution_runs();
1518 let pos = runs.iter().rposition(|r| r == run)?;
1519 runs.get(pos + 1)
1520 }
1521
1522 pub fn earlier_attempts(&self) -> &[String] {
1529 self.execution_runs()
1530 }
1531
1532 fn preserve_legacy_build_cache(&mut self) {
1533 if self.schema < 23 {
1534 let overrides = self.overrides.get_or_insert_with(|| RunOverrides {
1535 inherit_daemon_options: true,
1536 ..RunOverrides::default()
1537 });
1538 overrides.build_cache.get_or_insert(BuildCache::Shared);
1539 }
1540 }
1541
1542 fn stamp_schema(&mut self) {
1547 self.preserve_legacy_build_cache();
1548 self.schema = self.schema.max(SCHEMA);
1549 }
1550
1551 fn note_held(&mut self) {
1554 self.stamp_schema();
1555 let continuing = self.status == TaskStatus::Held
1556 || (self.status == TaskStatus::Blocked
1557 && (self.blocked_from == Some(TaskStatus::Held)
1558 || (self.blocked_from.is_none()
1559 && (self.hold_reason.is_some() || self.hold_source.is_some()))));
1560 if !continuing || self.held_at.is_none() {
1561 self.held_at = Some(Timestamp::now());
1562 }
1563 }
1564
1565 pub fn fail(&mut self, why: impl Into<String>, max_attempts: usize) {
1581 let why = why.into();
1582 if self.status == TaskStatus::Running {
1583 let run = self.runs.last().cloned();
1584 self.notice_cause("run-failed", run.as_deref(), Some(self.attempts));
1585 }
1586 self.diagnostic = None;
1587 self.park_reason = None;
1588 self.status = if self.attempts >= self.effective_attempt_limit(max_attempts) {
1589 self.note_held();
1590 self.hold_source = Some(HoldSource::Machine);
1591 self.machine_hold_cause = None;
1592 self.disk_resume_run = None;
1593 self.disk_checked_at = None;
1594 self.hold_reason = Some(why.clone());
1595 TaskStatus::Held
1596 } else {
1597 TaskStatus::Failed
1598 };
1599 self.last_error = Some(why);
1600 }
1601
1602 pub fn stall(&mut self, why: impl Into<String>) {
1612 self.last_error = Some(why.into());
1613 self.diagnostic = None;
1614 self.attempts = self.attempts.saturating_sub(1);
1615 self.status = TaskStatus::Failed;
1616 }
1617
1618 pub fn park(&mut self, why: impl Into<String>) {
1624 self.diagnostic = None;
1625 self.attempts = self.attempts.saturating_sub(1);
1626 self.status = TaskStatus::Parked;
1627 self.park_reason = Some(why.into());
1628 self.stamp_schema();
1631 }
1632
1633 pub fn record_pass(
1635 &mut self,
1636 run: &str,
1637 pass_index: usize,
1638 attempts_before: usize,
1639 parked: Option<crate::run::ParkCause>,
1640 ) {
1641 self.record_pass_at(run, pass_index, None, attempts_before, parked);
1642 }
1643
1644 pub fn record_pass_at(
1647 &mut self,
1648 run: &str,
1649 pass_index: usize,
1650 started_at: Option<Timestamp>,
1651 attempts_before: usize,
1652 parked: Option<crate::run::ParkCause>,
1653 ) {
1654 if self
1655 .passes
1656 .iter()
1657 .any(|p| p.run == run && p.pass_index == pass_index)
1658 {
1659 return;
1660 }
1661 let cost = if self.attempts < attempts_before {
1662 PassCost::Refunded
1663 } else {
1664 PassCost::Spent
1665 };
1666 self.passes.push(TaskPass {
1667 run: run.to_owned(),
1668 pass_index,
1669 started_at: self.dispatched_at.or(started_at),
1670 cost,
1671 attempts_after: self.attempts,
1672 settled_at: Timestamp::now(),
1673 parked,
1674 });
1675 self.stamp_schema();
1676 }
1677
1678 pub(crate) fn upgrade_resume_key(&self) -> Option<(Timestamp, &str)> {
1681 if self.status != TaskStatus::Parked || self.fresh_start {
1682 return None;
1683 }
1684 let pass = self.passes.last()?;
1685 if self.execution_runs().last() != Some(&pass.run)
1686 || !matches!(pass.parked, Some(crate::run::ParkCause::Upgrade { .. }))
1687 {
1688 return None;
1689 }
1690 Some((pass.started_at.unwrap_or(pass.settled_at), &self.id))
1693 }
1694
1695 pub fn operator_held(&self) -> bool {
1701 self.status == TaskStatus::Held && !matches!(self.hold_source, Some(HoldSource::Machine))
1702 }
1703
1704 pub fn hold_manual(&mut self, reason: Option<String>) {
1714 self.operator_decided_at = Some(Timestamp::now());
1716 self.hold_unrequested_retry(reason);
1717 }
1718
1719 pub(crate) fn hold_unrequested_retry(&mut self, reason: Option<String>) {
1722 self.park_reason = None;
1723 self.note_held();
1724 self.status = TaskStatus::Held;
1725 if reason.is_some() {
1726 self.hold_reason = reason;
1727 }
1728 self.hold_source = Some(HoldSource::Manual);
1729 self.machine_hold_cause = None;
1730 self.disk_resume_run = None;
1731 self.disk_checked_at = None;
1732 self.blocked_by.clear();
1733 self.block_reason = None;
1734 self.blocked_from = None;
1735 }
1736
1737 pub fn hold_machine(&mut self, reason: Option<String>) {
1742 self.park_reason = None;
1743 self.note_held();
1744 self.status = TaskStatus::Held;
1745 if reason.is_some() {
1746 self.hold_reason = reason;
1747 }
1748 self.hold_source = Some(HoldSource::Machine);
1749 self.machine_hold_cause = None;
1750 self.disk_resume_run = None;
1751 self.disk_checked_at = None;
1752 self.blocked_by.clear();
1753 self.block_reason = None;
1754 self.blocked_from = None;
1755 }
1756
1757 pub(crate) fn mark_disk_capacity(&mut self) {
1759 self.machine_hold_cause = Some(MachineHoldCause::DiskCapacity {
1760 reason: self.hold_reason.clone().unwrap_or_default(),
1761 before_start: false,
1762 });
1763 }
1764
1765 pub(crate) fn mark_startup_disk_capacity(&mut self) {
1766 self.machine_hold_cause = Some(MachineHoldCause::DiskCapacity {
1767 reason: self.hold_reason.clone().unwrap_or_default(),
1768 before_start: true,
1769 });
1770 }
1771
1772 pub(crate) fn disk_wait_run(&self) -> Option<&String> {
1773 if self.has_disk_capacity_cause()
1774 && matches!(
1775 self.machine_hold_cause,
1776 Some(MachineHoldCause::DiskCapacity {
1777 before_start: true,
1778 ..
1779 })
1780 )
1781 {
1782 None
1783 } else {
1784 self.execution_runs().last()
1785 }
1786 }
1787
1788 pub fn has_disk_capacity_cause(&self) -> bool {
1790 matches!(
1791 (&self.machine_hold_cause, self.hold_reason.as_deref()),
1792 (Some(MachineHoldCause::DiskCapacity { reason, .. }), Some(now)) if reason == now
1793 )
1794 }
1795
1796 pub(crate) fn resume_disk_wait(&mut self) {
1798 let retained = self.disk_wait_run().cloned();
1799 self.status = TaskStatus::Queued;
1800 self.held_at = None;
1801 self.hold_source = None;
1802 self.hold_reason = None;
1803 self.machine_hold_cause = None;
1804 self.last_error = None;
1805 self.disk_resume_run = retained;
1806 }
1807
1808 pub fn block(&mut self, blocked_by: Vec<String>, reason: Option<String>) {
1818 self.stamp_schema();
1819 let before: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1820 let after: std::collections::BTreeSet<_> = blocked_by.iter().collect();
1821 if self.status != TaskStatus::Blocked || before != after {
1822 self.blocked_at = Some(Timestamp::now());
1823 }
1824 if self.status != TaskStatus::Blocked {
1825 self.blocked_from = Some(self.status);
1826 }
1827 self.status = TaskStatus::Blocked;
1828 self.blocked_by = blocked_by;
1829 self.block_reason = reason;
1830 }
1831
1832 pub fn unblock(&mut self, resolved_id: &str) {
1855 if self.status != TaskStatus::Blocked {
1856 return;
1857 }
1858 self.blocked_by.retain(|id| id != resolved_id);
1859 self.restore_if_unblocked();
1860 }
1861
1862 pub fn dependency_deleted(&mut self, deleted_id: &str) -> bool {
1873 if self.status != TaskStatus::Blocked || !self.blocked_by.iter().any(|b| b == deleted_id) {
1874 return false;
1875 }
1876 self.blocked_by.retain(|id| id != deleted_id);
1877 self.restore_if_unblocked();
1878 true
1879 }
1880
1881 fn restore_if_unblocked(&mut self) {
1885 if self.blocked_by.is_empty() {
1886 self.status = match self.blocked_from {
1887 Some(TaskStatus::Running) => TaskStatus::Queued,
1888 Some(other) => other,
1889 None if self.hold_reason.is_some() || self.hold_source.is_some() => {
1890 TaskStatus::Held
1891 }
1892 None => TaskStatus::Queued,
1893 };
1894 self.block_reason = None;
1895 self.blocked_from = None;
1896 }
1897 }
1898
1899 pub fn record_answer(&mut self, question: String, answer: String) {
1904 self.answers.push(AnsweredQuestion { question, answer });
1905 }
1906
1907 pub fn request_review(&mut self, branch: String) {
1911 self.release();
1912 self.review_branch = Some(branch);
1913 self.review_instruction = Some(self.instruction.clone());
1914 }
1915
1916 pub fn requeue(&mut self) {
1919 self.release();
1920 self.review_branch = None;
1922 self.review_instruction = None;
1923 self.fresh_start = true;
1924 }
1925
1926 pub fn hold_for_handover(&mut self, branch: Option<String>, reason: String) {
1935 if branch.is_some() {
1936 self.review_branch = branch;
1937 self.review_instruction = Some(self.instruction.clone());
1938 }
1939 self.hold_machine(Some(reason));
1940 }
1941
1942 pub fn set_urgent(&mut self, urgent: bool) -> Result<()> {
1944 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1945 bail!(
1946 "task {} is {}; urgent can only change while queued or held; \
1947 wait for the active attempt to finish and requeue or hold the task first",
1948 self.short(),
1949 self.status.as_str()
1950 );
1951 }
1952 self.urgent = urgent;
1953 Ok(())
1954 }
1955
1956 pub fn set_priority(&mut self, priority: i32) -> Result<()> {
1965 if self.status == TaskStatus::Running {
1966 bail!(
1967 "task {} is running; its priority cannot be changed until \
1968 this attempt finishes",
1969 self.short()
1970 );
1971 }
1972 self.priority = priority;
1973 Ok(())
1974 }
1975
1976 pub fn set_interrupt(&mut self, interrupt: bool) -> Result<()> {
1991 if interrupt && !self.status.runnable() {
1992 bail!(
1993 "task {} is {}; only a queued or failed task can be marked \
1994 to interrupt",
1995 self.short(),
1996 self.status.as_str()
1997 );
1998 }
1999 self.interrupt = interrupt;
2000 Ok(())
2001 }
2002
2003 pub fn effective_attempt_limit(&self, daemon_limit: usize) -> usize {
2005 self.attempt_limit.map_or(daemon_limit, NonZeroUsize::get)
2006 }
2007
2008 pub fn change_attempt_limit(&mut self, limit: Option<NonZeroUsize>) -> Result<()> {
2011 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
2012 bail!("only queued or held tasks can change the attempt limit");
2013 }
2014 self.stamp_schema();
2015 self.attempt_limit = limit;
2016 Ok(())
2017 }
2018
2019 pub fn implementers(&self) -> Option<usize> {
2021 if self.solo {
2022 Some(1)
2023 } else {
2024 self.overrides.as_ref().and_then(|o| o.candidates)
2025 }
2026 }
2027
2028 pub fn build_cache_override(
2032 &self,
2033 config: &crate::config::Config,
2034 ) -> Option<(BuildCache, &'static str)> {
2035 self.overrides
2036 .as_ref()
2037 .and_then(|o| o.build_cache)
2038 .map(|mode| (mode, "task-override"))
2039 .or_else(|| {
2040 (self.urgent && crate::build_target::urgent_isolation(config))
2041 .then_some((BuildCache::Isolated, "urgent"))
2042 })
2043 }
2044
2045 pub fn change_build_cache(&mut self, mode: Option<BuildCache>) -> Result<()> {
2048 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
2049 bail!(
2050 "task {} is {}; only queued or held tasks can change build cache",
2051 self.short(),
2052 self.status.as_str()
2053 );
2054 }
2055 self.stamp_schema();
2056 if self.overrides.is_some() || mode.is_some() {
2057 let overrides = self.overrides.get_or_insert_with(|| RunOverrides {
2058 inherit_daemon_options: true,
2059 ..RunOverrides::default()
2060 });
2061 overrides.build_cache = mode;
2062 }
2063 Ok(())
2064 }
2065
2066 pub fn change_implementers(&mut self, count: u8) -> Result<()> {
2069 self.check_implementers_editable()?;
2070 let explicit = self.overrides.as_ref().and_then(|o| o.config.as_deref());
2071 let (mut config, _) = crate::config::Config::discover(&self.repo, explicit)?;
2072 if let Some(o) = &self.overrides {
2073 o.apply(&mut config);
2074 }
2075 self.set_implementers(count, &config)
2076 }
2077
2078 fn check_implementers_editable(&self) -> Result<()> {
2079 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
2080 bail!(
2081 "task {} is {}; only queued or held tasks can change implementers",
2082 self.short(),
2083 self.status.as_str()
2084 );
2085 }
2086 Ok(())
2087 }
2088
2089 pub fn set_implementers(&mut self, count: u8, config: &crate::config::Config) -> Result<()> {
2091 self.check_implementers_editable()?;
2092 crate::talk::check_implementers(count, config).map_err(anyhow::Error::msg)?;
2093 self.stamp_schema();
2094 self.solo = count == 1;
2095 let mut overrides = self.overrides.clone().unwrap_or_else(|| RunOverrides {
2096 inherit_daemon_options: true,
2097 ..Default::default()
2098 });
2099 overrides.candidates = (count > 1).then_some(usize::from(count));
2100 let mut settings = overrides.clone();
2101 settings.inherit_daemon_options = false;
2102 self.overrides = if overrides.inherit_daemon_options && settings == RunOverrides::default()
2104 {
2105 None
2106 } else {
2107 Some(overrides)
2108 };
2109 Ok(())
2110 }
2111
2112 pub fn edit(&mut self, title: String, instruction: String) -> Result<()> {
2124 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
2125 bail!(
2126 "task {} is {}; only a queued or held task's instruction can \
2127 be edited",
2128 self.short(),
2129 self.status.as_str()
2130 );
2131 }
2132 if self.review_branch.is_some() && self.review_instruction.is_none() {
2135 self.review_instruction = Some(self.instruction.clone());
2136 }
2137 if self.instruction != instruction {
2138 self.resume_override = None;
2141 }
2142 if self.status == TaskStatus::Held && self.machine_hold_cause.is_some() {
2143 self.hold_manual(self.hold_reason.clone());
2144 }
2145 self.machine_hold_cause = None;
2146 self.disk_resume_run = None;
2147 self.disk_checked_at = None;
2148 self.title = title;
2149 self.instruction = instruction;
2150 Ok(())
2151 }
2152
2153 pub fn handed_off(&mut self, why: impl Into<String>) {
2173 let why = why.into();
2174 self.diagnostic = None;
2175 self.note_held();
2176 self.status = TaskStatus::Held;
2177 self.hold_source = Some(HoldSource::Machine);
2178 self.machine_hold_cause = None;
2179 self.disk_resume_run = None;
2180 self.disk_checked_at = None;
2181 self.hold_reason = Some(why.clone());
2182 self.last_error = Some(why);
2183 }
2184
2185 pub fn release_operator(&mut self, source: &str, question_id: Option<&str>) {
2187 self.recovery_budget = None;
2188 self.recovery_generations.clear();
2189 self.recovery_admission = None;
2190 if self
2191 .resumption
2192 .as_ref()
2193 .is_some_and(|r| r.instruction != self.instruction)
2194 {
2195 self.resume_override = None;
2198 }
2199 let approval = Resumption {
2200 generation: self.resumption.as_ref().map_or(1, |r| r.generation + 1),
2201 source: source.to_owned(),
2202 question_id: question_id.map(str::to_owned),
2203 at: Timestamp::now(),
2204 instruction: self.instruction.clone(),
2205 answers_through: self.answers.len(),
2206 runs_through: self.runs.len(),
2207 blocking_evidence: Vec::new(),
2208 cleared_reason: self
2209 .hold_reason
2210 .clone()
2211 .or_else(|| self.block_reason.clone()),
2212 };
2213 self.release();
2214 self.operator_decided_at = Some(approval.at);
2215 if let Some(previous) = self.resumption.replace(approval) {
2216 self.resumption_history.push(previous);
2217 }
2218 }
2219
2220 pub(crate) fn recovery_failed(&mut self, state: &crate::run::RunState) {
2222 let (base, inputs) = self.recovery_admission.as_ref().map_or_else(
2223 || (state.base_commit.clone(), recovery_inputs(&state.sources)),
2224 |a| (a.base.clone(), a.inputs.clone()),
2225 );
2226 let progress = recovery_progress(state);
2227 let stage = if state.prep_failure.is_some() {
2228 "preparation"
2229 } else {
2230 "execution"
2231 };
2232 let matches = |b: &RecoveryBudget| {
2233 b.instruction == self.instruction
2234 && b.base == base
2235 && b.inputs == inputs
2236 && (progress.is_none() || b.progress == progress)
2237 };
2238 if !self.recovery_budget.as_ref().is_some_and(matches) {
2239 let previous = self
2240 .recovery_generations
2241 .iter()
2242 .position(matches)
2243 .map(|index| self.recovery_generations.remove(index));
2244 if let Some(old) = self.recovery_budget.take() {
2245 self.recovery_generations.push(old);
2246 }
2247 self.recovery_budget = previous;
2248 }
2249 if let Some(budget) = &mut self.recovery_budget {
2250 budget.failures = budget.failures.saturating_add(1);
2251 budget.stage = stage.to_owned();
2252 } else {
2253 self.recovery_budget = Some(RecoveryBudget {
2254 instruction: self.instruction.clone(),
2255 base,
2256 inputs,
2257 stage: stage.to_owned(),
2258 failures: 1,
2259 progress,
2260 });
2261 }
2262 }
2263
2264 pub(crate) fn recovery_exhausted(
2265 &self,
2266 base: &str,
2267 inputs: &str,
2268 limit: usize,
2269 ) -> Option<String> {
2270 let same_input = |b: &&RecoveryBudget| {
2271 b.instruction == self.instruction && b.base == base && b.inputs == inputs
2272 };
2273 let budget = self
2274 .recovery_budget
2275 .as_ref()
2276 .filter(same_input)
2277 .or_else(|| self.recovery_generations.iter().rev().find(same_input))?;
2278 (budget.instruction == self.instruction && budget.base == base && budget.inputs == inputs
2279 && budget.failures >= limit.max(1)).then(|| format!(
2280 "[recovery-budget] {} failed passes against unchanged inputs ({}). Inspect the retained recovery sources and correct the cause, change the inputs, or explicitly release the task to approve another generation. Automatic requeue does not reset this budget.",
2281 budget.failures, budget.stage))
2282 }
2283
2284 pub fn release(&mut self) {
2288 self.import_chat_notice();
2291 if self.hold_notice_cause.is_none()
2292 && matches!(self.status, TaskStatus::Held | TaskStatus::Blocked)
2293 {
2294 self.hold_notice_cause = Some(self.unresolved_hold_cause());
2295 }
2296 let refused_handover = self.status == TaskStatus::Held
2297 && self.hold_source == Some(HoldSource::Machine)
2298 && self.review_branch.is_some()
2299 && self
2300 .review_instruction
2301 .as_ref()
2302 .is_none_or(|i| *i == self.instruction);
2303 self.status = TaskStatus::Queued;
2304 self.held_at = None;
2305 self.attempts = 0;
2306 self.last_error = None;
2307 self.park_reason = None;
2308 self.hold_reason = None;
2311 self.hold_source = None;
2312 self.machine_hold_cause = None;
2313 self.disk_resume_run = None;
2314 self.disk_checked_at = None;
2315 self.diagnostic = None;
2316 self.blocked_by.clear();
2321 self.block_reason = None;
2322 self.blocked_from = None;
2323 if !refused_handover {
2327 self.review_branch = None;
2328 self.review_instruction = None;
2329 }
2330 self.fresh_start = false;
2331 }
2332}
2333
2334fn copy_new(dir: &Path, src: &Path, name: &str) -> Result<(String, PathBuf)> {
2338 use std::io::ErrorKind;
2339 let (stem, ext) = match name.rfind('.') {
2340 Some(i) if i > 0 => (&name[..i], &name[i..]),
2341 _ => (name, ""),
2342 };
2343 for n in 1u32.. {
2344 let candidate = if n == 1 {
2345 name.to_owned()
2346 } else {
2347 let suffix = format!("-{n}");
2348 let room = 64usize.saturating_sub(suffix.len() + ext.len());
2349 let stem: String = stem.chars().take(room).collect();
2350 format!("{stem}{suffix}{ext}")
2351 };
2352 if !crate::ask::valid_asset_name(&candidate) {
2353 bail!("no valid attachment name is left for `{name}`");
2354 }
2355 let path = dir.join(&candidate);
2356 match std::fs::OpenOptions::new()
2357 .write(true)
2358 .create_new(true)
2359 .open(&path)
2360 {
2361 Ok(mut out) => {
2362 let copied = std::fs::File::open(src)
2363 .and_then(|mut input| std::io::copy(&mut input, &mut out));
2364 if let Err(e) = copied {
2365 drop(out);
2366 let _ = std::fs::remove_file(&path);
2367 return Err(e).with_context(|| format!("copy {}", src.display()));
2368 }
2369 return Ok((candidate, path));
2370 }
2371 Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
2372 Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
2373 }
2374 }
2375 unreachable!("the counter never runs out")
2376}
2377
2378const TASK_LOCK_STALE: std::time::Duration = std::time::Duration::from_secs(10);
2381
2382pub(crate) struct TaskLock {
2385 path: PathBuf,
2386 token: String,
2387}
2388
2389fn owner_token() -> String {
2392 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
2393 let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2394 let mut r = crate::rng::SplitMix64::new(crate::rng::entropy() ^ n.rotate_left(32));
2395 format!("{}-{:016x}", std::process::id(), r.next_u64())
2396}
2397
2398fn break_marker(path: &Path, token: &str) -> PathBuf {
2401 let mut h: u64 = 0xcbf29ce484222325;
2402 for b in token.bytes() {
2403 h ^= u64::from(b);
2404 h = h.wrapping_mul(0x100000001b3);
2405 }
2406 let mut name = path.as_os_str().to_owned();
2407 name.push(format!(".break-{h:016x}"));
2408 PathBuf::from(name)
2409}
2410
2411fn older_than_stale(path: &Path) -> bool {
2412 std::fs::metadata(path)
2413 .and_then(|m| m.modified())
2414 .ok()
2415 .and_then(|t| t.elapsed().ok())
2416 .is_some_and(|age| age > TASK_LOCK_STALE)
2417}
2418
2419fn writer_may_be_gone(token: &str) -> bool {
2420 let Some(pid) = token
2421 .split_once('-')
2422 .and_then(|(pid, _)| pid.parse::<u32>().ok())
2423 else {
2424 return true;
2427 };
2428 if pid == std::process::id() {
2429 return false;
2430 }
2431 crate::proc::pid_status(pid) == Some(false)
2433}
2434
2435const MAX_MARKER_DEPTH: u8 = 3;
2439
2440fn take_marker(
2446 path: &Path,
2447 token: &str,
2448 depth: u8,
2449 retry: crate::lock_file::Retry,
2450) -> Option<(PathBuf, String)> {
2451 use std::io::Write;
2452 let marker = break_marker(path, token);
2453 for _ in 0..2 {
2454 match retry.create(&marker) {
2455 Ok(mut f) => {
2456 let mine = owner_token();
2457 if f.write_all(mine.as_bytes()).is_err() {
2458 drop(f);
2459 let _ = std::fs::remove_file(&marker);
2460 return None;
2461 }
2462 return Some((marker, mine));
2463 }
2464 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2465 let judged = std::fs::read_to_string(&marker).ok();
2466 match judged.filter(|_| older_than_stale(&marker)) {
2467 Some(judged) if depth < MAX_MARKER_DEPTH => {
2468 remove_lock_if(&marker, &judged, true, depth + 1, retry)?;
2469 }
2470 _ => return None,
2471 }
2472 }
2473 Err(_) => return None,
2474 }
2475 }
2476 None
2477}
2478
2479fn remove_lock_if(
2484 path: &Path,
2485 token: &str,
2486 require_stale: bool,
2487 depth: u8,
2488 retry: crate::lock_file::Retry,
2489) -> Option<bool> {
2490 let (marker, mine) = take_marker(path, token, depth, retry)?;
2491 let still = std::fs::read_to_string(path).is_ok_and(|c| c == token)
2492 && (!require_stale || older_than_stale(path) && writer_may_be_gone(token));
2493 if still {
2494 let _ = std::fs::remove_file(path);
2495 }
2496 if depth >= MAX_MARKER_DEPTH {
2498 if std::fs::read_to_string(&marker).is_ok_and(|c| c == mine) {
2499 let _ = std::fs::remove_file(&marker);
2500 }
2501 } else {
2502 let _ = remove_lock_if(&marker, &mine, false, depth + 1, retry);
2503 }
2504 Some(still)
2505}
2506
2507#[cfg(test)]
2510fn break_stale(path: &Path, judged: &str) -> bool {
2511 break_stale_with(path, judged, crate::lock_file::Retry::new(None))
2512}
2513
2514fn break_stale_with(path: &Path, judged: &str, retry: crate::lock_file::Retry) -> bool {
2515 remove_lock_if(path, judged, true, 0, retry).unwrap_or(false)
2516}
2517
2518impl Drop for TaskLock {
2519 fn drop(&mut self) {
2520 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
2521 let retry = crate::lock_file::Retry::new(Some(deadline));
2522 loop {
2523 match remove_lock_if(&self.path, &self.token, false, 0, retry) {
2524 Some(_) => return,
2525 None if std::time::Instant::now() > deadline => return,
2526 None => std::thread::sleep(std::time::Duration::from_millis(5)),
2527 }
2528 }
2529 }
2530}
2531
2532#[derive(Debug, Clone)]
2534pub struct Queue {
2535 root: PathBuf,
2536 ids: IdSource,
2537}
2538
2539fn validate_attachments(sources: &[PathBuf]) -> Result<Vec<(&PathBuf, &str)>> {
2540 let mut wanted = Vec::new();
2541 for src in sources {
2542 let name = src
2543 .file_name()
2544 .and_then(|n| n.to_str())
2545 .with_context(|| format!("`{}` has no usable file name", src.display()))?;
2546 if !crate::ask::valid_asset_name(name) {
2547 bail!(
2548 "attachment name `{name}` must match ^[A-Za-z0-9][A-Za-z0-9._-]{{0,63}}$ \
2549 with no `..`; rename the file and try again"
2550 );
2551 }
2552 if !src.is_file() {
2553 bail!("attachment `{}` is not a file", src.display());
2554 }
2555 wanted.push((src, name));
2556 }
2557 Ok(wanted)
2558}
2559
2560impl Queue {
2561 pub fn open() -> Self {
2563 Self::at(crate::run::home().join("queue"))
2564 }
2565
2566 pub fn at(root: PathBuf) -> Self {
2569 Self {
2570 root,
2571 ids: IdSource::new(new_id),
2572 }
2573 }
2574
2575 #[doc(hidden)]
2578 pub fn with_id_source(mut self, f: impl Fn() -> String + Send + Sync + 'static) -> Self {
2579 self.ids = IdSource::custom(f);
2580 self
2581 }
2582
2583 fn file_with(
2594 &self,
2595 task: &mut Task,
2596 hook: impl FnMut(&mut Task) -> Result<bool>,
2597 undo: impl FnMut(&mut Task),
2598 ) -> Result<()> {
2599 self.reserve_id(task, hook, undo, true)
2600 }
2601
2602 fn reserve_id(
2610 &self,
2611 task: &mut Task,
2612 mut hook: impl FnMut(&mut Task) -> Result<bool>,
2613 mut undo: impl FnMut(&mut Task),
2614 store: bool,
2615 ) -> Result<()> {
2616 let redraw_first = self.ids.is_custom() && task.attachments.is_empty();
2617 let initial = task.id.clone();
2618 self.ids
2619 .reserve((!redraw_first).then_some(initial.as_str()), "task", |id| {
2620 task.id = id.to_owned();
2621 let _lock = self.lock_task(&task.id)?;
2622 if path_exists(&self.path_of(&task.id))? || !hook(task)? {
2623 return Ok(false);
2624 }
2625 if store {
2626 if let Err(e) = self.write_record(task) {
2627 undo(task);
2628 return Err(e);
2629 }
2630 task.unfiled = false;
2631 }
2632 Ok(true)
2633 })?;
2634 Ok(())
2635 }
2636
2637 fn id_in_use(&self, id: &str, check_attachments: bool) -> Result<bool> {
2642 if path_exists(&self.lock_path(id))? {
2643 return Ok(true);
2644 }
2645 Ok(check_attachments && path_exists(&self.attachments_dir(id))?)
2646 }
2647
2648 fn plain_filing_hook(&self, task: &Task) -> Result<bool> {
2652 Ok(!self.id_in_use(&task.id, task.attachments.is_empty())?)
2653 }
2654
2655 pub fn file_claimed(&self, task: &mut Task) -> Result<Claim> {
2658 let mut claim = None;
2659 self.file_with(
2660 task,
2661 |t| {
2662 if self.id_in_use(&t.id, true)? {
2663 return Ok(false);
2664 }
2665 claim = Some(self.claim(&t.id)?);
2666 Ok(true)
2667 },
2668 |_| {},
2669 )?;
2670 claim.context("claim missing after filing")
2671 }
2672
2673 pub fn root(&self) -> &Path {
2675 &self.root
2676 }
2677
2678 pub fn path_of(&self, id: &str) -> PathBuf {
2680 self.root.join(format!("{id}.json"))
2681 }
2682
2683 pub fn attachments_dir(&self, id: &str) -> PathBuf {
2686 self.root.join(format!("{id}.attachments"))
2687 }
2688
2689 pub fn attach(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2700 let wanted = validate_attachments(sources)?;
2701 if task.unfiled && task.attachments.is_empty() {
2702 let names = std::cell::RefCell::new(Vec::new());
2705 self.reserve_id(
2706 task,
2707 |t| {
2708 if self.id_in_use(&t.id, true)? {
2709 return Ok(false);
2710 }
2711 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2712 Ok(true)
2713 },
2714 |_| {},
2715 false,
2716 )?;
2717 return Ok(names.into_inner());
2718 }
2719 self.copy_attachments(task, &wanted)
2720 }
2721
2722 fn copy_attachments(
2724 &self,
2725 task: &mut Task,
2726 wanted: &[(&PathBuf, &str)],
2727 ) -> Result<Vec<String>> {
2728 let dir = self.attachments_dir(&task.id);
2729 let existed = dir.is_dir();
2730 std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
2731 let mut created: Vec<PathBuf> = Vec::new();
2732 let mut names = Vec::new();
2733 let mut copy_all = || -> Result<()> {
2734 for (src, name) in wanted {
2735 let (stored, path) = copy_new(&dir, src, name)?;
2736 created.push(path);
2737 names.push(stored);
2738 }
2739 Ok(())
2740 };
2741 if let Err(e) = copy_all() {
2742 for path in &created {
2743 let _ = std::fs::remove_file(path);
2744 }
2745 if !existed {
2746 let _ = std::fs::remove_dir(&dir);
2747 }
2748 return Err(e);
2749 }
2750 task.attachments.extend(names.iter().cloned());
2751 Ok(names)
2752 }
2753
2754 pub fn attach_and_put(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2765 if task.unfiled && task.attachments.is_empty() {
2766 return self.attach_and_file(task, sources);
2767 }
2768 let dir = self.attachments_dir(&task.id);
2769 let existed = dir.is_dir();
2770 let before = task.attachments.len();
2771 let names = self.attach(task, sources)?;
2772 if let Err(e) = self.put(task) {
2773 for name in &names {
2774 let _ = std::fs::remove_file(dir.join(name));
2775 }
2776 if !existed {
2777 let _ = std::fs::remove_dir(&dir);
2778 }
2779 task.attachments.truncate(before);
2780 return Err(e);
2781 }
2782 Ok(names)
2783 }
2784
2785 fn attach_and_file(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2789 let wanted = validate_attachments(sources)?;
2790 let before = task.attachments.len();
2791 let names = std::cell::RefCell::new(Vec::new());
2792 let result = self.file_with(
2793 task,
2794 |t| {
2795 if self.id_in_use(&t.id, true)? {
2796 return Ok(false);
2797 }
2798 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2799 Ok(true)
2800 },
2801 |t| {
2802 let dir = self.attachments_dir(&t.id);
2803 for name in names.borrow().iter() {
2804 let _ = std::fs::remove_file(dir.join(name));
2805 }
2806 let _ = std::fs::remove_dir(&dir);
2808 t.attachments.truncate(before);
2809 },
2810 );
2811 result.map(|()| names.into_inner())
2812 }
2813
2814 pub fn attachment_paths(&self, task: &Task) -> Vec<PathBuf> {
2818 let dir = self.attachments_dir(&task.id);
2819 task.attachments
2820 .iter()
2821 .map(|n| {
2822 let p = dir.join(n);
2823 std::path::absolute(&p).unwrap_or(p)
2824 })
2825 .collect()
2826 }
2827
2828 pub fn put(&self, task: &mut Task) -> Result<()> {
2837 if task.unfiled {
2838 return self.file_with(task, |t| self.plain_filing_hook(t), |_| {});
2840 }
2841 let _lock = self.lock_task(&task.id)?;
2842 self.put_unlocked(task)
2843 }
2844
2845 pub fn create_new(&self, task: &mut Task) -> Result<bool> {
2850 let _lock = self.lock_task(&task.id)?;
2851 if path_exists(&self.path_of(&task.id))? {
2852 return Ok(false);
2853 }
2854 self.write_record(task)?;
2855 task.unfiled = false;
2856 Ok(true)
2857 }
2858
2859 fn put_unlocked(&self, task: &mut Task) -> Result<()> {
2861 self.merge_stored(task);
2862 if let Some(boundary) = task.reassignments.last() {
2863 let stored = self.get(&task.id)?;
2864 anyhow::ensure!(
2865 task.reassignments == stored.reassignments
2866 && task.repo == stored.repo
2867 && task.runs.get(..boundary.run_boundary)
2868 == stored.runs.get(..boundary.run_boundary),
2869 "historical repository execution cannot be restarted; reload the task"
2870 );
2871 }
2872 self.write_record(task)
2873 }
2874
2875 pub(crate) fn commit_base_recovery(
2877 &self,
2878 task: &mut Task,
2879 expected: &serde_json::Value,
2880 ) -> Result<()> {
2881 let _lock = self.lock_task(&task.id)?;
2882 anyhow::ensure!(
2883 serde_json::to_value(self.get(&task.id)?)? == *expected,
2884 "task changed during base recovery; inspect the pending authorization"
2885 );
2886 self.put_unlocked(task)
2888 }
2889
2890 pub(crate) fn commit_disk_wait(&self, task: &mut Task, expected: &Task) -> Result<()> {
2892 let _lock = self.lock_task(&task.id)?;
2893 anyhow::ensure!(
2894 serde_json::to_value(self.get(&task.id)?)? == serde_json::to_value(expected)?,
2895 "task changed during disk monitoring"
2896 );
2897 task.stamp_schema();
2898 self.write_record_inner(task, false)
2899 }
2900
2901 pub(crate) fn commit_reassignment(&self, task: &mut Task, expected: &Task) -> Result<()> {
2903 let _lock = self.lock_task(&task.id)?;
2904 let stored = self.get(&task.id)?;
2905 anyhow::ensure!(
2906 serde_json::to_value(&stored)? == serde_json::to_value(expected)?
2907 && stored.reassignments.len() + 1 == task.reassignments.len()
2908 && stored.repo == task.reassignments.last().expect("new boundary").from
2909 && stored.runs == task.runs,
2910 "task changed during reassignment; retry after inspecting it"
2911 );
2912 self.write_record(task)
2913 }
2914
2915 fn merge_stored(&self, task: &mut Task) {
2917 if let Ok(mut stored) = read_path(&self.path_of(&task.id)) {
2918 stored.import_chat_notice();
2919 if stored.reassignments.len() > task.reassignments.len() {
2920 let stale = std::mem::replace(task, stored.clone());
2923 for answer in stale.answers {
2924 if !task.answers.contains(&answer) {
2925 task.answers.push(answer);
2926 }
2927 }
2928 }
2929 for key in &stored.base_recovery_requests {
2930 if !task.base_recovery_requests.contains(key) {
2931 task.base_recovery_requests.push(key.clone());
2932 }
2933 }
2934 for receipt in &stored.repair_grants {
2935 if !task
2936 .repair_grants
2937 .iter()
2938 .any(|r| r.request.request == receipt.request.request)
2939 {
2940 task.repair_grants.push(receipt.clone());
2942 task.resume_override = stored.resume_override.clone();
2943 task.status = stored.status;
2944 task.hold_source = stored.hold_source;
2945 task.machine_hold_cause = stored.machine_hold_cause.clone();
2946 task.disk_resume_run = stored.disk_resume_run.clone();
2947 task.disk_checked_at = stored.disk_checked_at;
2948 task.hold_reason = stored.hold_reason.clone();
2949 task.held_at = stored.held_at;
2950 }
2951 }
2952 for receipt in &stored.repair_reconciliations {
2953 if !task.repair_reconciliations.contains(receipt) {
2954 task.repair_reconciliations.push(receipt.clone());
2957 task.resume_override = stored.resume_override.clone();
2958 task.status = stored.status;
2959 task.hold_source = stored.hold_source;
2960 task.machine_hold_cause = stored.machine_hold_cause.clone();
2961 task.disk_resume_run = stored.disk_resume_run.clone();
2962 task.disk_checked_at = stored.disk_checked_at;
2963 task.hold_reason = stored.hold_reason.clone();
2964 task.held_at = stored.held_at;
2965 }
2966 }
2967 task.superseded_resumes.extend(stored.superseded_resumes);
2968 task.dispatched_resumes.extend(stored.dispatched_resumes);
2969 if task.status != TaskStatus::Running
2970 && let Some(pin) = &stored.resume_override
2971 && !pin.question_id.is_empty()
2972 && let Some(run) = &pin.pinned_run
2973 && task.resume_override.as_ref().is_none_or(|next| {
2974 next.question_id != pin.question_id || next.pinned_run != pin.pinned_run
2975 })
2976 {
2977 task.superseded_resumes
2978 .insert(pin.question_id.clone(), run.clone());
2979 }
2980 for (id, receipt) in stored.decision_effects {
2983 task.decision_effects.insert(id, receipt);
2984 }
2985 if !task.decision_effects.is_empty() {
2986 task.stamp_schema();
2987 }
2988 if task.chat.is_none() {
2991 task.chat = stored.chat.clone();
2992 }
2993 for key in stored.chat_notices {
2994 if !task.chat_notices.contains(&key) {
2995 task.chat_notices.push(key);
2996 }
2997 }
2998 for failure in stored.startup_failures {
2999 task.record_startup_failure(failure);
3000 }
3001 for run in stored.runs {
3002 if !task.runs.contains(&run) {
3003 task.runs.push(run);
3004 }
3005 }
3006 for p in stored.passes {
3007 if !task
3008 .passes
3009 .iter()
3010 .any(|q| q.run == p.run && q.pass_index == p.pass_index)
3011 {
3012 task.passes.push(p);
3013 }
3014 }
3015 if let Some(n) = stored.chat_notice
3018 && task.chat_notice.as_ref().is_none_or(|mine| mine.at < n.at)
3019 {
3020 task.chat_notice = Some(n);
3021 }
3022 }
3023 }
3024
3025 pub(crate) fn write_startup_history(&self, task: &Task) -> Result<()> {
3028 let path = self.path_of(&task.id);
3029 let tmp = path.with_extension("json.tmp");
3030 std::fs::write(&tmp, serde_json::to_vec_pretty(task)?)?;
3031 std::fs::rename(tmp, path)?;
3032 Ok(())
3033 }
3034
3035 fn write_record(&self, task: &mut Task) -> Result<()> {
3037 self.write_record_inner(task, true)
3038 }
3039
3040 fn write_record_inner(&self, task: &mut Task, notify: bool) -> Result<()> {
3041 task.import_chat_notice();
3042 task.updated_at = Timestamp::now();
3043 std::fs::create_dir_all(&self.root)
3044 .with_context(|| format!("create {}", self.root.display()))?;
3045 let body = serde_json::to_string_pretty(task).context("serialize task")?;
3046 let path = self.path_of(&task.id);
3047 let tmp = path.with_extension("json.tmp");
3048 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
3049 std::fs::rename(&tmp, &path).with_context(|| format!("replace {}", path.display()))?;
3050 if let (Some(notice), Some(home)) = (
3055 notify.then(|| crate::notices::task_held(task)).flatten(),
3056 self.root.parent().filter(|p| !p.as_os_str().is_empty()),
3057 ) {
3058 crate::notices::raise_in(home, notice);
3059 }
3060 Ok(())
3061 }
3062
3063 pub fn modify(&self, id: &str, f: impl FnOnce(&mut Task) -> bool) -> Result<bool> {
3068 let id = self.resolve_id(id)?;
3069 let _lock = self.lock_task(&id)?;
3070 let mut task = self.get(&id)?;
3071 if !f(&mut task) {
3072 return Ok(false);
3073 }
3074 self.put_unlocked(&mut task)?;
3075 Ok(true)
3076 }
3077
3078 pub fn link_run(&self, id: &str, run: &str) -> Result<Task> {
3083 let id = self.resolve_id(id)?;
3084 let _lock = self.lock_task(&id)?;
3087 let mut task = self.get(&id)?;
3088 if task.link_run(run) {
3089 self.put_unlocked(&mut task)?;
3090 }
3091 Ok(task)
3092 }
3093
3094 pub(crate) fn lock_task(&self, id: &str) -> Result<TaskLock> {
3102 std::fs::create_dir_all(&self.root)
3103 .with_context(|| format!("create {}", self.root.display()))?;
3104 let path = self.root.join(format!("{id}.write-lock"));
3105 let started = std::time::Instant::now();
3106 let retry = crate::lock_file::Retry::task_write(started + TASK_LOCK_STALE);
3107 let marker_retry = crate::lock_file::Retry::new(Some(started + TASK_LOCK_STALE));
3108 loop {
3109 match retry.create(&path) {
3110 Ok(mut file) => {
3111 use std::io::Write;
3112 let token = owner_token();
3113 if let Err(e) = file.write_all(token.as_bytes()) {
3114 drop(file);
3115 let _ = std::fs::remove_file(&path);
3116 return Err(e).with_context(|| format!("lock {}", path.display()));
3117 }
3118 return Ok(TaskLock { path, token });
3119 }
3120 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
3121 let judged = std::fs::read_to_string(&path).ok();
3124 let broken = judged
3125 .filter(|_| older_than_stale(&path))
3126 .is_some_and(|judged| break_stale_with(&path, &judged, marker_retry));
3127 if !broken {
3131 if started.elapsed() > TASK_LOCK_STALE {
3132 bail!("could not lock task {id}");
3133 }
3134 std::thread::sleep(std::time::Duration::from_millis(15));
3135 }
3136 }
3137 Err(e) => return Err(e).with_context(|| format!("lock {}", path.display())),
3138 }
3139 }
3140 }
3141
3142 pub fn get(&self, id: &str) -> Result<Task> {
3144 let resolved = self.resolve_id(id)?;
3145 read_path(&self.path_of(&resolved))
3146 }
3147
3148 pub fn remove(&self, id: &str, in_flight: bool, questions: &Questions) -> Result<Removal> {
3180 self.remove_with_receipt(id, in_flight, questions, None)
3181 }
3182
3183 pub(crate) fn discard_triage(
3185 &self,
3186 id: &str,
3187 questions: &Questions,
3188 inquiry: &str,
3189 ) -> Result<Removal> {
3190 self.remove_with_receipt(id, false, questions, Some(inquiry))
3191 }
3192
3193 pub(crate) fn triage_discard_applied(&self, id: &str, inquiry: &str) -> Result<bool> {
3194 let path = self
3195 .root
3196 .join("triage-discard")
3197 .join(format!("{inquiry}.json"));
3198 let bytes = match std::fs::read(path) {
3199 Ok(bytes) => bytes,
3200 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
3201 Err(e) => return Err(e.into()),
3202 };
3203 let receipt: Task = serde_json::from_slice(&bytes)?;
3204 Ok(receipt.id == id && !self.path_of(id).exists())
3205 }
3206
3207 fn remove_with_receipt(
3208 &self,
3209 id: &str,
3210 in_flight: bool,
3211 questions: &Questions,
3212 inquiry: Option<&str>,
3213 ) -> Result<Removal> {
3214 let resolved = self.resolve_id(id)?;
3215 if in_flight {
3216 bail!("task {resolved} is being run by a live daemon right now");
3217 }
3218 let receipt = inquiry.map(|q| self.root.join("triage-discard").join(format!("{q}.json")));
3219 if let Some(path) = &receipt {
3220 std::fs::create_dir_all(path.parent().expect("receipt directory"))?;
3221 }
3222 self.write_tombstone(&resolved)?;
3223 if let Err(e) = self.remove_record_with_attachments(&resolved, |p| match &receipt {
3224 Some(path) => std::fs::rename(p, path),
3225 None => std::fs::remove_file(p),
3226 }) {
3227 if self.path_of(&resolved).exists() {
3231 let _ = std::fs::remove_file(self.tombstone_path(&resolved));
3232 }
3233 return Err(e);
3234 }
3235 let (released, still_blocked) = self.release_dependents_of(&resolved, questions);
3236 Ok(Removal {
3237 id: resolved,
3238 released,
3239 still_blocked,
3240 })
3241 }
3242
3243 fn tombstone_path(&self, id: &str) -> PathBuf {
3246 self.root.join(format!("{id}.removed"))
3247 }
3248
3249 fn write_tombstone(&self, id: &str) -> Result<()> {
3250 let path = self.tombstone_path(id);
3251 let tmp = self.root.join(format!("{id}.removed.tmp"));
3252 std::fs::write(&tmp, b"")
3253 .and_then(|()| std::fs::rename(&tmp, &path))
3254 .with_context(|| format!("write {}", path.display()))
3255 }
3256
3257 fn was_deleted(&self, id: &str) -> bool {
3261 self.tombstone_path(id).is_file() && !self.path_of(id).exists()
3262 }
3263
3264 pub fn apply_deleted_blockers(&self, task: &mut Task, questions: &Questions) -> Vec<String> {
3268 let deleted = deleted_blockers(self, questions, &task.blocked_by);
3269 deleted
3270 .into_iter()
3271 .filter(|id| task.dependency_deleted(id))
3272 .collect()
3273 }
3274
3275 pub fn note_dependency_deleted(&self, dependent: &Task, deleted: &str) {
3279 let Some(home) = self.root.parent().filter(|p| !p.as_os_str().is_empty()) else {
3280 return;
3281 };
3282 let ja = crate::lang::is_japanese(&crate::lang::of_repo(&dependent.repo));
3283 let waiting = dependent.status == TaskStatus::Blocked;
3284 let message = match (ja, waiting) {
3285 (true, false) => format!(
3286 "タスク {} は、待っていた {} が削除されたため待機を解除し、元の状態に戻しました",
3287 dependent.short(),
3288 short(deleted)
3289 ),
3290 (true, true) => format!(
3291 "タスク {} は、待っていた {} が削除されたため、残りの依存を待っています",
3292 dependent.short(),
3293 short(deleted)
3294 ),
3295 (false, false) => format!(
3296 "Task {} stopped waiting on {} because it was deleted, and returned to its previous state",
3297 dependent.short(),
3298 short(deleted)
3299 ),
3300 (false, true) => format!(
3301 "Task {} stopped waiting on {} because it was deleted, and is still waiting on its other dependencies",
3302 dependent.short(),
3303 short(deleted)
3304 ),
3305 };
3306 crate::notices::raise_in(
3307 home,
3308 crate::notices::Notice::info(
3309 &format!("unblocked:{}:{}", dependent.id, deleted),
3310 message,
3311 )
3312 .link(crate::notices::Link::Task {
3313 id: dependent.id.clone(),
3314 }),
3315 );
3316 }
3317
3318 fn release_dependents_of(
3323 &self,
3324 dependency: &str,
3325 questions: &Questions,
3326 ) -> (Vec<String>, Vec<String>) {
3327 let (mut released, mut still_blocked) = (Vec::new(), Vec::new());
3328 for listed in self.list() {
3329 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| {
3330 matches!(crate::queue::dependency(self, questions, b), Dependency::Deleted(id) if id == dependency)
3331 }) {
3332 continue;
3333 }
3334 let Ok(_claim) = self.claim(&listed.id) else {
3335 continue;
3336 };
3337 let Ok(mut task) = self.get(&listed.id) else {
3338 continue;
3339 };
3340 let references: Vec<String> = task.blocked_by.iter().filter(|id| {
3341 matches!(crate::queue::dependency(self, questions, id), Dependency::Deleted(full) if full == dependency)
3342 }).cloned().collect();
3343 if references.is_empty() {
3344 continue;
3345 }
3346 for reference in references {
3347 task.dependency_deleted(&reference);
3348 }
3349 if self.put(&mut task).is_err() {
3350 continue;
3351 }
3352 self.note_dependency_deleted(&task, dependency);
3353 if task.status == TaskStatus::Blocked {
3354 still_blocked.push(task.id.clone());
3355 } else {
3356 released.push(task.id.clone());
3357 }
3358 }
3359 (released, still_blocked)
3360 }
3361
3362 fn remove_record_with_attachments(
3366 &self,
3367 resolved: &str,
3368 remove_record: impl FnOnce(&Path) -> std::io::Result<()>,
3369 ) -> Result<()> {
3370 self.sweep_removed_attachments();
3377 let attachments = self.attachments_dir(resolved);
3378 let aside = self.root.join(format!("{resolved}.attachments.removing"));
3379 let moved = match std::fs::rename(&attachments, &aside) {
3380 Ok(()) => true,
3381 Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
3382 Err(e) => {
3383 return Err(e).with_context(|| format!("remove {}", attachments.display()));
3384 }
3385 };
3386 let path = self.path_of(resolved);
3387 if let Err(e) = remove_record(&path) {
3388 if moved {
3389 let _ = std::fs::rename(&aside, &attachments);
3390 }
3391 return Err(e).with_context(|| format!("remove {}", path.display()));
3392 }
3393 if moved {
3394 if let Err(e) = std::fs::remove_dir_all(&aside) {
3395 tracing::warn!("leftover attachments {}: {e}", aside.display());
3396 }
3397 }
3398 let lock = self.lock_path(resolved);
3399 if let Err(e) = std::fs::remove_file(&lock) {
3400 if e.kind() != std::io::ErrorKind::NotFound {
3401 return Err(e).with_context(|| format!("remove {}", lock.display()));
3402 }
3403 }
3404 Ok(())
3405 }
3406
3407 fn sweep_removed_attachments(&self) {
3413 let Ok(entries) = std::fs::read_dir(&self.root) else {
3414 return;
3415 };
3416 for entry in entries.flatten() {
3417 let name = entry.file_name();
3418 let name = name.to_string_lossy();
3419 let Some(id) = name.strip_suffix(".attachments.removing") else {
3420 continue;
3421 };
3422 if !self.path_of(id).exists() {
3425 if let Err(e) = std::fs::remove_dir_all(entry.path()) {
3426 tracing::warn!("leftover attachments {}: {e}", entry.path().display());
3427 }
3428 }
3429 }
3430 self.sweep_tombstones();
3431 }
3432
3433 fn sweep_tombstones(&self) {
3442 let Ok(entries) = std::fs::read_dir(&self.root) else {
3443 return;
3444 };
3445 let tasks = self.list();
3446 for entry in entries.flatten() {
3447 let name = entry.file_name();
3448 let name = name.to_string_lossy();
3449 let Some(id) = name.strip_suffix(".removed") else {
3450 continue;
3451 };
3452 if self.path_of(id).exists()
3453 || tasks.iter().any(|t| {
3454 t.blocked_by
3455 .iter()
3456 .any(|b| id.starts_with(b) || id.ends_with(b))
3457 })
3458 {
3459 continue;
3460 }
3461 let old = entry
3462 .metadata()
3463 .and_then(|m| m.modified())
3464 .ok()
3465 .and_then(|m| m.elapsed().ok())
3466 .is_some_and(|age| age >= TOMBSTONE_GRACE);
3467 if old {
3468 let _ = std::fs::remove_file(entry.path());
3469 }
3470 }
3471 }
3472
3473 fn lock_path(&self, id: &str) -> PathBuf {
3476 self.root.join(format!("{id}.lock"))
3477 }
3478
3479 pub fn list(&self) -> Vec<Task> {
3492 #[cfg(test)]
3493 LIST_READS.with(|count| count.set(count.get() + 1));
3494 let mut tasks: Vec<Task> = std::fs::read_dir(&self.root)
3495 .into_iter()
3496 .flatten()
3497 .flatten()
3498 .map(|e| e.path())
3499 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3500 .filter_map(|p| read_path(&p).ok())
3501 .collect();
3502 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then_with(|| b.id.cmp(&a.id)));
3503 tasks
3504 }
3505
3506 pub fn unreadable(&self) -> Vec<PathBuf> {
3510 std::fs::read_dir(&self.root)
3511 .into_iter()
3512 .flatten()
3513 .flatten()
3514 .map(|e| e.path())
3515 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3516 .filter(|p| read_path(p).is_err())
3517 .collect()
3518 }
3519
3520 pub fn superseded(&self) -> HashMap<String, String> {
3533 let mut by = HashMap::new();
3534 for task in self.list() {
3535 for earlier in &task.runs {
3536 if let Some(later) = task.successor_of(earlier) {
3537 by.insert(earlier.clone(), later.clone());
3538 }
3539 }
3540 }
3541 by
3542 }
3543
3544 pub fn superseded_by(&self, run: &str) -> Option<String> {
3552 for task in self.list() {
3553 if task.runs.iter().any(|r| r == run) {
3554 return task.successor_of(run).cloned();
3555 }
3556 }
3557 None
3558 }
3559
3560 pub fn latest_attempt(&self, run: &str) -> Option<String> {
3571 for task in self.list() {
3572 if task.runs.iter().any(|r| r == run) {
3573 return task.runs.last().filter(|last| **last != run).cloned();
3574 }
3575 }
3576 None
3577 }
3578
3579 pub fn next_runnable(&self) -> Option<Task> {
3584 let mut runnable: Vec<Task> = self
3585 .list()
3586 .into_iter()
3587 .filter(|t| t.status.runnable())
3588 .collect();
3589 runnable.sort_unstable_by(dispatch_order);
3590 runnable.into_iter().next()
3591 }
3592
3593 pub fn claim(&self, id: &str) -> Result<Claim> {
3600 std::fs::create_dir_all(&self.root)
3601 .with_context(|| format!("create {}", self.root.display()))?;
3602 let path = self.lock_path(id);
3603 let retry = crate::lock_file::Retry::new(None);
3604 match retry.create(&path) {
3605 Ok(mut f) => {
3606 use std::io::Write as _;
3607 let _ = writeln!(f, "{}", std::process::id());
3609 Ok(Claim { path })
3610 }
3611 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
3612 bail!("task {id} is already claimed ({} exists)", path.display())
3613 }
3614 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
3615 }
3616 }
3617
3618 pub fn resolve_id(&self, prefix: &str) -> Result<String> {
3620 if !prefix.is_empty()
3621 && prefix != "."
3622 && prefix != ".."
3623 && !prefix.contains(['/', '\\'])
3624 && self.path_of(prefix).is_file()
3625 {
3626 return Ok(prefix.to_owned());
3627 }
3628 let ids = record_ids(&self.root, "json")?;
3629 match resolve_reference(prefix, ids.iter().map(String::as_str)) {
3630 Reference::Found(id) => Ok(id),
3631 Reference::Missing => bail!("no task matches `{prefix}`"),
3632 Reference::Ambiguous => bail!("`{prefix}` matches multiple tasks"),
3633 }
3634 }
3635
3636 pub fn revision(&self) -> u64 {
3643 self.revision_excluding(&std::collections::BTreeSet::new())
3644 }
3645
3646 pub fn revision_excluding(&self, skip: &std::collections::BTreeSet<String>) -> u64 {
3649 use std::hash::{Hash as _, Hasher as _};
3650
3651 let mut entries: Vec<(String, u64)> = std::fs::read_dir(&self.root)
3652 .into_iter()
3653 .flatten()
3654 .flatten()
3655 .filter(|e| e.path().extension().is_some_and(|ext| ext == "json"))
3656 .filter(|e| {
3657 let path = e.path();
3658 !path
3659 .file_stem()
3660 .is_some_and(|stem| skip.contains(stem.to_string_lossy().as_ref()))
3661 })
3662 .filter_map(|e| {
3663 let name = e.file_name().to_string_lossy().into_owned();
3664 let mtime = e
3665 .metadata()
3666 .ok()?
3667 .modified()
3668 .ok()?
3669 .duration_since(std::time::UNIX_EPOCH)
3670 .ok()?
3671 .as_millis() as u64;
3672 Some((name, mtime))
3673 })
3674 .collect();
3675
3676 if entries.is_empty() {
3677 return 0;
3678 }
3679
3680 entries.sort_unstable();
3681 let mut hasher = std::hash::DefaultHasher::new();
3682 for (name, mtime) in &entries {
3683 name.hash(&mut hasher);
3684 mtime.hash(&mut hasher);
3685 }
3686 let h = hasher.finish();
3687 if h == 0 { 1 } else { h }
3688 }
3689}
3690
3691const TOMBSTONE_GRACE: std::time::Duration = std::time::Duration::from_secs(3600);
3694
3695#[derive(Debug, Clone)]
3697pub struct Removal {
3698 pub id: String,
3700 pub released: Vec<String>,
3704 pub still_blocked: Vec<String>,
3707}
3708
3709#[derive(Debug)]
3711pub struct Claim {
3712 path: PathBuf,
3713}
3714
3715impl Drop for Claim {
3716 fn drop(&mut self) {
3717 let _ = std::fs::remove_file(&self.path);
3718 }
3719}
3720
3721pub fn title_from(instruction: &str, max: usize) -> String {
3724 let Some(line) = first_line(instruction) else {
3725 return "(empty task)".to_owned();
3726 };
3727 if line.chars().count() <= max {
3728 return line.to_owned();
3729 }
3730 let head: String = line.chars().take(max.saturating_sub(1)).collect();
3731 format!("{head}…")
3732}
3733
3734pub(crate) fn first_line(instruction: &str) -> Option<&str> {
3740 let line = instruction
3741 .lines()
3742 .map(str::trim)
3743 .find(|l| !l.is_empty())?
3744 .trim_start_matches(['#', '-', '*', '>', ' '])
3745 .trim();
3746 (!line.is_empty()).then_some(line)
3747}
3748
3749#[cfg(test)]
3750thread_local! {
3751 pub(crate) static TASK_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
3752}
3753
3754fn read_path(path: &Path) -> Result<Task> {
3755 #[cfg(test)]
3756 TASK_READS.with(|count| count.set(count.get() + 1));
3757 let body = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
3758 let mut task: Task =
3759 serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
3760 if task.schema > SCHEMA {
3766 bail!(
3767 "task {} was written by a different magi (schema {}, this build \
3768 speaks {SCHEMA})",
3769 task.id,
3770 task.schema
3771 );
3772 }
3773 task.preserve_legacy_build_cache();
3774 if task.schema < 22
3775 && task.status == TaskStatus::Held
3776 && task.hold_source == Some(HoldSource::Machine)
3777 && task.resume_override.is_none()
3778 && task
3779 .hold_reason
3780 .as_deref()
3781 .is_some_and(crate::disk::is_gate_reason)
3782 {
3783 task.mark_disk_capacity();
3784 }
3785 Ok(task)
3786}
3787
3788#[derive(Debug, PartialEq, Eq)]
3791pub(crate) enum Reference {
3792 Found(String),
3793 Missing,
3794 Ambiguous,
3795}
3796
3797pub(crate) fn resolve_reference<'a>(id: &str, ids: impl Iterator<Item = &'a str>) -> Reference {
3798 let mut hits = Vec::new();
3799 for candidate in ids {
3800 if candidate == id {
3801 return Reference::Found(candidate.to_owned());
3802 }
3803 if candidate.starts_with(id) || candidate.ends_with(id) {
3804 hits.push(candidate);
3805 }
3806 }
3807 match hits.as_slice() {
3808 [] => Reference::Missing,
3809 [one] => Reference::Found((*one).to_owned()),
3810 _ => Reference::Ambiguous,
3811 }
3812}
3813
3814pub(crate) fn record_ids(root: &Path, extension: &str) -> Result<Vec<String>> {
3817 let entries = match std::fs::read_dir(root) {
3818 Ok(entries) => entries,
3819 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
3820 Err(e) => return Err(e.into()),
3821 };
3822 let mut ids = Vec::new();
3823 for entry in entries {
3824 let path = entry?.path();
3825 if path.extension().is_some_and(|e| e == extension)
3826 && let Some(id) = path.file_stem().and_then(|s| s.to_str())
3827 {
3828 ids.push(id.to_owned());
3829 }
3830 }
3831 Ok(ids)
3832}
3833
3834#[derive(Debug, PartialEq, Eq)]
3837pub enum Dependency {
3838 Task(String),
3840 Question(String),
3842 Deleted(String),
3844 Missing,
3846 Ambiguous,
3848 Unknown,
3850}
3851
3852pub fn dependency(queue: &Queue, questions: &Questions, id: &str) -> Dependency {
3854 let (Ok(tasks), Ok(qs), Ok(removed)) = (
3855 record_ids(queue.root(), "json"),
3856 record_ids(questions.root(), "json"),
3857 record_ids(queue.root(), "removed"),
3858 ) else {
3859 return Dependency::Unknown;
3860 };
3861 let mut ids = tasks
3864 .iter()
3865 .chain(qs.iter())
3866 .chain(removed.iter())
3867 .cloned()
3868 .collect::<Vec<_>>();
3869 ids.sort();
3870 ids.dedup();
3871 match resolve_reference(id, ids.iter().map(String::as_str)) {
3872 Reference::Found(id) if tasks.contains(&id) => Dependency::Task(id),
3873 Reference::Found(id) if qs.contains(&id) => Dependency::Question(id),
3874 Reference::Found(id) if queue.was_deleted(&id) => Dependency::Deleted(id),
3875 Reference::Found(_) => Dependency::Unknown,
3876 Reference::Missing => Dependency::Missing,
3877 Reference::Ambiguous => Dependency::Ambiguous,
3878 }
3879}
3880
3881pub fn normalize_blockers(queue: &Queue, questions: &Questions, ids: &mut [String]) {
3883 for id in ids {
3884 if let Dependency::Task(full) | Dependency::Question(full) | Dependency::Deleted(full) =
3885 dependency(queue, questions, id)
3886 {
3887 *id = full;
3888 }
3889 }
3890}
3891
3892pub fn dependency_facts(queue: &Queue, questions: &Questions) -> String {
3895 let inventory = crate::blockers::Inventory::load(queue, questions);
3896 let mut text = String::from(
3897 "Current authoritative queue (historical missing-task claims may be stale; unreadable or ambiguous references must keep waiting):\n",
3898 );
3899 match record_ids(queue.root(), "json") {
3900 Ok(ids) => {
3901 for id in ids {
3902 match queue.get(&id) {
3903 Ok(task) => text.push_str(&format!(
3904 "{}: {}; blocked_by={:?}; chain={:?}\n",
3905 task.id,
3906 task.status.as_str(),
3907 task.blocked_by,
3908 inventory.waits_on(&task)
3909 )),
3910 Err(_) => text.push_str(&format!("{id}: exists, state unreadable\n")),
3911 }
3912 }
3913 }
3914 Err(_) => text.push_str("Queue inventory unreadable: existence is unknown.\n"),
3915 }
3916 text
3917}
3918
3919pub fn missing_blockers(
3921 queue: &Queue,
3922 questions: &Questions,
3923 blocked_by: &[String],
3924) -> Vec<String> {
3925 blocked_by
3926 .iter()
3927 .filter(|id| dependency(queue, questions, id) == Dependency::Missing)
3928 .cloned()
3929 .collect()
3930}
3931
3932pub fn deleted_blockers(
3935 queue: &Queue,
3936 questions: &Questions,
3937 blocked_by: &[String],
3938) -> Vec<String> {
3939 blocked_by
3940 .iter()
3941 .filter(|id| matches!(dependency(queue, questions, id), Dependency::Deleted(_)))
3942 .cloned()
3943 .collect()
3944}
3945
3946pub fn missing_blocker_hold_reason(blocked_by: &[String], missing: &[String]) -> String {
3958 missing_blocker_hold_reason_in(blocked_by, missing, "en")
3959}
3960
3961pub fn missing_blocker_hold_reason_in(
3963 blocked_by: &[String],
3964 missing: &[String],
3965 language: &str,
3966) -> String {
3967 if crate::lang::is_japanese(language) {
3968 format!(
3969 "{} を待っていましたが、{} はディスク上に存在しません - `magi task triage` を参照",
3970 blocked_by.join(", "),
3971 missing.join(", "),
3972 )
3973 } else {
3974 format!(
3975 "blocked on {} but {} no longer exist(s) on disk - see `magi task triage`",
3976 blocked_by.join(", "),
3977 missing.join(", "),
3978 )
3979 }
3980}
3981
3982pub fn short(id: &str) -> &str {
3984 id.split('-').next_back().unwrap_or(id)
3985}
3986
3987fn new_id() -> String {
3988 let stamp = jiff::Zoned::now().strftime("%Y%m%d-%H%M%S");
3989 let seed = crate::rng::entropy();
3990 format!("{stamp}-{:04x}", (seed ^ (seed >> 32)) & 0xffff)
3991}
3992
3993#[cfg(test)]
3994mod tests {
3995 #[test]
3996 fn reassignment_clears_repository_bound_execution_selections() {
3997 let mut task = Task::new(
3998 "move".into(),
3999 "evidence".into(),
4000 "old".into(),
4001 Source::Human,
4002 );
4003 task.review_branch = Some("old-review".into());
4004 task.review_instruction = Some("old criteria".into());
4005 task.pr_recovery = Some(crate::pr_recovery::Selection {
4006 url: "https://example.invalid/pull/1".into(),
4007 branch: "old-pr".into(),
4008 head: "head".into(),
4009 source_head: None,
4010 source_run: None,
4011 source_proof: None,
4012 disposition: None,
4013 instruction: "old criteria".into(),
4014 approved_at: Timestamp::now(),
4015 });
4016 task.replacement = Some(crate::merge_decision::Replacement {
4017 question: "owner".into(),
4018 run: "old-run".into(),
4019 pr: "https://example.invalid/pull/1".into(),
4020 agent: "worker".into(),
4021 context: "evidence".into(),
4022 });
4023 task.resume_override = Some(OperatorResume {
4024 question_id: "old-question".into(),
4025 at: Timestamp::now(),
4026 conductor_rehold: None,
4027 forced: true,
4028 pinned_run: Some("old-run".into()),
4029 });
4030 task.interrupt = true;
4031 task.reassign_repository("new".into(), "test");
4032 assert!(task.review_branch.is_none());
4033 assert!(task.review_instruction.is_none());
4034 assert!(task.pr_recovery.is_none());
4035 assert!(task.replacement.is_none());
4036 assert!(task.resume_override.is_none());
4037 assert!(!task.interrupt);
4038 assert!(task.fresh_start);
4039 assert_eq!(task.instruction, "evidence");
4040 }
4041
4042 #[test]
4043 fn explicit_resumption_is_not_invented_for_legacy_or_automatic_release() {
4044 let t = Task::new(
4045 "retry".into(),
4046 "do it".into(),
4047 PathBuf::from("."),
4048 Source::Human,
4049 );
4050 let mut value = serde_json::to_value(&t).unwrap();
4051 for field in ["resumption", "resumption_history", "operator_decided_at"] {
4052 value.as_object_mut().unwrap().remove(field);
4053 }
4054 value["schema"] = serde_json::json!(15);
4055 let mut legacy: Task = serde_json::from_value(value).unwrap();
4056 legacy.hold_machine(Some("automatic retry".into()));
4057 legacy.release();
4058 assert!(legacy.resumption.is_none());
4059 assert!(legacy.resumption_history.is_empty());
4060 assert!(legacy.operator_decided_at.is_none());
4061 legacy.release_operator("cli", None);
4062 legacy.start("run".into());
4063 let reloaded: Task =
4064 serde_json::from_str(&serde_json::to_string(&legacy).unwrap()).unwrap();
4065 assert!(reloaded.resume_override.is_none());
4066 assert_eq!(reloaded.resumption.as_ref().unwrap().generation, 1);
4067 }
4068
4069 #[test]
4070 fn merge_over_lets_the_later_choice_win_and_keeps_the_rest() {
4071 let mut base = RunOverrides {
4072 merge: Some("pr".to_owned()),
4073 candidates: Some(3),
4074 ..RunOverrides::default()
4075 };
4076 base.merge_over(&RunOverrides {
4077 merge: Some("none".to_owned()),
4078 seed: Some(7),
4079 ..RunOverrides::default()
4080 });
4081 assert_eq!(base.merge.as_deref(), Some("none"));
4082 assert_eq!(base.candidates, Some(3));
4083 assert_eq!(base.seed, Some(7));
4084 }
4085
4086 #[test]
4087 fn an_already_landed_task_is_done_with_its_attempt_refunded() {
4088 let mut t = task("relanded");
4089 t.attempts = 1;
4090 t.status = TaskStatus::Running;
4091 t.already_landed("already in main as 0e368de");
4092 assert_eq!(t.status, TaskStatus::Done);
4093 assert_eq!(t.attempts, 0);
4094 assert!(t.hold_reason.is_none());
4095 assert_eq!(t.last_error.as_deref(), Some("already in main as 0e368de"));
4096 }
4097
4098 #[test]
4099 fn missing_blocker_reason_follows_the_language() {
4100 let b = vec!["a".to_owned()];
4101 let en = missing_blocker_hold_reason_in(&b, &b, "en");
4102 assert_eq!(en, missing_blocker_hold_reason(&b, &b));
4103 assert!(en.starts_with("blocked on a"));
4104 assert!(missing_blocker_hold_reason_in(&b, &b, "ja").contains("存在しません"));
4105 assert_eq!(missing_blocker_hold_reason_in(&b, &b, "de"), en);
4106 }
4107
4108 use super::*;
4109
4110 #[test]
4111 fn triage_applied_survives_release_and_old_records_read_as_empty() {
4112 let mut t = Task::new(
4113 "t".to_owned(),
4114 "i".to_owned(),
4115 PathBuf::from("r"),
4116 Source::Human,
4117 );
4118 t.mark_triage_applied("q1");
4119 t.mark_triage_applied("q1");
4120 t.hold_machine(Some("x".to_owned()));
4121 t.release();
4122 assert_eq!(t.triage_applied, ["q1"]);
4123 assert!(t.triage_applied("q1") && !t.triage_applied("q2"));
4124
4125 let mut v = serde_json::to_value(&t).unwrap();
4126 v.as_object_mut().unwrap().remove("triage_applied");
4127 let old: Task = serde_json::from_value(v).unwrap();
4128 assert!(old.triage_applied.is_empty());
4129 }
4130
4131 #[test]
4132 fn task_counts_of_empty_is_all_zero() {
4133 assert_eq!(TaskCounts::of(&[]), TaskCounts::default());
4134 }
4135
4136 #[test]
4137 fn task_counts_of_tallies_every_status() {
4138 let mut queued = Task::new(
4139 "q".to_owned(),
4140 "i".to_owned(),
4141 PathBuf::from("."),
4142 Source::Human,
4143 );
4144 queued.status = TaskStatus::Queued;
4145 let mut running = queued.clone();
4146 running.status = TaskStatus::Running;
4147 let mut done = queued.clone();
4148 done.status = TaskStatus::Done;
4149 let mut failed = queued.clone();
4150 failed.status = TaskStatus::Failed;
4151 let mut held = queued.clone();
4152 held.status = TaskStatus::Held;
4153 let mut blocked = queued.clone();
4154 blocked.status = TaskStatus::Blocked;
4155
4156 let mut parked = queued.clone();
4157 parked.status = TaskStatus::Parked;
4158
4159 let counts = TaskCounts::of(&[
4160 queued,
4161 running,
4162 done.clone(),
4163 done,
4164 failed,
4165 held,
4166 blocked,
4167 parked,
4168 ]);
4169 assert_eq!(
4170 counts,
4171 TaskCounts {
4172 queued: 1,
4173 running: 1,
4174 done: 2,
4175 failed: 1,
4176 held: 1,
4177 blocked: 1,
4178 parked: 1,
4179 }
4180 );
4181 }
4182
4183 #[test]
4184 fn task_implementers_persist_normalized_without_losing_history() {
4185 let (_dir, q) = queue();
4186 let mut t = task("keep me");
4187 t.hold_manual(Some("waiting".into()));
4188 t.runs.push("earlier-run".into());
4189 t.attachments.push("reference.txt".into());
4190 let cfg: crate::config::Config =
4191 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
4192 .unwrap();
4193 q.put(&mut t).unwrap();
4194 let original = serde_json::to_value(&t).unwrap();
4195 for n in [2, 1, 2] {
4196 t.set_implementers(n, &cfg).unwrap();
4197 q.put(&mut t).unwrap();
4198 t = q.get(&t.id).unwrap();
4199 assert_eq!(t.implementers(), Some(usize::from(n)));
4200 assert_eq!(t.solo, n == 1);
4201 if n == 1 {
4202 assert!(t.overrides.is_none());
4203 }
4204 let mut actual = serde_json::to_value(&t).unwrap();
4205 for field in ["solo", "overrides", "updated_at"] {
4206 actual[field] = original[field].clone();
4207 }
4208 assert_eq!(actual, original);
4209 }
4210 t.overrides.as_mut().unwrap().judges = Some(2);
4211 t.set_implementers(1, &cfg).unwrap();
4212 assert_eq!(t.overrides.as_ref().unwrap().judges, Some(2));
4213 assert_eq!(t.overrides.as_ref().unwrap().candidates, None);
4214 }
4215
4216 #[test]
4217 fn task_implementers_old_records_keep_inheritance_until_changed() {
4218 let mut value = serde_json::to_value(task("old task")).unwrap();
4219 value.as_object_mut().unwrap().remove("solo");
4220 value.as_object_mut().unwrap().remove("overrides");
4221 let mut old: Task = serde_json::from_value(value).unwrap();
4222 assert_eq!(old.implementers(), None);
4223 old.set_implementers(1, &crate::config::Config::default())
4224 .unwrap();
4225 assert!(old.solo);
4226 assert!(old.overrides.is_none());
4227 }
4228
4229 #[test]
4230 fn task_implementers_reject_invalid_counts_and_states() {
4231 let cfg = crate::config::Config::default();
4232 let mut t = task("unchanged");
4233 for n in [0, 4, 255] {
4234 assert!(t.set_implementers(n, &cfg).is_err());
4235 }
4236 for status in [
4237 TaskStatus::Running,
4238 TaskStatus::Done,
4239 TaskStatus::Failed,
4240 TaskStatus::Blocked,
4241 TaskStatus::Parked,
4242 ] {
4243 t.status = status;
4244 let before = serde_json::to_value(&t).unwrap();
4245 assert!(t.set_implementers(1, &cfg).is_err());
4246 assert_eq!(serde_json::to_value(&t).unwrap(), before);
4247 }
4248 }
4249
4250 fn queue() -> (tempfile::TempDir, Queue) {
4253 let dir = tempfile::tempdir().unwrap();
4254 let q = Queue::at(dir.path().join("queue"));
4255 (dir, q)
4256 }
4257
4258 #[test]
4259 fn base_recovery_commit_retains_decision_and_superseded_resume_evidence() {
4260 let (_dir, q) = queue();
4261 let mut original = task("base recovery");
4262 original
4263 .decision_effects
4264 .insert("old-effect".into(), "receipt".into());
4265 original
4266 .dispatched_resumes
4267 .insert("dispatched".into(), "prior-run".into());
4268 original.resume_override = Some(OperatorResume {
4269 question_id: "prior-decision".into(),
4270 at: Timestamp::now(),
4271 conductor_rehold: None,
4272 forced: true,
4273 pinned_run: Some("retained-run".into()),
4274 });
4275 q.put(&mut original).unwrap();
4276 let expected = serde_json::to_value(&original).unwrap();
4277 let mut recovered = original.clone();
4278 recovered.base_recovery_requests.push("chat/request".into());
4279 recovered
4280 .resume_override
4281 .as_mut()
4282 .unwrap()
4283 .question_id
4284 .clear();
4285 q.commit_base_recovery(&mut recovered, &expected).unwrap();
4286 let saved = q.get(&original.id).unwrap();
4287 assert_eq!(saved.decision_effects, original.decision_effects);
4288 assert_eq!(saved.dispatched_resumes, original.dispatched_resumes);
4289 assert_eq!(
4290 saved
4291 .superseded_resumes
4292 .get("prior-decision")
4293 .map(String::as_str),
4294 Some("retained-run")
4295 );
4296 assert_eq!(saved.base_recovery_requests, ["chat/request"]);
4297 assert!(saved.resume_override.unwrap().question_id.is_empty());
4298 assert!(q.commit_base_recovery(&mut recovered, &expected).is_err());
4299 }
4300
4301 #[test]
4302 fn a_stale_write_does_not_drop_the_recorded_task_chat() {
4303 let (_dir, q) = queue();
4304 let mut stale = task("chat");
4305 q.put(&mut stale).unwrap();
4306 let mut fresh = q.get(&stale.id).unwrap();
4307 fresh.chat = Some("talk-1".to_owned());
4308 q.put(&mut fresh).unwrap();
4309 q.put(&mut stale).unwrap();
4310 assert_eq!(q.get(&stale.id).unwrap().chat.as_deref(), Some("talk-1"));
4311 }
4312
4313 #[test]
4314 fn putting_a_machine_held_task_files_a_notification_beside_the_queue() {
4315 let dir = tempfile::tempdir().unwrap();
4316 let q = Queue::at(dir.path().join("queue"));
4317 let mut t = task("held");
4318 q.put(&mut t).unwrap();
4319 assert_eq!(
4320 crate::notices::Notices::at(dir.path().join("notifications"))
4321 .list()
4322 .len(),
4323 0
4324 );
4325 t.hold_machine(Some("out of attempts".to_owned()));
4326 q.put(&mut t).unwrap();
4327 let listed = crate::notices::Notices::at(dir.path().join("notifications")).list();
4328 assert_eq!(listed.len(), 1);
4329 assert!(listed[0].message.contains("out of attempts"));
4330 }
4331
4332 #[test]
4333 fn start_consumes_the_disk_resume_pin() {
4334 let mut t = task("pin");
4335 t.disk_resume_run = Some("run-1".into());
4336 t.start("run-1".into());
4337 assert!(t.disk_resume_run.is_none());
4338 }
4339
4340 fn task(title: &str) -> Task {
4341 Task::new(
4342 title.to_owned(),
4343 format!("do {title}"),
4344 PathBuf::from("."),
4345 Source::Human,
4346 )
4347 }
4348
4349 #[test]
4350 fn attempt_limit_inherits_old_records_and_controls_failure() {
4351 let mut t = task("budget");
4352 let mut old = serde_json::to_value(&t).unwrap();
4353 old.as_object_mut().unwrap().remove("attempt_limit");
4354 old["schema"] = serde_json::json!(15);
4355 let legacy: Task = serde_json::from_value(old).unwrap();
4356 assert_eq!(legacy.attempt_limit, None);
4357 assert_eq!(legacy.effective_attempt_limit(2), 2);
4358 t.attempts = 2;
4359 t.change_attempt_limit(NonZeroUsize::new(3)).unwrap();
4360 t.fail("retry", 1);
4361 assert_eq!(t.status, TaskStatus::Failed);
4362 t.attempts = 3;
4363 t.fail("stop", 10);
4364 assert_eq!(t.status, TaskStatus::Held);
4365 t.change_attempt_limit(None).unwrap();
4366 assert_eq!(t.status, TaskStatus::Held);
4367 assert_eq!(t.attempts, 3);
4368 t.fail("inherit", 4);
4369 assert_eq!(t.status, TaskStatus::Failed);
4370 for status in [
4371 TaskStatus::Running,
4372 TaskStatus::Done,
4373 TaskStatus::Failed,
4374 TaskStatus::Blocked,
4375 TaskStatus::Parked,
4376 ] {
4377 t.status = status;
4378 assert!(t.change_attempt_limit(NonZeroUsize::new(1)).is_err());
4379 assert_eq!(t.attempt_limit, None);
4380 }
4381 }
4382
4383 #[test]
4384 fn earlier_attempts_is_every_recorded_run_and_agrees_with_the_display() {
4385 let mut t = task("retried");
4386 assert!(
4387 t.earlier_attempts().is_empty(),
4388 "a first attempt takes nothing over"
4389 );
4390 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4391 assert_eq!(t.earlier_attempts(), ["aaaa", "bbbb"]);
4392 assert_eq!(t.successor_of("aaaa"), Some(&"bbbb".to_owned()));
4394 assert_eq!(t.successor_of("bbbb"), None);
4395 assert_eq!(t.successor_of("zzzz"), None);
4396 }
4397
4398 #[test]
4399 fn resumed_runs_are_superseded_only_after_their_last_occurrence() {
4400 let (_dir, q) = queue();
4401 let mut t = task("resumed");
4402 for (runs, successor) in [
4403 (vec!["aaaa", "aaaa"], None),
4404 (vec!["aaaa", "bbbb", "aaaa"], None),
4405 (vec!["aaaa", "aaaa", "bbbb"], Some("bbbb")),
4406 (vec!["aaaa", "bbbb", "aaaa", "cccc"], Some("cccc")),
4407 ] {
4408 t.runs = runs.into_iter().map(str::to_owned).collect();
4409 q.put(&mut t).unwrap();
4410
4411 assert_eq!(t.successor_of("aaaa").map(String::as_str), successor);
4412 assert_eq!(q.superseded_by("aaaa").as_deref(), successor);
4413 assert_eq!(q.superseded().get("aaaa").map(String::as_str), successor);
4414 }
4415 }
4416
4417 #[test]
4418 fn superseded_by_names_the_next_attempt_and_none_for_the_last() {
4419 let (_dir, q) = queue();
4420 let mut t = task("retried");
4421 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4422 q.put(&mut t).unwrap();
4423
4424 assert_eq!(q.superseded_by("aaaa"), Some("bbbb".to_owned()));
4425 assert_eq!(q.superseded_by("bbbb"), Some("cccc".to_owned()));
4426 assert_eq!(
4427 q.superseded_by("cccc"),
4428 None,
4429 "the latest attempt replaces nothing"
4430 );
4431 assert_eq!(
4432 q.superseded_by("never-heard-of-it"),
4433 None,
4434 "a run belonging to no task on this queue is not superseded"
4435 );
4436
4437 let mut by = HashMap::new();
4438 by.insert("aaaa".to_owned(), "bbbb".to_owned());
4439 by.insert("bbbb".to_owned(), "cccc".to_owned());
4440 assert_eq!(
4441 q.superseded(),
4442 by,
4443 "the whole-map and single-run forms must agree"
4444 );
4445 }
4446
4447 #[test]
4448 fn superseded_attempts_is_empty_until_the_task_is_done() {
4449 let mut t = task("retried");
4450 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4451 t.status = TaskStatus::Failed;
4452 assert_eq!(
4453 t.superseded_attempts(true),
4454 &[] as &[String],
4455 "a task still retrying has no attempt yet that a later one made moot"
4456 );
4457
4458 t.status = TaskStatus::Running;
4459 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
4460 }
4461
4462 #[test]
4463 fn superseded_attempts_names_every_run_before_the_one_that_succeeded() {
4464 let mut t = task("retried");
4465 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4466 t.status = TaskStatus::Done;
4467 assert_eq!(
4468 t.superseded_attempts(true),
4469 &["aaaa".to_owned(), "bbbb".to_owned()],
4470 "cccc is the attempt whose success made the task done, and stays out"
4471 );
4472 }
4473
4474 #[test]
4475 fn superseded_attempts_is_empty_for_a_done_task_with_only_one_attempt() {
4476 let mut t = task("first try landed");
4477 t.runs = vec!["aaaa".to_owned()];
4478 t.status = TaskStatus::Done;
4479 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
4480 }
4481
4482 #[test]
4483 fn superseded_attempts_is_empty_when_the_last_run_never_actually_succeeded() {
4484 let mut t = task("closed by hand after a manual merge");
4491 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4492 t.status = TaskStatus::Done;
4493 assert_eq!(
4494 t.superseded_attempts(false),
4495 &[] as &[String],
4496 "nothing here is provably why the task is done, so nothing is superseded"
4497 );
4498 }
4499
4500 #[test]
4501 fn latest_attempt_names_the_chain_s_current_head_not_just_the_next_one() {
4502 let (_dir, q) = queue();
4503 let mut t = task("retried twice");
4504 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4505 q.put(&mut t).unwrap();
4506
4507 assert_eq!(
4508 q.latest_attempt("aaaa"),
4509 Some("cccc".to_owned()),
4510 "an old attempt points straight at the chain's current head, not the \
4511 next attempt in the middle of it"
4512 );
4513 assert_eq!(q.latest_attempt("bbbb"), Some("cccc".to_owned()));
4514 assert_eq!(
4515 q.latest_attempt("cccc"),
4516 None,
4517 "the latest attempt is not superseded by anything"
4518 );
4519 assert_eq!(
4520 q.latest_attempt("never-heard-of-it"),
4521 None,
4522 "a run belonging to no task on this queue is not superseded"
4523 );
4524 }
4525
4526 #[test]
4527 fn a_markdown_heading_is_the_title_not_decoration() {
4528 assert_eq!(
4533 title_from("# Rework the config loader\n\nIt re-reads it.\n", 40),
4534 "Rework the config loader"
4535 );
4536 assert_eq!(title_from("- fix the thing", 40), "fix the thing");
4537 assert_eq!(title_from("> quoted task", 40), "quoted task");
4538 assert_eq!(title_from(" \n\n", 40), "(empty task)");
4540 assert_eq!(title_from("###\n", 40), "(empty task)");
4541 }
4542
4543 #[test]
4544 fn a_long_title_is_elided_by_characters_not_bytes() {
4545 let long = "課題".repeat(30);
4547 let title = title_from(&long, 10);
4548 assert_eq!(title.chars().count(), 10);
4549 assert!(title.ends_with('…'));
4550 }
4551
4552 #[test]
4553 fn priority_wins_and_ties_break_oldest_first() {
4554 let (_dir, q) = queue();
4555 let mut a = task("first");
4556 let mut b = task("second");
4557 let mut c = task("urgent");
4558 a.id = "20260101-000001-aaaa".to_owned();
4560 b.id = "20260101-000002-bbbb".to_owned();
4561 c.id = "20260101-000003-cccc".to_owned();
4562 c.priority = 5;
4563 for t in [&mut a, &mut b, &mut c] {
4564 q.put(t).unwrap();
4565 }
4566
4567 assert_eq!(q.next_runnable().unwrap().id, c.id);
4569 c.hold_machine(None);
4570 q.put(&mut c).unwrap();
4571 assert_eq!(q.next_runnable().unwrap().id, a.id);
4573 assert_eq!(q.list().len(), 3, "b is still waiting its turn");
4574 }
4575
4576 #[test]
4577 fn a_blocked_task_never_starves_another_runnable_one() {
4578 let (_dir, q) = queue();
4579 let mut blocked = task("blocked");
4580 blocked.block(vec!["something".to_owned()], None);
4581 q.put(&mut blocked).unwrap();
4582
4583 let mut runnable = task("free to go");
4584 q.put(&mut runnable).unwrap();
4585
4586 let next = q.next_runnable().expect("a runnable task is still offered");
4587 assert_eq!(next.id, runnable.id);
4588 }
4589
4590 #[test]
4591 fn a_held_task_is_never_offered_to_the_loop() {
4592 let (_dir, q) = queue();
4593 let mut t = task("held");
4594 q.put(&mut t).unwrap();
4595 assert!(q.next_runnable().is_some());
4596
4597 t.hold_machine(None);
4598 q.put(&mut t).unwrap();
4599 assert!(
4600 q.next_runnable().is_none(),
4601 "a held task must wait for a human"
4602 );
4603
4604 t.status = TaskStatus::Failed;
4606 q.put(&mut t).unwrap();
4607 assert!(q.next_runnable().is_some());
4608 }
4609
4610 #[test]
4611 fn attempts_are_capped_and_then_the_task_is_held() {
4612 let mut t = task("doomed");
4613
4614 t.start("run-1".to_owned());
4615 t.fail("gate red", 2);
4616 assert_eq!(t.status, TaskStatus::Failed, "one attempt of two: retry");
4617
4618 t.start("run-2".to_owned());
4619 t.fail("gate red", 2);
4620 assert_eq!(
4621 t.status,
4622 TaskStatus::Held,
4623 "out of attempts: stop spending money on it"
4624 );
4625 assert_eq!(t.runs, ["run-1", "run-2"]);
4626 assert_eq!(t.last_error.as_deref(), Some("gate red"));
4627 assert_eq!(
4628 t.hold_reason.as_deref(),
4629 Some("gate red"),
4630 "the hold must say why, not leave hold_reason null next to a \
4631 populated last_error"
4632 );
4633 }
4634
4635 #[test]
4636 fn handing_off_a_task_records_a_hold_reason_too() {
4637 let mut t = task("left a pull request");
4638 t.start("run-1".to_owned());
4639 t.handed_off("run ended with a pull request open [run run-1]");
4640 assert_eq!(t.status, TaskStatus::Held);
4641 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4642 assert_eq!(
4643 t.hold_reason.as_deref(),
4644 Some("run ended with a pull request open [run run-1]")
4645 );
4646 assert_eq!(t.hold_reason, t.last_error);
4647 }
4648
4649 #[test]
4650 fn a_quota_stall_is_refunded_so_the_backlog_survives_the_night() {
4651 let mut t = task("stalled by quota");
4652
4653 t.start("run-1".to_owned());
4654 assert_eq!(t.attempts, 1);
4655 t.stall("judge-1, judge-2 out of quota");
4656 assert_eq!(
4657 t.attempts, 0,
4658 "a closed quota window must not spend the task's retry budget"
4659 );
4660 assert_eq!(t.status, TaskStatus::Failed, "the loop should retry it");
4661 assert_eq!(
4662 t.last_error.as_deref(),
4663 Some("judge-1, judge-2 out of quota")
4664 );
4665
4666 for _ in 0..20 {
4669 t.start("run-n".to_owned());
4670 t.stall("still out of quota");
4671 }
4672 t.start("run-real".to_owned());
4673 t.fail("gate red", 2);
4674 assert_eq!(
4675 t.status,
4676 TaskStatus::Failed,
4677 "the first attempt that was really judged is attempt one"
4678 );
4679 }
4680
4681 #[test]
4682 fn releasing_a_held_task_gives_it_a_real_second_chance() {
4683 let mut t = task("retry me");
4684 t.start("run-1".to_owned());
4685 t.fail("gate red", 1);
4686 assert_eq!(t.status, TaskStatus::Held);
4687
4688 t.release();
4689 assert_eq!(t.status, TaskStatus::Queued);
4690 assert_eq!(t.attempts, 0);
4693 assert!(t.last_error.is_none());
4694 assert_eq!(
4695 t.runs.len(),
4696 1,
4697 "history is kept: attempts reset, evidence does not"
4698 );
4699 }
4700
4701 #[test]
4702 fn a_hold_reason_survives_and_a_release_clears_it() {
4703 let mut t = task("waiting on something else");
4704 t.hold_manual(Some(
4705 "waiting for 20260101-000000-aaaa to land first".to_owned(),
4706 ));
4707 assert_eq!(t.status, TaskStatus::Held);
4708 assert_eq!(
4709 t.hold_reason.as_deref(),
4710 Some("waiting for 20260101-000000-aaaa to land first")
4711 );
4712
4713 t.hold_manual(None);
4715 assert_eq!(
4716 t.hold_reason.as_deref(),
4717 Some("waiting for 20260101-000000-aaaa to land first"),
4718 "a bare re-hold keeps whatever a human already wrote down"
4719 );
4720
4721 let mut plain = task("no reason given");
4723 plain.hold_manual(None);
4724 assert_eq!(plain.status, TaskStatus::Held);
4725 assert!(plain.hold_reason.is_none());
4726
4727 t.release();
4728 assert_eq!(t.status, TaskStatus::Queued);
4729 assert!(
4730 t.hold_reason.is_none(),
4731 "a stale reason must not greet the next person who holds this task"
4732 );
4733 }
4734
4735 #[test]
4736 fn closing_a_held_task_as_done_clears_its_hold_reason_too() {
4737 let mut t = task("landed by hand while held");
4742 t.hold_manual(Some("waiting on 3ed9".to_owned()));
4743 assert_eq!(t.hold_reason.as_deref(), Some("waiting on 3ed9"));
4744
4745 t.succeed();
4746 assert_eq!(t.status, TaskStatus::Done);
4747 assert!(
4748 t.hold_reason.is_none(),
4749 "a done task cannot still be waiting on something"
4750 );
4751 }
4752
4753 #[test]
4754 fn holding_or_closing_a_blocked_task_clears_its_dependency_too() {
4755 let mut held = task("held straight out of blocked");
4762 held.block(
4763 vec!["20260101-000000-dead".to_owned()],
4764 Some("waiting on the migration script".to_owned()),
4765 );
4766 assert_eq!(held.status, TaskStatus::Blocked);
4767
4768 held.hold_manual(None);
4769 assert_eq!(held.status, TaskStatus::Held);
4770 assert!(
4771 held.blocked_by.is_empty(),
4772 "hold overrides the wait, same as release"
4773 );
4774 assert!(held.block_reason.is_none());
4775
4776 let mut done = task("closed straight out of blocked");
4777 done.block(
4778 vec!["20260101-000000-dead".to_owned()],
4779 Some("waiting on the migration script".to_owned()),
4780 );
4781 done.succeed();
4782 assert_eq!(done.status, TaskStatus::Done);
4783 assert!(
4784 done.blocked_by.is_empty(),
4785 "a done task cannot still be waiting on a dependency"
4786 );
4787 assert!(done.block_reason.is_none());
4788 }
4789
4790 #[test]
4791 fn a_blocked_task_is_never_offered_to_the_loop() {
4792 let mut t = task("blocked");
4793 assert!(t.status.runnable());
4794 t.block(
4795 vec!["dep-id".to_owned()],
4796 Some("waits on dep-id".to_owned()),
4797 );
4798 assert_eq!(t.status, TaskStatus::Blocked);
4799 assert!(!t.status.runnable());
4800 assert_eq!(TaskStatus::Blocked.as_str(), "blocked");
4801 }
4802
4803 #[test]
4804 fn unblocking_the_last_dependency_returns_the_task_to_queued() {
4805 let mut t = task("blocked on two");
4806 t.block(
4807 vec!["a".to_owned(), "b".to_owned()],
4808 Some("waits on a and b".to_owned()),
4809 );
4810
4811 t.unblock("a");
4812 assert_eq!(t.status, TaskStatus::Blocked, "b is still outstanding");
4813 assert_eq!(t.blocked_by, ["b"]);
4814
4815 t.unblock("b");
4816 assert_eq!(t.status, TaskStatus::Queued);
4817 assert!(t.blocked_by.is_empty());
4818 assert!(t.block_reason.is_none());
4819 }
4820
4821 #[test]
4822 fn unblocking_an_id_on_a_task_that_is_not_blocked_is_a_no_op() {
4823 let mut t = task("never blocked");
4824 t.unblock("whatever");
4825 assert_eq!(t.status, TaskStatus::Queued);
4826 }
4827
4828 #[test]
4829 fn a_held_task_blocked_on_a_question_returns_to_held_not_queued() {
4830 let mut t = task("held, then asked about");
4835 t.hold_machine(Some("out of attempts".to_owned()));
4836 assert_eq!(t.status, TaskStatus::Held);
4837
4838 t.block(vec!["q1".to_owned()], Some("what now?".to_owned()));
4839 assert_eq!(t.status, TaskStatus::Blocked);
4840
4841 t.record_answer("what now?".to_owned(), "leave it held".to_owned());
4842 t.unblock("q1");
4843 assert_eq!(t.status, TaskStatus::Held, "must restore, not requeue");
4844 assert_eq!(t.hold_reason.as_deref(), Some("out of attempts"));
4845 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4846 assert!(t.blocked_from.is_none(), "consumed once restored");
4847 }
4848
4849 #[test]
4850 fn a_manually_held_task_blocked_on_a_question_returns_to_held() {
4851 let mut t = task("manually held, then asked about");
4852 t.hold_manual(Some("waiting on a dependency".to_owned()));
4853
4854 t.block(vec!["q1".to_owned()], None);
4855 t.unblock("q1");
4856
4857 assert_eq!(t.status, TaskStatus::Held);
4858 assert_eq!(t.hold_source, Some(HoldSource::Manual));
4859 }
4860
4861 #[test]
4862 fn re_blocking_an_already_blocked_task_keeps_the_original_blocked_from() {
4863 let mut t = task("held, blocked twice");
4867 t.hold_machine(None);
4868 t.block(vec!["q1".to_owned()], Some("first".to_owned()));
4869 t.block(
4870 vec!["q1".to_owned(), "q2".to_owned()],
4871 Some("second".to_owned()),
4872 );
4873
4874 t.unblock("q1");
4875 assert_eq!(t.status, TaskStatus::Blocked, "q2 still outstanding");
4876 t.unblock("q2");
4877 assert_eq!(t.status, TaskStatus::Held);
4878 }
4879
4880 #[test]
4881 fn unblocking_a_task_blocked_while_running_lands_on_queued_not_running() {
4882 let mut t = task("blocked mid-run");
4886 t.start("run-1".to_owned());
4887 assert_eq!(t.status, TaskStatus::Running);
4888
4889 t.block(vec!["q1".to_owned()], None);
4890 t.unblock("q1");
4891 assert_eq!(t.status, TaskStatus::Queued);
4892 }
4893
4894 #[test]
4895 fn a_pre_schema_4_blocked_record_with_hold_evidence_restores_to_held() {
4896 let mut t = task("legacy record, held before it was blocked");
4902 t.hold_source = Some(HoldSource::Machine);
4903 t.hold_reason = Some("legacy hold reason".to_owned());
4904 t.status = TaskStatus::Blocked;
4905 t.blocked_by = vec!["q1".to_owned()];
4906 t.blocked_from = None;
4907
4908 t.unblock("q1");
4909 assert_eq!(t.status, TaskStatus::Held);
4910 }
4911
4912 #[test]
4913 fn a_pre_schema_4_blocked_record_with_no_hold_evidence_restores_to_queued() {
4914 let mut t = task("legacy record, ordinary dependency block");
4915 t.status = TaskStatus::Blocked;
4916 t.blocked_by = vec!["dep".to_owned()];
4917 t.blocked_from = None;
4918
4919 t.unblock("dep");
4920 assert_eq!(t.status, TaskStatus::Queued);
4921 }
4922
4923 #[test]
4924 fn answering_a_question_is_recorded_and_survives_a_release() {
4925 let mut t = task("asked something");
4926 t.block(vec!["q1".to_owned()], Some("which backend?".to_owned()));
4927 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
4928 t.unblock("q1");
4929 assert_eq!(t.status, TaskStatus::Queued);
4930 assert_eq!(t.answers.len(), 1);
4931 assert_eq!(t.answers[0].answer, "SQLite");
4932
4933 t.release();
4937 assert_eq!(t.answers.len(), 1, "the answer is not lost on release");
4938 }
4939
4940 #[test]
4941 fn resumption_review_branch_requires_unchanged_authored_criteria() {
4942 for source in ["cli", "web"] {
4943 let mut t = task("review old candidate");
4944 t.request_review("candidate-A".into());
4945 t.hold_for_handover(Some("candidate-A".into()), "handover refused".into());
4946 let mut unchanged = t.clone();
4947 unchanged.release_operator(source, None);
4948 assert_eq!(unchanged.review_branch.as_deref(), Some("candidate-A"));
4949 t.edit(
4950 "review approved head".into(),
4951 "Reuse measured PR head B; fix Clippy; review".into(),
4952 )
4953 .unwrap();
4954 t.release_operator(source, None);
4955 assert!(t.review_branch.is_none());
4956 assert!(t.review_instruction.is_none());
4957 let mut legacy = unchanged;
4959 legacy.review_instruction = None;
4960 legacy.hold_for_handover(None, "still checked out".into());
4961 legacy
4962 .edit("new plan".into(), "Reuse measured PR head B".into())
4963 .unwrap();
4964 legacy.release_operator(source, None);
4965 assert!(legacy.review_branch.is_none());
4966 }
4967 }
4968
4969 #[test]
4970 fn a_refused_handover_keeps_the_review_branch_across_release() {
4971 let mut t = task("refused takeover");
4972 t.start("run-1".to_owned());
4973 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
4974 assert_eq!(t.status, TaskStatus::Held);
4975 assert_eq!(t.attempts, 1);
4976 t.release();
4977 assert_eq!(t.status, TaskStatus::Queued);
4978 assert_eq!(t.attempts, 0);
4979 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4980
4981 t.hold_for_handover(None, "again".to_owned());
4983 t.requeue();
4984 assert!(t.review_branch.is_none());
4985
4986 let mut m = task("manual");
4988 m.review_branch = Some("magi/x/A".to_owned());
4989 m.hold_manual(None);
4990 m.release();
4991 assert!(m.review_branch.is_none());
4992 }
4993
4994 #[test]
4995 fn requesting_review_requeues_the_task_and_remembers_the_branch() {
4996 let mut t = task("blocked run with a surviving branch");
4997 t.start("run-1".to_owned());
4998 t.fail("blocked with major findings", 5);
4999 assert_eq!(t.status, TaskStatus::Failed);
5000
5001 t.request_review("magi/eba2/A".to_owned());
5002 assert_eq!(t.status, TaskStatus::Queued);
5003 assert_eq!(t.attempts, 0);
5004 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
5005
5006 t.release();
5008 assert!(t.review_branch.is_none());
5009 }
5010
5011 #[test]
5012 fn conductor_requeue_but_not_an_ordinary_release_forces_a_fresh_start() {
5013 let mut t = task("retry");
5014 t.start("run-1".to_owned());
5015 t.requeue();
5016 assert!(t.fresh_start);
5017
5018 t.release();
5019 assert!(!t.fresh_start);
5020 }
5021
5022 #[test]
5023 fn urgent_only_changes_queued_or_held_and_preserves_settings() {
5024 for status in [
5025 TaskStatus::Queued,
5026 TaskStatus::Held,
5027 TaskStatus::Running,
5028 TaskStatus::Done,
5029 TaskStatus::Failed,
5030 TaskStatus::Blocked,
5031 TaskStatus::Parked,
5032 ] {
5033 let mut t = task("urgent settings");
5034 t.status = status;
5035 t.attempts = 2;
5036 t.hold_reason = Some("waiting".into());
5037 t.priority = 7;
5038 t.interrupt = true;
5039 let before = serde_json::to_value(&t).unwrap();
5040 for urgent in [true, true, false, false] {
5041 let result = t.set_urgent(urgent);
5042 let mut expected = before.clone();
5043 if matches!(status, TaskStatus::Queued | TaskStatus::Held) {
5044 result.unwrap();
5045 expected["urgent"] = urgent.into();
5046 } else {
5047 let error = result.unwrap_err().to_string();
5048 assert!(error.contains(status.as_str()), "{error}");
5049 assert!(error.contains("queued or held"), "{error}");
5050 }
5051 assert_eq!(serde_json::to_value(&t).unwrap(), expected);
5052 }
5053 }
5054 }
5055
5056 #[test]
5057 fn priority_can_be_changed_while_queued_but_not_while_running() {
5058 let mut t = task("reprioritise me");
5059 t.set_priority(5).unwrap();
5060 assert_eq!(t.priority, 5);
5061
5062 t.start("run-1".to_owned());
5063 let err = t.set_priority(9).unwrap_err().to_string();
5064 assert!(err.contains("running"), "{err}");
5065 assert_eq!(t.priority, 5, "the rejected write must not partially apply");
5066 }
5067
5068 #[test]
5069 fn interrupt_can_be_marked_while_queued_but_not_while_running() {
5070 let mut t = task("interrupt me");
5071 assert!(!t.interrupt, "off unless asked, same as any other task");
5072
5073 t.set_interrupt(true).unwrap();
5074 assert!(t.interrupt);
5075
5076 t.start("run-1".to_owned());
5077 assert!(
5078 !t.interrupt,
5079 "the mark is one-shot: dispatching the task fulfils it, \
5080 whatever the run that follows ends up doing"
5081 );
5082 let err = t.set_interrupt(true).unwrap_err().to_string();
5083 assert!(err.contains("running"), "{err}");
5084 t.set_interrupt(false).unwrap();
5087 assert!(!t.interrupt);
5088 }
5089
5090 #[test]
5094 fn a_failed_run_does_not_leave_the_task_still_marked_to_interrupt() {
5095 let mut t = task("interrupt me");
5096 t.set_interrupt(true).unwrap();
5097 t.start("run-1".to_owned());
5098 t.fail("mock failure", 5);
5099 assert_eq!(t.status, TaskStatus::Failed);
5100 assert!(
5101 !t.interrupt,
5102 "one attempt already spent the mark; a retry is an ordinary \
5103 requeue, not a fresh interrupt request"
5104 );
5105 }
5106
5107 #[test]
5108 fn changing_priority_moves_a_task_ahead_in_the_real_queue_order() {
5109 let (_dir, q) = queue();
5110 let mut a = task("first filed");
5111 let mut b = task("second filed");
5112 a.id = "20260101-000001-aaaa".to_owned();
5113 b.id = "20260101-000002-bbbb".to_owned();
5114 q.put(&mut a).unwrap();
5115 q.put(&mut b).unwrap();
5116
5117 assert_eq!(
5118 q.next_runnable().unwrap().id,
5119 a.id,
5120 "with equal priority the older task goes first, so a burst of \
5121 new work cannot starve it"
5122 );
5123 assert_eq!(
5124 q.list()[0].id,
5125 b.id,
5126 "but the list an operator reads is newest first, the same as \
5127 before priority existed - a's turn to run does not make it the \
5128 newest task"
5129 );
5130
5131 let mut a = q.get(&a.id).unwrap();
5132 a.set_priority(10).unwrap();
5133 q.put(&mut a).unwrap();
5134
5135 assert_eq!(
5136 q.next_runnable().unwrap().id,
5137 a.id,
5138 "a raised priority must be reflected the moment it is saved"
5139 );
5140 assert_eq!(
5144 q.list()[0].id,
5145 a.id,
5146 "the raised task must sort first in the list an operator reads, \
5147 not only in next_runnable's own ordering"
5148 );
5149 }
5150
5151 #[test]
5152 fn editing_replaces_title_and_instruction_but_keeps_identity_and_history() {
5153 let mut t = Task::new(
5154 "old title".to_owned(),
5155 "old instruction".to_owned(),
5156 PathBuf::from("/repo"),
5157 Source::Agent {
5158 run: "20260101-000000-beef".to_owned(),
5159 node: "implement".to_owned(),
5160 },
5161 );
5162 let id = t.id.clone();
5163 let created_at = t.created_at;
5164 t.runs.push("20260101-000000-beef".to_owned());
5165
5166 t.edit("new title".to_owned(), "new instruction".to_owned())
5167 .unwrap();
5168
5169 assert_eq!(t.title, "new title");
5170 assert_eq!(t.instruction, "new instruction");
5171 assert_eq!(t.id, id, "editing must not mint a new id");
5172 assert_eq!(t.created_at, created_at);
5173 assert_eq!(
5174 t.source,
5175 Source::Agent {
5176 run: "20260101-000000-beef".to_owned(),
5177 node: "implement".to_owned(),
5178 },
5179 "editing must not turn agent attribution into human"
5180 );
5181 assert_eq!(t.runs, ["20260101-000000-beef"]);
5182 }
5183
5184 #[test]
5185 fn editing_is_refused_once_a_task_is_running_or_finished() {
5186 let mut running = task("in flight");
5187 running.start("run-1".to_owned());
5188 let err = running
5189 .edit("x".to_owned(), "y".to_owned())
5190 .unwrap_err()
5191 .to_string();
5192 assert!(err.contains("running"), "{err}");
5193
5194 let mut done = task("finished");
5195 done.succeed();
5196 let err = done
5197 .edit("x".to_owned(), "y".to_owned())
5198 .unwrap_err()
5199 .to_string();
5200 assert!(err.contains("done"), "{err}");
5201
5202 let mut queued = task("waiting");
5204 queued.edit("x".to_owned(), "y".to_owned()).unwrap();
5205 let mut held = task("parked");
5206 held.hold_machine(None);
5207 held.edit("x".to_owned(), "y".to_owned()).unwrap();
5208 }
5209
5210 #[test]
5211 fn a_task_recorded_without_a_hold_reason_still_reads_as_none() {
5212 let (_dir, q) = queue();
5213 let path = q.path_of("20260101-000000-aaaa");
5214 std::fs::create_dir_all(q.root()).unwrap();
5215 std::fs::write(
5216 &path,
5217 serde_json::json!({
5218 "schema": SCHEMA,
5219 "id": "20260101-000000-aaaa",
5220 "title": "from before hold reasons existed",
5221 "instruction": "from before hold reasons existed",
5222 "repo": ".",
5223 "source": { "kind": "human" },
5224 "status": "held",
5225 "created_at": Timestamp::now().to_string(),
5226 "updated_at": Timestamp::now().to_string(),
5227 })
5228 .to_string(),
5229 )
5230 .unwrap();
5231
5232 let task = q.get("20260101-000000-aaaa").expect("must still read");
5233 assert!(task.hold_reason.is_none());
5234 assert!(task.operator_held());
5235 }
5236
5237 #[test]
5238 fn a_legacy_reasoned_hold_defaults_to_operator_protection() {
5239 let (_dir, q) = queue();
5240 let path = q.path_of("20260101-000000-bbbb");
5241 std::fs::create_dir_all(q.root()).unwrap();
5242 std::fs::write(
5243 &path,
5244 serde_json::json!({
5245 "schema": 2,
5246 "id": "20260101-000000-bbbb",
5247 "title": "old manual recovery",
5248 "instruction": "old manual recovery",
5249 "repo": ".",
5250 "source": { "kind": "human" },
5251 "status": "held",
5252 "hold_reason": "active manual recovery run20260912-224242-daf5",
5253 "created_at": Timestamp::now().to_string(),
5254 "updated_at": Timestamp::now().to_string(),
5255 })
5256 .to_string(),
5257 )
5258 .unwrap();
5259
5260 let task = q.get("20260101-000000-bbbb").expect("must still read");
5261 assert_eq!(task.hold_source, None);
5262 assert!(task.operator_held());
5263 }
5264
5265 #[test]
5266 fn a_task_recorded_without_a_diagnostic_still_reads_as_none() {
5267 let (_dir, q) = queue();
5268 let path = q.path_of("20260101-000000-aaaa");
5269 std::fs::create_dir_all(q.root()).unwrap();
5270 std::fs::write(
5271 &path,
5272 serde_json::json!({
5273 "schema": SCHEMA,
5274 "id": "20260101-000000-aaaa",
5275 "title": "from before diagnostics existed",
5276 "instruction": "from before diagnostics existed",
5277 "repo": ".",
5278 "source": { "kind": "human" },
5279 "status": "held",
5280 "created_at": Timestamp::now().to_string(),
5281 "updated_at": Timestamp::now().to_string(),
5282 })
5283 .to_string(),
5284 )
5285 .unwrap();
5286
5287 let task = q.get("20260101-000000-aaaa").expect("must still read");
5288 assert!(task.diagnostic.is_none());
5289 }
5290
5291 #[test]
5292 fn a_schema_1_task_with_no_blocking_fields_still_reads() {
5293 let (_dir, q) = queue();
5297 let path = q.path_of("20260101-000000-aaaa");
5298 std::fs::create_dir_all(q.root()).unwrap();
5299 std::fs::write(
5300 &path,
5301 serde_json::json!({
5302 "schema": 1,
5303 "id": "20260101-000000-aaaa",
5304 "title": "from before blocking existed",
5305 "instruction": "from before blocking existed",
5306 "repo": ".",
5307 "source": { "kind": "human" },
5308 "status": "queued",
5309 "created_at": Timestamp::now().to_string(),
5310 "updated_at": Timestamp::now().to_string(),
5311 })
5312 .to_string(),
5313 )
5314 .unwrap();
5315
5316 let task = q.get("20260101-000000-aaaa").expect("must still read");
5317 assert!(task.blocked_by.is_empty());
5318 assert!(task.block_reason.is_none());
5319 assert!(task.answers.is_empty());
5320 assert!(task.review_branch.is_none());
5321 }
5322
5323 #[test]
5324 fn releasing_or_finishing_a_task_clears_its_stale_diagnostic() {
5325 let mut held = task("diagnosed");
5330 held.start("run-1".to_owned());
5331 held.fail("gate red", 1);
5332 held.diagnostic = Some("cargo test failed: ...".to_owned());
5333 assert_eq!(held.status, TaskStatus::Held);
5334
5335 held.release();
5336 assert!(held.diagnostic.is_none());
5337
5338 held.diagnostic = Some("cargo test failed: ...".to_owned());
5339 held.succeed();
5340 assert!(held.diagnostic.is_none());
5341 }
5342
5343 #[test]
5344 fn failing_a_task_always_clears_whatever_diagnostic_it_carried() {
5345 let mut t = task("retried");
5346 t.start("run-1".to_owned());
5347 t.diagnostic = Some("stale evidence from a previous hold".to_owned());
5348 t.fail("unrelated config error", 5);
5349 assert_eq!(t.status, TaskStatus::Failed);
5350 assert!(
5351 t.diagnostic.is_none(),
5352 "fail() must not let an old diagnostic outlive the run that produced it"
5353 );
5354 }
5355
5356 #[test]
5357 fn a_claim_is_exclusive_and_releases_on_drop() {
5358 let (_dir, q) = queue();
5359 let mut t = task("contended");
5360 q.put(&mut t).unwrap();
5361
5362 let held = q.claim(&t.id).unwrap();
5363 assert!(
5364 q.claim(&t.id).is_err(),
5365 "two daemons must not drive one task into two runs"
5366 );
5367 drop(held);
5368 assert!(q.claim(&t.id).is_ok(), "a released claim is reclaimable");
5369 }
5370
5371 #[test]
5372 fn a_round_trip_survives_disk() {
5373 let (_dir, q) = queue();
5374 let mut t = Task::new(
5375 "titled".to_owned(),
5376 "body".to_owned(),
5377 PathBuf::from("/repo"),
5378 Source::Agent {
5379 run: "20260101-000000-beef".to_owned(),
5380 node: "implement".to_owned(),
5381 },
5382 );
5383 t.priority = 3;
5384 q.put(&mut t).unwrap();
5385
5386 let back = q.get(&t.id).unwrap();
5387 assert_eq!(back.id, t.id);
5388 assert_eq!(back.priority, 3);
5389 assert_eq!(back.source.label(), "implement@beef");
5390 assert_eq!(q.get(t.short()).unwrap().id, t.id);
5392 }
5393
5394 #[test]
5395 fn an_unreadable_task_does_not_take_the_queue_down() {
5396 let (_dir, q) = queue();
5397 let mut t = task("fine");
5398 q.put(&mut t).unwrap();
5399 std::fs::write(q.root().join("broken.json"), "{ not json").unwrap();
5400
5401 let listed = q.list();
5402 assert_eq!(listed.len(), 1, "the readable task still lists");
5403 assert_eq!(listed[0].id, t.id);
5404 }
5405
5406 #[test]
5407 fn a_task_recorded_without_a_solo_field_still_reads_as_not_solo() {
5408 let (_dir, q) = queue();
5409 let path = q.path_of("20260101-000000-aaaa");
5410 std::fs::create_dir_all(q.root()).unwrap();
5411 std::fs::write(
5412 &path,
5413 serde_json::json!({
5414 "schema": SCHEMA,
5415 "id": "20260101-000000-aaaa",
5416 "title": "from before solo existed",
5417 "instruction": "from before solo existed",
5418 "repo": ".",
5419 "source": { "kind": "human" },
5420 "status": "queued",
5421 "created_at": Timestamp::now().to_string(),
5422 "updated_at": Timestamp::now().to_string(),
5423 })
5424 .to_string(),
5425 )
5426 .unwrap();
5427
5428 let task = q.get("20260101-000000-aaaa").expect("must still read");
5429 assert!(!task.solo, "a queue file with no `solo` field means false");
5430 }
5431
5432 #[test]
5433 fn a_task_recorded_without_an_urgent_field_still_reads_as_not_urgent() {
5434 let (_dir, q) = queue();
5435 let path = q.path_of("20260101-000000-bbbb");
5436 std::fs::create_dir_all(q.root()).unwrap();
5437 std::fs::write(
5438 &path,
5439 serde_json::json!({
5440 "schema": SCHEMA,
5441 "id": "20260101-000000-bbbb",
5442 "title": "from before urgent existed",
5443 "instruction": "from before urgent existed",
5444 "repo": ".",
5445 "source": { "kind": "human" },
5446 "status": "queued",
5447 "created_at": Timestamp::now().to_string(),
5448 "updated_at": Timestamp::now().to_string(),
5449 })
5450 .to_string(),
5451 )
5452 .unwrap();
5453
5454 let task = q.get("20260101-000000-bbbb").expect("must still read");
5455 assert!(
5456 !task.urgent,
5457 "a queue file with no `urgent` field means false, same as `solo`"
5458 );
5459 }
5460
5461 #[test]
5462 fn a_task_from_a_future_schema_is_refused_rather_than_guessed_at() {
5463 let (_dir, q) = queue();
5464 let mut t = task("from the future");
5465 q.put(&mut t).unwrap();
5466 let path = q.path_of(&t.id);
5467 let body = std::fs::read_to_string(&path)
5468 .unwrap()
5469 .replace(&format!("\"schema\": {SCHEMA}"), "\"schema\": 99");
5470 std::fs::write(&path, body).unwrap();
5471
5472 let err = q.get(&t.id).unwrap_err().to_string();
5473 assert!(err.contains("schema 99"), "{err}");
5474 }
5475
5476 #[test]
5477 fn revision_moves_when_the_queue_changes() {
5478 let (_dir, q) = queue();
5479 assert_eq!(q.revision(), 0, "an empty queue has no revision");
5480 let mut t = task("first");
5481 q.put(&mut t).unwrap();
5482 assert!(q.revision() > 0, "a written task moves the revision");
5483 }
5484
5485 #[test]
5486 fn revision_moves_when_deleting_an_older_task() {
5487 let (dir, q) = queue();
5488 let questions = Questions::at(dir.path().join("questions"));
5489 let mut t1 = task("older");
5490 q.put(&mut t1).unwrap();
5491 std::thread::sleep(std::time::Duration::from_millis(10));
5493 let mut t2 = task("newer");
5494 q.put(&mut t2).unwrap();
5495
5496 let rev_before = q.revision();
5497 q.remove(&t1.id, false, &questions).unwrap();
5498 let rev_after = q.revision();
5499
5500 assert_ne!(
5501 rev_before, rev_after,
5502 "deleting an older task must change the revision so other clients see the deletion"
5503 );
5504 }
5505
5506 #[test]
5507 fn removing_a_task_takes_it_out_of_the_listing() {
5508 let (dir, q) = queue();
5509 let questions = Questions::at(dir.path().join("questions"));
5510 let mut t = task("delete me");
5511 q.put(&mut t).unwrap();
5512 let removed = q.remove(t.short(), false, &questions).unwrap();
5513 assert_eq!(removed.id, t.id, "a prefix resolves before deleting");
5514 assert!(removed.released.is_empty() && removed.still_blocked.is_empty());
5515 assert!(q.list().is_empty());
5516 assert!(
5517 q.remove(&t.id, false, &questions).is_err(),
5518 "removing twice is an error"
5519 );
5520 }
5521
5522 #[test]
5523 fn removing_a_task_takes_its_stale_lock_with_it() {
5524 let (dir, q) = queue();
5525 let questions = Questions::at(dir.path().join("questions"));
5526 let mut t = task("interrupted");
5527 q.put(&mut t).unwrap();
5528
5529 let claim = q.claim(&t.id).unwrap();
5532 std::mem::forget(claim);
5533 assert!(
5534 q.claim(&t.id).is_err(),
5535 "the orphaned lock is what makes the task look claimed"
5536 );
5537
5538 let err = q.remove(&t.id, true, &questions).unwrap_err().to_string();
5540 assert!(err.contains("live daemon"), "{err}");
5541 assert!(q.get(&t.id).is_ok(), "a refused delete keeps the task");
5542
5543 q.remove(&t.id, false, &questions).unwrap();
5545 assert!(q.list().is_empty());
5546 let mut again = task("interrupted");
5547 again.id = t.id.clone();
5548 q.put(&mut again).unwrap();
5549 assert!(
5550 q.claim(&t.id).is_ok(),
5551 "a task that comes back must be claimable, which a left-behind lock would prevent"
5552 );
5553 }
5554
5555 fn notices_of(dir: &Path) -> Vec<crate::notices::Notice> {
5556 crate::notices::Notices::at(dir.join("notifications")).list()
5557 }
5558
5559 #[test]
5560 fn removing_a_sole_dependency_releases_the_dependent_without_a_hold() {
5561 let (dir, q) = queue();
5562 let questions = Questions::at(dir.path().join("questions"));
5563 let mut dep = task("dependency");
5564 q.put(&mut dep).unwrap();
5565 let mut blocked = task("waiting");
5566 blocked.block(vec![dep.id.clone()], Some("waits".to_owned()));
5567 q.put(&mut blocked).unwrap();
5568
5569 let removed = q.remove(&dep.id, false, &questions).unwrap();
5570 assert_eq!(removed.released, [blocked.id.clone()]);
5571 assert!(removed.still_blocked.is_empty());
5572
5573 let after = q.get(&blocked.id).unwrap();
5574 assert_eq!(after.status, TaskStatus::Queued);
5575 assert!(after.blocked_by.is_empty());
5576 assert!(after.block_reason.is_none());
5577 let notes = notices_of(dir.path());
5578 assert_eq!(notes.len(), 1, "{notes:?}");
5579 assert_eq!(notes[0].severity, crate::notices::Severity::Info);
5580 assert_eq!(
5581 notes[0].link,
5582 Some(crate::notices::Link::Task {
5583 id: blocked.id.clone(),
5584 })
5585 );
5586 }
5587
5588 #[test]
5589 fn removing_one_of_two_dependencies_keeps_the_other() {
5590 let (dir, q) = queue();
5591 let questions = Questions::at(dir.path().join("questions"));
5592 let mut dep = task("dependency");
5593 q.put(&mut dep).unwrap();
5594 let mut other = task("other");
5595 q.put(&mut other).unwrap();
5596 let mut blocked = task("waiting");
5597 blocked.block(
5598 vec![dep.id.clone(), other.id.clone()],
5599 Some("waits on both".to_owned()),
5600 );
5601 q.put(&mut blocked).unwrap();
5602
5603 let removed = q.remove(&dep.id, false, &questions).unwrap();
5604 assert!(removed.released.is_empty());
5605 assert_eq!(removed.still_blocked, [blocked.id.clone()]);
5606
5607 let after = q.get(&blocked.id).unwrap();
5608 assert_eq!(after.status, TaskStatus::Blocked);
5609 assert_eq!(after.blocked_by, [other.id.clone()]);
5610 assert!(missing_blockers(&q, &questions, &after.blocked_by).is_empty());
5613 }
5614
5615 #[test]
5616 fn a_dependency_deleted_before_its_dependents_were_rewritten_is_released_later() {
5617 let (dir, q) = queue();
5618 let questions = Questions::at(dir.path().join("questions"));
5619 let mut dep = task("dependency");
5620 q.put(&mut dep).unwrap();
5621 let mut blocked = task("waiting");
5622 blocked.block(vec![dep.id.clone()], None);
5623 q.put(&mut blocked).unwrap();
5624
5625 let claim = q.claim(&blocked.id).unwrap();
5627 let removed = q.remove(&dep.id, false, &questions).unwrap();
5628 assert!(removed.released.is_empty());
5629 drop(claim);
5630
5631 let mut task = q.get(&blocked.id).unwrap();
5632 assert_eq!(task.status, TaskStatus::Blocked);
5633 assert!(missing_blockers(&q, &questions, &task.blocked_by).is_empty());
5634 assert_eq!(
5635 q.apply_deleted_blockers(&mut task, &questions),
5636 [dep.id.clone()]
5637 );
5638 assert_eq!(task.status, TaskStatus::Queued);
5639 }
5640
5641 #[test]
5642 fn dependency_resolution_short_deleted_reference_releases_only_its_waiters() {
5643 let (dir, q) = queue();
5644 let questions = Questions::at(dir.path().join("questions"));
5645 let mut dep = task("dependency");
5646 dep.id = "20261013-193611-ca5b".into();
5647 q.put(&mut dep).unwrap();
5648 let mut waiting = task("waiting");
5649 waiting.block(vec!["ca5b".into()], None);
5650 q.put(&mut waiting).unwrap();
5651 q.remove(&dep.id, false, &questions).unwrap();
5652 assert_eq!(q.get(&waiting.id).unwrap().status, TaskStatus::Queued);
5653 assert_eq!(
5654 dependency(&q, &questions, "ca5b"),
5655 Dependency::Deleted(dep.id)
5656 );
5657 }
5658
5659 #[test]
5660 fn dependency_resolution_preserves_cross_store_ambiguity_and_exact_ids() {
5661 let (dir, queue) = queue();
5662 let questions = Questions::at(dir.path().join("questions"));
5663 let mut dep = task("dependency");
5664 dep.id = "20261013-193611-ca5b".into();
5665 queue.put(&mut dep).unwrap();
5666 std::fs::create_dir_all(questions.root()).unwrap();
5667 let question_id = "20261014-193611-ca5b";
5668 std::fs::write(questions.path_of(question_id), "unreadable question").unwrap();
5669 assert_eq!(questions.resolve_id("ca5b").unwrap(), question_id);
5670 assert_eq!(
5671 dependency(&queue, &questions, "ca5b"),
5672 Dependency::Ambiguous
5673 );
5674 assert_eq!(
5675 dependency(&queue, &questions, &dep.id),
5676 Dependency::Task(dep.id)
5677 );
5678 assert_eq!(
5679 dependency(&queue, &questions, question_id),
5680 Dependency::Question(question_id.into())
5681 );
5682 assert_eq!(
5683 dependency(&queue, &questions, "missing"),
5684 Dependency::Missing
5685 );
5686 }
5687
5688 #[test]
5689 fn dependency_resolution_directory_errors_are_unknown_not_missing() {
5690 let dir = tempfile::tempdir().unwrap();
5691 let q = Queue::at(dir.path().join("not-a-directory"));
5692 std::fs::write(q.root(), "file").unwrap();
5693 let questions = Questions::at(dir.path().join("questions"));
5694 assert_eq!(dependency(&q, &questions, "ca5b"), Dependency::Unknown);
5695 assert!(missing_blockers(&q, &questions, &["ca5b".into()]).is_empty());
5696 }
5697
5698 #[test]
5699 fn a_dependency_without_a_tombstone_is_still_missing() {
5700 let (dir, q) = queue();
5701 let questions = Questions::at(dir.path().join("questions"));
5702 let mut dep = task("dependency");
5703 q.put(&mut dep).unwrap();
5704 std::fs::remove_file(q.path_of(&dep.id)).unwrap();
5705 assert_eq!(
5706 missing_blockers(&q, &questions, std::slice::from_ref(&dep.id)),
5707 [dep.id.clone()]
5708 );
5709 assert!(deleted_blockers(&q, &questions, std::slice::from_ref(&dep.id)).is_empty());
5710 }
5711
5712 #[test]
5713 fn a_held_dependent_is_left_alone_by_a_dependency_deletion() {
5714 let mut t = task("held");
5715 t.hold_machine(Some("because".to_owned()));
5716 t.blocked_by = vec!["gone".to_owned()];
5717 assert!(!t.dependency_deleted("gone"));
5718 assert_eq!(t.status, TaskStatus::Held);
5719 }
5720
5721 #[test]
5722 fn a_failed_record_removal_takes_the_tombstone_back() {
5723 let (_dir, q) = queue();
5724 let mut t = task("stays");
5725 q.put(&mut t).unwrap();
5726 q.write_tombstone(&t.id).unwrap();
5727 let err = q.remove_record_with_attachments(&t.id, |_| Err(std::io::Error::other("nope")));
5728 assert!(err.is_err());
5729 assert!(!q.was_deleted(&t.id));
5732 }
5733
5734 fn source_file(dir: &Path, name: &str, body: &str) -> PathBuf {
5735 let p = dir.join(name);
5736 std::fs::write(&p, body).unwrap();
5737 p
5738 }
5739
5740 #[test]
5741 fn an_attachment_copy_survives_deleting_its_source() {
5742 let (dir, q) = queue();
5743 let src = source_file(dir.path(), "shot.png", "pixels");
5744 let mut t = task("with a picture");
5745 let names = q.attach(&mut t, std::slice::from_ref(&src)).unwrap();
5746 q.put(&mut t).unwrap();
5747 std::fs::remove_file(&src).unwrap();
5748 assert_eq!(names, ["shot.png"]);
5749 let loaded = q.get(&t.id).unwrap();
5750 let paths = q.attachment_paths(&loaded);
5751 assert_eq!(paths.len(), 1);
5752 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "pixels");
5753 }
5754
5755 #[test]
5756 fn attachment_names_that_could_traverse_or_are_odd_are_refused() {
5757 let (dir, q) = queue();
5758 let mut t = task("bad names");
5759 for name in ["a..b.png", ".hidden", "with space.png", "-x.png"] {
5760 let src = source_file(dir.path(), name, "x");
5761 assert!(
5762 q.attach(&mut t, &[src]).is_err(),
5763 "`{name}` must be refused"
5764 );
5765 }
5766 assert!(!crate::ask::valid_asset_name("C:foo.png"));
5770 #[cfg(not(windows))]
5771 {
5772 let src = source_file(dir.path(), "C:foo.png", "x");
5773 assert!(q.attach(&mut t, &[src]).is_err());
5774 }
5775 let long = format!("{}.png", "a".repeat(70));
5776 let src = source_file(dir.path(), &long, "x");
5777 assert!(q.attach(&mut t, &[src]).is_err());
5778 assert!(t.attachments.is_empty());
5779 assert!(!q.attachments_dir(&t.id).exists());
5780 }
5781
5782 #[test]
5783 fn a_taken_attachment_name_is_numbered_not_overwritten() {
5784 let (dir, q) = queue();
5785 let a = source_file(dir.path(), "shot.png", "one");
5786 let sub = dir.path().join("other");
5787 std::fs::create_dir_all(&sub).unwrap();
5788 let b = source_file(&sub, "shot.png", "two");
5789 let mut t = task("collision");
5790 q.attach(&mut t, &[a]).unwrap();
5791 q.attach(&mut t, &[b]).unwrap();
5792 assert_eq!(t.attachments, ["shot.png", "shot-2.png"]);
5793 let paths = q.attachment_paths(&t);
5794 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "one");
5795 assert_eq!(std::fs::read_to_string(&paths[1]).unwrap(), "two");
5796 }
5797
5798 #[test]
5799 fn a_renumbered_name_stays_inside_the_length_bound() {
5800 let (dir, q) = queue();
5801 let name = format!("{}.png", "a".repeat(60));
5802 assert_eq!(name.len(), 64);
5803 let a = source_file(dir.path(), &name, "one");
5804 let sub = dir.path().join("other");
5805 std::fs::create_dir_all(&sub).unwrap();
5806 let b = source_file(&sub, &name, "two");
5807 let mut t = task("long");
5808 q.attach(&mut t, &[a, b]).unwrap();
5809 assert_eq!(t.attachments.len(), 2);
5810 assert!(
5811 t.attachments
5812 .iter()
5813 .all(|n| crate::ask::valid_asset_name(n))
5814 );
5815 assert!(t.attachments[1].ends_with("-2.png"));
5816 }
5817
5818 #[test]
5819 fn a_failed_attach_keeps_existing_attachments_and_leaves_no_partial_copy() {
5820 let (dir, q) = queue();
5821 let good = source_file(dir.path(), "good.png", "ok");
5822 let mut t = task("partial");
5823 q.attach(&mut t, &[good]).unwrap();
5824 let more = source_file(dir.path(), "more.png", "ok");
5825 let missing = dir.path().join("missing.png");
5826 assert!(q.attach(&mut t, &[more, missing]).is_err());
5827 assert_eq!(t.attachments, ["good.png"]);
5828 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5829 .unwrap()
5830 .flatten()
5831 .collect();
5832 assert_eq!(on_disk.len(), 1);
5833 }
5834
5835 fn block_put(q: &Queue, t: &Task) -> PathBuf {
5837 let tmp = q.path_of(&t.id).with_extension("json.tmp");
5838 std::fs::create_dir_all(&tmp).unwrap();
5839 tmp
5840 }
5841
5842 #[test]
5843 fn a_failed_put_leaves_no_new_attachment_directory() {
5844 let (dir, q) = queue();
5845 let mut t = task("fresh");
5846 let tmp = block_put(&q, &t);
5847 let src = source_file(dir.path(), "shot.png", "x");
5848 assert!(q.attach_and_put(&mut t, &[src]).is_err());
5849 assert!(t.attachments.is_empty());
5850 assert!(!q.attachments_dir(&t.id).exists());
5851 assert!(!q.path_of(&t.id).exists());
5852 std::fs::remove_dir(tmp).unwrap();
5853 }
5854
5855 #[test]
5856 fn a_failed_put_removes_only_the_copy_it_just_made() {
5857 let (dir, q) = queue();
5858 let mut t = task("edited");
5859 let first = source_file(dir.path(), "first.png", "1");
5860 q.attach_and_put(&mut t, &[first]).unwrap();
5861 block_put(&q, &t);
5862 let second = source_file(dir.path(), "second.png", "2");
5863 assert!(q.attach_and_put(&mut t, &[second]).is_err());
5864 assert_eq!(t.attachments, ["first.png"]);
5865 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5866 .unwrap()
5867 .flatten()
5868 .map(|e| e.file_name().to_string_lossy().into_owned())
5869 .collect();
5870 assert_eq!(on_disk, ["first.png"]);
5871 assert_eq!(q.get(&t.id).unwrap().attachments, ["first.png"]);
5872 }
5873
5874 #[test]
5875 fn a_leftover_removing_directory_is_swept_by_the_next_removal() {
5876 let (dir, q) = queue();
5877 let questions = Questions::at(dir.path().join("questions"));
5878 let gone = task("gone");
5879 let mut other = task("other");
5880 let mut live = task("live");
5881 q.put(&mut other).unwrap();
5882 q.put(&mut live).unwrap();
5883 let orphan = q.root.join(format!("{}.attachments.removing", gone.id));
5886 std::fs::create_dir_all(&orphan).unwrap();
5887 std::fs::write(orphan.join("shot.png"), "x").unwrap();
5888 let busy = q.root.join(format!("{}.attachments.removing", live.id));
5891 std::fs::create_dir_all(&busy).unwrap();
5892
5893 q.remove(&other.id, false, &questions).unwrap();
5894 assert!(!orphan.exists(), "an orphan is swept");
5895 assert!(busy.exists(), "a removal in progress is left alone");
5896 }
5897
5898 #[test]
5899 fn a_blocked_aside_rename_fails_the_removal_and_loses_nothing() {
5900 let (dir, q) = queue();
5901 let questions = Questions::at(dir.path().join("questions"));
5902 let mut t = task("stuck");
5903 let src = source_file(dir.path(), "shot.png", "x");
5904 q.attach_and_put(&mut t, &[src]).unwrap();
5905 let aside = q.root.join(format!("{}.attachments.removing", t.id));
5908 std::fs::create_dir_all(&aside).unwrap();
5909 std::fs::write(aside.join("old.png"), "o").unwrap();
5910 assert!(q.remove(&t.id, false, &questions).is_err());
5911 assert!(q.path_of(&t.id).exists());
5912 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5913 }
5914
5915 #[test]
5916 fn a_failed_record_removal_puts_the_attachments_back() {
5917 let (dir, q) = queue();
5918 let mut t = task("rollback");
5919 let src = source_file(dir.path(), "shot.png", "x");
5920 q.attach_and_put(&mut t, &[src]).unwrap();
5921 let err = q
5922 .remove_record_with_attachments(&t.id, |_| {
5923 Err(std::io::Error::other("injected failure"))
5924 })
5925 .unwrap_err();
5926 assert!(format!("{err:#}").contains("injected failure"));
5927 assert!(q.path_of(&t.id).exists());
5928 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5929 assert!(
5930 !q.root
5931 .join(format!("{}.attachments.removing", t.id))
5932 .exists()
5933 );
5934 }
5935
5936 #[test]
5937 fn editing_a_task_keeps_its_attachments() {
5938 let (dir, q) = queue();
5939 let src = source_file(dir.path(), "shot.png", "x");
5940 let mut t = task("editable");
5941 q.attach(&mut t, &[src]).unwrap();
5942 t.edit("new".to_owned(), "new text".to_owned()).unwrap();
5943 q.put(&mut t).unwrap();
5944 assert_eq!(q.get(&t.id).unwrap().attachments, ["shot.png"]);
5945 }
5946
5947 #[test]
5948 fn removing_a_task_deletes_its_attachments() {
5949 let (dir, q) = queue();
5950 let questions = Questions::at(dir.path().join("questions"));
5951 let src = source_file(dir.path(), "shot.png", "x");
5952 let mut t = task("doomed");
5953 q.attach(&mut t, &[src]).unwrap();
5954 q.put(&mut t).unwrap();
5955 assert!(q.attachments_dir(&t.id).is_dir());
5956 q.remove(&t.id, false, &questions).unwrap();
5957 assert!(!q.attachments_dir(&t.id).exists());
5958 assert!(q.list().is_empty());
5959 }
5960
5961 #[test]
5962 fn a_task_written_before_attachments_still_reads() {
5963 let (_dir, q) = queue();
5964 let mut t = task("old");
5965 q.put(&mut t).unwrap();
5966 let path = q.path_of(&t.id);
5967 let mut v: serde_json::Value =
5968 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5969 v.as_object_mut().unwrap().remove("attachments");
5970 std::fs::write(&path, v.to_string()).unwrap();
5971 assert!(q.get(&t.id).unwrap().attachments.is_empty());
5972 }
5973
5974 #[test]
5975 fn attachment_paths_are_absolute_even_when_the_root_is_relative() {
5976 let q = Queue::at(PathBuf::from("relative-queue"));
5977 let mut t = task("rel");
5978 t.attachments.push("shot.png".to_owned());
5979 let paths = q.attachment_paths(&t);
5980 assert!(paths[0].is_absolute(), "{}", paths[0].display());
5981 assert!(paths[0].ends_with(format!("{}.attachments/shot.png", t.id)));
5982 }
5983
5984 fn upgrade_parked(id: &str, started: &str) -> Task {
5985 let mut t = task(id);
5986 t.id = id.into();
5987 t.start(format!("run-{id}"));
5988 t.park("upgrade");
5989 t.record_pass_at(
5990 &format!("run-{id}"),
5991 0,
5992 Some(started.parse().unwrap()),
5993 1,
5994 Some(crate::run::ParkCause::Upgrade {
5995 from: "old".into(),
5996 to: None,
5997 }),
5998 );
5999 t
6000 }
6001
6002 #[test]
6003 fn upgrade_resume_order_follows_dispatch_claim_not_graph_start() {
6004 let stamp = |id: &str, dispatched: &str, started: &str| {
6006 let mut t = task(id);
6007 t.id = id.into();
6008 t.start(format!("run-{id}"));
6009 t.park("upgrade");
6010 t.dispatched_at = Some(dispatched.parse().unwrap());
6011 t.record_pass_at(
6012 &format!("run-{id}"),
6013 0,
6014 Some(started.parse().unwrap()),
6015 1,
6016 Some(crate::run::ParkCause::Upgrade {
6017 from: "old".into(),
6018 to: None,
6019 }),
6020 );
6021 t
6022 };
6023 let a = stamp("a", "2026-10-22T00:00:01Z", "2026-10-22T00:00:09Z");
6024 let b = stamp("b", "2026-10-22T00:00:02Z", "2026-10-22T00:00:03Z");
6025 assert!(dispatch_order(&a, &b).is_lt());
6026 }
6027
6028 #[test]
6029 fn upgrade_resume_order_survives_queue_reload_and_reversed_settlement() {
6030 let (_dir, q) = queue();
6031 let mut later = upgrade_parked("later", "2026-10-22T00:00:02Z");
6032 let mut earlier = upgrade_parked("earlier", "2026-10-22T00:00:01Z");
6033 earlier.priority = -10;
6034 let mut fresh = task("fresh");
6035 fresh.id = "newer".into();
6036 fresh.priority = 100;
6037 for t in [&mut fresh, &mut later, &mut earlier] {
6038 q.put(t).unwrap();
6039 }
6040 let reloaded = Queue::at(q.root().to_owned());
6041 let mut tasks = reloaded.list();
6042 tasks.sort_unstable_by(dispatch_order);
6043 assert_eq!(
6044 tasks.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
6045 vec!["earlier", "later", "newer"]
6046 );
6047 assert_eq!(reloaded.next_runnable().unwrap().id, "earlier");
6048 assert_eq!(reloaded.get("earlier").unwrap().attempts, 0);
6049 }
6050
6051 #[test]
6052 fn upgrade_resume_legacy_order_uses_settlement_and_id() {
6053 let mut a = upgrade_parked("a", "2026-10-22T00:00:02Z");
6054 let mut b = upgrade_parked("b", "2026-10-22T00:00:01Z");
6055 for t in [&mut a, &mut b] {
6056 let mut value = serde_json::to_value(&*t).unwrap();
6057 value["passes"][0]
6058 .as_object_mut()
6059 .unwrap()
6060 .remove("started_at");
6061 *t = serde_json::from_value(value).unwrap();
6062 }
6063 assert!(dispatch_order(&a, &b).is_lt());
6064 b.passes[0].settled_at = a.passes[0].settled_at;
6065 assert!(dispatch_order(&a, &b).is_lt());
6066 }
6067
6068 #[test]
6069 fn upgrade_resume_key_excludes_other_parks_and_explicit_task_changes() {
6070 let original = upgrade_parked("a", "2026-10-22T00:00:01Z");
6071 assert!(original.upgrade_resume_key().is_some());
6072 for cause in [
6073 crate::run::ParkCause::Operator,
6074 crate::run::ParkCause::Question,
6075 crate::run::ParkCause::Interrupt {
6076 reason: "interrupt".into(),
6077 },
6078 crate::run::ParkCause::Unspecified,
6079 ] {
6080 let mut t = original.clone();
6081 t.passes[0].parked = Some(cause);
6082 assert!(t.upgrade_resume_key().is_none());
6083 let mut fresh = task("fresh");
6084 fresh.priority = 1;
6085 assert!(dispatch_order(&fresh, &t).is_lt());
6086 }
6087 let mut t = original.clone();
6088 t.release();
6089 assert!(t.upgrade_resume_key().is_none());
6090 t.edit("edited".into(), "new instruction".into()).unwrap();
6091 assert!(t.upgrade_resume_key().is_none());
6092 let mut t = original.clone();
6093 t.requeue();
6094 t.status = TaskStatus::Parked;
6095 assert!(t.upgrade_resume_key().is_none());
6096 let mut t = original.clone();
6097 t.start("different-run".into());
6098 t.park("no upgrade");
6099 assert!(t.upgrade_resume_key().is_none());
6100 }
6101
6102 #[test]
6103 fn explicitly_resumed_older_run_becomes_current_without_duplicate_history() {
6104 let dir = tempfile::tempdir().unwrap();
6105 let q = Queue::at(dir.path().join("queue"));
6106 let mut t = task("resume");
6107 t.start("A".into());
6108 t.record_pass("A", 0, 1, None);
6109 t.start("B".into());
6110 q.put(&mut t).unwrap();
6111 assert_eq!(t.runs, vec!["A", "B"]);
6112 t.start("A".into());
6113 t.record_pass("A", 1, 3, None);
6114 q.put(&mut t).unwrap();
6115 let saved = q.get(&t.id).unwrap();
6116 assert_eq!(saved.runs, vec!["B", "A"]);
6117 assert_eq!(saved.runs.last().map(String::as_str), Some("A"));
6118 assert_eq!(saved.attempts, 3);
6119 assert_eq!(saved.passes.len(), 2);
6120 assert_eq!(saved.passes[0].run, "A");
6121 assert_eq!(saved.passes[1].run, "A");
6122 }
6123
6124 #[test]
6125 fn repeated_start_records_one_run_and_separate_passes() {
6126 let mut t = task("resume");
6127 t.start("same-run".into());
6128 t.record_pass("same-run", 0, 1, None);
6129 t.start("same-run".into());
6130 t.record_pass("same-run", 1, 2, None);
6131 assert_eq!(t.runs, vec!["same-run"]);
6132 assert_eq!(t.attempts, 2);
6133 assert_eq!(t.passes.len(), 2);
6134 }
6135
6136 #[test]
6137 fn link_run_adds_a_run_once_and_touches_nothing_else() {
6138 let dir = tempfile::tempdir().unwrap();
6139 let queue = Queue::at(dir.path().join("queue"));
6140 let mut t = Task::new(
6141 "t".to_owned(),
6142 "do it".to_owned(),
6143 PathBuf::from("."),
6144 Source::Human,
6145 );
6146 queue.put(&mut t).unwrap();
6147 let before = queue.get(&t.id).unwrap();
6148
6149 let linked = queue.link_run(&t.id[..4], "20260930-092817-ec34").unwrap();
6150 assert_eq!(linked.runs, vec!["20260930-092817-ec34".to_owned()]);
6151 assert_eq!(linked.status, before.status);
6152 assert_eq!(linked.attempts, before.attempts);
6153 assert_eq!(linked.interrupt, before.interrupt);
6154
6155 let again = queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
6156 assert_eq!(again.runs.len(), 1, "linking twice must not duplicate");
6157 assert_eq!(queue.get(&t.id).unwrap().runs.len(), 1);
6158 assert!(queue.link_run("no-such-task", "r").is_err());
6159 }
6160
6161 #[test]
6162 fn put_keeps_a_run_linked_after_the_writer_took_its_snapshot() {
6163 let dir = tempfile::tempdir().unwrap();
6164 let queue = Queue::at(dir.path().join("queue"));
6165 let mut t = Task::new(
6166 "t".to_owned(),
6167 "do it".to_owned(),
6168 PathBuf::from("."),
6169 Source::Human,
6170 );
6171 queue.put(&mut t).unwrap();
6172 let mut snapshot = queue.get(&t.id).unwrap();
6174 queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
6175
6176 snapshot.start("20260930-000000-aaaa".to_owned());
6177 queue.put(&mut snapshot).unwrap();
6178
6179 let stored = queue.get(&t.id).unwrap();
6180 assert!(stored.runs.contains(&"20260930-092817-ec34".to_owned()));
6181 assert!(stored.runs.contains(&"20260930-000000-aaaa".to_owned()));
6182 assert_eq!(stored.attempts, 1);
6183 }
6184
6185 #[test]
6186 fn concurrent_links_and_daemon_saves_lose_nothing() {
6187 let dir = tempfile::tempdir().unwrap();
6188 let queue = Queue::at(dir.path().join("queue"));
6189 let mut t = Task::new(
6190 "t".to_owned(),
6191 "do it".to_owned(),
6192 PathBuf::from("."),
6193 Source::Human,
6194 );
6195 queue.put(&mut t).unwrap();
6196 let id = t.id.clone();
6197
6198 let linkers: Vec<_> = (0..4)
6199 .map(|n| {
6200 let (queue, id) = (queue.clone(), id.clone());
6201 std::thread::spawn(move || {
6202 for k in 0..10 {
6203 queue
6204 .link_run(&id, &format!("20260930-00000{n}-l{k:03}"))
6205 .unwrap();
6206 }
6207 })
6208 })
6209 .collect();
6210 let mut mine = queue.get(&id).unwrap();
6213 for k in 0..10 {
6214 mine.start(format!("20260930-000009-d{k:03}"));
6215 queue.put(&mut mine).unwrap();
6216 }
6217 for l in linkers {
6218 l.join().unwrap();
6219 }
6220
6221 let stored = queue.get(&id).unwrap();
6222 assert_eq!(stored.runs.len(), 50, "{:?}", stored.runs);
6223 assert_eq!(
6224 stored.attempts, 10,
6225 "linking never rewinds the daemon's work"
6226 );
6227 }
6228
6229 fn age_lock(path: &Path) {
6230 let f = std::fs::OpenOptions::new().write(true).open(path).unwrap();
6231 f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(60))
6232 .unwrap();
6233 }
6234
6235 #[test]
6236 fn concurrent_stale_takeover_yields_one_holder() {
6237 use std::sync::atomic::{AtomicUsize, Ordering};
6238 use std::sync::{Arc, Barrier};
6239 for _ in 0..5 {
6240 let dir = tempfile::tempdir().unwrap();
6241 let q = Queue::at(dir.path().to_path_buf());
6242 let lock = dir.path().join("t.write-lock");
6243 std::fs::write(&lock, "dead-0000").unwrap();
6244 age_lock(&lock);
6245 let n = 6;
6246 let barrier = Arc::new(Barrier::new(n));
6247 let (now, max) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
6248 let handles: Vec<_> = (0..n)
6249 .map(|_| {
6250 let (q, b, now, max) = (q.clone(), barrier.clone(), now.clone(), max.clone());
6251 std::thread::spawn(move || {
6252 b.wait();
6253 let g = q.lock_task("t").unwrap();
6254 let held = now.fetch_add(1, Ordering::SeqCst) + 1;
6255 max.fetch_max(held, Ordering::SeqCst);
6256 std::thread::sleep(std::time::Duration::from_millis(20));
6257 now.fetch_sub(1, Ordering::SeqCst);
6258 drop(g);
6259 })
6260 })
6261 .collect();
6262 for h in handles {
6263 h.join().unwrap();
6264 }
6265 assert_eq!(max.load(Ordering::SeqCst), 1);
6266 assert!(!lock.exists());
6267 }
6268 }
6269
6270 #[test]
6271 fn dropping_a_replaced_lock_leaves_the_new_holders_token() {
6272 let dir = tempfile::tempdir().unwrap();
6273 let q = Queue::at(dir.path().to_path_buf());
6274 let lock = dir.path().join("t.write-lock");
6275 let a = q.lock_task("t").unwrap();
6276 let replacement = owner_token();
6279 std::fs::write(&lock, &replacement).unwrap();
6280 drop(a);
6281 assert_eq!(std::fs::read_to_string(&lock).unwrap(), replacement);
6282 }
6283
6284 #[test]
6285 fn aged_live_task_lock_excludes_updates_until_retirement_releases_it() {
6286 let dir = tempfile::tempdir().unwrap();
6287 let q = Queue::at(dir.path().to_path_buf());
6288 let held = q.lock_task("t").unwrap();
6289 let path = dir.path().join("t.write-lock");
6290 age_lock(&path);
6291 assert!(!break_stale(&path, &held.token));
6292 let (started_tx, started_rx) = std::sync::mpsc::channel();
6293 let (acquired_tx, acquired_rx) = std::sync::mpsc::channel();
6294 let updater = std::thread::spawn(move || {
6295 started_tx.send(()).unwrap();
6296 let _write = q.lock_task("t").unwrap();
6297 acquired_tx.send(()).unwrap();
6298 });
6299 started_rx.recv().unwrap();
6300 assert!(
6301 acquired_rx
6302 .recv_timeout(std::time::Duration::from_millis(100))
6303 .is_err()
6304 );
6305 assert_eq!(std::fs::read_to_string(&path).unwrap(), held.token);
6306 drop(held);
6307 acquired_rx
6308 .recv_timeout(std::time::Duration::from_secs(2))
6309 .unwrap();
6310 updater.join().unwrap();
6311 assert!(!path.exists());
6312 }
6313
6314 #[test]
6315 fn break_stale_leaves_a_lock_that_replaced_the_one_judged() {
6316 let dir = tempfile::tempdir().unwrap();
6317 let lock = dir.path().join("t.write-lock");
6318 std::fs::write(&lock, "new-token").unwrap();
6319 assert!(!break_stale(&lock, "old-token"));
6320 assert_eq!(std::fs::read_to_string(&lock).unwrap(), "new-token");
6321 assert!(!break_stale(&lock, "new-token"));
6323 assert!(lock.exists());
6324 age_lock(&lock);
6325 assert!(break_stale(&lock, "new-token"));
6326 assert!(!lock.exists());
6327 }
6328
6329 #[test]
6330 fn a_stale_break_marker_is_recovered_and_a_fresh_one_is_respected() {
6331 let dir = tempfile::tempdir().unwrap();
6332 let lock = dir.path().join("t.write-lock");
6333 std::fs::write(&lock, "dead-1").unwrap();
6334 age_lock(&lock);
6335 let marker = break_marker(&lock, "dead-1");
6336 std::fs::write(&marker, "crashed-remover").unwrap();
6337 assert!(!break_stale(&lock, "dead-1"));
6339 assert!(lock.exists());
6340 age_lock(&marker);
6342 assert!(break_stale(&lock, "dead-1"));
6343 assert!(!lock.exists());
6344 assert!(!marker.exists());
6345 }
6346
6347 fn chat_task() -> Task {
6348 Task::new(
6349 "t".to_owned(),
6350 "do t".to_owned(),
6351 PathBuf::from("."),
6352 Source::Agent {
6353 run: "20260902-000000-beef".to_owned(),
6354 node: CHAT_NODE.to_owned(),
6355 },
6356 )
6357 }
6358
6359 #[test]
6360 fn a_blocked_hold_keeps_its_anchor_but_a_new_run_starts_a_new_hold() {
6361 let mut t = chat_task();
6362 t.hold_machine(Some("waiting".into()));
6363 let old: Timestamp = "2020-01-01T00:00:00Z".parse().unwrap();
6364 t.held_at = Some(old);
6365 t.block(vec!["decision".into()], None);
6366 t.hold_machine(Some("same waiting decision".into()));
6367 assert_eq!(t.held_at, Some(old));
6368 t.start("real-new-run".into());
6369 t.fail("new failure", 1);
6370 assert_ne!(
6371 t.held_at,
6372 Some(old),
6373 "old answers cannot approve a new hold"
6374 );
6375 }
6376
6377 #[test]
6378 fn legacy_delivery_survives_upgrade_and_a_no_progress_run_link() {
6379 let (_dir, queue) = queue();
6380 let mut t = chat_task();
6381 t.start("old-run".into());
6382 t.hold_machine(Some("operator is considering recovery".into()));
6383 t.chat_notice = Some(ChatNotice {
6384 key: format!("held#{}", t.held_at.unwrap()),
6385 at: Timestamp::now(),
6386 outcome: ChatNoticeOutcome::Posted,
6387 });
6388 let mut raw = serde_json::to_value(&t).unwrap();
6389 raw.as_object_mut().unwrap().remove("chat_notices");
6390 raw.as_object_mut().unwrap().remove("hold_notice_cause");
6391 std::fs::create_dir_all(queue.path_of(&t.id).parent().unwrap()).unwrap();
6392 std::fs::write(queue.path_of(&t.id), serde_json::to_string(&raw).unwrap()).unwrap();
6393 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
6394 queue
6395 .modify(&t.id, |task| {
6396 task.requeue();
6397 task.link_run("unprepared-run");
6398 task.startup_notice_cause("startup-source");
6399 task.fail("same unresolved preflight", 0);
6400 true
6401 })
6402 .unwrap();
6403 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
6404 queue
6405 .modify(&t.id, |task| {
6406 task.succeed();
6407 true
6408 })
6409 .unwrap();
6410 assert_eq!(
6411 queue.get(&t.id).unwrap().chat_notice_due().as_deref(),
6412 Some("done")
6413 );
6414 }
6415
6416 #[test]
6417 fn a_dependency_wait_does_not_change_when_released_and_blocked_again() {
6418 let mut t = chat_task();
6419 t.block(vec!["a".into(), "b".into()], Some("waiting".into()));
6420 let key = t.chat_notice_key().unwrap();
6421 t.chat_notices.push(key);
6422 t.release();
6423 t.block(
6424 vec!["b".into(), "a".into(), "a".into()],
6425 Some("still waiting on prerequisites".into()),
6426 );
6427 assert_eq!(t.chat_notice_due(), None);
6428 t.block(
6429 vec!["new-question".into()],
6430 Some("different decision".into()),
6431 );
6432 assert!(t.chat_notice_due().is_some());
6433 }
6434
6435 #[test]
6436 fn running_startup_failures_preserve_stage_and_report_distinct_blockers() {
6437 let mut t = chat_task();
6438 t.start("existing-run".into());
6439 t.fail("implementation failed", 1);
6440 let failed = t.chat_notice_key().unwrap();
6441 t.chat_notices.push(failed.clone());
6442 t.release_operator("test", None);
6443 t.start("existing-run".into());
6444 t.fail_startup("startup-source", "same unresolved recovery source", 1);
6445 assert_eq!(
6446 t.chat_notice_due(),
6447 None,
6448 "no-progress recovery stays quiet"
6449 );
6450 t.release_operator("test", None);
6451 t.start("existing-run".into());
6452 t.fail_startup("startup-attachments", "attachment missing", 1);
6453 let attachments = t.chat_notice_due().expect("new attachment blocker");
6454 assert_ne!(attachments, failed);
6455 assert_eq!(
6456 t.hold_notice_cause.as_ref().unwrap().stage,
6457 "startup-attachments"
6458 );
6459 t.chat_notices.push(attachments);
6460 t.release_operator("test", None);
6461 t.start("existing-run".into());
6462 t.fail_startup(
6463 "startup-attachments",
6464 "same missing attachment, reworded",
6465 1,
6466 );
6467 assert_eq!(
6468 t.chat_notice_due(),
6469 None,
6470 "same stage after attempt counter reset"
6471 );
6472 t.release_operator("test", None);
6473 t.start("existing-run".into());
6474 t.fail_startup("startup-config", "different preflight stage", 1);
6475 assert!(t.chat_notice_due().is_some());
6476 assert_eq!(
6477 t.hold_notice_cause.as_ref().unwrap().stage,
6478 "startup-config"
6479 );
6480 }
6481
6482 #[test]
6483 fn structured_causes_preserve_no_progress_retries_and_report_new_passes() {
6484 let mut t = chat_task();
6485 t.hold_machine(Some("startup source missing".into()));
6486 t.startup_notice_cause("startup-source");
6487 let first = t.chat_notice_key().unwrap();
6488 t.chat_notices.push(first.clone());
6489 t.release_operator("test", None);
6490 t.startup_notice_cause("startup-source");
6491 t.hold_machine(Some("same source cannot be read".into()));
6492 assert_eq!(t.chat_notice_due(), None);
6493 t.startup_notice_cause("startup-config");
6494 assert!(t.chat_notice_due().is_some(), "different startup stage");
6495 t.notice_cause("failed", Some("existing-run"), Some(0));
6496 let run_failure = t.chat_notice_key().unwrap();
6497 t.chat_notices.push(run_failure.clone());
6498 t.release_operator("test", None);
6499 t.startup_notice_cause("startup-source");
6500 t.hold_machine(Some("no new work happened".into()));
6501 assert_eq!(t.chat_notice_due(), None);
6502 t.notice_cause("failed", Some("existing-run"), Some(1));
6503 assert!(t.chat_notice_due().is_some(), "a real resumed pass failed");
6504 assert_ne!(t.chat_notice_key().unwrap(), run_failure);
6505 }
6506
6507 #[test]
6508 fn legacy_hold_baseline_does_not_replay_or_hide_new_failures_and_completion() {
6509 let mut t = chat_task();
6510 t.hold_machine(Some("still waiting".into()));
6511 t.chat_notice = Some(ChatNotice {
6512 key: "held#2026-10-14T00:00:00Z".into(),
6513 at: Timestamp::now(),
6514 outcome: ChatNoticeOutcome::Posted,
6515 });
6516 assert_eq!(t.chat_notice_due(), None, "old hold baseline");
6517 t.import_chat_notice();
6518 t.notice_cause("failed", Some("new-run"), Some(0));
6519 assert!(t.chat_notice_due().is_some(), "new producer evidence");
6520 t.succeed();
6521 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
6522 }
6523
6524 #[test]
6525 fn a_chat_notice_is_due_only_for_a_final_state_and_only_once() {
6526 let mut t = chat_task();
6527 assert_eq!(t.chat_notice_due(), None, "queued");
6528 t.start("r1".to_owned());
6529 assert_eq!(t.chat_notice_due(), None, "running");
6530 t.fail("boom", 3);
6531 assert_eq!(t.status, TaskStatus::Failed);
6532 assert_eq!(t.chat_notice_due(), None, "a retried failure is not final");
6533 t.start("r2".to_owned());
6534 t.park("upgrade");
6535 assert_eq!(t.chat_notice_due(), None, "parked");
6536 t.fail("boom", 1);
6537 assert_eq!(t.status, TaskStatus::Held);
6538 let key = t
6539 .chat_notice_due()
6540 .expect("held on the last attempt is final");
6541 t.chat_notice = Some(ChatNotice {
6542 key,
6543 at: Timestamp::now(),
6544 outcome: ChatNoticeOutcome::Posted,
6545 });
6546 assert_eq!(t.chat_notice_due(), None, "same hold, no second notice");
6547 t.release();
6548 assert_eq!(t.chat_notice_due(), None, "requeued after a release");
6549 t.start("r3".to_owned());
6550 t.succeed();
6551 assert!(t.chat_notice_due().is_some(), "done is a new final state");
6552 }
6553
6554 #[test]
6555 fn completed_chat_notices_keep_current_and_legacy_keys_after_requeue() {
6556 for key in ["done", "done#0", "done#1", "done#12"] {
6557 let mut t = chat_task();
6558 t.start("r1".to_owned());
6559 t.succeed();
6560 t.chat_notice = Some(ChatNotice {
6561 key: key.to_owned(),
6562 at: Timestamp::now(),
6563 outcome: ChatNoticeOutcome::Posted,
6564 });
6565 t.requeue();
6566 t.start("r2".to_owned());
6567 t.succeed();
6568 assert_eq!(t.chat_notice_key().as_deref(), Some("done"));
6569 assert_eq!(t.chat_notice_due(), None, "recorded key: {key}");
6570 }
6571 }
6572
6573 #[test]
6574 fn a_release_without_progress_keeps_the_unresolved_notice_cause() {
6575 let mut t = chat_task();
6576 t.hold_manual(Some("wait".to_owned()));
6577 let first = t.chat_notice_due().expect("held");
6578 t.chat_notice = Some(ChatNotice {
6579 key: first.clone(),
6580 at: Timestamp::now(),
6581 outcome: ChatNoticeOutcome::Posted,
6582 });
6583 t.hold_manual(Some("still waiting".to_owned()));
6584 assert_eq!(t.chat_notice_due(), None, "a re-hold keeps its start");
6585 t.release();
6586 std::thread::sleep(std::time::Duration::from_millis(5));
6587 t.hold_manual(None);
6588 assert_eq!(t.chat_notice_due(), None, "no progress, same cause");
6589 }
6590
6591 #[test]
6592 fn a_chat_notice_needs_a_chat_source_and_a_current_schema() {
6593 let mut human = task("h");
6594 human.succeed();
6595 assert_eq!(human.chat_notice_due(), None);
6596 let mut inherited = task("i");
6598 inherited.origin_chat = Some("20260902-000000-beef".to_owned());
6599 inherited.succeed();
6600 assert_eq!(inherited.chat_notice_due(), None);
6601 let mut old = chat_task();
6602 old.schema = 12;
6603 old.status = TaskStatus::Done;
6604 assert_eq!(old.chat_notice_due(), None, "history is not replayed");
6605 let mut live = chat_task();
6606 live.schema = 12;
6607 live.start("r1".to_owned());
6608 live.succeed();
6609 assert!(live.chat_notice_due().is_some(), "started by this build");
6610 let mut held = chat_task();
6611 held.schema = 12;
6612 held.hold_machine(Some("config".to_owned()));
6613 assert!(held.chat_notice_due().is_some(), "held before any start");
6614 }
6615
6616 #[test]
6617 fn a_stale_snapshot_does_not_rewind_a_recorded_chat_notice() {
6618 let tmp = tempfile::tempdir().expect("tempdir");
6619 let q = Queue::at(tmp.path().join("queue"));
6620 let mut t = chat_task();
6621 t.succeed();
6622 q.put(&mut t).expect("put");
6623 let mut stale = q.get(&t.id).expect("get");
6624 let key = t.chat_notice_due().expect("due");
6625 q.modify(&t.id, |x| {
6626 x.chat_notice = Some(ChatNotice {
6627 key: key.clone(),
6628 at: Timestamp::now(),
6629 outcome: ChatNoticeOutcome::Posted,
6630 });
6631 true
6632 })
6633 .expect("modify");
6634 q.put(&mut stale).expect("stale put");
6635 assert_eq!(q.get(&t.id).expect("get").chat_notice_due(), None);
6636 }
6637
6638 #[test]
6639 fn empty_sources_are_omitted_so_older_builds_still_read_the_task() {
6640 let mut t = task("plain");
6643 let value = serde_json::to_value(&t).unwrap();
6644 assert!(value.get("sources").is_none(), "{value}");
6645 t.sources.pick_commits.push(crate::refs::Input {
6646 token: "abc1234".into(),
6647 sha: "a".repeat(40),
6648 provenance: crate::refs::Provenance::ExplicitCli,
6649 });
6650 let value = serde_json::to_value(&t).unwrap();
6651 assert_eq!(value["sources"]["pick_commits"][0]["sha"], "a".repeat(40));
6652 let restored: Task = serde_json::from_value(value).unwrap();
6653 assert_eq!(restored.sources, t.sources);
6654 }
6655
6656 fn queue_with_ids(dir: &Path, ids: &[&str]) -> Queue {
6658 let ids: Vec<String> = ids.iter().map(|s| (*s).to_owned()).collect();
6659 let n = std::sync::atomic::AtomicUsize::new(0);
6660 Queue::at(dir.join("queue")).with_id_source(move || {
6661 let i = n.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6662 ids.get(i)
6663 .cloned()
6664 .unwrap_or_else(|| format!("20260101-000000-f{i:03x}"))
6665 })
6666 }
6667
6668 const TAKEN: &str = "20260101-000000-aaaa";
6669
6670 #[test]
6671 fn a_new_task_never_merges_into_a_record_with_its_candidate_id() {
6672 let dir = tempfile::tempdir().unwrap();
6673 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6674 let mut first = task("first");
6675 first.start("run-1".into());
6676 first.succeed();
6677 q.put(&mut first).unwrap();
6678 assert_eq!(first.id, TAKEN);
6679 let stored_before = q.get(&first.id).unwrap();
6680 assert!(!stored_before.runs.is_empty());
6681
6682 let mut second = task("second");
6683 q.put(&mut second).unwrap();
6684
6685 assert_ne!(second.id, first.id);
6686 assert!(second.runs.is_empty());
6687 assert!(second.passes.is_empty());
6688 assert!(second.chat_notices.is_empty());
6689 assert!(second.chat_notice.is_none());
6690 assert_eq!(second.chat_notice_due(), task("fresh").chat_notice_due());
6691 let a = q.get(&first.id).unwrap();
6692 let b = q.get(&second.id).unwrap();
6693 assert_eq!(a.title, "first");
6694 assert_eq!(a.runs, stored_before.runs);
6695 assert_eq!(a.passes, stored_before.passes);
6696 assert_eq!(a.chat_notices, stored_before.chat_notices);
6697 assert_eq!(a.chat_notice, stored_before.chat_notice);
6698 assert_eq!(a.chat_notice_due(), stored_before.chat_notice_due());
6699 assert_eq!(b.title, "second");
6700 assert!(b.runs.is_empty());
6701 assert!(b.passes.is_empty());
6702 assert!(b.chat_notice.is_none());
6703
6704 let mut again = q.get(&second.id).unwrap();
6706 again.title = "second, edited".into();
6707 q.put(&mut again).unwrap();
6708 assert_eq!(q.get(&second.id).unwrap().title, "second, edited");
6709 assert_eq!(q.list().len(), 2);
6710 }
6711
6712 #[test]
6713 fn a_new_task_skips_an_id_with_an_orphaned_claim_or_attachments() {
6714 let dir = tempfile::tempdir().unwrap();
6715 let q = queue_with_ids(dir.path(), &["orphan-lock", "orphan-dir", "free"]);
6716 std::fs::create_dir_all(q.root()).unwrap();
6717 std::fs::write(q.root().join("orphan-lock.lock"), "x").unwrap();
6718 std::fs::create_dir_all(q.attachments_dir("orphan-dir")).unwrap();
6719 let mut t = task("t");
6720 q.put(&mut t).unwrap();
6721 assert_eq!(t.id, "free");
6722 }
6723
6724 #[test]
6725 fn a_claimed_new_task_holds_the_claim_on_the_reserved_id() {
6726 let dir = tempfile::tempdir().unwrap();
6727 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6728 let mut first = task("first");
6729 q.put(&mut first).unwrap();
6730 let mut second = task("second");
6731 let claim = q.file_claimed(&mut second).unwrap();
6732 assert_ne!(second.id, first.id);
6733 assert!(q.claim(&second.id).is_err());
6734 assert!(q.claim(&first.id).is_ok(), "the first task was not claimed");
6735 drop(claim);
6736 }
6737
6738 #[test]
6739 fn an_attached_new_task_skips_a_taken_id() {
6740 let dir = tempfile::tempdir().unwrap();
6741 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6742 let mut first = task("first");
6743 q.put(&mut first).unwrap();
6744 let src = source_file(dir.path(), "shot.png", "x");
6745 let mut second = task("second");
6746 q.attach_and_put(&mut second, &[src]).unwrap();
6747 assert_ne!(second.id, first.id);
6748 assert!(!q.attachments_dir(&first.id).exists());
6749 assert_eq!(q.attachment_paths(&second).len(), 1);
6750 assert!(q.attachment_paths(&second)[0].is_file());
6751 }
6752
6753 #[test]
6754 fn attaching_before_putting_a_new_task_reserves_the_id() {
6755 let dir = tempfile::tempdir().unwrap();
6756 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6757 let mut first = task("first");
6758 q.put(&mut first).unwrap();
6759 let src = source_file(dir.path(), "shot.png", "x");
6760 let mut second = task("second");
6761 q.attach(&mut second, &[src]).unwrap();
6762 assert_ne!(second.id, first.id);
6763 assert!(q.attachments_dir(&second.id).is_dir());
6764 q.put(&mut second).unwrap();
6765 assert_eq!(q.get(&second.id).unwrap().attachments.len(), 1);
6766 assert_eq!(q.get(&first.id).unwrap().title, "first");
6767 assert!(q.get(&first.id).unwrap().attachments.is_empty());
6768 }
6769
6770 #[test]
6771 fn create_new_keeps_its_already_exists_meaning() {
6772 let dir = tempfile::tempdir().unwrap();
6773 let q = Queue::at(dir.path().join("queue"));
6774 let mut a = task("a");
6775 a.id = "fixed-id".into();
6776 assert!(q.create_new(&mut a).unwrap());
6777 let mut b = task("b");
6778 b.id = "fixed-id".into();
6779 assert!(!q.create_new(&mut b).unwrap());
6780 assert_eq!(q.get("fixed-id").unwrap().title, "a");
6781 }
6782
6783 #[test]
6784 fn filing_gives_up_when_no_id_is_ever_free() {
6785 let dir = tempfile::tempdir().unwrap();
6786 let q = Queue::at(dir.path().join("queue")).with_id_source(|| TAKEN.to_owned());
6787 let mut first = task("first");
6788 q.put(&mut first).unwrap();
6789 let mut second = task("second");
6790 assert!(q.put(&mut second).is_err());
6791 assert_eq!(q.get(&first.id).unwrap().title, "first");
6792 }
6793}