1use std::collections::HashMap;
34use std::num::NonZeroUsize;
35use std::path::{Path, PathBuf};
36
37use anyhow::{Context, Result, bail};
38use jiff::Timestamp;
39use serde::{Deserialize, Serialize};
40
41use crate::ask::Questions;
42use crate::id_reserve::{IdSource, path_exists};
43
44pub const SCHEMA: u32 = 21;
136
137pub(crate) fn dispatch_order(a: &Task, b: &Task) -> std::cmp::Ordering {
143 match (a.upgrade_resume_key(), b.upgrade_resume_key()) {
144 (Some(a), Some(b)) => a.cmp(&b),
145 (Some(_), None) => std::cmp::Ordering::Less,
146 (None, Some(_)) => std::cmp::Ordering::Greater,
147 (None, None) => b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)),
148 }
149}
150
151#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
153#[serde(rename_all = "lowercase")]
154pub enum HoldSource {
155 Manual,
157 Machine,
159}
160
161impl HoldSource {
162 pub fn label(self) -> &'static str {
164 match self {
165 Self::Manual => "manual",
166 Self::Machine => "machine",
167 }
168 }
169}
170
171#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
174#[serde(tag = "kind", rename_all = "lowercase")]
175pub enum Source {
176 Human,
178 Agent {
181 run: String,
183 node: String,
185 },
186 Issue {
188 number: u64,
190 repo: String,
192 },
193}
194
195pub const CHAT_NODE: &str = "chat";
198
199impl Source {
200 pub fn label(&self) -> String {
202 match self {
203 Self::Human => "human".to_owned(),
204 Self::Agent { run, node } => format!("{node}@{}", short(run)),
205 Self::Issue { number, .. } => format!("issue #{number}"),
206 }
207 }
208}
209
210#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
212#[serde(rename_all = "lowercase")]
213pub enum TaskStatus {
214 Queued,
216 Running,
218 Done,
220 Failed,
222 Held,
224 Blocked,
228 Parked,
233}
234
235impl TaskStatus {
236 pub fn runnable(self) -> bool {
238 matches!(self, Self::Queued | Self::Failed | Self::Parked)
239 }
240
241 pub fn as_str(self) -> &'static str {
243 match self {
244 Self::Queued => "queued",
245 Self::Running => "running",
246 Self::Done => "done",
247 Self::Failed => "failed",
248 Self::Held => "held",
249 Self::Blocked => "blocked",
250 Self::Parked => "parked",
251 }
252 }
253}
254
255#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
258pub struct TaskCounts {
259 pub queued: usize,
261 pub running: usize,
263 pub done: usize,
265 pub failed: usize,
267 pub held: usize,
269 pub blocked: usize,
271 pub parked: usize,
273}
274
275impl TaskCounts {
276 pub fn of(tasks: &[Task]) -> Self {
280 Self::of_statuses(tasks.iter().map(|task| task.status))
281 }
282
283 pub(crate) fn of_statuses(statuses: impl IntoIterator<Item = TaskStatus>) -> Self {
285 let mut counts = Self::default();
286 for status in statuses {
287 match status {
288 TaskStatus::Queued => counts.queued += 1,
289 TaskStatus::Running => counts.running += 1,
290 TaskStatus::Done => counts.done += 1,
291 TaskStatus::Failed => counts.failed += 1,
292 TaskStatus::Held => counts.held += 1,
293 TaskStatus::Blocked => counts.blocked += 1,
294 TaskStatus::Parked => counts.parked += 1,
295 }
296 }
297 counts
298 }
299}
300
301#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
303pub struct Reassignment {
304 pub from: PathBuf,
306 pub to: PathBuf,
308 pub actor: String,
310 pub at: Timestamp,
312 pub run_boundary: usize,
314}
315
316#[derive(Debug, Clone, Serialize, Deserialize)]
318#[serde(deny_unknown_fields)]
319pub struct Task {
320 pub schema: u32,
322 pub id: String,
324 pub title: String,
326 pub instruction: String,
328 #[serde(default, skip_serializing_if = "crate::refs::Sources::is_empty")]
330 pub sources: crate::refs::Sources,
331 pub repo: PathBuf,
333 #[serde(default)]
335 pub reassignments: Vec<Reassignment>,
336 pub source: Source,
338 #[serde(default)]
340 pub chat: Option<String>,
341 #[serde(default)]
343 pub priority: i32,
344 #[serde(default)]
355 pub solo: bool,
356 pub status: TaskStatus,
358 #[serde(default)]
360 pub attempts: usize,
361 #[serde(default)]
363 pub attempt_limit: Option<NonZeroUsize>,
364 #[serde(default)]
367 pub runs: Vec<String>,
368 #[serde(default)]
370 pub startup_failures: Vec<crate::run::StartupFailure>,
371 #[serde(default)]
373 pub last_error: Option<String>,
374 #[serde(default)]
387 pub hold_reason: Option<String>,
388 #[serde(default)]
391 pub hold_source: Option<HoldSource>,
392 #[serde(default)]
404 pub diagnostic: Option<String>,
405 #[serde(default)]
415 pub blocked_by: Vec<String>,
416 #[serde(default)]
419 pub block_reason: Option<String>,
420 #[serde(default)]
433 pub blocked_from: Option<TaskStatus>,
434 #[serde(default)]
436 pub blocked_at: Option<Timestamp>,
437 #[serde(default)]
448 pub answers: Vec<AnsweredQuestion>,
449 #[serde(default)]
455 pub triage_applied: Vec<String>,
456 #[serde(default)]
462 pub actions_applied: Vec<String>,
463 #[serde(default)]
468 pub resume_override: Option<OperatorResume>,
469 #[serde(default)]
471 pub repair_grants: Vec<crate::repair_budget::Receipt>,
472 #[serde(default)]
475 pub resumption: Option<Resumption>,
476 #[serde(default)]
478 pub resumption_history: Vec<Resumption>,
479 #[serde(default)]
481 pub operator_decided_at: Option<Timestamp>,
482 #[serde(default)]
494 pub review_branch: Option<String>,
495 #[serde(default)]
497 pub review_instruction: Option<String>,
498 #[serde(default)]
500 pub pr_recovery: Option<crate::pr_recovery::Selection>,
501 #[serde(default)]
504 pub fresh_start: bool,
505 #[serde(default)]
507 pub replacement: Option<crate::merge_decision::Replacement>,
508 #[serde(default)]
519 pub interrupt: bool,
520 #[serde(default)]
543 pub urgent: bool,
544 #[serde(skip)]
549 pub(crate) unfiled: bool,
550 #[serde(default)]
556 pub attachments: Vec<String>,
557 #[serde(default)]
561 pub followup: Option<FollowUp>,
562 #[serde(default)]
568 pub origin_chat: Option<String>,
569 #[serde(default)]
576 pub overrides: Option<RunOverrides>,
577 #[serde(default)]
584 pub review_of: Option<String>,
585 #[serde(default)]
590 pub held_at: Option<Timestamp>,
591 #[serde(default)]
595 pub park_reason: Option<String>,
596 #[serde(default)]
599 pub chat_notice: Option<ChatNotice>,
600 #[serde(default)]
602 pub chat_notices: Vec<String>,
603 #[serde(default)]
606 pub hold_notice_cause: Option<NoticeCause>,
607 #[serde(default)]
612 pub passes: Vec<TaskPass>,
613 #[serde(default)]
617 pub dispatched_at: Option<Timestamp>,
618 #[serde(default)]
620 pub recovery_budget: Option<RecoveryBudget>,
621 #[serde(default)]
624 pub recovery_admission: Option<RecoveryAdmission>,
625 #[serde(default)]
628 pub recovery_generations: Vec<RecoveryBudget>,
629 pub created_at: Timestamp,
631 pub updated_at: Timestamp,
633}
634
635#[derive(Debug, Clone, Serialize, Deserialize)]
637pub struct RecoveryAdmission {
638 pub base: String,
640 pub inputs: String,
642}
643
644#[derive(Debug, Clone, Serialize, Deserialize)]
647pub struct RecoveryBudget {
648 pub instruction: String,
650 pub base: String,
652 pub inputs: String,
654 pub stage: String,
656 pub failures: usize,
658 #[serde(default)]
661 pub progress: Option<String>,
662}
663
664fn recovery_progress(state: &crate::run::RunState) -> Option<String> {
665 use crate::proc::Quiet as _;
666 use std::hash::{Hash, Hasher};
667 let mut patches = Vec::new();
668 for candidate in state.candidates.iter().filter(|c| c.commits > 0) {
669 let output = std::process::Command::new("git")
670 .current_dir(&state.repo)
671 .args(["diff", "--binary", &state.base_commit, &candidate.branch])
672 .quiet()
673 .output()
674 .ok()?;
675 if !output.status.success() {
676 return None;
677 }
678 patches.push(output.stdout);
679 }
680 patches.sort();
681 patches.dedup();
682 let mut hash = std::collections::hash_map::DefaultHasher::new();
683 patches.hash(&mut hash);
684 Some(format!("{:x}", hash.finish()))
685}
686
687pub(crate) fn recovery_inputs(sources: &crate::refs::Sources) -> String {
691 serde_json::to_string(
692 &sources
693 .start_from
694 .iter()
695 .map(|i| ("start", &i.token, &i.sha))
696 .chain(
697 sources
698 .pick_commits
699 .iter()
700 .map(|i| ("pick", &i.token, &i.sha)),
701 )
702 .collect::<Vec<_>>(),
703 )
704 .unwrap_or_default()
705}
706
707#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
709pub struct NoticeCause {
710 pub stage: String,
712 #[serde(default, skip_serializing_if = "Option::is_none")]
715 pub target: Option<String>,
716 pub run: Option<String>,
718 pub pass: Option<usize>,
720}
721
722pub(crate) fn is_done_notice_key(key: &str) -> bool {
724 key == "done"
725 || key
726 .strip_prefix("done#")
727 .is_some_and(|count| !count.is_empty() && count.bytes().all(|b| b.is_ascii_digit()))
728}
729
730#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
732#[serde(rename_all = "snake_case")]
733pub enum PassCost {
734 Spent,
736 Refunded,
738}
739
740#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
742pub struct TaskPass {
743 pub run: String,
745 pub pass_index: usize,
747 pub cost: PassCost,
749 pub attempts_after: usize,
751 #[serde(default)]
753 pub started_at: Option<Timestamp>,
754 pub settled_at: Timestamp,
756 #[serde(default)]
758 pub parked: Option<crate::run::ParkCause>,
759}
760
761#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
763#[serde(rename_all = "lowercase")]
764pub enum ChatNoticeOutcome {
765 Posted,
767 Skipped,
769}
770
771#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
773pub struct ChatNotice {
774 pub key: String,
776 pub at: Timestamp,
778 pub outcome: ChatNoticeOutcome,
780}
781
782#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
784pub struct RunOverrides {
785 #[serde(default)]
788 pub inherit_daemon_options: bool,
789 #[serde(default)]
791 pub merge: Option<String>,
792 #[serde(default)]
794 pub candidates: Option<usize>,
795 #[serde(default)]
797 pub judges: Option<usize>,
798 #[serde(default)]
800 pub reviewers: Option<usize>,
801 #[serde(default)]
803 pub review_rounds: Option<usize>,
804 #[serde(default)]
806 pub seed: Option<u64>,
807 #[serde(default)]
809 pub config: Option<PathBuf>,
810}
811
812impl RunOverrides {
813 pub fn merge_over(&mut self, other: &RunOverrides) {
815 macro_rules! take {
816 ($($f:ident),*) => {$(
817 if other.$f.is_some() {
818 self.$f.clone_from(&other.$f);
819 }
820 )*};
821 }
822 take!(
823 merge,
824 candidates,
825 judges,
826 reviewers,
827 review_rounds,
828 seed,
829 config
830 );
831 }
832
833 pub fn apply(&self, config: &mut crate::config::Config) {
835 if let Some(n) = self.candidates {
836 config.graph.implementers = n;
837 }
838 if let Some(n) = self.judges {
839 config.graph.judges = n;
840 }
841 if let Some(n) = self.reviewers {
842 config.graph.reviewers = n;
843 }
844 if let Some(n) = self.review_rounds {
845 config.graph.review_rounds = n;
846 }
847 if let Some(m) = &self.merge
848 && let Ok(mode) = crate::daemon::merge_mode(m)
849 {
850 config.merge.mode = mode;
851 }
852 if let Some(s) = self.seed {
853 config.blind.seed = Some(s);
854 }
855 }
856}
857
858#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
860pub struct FollowUp {
861 pub run: String,
863 #[serde(default)]
865 pub origin_task: Option<String>,
866 pub pr: String,
868 pub findings: Vec<String>,
870 pub generation: u32,
873}
874
875#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
877pub struct Resumption {
878 pub generation: u64,
880 pub source: String,
882 pub question_id: Option<String>,
884 pub at: Timestamp,
886 pub instruction: String,
888 pub answers_through: usize,
890 pub runs_through: usize,
893 pub cleared_reason: Option<String>,
895 #[serde(default)]
897 pub blocking_evidence: Vec<String>,
898}
899
900#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
903pub struct OperatorResume {
904 pub question_id: String,
906 pub at: Timestamp,
908 #[serde(default)]
911 pub conductor_rehold: Option<String>,
912 #[serde(default)]
915 pub forced: bool,
916 #[serde(default)]
923 pub pinned_run: Option<String>,
924}
925
926#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
929pub struct AnsweredQuestion {
930 pub question: String,
932 pub answer: String,
934}
935
936impl Task {
937 pub fn new(title: String, instruction: String, repo: PathBuf, source: Source) -> Self {
939 let now = Timestamp::now();
940 let origin_chat = match &source {
941 Source::Agent { run, node } if node == CHAT_NODE => Some(run.clone()),
942 _ => None,
943 };
944 Self {
945 schema: SCHEMA,
946 id: new_id(),
947 title,
948 instruction,
949 sources: crate::refs::Sources::default(),
950 repo,
951 reassignments: Vec::new(),
952 source,
953 chat: None,
954 priority: 0,
955 solo: false,
956 status: TaskStatus::Queued,
957 attempts: 0,
958 attempt_limit: None,
959 runs: Vec::new(),
960 startup_failures: Vec::new(),
961 last_error: None,
962 hold_reason: None,
963 hold_source: None,
964 diagnostic: None,
965 blocked_by: Vec::new(),
966 block_reason: None,
967 blocked_from: None,
968 blocked_at: None,
969 answers: Vec::new(),
970 triage_applied: Vec::new(),
971 actions_applied: Vec::new(),
972 resume_override: None,
973 repair_grants: Vec::new(),
974 resumption: None,
975 resumption_history: Vec::new(),
976 operator_decided_at: None,
977 review_branch: None,
978 review_instruction: None,
979 pr_recovery: None,
980 fresh_start: false,
981 replacement: None,
982 interrupt: false,
983 urgent: false,
984 unfiled: true,
985 attachments: Vec::new(),
986 followup: None,
987 origin_chat,
988 overrides: None,
989 review_of: None,
990 held_at: None,
991 park_reason: None,
992 chat_notice: None,
993 chat_notices: Vec::new(),
994 hold_notice_cause: None,
995 passes: Vec::new(),
996 dispatched_at: None,
997 recovery_budget: None,
998 recovery_admission: None,
999 recovery_generations: Vec::new(),
1000 created_at: now,
1001 updated_at: now,
1002 }
1003 }
1004
1005 pub(crate) fn reassign_repository(&mut self, repo: PathBuf, actor: &str) {
1007 self.stamp_schema();
1008 if matches!(self.status, TaskStatus::Failed | TaskStatus::Parked) {
1009 self.hold_manual(Some(
1010 "Repository reassigned; inspect history and release to start fresh.".into(),
1011 ));
1012 }
1013 let at = Timestamp::now();
1014 self.reassignments.push(Reassignment {
1015 from: self.repo.clone(),
1016 to: repo.clone(),
1017 actor: actor.to_owned(),
1018 at,
1019 run_boundary: self.runs.len(),
1020 });
1021 self.repo = repo;
1022 self.review_branch = None;
1023 self.review_instruction = None;
1024 self.pr_recovery = None;
1025 self.replacement = None;
1026 self.interrupt = false;
1027 self.resume_override = None;
1028 self.fresh_start = true;
1029 self.operator_decided_at = Some(at);
1030 }
1031
1032 pub fn publication_branch(&self) -> String {
1034 crate::run::task_pr_branch_name(&self.id, self.reassignments.len())
1035 }
1036
1037 pub fn execution_runs(&self) -> &[String] {
1039 let boundary = self.reassignments.last().map_or(0, |r| r.run_boundary);
1040 self.runs.get(boundary..).unwrap_or_default()
1041 }
1042
1043 pub fn execution_run(&self, run: &str) -> bool {
1045 self.execution_runs().iter().any(|id| id == run)
1046 }
1047
1048 pub(crate) fn assignment_accepts(&self, run: &crate::run::RunState) -> bool {
1050 self.reassignments.last().is_none_or(|boundary| {
1051 self.execution_run(&run.id)
1052 && run.task_assignment == self.reassignments.len()
1053 && run.created_at > boundary.at
1054 && run
1055 .repo
1056 .canonicalize()
1057 .ok()
1058 .zip(self.repo.canonicalize().ok())
1059 .is_some_and(|(run_repo, task_repo)| run_repo == task_repo)
1060 })
1061 }
1062
1063 pub(crate) fn stale_question(&self, q: &crate::ask::Question) -> bool {
1065 self.reassignments
1066 .last()
1067 .is_some_and(|r| q.asked_at <= r.at)
1068 || (self.runs.contains(&q.run) && !self.execution_run(&q.run))
1069 }
1070
1071 pub fn chat_talk(&self) -> Option<&str> {
1073 if let Some(id) = &self.origin_chat {
1074 return Some(id);
1075 }
1076 match &self.source {
1077 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
1078 _ => None,
1079 }
1080 }
1081
1082 pub fn filed_by_chat(&self) -> Option<&str> {
1086 match &self.source {
1087 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
1088 _ => None,
1089 }
1090 }
1091
1092 pub fn chat_notice_key(&self) -> Option<String> {
1095 match self.status {
1096 TaskStatus::Done => Some("done".to_owned()),
1097 TaskStatus::Held => {
1098 let cause = self
1099 .hold_notice_cause
1100 .clone()
1101 .unwrap_or_else(|| self.unresolved_hold_cause());
1102 Some(format!(
1103 "cause:{}",
1104 serde_json::to_string(&cause).expect("cause")
1105 ))
1106 }
1107 TaskStatus::Blocked => {
1108 let ids: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1109 let evidence = self
1112 .hold_notice_cause
1113 .as_ref()
1114 .filter(|c| c.stage.starts_with("conductor-evidence:"));
1115 Some(format!(
1116 "dependencies:{}",
1117 serde_json::to_string(&(ids, evidence)).expect("ids")
1118 ))
1119 }
1120 _ => None,
1121 }
1122 }
1123
1124 fn unresolved_hold_cause(&self) -> NoticeCause {
1125 NoticeCause {
1126 stage: "hold".into(),
1127 target: None,
1128 run: self.runs.last().cloned(),
1129 pass: None,
1130 }
1131 }
1132
1133 pub(crate) fn legacy_chat_notice_key(&self) -> Option<String> {
1135 let key = match self.status {
1136 TaskStatus::Held if self.hold_notice_cause.is_none() => Some(format!(
1137 "held#{}",
1138 self.held_at.map(|t| t.to_string()).unwrap_or_default()
1139 )),
1140 TaskStatus::Blocked => Some(format!(
1141 "blocked#{}#{}",
1142 self.blocked_by.join(","),
1143 self.block_reason.as_deref().unwrap_or_default()
1144 )),
1145 _ => None,
1146 };
1147 key.filter(|key| self.legacy_notice_matches(key))
1148 }
1149
1150 pub(crate) fn legacy_notice_matches(&self, key: &str) -> bool {
1151 match self.status {
1152 TaskStatus::Held if self.hold_notice_cause.is_none() => key
1153 .strip_prefix("held#")
1154 .is_some_and(|time| time.is_empty() || time.parse::<Timestamp>().is_ok()),
1155 TaskStatus::Blocked
1156 if self
1157 .hold_notice_cause
1158 .as_ref()
1159 .is_none_or(|c| !c.stage.starts_with("conductor-evidence:")) =>
1160 {
1161 key.strip_prefix("blocked#").is_some_and(|tail| {
1162 let Some((ids, _)) = tail.split_once('#') else {
1163 return false;
1164 };
1165 let old: std::collections::BTreeSet<_> =
1166 ids.split(',').filter(|id| !id.is_empty()).collect();
1167 let current: std::collections::BTreeSet<_> =
1168 self.blocked_by.iter().map(String::as_str).collect();
1169 old == current
1170 })
1171 }
1172 _ => false,
1173 }
1174 }
1175
1176 fn import_chat_notice(&mut self) {
1177 if let Some(n) = &self.chat_notice {
1178 let key = if is_done_notice_key(&n.key) {
1179 "done".to_owned()
1180 } else if self.legacy_notice_matches(&n.key) {
1181 self.chat_notice_key().unwrap_or_else(|| n.key.clone())
1182 } else {
1183 n.key.clone()
1184 };
1185 if !self.chat_notices.contains(&key) {
1186 self.chat_notices.push(key);
1187 }
1188 }
1189 }
1190
1191 pub(crate) fn notice_cause(&mut self, stage: &str, run: Option<&str>, pass: Option<usize>) {
1194 self.import_chat_notice();
1195 self.hold_notice_cause = Some(NoticeCause {
1196 stage: stage.to_owned(),
1197 target: None,
1198 run: run.map(str::to_owned),
1199 pass,
1200 });
1201 }
1202
1203 pub(crate) fn startup_notice_cause(&mut self, stage: &str) {
1206 let recovery_only = matches!(
1207 stage,
1208 "startup-source" | "startup-candidates" | "startup-public-head"
1209 );
1210 match self.hold_notice_cause.as_ref() {
1211 Some(cause) if recovery_only && !cause.stage.starts_with("startup-") => return,
1212 None if recovery_only
1213 && self
1214 .chat_notice
1215 .as_ref()
1216 .is_some_and(|n| !is_done_notice_key(&n.key)) =>
1217 {
1218 return;
1219 }
1220 _ => {}
1221 }
1222 self.notice_cause(stage, None, None);
1223 }
1224
1225 pub(crate) fn fail_startup(
1228 &mut self,
1229 stage: &str,
1230 why: impl Into<String>,
1231 max_attempts: usize,
1232 ) {
1233 let unresolved = self.hold_notice_cause.clone();
1234 self.fail(why, max_attempts);
1235 self.hold_notice_cause = unresolved;
1236 self.startup_notice_cause(stage);
1237 }
1238
1239 pub(crate) fn fail_startup_target(
1241 &mut self,
1242 stage: &str,
1243 target: String,
1244 why: impl Into<String>,
1245 max_attempts: usize,
1246 ) {
1247 self.fail_startup(stage, why, max_attempts);
1248 if let Some(cause) = self.hold_notice_cause.as_mut() {
1249 cause.target = Some(target);
1250 }
1251 }
1252
1253 pub fn chat_notice_due(&self) -> Option<String> {
1255 self.filed_by_chat()?;
1256 if self.schema < 13 {
1257 return None;
1258 }
1259 let key = self.chat_notice_key()?;
1260 if self.chat_notices.contains(&key) {
1261 return None;
1262 }
1263 match &self.chat_notice {
1264 Some(n)
1265 if n.key == key
1266 || (key == "done" && is_done_notice_key(&n.key))
1267 || self.legacy_notice_matches(&n.key) =>
1268 {
1269 None
1270 }
1271 _ => Some(key),
1272 }
1273 }
1274
1275 pub fn short(&self) -> &str {
1277 short(&self.id)
1278 }
1279
1280 pub fn mark_triage_applied(&mut self, question_id: &str) {
1282 if !self.triage_applied(question_id) {
1283 self.triage_applied.push(question_id.to_owned());
1284 }
1285 }
1286
1287 pub fn action_applied(&self, question_id: &str) -> bool {
1290 self.actions_applied.iter().any(|id| id == question_id)
1291 }
1292
1293 pub fn mark_action_applied(&mut self, question_id: &str) {
1295 if !self.action_applied(question_id) {
1296 self.actions_applied.push(question_id.to_owned());
1297 }
1298 }
1299
1300 pub fn triage_applied(&self, question_id: &str) -> bool {
1302 self.triage_applied.iter().any(|id| id == question_id)
1303 }
1304
1305 pub fn start(&mut self, run: String) {
1318 self.stamp_schema();
1319 self.status = TaskStatus::Running;
1320 self.attempts += 1;
1321 self.runs.retain(|id| id != &run);
1322 self.runs.push(run);
1323 self.last_error = None;
1324 self.park_reason = None;
1325 self.fresh_start = false;
1326 self.interrupt = false;
1327 self.resume_override = None;
1329 }
1330
1331 pub fn record_startup_failure(&mut self, failure: crate::run::StartupFailure) {
1333 if !self.startup_failures.iter().any(|f| f.run == failure.run) {
1334 self.startup_failures.push(failure);
1335 }
1336 }
1337
1338 pub fn link_run(&mut self, run: &str) -> bool {
1343 if self.runs.iter().any(|r| r == run) {
1344 return false;
1345 }
1346 self.runs.push(run.to_owned());
1347 true
1348 }
1349
1350 pub fn succeed(&mut self) {
1360 self.stamp_schema();
1361 self.status = TaskStatus::Done;
1362 self.resume_override = None;
1363 self.last_error = None;
1364 self.park_reason = None;
1365 self.hold_reason = None;
1366 self.hold_source = None;
1367 self.diagnostic = None;
1368 self.blocked_by.clear();
1369 self.block_reason = None;
1370 self.blocked_from = None;
1371 }
1372
1373 pub fn already_landed(&mut self, note: impl Into<String>) {
1381 self.succeed();
1382 self.attempts = self.attempts.saturating_sub(1);
1383 self.last_error = Some(note.into());
1384 }
1385
1386 pub fn superseded_attempts(&self, last_run_succeeded: bool) -> &[String] {
1416 if self.status != TaskStatus::Done || !last_run_succeeded || self.execution_runs().len() < 2
1417 {
1418 return &[];
1419 }
1420 &self.execution_runs()[..self.execution_runs().len() - 1]
1421 }
1422
1423 pub fn successor_of(&self, run: &str) -> Option<&String> {
1431 let runs = self.execution_runs();
1432 let pos = runs.iter().rposition(|r| r == run)?;
1433 runs.get(pos + 1)
1434 }
1435
1436 pub fn earlier_attempts(&self) -> &[String] {
1443 self.execution_runs()
1444 }
1445
1446 fn stamp_schema(&mut self) {
1453 self.schema = self.schema.max(SCHEMA);
1454 }
1455
1456 fn note_held(&mut self) {
1457 self.stamp_schema();
1458 let continuing = self.status == TaskStatus::Held
1459 || (self.status == TaskStatus::Blocked
1460 && (self.blocked_from == Some(TaskStatus::Held)
1461 || (self.blocked_from.is_none()
1462 && (self.hold_reason.is_some() || self.hold_source.is_some()))));
1463 if !continuing || self.held_at.is_none() {
1464 self.held_at = Some(Timestamp::now());
1465 }
1466 }
1467
1468 pub fn fail(&mut self, why: impl Into<String>, max_attempts: usize) {
1484 let why = why.into();
1485 if self.status == TaskStatus::Running {
1486 let run = self.runs.last().cloned();
1487 self.notice_cause("run-failed", run.as_deref(), Some(self.attempts));
1488 }
1489 self.diagnostic = None;
1490 self.park_reason = None;
1491 self.status = if self.attempts >= self.effective_attempt_limit(max_attempts) {
1492 self.note_held();
1493 self.hold_source = Some(HoldSource::Machine);
1494 self.hold_reason = Some(why.clone());
1495 TaskStatus::Held
1496 } else {
1497 TaskStatus::Failed
1498 };
1499 self.last_error = Some(why);
1500 }
1501
1502 pub fn stall(&mut self, why: impl Into<String>) {
1512 self.last_error = Some(why.into());
1513 self.diagnostic = None;
1514 self.attempts = self.attempts.saturating_sub(1);
1515 self.status = TaskStatus::Failed;
1516 }
1517
1518 pub fn park(&mut self, why: impl Into<String>) {
1524 self.diagnostic = None;
1525 self.attempts = self.attempts.saturating_sub(1);
1526 self.status = TaskStatus::Parked;
1527 self.park_reason = Some(why.into());
1528 self.schema = self.schema.max(SCHEMA);
1531 }
1532
1533 pub fn record_pass(
1535 &mut self,
1536 run: &str,
1537 pass_index: usize,
1538 attempts_before: usize,
1539 parked: Option<crate::run::ParkCause>,
1540 ) {
1541 self.record_pass_at(run, pass_index, None, attempts_before, parked);
1542 }
1543
1544 pub fn record_pass_at(
1547 &mut self,
1548 run: &str,
1549 pass_index: usize,
1550 started_at: Option<Timestamp>,
1551 attempts_before: usize,
1552 parked: Option<crate::run::ParkCause>,
1553 ) {
1554 if self
1555 .passes
1556 .iter()
1557 .any(|p| p.run == run && p.pass_index == pass_index)
1558 {
1559 return;
1560 }
1561 let cost = if self.attempts < attempts_before {
1562 PassCost::Refunded
1563 } else {
1564 PassCost::Spent
1565 };
1566 self.passes.push(TaskPass {
1567 run: run.to_owned(),
1568 pass_index,
1569 started_at: self.dispatched_at.or(started_at),
1570 cost,
1571 attempts_after: self.attempts,
1572 settled_at: Timestamp::now(),
1573 parked,
1574 });
1575 self.schema = self.schema.max(SCHEMA);
1576 }
1577
1578 pub(crate) fn upgrade_resume_key(&self) -> Option<(Timestamp, &str)> {
1581 if self.status != TaskStatus::Parked || self.fresh_start {
1582 return None;
1583 }
1584 let pass = self.passes.last()?;
1585 if self.execution_runs().last() != Some(&pass.run)
1586 || !matches!(pass.parked, Some(crate::run::ParkCause::Upgrade { .. }))
1587 {
1588 return None;
1589 }
1590 Some((pass.started_at.unwrap_or(pass.settled_at), &self.id))
1593 }
1594
1595 pub fn operator_held(&self) -> bool {
1601 self.status == TaskStatus::Held && !matches!(self.hold_source, Some(HoldSource::Machine))
1602 }
1603
1604 pub fn hold_manual(&mut self, reason: Option<String>) {
1614 self.operator_decided_at = Some(Timestamp::now());
1616 self.hold_unrequested_retry(reason);
1617 }
1618
1619 pub(crate) fn hold_unrequested_retry(&mut self, reason: Option<String>) {
1622 self.park_reason = None;
1623 self.note_held();
1624 self.status = TaskStatus::Held;
1625 if reason.is_some() {
1626 self.hold_reason = reason;
1627 }
1628 self.hold_source = Some(HoldSource::Manual);
1629 self.blocked_by.clear();
1630 self.block_reason = None;
1631 self.blocked_from = None;
1632 }
1633
1634 pub fn hold_machine(&mut self, reason: Option<String>) {
1639 self.park_reason = None;
1640 self.note_held();
1641 self.status = TaskStatus::Held;
1642 if reason.is_some() {
1643 self.hold_reason = reason;
1644 }
1645 self.hold_source = Some(HoldSource::Machine);
1646 self.blocked_by.clear();
1647 self.block_reason = None;
1648 self.blocked_from = None;
1649 }
1650
1651 pub fn block(&mut self, blocked_by: Vec<String>, reason: Option<String>) {
1661 self.stamp_schema();
1662 let before: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1663 let after: std::collections::BTreeSet<_> = blocked_by.iter().collect();
1664 if self.status != TaskStatus::Blocked || before != after {
1665 self.blocked_at = Some(Timestamp::now());
1666 }
1667 if self.status != TaskStatus::Blocked {
1668 self.blocked_from = Some(self.status);
1669 }
1670 self.status = TaskStatus::Blocked;
1671 self.blocked_by = blocked_by;
1672 self.block_reason = reason;
1673 }
1674
1675 pub fn unblock(&mut self, resolved_id: &str) {
1698 if self.status != TaskStatus::Blocked {
1699 return;
1700 }
1701 self.blocked_by.retain(|id| id != resolved_id);
1702 self.restore_if_unblocked();
1703 }
1704
1705 pub fn dependency_deleted(&mut self, deleted_id: &str) -> bool {
1716 if self.status != TaskStatus::Blocked || !self.blocked_by.iter().any(|b| b == deleted_id) {
1717 return false;
1718 }
1719 self.blocked_by.retain(|id| id != deleted_id);
1720 self.restore_if_unblocked();
1721 true
1722 }
1723
1724 fn restore_if_unblocked(&mut self) {
1728 if self.blocked_by.is_empty() {
1729 self.status = match self.blocked_from {
1730 Some(TaskStatus::Running) => TaskStatus::Queued,
1731 Some(other) => other,
1732 None if self.hold_reason.is_some() || self.hold_source.is_some() => {
1733 TaskStatus::Held
1734 }
1735 None => TaskStatus::Queued,
1736 };
1737 self.block_reason = None;
1738 self.blocked_from = None;
1739 }
1740 }
1741
1742 pub fn record_answer(&mut self, question: String, answer: String) {
1747 self.answers.push(AnsweredQuestion { question, answer });
1748 }
1749
1750 pub fn request_review(&mut self, branch: String) {
1754 self.release();
1755 self.review_branch = Some(branch);
1756 self.review_instruction = Some(self.instruction.clone());
1757 }
1758
1759 pub fn requeue(&mut self) {
1762 self.release();
1763 self.review_branch = None;
1765 self.review_instruction = None;
1766 self.fresh_start = true;
1767 }
1768
1769 pub fn hold_for_handover(&mut self, branch: Option<String>, reason: String) {
1778 if branch.is_some() {
1779 self.review_branch = branch;
1780 self.review_instruction = Some(self.instruction.clone());
1781 }
1782 self.hold_machine(Some(reason));
1783 }
1784
1785 pub fn set_urgent(&mut self, urgent: bool) -> Result<()> {
1787 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1788 bail!(
1789 "task {} is {}; urgent can only change while queued or held; \
1790 wait for the active attempt to finish and requeue or hold the task first",
1791 self.short(),
1792 self.status.as_str()
1793 );
1794 }
1795 self.urgent = urgent;
1796 Ok(())
1797 }
1798
1799 pub fn set_priority(&mut self, priority: i32) -> Result<()> {
1808 if self.status == TaskStatus::Running {
1809 bail!(
1810 "task {} is running; its priority cannot be changed until \
1811 this attempt finishes",
1812 self.short()
1813 );
1814 }
1815 self.priority = priority;
1816 Ok(())
1817 }
1818
1819 pub fn set_interrupt(&mut self, interrupt: bool) -> Result<()> {
1834 if interrupt && !self.status.runnable() {
1835 bail!(
1836 "task {} is {}; only a queued or failed task can be marked \
1837 to interrupt",
1838 self.short(),
1839 self.status.as_str()
1840 );
1841 }
1842 self.interrupt = interrupt;
1843 Ok(())
1844 }
1845
1846 pub fn effective_attempt_limit(&self, daemon_limit: usize) -> usize {
1848 self.attempt_limit.map_or(daemon_limit, NonZeroUsize::get)
1849 }
1850
1851 pub fn change_attempt_limit(&mut self, limit: Option<NonZeroUsize>) -> Result<()> {
1854 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1855 bail!("only queued or held tasks can change the attempt limit");
1856 }
1857 self.stamp_schema();
1858 self.attempt_limit = limit;
1859 Ok(())
1860 }
1861
1862 pub fn implementers(&self) -> Option<usize> {
1864 if self.solo {
1865 Some(1)
1866 } else {
1867 self.overrides.as_ref().and_then(|o| o.candidates)
1868 }
1869 }
1870
1871 pub fn change_implementers(&mut self, count: u8) -> Result<()> {
1874 self.check_implementers_editable()?;
1875 let explicit = self.overrides.as_ref().and_then(|o| o.config.as_deref());
1876 let (mut config, _) = crate::config::Config::discover(&self.repo, explicit)?;
1877 if let Some(o) = &self.overrides {
1878 o.apply(&mut config);
1879 }
1880 self.set_implementers(count, &config)
1881 }
1882
1883 fn check_implementers_editable(&self) -> Result<()> {
1884 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1885 bail!(
1886 "task {} is {}; only queued or held tasks can change implementers",
1887 self.short(),
1888 self.status.as_str()
1889 );
1890 }
1891 Ok(())
1892 }
1893
1894 pub fn set_implementers(&mut self, count: u8, config: &crate::config::Config) -> Result<()> {
1896 self.check_implementers_editable()?;
1897 crate::talk::check_implementers(count, config).map_err(anyhow::Error::msg)?;
1898 self.stamp_schema();
1899 self.solo = count == 1;
1900 let mut overrides = self.overrides.clone().unwrap_or_else(|| RunOverrides {
1901 inherit_daemon_options: true,
1902 ..Default::default()
1903 });
1904 overrides.candidates = (count > 1).then_some(usize::from(count));
1905 let mut settings = overrides.clone();
1906 settings.inherit_daemon_options = false;
1907 self.overrides = if overrides.inherit_daemon_options && settings == RunOverrides::default()
1909 {
1910 None
1911 } else {
1912 Some(overrides)
1913 };
1914 Ok(())
1915 }
1916
1917 pub fn edit(&mut self, title: String, instruction: String) -> Result<()> {
1929 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1930 bail!(
1931 "task {} is {}; only a queued or held task's instruction can \
1932 be edited",
1933 self.short(),
1934 self.status.as_str()
1935 );
1936 }
1937 if self.review_branch.is_some() && self.review_instruction.is_none() {
1940 self.review_instruction = Some(self.instruction.clone());
1941 }
1942 if self.instruction != instruction {
1943 self.resume_override = None;
1946 }
1947 self.title = title;
1948 self.instruction = instruction;
1949 Ok(())
1950 }
1951
1952 pub fn handed_off(&mut self, why: impl Into<String>) {
1972 let why = why.into();
1973 self.diagnostic = None;
1974 self.note_held();
1975 self.status = TaskStatus::Held;
1976 self.hold_source = Some(HoldSource::Machine);
1977 self.hold_reason = Some(why.clone());
1978 self.last_error = Some(why);
1979 }
1980
1981 pub fn release_operator(&mut self, source: &str, question_id: Option<&str>) {
1983 self.recovery_budget = None;
1984 self.recovery_generations.clear();
1985 self.recovery_admission = None;
1986 if self
1987 .resumption
1988 .as_ref()
1989 .is_some_and(|r| r.instruction != self.instruction)
1990 {
1991 self.resume_override = None;
1994 }
1995 let approval = Resumption {
1996 generation: self.resumption.as_ref().map_or(1, |r| r.generation + 1),
1997 source: source.to_owned(),
1998 question_id: question_id.map(str::to_owned),
1999 at: Timestamp::now(),
2000 instruction: self.instruction.clone(),
2001 answers_through: self.answers.len(),
2002 runs_through: self.runs.len(),
2003 blocking_evidence: Vec::new(),
2004 cleared_reason: self
2005 .hold_reason
2006 .clone()
2007 .or_else(|| self.block_reason.clone()),
2008 };
2009 self.release();
2010 self.operator_decided_at = Some(approval.at);
2011 if let Some(previous) = self.resumption.replace(approval) {
2012 self.resumption_history.push(previous);
2013 }
2014 }
2015
2016 pub(crate) fn recovery_failed(&mut self, state: &crate::run::RunState) {
2018 let (base, inputs) = self.recovery_admission.as_ref().map_or_else(
2019 || (state.base_commit.clone(), recovery_inputs(&state.sources)),
2020 |a| (a.base.clone(), a.inputs.clone()),
2021 );
2022 let progress = recovery_progress(state);
2023 let stage = if state.prep_failure.is_some() {
2024 "preparation"
2025 } else {
2026 "execution"
2027 };
2028 let matches = |b: &RecoveryBudget| {
2029 b.instruction == self.instruction
2030 && b.base == base
2031 && b.inputs == inputs
2032 && (progress.is_none() || b.progress == progress)
2033 };
2034 if !self.recovery_budget.as_ref().is_some_and(matches) {
2035 let previous = self
2036 .recovery_generations
2037 .iter()
2038 .position(matches)
2039 .map(|index| self.recovery_generations.remove(index));
2040 if let Some(old) = self.recovery_budget.take() {
2041 self.recovery_generations.push(old);
2042 }
2043 self.recovery_budget = previous;
2044 }
2045 if let Some(budget) = &mut self.recovery_budget {
2046 budget.failures = budget.failures.saturating_add(1);
2047 budget.stage = stage.to_owned();
2048 } else {
2049 self.recovery_budget = Some(RecoveryBudget {
2050 instruction: self.instruction.clone(),
2051 base,
2052 inputs,
2053 stage: stage.to_owned(),
2054 failures: 1,
2055 progress,
2056 });
2057 }
2058 }
2059
2060 pub(crate) fn recovery_exhausted(
2061 &self,
2062 base: &str,
2063 inputs: &str,
2064 limit: usize,
2065 ) -> Option<String> {
2066 let same_input = |b: &&RecoveryBudget| {
2067 b.instruction == self.instruction && b.base == base && b.inputs == inputs
2068 };
2069 let budget = self
2070 .recovery_budget
2071 .as_ref()
2072 .filter(same_input)
2073 .or_else(|| self.recovery_generations.iter().rev().find(same_input))?;
2074 (budget.instruction == self.instruction && budget.base == base && budget.inputs == inputs
2075 && budget.failures >= limit.max(1)).then(|| format!(
2076 "[recovery-budget] {} failed passes against unchanged inputs ({}). Inspect the retained recovery sources and correct the cause, change the inputs, or explicitly release the task to approve another generation. Automatic requeue does not reset this budget.",
2077 budget.failures, budget.stage))
2078 }
2079
2080 pub fn release(&mut self) {
2084 self.import_chat_notice();
2087 if self.hold_notice_cause.is_none()
2088 && matches!(self.status, TaskStatus::Held | TaskStatus::Blocked)
2089 {
2090 self.hold_notice_cause = Some(self.unresolved_hold_cause());
2091 }
2092 let refused_handover = self.status == TaskStatus::Held
2093 && self.hold_source == Some(HoldSource::Machine)
2094 && self.review_branch.is_some()
2095 && self
2096 .review_instruction
2097 .as_ref()
2098 .is_none_or(|i| *i == self.instruction);
2099 self.status = TaskStatus::Queued;
2100 self.held_at = None;
2101 self.attempts = 0;
2102 self.last_error = None;
2103 self.park_reason = None;
2104 self.hold_reason = None;
2107 self.hold_source = None;
2108 self.diagnostic = None;
2109 self.blocked_by.clear();
2114 self.block_reason = None;
2115 self.blocked_from = None;
2116 if !refused_handover {
2120 self.review_branch = None;
2121 self.review_instruction = None;
2122 }
2123 self.fresh_start = false;
2124 }
2125}
2126
2127fn copy_new(dir: &Path, src: &Path, name: &str) -> Result<(String, PathBuf)> {
2131 use std::io::ErrorKind;
2132 let (stem, ext) = match name.rfind('.') {
2133 Some(i) if i > 0 => (&name[..i], &name[i..]),
2134 _ => (name, ""),
2135 };
2136 for n in 1u32.. {
2137 let candidate = if n == 1 {
2138 name.to_owned()
2139 } else {
2140 let suffix = format!("-{n}");
2141 let room = 64usize.saturating_sub(suffix.len() + ext.len());
2142 let stem: String = stem.chars().take(room).collect();
2143 format!("{stem}{suffix}{ext}")
2144 };
2145 if !crate::ask::valid_asset_name(&candidate) {
2146 bail!("no valid attachment name is left for `{name}`");
2147 }
2148 let path = dir.join(&candidate);
2149 match std::fs::OpenOptions::new()
2150 .write(true)
2151 .create_new(true)
2152 .open(&path)
2153 {
2154 Ok(mut out) => {
2155 let copied = std::fs::File::open(src)
2156 .and_then(|mut input| std::io::copy(&mut input, &mut out));
2157 if let Err(e) = copied {
2158 drop(out);
2159 let _ = std::fs::remove_file(&path);
2160 return Err(e).with_context(|| format!("copy {}", src.display()));
2161 }
2162 return Ok((candidate, path));
2163 }
2164 Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
2165 Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
2166 }
2167 }
2168 unreachable!("the counter never runs out")
2169}
2170
2171const TASK_LOCK_STALE: std::time::Duration = std::time::Duration::from_secs(10);
2174
2175pub(crate) struct TaskLock {
2178 path: PathBuf,
2179 token: String,
2180}
2181
2182fn owner_token() -> String {
2185 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
2186 let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2187 let mut r = crate::rng::SplitMix64::new(crate::rng::entropy() ^ n.rotate_left(32));
2188 format!("{}-{:016x}", std::process::id(), r.next_u64())
2189}
2190
2191fn break_marker(path: &Path, token: &str) -> PathBuf {
2194 let mut h: u64 = 0xcbf29ce484222325;
2195 for b in token.bytes() {
2196 h ^= u64::from(b);
2197 h = h.wrapping_mul(0x100000001b3);
2198 }
2199 let mut name = path.as_os_str().to_owned();
2200 name.push(format!(".break-{h:016x}"));
2201 PathBuf::from(name)
2202}
2203
2204fn older_than_stale(path: &Path) -> bool {
2205 std::fs::metadata(path)
2206 .and_then(|m| m.modified())
2207 .ok()
2208 .and_then(|t| t.elapsed().ok())
2209 .is_some_and(|age| age > TASK_LOCK_STALE)
2210}
2211
2212fn writer_may_be_gone(token: &str) -> bool {
2213 let Some(pid) = token
2214 .split_once('-')
2215 .and_then(|(pid, _)| pid.parse::<u32>().ok())
2216 else {
2217 return true;
2220 };
2221 if pid == std::process::id() {
2222 return false;
2223 }
2224 crate::proc::pid_status(pid) == Some(false)
2226}
2227
2228const MAX_MARKER_DEPTH: u8 = 3;
2232
2233fn take_marker(
2239 path: &Path,
2240 token: &str,
2241 depth: u8,
2242 retry: crate::lock_file::Retry,
2243) -> Option<(PathBuf, String)> {
2244 use std::io::Write;
2245 let marker = break_marker(path, token);
2246 for _ in 0..2 {
2247 match retry.create(&marker) {
2248 Ok(mut f) => {
2249 let mine = owner_token();
2250 if f.write_all(mine.as_bytes()).is_err() {
2251 drop(f);
2252 let _ = std::fs::remove_file(&marker);
2253 return None;
2254 }
2255 return Some((marker, mine));
2256 }
2257 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2258 let judged = std::fs::read_to_string(&marker).ok();
2259 match judged.filter(|_| older_than_stale(&marker)) {
2260 Some(judged) if depth < MAX_MARKER_DEPTH => {
2261 remove_lock_if(&marker, &judged, true, depth + 1, retry)?;
2262 }
2263 _ => return None,
2264 }
2265 }
2266 Err(_) => return None,
2267 }
2268 }
2269 None
2270}
2271
2272fn remove_lock_if(
2277 path: &Path,
2278 token: &str,
2279 require_stale: bool,
2280 depth: u8,
2281 retry: crate::lock_file::Retry,
2282) -> Option<bool> {
2283 let (marker, mine) = take_marker(path, token, depth, retry)?;
2284 let still = std::fs::read_to_string(path).is_ok_and(|c| c == token)
2285 && (!require_stale || older_than_stale(path) && writer_may_be_gone(token));
2286 if still {
2287 let _ = std::fs::remove_file(path);
2288 }
2289 if depth >= MAX_MARKER_DEPTH {
2291 if std::fs::read_to_string(&marker).is_ok_and(|c| c == mine) {
2292 let _ = std::fs::remove_file(&marker);
2293 }
2294 } else {
2295 let _ = remove_lock_if(&marker, &mine, false, depth + 1, retry);
2296 }
2297 Some(still)
2298}
2299
2300#[cfg(test)]
2303fn break_stale(path: &Path, judged: &str) -> bool {
2304 break_stale_with(path, judged, crate::lock_file::Retry::new(None))
2305}
2306
2307fn break_stale_with(path: &Path, judged: &str, retry: crate::lock_file::Retry) -> bool {
2308 remove_lock_if(path, judged, true, 0, retry).unwrap_or(false)
2309}
2310
2311impl Drop for TaskLock {
2312 fn drop(&mut self) {
2313 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
2314 let retry = crate::lock_file::Retry::new(Some(deadline));
2315 loop {
2316 match remove_lock_if(&self.path, &self.token, false, 0, retry) {
2317 Some(_) => return,
2318 None if std::time::Instant::now() > deadline => return,
2319 None => std::thread::sleep(std::time::Duration::from_millis(5)),
2320 }
2321 }
2322 }
2323}
2324
2325#[derive(Debug, Clone)]
2327pub struct Queue {
2328 root: PathBuf,
2329 ids: IdSource,
2330}
2331
2332fn validate_attachments(sources: &[PathBuf]) -> Result<Vec<(&PathBuf, &str)>> {
2333 let mut wanted = Vec::new();
2334 for src in sources {
2335 let name = src
2336 .file_name()
2337 .and_then(|n| n.to_str())
2338 .with_context(|| format!("`{}` has no usable file name", src.display()))?;
2339 if !crate::ask::valid_asset_name(name) {
2340 bail!(
2341 "attachment name `{name}` must match ^[A-Za-z0-9][A-Za-z0-9._-]{{0,63}}$ \
2342 with no `..`; rename the file and try again"
2343 );
2344 }
2345 if !src.is_file() {
2346 bail!("attachment `{}` is not a file", src.display());
2347 }
2348 wanted.push((src, name));
2349 }
2350 Ok(wanted)
2351}
2352
2353impl Queue {
2354 pub fn open() -> Self {
2356 Self::at(crate::run::home().join("queue"))
2357 }
2358
2359 pub fn at(root: PathBuf) -> Self {
2362 Self {
2363 root,
2364 ids: IdSource::new(new_id),
2365 }
2366 }
2367
2368 #[doc(hidden)]
2371 pub fn with_id_source(mut self, f: impl Fn() -> String + Send + Sync + 'static) -> Self {
2372 self.ids = IdSource::custom(f);
2373 self
2374 }
2375
2376 fn file_with(
2387 &self,
2388 task: &mut Task,
2389 hook: impl FnMut(&mut Task) -> Result<bool>,
2390 undo: impl FnMut(&mut Task),
2391 ) -> Result<()> {
2392 self.reserve_id(task, hook, undo, true)
2393 }
2394
2395 fn reserve_id(
2403 &self,
2404 task: &mut Task,
2405 mut hook: impl FnMut(&mut Task) -> Result<bool>,
2406 mut undo: impl FnMut(&mut Task),
2407 store: bool,
2408 ) -> Result<()> {
2409 let redraw_first = self.ids.is_custom() && task.attachments.is_empty();
2410 let initial = task.id.clone();
2411 self.ids
2412 .reserve((!redraw_first).then_some(initial.as_str()), "task", |id| {
2413 task.id = id.to_owned();
2414 let _lock = self.lock_task(&task.id)?;
2415 if path_exists(&self.path_of(&task.id))? || !hook(task)? {
2416 return Ok(false);
2417 }
2418 if store {
2419 if let Err(e) = self.write_record(task) {
2420 undo(task);
2421 return Err(e);
2422 }
2423 task.unfiled = false;
2424 }
2425 Ok(true)
2426 })?;
2427 Ok(())
2428 }
2429
2430 fn id_in_use(&self, id: &str, check_attachments: bool) -> Result<bool> {
2435 if path_exists(&self.lock_path(id))? {
2436 return Ok(true);
2437 }
2438 Ok(check_attachments && path_exists(&self.attachments_dir(id))?)
2439 }
2440
2441 fn plain_filing_hook(&self, task: &Task) -> Result<bool> {
2445 Ok(!self.id_in_use(&task.id, task.attachments.is_empty())?)
2446 }
2447
2448 pub fn file_claimed(&self, task: &mut Task) -> Result<Claim> {
2451 let mut claim = None;
2452 self.file_with(
2453 task,
2454 |t| {
2455 if self.id_in_use(&t.id, true)? {
2456 return Ok(false);
2457 }
2458 claim = Some(self.claim(&t.id)?);
2459 Ok(true)
2460 },
2461 |_| {},
2462 )?;
2463 claim.context("claim missing after filing")
2464 }
2465
2466 pub fn root(&self) -> &Path {
2468 &self.root
2469 }
2470
2471 pub fn path_of(&self, id: &str) -> PathBuf {
2473 self.root.join(format!("{id}.json"))
2474 }
2475
2476 pub fn attachments_dir(&self, id: &str) -> PathBuf {
2479 self.root.join(format!("{id}.attachments"))
2480 }
2481
2482 pub fn attach(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2493 let wanted = validate_attachments(sources)?;
2494 if task.unfiled && task.attachments.is_empty() {
2495 let names = std::cell::RefCell::new(Vec::new());
2498 self.reserve_id(
2499 task,
2500 |t| {
2501 if self.id_in_use(&t.id, true)? {
2502 return Ok(false);
2503 }
2504 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2505 Ok(true)
2506 },
2507 |_| {},
2508 false,
2509 )?;
2510 return Ok(names.into_inner());
2511 }
2512 self.copy_attachments(task, &wanted)
2513 }
2514
2515 fn copy_attachments(
2517 &self,
2518 task: &mut Task,
2519 wanted: &[(&PathBuf, &str)],
2520 ) -> Result<Vec<String>> {
2521 let dir = self.attachments_dir(&task.id);
2522 let existed = dir.is_dir();
2523 std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
2524 let mut created: Vec<PathBuf> = Vec::new();
2525 let mut names = Vec::new();
2526 let mut copy_all = || -> Result<()> {
2527 for (src, name) in wanted {
2528 let (stored, path) = copy_new(&dir, src, name)?;
2529 created.push(path);
2530 names.push(stored);
2531 }
2532 Ok(())
2533 };
2534 if let Err(e) = copy_all() {
2535 for path in &created {
2536 let _ = std::fs::remove_file(path);
2537 }
2538 if !existed {
2539 let _ = std::fs::remove_dir(&dir);
2540 }
2541 return Err(e);
2542 }
2543 task.attachments.extend(names.iter().cloned());
2544 Ok(names)
2545 }
2546
2547 pub fn attach_and_put(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2558 if task.unfiled && task.attachments.is_empty() {
2559 return self.attach_and_file(task, sources);
2560 }
2561 let dir = self.attachments_dir(&task.id);
2562 let existed = dir.is_dir();
2563 let before = task.attachments.len();
2564 let names = self.attach(task, sources)?;
2565 if let Err(e) = self.put(task) {
2566 for name in &names {
2567 let _ = std::fs::remove_file(dir.join(name));
2568 }
2569 if !existed {
2570 let _ = std::fs::remove_dir(&dir);
2571 }
2572 task.attachments.truncate(before);
2573 return Err(e);
2574 }
2575 Ok(names)
2576 }
2577
2578 fn attach_and_file(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2582 let wanted = validate_attachments(sources)?;
2583 let before = task.attachments.len();
2584 let names = std::cell::RefCell::new(Vec::new());
2585 let result = self.file_with(
2586 task,
2587 |t| {
2588 if self.id_in_use(&t.id, true)? {
2589 return Ok(false);
2590 }
2591 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2592 Ok(true)
2593 },
2594 |t| {
2595 let dir = self.attachments_dir(&t.id);
2596 for name in names.borrow().iter() {
2597 let _ = std::fs::remove_file(dir.join(name));
2598 }
2599 let _ = std::fs::remove_dir(&dir);
2601 t.attachments.truncate(before);
2602 },
2603 );
2604 result.map(|()| names.into_inner())
2605 }
2606
2607 pub fn attachment_paths(&self, task: &Task) -> Vec<PathBuf> {
2611 let dir = self.attachments_dir(&task.id);
2612 task.attachments
2613 .iter()
2614 .map(|n| {
2615 let p = dir.join(n);
2616 std::path::absolute(&p).unwrap_or(p)
2617 })
2618 .collect()
2619 }
2620
2621 pub fn put(&self, task: &mut Task) -> Result<()> {
2630 if task.unfiled {
2631 return self.file_with(task, |t| self.plain_filing_hook(t), |_| {});
2633 }
2634 let _lock = self.lock_task(&task.id)?;
2635 self.put_unlocked(task)
2636 }
2637
2638 pub fn create_new(&self, task: &mut Task) -> Result<bool> {
2643 let _lock = self.lock_task(&task.id)?;
2644 if path_exists(&self.path_of(&task.id))? {
2645 return Ok(false);
2646 }
2647 self.write_record(task)?;
2648 task.unfiled = false;
2649 Ok(true)
2650 }
2651
2652 fn put_unlocked(&self, task: &mut Task) -> Result<()> {
2654 self.merge_stored(task);
2655 if let Some(boundary) = task.reassignments.last() {
2656 let stored = self.get(&task.id)?;
2657 anyhow::ensure!(
2658 task.reassignments == stored.reassignments
2659 && task.repo == stored.repo
2660 && task.runs.get(..boundary.run_boundary)
2661 == stored.runs.get(..boundary.run_boundary),
2662 "historical repository execution cannot be restarted; reload the task"
2663 );
2664 }
2665 self.write_record(task)
2666 }
2667
2668 pub(crate) fn commit_reassignment(&self, task: &mut Task, expected: &Task) -> Result<()> {
2670 let _lock = self.lock_task(&task.id)?;
2671 let stored = self.get(&task.id)?;
2672 anyhow::ensure!(
2673 serde_json::to_value(&stored)? == serde_json::to_value(expected)?
2674 && stored.reassignments.len() + 1 == task.reassignments.len()
2675 && stored.repo == task.reassignments.last().expect("new boundary").from
2676 && stored.runs == task.runs,
2677 "task changed during reassignment; retry after inspecting it"
2678 );
2679 self.write_record(task)
2680 }
2681
2682 fn merge_stored(&self, task: &mut Task) {
2684 if let Ok(mut stored) = read_path(&self.path_of(&task.id)) {
2685 stored.import_chat_notice();
2686 if stored.reassignments.len() > task.reassignments.len() {
2687 let stale = std::mem::replace(task, stored.clone());
2690 for answer in stale.answers {
2691 if !task.answers.contains(&answer) {
2692 task.answers.push(answer);
2693 }
2694 }
2695 }
2696 for receipt in &stored.repair_grants {
2697 if !task
2698 .repair_grants
2699 .iter()
2700 .any(|r| r.request.request == receipt.request.request)
2701 {
2702 task.repair_grants.push(receipt.clone());
2704 task.resume_override = stored.resume_override.clone();
2705 task.status = stored.status;
2706 task.hold_source = stored.hold_source;
2707 task.hold_reason = stored.hold_reason.clone();
2708 task.held_at = stored.held_at;
2709 }
2710 }
2711 if task.chat.is_none() {
2714 task.chat = stored.chat.clone();
2715 }
2716 for key in stored.chat_notices {
2717 if !task.chat_notices.contains(&key) {
2718 task.chat_notices.push(key);
2719 }
2720 }
2721 for failure in stored.startup_failures {
2722 task.record_startup_failure(failure);
2723 }
2724 for run in stored.runs {
2725 if !task.runs.contains(&run) {
2726 task.runs.push(run);
2727 }
2728 }
2729 for p in stored.passes {
2730 if !task
2731 .passes
2732 .iter()
2733 .any(|q| q.run == p.run && q.pass_index == p.pass_index)
2734 {
2735 task.passes.push(p);
2736 }
2737 }
2738 if let Some(n) = stored.chat_notice
2741 && task.chat_notice.as_ref().is_none_or(|mine| mine.at < n.at)
2742 {
2743 task.chat_notice = Some(n);
2744 }
2745 }
2746 }
2747
2748 pub(crate) fn write_startup_history(&self, task: &Task) -> Result<()> {
2751 let path = self.path_of(&task.id);
2752 let tmp = path.with_extension("json.tmp");
2753 std::fs::write(&tmp, serde_json::to_vec_pretty(task)?)?;
2754 std::fs::rename(tmp, path)?;
2755 Ok(())
2756 }
2757
2758 fn write_record(&self, task: &mut Task) -> Result<()> {
2760 task.import_chat_notice();
2761 task.updated_at = Timestamp::now();
2762 std::fs::create_dir_all(&self.root)
2763 .with_context(|| format!("create {}", self.root.display()))?;
2764 let body = serde_json::to_string_pretty(task).context("serialize task")?;
2765 let path = self.path_of(&task.id);
2766 let tmp = path.with_extension("json.tmp");
2767 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
2768 std::fs::rename(&tmp, &path).with_context(|| format!("replace {}", path.display()))?;
2769 if let (Some(notice), Some(home)) = (
2774 crate::notices::task_held(task),
2775 self.root.parent().filter(|p| !p.as_os_str().is_empty()),
2776 ) {
2777 crate::notices::raise_in(home, notice);
2778 }
2779 Ok(())
2780 }
2781
2782 pub fn modify(&self, id: &str, f: impl FnOnce(&mut Task) -> bool) -> Result<bool> {
2787 let id = self.resolve_id(id)?;
2788 let _lock = self.lock_task(&id)?;
2789 let mut task = self.get(&id)?;
2790 if !f(&mut task) {
2791 return Ok(false);
2792 }
2793 self.put_unlocked(&mut task)?;
2794 Ok(true)
2795 }
2796
2797 pub fn link_run(&self, id: &str, run: &str) -> Result<Task> {
2802 let id = self.resolve_id(id)?;
2803 let _lock = self.lock_task(&id)?;
2806 let mut task = self.get(&id)?;
2807 if task.link_run(run) {
2808 self.put_unlocked(&mut task)?;
2809 }
2810 Ok(task)
2811 }
2812
2813 pub(crate) fn lock_task(&self, id: &str) -> Result<TaskLock> {
2821 std::fs::create_dir_all(&self.root)
2822 .with_context(|| format!("create {}", self.root.display()))?;
2823 let path = self.root.join(format!("{id}.write-lock"));
2824 let started = std::time::Instant::now();
2825 let retry = crate::lock_file::Retry::task_write(started + TASK_LOCK_STALE);
2826 let marker_retry = crate::lock_file::Retry::new(Some(started + TASK_LOCK_STALE));
2827 loop {
2828 match retry.create(&path) {
2829 Ok(mut file) => {
2830 use std::io::Write;
2831 let token = owner_token();
2832 if let Err(e) = file.write_all(token.as_bytes()) {
2833 drop(file);
2834 let _ = std::fs::remove_file(&path);
2835 return Err(e).with_context(|| format!("lock {}", path.display()));
2836 }
2837 return Ok(TaskLock { path, token });
2838 }
2839 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2840 let judged = std::fs::read_to_string(&path).ok();
2843 let broken = judged
2844 .filter(|_| older_than_stale(&path))
2845 .is_some_and(|judged| break_stale_with(&path, &judged, marker_retry));
2846 if !broken {
2850 if started.elapsed() > TASK_LOCK_STALE {
2851 bail!("could not lock task {id}");
2852 }
2853 std::thread::sleep(std::time::Duration::from_millis(15));
2854 }
2855 }
2856 Err(e) => return Err(e).with_context(|| format!("lock {}", path.display())),
2857 }
2858 }
2859 }
2860
2861 pub fn get(&self, id: &str) -> Result<Task> {
2863 let resolved = self.resolve_id(id)?;
2864 read_path(&self.path_of(&resolved))
2865 }
2866
2867 pub fn remove(&self, id: &str, in_flight: bool, questions: &Questions) -> Result<Removal> {
2899 let resolved = self.resolve_id(id)?;
2900 if in_flight {
2901 bail!("task {resolved} is being run by a live daemon right now");
2902 }
2903 self.write_tombstone(&resolved)?;
2904 if let Err(e) = self.remove_record_with_attachments(&resolved, |p| std::fs::remove_file(p))
2905 {
2906 if self.path_of(&resolved).exists() {
2910 let _ = std::fs::remove_file(self.tombstone_path(&resolved));
2911 }
2912 return Err(e);
2913 }
2914 let (released, still_blocked) = self.release_dependents_of(&resolved, questions);
2915 Ok(Removal {
2916 id: resolved,
2917 released,
2918 still_blocked,
2919 })
2920 }
2921
2922 fn tombstone_path(&self, id: &str) -> PathBuf {
2925 self.root.join(format!("{id}.removed"))
2926 }
2927
2928 fn write_tombstone(&self, id: &str) -> Result<()> {
2929 let path = self.tombstone_path(id);
2930 let tmp = self.root.join(format!("{id}.removed.tmp"));
2931 std::fs::write(&tmp, b"")
2932 .and_then(|()| std::fs::rename(&tmp, &path))
2933 .with_context(|| format!("write {}", path.display()))
2934 }
2935
2936 fn was_deleted(&self, id: &str) -> bool {
2940 self.tombstone_path(id).is_file() && !self.path_of(id).exists()
2941 }
2942
2943 pub fn apply_deleted_blockers(&self, task: &mut Task, questions: &Questions) -> Vec<String> {
2947 let deleted = deleted_blockers(self, questions, &task.blocked_by);
2948 deleted
2949 .into_iter()
2950 .filter(|id| task.dependency_deleted(id))
2951 .collect()
2952 }
2953
2954 pub fn note_dependency_deleted(&self, dependent: &Task, deleted: &str) {
2958 let Some(home) = self.root.parent().filter(|p| !p.as_os_str().is_empty()) else {
2959 return;
2960 };
2961 let ja = crate::lang::is_japanese(&crate::lang::of_repo(&dependent.repo));
2962 let waiting = dependent.status == TaskStatus::Blocked;
2963 let message = match (ja, waiting) {
2964 (true, false) => format!(
2965 "タスク {} は、待っていた {} が削除されたため待機を解除し、元の状態に戻しました",
2966 dependent.short(),
2967 short(deleted)
2968 ),
2969 (true, true) => format!(
2970 "タスク {} は、待っていた {} が削除されたため、残りの依存を待っています",
2971 dependent.short(),
2972 short(deleted)
2973 ),
2974 (false, false) => format!(
2975 "Task {} stopped waiting on {} because it was deleted, and returned to its previous state",
2976 dependent.short(),
2977 short(deleted)
2978 ),
2979 (false, true) => format!(
2980 "Task {} stopped waiting on {} because it was deleted, and is still waiting on its other dependencies",
2981 dependent.short(),
2982 short(deleted)
2983 ),
2984 };
2985 crate::notices::raise_in(
2986 home,
2987 crate::notices::Notice::info(
2988 &format!("unblocked:{}:{}", dependent.id, deleted),
2989 message,
2990 )
2991 .link(crate::notices::Link::Task {
2992 id: dependent.id.clone(),
2993 }),
2994 );
2995 }
2996
2997 fn release_dependents_of(
3002 &self,
3003 dependency: &str,
3004 questions: &Questions,
3005 ) -> (Vec<String>, Vec<String>) {
3006 let (mut released, mut still_blocked) = (Vec::new(), Vec::new());
3007 for listed in self.list() {
3008 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| {
3009 matches!(crate::queue::dependency(self, questions, b), Dependency::Deleted(id) if id == dependency)
3010 }) {
3011 continue;
3012 }
3013 let Ok(_claim) = self.claim(&listed.id) else {
3014 continue;
3015 };
3016 let Ok(mut task) = self.get(&listed.id) else {
3017 continue;
3018 };
3019 let references: Vec<String> = task.blocked_by.iter().filter(|id| {
3020 matches!(crate::queue::dependency(self, questions, id), Dependency::Deleted(full) if full == dependency)
3021 }).cloned().collect();
3022 if references.is_empty() {
3023 continue;
3024 }
3025 for reference in references {
3026 task.dependency_deleted(&reference);
3027 }
3028 if self.put(&mut task).is_err() {
3029 continue;
3030 }
3031 self.note_dependency_deleted(&task, dependency);
3032 if task.status == TaskStatus::Blocked {
3033 still_blocked.push(task.id.clone());
3034 } else {
3035 released.push(task.id.clone());
3036 }
3037 }
3038 (released, still_blocked)
3039 }
3040
3041 fn remove_record_with_attachments(
3045 &self,
3046 resolved: &str,
3047 remove_record: impl FnOnce(&Path) -> std::io::Result<()>,
3048 ) -> Result<()> {
3049 self.sweep_removed_attachments();
3056 let attachments = self.attachments_dir(resolved);
3057 let aside = self.root.join(format!("{resolved}.attachments.removing"));
3058 let moved = match std::fs::rename(&attachments, &aside) {
3059 Ok(()) => true,
3060 Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
3061 Err(e) => {
3062 return Err(e).with_context(|| format!("remove {}", attachments.display()));
3063 }
3064 };
3065 let path = self.path_of(resolved);
3066 if let Err(e) = remove_record(&path) {
3067 if moved {
3068 let _ = std::fs::rename(&aside, &attachments);
3069 }
3070 return Err(e).with_context(|| format!("remove {}", path.display()));
3071 }
3072 if moved {
3073 if let Err(e) = std::fs::remove_dir_all(&aside) {
3074 tracing::warn!("leftover attachments {}: {e}", aside.display());
3075 }
3076 }
3077 let lock = self.lock_path(resolved);
3078 if let Err(e) = std::fs::remove_file(&lock) {
3079 if e.kind() != std::io::ErrorKind::NotFound {
3080 return Err(e).with_context(|| format!("remove {}", lock.display()));
3081 }
3082 }
3083 Ok(())
3084 }
3085
3086 fn sweep_removed_attachments(&self) {
3092 let Ok(entries) = std::fs::read_dir(&self.root) else {
3093 return;
3094 };
3095 for entry in entries.flatten() {
3096 let name = entry.file_name();
3097 let name = name.to_string_lossy();
3098 let Some(id) = name.strip_suffix(".attachments.removing") else {
3099 continue;
3100 };
3101 if !self.path_of(id).exists() {
3104 if let Err(e) = std::fs::remove_dir_all(entry.path()) {
3105 tracing::warn!("leftover attachments {}: {e}", entry.path().display());
3106 }
3107 }
3108 }
3109 self.sweep_tombstones();
3110 }
3111
3112 fn sweep_tombstones(&self) {
3121 let Ok(entries) = std::fs::read_dir(&self.root) else {
3122 return;
3123 };
3124 let tasks = self.list();
3125 for entry in entries.flatten() {
3126 let name = entry.file_name();
3127 let name = name.to_string_lossy();
3128 let Some(id) = name.strip_suffix(".removed") else {
3129 continue;
3130 };
3131 if self.path_of(id).exists()
3132 || tasks.iter().any(|t| {
3133 t.blocked_by
3134 .iter()
3135 .any(|b| id.starts_with(b) || id.ends_with(b))
3136 })
3137 {
3138 continue;
3139 }
3140 let old = entry
3141 .metadata()
3142 .and_then(|m| m.modified())
3143 .ok()
3144 .and_then(|m| m.elapsed().ok())
3145 .is_some_and(|age| age >= TOMBSTONE_GRACE);
3146 if old {
3147 let _ = std::fs::remove_file(entry.path());
3148 }
3149 }
3150 }
3151
3152 fn lock_path(&self, id: &str) -> PathBuf {
3155 self.root.join(format!("{id}.lock"))
3156 }
3157
3158 pub fn list(&self) -> Vec<Task> {
3171 let mut tasks: Vec<Task> = std::fs::read_dir(&self.root)
3172 .into_iter()
3173 .flatten()
3174 .flatten()
3175 .map(|e| e.path())
3176 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3177 .filter_map(|p| read_path(&p).ok())
3178 .collect();
3179 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then_with(|| b.id.cmp(&a.id)));
3180 tasks
3181 }
3182
3183 pub fn unreadable(&self) -> Vec<PathBuf> {
3187 std::fs::read_dir(&self.root)
3188 .into_iter()
3189 .flatten()
3190 .flatten()
3191 .map(|e| e.path())
3192 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3193 .filter(|p| read_path(p).is_err())
3194 .collect()
3195 }
3196
3197 pub fn superseded(&self) -> HashMap<String, String> {
3210 let mut by = HashMap::new();
3211 for task in self.list() {
3212 for earlier in &task.runs {
3213 if let Some(later) = task.successor_of(earlier) {
3214 by.insert(earlier.clone(), later.clone());
3215 }
3216 }
3217 }
3218 by
3219 }
3220
3221 pub fn superseded_by(&self, run: &str) -> Option<String> {
3229 for task in self.list() {
3230 if task.runs.iter().any(|r| r == run) {
3231 return task.successor_of(run).cloned();
3232 }
3233 }
3234 None
3235 }
3236
3237 pub fn latest_attempt(&self, run: &str) -> Option<String> {
3248 for task in self.list() {
3249 if task.runs.iter().any(|r| r == run) {
3250 return task.runs.last().filter(|last| **last != run).cloned();
3251 }
3252 }
3253 None
3254 }
3255
3256 pub fn next_runnable(&self) -> Option<Task> {
3261 let mut runnable: Vec<Task> = self
3262 .list()
3263 .into_iter()
3264 .filter(|t| t.status.runnable())
3265 .collect();
3266 runnable.sort_unstable_by(dispatch_order);
3267 runnable.into_iter().next()
3268 }
3269
3270 pub fn claim(&self, id: &str) -> Result<Claim> {
3277 std::fs::create_dir_all(&self.root)
3278 .with_context(|| format!("create {}", self.root.display()))?;
3279 let path = self.lock_path(id);
3280 let retry = crate::lock_file::Retry::new(None);
3281 match retry.create(&path) {
3282 Ok(mut f) => {
3283 use std::io::Write as _;
3284 let _ = writeln!(f, "{}", std::process::id());
3286 Ok(Claim { path })
3287 }
3288 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
3289 bail!("task {id} is already claimed ({} exists)", path.display())
3290 }
3291 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
3292 }
3293 }
3294
3295 pub fn resolve_id(&self, prefix: &str) -> Result<String> {
3297 if !prefix.is_empty()
3298 && prefix != "."
3299 && prefix != ".."
3300 && !prefix.contains(['/', '\\'])
3301 && self.path_of(prefix).is_file()
3302 {
3303 return Ok(prefix.to_owned());
3304 }
3305 let ids = record_ids(&self.root, "json")?;
3306 match resolve_reference(prefix, ids.iter().map(String::as_str)) {
3307 Reference::Found(id) => Ok(id),
3308 Reference::Missing => bail!("no task matches `{prefix}`"),
3309 Reference::Ambiguous => bail!("`{prefix}` matches multiple tasks"),
3310 }
3311 }
3312
3313 pub fn revision(&self) -> u64 {
3320 self.revision_excluding(&std::collections::BTreeSet::new())
3321 }
3322
3323 pub fn revision_excluding(&self, skip: &std::collections::BTreeSet<String>) -> u64 {
3326 use std::hash::{Hash as _, Hasher as _};
3327
3328 let mut entries: Vec<(String, u64)> = std::fs::read_dir(&self.root)
3329 .into_iter()
3330 .flatten()
3331 .flatten()
3332 .filter(|e| e.path().extension().is_some_and(|ext| ext == "json"))
3333 .filter(|e| {
3334 let path = e.path();
3335 !path
3336 .file_stem()
3337 .is_some_and(|stem| skip.contains(stem.to_string_lossy().as_ref()))
3338 })
3339 .filter_map(|e| {
3340 let name = e.file_name().to_string_lossy().into_owned();
3341 let mtime = e
3342 .metadata()
3343 .ok()?
3344 .modified()
3345 .ok()?
3346 .duration_since(std::time::UNIX_EPOCH)
3347 .ok()?
3348 .as_millis() as u64;
3349 Some((name, mtime))
3350 })
3351 .collect();
3352
3353 if entries.is_empty() {
3354 return 0;
3355 }
3356
3357 entries.sort_unstable();
3358 let mut hasher = std::hash::DefaultHasher::new();
3359 for (name, mtime) in &entries {
3360 name.hash(&mut hasher);
3361 mtime.hash(&mut hasher);
3362 }
3363 let h = hasher.finish();
3364 if h == 0 { 1 } else { h }
3365 }
3366}
3367
3368const TOMBSTONE_GRACE: std::time::Duration = std::time::Duration::from_secs(3600);
3371
3372#[derive(Debug, Clone)]
3374pub struct Removal {
3375 pub id: String,
3377 pub released: Vec<String>,
3381 pub still_blocked: Vec<String>,
3384}
3385
3386#[derive(Debug)]
3388pub struct Claim {
3389 path: PathBuf,
3390}
3391
3392impl Drop for Claim {
3393 fn drop(&mut self) {
3394 let _ = std::fs::remove_file(&self.path);
3395 }
3396}
3397
3398pub fn title_from(instruction: &str, max: usize) -> String {
3401 let Some(line) = first_line(instruction) else {
3402 return "(empty task)".to_owned();
3403 };
3404 if line.chars().count() <= max {
3405 return line.to_owned();
3406 }
3407 let head: String = line.chars().take(max.saturating_sub(1)).collect();
3408 format!("{head}…")
3409}
3410
3411pub(crate) fn first_line(instruction: &str) -> Option<&str> {
3417 let line = instruction
3418 .lines()
3419 .map(str::trim)
3420 .find(|l| !l.is_empty())?
3421 .trim_start_matches(['#', '-', '*', '>', ' '])
3422 .trim();
3423 (!line.is_empty()).then_some(line)
3424}
3425
3426fn read_path(path: &Path) -> Result<Task> {
3427 let body = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
3428 let task: Task =
3429 serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
3430 if task.schema > SCHEMA {
3436 bail!(
3437 "task {} was written by a different magi (schema {}, this build \
3438 speaks {SCHEMA})",
3439 task.id,
3440 task.schema
3441 );
3442 }
3443 Ok(task)
3444}
3445
3446#[derive(Debug, PartialEq, Eq)]
3449pub(crate) enum Reference {
3450 Found(String),
3451 Missing,
3452 Ambiguous,
3453}
3454
3455pub(crate) fn resolve_reference<'a>(id: &str, ids: impl Iterator<Item = &'a str>) -> Reference {
3456 let mut hits = Vec::new();
3457 for candidate in ids {
3458 if candidate == id {
3459 return Reference::Found(candidate.to_owned());
3460 }
3461 if candidate.starts_with(id) || candidate.ends_with(id) {
3462 hits.push(candidate);
3463 }
3464 }
3465 match hits.as_slice() {
3466 [] => Reference::Missing,
3467 [one] => Reference::Found((*one).to_owned()),
3468 _ => Reference::Ambiguous,
3469 }
3470}
3471
3472pub(crate) fn record_ids(root: &Path, extension: &str) -> Result<Vec<String>> {
3475 let entries = match std::fs::read_dir(root) {
3476 Ok(entries) => entries,
3477 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
3478 Err(e) => return Err(e.into()),
3479 };
3480 let mut ids = Vec::new();
3481 for entry in entries {
3482 let path = entry?.path();
3483 if path.extension().is_some_and(|e| e == extension)
3484 && let Some(id) = path.file_stem().and_then(|s| s.to_str())
3485 {
3486 ids.push(id.to_owned());
3487 }
3488 }
3489 Ok(ids)
3490}
3491
3492#[derive(Debug, PartialEq, Eq)]
3495pub enum Dependency {
3496 Task(String),
3498 Question(String),
3500 Deleted(String),
3502 Missing,
3504 Ambiguous,
3506 Unknown,
3508}
3509
3510pub fn dependency(queue: &Queue, questions: &Questions, id: &str) -> Dependency {
3512 let (Ok(tasks), Ok(qs), Ok(removed)) = (
3513 record_ids(queue.root(), "json"),
3514 record_ids(questions.root(), "json"),
3515 record_ids(queue.root(), "removed"),
3516 ) else {
3517 return Dependency::Unknown;
3518 };
3519 let mut ids = tasks
3522 .iter()
3523 .chain(qs.iter())
3524 .chain(removed.iter())
3525 .cloned()
3526 .collect::<Vec<_>>();
3527 ids.sort();
3528 ids.dedup();
3529 match resolve_reference(id, ids.iter().map(String::as_str)) {
3530 Reference::Found(id) if tasks.contains(&id) => Dependency::Task(id),
3531 Reference::Found(id) if qs.contains(&id) => Dependency::Question(id),
3532 Reference::Found(id) if queue.was_deleted(&id) => Dependency::Deleted(id),
3533 Reference::Found(_) => Dependency::Unknown,
3534 Reference::Missing => Dependency::Missing,
3535 Reference::Ambiguous => Dependency::Ambiguous,
3536 }
3537}
3538
3539pub fn normalize_blockers(queue: &Queue, questions: &Questions, ids: &mut [String]) {
3541 for id in ids {
3542 if let Dependency::Task(full) | Dependency::Question(full) | Dependency::Deleted(full) =
3543 dependency(queue, questions, id)
3544 {
3545 *id = full;
3546 }
3547 }
3548}
3549
3550pub fn dependency_facts(queue: &Queue, questions: &Questions) -> String {
3553 let inventory = crate::blockers::Inventory::load(queue, questions);
3554 let mut text = String::from(
3555 "Current authoritative queue (historical missing-task claims may be stale; unreadable or ambiguous references must keep waiting):\n",
3556 );
3557 match record_ids(queue.root(), "json") {
3558 Ok(ids) => {
3559 for id in ids {
3560 match queue.get(&id) {
3561 Ok(task) => text.push_str(&format!(
3562 "{}: {}; blocked_by={:?}; chain={:?}\n",
3563 task.id,
3564 task.status.as_str(),
3565 task.blocked_by,
3566 inventory.waits_on(&task)
3567 )),
3568 Err(_) => text.push_str(&format!("{id}: exists, state unreadable\n")),
3569 }
3570 }
3571 }
3572 Err(_) => text.push_str("Queue inventory unreadable: existence is unknown.\n"),
3573 }
3574 text
3575}
3576
3577pub fn missing_blockers(
3579 queue: &Queue,
3580 questions: &Questions,
3581 blocked_by: &[String],
3582) -> Vec<String> {
3583 blocked_by
3584 .iter()
3585 .filter(|id| dependency(queue, questions, id) == Dependency::Missing)
3586 .cloned()
3587 .collect()
3588}
3589
3590pub fn deleted_blockers(
3593 queue: &Queue,
3594 questions: &Questions,
3595 blocked_by: &[String],
3596) -> Vec<String> {
3597 blocked_by
3598 .iter()
3599 .filter(|id| matches!(dependency(queue, questions, id), Dependency::Deleted(_)))
3600 .cloned()
3601 .collect()
3602}
3603
3604pub fn missing_blocker_hold_reason(blocked_by: &[String], missing: &[String]) -> String {
3616 missing_blocker_hold_reason_in(blocked_by, missing, "en")
3617}
3618
3619pub fn missing_blocker_hold_reason_in(
3621 blocked_by: &[String],
3622 missing: &[String],
3623 language: &str,
3624) -> String {
3625 if crate::lang::is_japanese(language) {
3626 format!(
3627 "{} を待っていましたが、{} はディスク上に存在しません - `magi task triage` を参照",
3628 blocked_by.join(", "),
3629 missing.join(", "),
3630 )
3631 } else {
3632 format!(
3633 "blocked on {} but {} no longer exist(s) on disk - see `magi task triage`",
3634 blocked_by.join(", "),
3635 missing.join(", "),
3636 )
3637 }
3638}
3639
3640pub fn short(id: &str) -> &str {
3642 id.split('-').next_back().unwrap_or(id)
3643}
3644
3645fn new_id() -> String {
3646 let stamp = jiff::Zoned::now().strftime("%Y%m%d-%H%M%S");
3647 let seed = crate::rng::entropy();
3648 format!("{stamp}-{:04x}", (seed ^ (seed >> 32)) & 0xffff)
3649}
3650
3651#[cfg(test)]
3652mod tests {
3653 #[test]
3654 fn reassignment_clears_repository_bound_execution_selections() {
3655 let mut task = Task::new(
3656 "move".into(),
3657 "evidence".into(),
3658 "old".into(),
3659 Source::Human,
3660 );
3661 task.review_branch = Some("old-review".into());
3662 task.review_instruction = Some("old criteria".into());
3663 task.pr_recovery = Some(crate::pr_recovery::Selection {
3664 url: "https://example.invalid/pull/1".into(),
3665 branch: "old-pr".into(),
3666 head: "head".into(),
3667 source_head: None,
3668 source_run: None,
3669 source_proof: None,
3670 disposition: None,
3671 instruction: "old criteria".into(),
3672 approved_at: Timestamp::now(),
3673 });
3674 task.replacement = Some(crate::merge_decision::Replacement {
3675 question: "owner".into(),
3676 run: "old-run".into(),
3677 pr: "https://example.invalid/pull/1".into(),
3678 agent: "worker".into(),
3679 context: "evidence".into(),
3680 });
3681 task.resume_override = Some(OperatorResume {
3682 question_id: "old-question".into(),
3683 at: Timestamp::now(),
3684 conductor_rehold: None,
3685 forced: true,
3686 pinned_run: Some("old-run".into()),
3687 });
3688 task.interrupt = true;
3689 task.reassign_repository("new".into(), "test");
3690 assert!(task.review_branch.is_none());
3691 assert!(task.review_instruction.is_none());
3692 assert!(task.pr_recovery.is_none());
3693 assert!(task.replacement.is_none());
3694 assert!(task.resume_override.is_none());
3695 assert!(!task.interrupt);
3696 assert!(task.fresh_start);
3697 assert_eq!(task.instruction, "evidence");
3698 }
3699
3700 #[test]
3701 fn explicit_resumption_is_not_invented_for_legacy_or_automatic_release() {
3702 let t = Task::new(
3703 "retry".into(),
3704 "do it".into(),
3705 PathBuf::from("."),
3706 Source::Human,
3707 );
3708 let mut value = serde_json::to_value(&t).unwrap();
3709 for field in ["resumption", "resumption_history", "operator_decided_at"] {
3710 value.as_object_mut().unwrap().remove(field);
3711 }
3712 value["schema"] = serde_json::json!(15);
3713 let mut legacy: Task = serde_json::from_value(value).unwrap();
3714 legacy.hold_machine(Some("automatic retry".into()));
3715 legacy.release();
3716 assert!(legacy.resumption.is_none());
3717 assert!(legacy.resumption_history.is_empty());
3718 assert!(legacy.operator_decided_at.is_none());
3719 legacy.release_operator("cli", None);
3720 legacy.start("run".into());
3721 let reloaded: Task =
3722 serde_json::from_str(&serde_json::to_string(&legacy).unwrap()).unwrap();
3723 assert!(reloaded.resume_override.is_none());
3724 assert_eq!(reloaded.resumption.as_ref().unwrap().generation, 1);
3725 }
3726
3727 #[test]
3728 fn merge_over_lets_the_later_choice_win_and_keeps_the_rest() {
3729 let mut base = RunOverrides {
3730 merge: Some("pr".to_owned()),
3731 candidates: Some(3),
3732 ..RunOverrides::default()
3733 };
3734 base.merge_over(&RunOverrides {
3735 merge: Some("none".to_owned()),
3736 seed: Some(7),
3737 ..RunOverrides::default()
3738 });
3739 assert_eq!(base.merge.as_deref(), Some("none"));
3740 assert_eq!(base.candidates, Some(3));
3741 assert_eq!(base.seed, Some(7));
3742 }
3743
3744 #[test]
3745 fn an_already_landed_task_is_done_with_its_attempt_refunded() {
3746 let mut t = task("relanded");
3747 t.attempts = 1;
3748 t.status = TaskStatus::Running;
3749 t.already_landed("already in main as 0e368de");
3750 assert_eq!(t.status, TaskStatus::Done);
3751 assert_eq!(t.attempts, 0);
3752 assert!(t.hold_reason.is_none());
3753 assert_eq!(t.last_error.as_deref(), Some("already in main as 0e368de"));
3754 }
3755
3756 #[test]
3757 fn missing_blocker_reason_follows_the_language() {
3758 let b = vec!["a".to_owned()];
3759 let en = missing_blocker_hold_reason_in(&b, &b, "en");
3760 assert_eq!(en, missing_blocker_hold_reason(&b, &b));
3761 assert!(en.starts_with("blocked on a"));
3762 assert!(missing_blocker_hold_reason_in(&b, &b, "ja").contains("存在しません"));
3763 assert_eq!(missing_blocker_hold_reason_in(&b, &b, "de"), en);
3764 }
3765
3766 use super::*;
3767
3768 #[test]
3769 fn triage_applied_survives_release_and_old_records_read_as_empty() {
3770 let mut t = Task::new(
3771 "t".to_owned(),
3772 "i".to_owned(),
3773 PathBuf::from("r"),
3774 Source::Human,
3775 );
3776 t.mark_triage_applied("q1");
3777 t.mark_triage_applied("q1");
3778 t.hold_machine(Some("x".to_owned()));
3779 t.release();
3780 assert_eq!(t.triage_applied, ["q1"]);
3781 assert!(t.triage_applied("q1") && !t.triage_applied("q2"));
3782
3783 let mut v = serde_json::to_value(&t).unwrap();
3784 v.as_object_mut().unwrap().remove("triage_applied");
3785 let old: Task = serde_json::from_value(v).unwrap();
3786 assert!(old.triage_applied.is_empty());
3787 }
3788
3789 #[test]
3790 fn task_counts_of_empty_is_all_zero() {
3791 assert_eq!(TaskCounts::of(&[]), TaskCounts::default());
3792 }
3793
3794 #[test]
3795 fn task_counts_of_tallies_every_status() {
3796 let mut queued = Task::new(
3797 "q".to_owned(),
3798 "i".to_owned(),
3799 PathBuf::from("."),
3800 Source::Human,
3801 );
3802 queued.status = TaskStatus::Queued;
3803 let mut running = queued.clone();
3804 running.status = TaskStatus::Running;
3805 let mut done = queued.clone();
3806 done.status = TaskStatus::Done;
3807 let mut failed = queued.clone();
3808 failed.status = TaskStatus::Failed;
3809 let mut held = queued.clone();
3810 held.status = TaskStatus::Held;
3811 let mut blocked = queued.clone();
3812 blocked.status = TaskStatus::Blocked;
3813
3814 let mut parked = queued.clone();
3815 parked.status = TaskStatus::Parked;
3816
3817 let counts = TaskCounts::of(&[
3818 queued,
3819 running,
3820 done.clone(),
3821 done,
3822 failed,
3823 held,
3824 blocked,
3825 parked,
3826 ]);
3827 assert_eq!(
3828 counts,
3829 TaskCounts {
3830 queued: 1,
3831 running: 1,
3832 done: 2,
3833 failed: 1,
3834 held: 1,
3835 blocked: 1,
3836 parked: 1,
3837 }
3838 );
3839 }
3840
3841 #[test]
3842 fn task_implementers_persist_normalized_without_losing_history() {
3843 let (_dir, q) = queue();
3844 let mut t = task("keep me");
3845 t.hold_manual(Some("waiting".into()));
3846 t.runs.push("earlier-run".into());
3847 t.attachments.push("reference.txt".into());
3848 let cfg: crate::config::Config =
3849 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
3850 .unwrap();
3851 q.put(&mut t).unwrap();
3852 let original = serde_json::to_value(&t).unwrap();
3853 for n in [2, 1, 2] {
3854 t.set_implementers(n, &cfg).unwrap();
3855 q.put(&mut t).unwrap();
3856 t = q.get(&t.id).unwrap();
3857 assert_eq!(t.implementers(), Some(usize::from(n)));
3858 assert_eq!(t.solo, n == 1);
3859 if n == 1 {
3860 assert!(t.overrides.is_none());
3861 }
3862 let mut actual = serde_json::to_value(&t).unwrap();
3863 for field in ["solo", "overrides", "updated_at"] {
3864 actual[field] = original[field].clone();
3865 }
3866 assert_eq!(actual, original);
3867 }
3868 t.overrides.as_mut().unwrap().judges = Some(2);
3869 t.set_implementers(1, &cfg).unwrap();
3870 assert_eq!(t.overrides.as_ref().unwrap().judges, Some(2));
3871 assert_eq!(t.overrides.as_ref().unwrap().candidates, None);
3872 }
3873
3874 #[test]
3875 fn task_implementers_old_records_keep_inheritance_until_changed() {
3876 let mut value = serde_json::to_value(task("old task")).unwrap();
3877 value.as_object_mut().unwrap().remove("solo");
3878 value.as_object_mut().unwrap().remove("overrides");
3879 let mut old: Task = serde_json::from_value(value).unwrap();
3880 assert_eq!(old.implementers(), None);
3881 old.set_implementers(1, &crate::config::Config::default())
3882 .unwrap();
3883 assert!(old.solo);
3884 assert!(old.overrides.is_none());
3885 }
3886
3887 #[test]
3888 fn task_implementers_reject_invalid_counts_and_states() {
3889 let cfg = crate::config::Config::default();
3890 let mut t = task("unchanged");
3891 for n in [0, 4, 255] {
3892 assert!(t.set_implementers(n, &cfg).is_err());
3893 }
3894 for status in [
3895 TaskStatus::Running,
3896 TaskStatus::Done,
3897 TaskStatus::Failed,
3898 TaskStatus::Blocked,
3899 TaskStatus::Parked,
3900 ] {
3901 t.status = status;
3902 let before = serde_json::to_value(&t).unwrap();
3903 assert!(t.set_implementers(1, &cfg).is_err());
3904 assert_eq!(serde_json::to_value(&t).unwrap(), before);
3905 }
3906 }
3907
3908 fn queue() -> (tempfile::TempDir, Queue) {
3911 let dir = tempfile::tempdir().unwrap();
3912 let q = Queue::at(dir.path().join("queue"));
3913 (dir, q)
3914 }
3915
3916 #[test]
3917 fn a_stale_write_does_not_drop_the_recorded_task_chat() {
3918 let (_dir, q) = queue();
3919 let mut stale = task("chat");
3920 q.put(&mut stale).unwrap();
3921 let mut fresh = q.get(&stale.id).unwrap();
3922 fresh.chat = Some("talk-1".to_owned());
3923 q.put(&mut fresh).unwrap();
3924 q.put(&mut stale).unwrap();
3925 assert_eq!(q.get(&stale.id).unwrap().chat.as_deref(), Some("talk-1"));
3926 }
3927
3928 #[test]
3929 fn putting_a_machine_held_task_files_a_notification_beside_the_queue() {
3930 let dir = tempfile::tempdir().unwrap();
3931 let q = Queue::at(dir.path().join("queue"));
3932 let mut t = task("held");
3933 q.put(&mut t).unwrap();
3934 assert_eq!(
3935 crate::notices::Notices::at(dir.path().join("notifications"))
3936 .list()
3937 .len(),
3938 0
3939 );
3940 t.hold_machine(Some("out of attempts".to_owned()));
3941 q.put(&mut t).unwrap();
3942 let listed = crate::notices::Notices::at(dir.path().join("notifications")).list();
3943 assert_eq!(listed.len(), 1);
3944 assert!(listed[0].message.contains("out of attempts"));
3945 }
3946
3947 fn task(title: &str) -> Task {
3948 Task::new(
3949 title.to_owned(),
3950 format!("do {title}"),
3951 PathBuf::from("."),
3952 Source::Human,
3953 )
3954 }
3955
3956 #[test]
3957 fn attempt_limit_inherits_old_records_and_controls_failure() {
3958 let mut t = task("budget");
3959 let mut old = serde_json::to_value(&t).unwrap();
3960 old.as_object_mut().unwrap().remove("attempt_limit");
3961 old["schema"] = serde_json::json!(15);
3962 let legacy: Task = serde_json::from_value(old).unwrap();
3963 assert_eq!(legacy.attempt_limit, None);
3964 assert_eq!(legacy.effective_attempt_limit(2), 2);
3965 t.attempts = 2;
3966 t.change_attempt_limit(NonZeroUsize::new(3)).unwrap();
3967 t.fail("retry", 1);
3968 assert_eq!(t.status, TaskStatus::Failed);
3969 t.attempts = 3;
3970 t.fail("stop", 10);
3971 assert_eq!(t.status, TaskStatus::Held);
3972 t.change_attempt_limit(None).unwrap();
3973 assert_eq!(t.status, TaskStatus::Held);
3974 assert_eq!(t.attempts, 3);
3975 t.fail("inherit", 4);
3976 assert_eq!(t.status, TaskStatus::Failed);
3977 for status in [
3978 TaskStatus::Running,
3979 TaskStatus::Done,
3980 TaskStatus::Failed,
3981 TaskStatus::Blocked,
3982 TaskStatus::Parked,
3983 ] {
3984 t.status = status;
3985 assert!(t.change_attempt_limit(NonZeroUsize::new(1)).is_err());
3986 assert_eq!(t.attempt_limit, None);
3987 }
3988 }
3989
3990 #[test]
3991 fn earlier_attempts_is_every_recorded_run_and_agrees_with_the_display() {
3992 let mut t = task("retried");
3993 assert!(
3994 t.earlier_attempts().is_empty(),
3995 "a first attempt takes nothing over"
3996 );
3997 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
3998 assert_eq!(t.earlier_attempts(), ["aaaa", "bbbb"]);
3999 assert_eq!(t.successor_of("aaaa"), Some(&"bbbb".to_owned()));
4001 assert_eq!(t.successor_of("bbbb"), None);
4002 assert_eq!(t.successor_of("zzzz"), None);
4003 }
4004
4005 #[test]
4006 fn resumed_runs_are_superseded_only_after_their_last_occurrence() {
4007 let (_dir, q) = queue();
4008 let mut t = task("resumed");
4009 for (runs, successor) in [
4010 (vec!["aaaa", "aaaa"], None),
4011 (vec!["aaaa", "bbbb", "aaaa"], None),
4012 (vec!["aaaa", "aaaa", "bbbb"], Some("bbbb")),
4013 (vec!["aaaa", "bbbb", "aaaa", "cccc"], Some("cccc")),
4014 ] {
4015 t.runs = runs.into_iter().map(str::to_owned).collect();
4016 q.put(&mut t).unwrap();
4017
4018 assert_eq!(t.successor_of("aaaa").map(String::as_str), successor);
4019 assert_eq!(q.superseded_by("aaaa").as_deref(), successor);
4020 assert_eq!(q.superseded().get("aaaa").map(String::as_str), successor);
4021 }
4022 }
4023
4024 #[test]
4025 fn superseded_by_names_the_next_attempt_and_none_for_the_last() {
4026 let (_dir, q) = queue();
4027 let mut t = task("retried");
4028 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4029 q.put(&mut t).unwrap();
4030
4031 assert_eq!(q.superseded_by("aaaa"), Some("bbbb".to_owned()));
4032 assert_eq!(q.superseded_by("bbbb"), Some("cccc".to_owned()));
4033 assert_eq!(
4034 q.superseded_by("cccc"),
4035 None,
4036 "the latest attempt replaces nothing"
4037 );
4038 assert_eq!(
4039 q.superseded_by("never-heard-of-it"),
4040 None,
4041 "a run belonging to no task on this queue is not superseded"
4042 );
4043
4044 let mut by = HashMap::new();
4045 by.insert("aaaa".to_owned(), "bbbb".to_owned());
4046 by.insert("bbbb".to_owned(), "cccc".to_owned());
4047 assert_eq!(
4048 q.superseded(),
4049 by,
4050 "the whole-map and single-run forms must agree"
4051 );
4052 }
4053
4054 #[test]
4055 fn superseded_attempts_is_empty_until_the_task_is_done() {
4056 let mut t = task("retried");
4057 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4058 t.status = TaskStatus::Failed;
4059 assert_eq!(
4060 t.superseded_attempts(true),
4061 &[] as &[String],
4062 "a task still retrying has no attempt yet that a later one made moot"
4063 );
4064
4065 t.status = TaskStatus::Running;
4066 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
4067 }
4068
4069 #[test]
4070 fn superseded_attempts_names_every_run_before_the_one_that_succeeded() {
4071 let mut t = task("retried");
4072 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4073 t.status = TaskStatus::Done;
4074 assert_eq!(
4075 t.superseded_attempts(true),
4076 &["aaaa".to_owned(), "bbbb".to_owned()],
4077 "cccc is the attempt whose success made the task done, and stays out"
4078 );
4079 }
4080
4081 #[test]
4082 fn superseded_attempts_is_empty_for_a_done_task_with_only_one_attempt() {
4083 let mut t = task("first try landed");
4084 t.runs = vec!["aaaa".to_owned()];
4085 t.status = TaskStatus::Done;
4086 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
4087 }
4088
4089 #[test]
4090 fn superseded_attempts_is_empty_when_the_last_run_never_actually_succeeded() {
4091 let mut t = task("closed by hand after a manual merge");
4098 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4099 t.status = TaskStatus::Done;
4100 assert_eq!(
4101 t.superseded_attempts(false),
4102 &[] as &[String],
4103 "nothing here is provably why the task is done, so nothing is superseded"
4104 );
4105 }
4106
4107 #[test]
4108 fn latest_attempt_names_the_chain_s_current_head_not_just_the_next_one() {
4109 let (_dir, q) = queue();
4110 let mut t = task("retried twice");
4111 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4112 q.put(&mut t).unwrap();
4113
4114 assert_eq!(
4115 q.latest_attempt("aaaa"),
4116 Some("cccc".to_owned()),
4117 "an old attempt points straight at the chain's current head, not the \
4118 next attempt in the middle of it"
4119 );
4120 assert_eq!(q.latest_attempt("bbbb"), Some("cccc".to_owned()));
4121 assert_eq!(
4122 q.latest_attempt("cccc"),
4123 None,
4124 "the latest attempt is not superseded by anything"
4125 );
4126 assert_eq!(
4127 q.latest_attempt("never-heard-of-it"),
4128 None,
4129 "a run belonging to no task on this queue is not superseded"
4130 );
4131 }
4132
4133 #[test]
4134 fn a_markdown_heading_is_the_title_not_decoration() {
4135 assert_eq!(
4140 title_from("# Rework the config loader\n\nIt re-reads it.\n", 40),
4141 "Rework the config loader"
4142 );
4143 assert_eq!(title_from("- fix the thing", 40), "fix the thing");
4144 assert_eq!(title_from("> quoted task", 40), "quoted task");
4145 assert_eq!(title_from(" \n\n", 40), "(empty task)");
4147 assert_eq!(title_from("###\n", 40), "(empty task)");
4148 }
4149
4150 #[test]
4151 fn a_long_title_is_elided_by_characters_not_bytes() {
4152 let long = "課題".repeat(30);
4154 let title = title_from(&long, 10);
4155 assert_eq!(title.chars().count(), 10);
4156 assert!(title.ends_with('…'));
4157 }
4158
4159 #[test]
4160 fn priority_wins_and_ties_break_oldest_first() {
4161 let (_dir, q) = queue();
4162 let mut a = task("first");
4163 let mut b = task("second");
4164 let mut c = task("urgent");
4165 a.id = "20260101-000001-aaaa".to_owned();
4167 b.id = "20260101-000002-bbbb".to_owned();
4168 c.id = "20260101-000003-cccc".to_owned();
4169 c.priority = 5;
4170 for t in [&mut a, &mut b, &mut c] {
4171 q.put(t).unwrap();
4172 }
4173
4174 assert_eq!(q.next_runnable().unwrap().id, c.id);
4176 c.hold_machine(None);
4177 q.put(&mut c).unwrap();
4178 assert_eq!(q.next_runnable().unwrap().id, a.id);
4180 assert_eq!(q.list().len(), 3, "b is still waiting its turn");
4181 }
4182
4183 #[test]
4184 fn a_blocked_task_never_starves_another_runnable_one() {
4185 let (_dir, q) = queue();
4186 let mut blocked = task("blocked");
4187 blocked.block(vec!["something".to_owned()], None);
4188 q.put(&mut blocked).unwrap();
4189
4190 let mut runnable = task("free to go");
4191 q.put(&mut runnable).unwrap();
4192
4193 let next = q.next_runnable().expect("a runnable task is still offered");
4194 assert_eq!(next.id, runnable.id);
4195 }
4196
4197 #[test]
4198 fn a_held_task_is_never_offered_to_the_loop() {
4199 let (_dir, q) = queue();
4200 let mut t = task("held");
4201 q.put(&mut t).unwrap();
4202 assert!(q.next_runnable().is_some());
4203
4204 t.hold_machine(None);
4205 q.put(&mut t).unwrap();
4206 assert!(
4207 q.next_runnable().is_none(),
4208 "a held task must wait for a human"
4209 );
4210
4211 t.status = TaskStatus::Failed;
4213 q.put(&mut t).unwrap();
4214 assert!(q.next_runnable().is_some());
4215 }
4216
4217 #[test]
4218 fn attempts_are_capped_and_then_the_task_is_held() {
4219 let mut t = task("doomed");
4220
4221 t.start("run-1".to_owned());
4222 t.fail("gate red", 2);
4223 assert_eq!(t.status, TaskStatus::Failed, "one attempt of two: retry");
4224
4225 t.start("run-2".to_owned());
4226 t.fail("gate red", 2);
4227 assert_eq!(
4228 t.status,
4229 TaskStatus::Held,
4230 "out of attempts: stop spending money on it"
4231 );
4232 assert_eq!(t.runs, ["run-1", "run-2"]);
4233 assert_eq!(t.last_error.as_deref(), Some("gate red"));
4234 assert_eq!(
4235 t.hold_reason.as_deref(),
4236 Some("gate red"),
4237 "the hold must say why, not leave hold_reason null next to a \
4238 populated last_error"
4239 );
4240 }
4241
4242 #[test]
4243 fn handing_off_a_task_records_a_hold_reason_too() {
4244 let mut t = task("left a pull request");
4245 t.start("run-1".to_owned());
4246 t.handed_off("run ended with a pull request open [run run-1]");
4247 assert_eq!(t.status, TaskStatus::Held);
4248 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4249 assert_eq!(
4250 t.hold_reason.as_deref(),
4251 Some("run ended with a pull request open [run run-1]")
4252 );
4253 assert_eq!(t.hold_reason, t.last_error);
4254 }
4255
4256 #[test]
4257 fn a_quota_stall_is_refunded_so_the_backlog_survives_the_night() {
4258 let mut t = task("stalled by quota");
4259
4260 t.start("run-1".to_owned());
4261 assert_eq!(t.attempts, 1);
4262 t.stall("judge-1, judge-2 out of quota");
4263 assert_eq!(
4264 t.attempts, 0,
4265 "a closed quota window must not spend the task's retry budget"
4266 );
4267 assert_eq!(t.status, TaskStatus::Failed, "the loop should retry it");
4268 assert_eq!(
4269 t.last_error.as_deref(),
4270 Some("judge-1, judge-2 out of quota")
4271 );
4272
4273 for _ in 0..20 {
4276 t.start("run-n".to_owned());
4277 t.stall("still out of quota");
4278 }
4279 t.start("run-real".to_owned());
4280 t.fail("gate red", 2);
4281 assert_eq!(
4282 t.status,
4283 TaskStatus::Failed,
4284 "the first attempt that was really judged is attempt one"
4285 );
4286 }
4287
4288 #[test]
4289 fn releasing_a_held_task_gives_it_a_real_second_chance() {
4290 let mut t = task("retry me");
4291 t.start("run-1".to_owned());
4292 t.fail("gate red", 1);
4293 assert_eq!(t.status, TaskStatus::Held);
4294
4295 t.release();
4296 assert_eq!(t.status, TaskStatus::Queued);
4297 assert_eq!(t.attempts, 0);
4300 assert!(t.last_error.is_none());
4301 assert_eq!(
4302 t.runs.len(),
4303 1,
4304 "history is kept: attempts reset, evidence does not"
4305 );
4306 }
4307
4308 #[test]
4309 fn a_hold_reason_survives_and_a_release_clears_it() {
4310 let mut t = task("waiting on something else");
4311 t.hold_manual(Some(
4312 "waiting for 20260101-000000-aaaa to land first".to_owned(),
4313 ));
4314 assert_eq!(t.status, TaskStatus::Held);
4315 assert_eq!(
4316 t.hold_reason.as_deref(),
4317 Some("waiting for 20260101-000000-aaaa to land first")
4318 );
4319
4320 t.hold_manual(None);
4322 assert_eq!(
4323 t.hold_reason.as_deref(),
4324 Some("waiting for 20260101-000000-aaaa to land first"),
4325 "a bare re-hold keeps whatever a human already wrote down"
4326 );
4327
4328 let mut plain = task("no reason given");
4330 plain.hold_manual(None);
4331 assert_eq!(plain.status, TaskStatus::Held);
4332 assert!(plain.hold_reason.is_none());
4333
4334 t.release();
4335 assert_eq!(t.status, TaskStatus::Queued);
4336 assert!(
4337 t.hold_reason.is_none(),
4338 "a stale reason must not greet the next person who holds this task"
4339 );
4340 }
4341
4342 #[test]
4343 fn closing_a_held_task_as_done_clears_its_hold_reason_too() {
4344 let mut t = task("landed by hand while held");
4349 t.hold_manual(Some("waiting on 3ed9".to_owned()));
4350 assert_eq!(t.hold_reason.as_deref(), Some("waiting on 3ed9"));
4351
4352 t.succeed();
4353 assert_eq!(t.status, TaskStatus::Done);
4354 assert!(
4355 t.hold_reason.is_none(),
4356 "a done task cannot still be waiting on something"
4357 );
4358 }
4359
4360 #[test]
4361 fn holding_or_closing_a_blocked_task_clears_its_dependency_too() {
4362 let mut held = task("held straight out of blocked");
4369 held.block(
4370 vec!["20260101-000000-dead".to_owned()],
4371 Some("waiting on the migration script".to_owned()),
4372 );
4373 assert_eq!(held.status, TaskStatus::Blocked);
4374
4375 held.hold_manual(None);
4376 assert_eq!(held.status, TaskStatus::Held);
4377 assert!(
4378 held.blocked_by.is_empty(),
4379 "hold overrides the wait, same as release"
4380 );
4381 assert!(held.block_reason.is_none());
4382
4383 let mut done = task("closed straight out of blocked");
4384 done.block(
4385 vec!["20260101-000000-dead".to_owned()],
4386 Some("waiting on the migration script".to_owned()),
4387 );
4388 done.succeed();
4389 assert_eq!(done.status, TaskStatus::Done);
4390 assert!(
4391 done.blocked_by.is_empty(),
4392 "a done task cannot still be waiting on a dependency"
4393 );
4394 assert!(done.block_reason.is_none());
4395 }
4396
4397 #[test]
4398 fn a_blocked_task_is_never_offered_to_the_loop() {
4399 let mut t = task("blocked");
4400 assert!(t.status.runnable());
4401 t.block(
4402 vec!["dep-id".to_owned()],
4403 Some("waits on dep-id".to_owned()),
4404 );
4405 assert_eq!(t.status, TaskStatus::Blocked);
4406 assert!(!t.status.runnable());
4407 assert_eq!(TaskStatus::Blocked.as_str(), "blocked");
4408 }
4409
4410 #[test]
4411 fn unblocking_the_last_dependency_returns_the_task_to_queued() {
4412 let mut t = task("blocked on two");
4413 t.block(
4414 vec!["a".to_owned(), "b".to_owned()],
4415 Some("waits on a and b".to_owned()),
4416 );
4417
4418 t.unblock("a");
4419 assert_eq!(t.status, TaskStatus::Blocked, "b is still outstanding");
4420 assert_eq!(t.blocked_by, ["b"]);
4421
4422 t.unblock("b");
4423 assert_eq!(t.status, TaskStatus::Queued);
4424 assert!(t.blocked_by.is_empty());
4425 assert!(t.block_reason.is_none());
4426 }
4427
4428 #[test]
4429 fn unblocking_an_id_on_a_task_that_is_not_blocked_is_a_no_op() {
4430 let mut t = task("never blocked");
4431 t.unblock("whatever");
4432 assert_eq!(t.status, TaskStatus::Queued);
4433 }
4434
4435 #[test]
4436 fn a_held_task_blocked_on_a_question_returns_to_held_not_queued() {
4437 let mut t = task("held, then asked about");
4442 t.hold_machine(Some("out of attempts".to_owned()));
4443 assert_eq!(t.status, TaskStatus::Held);
4444
4445 t.block(vec!["q1".to_owned()], Some("what now?".to_owned()));
4446 assert_eq!(t.status, TaskStatus::Blocked);
4447
4448 t.record_answer("what now?".to_owned(), "leave it held".to_owned());
4449 t.unblock("q1");
4450 assert_eq!(t.status, TaskStatus::Held, "must restore, not requeue");
4451 assert_eq!(t.hold_reason.as_deref(), Some("out of attempts"));
4452 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4453 assert!(t.blocked_from.is_none(), "consumed once restored");
4454 }
4455
4456 #[test]
4457 fn a_manually_held_task_blocked_on_a_question_returns_to_held() {
4458 let mut t = task("manually held, then asked about");
4459 t.hold_manual(Some("waiting on a dependency".to_owned()));
4460
4461 t.block(vec!["q1".to_owned()], None);
4462 t.unblock("q1");
4463
4464 assert_eq!(t.status, TaskStatus::Held);
4465 assert_eq!(t.hold_source, Some(HoldSource::Manual));
4466 }
4467
4468 #[test]
4469 fn re_blocking_an_already_blocked_task_keeps_the_original_blocked_from() {
4470 let mut t = task("held, blocked twice");
4474 t.hold_machine(None);
4475 t.block(vec!["q1".to_owned()], Some("first".to_owned()));
4476 t.block(
4477 vec!["q1".to_owned(), "q2".to_owned()],
4478 Some("second".to_owned()),
4479 );
4480
4481 t.unblock("q1");
4482 assert_eq!(t.status, TaskStatus::Blocked, "q2 still outstanding");
4483 t.unblock("q2");
4484 assert_eq!(t.status, TaskStatus::Held);
4485 }
4486
4487 #[test]
4488 fn unblocking_a_task_blocked_while_running_lands_on_queued_not_running() {
4489 let mut t = task("blocked mid-run");
4493 t.start("run-1".to_owned());
4494 assert_eq!(t.status, TaskStatus::Running);
4495
4496 t.block(vec!["q1".to_owned()], None);
4497 t.unblock("q1");
4498 assert_eq!(t.status, TaskStatus::Queued);
4499 }
4500
4501 #[test]
4502 fn a_pre_schema_4_blocked_record_with_hold_evidence_restores_to_held() {
4503 let mut t = task("legacy record, held before it was blocked");
4509 t.hold_source = Some(HoldSource::Machine);
4510 t.hold_reason = Some("legacy hold reason".to_owned());
4511 t.status = TaskStatus::Blocked;
4512 t.blocked_by = vec!["q1".to_owned()];
4513 t.blocked_from = None;
4514
4515 t.unblock("q1");
4516 assert_eq!(t.status, TaskStatus::Held);
4517 }
4518
4519 #[test]
4520 fn a_pre_schema_4_blocked_record_with_no_hold_evidence_restores_to_queued() {
4521 let mut t = task("legacy record, ordinary dependency block");
4522 t.status = TaskStatus::Blocked;
4523 t.blocked_by = vec!["dep".to_owned()];
4524 t.blocked_from = None;
4525
4526 t.unblock("dep");
4527 assert_eq!(t.status, TaskStatus::Queued);
4528 }
4529
4530 #[test]
4531 fn answering_a_question_is_recorded_and_survives_a_release() {
4532 let mut t = task("asked something");
4533 t.block(vec!["q1".to_owned()], Some("which backend?".to_owned()));
4534 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
4535 t.unblock("q1");
4536 assert_eq!(t.status, TaskStatus::Queued);
4537 assert_eq!(t.answers.len(), 1);
4538 assert_eq!(t.answers[0].answer, "SQLite");
4539
4540 t.release();
4544 assert_eq!(t.answers.len(), 1, "the answer is not lost on release");
4545 }
4546
4547 #[test]
4548 fn resumption_review_branch_requires_unchanged_authored_criteria() {
4549 for source in ["cli", "web"] {
4550 let mut t = task("review old candidate");
4551 t.request_review("candidate-A".into());
4552 t.hold_for_handover(Some("candidate-A".into()), "handover refused".into());
4553 let mut unchanged = t.clone();
4554 unchanged.release_operator(source, None);
4555 assert_eq!(unchanged.review_branch.as_deref(), Some("candidate-A"));
4556 t.edit(
4557 "review approved head".into(),
4558 "Reuse measured PR head B; fix Clippy; review".into(),
4559 )
4560 .unwrap();
4561 t.release_operator(source, None);
4562 assert!(t.review_branch.is_none());
4563 assert!(t.review_instruction.is_none());
4564 let mut legacy = unchanged;
4566 legacy.review_instruction = None;
4567 legacy.hold_for_handover(None, "still checked out".into());
4568 legacy
4569 .edit("new plan".into(), "Reuse measured PR head B".into())
4570 .unwrap();
4571 legacy.release_operator(source, None);
4572 assert!(legacy.review_branch.is_none());
4573 }
4574 }
4575
4576 #[test]
4577 fn a_refused_handover_keeps_the_review_branch_across_release() {
4578 let mut t = task("refused takeover");
4579 t.start("run-1".to_owned());
4580 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
4581 assert_eq!(t.status, TaskStatus::Held);
4582 assert_eq!(t.attempts, 1);
4583 t.release();
4584 assert_eq!(t.status, TaskStatus::Queued);
4585 assert_eq!(t.attempts, 0);
4586 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4587
4588 t.hold_for_handover(None, "again".to_owned());
4590 t.requeue();
4591 assert!(t.review_branch.is_none());
4592
4593 let mut m = task("manual");
4595 m.review_branch = Some("magi/x/A".to_owned());
4596 m.hold_manual(None);
4597 m.release();
4598 assert!(m.review_branch.is_none());
4599 }
4600
4601 #[test]
4602 fn requesting_review_requeues_the_task_and_remembers_the_branch() {
4603 let mut t = task("blocked run with a surviving branch");
4604 t.start("run-1".to_owned());
4605 t.fail("blocked with major findings", 5);
4606 assert_eq!(t.status, TaskStatus::Failed);
4607
4608 t.request_review("magi/eba2/A".to_owned());
4609 assert_eq!(t.status, TaskStatus::Queued);
4610 assert_eq!(t.attempts, 0);
4611 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4612
4613 t.release();
4615 assert!(t.review_branch.is_none());
4616 }
4617
4618 #[test]
4619 fn conductor_requeue_but_not_an_ordinary_release_forces_a_fresh_start() {
4620 let mut t = task("retry");
4621 t.start("run-1".to_owned());
4622 t.requeue();
4623 assert!(t.fresh_start);
4624
4625 t.release();
4626 assert!(!t.fresh_start);
4627 }
4628
4629 #[test]
4630 fn urgent_only_changes_queued_or_held_and_preserves_settings() {
4631 for status in [
4632 TaskStatus::Queued,
4633 TaskStatus::Held,
4634 TaskStatus::Running,
4635 TaskStatus::Done,
4636 TaskStatus::Failed,
4637 TaskStatus::Blocked,
4638 TaskStatus::Parked,
4639 ] {
4640 let mut t = task("urgent settings");
4641 t.status = status;
4642 t.attempts = 2;
4643 t.hold_reason = Some("waiting".into());
4644 t.priority = 7;
4645 t.interrupt = true;
4646 let before = serde_json::to_value(&t).unwrap();
4647 for urgent in [true, true, false, false] {
4648 let result = t.set_urgent(urgent);
4649 let mut expected = before.clone();
4650 if matches!(status, TaskStatus::Queued | TaskStatus::Held) {
4651 result.unwrap();
4652 expected["urgent"] = urgent.into();
4653 } else {
4654 let error = result.unwrap_err().to_string();
4655 assert!(error.contains(status.as_str()), "{error}");
4656 assert!(error.contains("queued or held"), "{error}");
4657 }
4658 assert_eq!(serde_json::to_value(&t).unwrap(), expected);
4659 }
4660 }
4661 }
4662
4663 #[test]
4664 fn priority_can_be_changed_while_queued_but_not_while_running() {
4665 let mut t = task("reprioritise me");
4666 t.set_priority(5).unwrap();
4667 assert_eq!(t.priority, 5);
4668
4669 t.start("run-1".to_owned());
4670 let err = t.set_priority(9).unwrap_err().to_string();
4671 assert!(err.contains("running"), "{err}");
4672 assert_eq!(t.priority, 5, "the rejected write must not partially apply");
4673 }
4674
4675 #[test]
4676 fn interrupt_can_be_marked_while_queued_but_not_while_running() {
4677 let mut t = task("interrupt me");
4678 assert!(!t.interrupt, "off unless asked, same as any other task");
4679
4680 t.set_interrupt(true).unwrap();
4681 assert!(t.interrupt);
4682
4683 t.start("run-1".to_owned());
4684 assert!(
4685 !t.interrupt,
4686 "the mark is one-shot: dispatching the task fulfils it, \
4687 whatever the run that follows ends up doing"
4688 );
4689 let err = t.set_interrupt(true).unwrap_err().to_string();
4690 assert!(err.contains("running"), "{err}");
4691 t.set_interrupt(false).unwrap();
4694 assert!(!t.interrupt);
4695 }
4696
4697 #[test]
4701 fn a_failed_run_does_not_leave_the_task_still_marked_to_interrupt() {
4702 let mut t = task("interrupt me");
4703 t.set_interrupt(true).unwrap();
4704 t.start("run-1".to_owned());
4705 t.fail("mock failure", 5);
4706 assert_eq!(t.status, TaskStatus::Failed);
4707 assert!(
4708 !t.interrupt,
4709 "one attempt already spent the mark; a retry is an ordinary \
4710 requeue, not a fresh interrupt request"
4711 );
4712 }
4713
4714 #[test]
4715 fn changing_priority_moves_a_task_ahead_in_the_real_queue_order() {
4716 let (_dir, q) = queue();
4717 let mut a = task("first filed");
4718 let mut b = task("second filed");
4719 a.id = "20260101-000001-aaaa".to_owned();
4720 b.id = "20260101-000002-bbbb".to_owned();
4721 q.put(&mut a).unwrap();
4722 q.put(&mut b).unwrap();
4723
4724 assert_eq!(
4725 q.next_runnable().unwrap().id,
4726 a.id,
4727 "with equal priority the older task goes first, so a burst of \
4728 new work cannot starve it"
4729 );
4730 assert_eq!(
4731 q.list()[0].id,
4732 b.id,
4733 "but the list an operator reads is newest first, the same as \
4734 before priority existed - a's turn to run does not make it the \
4735 newest task"
4736 );
4737
4738 let mut a = q.get(&a.id).unwrap();
4739 a.set_priority(10).unwrap();
4740 q.put(&mut a).unwrap();
4741
4742 assert_eq!(
4743 q.next_runnable().unwrap().id,
4744 a.id,
4745 "a raised priority must be reflected the moment it is saved"
4746 );
4747 assert_eq!(
4751 q.list()[0].id,
4752 a.id,
4753 "the raised task must sort first in the list an operator reads, \
4754 not only in next_runnable's own ordering"
4755 );
4756 }
4757
4758 #[test]
4759 fn editing_replaces_title_and_instruction_but_keeps_identity_and_history() {
4760 let mut t = Task::new(
4761 "old title".to_owned(),
4762 "old instruction".to_owned(),
4763 PathBuf::from("/repo"),
4764 Source::Agent {
4765 run: "20260101-000000-beef".to_owned(),
4766 node: "implement".to_owned(),
4767 },
4768 );
4769 let id = t.id.clone();
4770 let created_at = t.created_at;
4771 t.runs.push("20260101-000000-beef".to_owned());
4772
4773 t.edit("new title".to_owned(), "new instruction".to_owned())
4774 .unwrap();
4775
4776 assert_eq!(t.title, "new title");
4777 assert_eq!(t.instruction, "new instruction");
4778 assert_eq!(t.id, id, "editing must not mint a new id");
4779 assert_eq!(t.created_at, created_at);
4780 assert_eq!(
4781 t.source,
4782 Source::Agent {
4783 run: "20260101-000000-beef".to_owned(),
4784 node: "implement".to_owned(),
4785 },
4786 "editing must not turn agent attribution into human"
4787 );
4788 assert_eq!(t.runs, ["20260101-000000-beef"]);
4789 }
4790
4791 #[test]
4792 fn editing_is_refused_once_a_task_is_running_or_finished() {
4793 let mut running = task("in flight");
4794 running.start("run-1".to_owned());
4795 let err = running
4796 .edit("x".to_owned(), "y".to_owned())
4797 .unwrap_err()
4798 .to_string();
4799 assert!(err.contains("running"), "{err}");
4800
4801 let mut done = task("finished");
4802 done.succeed();
4803 let err = done
4804 .edit("x".to_owned(), "y".to_owned())
4805 .unwrap_err()
4806 .to_string();
4807 assert!(err.contains("done"), "{err}");
4808
4809 let mut queued = task("waiting");
4811 queued.edit("x".to_owned(), "y".to_owned()).unwrap();
4812 let mut held = task("parked");
4813 held.hold_machine(None);
4814 held.edit("x".to_owned(), "y".to_owned()).unwrap();
4815 }
4816
4817 #[test]
4818 fn a_task_recorded_without_a_hold_reason_still_reads_as_none() {
4819 let (_dir, q) = queue();
4820 let path = q.path_of("20260101-000000-aaaa");
4821 std::fs::create_dir_all(q.root()).unwrap();
4822 std::fs::write(
4823 &path,
4824 serde_json::json!({
4825 "schema": SCHEMA,
4826 "id": "20260101-000000-aaaa",
4827 "title": "from before hold reasons existed",
4828 "instruction": "from before hold reasons existed",
4829 "repo": ".",
4830 "source": { "kind": "human" },
4831 "status": "held",
4832 "created_at": Timestamp::now().to_string(),
4833 "updated_at": Timestamp::now().to_string(),
4834 })
4835 .to_string(),
4836 )
4837 .unwrap();
4838
4839 let task = q.get("20260101-000000-aaaa").expect("must still read");
4840 assert!(task.hold_reason.is_none());
4841 assert!(task.operator_held());
4842 }
4843
4844 #[test]
4845 fn a_legacy_reasoned_hold_defaults_to_operator_protection() {
4846 let (_dir, q) = queue();
4847 let path = q.path_of("20260101-000000-bbbb");
4848 std::fs::create_dir_all(q.root()).unwrap();
4849 std::fs::write(
4850 &path,
4851 serde_json::json!({
4852 "schema": 2,
4853 "id": "20260101-000000-bbbb",
4854 "title": "old manual recovery",
4855 "instruction": "old manual recovery",
4856 "repo": ".",
4857 "source": { "kind": "human" },
4858 "status": "held",
4859 "hold_reason": "active manual recovery run20260912-224242-daf5",
4860 "created_at": Timestamp::now().to_string(),
4861 "updated_at": Timestamp::now().to_string(),
4862 })
4863 .to_string(),
4864 )
4865 .unwrap();
4866
4867 let task = q.get("20260101-000000-bbbb").expect("must still read");
4868 assert_eq!(task.hold_source, None);
4869 assert!(task.operator_held());
4870 }
4871
4872 #[test]
4873 fn a_task_recorded_without_a_diagnostic_still_reads_as_none() {
4874 let (_dir, q) = queue();
4875 let path = q.path_of("20260101-000000-aaaa");
4876 std::fs::create_dir_all(q.root()).unwrap();
4877 std::fs::write(
4878 &path,
4879 serde_json::json!({
4880 "schema": SCHEMA,
4881 "id": "20260101-000000-aaaa",
4882 "title": "from before diagnostics existed",
4883 "instruction": "from before diagnostics existed",
4884 "repo": ".",
4885 "source": { "kind": "human" },
4886 "status": "held",
4887 "created_at": Timestamp::now().to_string(),
4888 "updated_at": Timestamp::now().to_string(),
4889 })
4890 .to_string(),
4891 )
4892 .unwrap();
4893
4894 let task = q.get("20260101-000000-aaaa").expect("must still read");
4895 assert!(task.diagnostic.is_none());
4896 }
4897
4898 #[test]
4899 fn a_schema_1_task_with_no_blocking_fields_still_reads() {
4900 let (_dir, q) = queue();
4904 let path = q.path_of("20260101-000000-aaaa");
4905 std::fs::create_dir_all(q.root()).unwrap();
4906 std::fs::write(
4907 &path,
4908 serde_json::json!({
4909 "schema": 1,
4910 "id": "20260101-000000-aaaa",
4911 "title": "from before blocking existed",
4912 "instruction": "from before blocking existed",
4913 "repo": ".",
4914 "source": { "kind": "human" },
4915 "status": "queued",
4916 "created_at": Timestamp::now().to_string(),
4917 "updated_at": Timestamp::now().to_string(),
4918 })
4919 .to_string(),
4920 )
4921 .unwrap();
4922
4923 let task = q.get("20260101-000000-aaaa").expect("must still read");
4924 assert!(task.blocked_by.is_empty());
4925 assert!(task.block_reason.is_none());
4926 assert!(task.answers.is_empty());
4927 assert!(task.review_branch.is_none());
4928 }
4929
4930 #[test]
4931 fn releasing_or_finishing_a_task_clears_its_stale_diagnostic() {
4932 let mut held = task("diagnosed");
4937 held.start("run-1".to_owned());
4938 held.fail("gate red", 1);
4939 held.diagnostic = Some("cargo test failed: ...".to_owned());
4940 assert_eq!(held.status, TaskStatus::Held);
4941
4942 held.release();
4943 assert!(held.diagnostic.is_none());
4944
4945 held.diagnostic = Some("cargo test failed: ...".to_owned());
4946 held.succeed();
4947 assert!(held.diagnostic.is_none());
4948 }
4949
4950 #[test]
4951 fn failing_a_task_always_clears_whatever_diagnostic_it_carried() {
4952 let mut t = task("retried");
4953 t.start("run-1".to_owned());
4954 t.diagnostic = Some("stale evidence from a previous hold".to_owned());
4955 t.fail("unrelated config error", 5);
4956 assert_eq!(t.status, TaskStatus::Failed);
4957 assert!(
4958 t.diagnostic.is_none(),
4959 "fail() must not let an old diagnostic outlive the run that produced it"
4960 );
4961 }
4962
4963 #[test]
4964 fn a_claim_is_exclusive_and_releases_on_drop() {
4965 let (_dir, q) = queue();
4966 let mut t = task("contended");
4967 q.put(&mut t).unwrap();
4968
4969 let held = q.claim(&t.id).unwrap();
4970 assert!(
4971 q.claim(&t.id).is_err(),
4972 "two daemons must not drive one task into two runs"
4973 );
4974 drop(held);
4975 assert!(q.claim(&t.id).is_ok(), "a released claim is reclaimable");
4976 }
4977
4978 #[test]
4979 fn a_round_trip_survives_disk() {
4980 let (_dir, q) = queue();
4981 let mut t = Task::new(
4982 "titled".to_owned(),
4983 "body".to_owned(),
4984 PathBuf::from("/repo"),
4985 Source::Agent {
4986 run: "20260101-000000-beef".to_owned(),
4987 node: "implement".to_owned(),
4988 },
4989 );
4990 t.priority = 3;
4991 q.put(&mut t).unwrap();
4992
4993 let back = q.get(&t.id).unwrap();
4994 assert_eq!(back.id, t.id);
4995 assert_eq!(back.priority, 3);
4996 assert_eq!(back.source.label(), "implement@beef");
4997 assert_eq!(q.get(t.short()).unwrap().id, t.id);
4999 }
5000
5001 #[test]
5002 fn an_unreadable_task_does_not_take_the_queue_down() {
5003 let (_dir, q) = queue();
5004 let mut t = task("fine");
5005 q.put(&mut t).unwrap();
5006 std::fs::write(q.root().join("broken.json"), "{ not json").unwrap();
5007
5008 let listed = q.list();
5009 assert_eq!(listed.len(), 1, "the readable task still lists");
5010 assert_eq!(listed[0].id, t.id);
5011 }
5012
5013 #[test]
5014 fn a_task_recorded_without_a_solo_field_still_reads_as_not_solo() {
5015 let (_dir, q) = queue();
5016 let path = q.path_of("20260101-000000-aaaa");
5017 std::fs::create_dir_all(q.root()).unwrap();
5018 std::fs::write(
5019 &path,
5020 serde_json::json!({
5021 "schema": SCHEMA,
5022 "id": "20260101-000000-aaaa",
5023 "title": "from before solo existed",
5024 "instruction": "from before solo existed",
5025 "repo": ".",
5026 "source": { "kind": "human" },
5027 "status": "queued",
5028 "created_at": Timestamp::now().to_string(),
5029 "updated_at": Timestamp::now().to_string(),
5030 })
5031 .to_string(),
5032 )
5033 .unwrap();
5034
5035 let task = q.get("20260101-000000-aaaa").expect("must still read");
5036 assert!(!task.solo, "a queue file with no `solo` field means false");
5037 }
5038
5039 #[test]
5040 fn a_task_recorded_without_an_urgent_field_still_reads_as_not_urgent() {
5041 let (_dir, q) = queue();
5042 let path = q.path_of("20260101-000000-bbbb");
5043 std::fs::create_dir_all(q.root()).unwrap();
5044 std::fs::write(
5045 &path,
5046 serde_json::json!({
5047 "schema": SCHEMA,
5048 "id": "20260101-000000-bbbb",
5049 "title": "from before urgent existed",
5050 "instruction": "from before urgent existed",
5051 "repo": ".",
5052 "source": { "kind": "human" },
5053 "status": "queued",
5054 "created_at": Timestamp::now().to_string(),
5055 "updated_at": Timestamp::now().to_string(),
5056 })
5057 .to_string(),
5058 )
5059 .unwrap();
5060
5061 let task = q.get("20260101-000000-bbbb").expect("must still read");
5062 assert!(
5063 !task.urgent,
5064 "a queue file with no `urgent` field means false, same as `solo`"
5065 );
5066 }
5067
5068 #[test]
5069 fn a_task_from_a_future_schema_is_refused_rather_than_guessed_at() {
5070 let (_dir, q) = queue();
5071 let mut t = task("from the future");
5072 q.put(&mut t).unwrap();
5073 let path = q.path_of(&t.id);
5074 let body = std::fs::read_to_string(&path)
5075 .unwrap()
5076 .replace(&format!("\"schema\": {SCHEMA}"), "\"schema\": 99");
5077 std::fs::write(&path, body).unwrap();
5078
5079 let err = q.get(&t.id).unwrap_err().to_string();
5080 assert!(err.contains("schema 99"), "{err}");
5081 }
5082
5083 #[test]
5084 fn revision_moves_when_the_queue_changes() {
5085 let (_dir, q) = queue();
5086 assert_eq!(q.revision(), 0, "an empty queue has no revision");
5087 let mut t = task("first");
5088 q.put(&mut t).unwrap();
5089 assert!(q.revision() > 0, "a written task moves the revision");
5090 }
5091
5092 #[test]
5093 fn revision_moves_when_deleting_an_older_task() {
5094 let (dir, q) = queue();
5095 let questions = Questions::at(dir.path().join("questions"));
5096 let mut t1 = task("older");
5097 q.put(&mut t1).unwrap();
5098 std::thread::sleep(std::time::Duration::from_millis(10));
5100 let mut t2 = task("newer");
5101 q.put(&mut t2).unwrap();
5102
5103 let rev_before = q.revision();
5104 q.remove(&t1.id, false, &questions).unwrap();
5105 let rev_after = q.revision();
5106
5107 assert_ne!(
5108 rev_before, rev_after,
5109 "deleting an older task must change the revision so other clients see the deletion"
5110 );
5111 }
5112
5113 #[test]
5114 fn removing_a_task_takes_it_out_of_the_listing() {
5115 let (dir, q) = queue();
5116 let questions = Questions::at(dir.path().join("questions"));
5117 let mut t = task("delete me");
5118 q.put(&mut t).unwrap();
5119 let removed = q.remove(t.short(), false, &questions).unwrap();
5120 assert_eq!(removed.id, t.id, "a prefix resolves before deleting");
5121 assert!(removed.released.is_empty() && removed.still_blocked.is_empty());
5122 assert!(q.list().is_empty());
5123 assert!(
5124 q.remove(&t.id, false, &questions).is_err(),
5125 "removing twice is an error"
5126 );
5127 }
5128
5129 #[test]
5130 fn removing_a_task_takes_its_stale_lock_with_it() {
5131 let (dir, q) = queue();
5132 let questions = Questions::at(dir.path().join("questions"));
5133 let mut t = task("interrupted");
5134 q.put(&mut t).unwrap();
5135
5136 let claim = q.claim(&t.id).unwrap();
5139 std::mem::forget(claim);
5140 assert!(
5141 q.claim(&t.id).is_err(),
5142 "the orphaned lock is what makes the task look claimed"
5143 );
5144
5145 let err = q.remove(&t.id, true, &questions).unwrap_err().to_string();
5147 assert!(err.contains("live daemon"), "{err}");
5148 assert!(q.get(&t.id).is_ok(), "a refused delete keeps the task");
5149
5150 q.remove(&t.id, false, &questions).unwrap();
5152 assert!(q.list().is_empty());
5153 let mut again = task("interrupted");
5154 again.id = t.id.clone();
5155 q.put(&mut again).unwrap();
5156 assert!(
5157 q.claim(&t.id).is_ok(),
5158 "a task that comes back must be claimable, which a left-behind lock would prevent"
5159 );
5160 }
5161
5162 fn notices_of(dir: &Path) -> Vec<crate::notices::Notice> {
5163 crate::notices::Notices::at(dir.join("notifications")).list()
5164 }
5165
5166 #[test]
5167 fn removing_a_sole_dependency_releases_the_dependent_without_a_hold() {
5168 let (dir, q) = queue();
5169 let questions = Questions::at(dir.path().join("questions"));
5170 let mut dep = task("dependency");
5171 q.put(&mut dep).unwrap();
5172 let mut blocked = task("waiting");
5173 blocked.block(vec![dep.id.clone()], Some("waits".to_owned()));
5174 q.put(&mut blocked).unwrap();
5175
5176 let removed = q.remove(&dep.id, false, &questions).unwrap();
5177 assert_eq!(removed.released, [blocked.id.clone()]);
5178 assert!(removed.still_blocked.is_empty());
5179
5180 let after = q.get(&blocked.id).unwrap();
5181 assert_eq!(after.status, TaskStatus::Queued);
5182 assert!(after.blocked_by.is_empty());
5183 assert!(after.block_reason.is_none());
5184 let notes = notices_of(dir.path());
5185 assert_eq!(notes.len(), 1, "{notes:?}");
5186 assert_eq!(notes[0].severity, crate::notices::Severity::Info);
5187 assert_eq!(
5188 notes[0].link,
5189 Some(crate::notices::Link::Task {
5190 id: blocked.id.clone(),
5191 })
5192 );
5193 }
5194
5195 #[test]
5196 fn removing_one_of_two_dependencies_keeps_the_other() {
5197 let (dir, q) = queue();
5198 let questions = Questions::at(dir.path().join("questions"));
5199 let mut dep = task("dependency");
5200 q.put(&mut dep).unwrap();
5201 let mut other = task("other");
5202 q.put(&mut other).unwrap();
5203 let mut blocked = task("waiting");
5204 blocked.block(
5205 vec![dep.id.clone(), other.id.clone()],
5206 Some("waits on both".to_owned()),
5207 );
5208 q.put(&mut blocked).unwrap();
5209
5210 let removed = q.remove(&dep.id, false, &questions).unwrap();
5211 assert!(removed.released.is_empty());
5212 assert_eq!(removed.still_blocked, [blocked.id.clone()]);
5213
5214 let after = q.get(&blocked.id).unwrap();
5215 assert_eq!(after.status, TaskStatus::Blocked);
5216 assert_eq!(after.blocked_by, [other.id.clone()]);
5217 assert!(missing_blockers(&q, &questions, &after.blocked_by).is_empty());
5220 }
5221
5222 #[test]
5223 fn a_dependency_deleted_before_its_dependents_were_rewritten_is_released_later() {
5224 let (dir, q) = queue();
5225 let questions = Questions::at(dir.path().join("questions"));
5226 let mut dep = task("dependency");
5227 q.put(&mut dep).unwrap();
5228 let mut blocked = task("waiting");
5229 blocked.block(vec![dep.id.clone()], None);
5230 q.put(&mut blocked).unwrap();
5231
5232 let claim = q.claim(&blocked.id).unwrap();
5234 let removed = q.remove(&dep.id, false, &questions).unwrap();
5235 assert!(removed.released.is_empty());
5236 drop(claim);
5237
5238 let mut task = q.get(&blocked.id).unwrap();
5239 assert_eq!(task.status, TaskStatus::Blocked);
5240 assert!(missing_blockers(&q, &questions, &task.blocked_by).is_empty());
5241 assert_eq!(
5242 q.apply_deleted_blockers(&mut task, &questions),
5243 [dep.id.clone()]
5244 );
5245 assert_eq!(task.status, TaskStatus::Queued);
5246 }
5247
5248 #[test]
5249 fn dependency_resolution_short_deleted_reference_releases_only_its_waiters() {
5250 let (dir, q) = queue();
5251 let questions = Questions::at(dir.path().join("questions"));
5252 let mut dep = task("dependency");
5253 dep.id = "20261013-193611-ca5b".into();
5254 q.put(&mut dep).unwrap();
5255 let mut waiting = task("waiting");
5256 waiting.block(vec!["ca5b".into()], None);
5257 q.put(&mut waiting).unwrap();
5258 q.remove(&dep.id, false, &questions).unwrap();
5259 assert_eq!(q.get(&waiting.id).unwrap().status, TaskStatus::Queued);
5260 assert_eq!(
5261 dependency(&q, &questions, "ca5b"),
5262 Dependency::Deleted(dep.id)
5263 );
5264 }
5265
5266 #[test]
5267 fn dependency_resolution_preserves_cross_store_ambiguity_and_exact_ids() {
5268 let (dir, queue) = queue();
5269 let questions = Questions::at(dir.path().join("questions"));
5270 let mut dep = task("dependency");
5271 dep.id = "20261013-193611-ca5b".into();
5272 queue.put(&mut dep).unwrap();
5273 std::fs::create_dir_all(questions.root()).unwrap();
5274 let question_id = "20261014-193611-ca5b";
5275 std::fs::write(questions.path_of(question_id), "unreadable question").unwrap();
5276 assert_eq!(questions.resolve_id("ca5b").unwrap(), question_id);
5277 assert_eq!(
5278 dependency(&queue, &questions, "ca5b"),
5279 Dependency::Ambiguous
5280 );
5281 assert_eq!(
5282 dependency(&queue, &questions, &dep.id),
5283 Dependency::Task(dep.id)
5284 );
5285 assert_eq!(
5286 dependency(&queue, &questions, question_id),
5287 Dependency::Question(question_id.into())
5288 );
5289 assert_eq!(
5290 dependency(&queue, &questions, "missing"),
5291 Dependency::Missing
5292 );
5293 }
5294
5295 #[test]
5296 fn dependency_resolution_directory_errors_are_unknown_not_missing() {
5297 let dir = tempfile::tempdir().unwrap();
5298 let q = Queue::at(dir.path().join("not-a-directory"));
5299 std::fs::write(q.root(), "file").unwrap();
5300 let questions = Questions::at(dir.path().join("questions"));
5301 assert_eq!(dependency(&q, &questions, "ca5b"), Dependency::Unknown);
5302 assert!(missing_blockers(&q, &questions, &["ca5b".into()]).is_empty());
5303 }
5304
5305 #[test]
5306 fn a_dependency_without_a_tombstone_is_still_missing() {
5307 let (dir, q) = queue();
5308 let questions = Questions::at(dir.path().join("questions"));
5309 let mut dep = task("dependency");
5310 q.put(&mut dep).unwrap();
5311 std::fs::remove_file(q.path_of(&dep.id)).unwrap();
5312 assert_eq!(
5313 missing_blockers(&q, &questions, std::slice::from_ref(&dep.id)),
5314 [dep.id.clone()]
5315 );
5316 assert!(deleted_blockers(&q, &questions, std::slice::from_ref(&dep.id)).is_empty());
5317 }
5318
5319 #[test]
5320 fn a_held_dependent_is_left_alone_by_a_dependency_deletion() {
5321 let mut t = task("held");
5322 t.hold_machine(Some("because".to_owned()));
5323 t.blocked_by = vec!["gone".to_owned()];
5324 assert!(!t.dependency_deleted("gone"));
5325 assert_eq!(t.status, TaskStatus::Held);
5326 }
5327
5328 #[test]
5329 fn a_failed_record_removal_takes_the_tombstone_back() {
5330 let (_dir, q) = queue();
5331 let mut t = task("stays");
5332 q.put(&mut t).unwrap();
5333 q.write_tombstone(&t.id).unwrap();
5334 let err = q.remove_record_with_attachments(&t.id, |_| Err(std::io::Error::other("nope")));
5335 assert!(err.is_err());
5336 assert!(!q.was_deleted(&t.id));
5339 }
5340
5341 fn source_file(dir: &Path, name: &str, body: &str) -> PathBuf {
5342 let p = dir.join(name);
5343 std::fs::write(&p, body).unwrap();
5344 p
5345 }
5346
5347 #[test]
5348 fn an_attachment_copy_survives_deleting_its_source() {
5349 let (dir, q) = queue();
5350 let src = source_file(dir.path(), "shot.png", "pixels");
5351 let mut t = task("with a picture");
5352 let names = q.attach(&mut t, std::slice::from_ref(&src)).unwrap();
5353 q.put(&mut t).unwrap();
5354 std::fs::remove_file(&src).unwrap();
5355 assert_eq!(names, ["shot.png"]);
5356 let loaded = q.get(&t.id).unwrap();
5357 let paths = q.attachment_paths(&loaded);
5358 assert_eq!(paths.len(), 1);
5359 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "pixels");
5360 }
5361
5362 #[test]
5363 fn attachment_names_that_could_traverse_or_are_odd_are_refused() {
5364 let (dir, q) = queue();
5365 let mut t = task("bad names");
5366 for name in ["a..b.png", ".hidden", "with space.png", "-x.png"] {
5367 let src = source_file(dir.path(), name, "x");
5368 assert!(
5369 q.attach(&mut t, &[src]).is_err(),
5370 "`{name}` must be refused"
5371 );
5372 }
5373 assert!(!crate::ask::valid_asset_name("C:foo.png"));
5377 #[cfg(not(windows))]
5378 {
5379 let src = source_file(dir.path(), "C:foo.png", "x");
5380 assert!(q.attach(&mut t, &[src]).is_err());
5381 }
5382 let long = format!("{}.png", "a".repeat(70));
5383 let src = source_file(dir.path(), &long, "x");
5384 assert!(q.attach(&mut t, &[src]).is_err());
5385 assert!(t.attachments.is_empty());
5386 assert!(!q.attachments_dir(&t.id).exists());
5387 }
5388
5389 #[test]
5390 fn a_taken_attachment_name_is_numbered_not_overwritten() {
5391 let (dir, q) = queue();
5392 let a = source_file(dir.path(), "shot.png", "one");
5393 let sub = dir.path().join("other");
5394 std::fs::create_dir_all(&sub).unwrap();
5395 let b = source_file(&sub, "shot.png", "two");
5396 let mut t = task("collision");
5397 q.attach(&mut t, &[a]).unwrap();
5398 q.attach(&mut t, &[b]).unwrap();
5399 assert_eq!(t.attachments, ["shot.png", "shot-2.png"]);
5400 let paths = q.attachment_paths(&t);
5401 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "one");
5402 assert_eq!(std::fs::read_to_string(&paths[1]).unwrap(), "two");
5403 }
5404
5405 #[test]
5406 fn a_renumbered_name_stays_inside_the_length_bound() {
5407 let (dir, q) = queue();
5408 let name = format!("{}.png", "a".repeat(60));
5409 assert_eq!(name.len(), 64);
5410 let a = source_file(dir.path(), &name, "one");
5411 let sub = dir.path().join("other");
5412 std::fs::create_dir_all(&sub).unwrap();
5413 let b = source_file(&sub, &name, "two");
5414 let mut t = task("long");
5415 q.attach(&mut t, &[a, b]).unwrap();
5416 assert_eq!(t.attachments.len(), 2);
5417 assert!(
5418 t.attachments
5419 .iter()
5420 .all(|n| crate::ask::valid_asset_name(n))
5421 );
5422 assert!(t.attachments[1].ends_with("-2.png"));
5423 }
5424
5425 #[test]
5426 fn a_failed_attach_keeps_existing_attachments_and_leaves_no_partial_copy() {
5427 let (dir, q) = queue();
5428 let good = source_file(dir.path(), "good.png", "ok");
5429 let mut t = task("partial");
5430 q.attach(&mut t, &[good]).unwrap();
5431 let more = source_file(dir.path(), "more.png", "ok");
5432 let missing = dir.path().join("missing.png");
5433 assert!(q.attach(&mut t, &[more, missing]).is_err());
5434 assert_eq!(t.attachments, ["good.png"]);
5435 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5436 .unwrap()
5437 .flatten()
5438 .collect();
5439 assert_eq!(on_disk.len(), 1);
5440 }
5441
5442 fn block_put(q: &Queue, t: &Task) -> PathBuf {
5444 let tmp = q.path_of(&t.id).with_extension("json.tmp");
5445 std::fs::create_dir_all(&tmp).unwrap();
5446 tmp
5447 }
5448
5449 #[test]
5450 fn a_failed_put_leaves_no_new_attachment_directory() {
5451 let (dir, q) = queue();
5452 let mut t = task("fresh");
5453 let tmp = block_put(&q, &t);
5454 let src = source_file(dir.path(), "shot.png", "x");
5455 assert!(q.attach_and_put(&mut t, &[src]).is_err());
5456 assert!(t.attachments.is_empty());
5457 assert!(!q.attachments_dir(&t.id).exists());
5458 assert!(!q.path_of(&t.id).exists());
5459 std::fs::remove_dir(tmp).unwrap();
5460 }
5461
5462 #[test]
5463 fn a_failed_put_removes_only_the_copy_it_just_made() {
5464 let (dir, q) = queue();
5465 let mut t = task("edited");
5466 let first = source_file(dir.path(), "first.png", "1");
5467 q.attach_and_put(&mut t, &[first]).unwrap();
5468 block_put(&q, &t);
5469 let second = source_file(dir.path(), "second.png", "2");
5470 assert!(q.attach_and_put(&mut t, &[second]).is_err());
5471 assert_eq!(t.attachments, ["first.png"]);
5472 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5473 .unwrap()
5474 .flatten()
5475 .map(|e| e.file_name().to_string_lossy().into_owned())
5476 .collect();
5477 assert_eq!(on_disk, ["first.png"]);
5478 assert_eq!(q.get(&t.id).unwrap().attachments, ["first.png"]);
5479 }
5480
5481 #[test]
5482 fn a_leftover_removing_directory_is_swept_by_the_next_removal() {
5483 let (dir, q) = queue();
5484 let questions = Questions::at(dir.path().join("questions"));
5485 let gone = task("gone");
5486 let mut other = task("other");
5487 let mut live = task("live");
5488 q.put(&mut other).unwrap();
5489 q.put(&mut live).unwrap();
5490 let orphan = q.root.join(format!("{}.attachments.removing", gone.id));
5493 std::fs::create_dir_all(&orphan).unwrap();
5494 std::fs::write(orphan.join("shot.png"), "x").unwrap();
5495 let busy = q.root.join(format!("{}.attachments.removing", live.id));
5498 std::fs::create_dir_all(&busy).unwrap();
5499
5500 q.remove(&other.id, false, &questions).unwrap();
5501 assert!(!orphan.exists(), "an orphan is swept");
5502 assert!(busy.exists(), "a removal in progress is left alone");
5503 }
5504
5505 #[test]
5506 fn a_blocked_aside_rename_fails_the_removal_and_loses_nothing() {
5507 let (dir, q) = queue();
5508 let questions = Questions::at(dir.path().join("questions"));
5509 let mut t = task("stuck");
5510 let src = source_file(dir.path(), "shot.png", "x");
5511 q.attach_and_put(&mut t, &[src]).unwrap();
5512 let aside = q.root.join(format!("{}.attachments.removing", t.id));
5515 std::fs::create_dir_all(&aside).unwrap();
5516 std::fs::write(aside.join("old.png"), "o").unwrap();
5517 assert!(q.remove(&t.id, false, &questions).is_err());
5518 assert!(q.path_of(&t.id).exists());
5519 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5520 }
5521
5522 #[test]
5523 fn a_failed_record_removal_puts_the_attachments_back() {
5524 let (dir, q) = queue();
5525 let mut t = task("rollback");
5526 let src = source_file(dir.path(), "shot.png", "x");
5527 q.attach_and_put(&mut t, &[src]).unwrap();
5528 let err = q
5529 .remove_record_with_attachments(&t.id, |_| {
5530 Err(std::io::Error::other("injected failure"))
5531 })
5532 .unwrap_err();
5533 assert!(format!("{err:#}").contains("injected failure"));
5534 assert!(q.path_of(&t.id).exists());
5535 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5536 assert!(
5537 !q.root
5538 .join(format!("{}.attachments.removing", t.id))
5539 .exists()
5540 );
5541 }
5542
5543 #[test]
5544 fn editing_a_task_keeps_its_attachments() {
5545 let (dir, q) = queue();
5546 let src = source_file(dir.path(), "shot.png", "x");
5547 let mut t = task("editable");
5548 q.attach(&mut t, &[src]).unwrap();
5549 t.edit("new".to_owned(), "new text".to_owned()).unwrap();
5550 q.put(&mut t).unwrap();
5551 assert_eq!(q.get(&t.id).unwrap().attachments, ["shot.png"]);
5552 }
5553
5554 #[test]
5555 fn removing_a_task_deletes_its_attachments() {
5556 let (dir, q) = queue();
5557 let questions = Questions::at(dir.path().join("questions"));
5558 let src = source_file(dir.path(), "shot.png", "x");
5559 let mut t = task("doomed");
5560 q.attach(&mut t, &[src]).unwrap();
5561 q.put(&mut t).unwrap();
5562 assert!(q.attachments_dir(&t.id).is_dir());
5563 q.remove(&t.id, false, &questions).unwrap();
5564 assert!(!q.attachments_dir(&t.id).exists());
5565 assert!(q.list().is_empty());
5566 }
5567
5568 #[test]
5569 fn a_task_written_before_attachments_still_reads() {
5570 let (_dir, q) = queue();
5571 let mut t = task("old");
5572 q.put(&mut t).unwrap();
5573 let path = q.path_of(&t.id);
5574 let mut v: serde_json::Value =
5575 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5576 v.as_object_mut().unwrap().remove("attachments");
5577 std::fs::write(&path, v.to_string()).unwrap();
5578 assert!(q.get(&t.id).unwrap().attachments.is_empty());
5579 }
5580
5581 #[test]
5582 fn attachment_paths_are_absolute_even_when_the_root_is_relative() {
5583 let q = Queue::at(PathBuf::from("relative-queue"));
5584 let mut t = task("rel");
5585 t.attachments.push("shot.png".to_owned());
5586 let paths = q.attachment_paths(&t);
5587 assert!(paths[0].is_absolute(), "{}", paths[0].display());
5588 assert!(paths[0].ends_with(format!("{}.attachments/shot.png", t.id)));
5589 }
5590
5591 fn upgrade_parked(id: &str, started: &str) -> Task {
5592 let mut t = task(id);
5593 t.id = id.into();
5594 t.start(format!("run-{id}"));
5595 t.park("upgrade");
5596 t.record_pass_at(
5597 &format!("run-{id}"),
5598 0,
5599 Some(started.parse().unwrap()),
5600 1,
5601 Some(crate::run::ParkCause::Upgrade {
5602 from: "old".into(),
5603 to: None,
5604 }),
5605 );
5606 t
5607 }
5608
5609 #[test]
5610 fn upgrade_resume_order_follows_dispatch_claim_not_graph_start() {
5611 let stamp = |id: &str, dispatched: &str, started: &str| {
5613 let mut t = task(id);
5614 t.id = id.into();
5615 t.start(format!("run-{id}"));
5616 t.park("upgrade");
5617 t.dispatched_at = Some(dispatched.parse().unwrap());
5618 t.record_pass_at(
5619 &format!("run-{id}"),
5620 0,
5621 Some(started.parse().unwrap()),
5622 1,
5623 Some(crate::run::ParkCause::Upgrade {
5624 from: "old".into(),
5625 to: None,
5626 }),
5627 );
5628 t
5629 };
5630 let a = stamp("a", "2026-10-22T00:00:01Z", "2026-10-22T00:00:09Z");
5631 let b = stamp("b", "2026-10-22T00:00:02Z", "2026-10-22T00:00:03Z");
5632 assert!(dispatch_order(&a, &b).is_lt());
5633 }
5634
5635 #[test]
5636 fn upgrade_resume_order_survives_queue_reload_and_reversed_settlement() {
5637 let (_dir, q) = queue();
5638 let mut later = upgrade_parked("later", "2026-10-22T00:00:02Z");
5639 let mut earlier = upgrade_parked("earlier", "2026-10-22T00:00:01Z");
5640 earlier.priority = -10;
5641 let mut fresh = task("fresh");
5642 fresh.id = "newer".into();
5643 fresh.priority = 100;
5644 for t in [&mut fresh, &mut later, &mut earlier] {
5645 q.put(t).unwrap();
5646 }
5647 let reloaded = Queue::at(q.root().to_owned());
5648 let mut tasks = reloaded.list();
5649 tasks.sort_unstable_by(dispatch_order);
5650 assert_eq!(
5651 tasks.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5652 vec!["earlier", "later", "newer"]
5653 );
5654 assert_eq!(reloaded.next_runnable().unwrap().id, "earlier");
5655 assert_eq!(reloaded.get("earlier").unwrap().attempts, 0);
5656 }
5657
5658 #[test]
5659 fn upgrade_resume_legacy_order_uses_settlement_and_id() {
5660 let mut a = upgrade_parked("a", "2026-10-22T00:00:02Z");
5661 let mut b = upgrade_parked("b", "2026-10-22T00:00:01Z");
5662 for t in [&mut a, &mut b] {
5663 let mut value = serde_json::to_value(&*t).unwrap();
5664 value["passes"][0]
5665 .as_object_mut()
5666 .unwrap()
5667 .remove("started_at");
5668 *t = serde_json::from_value(value).unwrap();
5669 }
5670 assert!(dispatch_order(&a, &b).is_lt());
5671 b.passes[0].settled_at = a.passes[0].settled_at;
5672 assert!(dispatch_order(&a, &b).is_lt());
5673 }
5674
5675 #[test]
5676 fn upgrade_resume_key_excludes_other_parks_and_explicit_task_changes() {
5677 let original = upgrade_parked("a", "2026-10-22T00:00:01Z");
5678 assert!(original.upgrade_resume_key().is_some());
5679 for cause in [
5680 crate::run::ParkCause::Operator,
5681 crate::run::ParkCause::Question,
5682 crate::run::ParkCause::Interrupt {
5683 reason: "interrupt".into(),
5684 },
5685 crate::run::ParkCause::Unspecified,
5686 ] {
5687 let mut t = original.clone();
5688 t.passes[0].parked = Some(cause);
5689 assert!(t.upgrade_resume_key().is_none());
5690 let mut fresh = task("fresh");
5691 fresh.priority = 1;
5692 assert!(dispatch_order(&fresh, &t).is_lt());
5693 }
5694 let mut t = original.clone();
5695 t.release();
5696 assert!(t.upgrade_resume_key().is_none());
5697 t.edit("edited".into(), "new instruction".into()).unwrap();
5698 assert!(t.upgrade_resume_key().is_none());
5699 let mut t = original.clone();
5700 t.requeue();
5701 t.status = TaskStatus::Parked;
5702 assert!(t.upgrade_resume_key().is_none());
5703 let mut t = original.clone();
5704 t.start("different-run".into());
5705 t.park("no upgrade");
5706 assert!(t.upgrade_resume_key().is_none());
5707 }
5708
5709 #[test]
5710 fn explicitly_resumed_older_run_becomes_current_without_duplicate_history() {
5711 let dir = tempfile::tempdir().unwrap();
5712 let q = Queue::at(dir.path().join("queue"));
5713 let mut t = task("resume");
5714 t.start("A".into());
5715 t.record_pass("A", 0, 1, None);
5716 t.start("B".into());
5717 q.put(&mut t).unwrap();
5718 assert_eq!(t.runs, vec!["A", "B"]);
5719 t.start("A".into());
5720 t.record_pass("A", 1, 3, None);
5721 q.put(&mut t).unwrap();
5722 let saved = q.get(&t.id).unwrap();
5723 assert_eq!(saved.runs, vec!["B", "A"]);
5724 assert_eq!(saved.runs.last().map(String::as_str), Some("A"));
5725 assert_eq!(saved.attempts, 3);
5726 assert_eq!(saved.passes.len(), 2);
5727 assert_eq!(saved.passes[0].run, "A");
5728 assert_eq!(saved.passes[1].run, "A");
5729 }
5730
5731 #[test]
5732 fn repeated_start_records_one_run_and_separate_passes() {
5733 let mut t = task("resume");
5734 t.start("same-run".into());
5735 t.record_pass("same-run", 0, 1, None);
5736 t.start("same-run".into());
5737 t.record_pass("same-run", 1, 2, None);
5738 assert_eq!(t.runs, vec!["same-run"]);
5739 assert_eq!(t.attempts, 2);
5740 assert_eq!(t.passes.len(), 2);
5741 }
5742
5743 #[test]
5744 fn link_run_adds_a_run_once_and_touches_nothing_else() {
5745 let dir = tempfile::tempdir().unwrap();
5746 let queue = Queue::at(dir.path().join("queue"));
5747 let mut t = Task::new(
5748 "t".to_owned(),
5749 "do it".to_owned(),
5750 PathBuf::from("."),
5751 Source::Human,
5752 );
5753 queue.put(&mut t).unwrap();
5754 let before = queue.get(&t.id).unwrap();
5755
5756 let linked = queue.link_run(&t.id[..4], "20260930-092817-ec34").unwrap();
5757 assert_eq!(linked.runs, vec!["20260930-092817-ec34".to_owned()]);
5758 assert_eq!(linked.status, before.status);
5759 assert_eq!(linked.attempts, before.attempts);
5760 assert_eq!(linked.interrupt, before.interrupt);
5761
5762 let again = queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
5763 assert_eq!(again.runs.len(), 1, "linking twice must not duplicate");
5764 assert_eq!(queue.get(&t.id).unwrap().runs.len(), 1);
5765 assert!(queue.link_run("no-such-task", "r").is_err());
5766 }
5767
5768 #[test]
5769 fn put_keeps_a_run_linked_after_the_writer_took_its_snapshot() {
5770 let dir = tempfile::tempdir().unwrap();
5771 let queue = Queue::at(dir.path().join("queue"));
5772 let mut t = Task::new(
5773 "t".to_owned(),
5774 "do it".to_owned(),
5775 PathBuf::from("."),
5776 Source::Human,
5777 );
5778 queue.put(&mut t).unwrap();
5779 let mut snapshot = queue.get(&t.id).unwrap();
5781 queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
5782
5783 snapshot.start("20260930-000000-aaaa".to_owned());
5784 queue.put(&mut snapshot).unwrap();
5785
5786 let stored = queue.get(&t.id).unwrap();
5787 assert!(stored.runs.contains(&"20260930-092817-ec34".to_owned()));
5788 assert!(stored.runs.contains(&"20260930-000000-aaaa".to_owned()));
5789 assert_eq!(stored.attempts, 1);
5790 }
5791
5792 #[test]
5793 fn concurrent_links_and_daemon_saves_lose_nothing() {
5794 let dir = tempfile::tempdir().unwrap();
5795 let queue = Queue::at(dir.path().join("queue"));
5796 let mut t = Task::new(
5797 "t".to_owned(),
5798 "do it".to_owned(),
5799 PathBuf::from("."),
5800 Source::Human,
5801 );
5802 queue.put(&mut t).unwrap();
5803 let id = t.id.clone();
5804
5805 let linkers: Vec<_> = (0..4)
5806 .map(|n| {
5807 let (queue, id) = (queue.clone(), id.clone());
5808 std::thread::spawn(move || {
5809 for k in 0..10 {
5810 queue
5811 .link_run(&id, &format!("20260930-00000{n}-l{k:03}"))
5812 .unwrap();
5813 }
5814 })
5815 })
5816 .collect();
5817 let mut mine = queue.get(&id).unwrap();
5820 for k in 0..10 {
5821 mine.start(format!("20260930-000009-d{k:03}"));
5822 queue.put(&mut mine).unwrap();
5823 }
5824 for l in linkers {
5825 l.join().unwrap();
5826 }
5827
5828 let stored = queue.get(&id).unwrap();
5829 assert_eq!(stored.runs.len(), 50, "{:?}", stored.runs);
5830 assert_eq!(
5831 stored.attempts, 10,
5832 "linking never rewinds the daemon's work"
5833 );
5834 }
5835
5836 fn age_lock(path: &Path) {
5837 let f = std::fs::OpenOptions::new().write(true).open(path).unwrap();
5838 f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(60))
5839 .unwrap();
5840 }
5841
5842 #[test]
5843 fn concurrent_stale_takeover_yields_one_holder() {
5844 use std::sync::atomic::{AtomicUsize, Ordering};
5845 use std::sync::{Arc, Barrier};
5846 for _ in 0..5 {
5847 let dir = tempfile::tempdir().unwrap();
5848 let q = Queue::at(dir.path().to_path_buf());
5849 let lock = dir.path().join("t.write-lock");
5850 std::fs::write(&lock, "dead-0000").unwrap();
5851 age_lock(&lock);
5852 let n = 6;
5853 let barrier = Arc::new(Barrier::new(n));
5854 let (now, max) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
5855 let handles: Vec<_> = (0..n)
5856 .map(|_| {
5857 let (q, b, now, max) = (q.clone(), barrier.clone(), now.clone(), max.clone());
5858 std::thread::spawn(move || {
5859 b.wait();
5860 let g = q.lock_task("t").unwrap();
5861 let held = now.fetch_add(1, Ordering::SeqCst) + 1;
5862 max.fetch_max(held, Ordering::SeqCst);
5863 std::thread::sleep(std::time::Duration::from_millis(20));
5864 now.fetch_sub(1, Ordering::SeqCst);
5865 drop(g);
5866 })
5867 })
5868 .collect();
5869 for h in handles {
5870 h.join().unwrap();
5871 }
5872 assert_eq!(max.load(Ordering::SeqCst), 1);
5873 assert!(!lock.exists());
5874 }
5875 }
5876
5877 #[test]
5878 fn dropping_a_replaced_lock_leaves_the_new_holders_token() {
5879 let dir = tempfile::tempdir().unwrap();
5880 let q = Queue::at(dir.path().to_path_buf());
5881 let lock = dir.path().join("t.write-lock");
5882 let a = q.lock_task("t").unwrap();
5883 let replacement = owner_token();
5886 std::fs::write(&lock, &replacement).unwrap();
5887 drop(a);
5888 assert_eq!(std::fs::read_to_string(&lock).unwrap(), replacement);
5889 }
5890
5891 #[test]
5892 fn aged_live_task_lock_excludes_updates_until_retirement_releases_it() {
5893 let dir = tempfile::tempdir().unwrap();
5894 let q = Queue::at(dir.path().to_path_buf());
5895 let held = q.lock_task("t").unwrap();
5896 let path = dir.path().join("t.write-lock");
5897 age_lock(&path);
5898 assert!(!break_stale(&path, &held.token));
5899 let (started_tx, started_rx) = std::sync::mpsc::channel();
5900 let (acquired_tx, acquired_rx) = std::sync::mpsc::channel();
5901 let updater = std::thread::spawn(move || {
5902 started_tx.send(()).unwrap();
5903 let _write = q.lock_task("t").unwrap();
5904 acquired_tx.send(()).unwrap();
5905 });
5906 started_rx.recv().unwrap();
5907 assert!(
5908 acquired_rx
5909 .recv_timeout(std::time::Duration::from_millis(100))
5910 .is_err()
5911 );
5912 assert_eq!(std::fs::read_to_string(&path).unwrap(), held.token);
5913 drop(held);
5914 acquired_rx
5915 .recv_timeout(std::time::Duration::from_secs(2))
5916 .unwrap();
5917 updater.join().unwrap();
5918 assert!(!path.exists());
5919 }
5920
5921 #[test]
5922 fn break_stale_leaves_a_lock_that_replaced_the_one_judged() {
5923 let dir = tempfile::tempdir().unwrap();
5924 let lock = dir.path().join("t.write-lock");
5925 std::fs::write(&lock, "new-token").unwrap();
5926 assert!(!break_stale(&lock, "old-token"));
5927 assert_eq!(std::fs::read_to_string(&lock).unwrap(), "new-token");
5928 assert!(!break_stale(&lock, "new-token"));
5930 assert!(lock.exists());
5931 age_lock(&lock);
5932 assert!(break_stale(&lock, "new-token"));
5933 assert!(!lock.exists());
5934 }
5935
5936 #[test]
5937 fn a_stale_break_marker_is_recovered_and_a_fresh_one_is_respected() {
5938 let dir = tempfile::tempdir().unwrap();
5939 let lock = dir.path().join("t.write-lock");
5940 std::fs::write(&lock, "dead-1").unwrap();
5941 age_lock(&lock);
5942 let marker = break_marker(&lock, "dead-1");
5943 std::fs::write(&marker, "crashed-remover").unwrap();
5944 assert!(!break_stale(&lock, "dead-1"));
5946 assert!(lock.exists());
5947 age_lock(&marker);
5949 assert!(break_stale(&lock, "dead-1"));
5950 assert!(!lock.exists());
5951 assert!(!marker.exists());
5952 }
5953
5954 fn chat_task() -> Task {
5955 Task::new(
5956 "t".to_owned(),
5957 "do t".to_owned(),
5958 PathBuf::from("."),
5959 Source::Agent {
5960 run: "20260902-000000-beef".to_owned(),
5961 node: CHAT_NODE.to_owned(),
5962 },
5963 )
5964 }
5965
5966 #[test]
5967 fn a_blocked_hold_keeps_its_anchor_but_a_new_run_starts_a_new_hold() {
5968 let mut t = chat_task();
5969 t.hold_machine(Some("waiting".into()));
5970 let old: Timestamp = "2020-01-01T00:00:00Z".parse().unwrap();
5971 t.held_at = Some(old);
5972 t.block(vec!["decision".into()], None);
5973 t.hold_machine(Some("same waiting decision".into()));
5974 assert_eq!(t.held_at, Some(old));
5975 t.start("real-new-run".into());
5976 t.fail("new failure", 1);
5977 assert_ne!(
5978 t.held_at,
5979 Some(old),
5980 "old answers cannot approve a new hold"
5981 );
5982 }
5983
5984 #[test]
5985 fn legacy_delivery_survives_upgrade_and_a_no_progress_run_link() {
5986 let (_dir, queue) = queue();
5987 let mut t = chat_task();
5988 t.start("old-run".into());
5989 t.hold_machine(Some("operator is considering recovery".into()));
5990 t.chat_notice = Some(ChatNotice {
5991 key: format!("held#{}", t.held_at.unwrap()),
5992 at: Timestamp::now(),
5993 outcome: ChatNoticeOutcome::Posted,
5994 });
5995 let mut raw = serde_json::to_value(&t).unwrap();
5996 raw.as_object_mut().unwrap().remove("chat_notices");
5997 raw.as_object_mut().unwrap().remove("hold_notice_cause");
5998 std::fs::create_dir_all(queue.path_of(&t.id).parent().unwrap()).unwrap();
5999 std::fs::write(queue.path_of(&t.id), serde_json::to_string(&raw).unwrap()).unwrap();
6000 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
6001 queue
6002 .modify(&t.id, |task| {
6003 task.requeue();
6004 task.link_run("unprepared-run");
6005 task.startup_notice_cause("startup-source");
6006 task.fail("same unresolved preflight", 0);
6007 true
6008 })
6009 .unwrap();
6010 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
6011 queue
6012 .modify(&t.id, |task| {
6013 task.succeed();
6014 true
6015 })
6016 .unwrap();
6017 assert_eq!(
6018 queue.get(&t.id).unwrap().chat_notice_due().as_deref(),
6019 Some("done")
6020 );
6021 }
6022
6023 #[test]
6024 fn a_dependency_wait_does_not_change_when_released_and_blocked_again() {
6025 let mut t = chat_task();
6026 t.block(vec!["a".into(), "b".into()], Some("waiting".into()));
6027 let key = t.chat_notice_key().unwrap();
6028 t.chat_notices.push(key);
6029 t.release();
6030 t.block(
6031 vec!["b".into(), "a".into(), "a".into()],
6032 Some("still waiting on prerequisites".into()),
6033 );
6034 assert_eq!(t.chat_notice_due(), None);
6035 t.block(
6036 vec!["new-question".into()],
6037 Some("different decision".into()),
6038 );
6039 assert!(t.chat_notice_due().is_some());
6040 }
6041
6042 #[test]
6043 fn running_startup_failures_preserve_stage_and_report_distinct_blockers() {
6044 let mut t = chat_task();
6045 t.start("existing-run".into());
6046 t.fail("implementation failed", 1);
6047 let failed = t.chat_notice_key().unwrap();
6048 t.chat_notices.push(failed.clone());
6049 t.release_operator("test", None);
6050 t.start("existing-run".into());
6051 t.fail_startup("startup-source", "same unresolved recovery source", 1);
6052 assert_eq!(
6053 t.chat_notice_due(),
6054 None,
6055 "no-progress recovery stays quiet"
6056 );
6057 t.release_operator("test", None);
6058 t.start("existing-run".into());
6059 t.fail_startup("startup-attachments", "attachment missing", 1);
6060 let attachments = t.chat_notice_due().expect("new attachment blocker");
6061 assert_ne!(attachments, failed);
6062 assert_eq!(
6063 t.hold_notice_cause.as_ref().unwrap().stage,
6064 "startup-attachments"
6065 );
6066 t.chat_notices.push(attachments);
6067 t.release_operator("test", None);
6068 t.start("existing-run".into());
6069 t.fail_startup(
6070 "startup-attachments",
6071 "same missing attachment, reworded",
6072 1,
6073 );
6074 assert_eq!(
6075 t.chat_notice_due(),
6076 None,
6077 "same stage after attempt counter reset"
6078 );
6079 t.release_operator("test", None);
6080 t.start("existing-run".into());
6081 t.fail_startup("startup-config", "different preflight stage", 1);
6082 assert!(t.chat_notice_due().is_some());
6083 assert_eq!(
6084 t.hold_notice_cause.as_ref().unwrap().stage,
6085 "startup-config"
6086 );
6087 }
6088
6089 #[test]
6090 fn structured_causes_preserve_no_progress_retries_and_report_new_passes() {
6091 let mut t = chat_task();
6092 t.hold_machine(Some("startup source missing".into()));
6093 t.startup_notice_cause("startup-source");
6094 let first = t.chat_notice_key().unwrap();
6095 t.chat_notices.push(first.clone());
6096 t.release_operator("test", None);
6097 t.startup_notice_cause("startup-source");
6098 t.hold_machine(Some("same source cannot be read".into()));
6099 assert_eq!(t.chat_notice_due(), None);
6100 t.startup_notice_cause("startup-config");
6101 assert!(t.chat_notice_due().is_some(), "different startup stage");
6102 t.notice_cause("failed", Some("existing-run"), Some(0));
6103 let run_failure = t.chat_notice_key().unwrap();
6104 t.chat_notices.push(run_failure.clone());
6105 t.release_operator("test", None);
6106 t.startup_notice_cause("startup-source");
6107 t.hold_machine(Some("no new work happened".into()));
6108 assert_eq!(t.chat_notice_due(), None);
6109 t.notice_cause("failed", Some("existing-run"), Some(1));
6110 assert!(t.chat_notice_due().is_some(), "a real resumed pass failed");
6111 assert_ne!(t.chat_notice_key().unwrap(), run_failure);
6112 }
6113
6114 #[test]
6115 fn legacy_hold_baseline_does_not_replay_or_hide_new_failures_and_completion() {
6116 let mut t = chat_task();
6117 t.hold_machine(Some("still waiting".into()));
6118 t.chat_notice = Some(ChatNotice {
6119 key: "held#2026-10-14T00:00:00Z".into(),
6120 at: Timestamp::now(),
6121 outcome: ChatNoticeOutcome::Posted,
6122 });
6123 assert_eq!(t.chat_notice_due(), None, "old hold baseline");
6124 t.import_chat_notice();
6125 t.notice_cause("failed", Some("new-run"), Some(0));
6126 assert!(t.chat_notice_due().is_some(), "new producer evidence");
6127 t.succeed();
6128 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
6129 }
6130
6131 #[test]
6132 fn a_chat_notice_is_due_only_for_a_final_state_and_only_once() {
6133 let mut t = chat_task();
6134 assert_eq!(t.chat_notice_due(), None, "queued");
6135 t.start("r1".to_owned());
6136 assert_eq!(t.chat_notice_due(), None, "running");
6137 t.fail("boom", 3);
6138 assert_eq!(t.status, TaskStatus::Failed);
6139 assert_eq!(t.chat_notice_due(), None, "a retried failure is not final");
6140 t.start("r2".to_owned());
6141 t.park("upgrade");
6142 assert_eq!(t.chat_notice_due(), None, "parked");
6143 t.fail("boom", 1);
6144 assert_eq!(t.status, TaskStatus::Held);
6145 let key = t
6146 .chat_notice_due()
6147 .expect("held on the last attempt is final");
6148 t.chat_notice = Some(ChatNotice {
6149 key,
6150 at: Timestamp::now(),
6151 outcome: ChatNoticeOutcome::Posted,
6152 });
6153 assert_eq!(t.chat_notice_due(), None, "same hold, no second notice");
6154 t.release();
6155 assert_eq!(t.chat_notice_due(), None, "requeued after a release");
6156 t.start("r3".to_owned());
6157 t.succeed();
6158 assert!(t.chat_notice_due().is_some(), "done is a new final state");
6159 }
6160
6161 #[test]
6162 fn completed_chat_notices_keep_current_and_legacy_keys_after_requeue() {
6163 for key in ["done", "done#0", "done#1", "done#12"] {
6164 let mut t = chat_task();
6165 t.start("r1".to_owned());
6166 t.succeed();
6167 t.chat_notice = Some(ChatNotice {
6168 key: key.to_owned(),
6169 at: Timestamp::now(),
6170 outcome: ChatNoticeOutcome::Posted,
6171 });
6172 t.requeue();
6173 t.start("r2".to_owned());
6174 t.succeed();
6175 assert_eq!(t.chat_notice_key().as_deref(), Some("done"));
6176 assert_eq!(t.chat_notice_due(), None, "recorded key: {key}");
6177 }
6178 }
6179
6180 #[test]
6181 fn a_release_without_progress_keeps_the_unresolved_notice_cause() {
6182 let mut t = chat_task();
6183 t.hold_manual(Some("wait".to_owned()));
6184 let first = t.chat_notice_due().expect("held");
6185 t.chat_notice = Some(ChatNotice {
6186 key: first.clone(),
6187 at: Timestamp::now(),
6188 outcome: ChatNoticeOutcome::Posted,
6189 });
6190 t.hold_manual(Some("still waiting".to_owned()));
6191 assert_eq!(t.chat_notice_due(), None, "a re-hold keeps its start");
6192 t.release();
6193 std::thread::sleep(std::time::Duration::from_millis(5));
6194 t.hold_manual(None);
6195 assert_eq!(t.chat_notice_due(), None, "no progress, same cause");
6196 }
6197
6198 #[test]
6199 fn a_chat_notice_needs_a_chat_source_and_a_current_schema() {
6200 let mut human = task("h");
6201 human.succeed();
6202 assert_eq!(human.chat_notice_due(), None);
6203 let mut inherited = task("i");
6205 inherited.origin_chat = Some("20260902-000000-beef".to_owned());
6206 inherited.succeed();
6207 assert_eq!(inherited.chat_notice_due(), None);
6208 let mut old = chat_task();
6209 old.schema = 12;
6210 old.status = TaskStatus::Done;
6211 assert_eq!(old.chat_notice_due(), None, "history is not replayed");
6212 let mut live = chat_task();
6213 live.schema = 12;
6214 live.start("r1".to_owned());
6215 live.succeed();
6216 assert!(live.chat_notice_due().is_some(), "started by this build");
6217 let mut held = chat_task();
6218 held.schema = 12;
6219 held.hold_machine(Some("config".to_owned()));
6220 assert!(held.chat_notice_due().is_some(), "held before any start");
6221 }
6222
6223 #[test]
6224 fn a_stale_snapshot_does_not_rewind_a_recorded_chat_notice() {
6225 let tmp = tempfile::tempdir().expect("tempdir");
6226 let q = Queue::at(tmp.path().join("queue"));
6227 let mut t = chat_task();
6228 t.succeed();
6229 q.put(&mut t).expect("put");
6230 let mut stale = q.get(&t.id).expect("get");
6231 let key = t.chat_notice_due().expect("due");
6232 q.modify(&t.id, |x| {
6233 x.chat_notice = Some(ChatNotice {
6234 key: key.clone(),
6235 at: Timestamp::now(),
6236 outcome: ChatNoticeOutcome::Posted,
6237 });
6238 true
6239 })
6240 .expect("modify");
6241 q.put(&mut stale).expect("stale put");
6242 assert_eq!(q.get(&t.id).expect("get").chat_notice_due(), None);
6243 }
6244
6245 #[test]
6246 fn empty_sources_are_omitted_so_older_builds_still_read_the_task() {
6247 let mut t = task("plain");
6250 let value = serde_json::to_value(&t).unwrap();
6251 assert!(value.get("sources").is_none(), "{value}");
6252 t.sources.pick_commits.push(crate::refs::Input {
6253 token: "abc1234".into(),
6254 sha: "a".repeat(40),
6255 provenance: crate::refs::Provenance::ExplicitCli,
6256 });
6257 let value = serde_json::to_value(&t).unwrap();
6258 assert_eq!(value["sources"]["pick_commits"][0]["sha"], "a".repeat(40));
6259 let restored: Task = serde_json::from_value(value).unwrap();
6260 assert_eq!(restored.sources, t.sources);
6261 }
6262
6263 fn queue_with_ids(dir: &Path, ids: &[&str]) -> Queue {
6265 let ids: Vec<String> = ids.iter().map(|s| (*s).to_owned()).collect();
6266 let n = std::sync::atomic::AtomicUsize::new(0);
6267 Queue::at(dir.join("queue")).with_id_source(move || {
6268 let i = n.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6269 ids.get(i)
6270 .cloned()
6271 .unwrap_or_else(|| format!("20260101-000000-f{i:03x}"))
6272 })
6273 }
6274
6275 const TAKEN: &str = "20260101-000000-aaaa";
6276
6277 #[test]
6278 fn a_new_task_never_merges_into_a_record_with_its_candidate_id() {
6279 let dir = tempfile::tempdir().unwrap();
6280 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6281 let mut first = task("first");
6282 first.start("run-1".into());
6283 first.succeed();
6284 q.put(&mut first).unwrap();
6285 assert_eq!(first.id, TAKEN);
6286 let stored_before = q.get(&first.id).unwrap();
6287 assert!(!stored_before.runs.is_empty());
6288
6289 let mut second = task("second");
6290 q.put(&mut second).unwrap();
6291
6292 assert_ne!(second.id, first.id);
6293 assert!(second.runs.is_empty());
6294 assert!(second.passes.is_empty());
6295 assert!(second.chat_notices.is_empty());
6296 assert!(second.chat_notice.is_none());
6297 assert_eq!(second.chat_notice_due(), task("fresh").chat_notice_due());
6298 let a = q.get(&first.id).unwrap();
6299 let b = q.get(&second.id).unwrap();
6300 assert_eq!(a.title, "first");
6301 assert_eq!(a.runs, stored_before.runs);
6302 assert_eq!(a.passes, stored_before.passes);
6303 assert_eq!(a.chat_notices, stored_before.chat_notices);
6304 assert_eq!(a.chat_notice, stored_before.chat_notice);
6305 assert_eq!(a.chat_notice_due(), stored_before.chat_notice_due());
6306 assert_eq!(b.title, "second");
6307 assert!(b.runs.is_empty());
6308 assert!(b.passes.is_empty());
6309 assert!(b.chat_notice.is_none());
6310
6311 let mut again = q.get(&second.id).unwrap();
6313 again.title = "second, edited".into();
6314 q.put(&mut again).unwrap();
6315 assert_eq!(q.get(&second.id).unwrap().title, "second, edited");
6316 assert_eq!(q.list().len(), 2);
6317 }
6318
6319 #[test]
6320 fn a_new_task_skips_an_id_with_an_orphaned_claim_or_attachments() {
6321 let dir = tempfile::tempdir().unwrap();
6322 let q = queue_with_ids(dir.path(), &["orphan-lock", "orphan-dir", "free"]);
6323 std::fs::create_dir_all(q.root()).unwrap();
6324 std::fs::write(q.root().join("orphan-lock.lock"), "x").unwrap();
6325 std::fs::create_dir_all(q.attachments_dir("orphan-dir")).unwrap();
6326 let mut t = task("t");
6327 q.put(&mut t).unwrap();
6328 assert_eq!(t.id, "free");
6329 }
6330
6331 #[test]
6332 fn a_claimed_new_task_holds_the_claim_on_the_reserved_id() {
6333 let dir = tempfile::tempdir().unwrap();
6334 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6335 let mut first = task("first");
6336 q.put(&mut first).unwrap();
6337 let mut second = task("second");
6338 let claim = q.file_claimed(&mut second).unwrap();
6339 assert_ne!(second.id, first.id);
6340 assert!(q.claim(&second.id).is_err());
6341 assert!(q.claim(&first.id).is_ok(), "the first task was not claimed");
6342 drop(claim);
6343 }
6344
6345 #[test]
6346 fn an_attached_new_task_skips_a_taken_id() {
6347 let dir = tempfile::tempdir().unwrap();
6348 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6349 let mut first = task("first");
6350 q.put(&mut first).unwrap();
6351 let src = source_file(dir.path(), "shot.png", "x");
6352 let mut second = task("second");
6353 q.attach_and_put(&mut second, &[src]).unwrap();
6354 assert_ne!(second.id, first.id);
6355 assert!(!q.attachments_dir(&first.id).exists());
6356 assert_eq!(q.attachment_paths(&second).len(), 1);
6357 assert!(q.attachment_paths(&second)[0].is_file());
6358 }
6359
6360 #[test]
6361 fn attaching_before_putting_a_new_task_reserves_the_id() {
6362 let dir = tempfile::tempdir().unwrap();
6363 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6364 let mut first = task("first");
6365 q.put(&mut first).unwrap();
6366 let src = source_file(dir.path(), "shot.png", "x");
6367 let mut second = task("second");
6368 q.attach(&mut second, &[src]).unwrap();
6369 assert_ne!(second.id, first.id);
6370 assert!(q.attachments_dir(&second.id).is_dir());
6371 q.put(&mut second).unwrap();
6372 assert_eq!(q.get(&second.id).unwrap().attachments.len(), 1);
6373 assert_eq!(q.get(&first.id).unwrap().title, "first");
6374 assert!(q.get(&first.id).unwrap().attachments.is_empty());
6375 }
6376
6377 #[test]
6378 fn create_new_keeps_its_already_exists_meaning() {
6379 let dir = tempfile::tempdir().unwrap();
6380 let q = Queue::at(dir.path().join("queue"));
6381 let mut a = task("a");
6382 a.id = "fixed-id".into();
6383 assert!(q.create_new(&mut a).unwrap());
6384 let mut b = task("b");
6385 b.id = "fixed-id".into();
6386 assert!(!q.create_new(&mut b).unwrap());
6387 assert_eq!(q.get("fixed-id").unwrap().title, "a");
6388 }
6389
6390 #[test]
6391 fn filing_gives_up_when_no_id_is_ever_free() {
6392 let dir = tempfile::tempdir().unwrap();
6393 let q = Queue::at(dir.path().join("queue")).with_id_source(|| TAKEN.to_owned());
6394 let mut first = task("first");
6395 q.put(&mut first).unwrap();
6396 let mut second = task("second");
6397 assert!(q.put(&mut second).is_err());
6398 assert_eq!(q.get(&first.id).unwrap().title, "first");
6399 }
6400}