1use std::collections::HashMap;
34use std::num::NonZeroUsize;
35use std::path::{Path, PathBuf};
36
37use anyhow::{Context, Result, bail};
38use jiff::Timestamp;
39use serde::{Deserialize, Serialize};
40
41use crate::ask::Questions;
42use crate::id_reserve::{IdSource, path_exists};
43
44pub const SCHEMA: u32 = 21;
138
139pub(crate) fn dispatch_order(a: &Task, b: &Task) -> std::cmp::Ordering {
145 match (a.upgrade_resume_key(), b.upgrade_resume_key()) {
146 (Some(a), Some(b)) => a.cmp(&b),
147 (Some(_), None) => std::cmp::Ordering::Less,
148 (None, Some(_)) => std::cmp::Ordering::Greater,
149 (None, None) => b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)),
150 }
151}
152
153#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
155#[serde(rename_all = "lowercase")]
156pub enum HoldSource {
157 Manual,
159 Machine,
161}
162
163impl HoldSource {
164 pub fn label(self) -> &'static str {
166 match self {
167 Self::Manual => "manual",
168 Self::Machine => "machine",
169 }
170 }
171}
172
173#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
176#[serde(tag = "kind", rename_all = "lowercase")]
177pub enum Source {
178 Human,
180 Agent {
183 run: String,
185 node: String,
187 },
188 Issue {
190 number: u64,
192 repo: String,
194 },
195}
196
197pub const CHAT_NODE: &str = "chat";
200
201impl Source {
202 pub fn label(&self) -> String {
204 match self {
205 Self::Human => "human".to_owned(),
206 Self::Agent { run, node } => format!("{node}@{}", short(run)),
207 Self::Issue { number, .. } => format!("issue #{number}"),
208 }
209 }
210}
211
212#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
214#[serde(rename_all = "lowercase")]
215pub enum TaskStatus {
216 Queued,
218 Running,
220 Done,
222 Failed,
224 Held,
226 Blocked,
230 Parked,
235}
236
237impl TaskStatus {
238 pub fn runnable(self) -> bool {
240 matches!(self, Self::Queued | Self::Failed | Self::Parked)
241 }
242
243 pub fn as_str(self) -> &'static str {
245 match self {
246 Self::Queued => "queued",
247 Self::Running => "running",
248 Self::Done => "done",
249 Self::Failed => "failed",
250 Self::Held => "held",
251 Self::Blocked => "blocked",
252 Self::Parked => "parked",
253 }
254 }
255}
256
257#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
260pub struct TaskCounts {
261 pub queued: usize,
263 pub running: usize,
265 pub done: usize,
267 pub failed: usize,
269 pub held: usize,
271 pub blocked: usize,
273 pub parked: usize,
275}
276
277impl TaskCounts {
278 pub fn of(tasks: &[Task]) -> Self {
282 Self::of_statuses(tasks.iter().map(|task| task.status))
283 }
284
285 pub(crate) fn of_statuses(statuses: impl IntoIterator<Item = TaskStatus>) -> Self {
287 let mut counts = Self::default();
288 for status in statuses {
289 match status {
290 TaskStatus::Queued => counts.queued += 1,
291 TaskStatus::Running => counts.running += 1,
292 TaskStatus::Done => counts.done += 1,
293 TaskStatus::Failed => counts.failed += 1,
294 TaskStatus::Held => counts.held += 1,
295 TaskStatus::Blocked => counts.blocked += 1,
296 TaskStatus::Parked => counts.parked += 1,
297 }
298 }
299 counts
300 }
301}
302
303#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
305pub struct Reassignment {
306 pub from: PathBuf,
308 pub to: PathBuf,
310 pub actor: String,
312 pub at: Timestamp,
314 pub run_boundary: usize,
316}
317
318#[derive(Debug, Clone, Serialize, Deserialize)]
320#[serde(deny_unknown_fields)]
321pub struct Task {
322 pub schema: u32,
324 pub id: String,
326 pub title: String,
328 pub instruction: String,
330 #[serde(default, skip_serializing_if = "crate::refs::Sources::is_empty")]
332 pub sources: crate::refs::Sources,
333 pub repo: PathBuf,
335 #[serde(default)]
337 pub reassignments: Vec<Reassignment>,
338 pub source: Source,
340 #[serde(default)]
342 pub chat: Option<String>,
343 #[serde(default)]
345 pub priority: i32,
346 #[serde(default)]
357 pub solo: bool,
358 pub status: TaskStatus,
360 #[serde(default)]
362 pub attempts: usize,
363 #[serde(default)]
365 pub attempt_limit: Option<NonZeroUsize>,
366 #[serde(default)]
369 pub runs: Vec<String>,
370 #[serde(default)]
372 pub startup_failures: Vec<crate::run::StartupFailure>,
373 #[serde(default)]
375 pub last_error: Option<String>,
376 #[serde(default)]
389 pub hold_reason: Option<String>,
390 #[serde(default)]
393 pub hold_source: Option<HoldSource>,
394 #[serde(default)]
406 pub diagnostic: Option<String>,
407 #[serde(default)]
417 pub blocked_by: Vec<String>,
418 #[serde(default)]
421 pub block_reason: Option<String>,
422 #[serde(default)]
435 pub blocked_from: Option<TaskStatus>,
436 #[serde(default)]
438 pub blocked_at: Option<Timestamp>,
439 #[serde(default)]
450 pub answers: Vec<AnsweredQuestion>,
451 #[serde(default)]
457 pub triage_applied: Vec<String>,
458 #[serde(default)]
464 pub actions_applied: Vec<String>,
465 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
468 pub decision_effects: std::collections::BTreeMap<String, String>,
469 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
472 pub superseded_resumes: std::collections::BTreeMap<String, String>,
473 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
477 pub dispatched_resumes: std::collections::BTreeMap<String, String>,
478 #[serde(default)]
483 pub resume_override: Option<OperatorResume>,
484 #[serde(default)]
486 pub repair_grants: Vec<crate::repair_budget::Receipt>,
487 #[serde(default)]
490 pub resumption: Option<Resumption>,
491 #[serde(default)]
493 pub resumption_history: Vec<Resumption>,
494 #[serde(default)]
496 pub operator_decided_at: Option<Timestamp>,
497 #[serde(default)]
509 pub review_branch: Option<String>,
510 #[serde(default)]
512 pub review_instruction: Option<String>,
513 #[serde(default)]
515 pub pr_recovery: Option<crate::pr_recovery::Selection>,
516 #[serde(default)]
519 pub fresh_start: bool,
520 #[serde(default)]
522 pub replacement: Option<crate::merge_decision::Replacement>,
523 #[serde(default)]
534 pub interrupt: bool,
535 #[serde(default)]
558 pub urgent: bool,
559 #[serde(skip)]
564 pub(crate) unfiled: bool,
565 #[serde(default)]
571 pub attachments: Vec<String>,
572 #[serde(default)]
576 pub followup: Option<FollowUp>,
577 #[serde(default)]
583 pub origin_chat: Option<String>,
584 #[serde(default)]
591 pub overrides: Option<RunOverrides>,
592 #[serde(default)]
599 pub review_of: Option<String>,
600 #[serde(default)]
605 pub held_at: Option<Timestamp>,
606 #[serde(default)]
610 pub park_reason: Option<String>,
611 #[serde(default)]
614 pub chat_notice: Option<ChatNotice>,
615 #[serde(default)]
617 pub chat_notices: Vec<String>,
618 #[serde(default)]
621 pub hold_notice_cause: Option<NoticeCause>,
622 #[serde(default)]
627 pub passes: Vec<TaskPass>,
628 #[serde(default)]
632 pub dispatched_at: Option<Timestamp>,
633 #[serde(default)]
635 pub recovery_budget: Option<RecoveryBudget>,
636 #[serde(default)]
639 pub recovery_admission: Option<RecoveryAdmission>,
640 #[serde(default)]
643 pub recovery_generations: Vec<RecoveryBudget>,
644 pub created_at: Timestamp,
646 pub updated_at: Timestamp,
648}
649
650#[derive(Debug, Clone, Serialize, Deserialize)]
652pub struct RecoveryAdmission {
653 pub base: String,
655 pub inputs: String,
657}
658
659#[derive(Debug, Clone, Serialize, Deserialize)]
662pub struct RecoveryBudget {
663 pub instruction: String,
665 pub base: String,
667 pub inputs: String,
669 pub stage: String,
671 pub failures: usize,
673 #[serde(default)]
676 pub progress: Option<String>,
677}
678
679fn recovery_progress(state: &crate::run::RunState) -> Option<String> {
680 use crate::proc::Quiet as _;
681 use std::hash::{Hash, Hasher};
682 let mut patches = Vec::new();
683 for candidate in state.candidates.iter().filter(|c| c.commits > 0) {
684 let output = std::process::Command::new("git")
685 .current_dir(&state.repo)
686 .args(["diff", "--binary", &state.base_commit, &candidate.branch])
687 .quiet()
688 .output()
689 .ok()?;
690 if !output.status.success() {
691 return None;
692 }
693 patches.push(output.stdout);
694 }
695 patches.sort();
696 patches.dedup();
697 let mut hash = std::collections::hash_map::DefaultHasher::new();
698 patches.hash(&mut hash);
699 Some(format!("{:x}", hash.finish()))
700}
701
702pub(crate) fn recovery_inputs(sources: &crate::refs::Sources) -> String {
706 serde_json::to_string(
707 &sources
708 .start_from
709 .iter()
710 .map(|i| ("start", &i.token, &i.sha))
711 .chain(
712 sources
713 .pick_commits
714 .iter()
715 .map(|i| ("pick", &i.token, &i.sha)),
716 )
717 .collect::<Vec<_>>(),
718 )
719 .unwrap_or_default()
720}
721
722#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
724pub struct NoticeCause {
725 pub stage: String,
727 #[serde(default, skip_serializing_if = "Option::is_none")]
730 pub target: Option<String>,
731 pub run: Option<String>,
733 pub pass: Option<usize>,
735}
736
737pub(crate) fn is_done_notice_key(key: &str) -> bool {
739 key == "done"
740 || key
741 .strip_prefix("done#")
742 .is_some_and(|count| !count.is_empty() && count.bytes().all(|b| b.is_ascii_digit()))
743}
744
745#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
747#[serde(rename_all = "snake_case")]
748pub enum PassCost {
749 Spent,
751 Refunded,
753}
754
755#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
757pub struct TaskPass {
758 pub run: String,
760 pub pass_index: usize,
762 pub cost: PassCost,
764 pub attempts_after: usize,
766 #[serde(default)]
768 pub started_at: Option<Timestamp>,
769 pub settled_at: Timestamp,
771 #[serde(default)]
773 pub parked: Option<crate::run::ParkCause>,
774}
775
776#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
778#[serde(rename_all = "lowercase")]
779pub enum ChatNoticeOutcome {
780 Posted,
782 Skipped,
784}
785
786#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
788pub struct ChatNotice {
789 pub key: String,
791 pub at: Timestamp,
793 pub outcome: ChatNoticeOutcome,
795}
796
797#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
799pub struct RunOverrides {
800 #[serde(default)]
803 pub inherit_daemon_options: bool,
804 #[serde(default)]
806 pub merge: Option<String>,
807 #[serde(default)]
809 pub candidates: Option<usize>,
810 #[serde(default)]
812 pub judges: Option<usize>,
813 #[serde(default)]
815 pub reviewers: Option<usize>,
816 #[serde(default)]
818 pub review_rounds: Option<usize>,
819 #[serde(default)]
821 pub seed: Option<u64>,
822 #[serde(default)]
824 pub config: Option<PathBuf>,
825}
826
827impl RunOverrides {
828 pub fn merge_over(&mut self, other: &RunOverrides) {
830 macro_rules! take {
831 ($($f:ident),*) => {$(
832 if other.$f.is_some() {
833 self.$f.clone_from(&other.$f);
834 }
835 )*};
836 }
837 take!(
838 merge,
839 candidates,
840 judges,
841 reviewers,
842 review_rounds,
843 seed,
844 config
845 );
846 }
847
848 pub fn apply(&self, config: &mut crate::config::Config) {
850 if let Some(n) = self.candidates {
851 config.graph.implementers = n;
852 }
853 if let Some(n) = self.judges {
854 config.graph.judges = n;
855 }
856 if let Some(n) = self.reviewers {
857 config.graph.reviewers = n;
858 }
859 if let Some(n) = self.review_rounds {
860 config.graph.review_rounds = n;
861 }
862 if let Some(m) = &self.merge
863 && let Ok(mode) = crate::daemon::merge_mode(m)
864 {
865 config.merge.mode = mode;
866 }
867 if let Some(s) = self.seed {
868 config.blind.seed = Some(s);
869 }
870 }
871}
872
873#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
875pub struct FollowUp {
876 pub run: String,
878 #[serde(default)]
880 pub origin_task: Option<String>,
881 pub pr: String,
883 pub findings: Vec<String>,
885 pub generation: u32,
888}
889
890#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
892pub struct Resumption {
893 pub generation: u64,
895 pub source: String,
897 pub question_id: Option<String>,
899 pub at: Timestamp,
901 pub instruction: String,
903 pub answers_through: usize,
905 pub runs_through: usize,
908 pub cleared_reason: Option<String>,
910 #[serde(default)]
912 pub blocking_evidence: Vec<String>,
913}
914
915#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
918pub struct OperatorResume {
919 pub question_id: String,
921 pub at: Timestamp,
923 #[serde(default)]
926 pub conductor_rehold: Option<String>,
927 #[serde(default)]
930 pub forced: bool,
931 #[serde(default)]
938 pub pinned_run: Option<String>,
939}
940
941#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
944pub struct AnsweredQuestion {
945 pub question: String,
947 pub answer: String,
949}
950
951impl Task {
952 pub fn new(title: String, instruction: String, repo: PathBuf, source: Source) -> Self {
954 let now = Timestamp::now();
955 let origin_chat = match &source {
956 Source::Agent { run, node } if node == CHAT_NODE => Some(run.clone()),
957 _ => None,
958 };
959 Self {
960 schema: SCHEMA,
961 id: new_id(),
962 title,
963 instruction,
964 sources: crate::refs::Sources::default(),
965 repo,
966 reassignments: Vec::new(),
967 source,
968 chat: None,
969 priority: 0,
970 solo: false,
971 status: TaskStatus::Queued,
972 attempts: 0,
973 attempt_limit: None,
974 runs: Vec::new(),
975 startup_failures: Vec::new(),
976 last_error: None,
977 hold_reason: None,
978 hold_source: None,
979 diagnostic: None,
980 blocked_by: Vec::new(),
981 block_reason: None,
982 blocked_from: None,
983 blocked_at: None,
984 answers: Vec::new(),
985 triage_applied: Vec::new(),
986 actions_applied: Vec::new(),
987 decision_effects: Default::default(),
988 superseded_resumes: Default::default(),
989 dispatched_resumes: Default::default(),
990 resume_override: None,
991 repair_grants: Vec::new(),
992 resumption: None,
993 resumption_history: Vec::new(),
994 operator_decided_at: None,
995 review_branch: None,
996 review_instruction: None,
997 pr_recovery: None,
998 fresh_start: false,
999 replacement: None,
1000 interrupt: false,
1001 urgent: false,
1002 unfiled: true,
1003 attachments: Vec::new(),
1004 followup: None,
1005 origin_chat,
1006 overrides: None,
1007 review_of: None,
1008 held_at: None,
1009 park_reason: None,
1010 chat_notice: None,
1011 chat_notices: Vec::new(),
1012 hold_notice_cause: None,
1013 passes: Vec::new(),
1014 dispatched_at: None,
1015 recovery_budget: None,
1016 recovery_admission: None,
1017 recovery_generations: Vec::new(),
1018 created_at: now,
1019 updated_at: now,
1020 }
1021 }
1022
1023 pub(crate) fn reassign_repository(&mut self, repo: PathBuf, actor: &str) {
1025 self.stamp_schema();
1026 if matches!(self.status, TaskStatus::Failed | TaskStatus::Parked) {
1027 self.hold_manual(Some(
1028 "Repository reassigned; inspect history and release to start fresh.".into(),
1029 ));
1030 }
1031 let at = Timestamp::now();
1032 self.reassignments.push(Reassignment {
1033 from: self.repo.clone(),
1034 to: repo.clone(),
1035 actor: actor.to_owned(),
1036 at,
1037 run_boundary: self.runs.len(),
1038 });
1039 self.repo = repo;
1040 self.review_branch = None;
1041 self.review_instruction = None;
1042 self.pr_recovery = None;
1043 self.replacement = None;
1044 self.interrupt = false;
1045 self.resume_override = None;
1046 self.fresh_start = true;
1047 self.operator_decided_at = Some(at);
1048 }
1049
1050 pub fn publication_branch(&self) -> String {
1052 crate::run::task_pr_branch_name(&self.id, self.reassignments.len())
1053 }
1054
1055 pub fn execution_runs(&self) -> &[String] {
1057 let boundary = self.reassignments.last().map_or(0, |r| r.run_boundary);
1058 self.runs.get(boundary..).unwrap_or_default()
1059 }
1060
1061 pub fn execution_run(&self, run: &str) -> bool {
1063 self.execution_runs().iter().any(|id| id == run)
1064 }
1065
1066 pub(crate) fn assignment_accepts(&self, run: &crate::run::RunState) -> bool {
1068 self.reassignments.last().is_none_or(|boundary| {
1069 self.execution_run(&run.id)
1070 && run.task_assignment == self.reassignments.len()
1071 && run.created_at > boundary.at
1072 && run
1073 .repo
1074 .canonicalize()
1075 .ok()
1076 .zip(self.repo.canonicalize().ok())
1077 .is_some_and(|(run_repo, task_repo)| run_repo == task_repo)
1078 })
1079 }
1080
1081 pub(crate) fn stale_question(&self, q: &crate::ask::Question) -> bool {
1083 self.reassignments
1084 .last()
1085 .is_some_and(|r| q.asked_at <= r.at)
1086 || (self.runs.contains(&q.run) && !self.execution_run(&q.run))
1087 }
1088
1089 pub fn chat_talk(&self) -> Option<&str> {
1091 if let Some(id) = &self.origin_chat {
1092 return Some(id);
1093 }
1094 match &self.source {
1095 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
1096 _ => None,
1097 }
1098 }
1099
1100 pub fn filed_by_chat(&self) -> Option<&str> {
1104 match &self.source {
1105 Source::Agent { run, node } if node == CHAT_NODE => Some(run),
1106 _ => None,
1107 }
1108 }
1109
1110 pub fn chat_notice_key(&self) -> Option<String> {
1113 match self.status {
1114 TaskStatus::Done => Some("done".to_owned()),
1115 TaskStatus::Held => {
1116 let cause = self
1117 .hold_notice_cause
1118 .clone()
1119 .unwrap_or_else(|| self.unresolved_hold_cause());
1120 Some(format!(
1121 "cause:{}",
1122 serde_json::to_string(&cause).expect("cause")
1123 ))
1124 }
1125 TaskStatus::Blocked => {
1126 let ids: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1127 let evidence = self
1130 .hold_notice_cause
1131 .as_ref()
1132 .filter(|c| c.stage.starts_with("conductor-evidence:"));
1133 Some(format!(
1134 "dependencies:{}",
1135 serde_json::to_string(&(ids, evidence)).expect("ids")
1136 ))
1137 }
1138 _ => None,
1139 }
1140 }
1141
1142 fn unresolved_hold_cause(&self) -> NoticeCause {
1143 NoticeCause {
1144 stage: "hold".into(),
1145 target: None,
1146 run: self.runs.last().cloned(),
1147 pass: None,
1148 }
1149 }
1150
1151 pub(crate) fn legacy_chat_notice_key(&self) -> Option<String> {
1153 let key = match self.status {
1154 TaskStatus::Held if self.hold_notice_cause.is_none() => Some(format!(
1155 "held#{}",
1156 self.held_at.map(|t| t.to_string()).unwrap_or_default()
1157 )),
1158 TaskStatus::Blocked => Some(format!(
1159 "blocked#{}#{}",
1160 self.blocked_by.join(","),
1161 self.block_reason.as_deref().unwrap_or_default()
1162 )),
1163 _ => None,
1164 };
1165 key.filter(|key| self.legacy_notice_matches(key))
1166 }
1167
1168 pub(crate) fn legacy_notice_matches(&self, key: &str) -> bool {
1169 match self.status {
1170 TaskStatus::Held if self.hold_notice_cause.is_none() => key
1171 .strip_prefix("held#")
1172 .is_some_and(|time| time.is_empty() || time.parse::<Timestamp>().is_ok()),
1173 TaskStatus::Blocked
1174 if self
1175 .hold_notice_cause
1176 .as_ref()
1177 .is_none_or(|c| !c.stage.starts_with("conductor-evidence:")) =>
1178 {
1179 key.strip_prefix("blocked#").is_some_and(|tail| {
1180 let Some((ids, _)) = tail.split_once('#') else {
1181 return false;
1182 };
1183 let old: std::collections::BTreeSet<_> =
1184 ids.split(',').filter(|id| !id.is_empty()).collect();
1185 let current: std::collections::BTreeSet<_> =
1186 self.blocked_by.iter().map(String::as_str).collect();
1187 old == current
1188 })
1189 }
1190 _ => false,
1191 }
1192 }
1193
1194 fn import_chat_notice(&mut self) {
1195 if let Some(n) = &self.chat_notice {
1196 let key = if is_done_notice_key(&n.key) {
1197 "done".to_owned()
1198 } else if self.legacy_notice_matches(&n.key) {
1199 self.chat_notice_key().unwrap_or_else(|| n.key.clone())
1200 } else {
1201 n.key.clone()
1202 };
1203 if !self.chat_notices.contains(&key) {
1204 self.chat_notices.push(key);
1205 }
1206 }
1207 }
1208
1209 pub(crate) fn notice_cause(&mut self, stage: &str, run: Option<&str>, pass: Option<usize>) {
1212 self.import_chat_notice();
1213 self.hold_notice_cause = Some(NoticeCause {
1214 stage: stage.to_owned(),
1215 target: None,
1216 run: run.map(str::to_owned),
1217 pass,
1218 });
1219 }
1220
1221 pub(crate) fn startup_notice_cause(&mut self, stage: &str) {
1224 let recovery_only = matches!(
1225 stage,
1226 "startup-source" | "startup-candidates" | "startup-public-head"
1227 );
1228 match self.hold_notice_cause.as_ref() {
1229 Some(cause) if recovery_only && !cause.stage.starts_with("startup-") => return,
1230 None if recovery_only
1231 && self
1232 .chat_notice
1233 .as_ref()
1234 .is_some_and(|n| !is_done_notice_key(&n.key)) =>
1235 {
1236 return;
1237 }
1238 _ => {}
1239 }
1240 self.notice_cause(stage, None, None);
1241 }
1242
1243 pub(crate) fn fail_startup(
1246 &mut self,
1247 stage: &str,
1248 why: impl Into<String>,
1249 max_attempts: usize,
1250 ) {
1251 let unresolved = self.hold_notice_cause.clone();
1252 self.fail(why, max_attempts);
1253 self.hold_notice_cause = unresolved;
1254 self.startup_notice_cause(stage);
1255 }
1256
1257 pub(crate) fn fail_startup_target(
1259 &mut self,
1260 stage: &str,
1261 target: String,
1262 why: impl Into<String>,
1263 max_attempts: usize,
1264 ) {
1265 self.fail_startup(stage, why, max_attempts);
1266 if let Some(cause) = self.hold_notice_cause.as_mut() {
1267 cause.target = Some(target);
1268 }
1269 }
1270
1271 pub fn chat_notice_due(&self) -> Option<String> {
1273 self.filed_by_chat()?;
1274 if self.schema < 13 {
1275 return None;
1276 }
1277 let key = self.chat_notice_key()?;
1278 if self.chat_notices.contains(&key) {
1279 return None;
1280 }
1281 match &self.chat_notice {
1282 Some(n)
1283 if n.key == key
1284 || (key == "done" && is_done_notice_key(&n.key))
1285 || self.legacy_notice_matches(&n.key) =>
1286 {
1287 None
1288 }
1289 _ => Some(key),
1290 }
1291 }
1292
1293 pub fn short(&self) -> &str {
1295 short(&self.id)
1296 }
1297
1298 pub fn mark_triage_applied(&mut self, question_id: &str) {
1300 if !self.triage_applied(question_id) {
1301 self.triage_applied.push(question_id.to_owned());
1302 }
1303 }
1304
1305 pub fn action_applied(&self, question_id: &str) -> bool {
1308 self.actions_applied.iter().any(|id| id == question_id)
1309 }
1310
1311 pub fn mark_action_applied(&mut self, question_id: &str) {
1313 if !self.action_applied(question_id) {
1314 self.actions_applied.push(question_id.to_owned());
1315 }
1316 }
1317
1318 pub fn triage_applied(&self, question_id: &str) -> bool {
1320 self.triage_applied.iter().any(|id| id == question_id)
1321 }
1322
1323 pub fn start(&mut self, run: String) {
1336 self.stamp_schema();
1337 self.status = TaskStatus::Running;
1338 self.attempts += 1;
1339 self.runs.retain(|id| id != &run);
1340 if let Some(pin) = &self.resume_override
1341 && pin.pinned_run.as_ref() == Some(&run)
1342 && !pin.question_id.is_empty()
1343 {
1344 self.dispatched_resumes
1345 .insert(pin.question_id.clone(), run.clone());
1346 }
1347 self.runs.push(run);
1348 self.last_error = None;
1349 self.park_reason = None;
1350 self.fresh_start = false;
1351 self.interrupt = false;
1352 self.resume_override = None;
1354 }
1355
1356 pub fn record_startup_failure(&mut self, failure: crate::run::StartupFailure) {
1358 if !self.startup_failures.iter().any(|f| f.run == failure.run) {
1359 self.startup_failures.push(failure);
1360 }
1361 }
1362
1363 pub fn link_run(&mut self, run: &str) -> bool {
1368 if self.runs.iter().any(|r| r == run) {
1369 return false;
1370 }
1371 self.runs.push(run.to_owned());
1372 true
1373 }
1374
1375 pub fn succeed(&mut self) {
1385 self.stamp_schema();
1386 self.status = TaskStatus::Done;
1387 self.resume_override = None;
1388 self.last_error = None;
1389 self.park_reason = None;
1390 self.hold_reason = None;
1391 self.hold_source = None;
1392 self.diagnostic = None;
1393 self.blocked_by.clear();
1394 self.block_reason = None;
1395 self.blocked_from = None;
1396 }
1397
1398 pub fn already_landed(&mut self, note: impl Into<String>) {
1406 self.succeed();
1407 self.attempts = self.attempts.saturating_sub(1);
1408 self.last_error = Some(note.into());
1409 }
1410
1411 pub fn superseded_attempts(&self, last_run_succeeded: bool) -> &[String] {
1441 if self.status != TaskStatus::Done || !last_run_succeeded || self.execution_runs().len() < 2
1442 {
1443 return &[];
1444 }
1445 &self.execution_runs()[..self.execution_runs().len() - 1]
1446 }
1447
1448 pub fn successor_of(&self, run: &str) -> Option<&String> {
1456 let runs = self.execution_runs();
1457 let pos = runs.iter().rposition(|r| r == run)?;
1458 runs.get(pos + 1)
1459 }
1460
1461 pub fn earlier_attempts(&self) -> &[String] {
1468 self.execution_runs()
1469 }
1470
1471 fn stamp_schema(&mut self) {
1478 self.schema = self.schema.max(SCHEMA);
1479 }
1480
1481 fn note_held(&mut self) {
1482 self.stamp_schema();
1483 let continuing = self.status == TaskStatus::Held
1484 || (self.status == TaskStatus::Blocked
1485 && (self.blocked_from == Some(TaskStatus::Held)
1486 || (self.blocked_from.is_none()
1487 && (self.hold_reason.is_some() || self.hold_source.is_some()))));
1488 if !continuing || self.held_at.is_none() {
1489 self.held_at = Some(Timestamp::now());
1490 }
1491 }
1492
1493 pub fn fail(&mut self, why: impl Into<String>, max_attempts: usize) {
1509 let why = why.into();
1510 if self.status == TaskStatus::Running {
1511 let run = self.runs.last().cloned();
1512 self.notice_cause("run-failed", run.as_deref(), Some(self.attempts));
1513 }
1514 self.diagnostic = None;
1515 self.park_reason = None;
1516 self.status = if self.attempts >= self.effective_attempt_limit(max_attempts) {
1517 self.note_held();
1518 self.hold_source = Some(HoldSource::Machine);
1519 self.hold_reason = Some(why.clone());
1520 TaskStatus::Held
1521 } else {
1522 TaskStatus::Failed
1523 };
1524 self.last_error = Some(why);
1525 }
1526
1527 pub fn stall(&mut self, why: impl Into<String>) {
1537 self.last_error = Some(why.into());
1538 self.diagnostic = None;
1539 self.attempts = self.attempts.saturating_sub(1);
1540 self.status = TaskStatus::Failed;
1541 }
1542
1543 pub fn park(&mut self, why: impl Into<String>) {
1549 self.diagnostic = None;
1550 self.attempts = self.attempts.saturating_sub(1);
1551 self.status = TaskStatus::Parked;
1552 self.park_reason = Some(why.into());
1553 self.schema = self.schema.max(SCHEMA);
1556 }
1557
1558 pub fn record_pass(
1560 &mut self,
1561 run: &str,
1562 pass_index: usize,
1563 attempts_before: usize,
1564 parked: Option<crate::run::ParkCause>,
1565 ) {
1566 self.record_pass_at(run, pass_index, None, attempts_before, parked);
1567 }
1568
1569 pub fn record_pass_at(
1572 &mut self,
1573 run: &str,
1574 pass_index: usize,
1575 started_at: Option<Timestamp>,
1576 attempts_before: usize,
1577 parked: Option<crate::run::ParkCause>,
1578 ) {
1579 if self
1580 .passes
1581 .iter()
1582 .any(|p| p.run == run && p.pass_index == pass_index)
1583 {
1584 return;
1585 }
1586 let cost = if self.attempts < attempts_before {
1587 PassCost::Refunded
1588 } else {
1589 PassCost::Spent
1590 };
1591 self.passes.push(TaskPass {
1592 run: run.to_owned(),
1593 pass_index,
1594 started_at: self.dispatched_at.or(started_at),
1595 cost,
1596 attempts_after: self.attempts,
1597 settled_at: Timestamp::now(),
1598 parked,
1599 });
1600 self.schema = self.schema.max(SCHEMA);
1601 }
1602
1603 pub(crate) fn upgrade_resume_key(&self) -> Option<(Timestamp, &str)> {
1606 if self.status != TaskStatus::Parked || self.fresh_start {
1607 return None;
1608 }
1609 let pass = self.passes.last()?;
1610 if self.execution_runs().last() != Some(&pass.run)
1611 || !matches!(pass.parked, Some(crate::run::ParkCause::Upgrade { .. }))
1612 {
1613 return None;
1614 }
1615 Some((pass.started_at.unwrap_or(pass.settled_at), &self.id))
1618 }
1619
1620 pub fn operator_held(&self) -> bool {
1626 self.status == TaskStatus::Held && !matches!(self.hold_source, Some(HoldSource::Machine))
1627 }
1628
1629 pub fn hold_manual(&mut self, reason: Option<String>) {
1639 self.operator_decided_at = Some(Timestamp::now());
1641 self.hold_unrequested_retry(reason);
1642 }
1643
1644 pub(crate) fn hold_unrequested_retry(&mut self, reason: Option<String>) {
1647 self.park_reason = None;
1648 self.note_held();
1649 self.status = TaskStatus::Held;
1650 if reason.is_some() {
1651 self.hold_reason = reason;
1652 }
1653 self.hold_source = Some(HoldSource::Manual);
1654 self.blocked_by.clear();
1655 self.block_reason = None;
1656 self.blocked_from = None;
1657 }
1658
1659 pub fn hold_machine(&mut self, reason: Option<String>) {
1664 self.park_reason = None;
1665 self.note_held();
1666 self.status = TaskStatus::Held;
1667 if reason.is_some() {
1668 self.hold_reason = reason;
1669 }
1670 self.hold_source = Some(HoldSource::Machine);
1671 self.blocked_by.clear();
1672 self.block_reason = None;
1673 self.blocked_from = None;
1674 }
1675
1676 pub fn block(&mut self, blocked_by: Vec<String>, reason: Option<String>) {
1686 self.stamp_schema();
1687 let before: std::collections::BTreeSet<_> = self.blocked_by.iter().collect();
1688 let after: std::collections::BTreeSet<_> = blocked_by.iter().collect();
1689 if self.status != TaskStatus::Blocked || before != after {
1690 self.blocked_at = Some(Timestamp::now());
1691 }
1692 if self.status != TaskStatus::Blocked {
1693 self.blocked_from = Some(self.status);
1694 }
1695 self.status = TaskStatus::Blocked;
1696 self.blocked_by = blocked_by;
1697 self.block_reason = reason;
1698 }
1699
1700 pub fn unblock(&mut self, resolved_id: &str) {
1723 if self.status != TaskStatus::Blocked {
1724 return;
1725 }
1726 self.blocked_by.retain(|id| id != resolved_id);
1727 self.restore_if_unblocked();
1728 }
1729
1730 pub fn dependency_deleted(&mut self, deleted_id: &str) -> bool {
1741 if self.status != TaskStatus::Blocked || !self.blocked_by.iter().any(|b| b == deleted_id) {
1742 return false;
1743 }
1744 self.blocked_by.retain(|id| id != deleted_id);
1745 self.restore_if_unblocked();
1746 true
1747 }
1748
1749 fn restore_if_unblocked(&mut self) {
1753 if self.blocked_by.is_empty() {
1754 self.status = match self.blocked_from {
1755 Some(TaskStatus::Running) => TaskStatus::Queued,
1756 Some(other) => other,
1757 None if self.hold_reason.is_some() || self.hold_source.is_some() => {
1758 TaskStatus::Held
1759 }
1760 None => TaskStatus::Queued,
1761 };
1762 self.block_reason = None;
1763 self.blocked_from = None;
1764 }
1765 }
1766
1767 pub fn record_answer(&mut self, question: String, answer: String) {
1772 self.answers.push(AnsweredQuestion { question, answer });
1773 }
1774
1775 pub fn request_review(&mut self, branch: String) {
1779 self.release();
1780 self.review_branch = Some(branch);
1781 self.review_instruction = Some(self.instruction.clone());
1782 }
1783
1784 pub fn requeue(&mut self) {
1787 self.release();
1788 self.review_branch = None;
1790 self.review_instruction = None;
1791 self.fresh_start = true;
1792 }
1793
1794 pub fn hold_for_handover(&mut self, branch: Option<String>, reason: String) {
1803 if branch.is_some() {
1804 self.review_branch = branch;
1805 self.review_instruction = Some(self.instruction.clone());
1806 }
1807 self.hold_machine(Some(reason));
1808 }
1809
1810 pub fn set_urgent(&mut self, urgent: bool) -> Result<()> {
1812 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1813 bail!(
1814 "task {} is {}; urgent can only change while queued or held; \
1815 wait for the active attempt to finish and requeue or hold the task first",
1816 self.short(),
1817 self.status.as_str()
1818 );
1819 }
1820 self.urgent = urgent;
1821 Ok(())
1822 }
1823
1824 pub fn set_priority(&mut self, priority: i32) -> Result<()> {
1833 if self.status == TaskStatus::Running {
1834 bail!(
1835 "task {} is running; its priority cannot be changed until \
1836 this attempt finishes",
1837 self.short()
1838 );
1839 }
1840 self.priority = priority;
1841 Ok(())
1842 }
1843
1844 pub fn set_interrupt(&mut self, interrupt: bool) -> Result<()> {
1859 if interrupt && !self.status.runnable() {
1860 bail!(
1861 "task {} is {}; only a queued or failed task can be marked \
1862 to interrupt",
1863 self.short(),
1864 self.status.as_str()
1865 );
1866 }
1867 self.interrupt = interrupt;
1868 Ok(())
1869 }
1870
1871 pub fn effective_attempt_limit(&self, daemon_limit: usize) -> usize {
1873 self.attempt_limit.map_or(daemon_limit, NonZeroUsize::get)
1874 }
1875
1876 pub fn change_attempt_limit(&mut self, limit: Option<NonZeroUsize>) -> Result<()> {
1879 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1880 bail!("only queued or held tasks can change the attempt limit");
1881 }
1882 self.stamp_schema();
1883 self.attempt_limit = limit;
1884 Ok(())
1885 }
1886
1887 pub fn implementers(&self) -> Option<usize> {
1889 if self.solo {
1890 Some(1)
1891 } else {
1892 self.overrides.as_ref().and_then(|o| o.candidates)
1893 }
1894 }
1895
1896 pub fn change_implementers(&mut self, count: u8) -> Result<()> {
1899 self.check_implementers_editable()?;
1900 let explicit = self.overrides.as_ref().and_then(|o| o.config.as_deref());
1901 let (mut config, _) = crate::config::Config::discover(&self.repo, explicit)?;
1902 if let Some(o) = &self.overrides {
1903 o.apply(&mut config);
1904 }
1905 self.set_implementers(count, &config)
1906 }
1907
1908 fn check_implementers_editable(&self) -> Result<()> {
1909 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1910 bail!(
1911 "task {} is {}; only queued or held tasks can change implementers",
1912 self.short(),
1913 self.status.as_str()
1914 );
1915 }
1916 Ok(())
1917 }
1918
1919 pub fn set_implementers(&mut self, count: u8, config: &crate::config::Config) -> Result<()> {
1921 self.check_implementers_editable()?;
1922 crate::talk::check_implementers(count, config).map_err(anyhow::Error::msg)?;
1923 self.stamp_schema();
1924 self.solo = count == 1;
1925 let mut overrides = self.overrides.clone().unwrap_or_else(|| RunOverrides {
1926 inherit_daemon_options: true,
1927 ..Default::default()
1928 });
1929 overrides.candidates = (count > 1).then_some(usize::from(count));
1930 let mut settings = overrides.clone();
1931 settings.inherit_daemon_options = false;
1932 self.overrides = if overrides.inherit_daemon_options && settings == RunOverrides::default()
1934 {
1935 None
1936 } else {
1937 Some(overrides)
1938 };
1939 Ok(())
1940 }
1941
1942 pub fn edit(&mut self, title: String, instruction: String) -> Result<()> {
1954 if !matches!(self.status, TaskStatus::Queued | TaskStatus::Held) {
1955 bail!(
1956 "task {} is {}; only a queued or held task's instruction can \
1957 be edited",
1958 self.short(),
1959 self.status.as_str()
1960 );
1961 }
1962 if self.review_branch.is_some() && self.review_instruction.is_none() {
1965 self.review_instruction = Some(self.instruction.clone());
1966 }
1967 if self.instruction != instruction {
1968 self.resume_override = None;
1971 }
1972 self.title = title;
1973 self.instruction = instruction;
1974 Ok(())
1975 }
1976
1977 pub fn handed_off(&mut self, why: impl Into<String>) {
1997 let why = why.into();
1998 self.diagnostic = None;
1999 self.note_held();
2000 self.status = TaskStatus::Held;
2001 self.hold_source = Some(HoldSource::Machine);
2002 self.hold_reason = Some(why.clone());
2003 self.last_error = Some(why);
2004 }
2005
2006 pub fn release_operator(&mut self, source: &str, question_id: Option<&str>) {
2008 self.recovery_budget = None;
2009 self.recovery_generations.clear();
2010 self.recovery_admission = None;
2011 if self
2012 .resumption
2013 .as_ref()
2014 .is_some_and(|r| r.instruction != self.instruction)
2015 {
2016 self.resume_override = None;
2019 }
2020 let approval = Resumption {
2021 generation: self.resumption.as_ref().map_or(1, |r| r.generation + 1),
2022 source: source.to_owned(),
2023 question_id: question_id.map(str::to_owned),
2024 at: Timestamp::now(),
2025 instruction: self.instruction.clone(),
2026 answers_through: self.answers.len(),
2027 runs_through: self.runs.len(),
2028 blocking_evidence: Vec::new(),
2029 cleared_reason: self
2030 .hold_reason
2031 .clone()
2032 .or_else(|| self.block_reason.clone()),
2033 };
2034 self.release();
2035 self.operator_decided_at = Some(approval.at);
2036 if let Some(previous) = self.resumption.replace(approval) {
2037 self.resumption_history.push(previous);
2038 }
2039 }
2040
2041 pub(crate) fn recovery_failed(&mut self, state: &crate::run::RunState) {
2043 let (base, inputs) = self.recovery_admission.as_ref().map_or_else(
2044 || (state.base_commit.clone(), recovery_inputs(&state.sources)),
2045 |a| (a.base.clone(), a.inputs.clone()),
2046 );
2047 let progress = recovery_progress(state);
2048 let stage = if state.prep_failure.is_some() {
2049 "preparation"
2050 } else {
2051 "execution"
2052 };
2053 let matches = |b: &RecoveryBudget| {
2054 b.instruction == self.instruction
2055 && b.base == base
2056 && b.inputs == inputs
2057 && (progress.is_none() || b.progress == progress)
2058 };
2059 if !self.recovery_budget.as_ref().is_some_and(matches) {
2060 let previous = self
2061 .recovery_generations
2062 .iter()
2063 .position(matches)
2064 .map(|index| self.recovery_generations.remove(index));
2065 if let Some(old) = self.recovery_budget.take() {
2066 self.recovery_generations.push(old);
2067 }
2068 self.recovery_budget = previous;
2069 }
2070 if let Some(budget) = &mut self.recovery_budget {
2071 budget.failures = budget.failures.saturating_add(1);
2072 budget.stage = stage.to_owned();
2073 } else {
2074 self.recovery_budget = Some(RecoveryBudget {
2075 instruction: self.instruction.clone(),
2076 base,
2077 inputs,
2078 stage: stage.to_owned(),
2079 failures: 1,
2080 progress,
2081 });
2082 }
2083 }
2084
2085 pub(crate) fn recovery_exhausted(
2086 &self,
2087 base: &str,
2088 inputs: &str,
2089 limit: usize,
2090 ) -> Option<String> {
2091 let same_input = |b: &&RecoveryBudget| {
2092 b.instruction == self.instruction && b.base == base && b.inputs == inputs
2093 };
2094 let budget = self
2095 .recovery_budget
2096 .as_ref()
2097 .filter(same_input)
2098 .or_else(|| self.recovery_generations.iter().rev().find(same_input))?;
2099 (budget.instruction == self.instruction && budget.base == base && budget.inputs == inputs
2100 && budget.failures >= limit.max(1)).then(|| format!(
2101 "[recovery-budget] {} failed passes against unchanged inputs ({}). Inspect the retained recovery sources and correct the cause, change the inputs, or explicitly release the task to approve another generation. Automatic requeue does not reset this budget.",
2102 budget.failures, budget.stage))
2103 }
2104
2105 pub fn release(&mut self) {
2109 self.import_chat_notice();
2112 if self.hold_notice_cause.is_none()
2113 && matches!(self.status, TaskStatus::Held | TaskStatus::Blocked)
2114 {
2115 self.hold_notice_cause = Some(self.unresolved_hold_cause());
2116 }
2117 let refused_handover = self.status == TaskStatus::Held
2118 && self.hold_source == Some(HoldSource::Machine)
2119 && self.review_branch.is_some()
2120 && self
2121 .review_instruction
2122 .as_ref()
2123 .is_none_or(|i| *i == self.instruction);
2124 self.status = TaskStatus::Queued;
2125 self.held_at = None;
2126 self.attempts = 0;
2127 self.last_error = None;
2128 self.park_reason = None;
2129 self.hold_reason = None;
2132 self.hold_source = None;
2133 self.diagnostic = None;
2134 self.blocked_by.clear();
2139 self.block_reason = None;
2140 self.blocked_from = None;
2141 if !refused_handover {
2145 self.review_branch = None;
2146 self.review_instruction = None;
2147 }
2148 self.fresh_start = false;
2149 }
2150}
2151
2152fn copy_new(dir: &Path, src: &Path, name: &str) -> Result<(String, PathBuf)> {
2156 use std::io::ErrorKind;
2157 let (stem, ext) = match name.rfind('.') {
2158 Some(i) if i > 0 => (&name[..i], &name[i..]),
2159 _ => (name, ""),
2160 };
2161 for n in 1u32.. {
2162 let candidate = if n == 1 {
2163 name.to_owned()
2164 } else {
2165 let suffix = format!("-{n}");
2166 let room = 64usize.saturating_sub(suffix.len() + ext.len());
2167 let stem: String = stem.chars().take(room).collect();
2168 format!("{stem}{suffix}{ext}")
2169 };
2170 if !crate::ask::valid_asset_name(&candidate) {
2171 bail!("no valid attachment name is left for `{name}`");
2172 }
2173 let path = dir.join(&candidate);
2174 match std::fs::OpenOptions::new()
2175 .write(true)
2176 .create_new(true)
2177 .open(&path)
2178 {
2179 Ok(mut out) => {
2180 let copied = std::fs::File::open(src)
2181 .and_then(|mut input| std::io::copy(&mut input, &mut out));
2182 if let Err(e) = copied {
2183 drop(out);
2184 let _ = std::fs::remove_file(&path);
2185 return Err(e).with_context(|| format!("copy {}", src.display()));
2186 }
2187 return Ok((candidate, path));
2188 }
2189 Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
2190 Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
2191 }
2192 }
2193 unreachable!("the counter never runs out")
2194}
2195
2196const TASK_LOCK_STALE: std::time::Duration = std::time::Duration::from_secs(10);
2199
2200pub(crate) struct TaskLock {
2203 path: PathBuf,
2204 token: String,
2205}
2206
2207fn owner_token() -> String {
2210 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
2211 let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2212 let mut r = crate::rng::SplitMix64::new(crate::rng::entropy() ^ n.rotate_left(32));
2213 format!("{}-{:016x}", std::process::id(), r.next_u64())
2214}
2215
2216fn break_marker(path: &Path, token: &str) -> PathBuf {
2219 let mut h: u64 = 0xcbf29ce484222325;
2220 for b in token.bytes() {
2221 h ^= u64::from(b);
2222 h = h.wrapping_mul(0x100000001b3);
2223 }
2224 let mut name = path.as_os_str().to_owned();
2225 name.push(format!(".break-{h:016x}"));
2226 PathBuf::from(name)
2227}
2228
2229fn older_than_stale(path: &Path) -> bool {
2230 std::fs::metadata(path)
2231 .and_then(|m| m.modified())
2232 .ok()
2233 .and_then(|t| t.elapsed().ok())
2234 .is_some_and(|age| age > TASK_LOCK_STALE)
2235}
2236
2237fn writer_may_be_gone(token: &str) -> bool {
2238 let Some(pid) = token
2239 .split_once('-')
2240 .and_then(|(pid, _)| pid.parse::<u32>().ok())
2241 else {
2242 return true;
2245 };
2246 if pid == std::process::id() {
2247 return false;
2248 }
2249 crate::proc::pid_status(pid) == Some(false)
2251}
2252
2253const MAX_MARKER_DEPTH: u8 = 3;
2257
2258fn take_marker(
2264 path: &Path,
2265 token: &str,
2266 depth: u8,
2267 retry: crate::lock_file::Retry,
2268) -> Option<(PathBuf, String)> {
2269 use std::io::Write;
2270 let marker = break_marker(path, token);
2271 for _ in 0..2 {
2272 match retry.create(&marker) {
2273 Ok(mut f) => {
2274 let mine = owner_token();
2275 if f.write_all(mine.as_bytes()).is_err() {
2276 drop(f);
2277 let _ = std::fs::remove_file(&marker);
2278 return None;
2279 }
2280 return Some((marker, mine));
2281 }
2282 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2283 let judged = std::fs::read_to_string(&marker).ok();
2284 match judged.filter(|_| older_than_stale(&marker)) {
2285 Some(judged) if depth < MAX_MARKER_DEPTH => {
2286 remove_lock_if(&marker, &judged, true, depth + 1, retry)?;
2287 }
2288 _ => return None,
2289 }
2290 }
2291 Err(_) => return None,
2292 }
2293 }
2294 None
2295}
2296
2297fn remove_lock_if(
2302 path: &Path,
2303 token: &str,
2304 require_stale: bool,
2305 depth: u8,
2306 retry: crate::lock_file::Retry,
2307) -> Option<bool> {
2308 let (marker, mine) = take_marker(path, token, depth, retry)?;
2309 let still = std::fs::read_to_string(path).is_ok_and(|c| c == token)
2310 && (!require_stale || older_than_stale(path) && writer_may_be_gone(token));
2311 if still {
2312 let _ = std::fs::remove_file(path);
2313 }
2314 if depth >= MAX_MARKER_DEPTH {
2316 if std::fs::read_to_string(&marker).is_ok_and(|c| c == mine) {
2317 let _ = std::fs::remove_file(&marker);
2318 }
2319 } else {
2320 let _ = remove_lock_if(&marker, &mine, false, depth + 1, retry);
2321 }
2322 Some(still)
2323}
2324
2325#[cfg(test)]
2328fn break_stale(path: &Path, judged: &str) -> bool {
2329 break_stale_with(path, judged, crate::lock_file::Retry::new(None))
2330}
2331
2332fn break_stale_with(path: &Path, judged: &str, retry: crate::lock_file::Retry) -> bool {
2333 remove_lock_if(path, judged, true, 0, retry).unwrap_or(false)
2334}
2335
2336impl Drop for TaskLock {
2337 fn drop(&mut self) {
2338 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
2339 let retry = crate::lock_file::Retry::new(Some(deadline));
2340 loop {
2341 match remove_lock_if(&self.path, &self.token, false, 0, retry) {
2342 Some(_) => return,
2343 None if std::time::Instant::now() > deadline => return,
2344 None => std::thread::sleep(std::time::Duration::from_millis(5)),
2345 }
2346 }
2347 }
2348}
2349
2350#[derive(Debug, Clone)]
2352pub struct Queue {
2353 root: PathBuf,
2354 ids: IdSource,
2355}
2356
2357fn validate_attachments(sources: &[PathBuf]) -> Result<Vec<(&PathBuf, &str)>> {
2358 let mut wanted = Vec::new();
2359 for src in sources {
2360 let name = src
2361 .file_name()
2362 .and_then(|n| n.to_str())
2363 .with_context(|| format!("`{}` has no usable file name", src.display()))?;
2364 if !crate::ask::valid_asset_name(name) {
2365 bail!(
2366 "attachment name `{name}` must match ^[A-Za-z0-9][A-Za-z0-9._-]{{0,63}}$ \
2367 with no `..`; rename the file and try again"
2368 );
2369 }
2370 if !src.is_file() {
2371 bail!("attachment `{}` is not a file", src.display());
2372 }
2373 wanted.push((src, name));
2374 }
2375 Ok(wanted)
2376}
2377
2378impl Queue {
2379 pub fn open() -> Self {
2381 Self::at(crate::run::home().join("queue"))
2382 }
2383
2384 pub fn at(root: PathBuf) -> Self {
2387 Self {
2388 root,
2389 ids: IdSource::new(new_id),
2390 }
2391 }
2392
2393 #[doc(hidden)]
2396 pub fn with_id_source(mut self, f: impl Fn() -> String + Send + Sync + 'static) -> Self {
2397 self.ids = IdSource::custom(f);
2398 self
2399 }
2400
2401 fn file_with(
2412 &self,
2413 task: &mut Task,
2414 hook: impl FnMut(&mut Task) -> Result<bool>,
2415 undo: impl FnMut(&mut Task),
2416 ) -> Result<()> {
2417 self.reserve_id(task, hook, undo, true)
2418 }
2419
2420 fn reserve_id(
2428 &self,
2429 task: &mut Task,
2430 mut hook: impl FnMut(&mut Task) -> Result<bool>,
2431 mut undo: impl FnMut(&mut Task),
2432 store: bool,
2433 ) -> Result<()> {
2434 let redraw_first = self.ids.is_custom() && task.attachments.is_empty();
2435 let initial = task.id.clone();
2436 self.ids
2437 .reserve((!redraw_first).then_some(initial.as_str()), "task", |id| {
2438 task.id = id.to_owned();
2439 let _lock = self.lock_task(&task.id)?;
2440 if path_exists(&self.path_of(&task.id))? || !hook(task)? {
2441 return Ok(false);
2442 }
2443 if store {
2444 if let Err(e) = self.write_record(task) {
2445 undo(task);
2446 return Err(e);
2447 }
2448 task.unfiled = false;
2449 }
2450 Ok(true)
2451 })?;
2452 Ok(())
2453 }
2454
2455 fn id_in_use(&self, id: &str, check_attachments: bool) -> Result<bool> {
2460 if path_exists(&self.lock_path(id))? {
2461 return Ok(true);
2462 }
2463 Ok(check_attachments && path_exists(&self.attachments_dir(id))?)
2464 }
2465
2466 fn plain_filing_hook(&self, task: &Task) -> Result<bool> {
2470 Ok(!self.id_in_use(&task.id, task.attachments.is_empty())?)
2471 }
2472
2473 pub fn file_claimed(&self, task: &mut Task) -> Result<Claim> {
2476 let mut claim = None;
2477 self.file_with(
2478 task,
2479 |t| {
2480 if self.id_in_use(&t.id, true)? {
2481 return Ok(false);
2482 }
2483 claim = Some(self.claim(&t.id)?);
2484 Ok(true)
2485 },
2486 |_| {},
2487 )?;
2488 claim.context("claim missing after filing")
2489 }
2490
2491 pub fn root(&self) -> &Path {
2493 &self.root
2494 }
2495
2496 pub fn path_of(&self, id: &str) -> PathBuf {
2498 self.root.join(format!("{id}.json"))
2499 }
2500
2501 pub fn attachments_dir(&self, id: &str) -> PathBuf {
2504 self.root.join(format!("{id}.attachments"))
2505 }
2506
2507 pub fn attach(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2518 let wanted = validate_attachments(sources)?;
2519 if task.unfiled && task.attachments.is_empty() {
2520 let names = std::cell::RefCell::new(Vec::new());
2523 self.reserve_id(
2524 task,
2525 |t| {
2526 if self.id_in_use(&t.id, true)? {
2527 return Ok(false);
2528 }
2529 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2530 Ok(true)
2531 },
2532 |_| {},
2533 false,
2534 )?;
2535 return Ok(names.into_inner());
2536 }
2537 self.copy_attachments(task, &wanted)
2538 }
2539
2540 fn copy_attachments(
2542 &self,
2543 task: &mut Task,
2544 wanted: &[(&PathBuf, &str)],
2545 ) -> Result<Vec<String>> {
2546 let dir = self.attachments_dir(&task.id);
2547 let existed = dir.is_dir();
2548 std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
2549 let mut created: Vec<PathBuf> = Vec::new();
2550 let mut names = Vec::new();
2551 let mut copy_all = || -> Result<()> {
2552 for (src, name) in wanted {
2553 let (stored, path) = copy_new(&dir, src, name)?;
2554 created.push(path);
2555 names.push(stored);
2556 }
2557 Ok(())
2558 };
2559 if let Err(e) = copy_all() {
2560 for path in &created {
2561 let _ = std::fs::remove_file(path);
2562 }
2563 if !existed {
2564 let _ = std::fs::remove_dir(&dir);
2565 }
2566 return Err(e);
2567 }
2568 task.attachments.extend(names.iter().cloned());
2569 Ok(names)
2570 }
2571
2572 pub fn attach_and_put(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2583 if task.unfiled && task.attachments.is_empty() {
2584 return self.attach_and_file(task, sources);
2585 }
2586 let dir = self.attachments_dir(&task.id);
2587 let existed = dir.is_dir();
2588 let before = task.attachments.len();
2589 let names = self.attach(task, sources)?;
2590 if let Err(e) = self.put(task) {
2591 for name in &names {
2592 let _ = std::fs::remove_file(dir.join(name));
2593 }
2594 if !existed {
2595 let _ = std::fs::remove_dir(&dir);
2596 }
2597 task.attachments.truncate(before);
2598 return Err(e);
2599 }
2600 Ok(names)
2601 }
2602
2603 fn attach_and_file(&self, task: &mut Task, sources: &[PathBuf]) -> Result<Vec<String>> {
2607 let wanted = validate_attachments(sources)?;
2608 let before = task.attachments.len();
2609 let names = std::cell::RefCell::new(Vec::new());
2610 let result = self.file_with(
2611 task,
2612 |t| {
2613 if self.id_in_use(&t.id, true)? {
2614 return Ok(false);
2615 }
2616 *names.borrow_mut() = self.copy_attachments(t, &wanted)?;
2617 Ok(true)
2618 },
2619 |t| {
2620 let dir = self.attachments_dir(&t.id);
2621 for name in names.borrow().iter() {
2622 let _ = std::fs::remove_file(dir.join(name));
2623 }
2624 let _ = std::fs::remove_dir(&dir);
2626 t.attachments.truncate(before);
2627 },
2628 );
2629 result.map(|()| names.into_inner())
2630 }
2631
2632 pub fn attachment_paths(&self, task: &Task) -> Vec<PathBuf> {
2636 let dir = self.attachments_dir(&task.id);
2637 task.attachments
2638 .iter()
2639 .map(|n| {
2640 let p = dir.join(n);
2641 std::path::absolute(&p).unwrap_or(p)
2642 })
2643 .collect()
2644 }
2645
2646 pub fn put(&self, task: &mut Task) -> Result<()> {
2655 if task.unfiled {
2656 return self.file_with(task, |t| self.plain_filing_hook(t), |_| {});
2658 }
2659 let _lock = self.lock_task(&task.id)?;
2660 self.put_unlocked(task)
2661 }
2662
2663 pub fn create_new(&self, task: &mut Task) -> Result<bool> {
2668 let _lock = self.lock_task(&task.id)?;
2669 if path_exists(&self.path_of(&task.id))? {
2670 return Ok(false);
2671 }
2672 self.write_record(task)?;
2673 task.unfiled = false;
2674 Ok(true)
2675 }
2676
2677 fn put_unlocked(&self, task: &mut Task) -> Result<()> {
2679 self.merge_stored(task);
2680 if let Some(boundary) = task.reassignments.last() {
2681 let stored = self.get(&task.id)?;
2682 anyhow::ensure!(
2683 task.reassignments == stored.reassignments
2684 && task.repo == stored.repo
2685 && task.runs.get(..boundary.run_boundary)
2686 == stored.runs.get(..boundary.run_boundary),
2687 "historical repository execution cannot be restarted; reload the task"
2688 );
2689 }
2690 self.write_record(task)
2691 }
2692
2693 pub(crate) fn commit_reassignment(&self, task: &mut Task, expected: &Task) -> Result<()> {
2695 let _lock = self.lock_task(&task.id)?;
2696 let stored = self.get(&task.id)?;
2697 anyhow::ensure!(
2698 serde_json::to_value(&stored)? == serde_json::to_value(expected)?
2699 && stored.reassignments.len() + 1 == task.reassignments.len()
2700 && stored.repo == task.reassignments.last().expect("new boundary").from
2701 && stored.runs == task.runs,
2702 "task changed during reassignment; retry after inspecting it"
2703 );
2704 self.write_record(task)
2705 }
2706
2707 fn merge_stored(&self, task: &mut Task) {
2709 if let Ok(mut stored) = read_path(&self.path_of(&task.id)) {
2710 stored.import_chat_notice();
2711 if stored.reassignments.len() > task.reassignments.len() {
2712 let stale = std::mem::replace(task, stored.clone());
2715 for answer in stale.answers {
2716 if !task.answers.contains(&answer) {
2717 task.answers.push(answer);
2718 }
2719 }
2720 }
2721 for receipt in &stored.repair_grants {
2722 if !task
2723 .repair_grants
2724 .iter()
2725 .any(|r| r.request.request == receipt.request.request)
2726 {
2727 task.repair_grants.push(receipt.clone());
2729 task.resume_override = stored.resume_override.clone();
2730 task.status = stored.status;
2731 task.hold_source = stored.hold_source;
2732 task.hold_reason = stored.hold_reason.clone();
2733 task.held_at = stored.held_at;
2734 }
2735 }
2736 task.superseded_resumes.extend(stored.superseded_resumes);
2737 task.dispatched_resumes.extend(stored.dispatched_resumes);
2738 if task.status != TaskStatus::Running
2739 && let Some(pin) = &stored.resume_override
2740 && !pin.question_id.is_empty()
2741 && let Some(run) = &pin.pinned_run
2742 && task.resume_override.as_ref().is_none_or(|next| {
2743 next.question_id != pin.question_id || next.pinned_run != pin.pinned_run
2744 })
2745 {
2746 task.superseded_resumes
2747 .insert(pin.question_id.clone(), run.clone());
2748 }
2749 for (id, receipt) in stored.decision_effects {
2752 task.decision_effects.insert(id, receipt);
2753 }
2754 if !task.decision_effects.is_empty() {
2755 task.schema = task.schema.max(SCHEMA);
2756 }
2757 if task.chat.is_none() {
2760 task.chat = stored.chat.clone();
2761 }
2762 for key in stored.chat_notices {
2763 if !task.chat_notices.contains(&key) {
2764 task.chat_notices.push(key);
2765 }
2766 }
2767 for failure in stored.startup_failures {
2768 task.record_startup_failure(failure);
2769 }
2770 for run in stored.runs {
2771 if !task.runs.contains(&run) {
2772 task.runs.push(run);
2773 }
2774 }
2775 for p in stored.passes {
2776 if !task
2777 .passes
2778 .iter()
2779 .any(|q| q.run == p.run && q.pass_index == p.pass_index)
2780 {
2781 task.passes.push(p);
2782 }
2783 }
2784 if let Some(n) = stored.chat_notice
2787 && task.chat_notice.as_ref().is_none_or(|mine| mine.at < n.at)
2788 {
2789 task.chat_notice = Some(n);
2790 }
2791 }
2792 }
2793
2794 pub(crate) fn write_startup_history(&self, task: &Task) -> Result<()> {
2797 let path = self.path_of(&task.id);
2798 let tmp = path.with_extension("json.tmp");
2799 std::fs::write(&tmp, serde_json::to_vec_pretty(task)?)?;
2800 std::fs::rename(tmp, path)?;
2801 Ok(())
2802 }
2803
2804 fn write_record(&self, task: &mut Task) -> Result<()> {
2806 task.import_chat_notice();
2807 task.updated_at = Timestamp::now();
2808 std::fs::create_dir_all(&self.root)
2809 .with_context(|| format!("create {}", self.root.display()))?;
2810 let body = serde_json::to_string_pretty(task).context("serialize task")?;
2811 let path = self.path_of(&task.id);
2812 let tmp = path.with_extension("json.tmp");
2813 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
2814 std::fs::rename(&tmp, &path).with_context(|| format!("replace {}", path.display()))?;
2815 if let (Some(notice), Some(home)) = (
2820 crate::notices::task_held(task),
2821 self.root.parent().filter(|p| !p.as_os_str().is_empty()),
2822 ) {
2823 crate::notices::raise_in(home, notice);
2824 }
2825 Ok(())
2826 }
2827
2828 pub fn modify(&self, id: &str, f: impl FnOnce(&mut Task) -> bool) -> Result<bool> {
2833 let id = self.resolve_id(id)?;
2834 let _lock = self.lock_task(&id)?;
2835 let mut task = self.get(&id)?;
2836 if !f(&mut task) {
2837 return Ok(false);
2838 }
2839 self.put_unlocked(&mut task)?;
2840 Ok(true)
2841 }
2842
2843 pub fn link_run(&self, id: &str, run: &str) -> Result<Task> {
2848 let id = self.resolve_id(id)?;
2849 let _lock = self.lock_task(&id)?;
2852 let mut task = self.get(&id)?;
2853 if task.link_run(run) {
2854 self.put_unlocked(&mut task)?;
2855 }
2856 Ok(task)
2857 }
2858
2859 pub(crate) fn lock_task(&self, id: &str) -> Result<TaskLock> {
2867 std::fs::create_dir_all(&self.root)
2868 .with_context(|| format!("create {}", self.root.display()))?;
2869 let path = self.root.join(format!("{id}.write-lock"));
2870 let started = std::time::Instant::now();
2871 let retry = crate::lock_file::Retry::task_write(started + TASK_LOCK_STALE);
2872 let marker_retry = crate::lock_file::Retry::new(Some(started + TASK_LOCK_STALE));
2873 loop {
2874 match retry.create(&path) {
2875 Ok(mut file) => {
2876 use std::io::Write;
2877 let token = owner_token();
2878 if let Err(e) = file.write_all(token.as_bytes()) {
2879 drop(file);
2880 let _ = std::fs::remove_file(&path);
2881 return Err(e).with_context(|| format!("lock {}", path.display()));
2882 }
2883 return Ok(TaskLock { path, token });
2884 }
2885 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2886 let judged = std::fs::read_to_string(&path).ok();
2889 let broken = judged
2890 .filter(|_| older_than_stale(&path))
2891 .is_some_and(|judged| break_stale_with(&path, &judged, marker_retry));
2892 if !broken {
2896 if started.elapsed() > TASK_LOCK_STALE {
2897 bail!("could not lock task {id}");
2898 }
2899 std::thread::sleep(std::time::Duration::from_millis(15));
2900 }
2901 }
2902 Err(e) => return Err(e).with_context(|| format!("lock {}", path.display())),
2903 }
2904 }
2905 }
2906
2907 pub fn get(&self, id: &str) -> Result<Task> {
2909 let resolved = self.resolve_id(id)?;
2910 read_path(&self.path_of(&resolved))
2911 }
2912
2913 pub fn remove(&self, id: &str, in_flight: bool, questions: &Questions) -> Result<Removal> {
2945 self.remove_with_receipt(id, in_flight, questions, None)
2946 }
2947
2948 pub(crate) fn discard_triage(
2950 &self,
2951 id: &str,
2952 questions: &Questions,
2953 inquiry: &str,
2954 ) -> Result<Removal> {
2955 self.remove_with_receipt(id, false, questions, Some(inquiry))
2956 }
2957
2958 pub(crate) fn triage_discard_applied(&self, id: &str, inquiry: &str) -> Result<bool> {
2959 let path = self
2960 .root
2961 .join("triage-discard")
2962 .join(format!("{inquiry}.json"));
2963 let bytes = match std::fs::read(path) {
2964 Ok(bytes) => bytes,
2965 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
2966 Err(e) => return Err(e.into()),
2967 };
2968 let receipt: Task = serde_json::from_slice(&bytes)?;
2969 Ok(receipt.id == id && !self.path_of(id).exists())
2970 }
2971
2972 fn remove_with_receipt(
2973 &self,
2974 id: &str,
2975 in_flight: bool,
2976 questions: &Questions,
2977 inquiry: Option<&str>,
2978 ) -> Result<Removal> {
2979 let resolved = self.resolve_id(id)?;
2980 if in_flight {
2981 bail!("task {resolved} is being run by a live daemon right now");
2982 }
2983 let receipt = inquiry.map(|q| self.root.join("triage-discard").join(format!("{q}.json")));
2984 if let Some(path) = &receipt {
2985 std::fs::create_dir_all(path.parent().expect("receipt directory"))?;
2986 }
2987 self.write_tombstone(&resolved)?;
2988 if let Err(e) = self.remove_record_with_attachments(&resolved, |p| match &receipt {
2989 Some(path) => std::fs::rename(p, path),
2990 None => std::fs::remove_file(p),
2991 }) {
2992 if self.path_of(&resolved).exists() {
2996 let _ = std::fs::remove_file(self.tombstone_path(&resolved));
2997 }
2998 return Err(e);
2999 }
3000 let (released, still_blocked) = self.release_dependents_of(&resolved, questions);
3001 Ok(Removal {
3002 id: resolved,
3003 released,
3004 still_blocked,
3005 })
3006 }
3007
3008 fn tombstone_path(&self, id: &str) -> PathBuf {
3011 self.root.join(format!("{id}.removed"))
3012 }
3013
3014 fn write_tombstone(&self, id: &str) -> Result<()> {
3015 let path = self.tombstone_path(id);
3016 let tmp = self.root.join(format!("{id}.removed.tmp"));
3017 std::fs::write(&tmp, b"")
3018 .and_then(|()| std::fs::rename(&tmp, &path))
3019 .with_context(|| format!("write {}", path.display()))
3020 }
3021
3022 fn was_deleted(&self, id: &str) -> bool {
3026 self.tombstone_path(id).is_file() && !self.path_of(id).exists()
3027 }
3028
3029 pub fn apply_deleted_blockers(&self, task: &mut Task, questions: &Questions) -> Vec<String> {
3033 let deleted = deleted_blockers(self, questions, &task.blocked_by);
3034 deleted
3035 .into_iter()
3036 .filter(|id| task.dependency_deleted(id))
3037 .collect()
3038 }
3039
3040 pub fn note_dependency_deleted(&self, dependent: &Task, deleted: &str) {
3044 let Some(home) = self.root.parent().filter(|p| !p.as_os_str().is_empty()) else {
3045 return;
3046 };
3047 let ja = crate::lang::is_japanese(&crate::lang::of_repo(&dependent.repo));
3048 let waiting = dependent.status == TaskStatus::Blocked;
3049 let message = match (ja, waiting) {
3050 (true, false) => format!(
3051 "タスク {} は、待っていた {} が削除されたため待機を解除し、元の状態に戻しました",
3052 dependent.short(),
3053 short(deleted)
3054 ),
3055 (true, true) => format!(
3056 "タスク {} は、待っていた {} が削除されたため、残りの依存を待っています",
3057 dependent.short(),
3058 short(deleted)
3059 ),
3060 (false, false) => format!(
3061 "Task {} stopped waiting on {} because it was deleted, and returned to its previous state",
3062 dependent.short(),
3063 short(deleted)
3064 ),
3065 (false, true) => format!(
3066 "Task {} stopped waiting on {} because it was deleted, and is still waiting on its other dependencies",
3067 dependent.short(),
3068 short(deleted)
3069 ),
3070 };
3071 crate::notices::raise_in(
3072 home,
3073 crate::notices::Notice::info(
3074 &format!("unblocked:{}:{}", dependent.id, deleted),
3075 message,
3076 )
3077 .link(crate::notices::Link::Task {
3078 id: dependent.id.clone(),
3079 }),
3080 );
3081 }
3082
3083 fn release_dependents_of(
3088 &self,
3089 dependency: &str,
3090 questions: &Questions,
3091 ) -> (Vec<String>, Vec<String>) {
3092 let (mut released, mut still_blocked) = (Vec::new(), Vec::new());
3093 for listed in self.list() {
3094 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| {
3095 matches!(crate::queue::dependency(self, questions, b), Dependency::Deleted(id) if id == dependency)
3096 }) {
3097 continue;
3098 }
3099 let Ok(_claim) = self.claim(&listed.id) else {
3100 continue;
3101 };
3102 let Ok(mut task) = self.get(&listed.id) else {
3103 continue;
3104 };
3105 let references: Vec<String> = task.blocked_by.iter().filter(|id| {
3106 matches!(crate::queue::dependency(self, questions, id), Dependency::Deleted(full) if full == dependency)
3107 }).cloned().collect();
3108 if references.is_empty() {
3109 continue;
3110 }
3111 for reference in references {
3112 task.dependency_deleted(&reference);
3113 }
3114 if self.put(&mut task).is_err() {
3115 continue;
3116 }
3117 self.note_dependency_deleted(&task, dependency);
3118 if task.status == TaskStatus::Blocked {
3119 still_blocked.push(task.id.clone());
3120 } else {
3121 released.push(task.id.clone());
3122 }
3123 }
3124 (released, still_blocked)
3125 }
3126
3127 fn remove_record_with_attachments(
3131 &self,
3132 resolved: &str,
3133 remove_record: impl FnOnce(&Path) -> std::io::Result<()>,
3134 ) -> Result<()> {
3135 self.sweep_removed_attachments();
3142 let attachments = self.attachments_dir(resolved);
3143 let aside = self.root.join(format!("{resolved}.attachments.removing"));
3144 let moved = match std::fs::rename(&attachments, &aside) {
3145 Ok(()) => true,
3146 Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
3147 Err(e) => {
3148 return Err(e).with_context(|| format!("remove {}", attachments.display()));
3149 }
3150 };
3151 let path = self.path_of(resolved);
3152 if let Err(e) = remove_record(&path) {
3153 if moved {
3154 let _ = std::fs::rename(&aside, &attachments);
3155 }
3156 return Err(e).with_context(|| format!("remove {}", path.display()));
3157 }
3158 if moved {
3159 if let Err(e) = std::fs::remove_dir_all(&aside) {
3160 tracing::warn!("leftover attachments {}: {e}", aside.display());
3161 }
3162 }
3163 let lock = self.lock_path(resolved);
3164 if let Err(e) = std::fs::remove_file(&lock) {
3165 if e.kind() != std::io::ErrorKind::NotFound {
3166 return Err(e).with_context(|| format!("remove {}", lock.display()));
3167 }
3168 }
3169 Ok(())
3170 }
3171
3172 fn sweep_removed_attachments(&self) {
3178 let Ok(entries) = std::fs::read_dir(&self.root) else {
3179 return;
3180 };
3181 for entry in entries.flatten() {
3182 let name = entry.file_name();
3183 let name = name.to_string_lossy();
3184 let Some(id) = name.strip_suffix(".attachments.removing") else {
3185 continue;
3186 };
3187 if !self.path_of(id).exists() {
3190 if let Err(e) = std::fs::remove_dir_all(entry.path()) {
3191 tracing::warn!("leftover attachments {}: {e}", entry.path().display());
3192 }
3193 }
3194 }
3195 self.sweep_tombstones();
3196 }
3197
3198 fn sweep_tombstones(&self) {
3207 let Ok(entries) = std::fs::read_dir(&self.root) else {
3208 return;
3209 };
3210 let tasks = self.list();
3211 for entry in entries.flatten() {
3212 let name = entry.file_name();
3213 let name = name.to_string_lossy();
3214 let Some(id) = name.strip_suffix(".removed") else {
3215 continue;
3216 };
3217 if self.path_of(id).exists()
3218 || tasks.iter().any(|t| {
3219 t.blocked_by
3220 .iter()
3221 .any(|b| id.starts_with(b) || id.ends_with(b))
3222 })
3223 {
3224 continue;
3225 }
3226 let old = entry
3227 .metadata()
3228 .and_then(|m| m.modified())
3229 .ok()
3230 .and_then(|m| m.elapsed().ok())
3231 .is_some_and(|age| age >= TOMBSTONE_GRACE);
3232 if old {
3233 let _ = std::fs::remove_file(entry.path());
3234 }
3235 }
3236 }
3237
3238 fn lock_path(&self, id: &str) -> PathBuf {
3241 self.root.join(format!("{id}.lock"))
3242 }
3243
3244 pub fn list(&self) -> Vec<Task> {
3257 let mut tasks: Vec<Task> = std::fs::read_dir(&self.root)
3258 .into_iter()
3259 .flatten()
3260 .flatten()
3261 .map(|e| e.path())
3262 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3263 .filter_map(|p| read_path(&p).ok())
3264 .collect();
3265 tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then_with(|| b.id.cmp(&a.id)));
3266 tasks
3267 }
3268
3269 pub fn unreadable(&self) -> Vec<PathBuf> {
3273 std::fs::read_dir(&self.root)
3274 .into_iter()
3275 .flatten()
3276 .flatten()
3277 .map(|e| e.path())
3278 .filter(|p| p.extension().is_some_and(|x| x == "json"))
3279 .filter(|p| read_path(p).is_err())
3280 .collect()
3281 }
3282
3283 pub fn superseded(&self) -> HashMap<String, String> {
3296 let mut by = HashMap::new();
3297 for task in self.list() {
3298 for earlier in &task.runs {
3299 if let Some(later) = task.successor_of(earlier) {
3300 by.insert(earlier.clone(), later.clone());
3301 }
3302 }
3303 }
3304 by
3305 }
3306
3307 pub fn superseded_by(&self, run: &str) -> Option<String> {
3315 for task in self.list() {
3316 if task.runs.iter().any(|r| r == run) {
3317 return task.successor_of(run).cloned();
3318 }
3319 }
3320 None
3321 }
3322
3323 pub fn latest_attempt(&self, run: &str) -> Option<String> {
3334 for task in self.list() {
3335 if task.runs.iter().any(|r| r == run) {
3336 return task.runs.last().filter(|last| **last != run).cloned();
3337 }
3338 }
3339 None
3340 }
3341
3342 pub fn next_runnable(&self) -> Option<Task> {
3347 let mut runnable: Vec<Task> = self
3348 .list()
3349 .into_iter()
3350 .filter(|t| t.status.runnable())
3351 .collect();
3352 runnable.sort_unstable_by(dispatch_order);
3353 runnable.into_iter().next()
3354 }
3355
3356 pub fn claim(&self, id: &str) -> Result<Claim> {
3363 std::fs::create_dir_all(&self.root)
3364 .with_context(|| format!("create {}", self.root.display()))?;
3365 let path = self.lock_path(id);
3366 let retry = crate::lock_file::Retry::new(None);
3367 match retry.create(&path) {
3368 Ok(mut f) => {
3369 use std::io::Write as _;
3370 let _ = writeln!(f, "{}", std::process::id());
3372 Ok(Claim { path })
3373 }
3374 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
3375 bail!("task {id} is already claimed ({} exists)", path.display())
3376 }
3377 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
3378 }
3379 }
3380
3381 pub fn resolve_id(&self, prefix: &str) -> Result<String> {
3383 if !prefix.is_empty()
3384 && prefix != "."
3385 && prefix != ".."
3386 && !prefix.contains(['/', '\\'])
3387 && self.path_of(prefix).is_file()
3388 {
3389 return Ok(prefix.to_owned());
3390 }
3391 let ids = record_ids(&self.root, "json")?;
3392 match resolve_reference(prefix, ids.iter().map(String::as_str)) {
3393 Reference::Found(id) => Ok(id),
3394 Reference::Missing => bail!("no task matches `{prefix}`"),
3395 Reference::Ambiguous => bail!("`{prefix}` matches multiple tasks"),
3396 }
3397 }
3398
3399 pub fn revision(&self) -> u64 {
3406 self.revision_excluding(&std::collections::BTreeSet::new())
3407 }
3408
3409 pub fn revision_excluding(&self, skip: &std::collections::BTreeSet<String>) -> u64 {
3412 use std::hash::{Hash as _, Hasher as _};
3413
3414 let mut entries: Vec<(String, u64)> = std::fs::read_dir(&self.root)
3415 .into_iter()
3416 .flatten()
3417 .flatten()
3418 .filter(|e| e.path().extension().is_some_and(|ext| ext == "json"))
3419 .filter(|e| {
3420 let path = e.path();
3421 !path
3422 .file_stem()
3423 .is_some_and(|stem| skip.contains(stem.to_string_lossy().as_ref()))
3424 })
3425 .filter_map(|e| {
3426 let name = e.file_name().to_string_lossy().into_owned();
3427 let mtime = e
3428 .metadata()
3429 .ok()?
3430 .modified()
3431 .ok()?
3432 .duration_since(std::time::UNIX_EPOCH)
3433 .ok()?
3434 .as_millis() as u64;
3435 Some((name, mtime))
3436 })
3437 .collect();
3438
3439 if entries.is_empty() {
3440 return 0;
3441 }
3442
3443 entries.sort_unstable();
3444 let mut hasher = std::hash::DefaultHasher::new();
3445 for (name, mtime) in &entries {
3446 name.hash(&mut hasher);
3447 mtime.hash(&mut hasher);
3448 }
3449 let h = hasher.finish();
3450 if h == 0 { 1 } else { h }
3451 }
3452}
3453
3454const TOMBSTONE_GRACE: std::time::Duration = std::time::Duration::from_secs(3600);
3457
3458#[derive(Debug, Clone)]
3460pub struct Removal {
3461 pub id: String,
3463 pub released: Vec<String>,
3467 pub still_blocked: Vec<String>,
3470}
3471
3472#[derive(Debug)]
3474pub struct Claim {
3475 path: PathBuf,
3476}
3477
3478impl Drop for Claim {
3479 fn drop(&mut self) {
3480 let _ = std::fs::remove_file(&self.path);
3481 }
3482}
3483
3484pub fn title_from(instruction: &str, max: usize) -> String {
3487 let Some(line) = first_line(instruction) else {
3488 return "(empty task)".to_owned();
3489 };
3490 if line.chars().count() <= max {
3491 return line.to_owned();
3492 }
3493 let head: String = line.chars().take(max.saturating_sub(1)).collect();
3494 format!("{head}…")
3495}
3496
3497pub(crate) fn first_line(instruction: &str) -> Option<&str> {
3503 let line = instruction
3504 .lines()
3505 .map(str::trim)
3506 .find(|l| !l.is_empty())?
3507 .trim_start_matches(['#', '-', '*', '>', ' '])
3508 .trim();
3509 (!line.is_empty()).then_some(line)
3510}
3511
3512fn read_path(path: &Path) -> Result<Task> {
3513 let body = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
3514 let task: Task =
3515 serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
3516 if task.schema > SCHEMA {
3522 bail!(
3523 "task {} was written by a different magi (schema {}, this build \
3524 speaks {SCHEMA})",
3525 task.id,
3526 task.schema
3527 );
3528 }
3529 Ok(task)
3530}
3531
3532#[derive(Debug, PartialEq, Eq)]
3535pub(crate) enum Reference {
3536 Found(String),
3537 Missing,
3538 Ambiguous,
3539}
3540
3541pub(crate) fn resolve_reference<'a>(id: &str, ids: impl Iterator<Item = &'a str>) -> Reference {
3542 let mut hits = Vec::new();
3543 for candidate in ids {
3544 if candidate == id {
3545 return Reference::Found(candidate.to_owned());
3546 }
3547 if candidate.starts_with(id) || candidate.ends_with(id) {
3548 hits.push(candidate);
3549 }
3550 }
3551 match hits.as_slice() {
3552 [] => Reference::Missing,
3553 [one] => Reference::Found((*one).to_owned()),
3554 _ => Reference::Ambiguous,
3555 }
3556}
3557
3558pub(crate) fn record_ids(root: &Path, extension: &str) -> Result<Vec<String>> {
3561 let entries = match std::fs::read_dir(root) {
3562 Ok(entries) => entries,
3563 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
3564 Err(e) => return Err(e.into()),
3565 };
3566 let mut ids = Vec::new();
3567 for entry in entries {
3568 let path = entry?.path();
3569 if path.extension().is_some_and(|e| e == extension)
3570 && let Some(id) = path.file_stem().and_then(|s| s.to_str())
3571 {
3572 ids.push(id.to_owned());
3573 }
3574 }
3575 Ok(ids)
3576}
3577
3578#[derive(Debug, PartialEq, Eq)]
3581pub enum Dependency {
3582 Task(String),
3584 Question(String),
3586 Deleted(String),
3588 Missing,
3590 Ambiguous,
3592 Unknown,
3594}
3595
3596pub fn dependency(queue: &Queue, questions: &Questions, id: &str) -> Dependency {
3598 let (Ok(tasks), Ok(qs), Ok(removed)) = (
3599 record_ids(queue.root(), "json"),
3600 record_ids(questions.root(), "json"),
3601 record_ids(queue.root(), "removed"),
3602 ) else {
3603 return Dependency::Unknown;
3604 };
3605 let mut ids = tasks
3608 .iter()
3609 .chain(qs.iter())
3610 .chain(removed.iter())
3611 .cloned()
3612 .collect::<Vec<_>>();
3613 ids.sort();
3614 ids.dedup();
3615 match resolve_reference(id, ids.iter().map(String::as_str)) {
3616 Reference::Found(id) if tasks.contains(&id) => Dependency::Task(id),
3617 Reference::Found(id) if qs.contains(&id) => Dependency::Question(id),
3618 Reference::Found(id) if queue.was_deleted(&id) => Dependency::Deleted(id),
3619 Reference::Found(_) => Dependency::Unknown,
3620 Reference::Missing => Dependency::Missing,
3621 Reference::Ambiguous => Dependency::Ambiguous,
3622 }
3623}
3624
3625pub fn normalize_blockers(queue: &Queue, questions: &Questions, ids: &mut [String]) {
3627 for id in ids {
3628 if let Dependency::Task(full) | Dependency::Question(full) | Dependency::Deleted(full) =
3629 dependency(queue, questions, id)
3630 {
3631 *id = full;
3632 }
3633 }
3634}
3635
3636pub fn dependency_facts(queue: &Queue, questions: &Questions) -> String {
3639 let inventory = crate::blockers::Inventory::load(queue, questions);
3640 let mut text = String::from(
3641 "Current authoritative queue (historical missing-task claims may be stale; unreadable or ambiguous references must keep waiting):\n",
3642 );
3643 match record_ids(queue.root(), "json") {
3644 Ok(ids) => {
3645 for id in ids {
3646 match queue.get(&id) {
3647 Ok(task) => text.push_str(&format!(
3648 "{}: {}; blocked_by={:?}; chain={:?}\n",
3649 task.id,
3650 task.status.as_str(),
3651 task.blocked_by,
3652 inventory.waits_on(&task)
3653 )),
3654 Err(_) => text.push_str(&format!("{id}: exists, state unreadable\n")),
3655 }
3656 }
3657 }
3658 Err(_) => text.push_str("Queue inventory unreadable: existence is unknown.\n"),
3659 }
3660 text
3661}
3662
3663pub fn missing_blockers(
3665 queue: &Queue,
3666 questions: &Questions,
3667 blocked_by: &[String],
3668) -> Vec<String> {
3669 blocked_by
3670 .iter()
3671 .filter(|id| dependency(queue, questions, id) == Dependency::Missing)
3672 .cloned()
3673 .collect()
3674}
3675
3676pub fn deleted_blockers(
3679 queue: &Queue,
3680 questions: &Questions,
3681 blocked_by: &[String],
3682) -> Vec<String> {
3683 blocked_by
3684 .iter()
3685 .filter(|id| matches!(dependency(queue, questions, id), Dependency::Deleted(_)))
3686 .cloned()
3687 .collect()
3688}
3689
3690pub fn missing_blocker_hold_reason(blocked_by: &[String], missing: &[String]) -> String {
3702 missing_blocker_hold_reason_in(blocked_by, missing, "en")
3703}
3704
3705pub fn missing_blocker_hold_reason_in(
3707 blocked_by: &[String],
3708 missing: &[String],
3709 language: &str,
3710) -> String {
3711 if crate::lang::is_japanese(language) {
3712 format!(
3713 "{} を待っていましたが、{} はディスク上に存在しません - `magi task triage` を参照",
3714 blocked_by.join(", "),
3715 missing.join(", "),
3716 )
3717 } else {
3718 format!(
3719 "blocked on {} but {} no longer exist(s) on disk - see `magi task triage`",
3720 blocked_by.join(", "),
3721 missing.join(", "),
3722 )
3723 }
3724}
3725
3726pub fn short(id: &str) -> &str {
3728 id.split('-').next_back().unwrap_or(id)
3729}
3730
3731fn new_id() -> String {
3732 let stamp = jiff::Zoned::now().strftime("%Y%m%d-%H%M%S");
3733 let seed = crate::rng::entropy();
3734 format!("{stamp}-{:04x}", (seed ^ (seed >> 32)) & 0xffff)
3735}
3736
3737#[cfg(test)]
3738mod tests {
3739 #[test]
3740 fn reassignment_clears_repository_bound_execution_selections() {
3741 let mut task = Task::new(
3742 "move".into(),
3743 "evidence".into(),
3744 "old".into(),
3745 Source::Human,
3746 );
3747 task.review_branch = Some("old-review".into());
3748 task.review_instruction = Some("old criteria".into());
3749 task.pr_recovery = Some(crate::pr_recovery::Selection {
3750 url: "https://example.invalid/pull/1".into(),
3751 branch: "old-pr".into(),
3752 head: "head".into(),
3753 source_head: None,
3754 source_run: None,
3755 source_proof: None,
3756 disposition: None,
3757 instruction: "old criteria".into(),
3758 approved_at: Timestamp::now(),
3759 });
3760 task.replacement = Some(crate::merge_decision::Replacement {
3761 question: "owner".into(),
3762 run: "old-run".into(),
3763 pr: "https://example.invalid/pull/1".into(),
3764 agent: "worker".into(),
3765 context: "evidence".into(),
3766 });
3767 task.resume_override = Some(OperatorResume {
3768 question_id: "old-question".into(),
3769 at: Timestamp::now(),
3770 conductor_rehold: None,
3771 forced: true,
3772 pinned_run: Some("old-run".into()),
3773 });
3774 task.interrupt = true;
3775 task.reassign_repository("new".into(), "test");
3776 assert!(task.review_branch.is_none());
3777 assert!(task.review_instruction.is_none());
3778 assert!(task.pr_recovery.is_none());
3779 assert!(task.replacement.is_none());
3780 assert!(task.resume_override.is_none());
3781 assert!(!task.interrupt);
3782 assert!(task.fresh_start);
3783 assert_eq!(task.instruction, "evidence");
3784 }
3785
3786 #[test]
3787 fn explicit_resumption_is_not_invented_for_legacy_or_automatic_release() {
3788 let t = Task::new(
3789 "retry".into(),
3790 "do it".into(),
3791 PathBuf::from("."),
3792 Source::Human,
3793 );
3794 let mut value = serde_json::to_value(&t).unwrap();
3795 for field in ["resumption", "resumption_history", "operator_decided_at"] {
3796 value.as_object_mut().unwrap().remove(field);
3797 }
3798 value["schema"] = serde_json::json!(15);
3799 let mut legacy: Task = serde_json::from_value(value).unwrap();
3800 legacy.hold_machine(Some("automatic retry".into()));
3801 legacy.release();
3802 assert!(legacy.resumption.is_none());
3803 assert!(legacy.resumption_history.is_empty());
3804 assert!(legacy.operator_decided_at.is_none());
3805 legacy.release_operator("cli", None);
3806 legacy.start("run".into());
3807 let reloaded: Task =
3808 serde_json::from_str(&serde_json::to_string(&legacy).unwrap()).unwrap();
3809 assert!(reloaded.resume_override.is_none());
3810 assert_eq!(reloaded.resumption.as_ref().unwrap().generation, 1);
3811 }
3812
3813 #[test]
3814 fn merge_over_lets_the_later_choice_win_and_keeps_the_rest() {
3815 let mut base = RunOverrides {
3816 merge: Some("pr".to_owned()),
3817 candidates: Some(3),
3818 ..RunOverrides::default()
3819 };
3820 base.merge_over(&RunOverrides {
3821 merge: Some("none".to_owned()),
3822 seed: Some(7),
3823 ..RunOverrides::default()
3824 });
3825 assert_eq!(base.merge.as_deref(), Some("none"));
3826 assert_eq!(base.candidates, Some(3));
3827 assert_eq!(base.seed, Some(7));
3828 }
3829
3830 #[test]
3831 fn an_already_landed_task_is_done_with_its_attempt_refunded() {
3832 let mut t = task("relanded");
3833 t.attempts = 1;
3834 t.status = TaskStatus::Running;
3835 t.already_landed("already in main as 0e368de");
3836 assert_eq!(t.status, TaskStatus::Done);
3837 assert_eq!(t.attempts, 0);
3838 assert!(t.hold_reason.is_none());
3839 assert_eq!(t.last_error.as_deref(), Some("already in main as 0e368de"));
3840 }
3841
3842 #[test]
3843 fn missing_blocker_reason_follows_the_language() {
3844 let b = vec!["a".to_owned()];
3845 let en = missing_blocker_hold_reason_in(&b, &b, "en");
3846 assert_eq!(en, missing_blocker_hold_reason(&b, &b));
3847 assert!(en.starts_with("blocked on a"));
3848 assert!(missing_blocker_hold_reason_in(&b, &b, "ja").contains("存在しません"));
3849 assert_eq!(missing_blocker_hold_reason_in(&b, &b, "de"), en);
3850 }
3851
3852 use super::*;
3853
3854 #[test]
3855 fn triage_applied_survives_release_and_old_records_read_as_empty() {
3856 let mut t = Task::new(
3857 "t".to_owned(),
3858 "i".to_owned(),
3859 PathBuf::from("r"),
3860 Source::Human,
3861 );
3862 t.mark_triage_applied("q1");
3863 t.mark_triage_applied("q1");
3864 t.hold_machine(Some("x".to_owned()));
3865 t.release();
3866 assert_eq!(t.triage_applied, ["q1"]);
3867 assert!(t.triage_applied("q1") && !t.triage_applied("q2"));
3868
3869 let mut v = serde_json::to_value(&t).unwrap();
3870 v.as_object_mut().unwrap().remove("triage_applied");
3871 let old: Task = serde_json::from_value(v).unwrap();
3872 assert!(old.triage_applied.is_empty());
3873 }
3874
3875 #[test]
3876 fn task_counts_of_empty_is_all_zero() {
3877 assert_eq!(TaskCounts::of(&[]), TaskCounts::default());
3878 }
3879
3880 #[test]
3881 fn task_counts_of_tallies_every_status() {
3882 let mut queued = Task::new(
3883 "q".to_owned(),
3884 "i".to_owned(),
3885 PathBuf::from("."),
3886 Source::Human,
3887 );
3888 queued.status = TaskStatus::Queued;
3889 let mut running = queued.clone();
3890 running.status = TaskStatus::Running;
3891 let mut done = queued.clone();
3892 done.status = TaskStatus::Done;
3893 let mut failed = queued.clone();
3894 failed.status = TaskStatus::Failed;
3895 let mut held = queued.clone();
3896 held.status = TaskStatus::Held;
3897 let mut blocked = queued.clone();
3898 blocked.status = TaskStatus::Blocked;
3899
3900 let mut parked = queued.clone();
3901 parked.status = TaskStatus::Parked;
3902
3903 let counts = TaskCounts::of(&[
3904 queued,
3905 running,
3906 done.clone(),
3907 done,
3908 failed,
3909 held,
3910 blocked,
3911 parked,
3912 ]);
3913 assert_eq!(
3914 counts,
3915 TaskCounts {
3916 queued: 1,
3917 running: 1,
3918 done: 2,
3919 failed: 1,
3920 held: 1,
3921 blocked: 1,
3922 parked: 1,
3923 }
3924 );
3925 }
3926
3927 #[test]
3928 fn task_implementers_persist_normalized_without_losing_history() {
3929 let (_dir, q) = queue();
3930 let mut t = task("keep me");
3931 t.hold_manual(Some("waiting".into()));
3932 t.runs.push("earlier-run".into());
3933 t.attachments.push("reference.txt".into());
3934 let cfg: crate::config::Config =
3935 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
3936 .unwrap();
3937 q.put(&mut t).unwrap();
3938 let original = serde_json::to_value(&t).unwrap();
3939 for n in [2, 1, 2] {
3940 t.set_implementers(n, &cfg).unwrap();
3941 q.put(&mut t).unwrap();
3942 t = q.get(&t.id).unwrap();
3943 assert_eq!(t.implementers(), Some(usize::from(n)));
3944 assert_eq!(t.solo, n == 1);
3945 if n == 1 {
3946 assert!(t.overrides.is_none());
3947 }
3948 let mut actual = serde_json::to_value(&t).unwrap();
3949 for field in ["solo", "overrides", "updated_at"] {
3950 actual[field] = original[field].clone();
3951 }
3952 assert_eq!(actual, original);
3953 }
3954 t.overrides.as_mut().unwrap().judges = Some(2);
3955 t.set_implementers(1, &cfg).unwrap();
3956 assert_eq!(t.overrides.as_ref().unwrap().judges, Some(2));
3957 assert_eq!(t.overrides.as_ref().unwrap().candidates, None);
3958 }
3959
3960 #[test]
3961 fn task_implementers_old_records_keep_inheritance_until_changed() {
3962 let mut value = serde_json::to_value(task("old task")).unwrap();
3963 value.as_object_mut().unwrap().remove("solo");
3964 value.as_object_mut().unwrap().remove("overrides");
3965 let mut old: Task = serde_json::from_value(value).unwrap();
3966 assert_eq!(old.implementers(), None);
3967 old.set_implementers(1, &crate::config::Config::default())
3968 .unwrap();
3969 assert!(old.solo);
3970 assert!(old.overrides.is_none());
3971 }
3972
3973 #[test]
3974 fn task_implementers_reject_invalid_counts_and_states() {
3975 let cfg = crate::config::Config::default();
3976 let mut t = task("unchanged");
3977 for n in [0, 4, 255] {
3978 assert!(t.set_implementers(n, &cfg).is_err());
3979 }
3980 for status in [
3981 TaskStatus::Running,
3982 TaskStatus::Done,
3983 TaskStatus::Failed,
3984 TaskStatus::Blocked,
3985 TaskStatus::Parked,
3986 ] {
3987 t.status = status;
3988 let before = serde_json::to_value(&t).unwrap();
3989 assert!(t.set_implementers(1, &cfg).is_err());
3990 assert_eq!(serde_json::to_value(&t).unwrap(), before);
3991 }
3992 }
3993
3994 fn queue() -> (tempfile::TempDir, Queue) {
3997 let dir = tempfile::tempdir().unwrap();
3998 let q = Queue::at(dir.path().join("queue"));
3999 (dir, q)
4000 }
4001
4002 #[test]
4003 fn a_stale_write_does_not_drop_the_recorded_task_chat() {
4004 let (_dir, q) = queue();
4005 let mut stale = task("chat");
4006 q.put(&mut stale).unwrap();
4007 let mut fresh = q.get(&stale.id).unwrap();
4008 fresh.chat = Some("talk-1".to_owned());
4009 q.put(&mut fresh).unwrap();
4010 q.put(&mut stale).unwrap();
4011 assert_eq!(q.get(&stale.id).unwrap().chat.as_deref(), Some("talk-1"));
4012 }
4013
4014 #[test]
4015 fn putting_a_machine_held_task_files_a_notification_beside_the_queue() {
4016 let dir = tempfile::tempdir().unwrap();
4017 let q = Queue::at(dir.path().join("queue"));
4018 let mut t = task("held");
4019 q.put(&mut t).unwrap();
4020 assert_eq!(
4021 crate::notices::Notices::at(dir.path().join("notifications"))
4022 .list()
4023 .len(),
4024 0
4025 );
4026 t.hold_machine(Some("out of attempts".to_owned()));
4027 q.put(&mut t).unwrap();
4028 let listed = crate::notices::Notices::at(dir.path().join("notifications")).list();
4029 assert_eq!(listed.len(), 1);
4030 assert!(listed[0].message.contains("out of attempts"));
4031 }
4032
4033 fn task(title: &str) -> Task {
4034 Task::new(
4035 title.to_owned(),
4036 format!("do {title}"),
4037 PathBuf::from("."),
4038 Source::Human,
4039 )
4040 }
4041
4042 #[test]
4043 fn attempt_limit_inherits_old_records_and_controls_failure() {
4044 let mut t = task("budget");
4045 let mut old = serde_json::to_value(&t).unwrap();
4046 old.as_object_mut().unwrap().remove("attempt_limit");
4047 old["schema"] = serde_json::json!(15);
4048 let legacy: Task = serde_json::from_value(old).unwrap();
4049 assert_eq!(legacy.attempt_limit, None);
4050 assert_eq!(legacy.effective_attempt_limit(2), 2);
4051 t.attempts = 2;
4052 t.change_attempt_limit(NonZeroUsize::new(3)).unwrap();
4053 t.fail("retry", 1);
4054 assert_eq!(t.status, TaskStatus::Failed);
4055 t.attempts = 3;
4056 t.fail("stop", 10);
4057 assert_eq!(t.status, TaskStatus::Held);
4058 t.change_attempt_limit(None).unwrap();
4059 assert_eq!(t.status, TaskStatus::Held);
4060 assert_eq!(t.attempts, 3);
4061 t.fail("inherit", 4);
4062 assert_eq!(t.status, TaskStatus::Failed);
4063 for status in [
4064 TaskStatus::Running,
4065 TaskStatus::Done,
4066 TaskStatus::Failed,
4067 TaskStatus::Blocked,
4068 TaskStatus::Parked,
4069 ] {
4070 t.status = status;
4071 assert!(t.change_attempt_limit(NonZeroUsize::new(1)).is_err());
4072 assert_eq!(t.attempt_limit, None);
4073 }
4074 }
4075
4076 #[test]
4077 fn earlier_attempts_is_every_recorded_run_and_agrees_with_the_display() {
4078 let mut t = task("retried");
4079 assert!(
4080 t.earlier_attempts().is_empty(),
4081 "a first attempt takes nothing over"
4082 );
4083 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4084 assert_eq!(t.earlier_attempts(), ["aaaa", "bbbb"]);
4085 assert_eq!(t.successor_of("aaaa"), Some(&"bbbb".to_owned()));
4087 assert_eq!(t.successor_of("bbbb"), None);
4088 assert_eq!(t.successor_of("zzzz"), None);
4089 }
4090
4091 #[test]
4092 fn resumed_runs_are_superseded_only_after_their_last_occurrence() {
4093 let (_dir, q) = queue();
4094 let mut t = task("resumed");
4095 for (runs, successor) in [
4096 (vec!["aaaa", "aaaa"], None),
4097 (vec!["aaaa", "bbbb", "aaaa"], None),
4098 (vec!["aaaa", "aaaa", "bbbb"], Some("bbbb")),
4099 (vec!["aaaa", "bbbb", "aaaa", "cccc"], Some("cccc")),
4100 ] {
4101 t.runs = runs.into_iter().map(str::to_owned).collect();
4102 q.put(&mut t).unwrap();
4103
4104 assert_eq!(t.successor_of("aaaa").map(String::as_str), successor);
4105 assert_eq!(q.superseded_by("aaaa").as_deref(), successor);
4106 assert_eq!(q.superseded().get("aaaa").map(String::as_str), successor);
4107 }
4108 }
4109
4110 #[test]
4111 fn superseded_by_names_the_next_attempt_and_none_for_the_last() {
4112 let (_dir, q) = queue();
4113 let mut t = task("retried");
4114 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4115 q.put(&mut t).unwrap();
4116
4117 assert_eq!(q.superseded_by("aaaa"), Some("bbbb".to_owned()));
4118 assert_eq!(q.superseded_by("bbbb"), Some("cccc".to_owned()));
4119 assert_eq!(
4120 q.superseded_by("cccc"),
4121 None,
4122 "the latest attempt replaces nothing"
4123 );
4124 assert_eq!(
4125 q.superseded_by("never-heard-of-it"),
4126 None,
4127 "a run belonging to no task on this queue is not superseded"
4128 );
4129
4130 let mut by = HashMap::new();
4131 by.insert("aaaa".to_owned(), "bbbb".to_owned());
4132 by.insert("bbbb".to_owned(), "cccc".to_owned());
4133 assert_eq!(
4134 q.superseded(),
4135 by,
4136 "the whole-map and single-run forms must agree"
4137 );
4138 }
4139
4140 #[test]
4141 fn superseded_attempts_is_empty_until_the_task_is_done() {
4142 let mut t = task("retried");
4143 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4144 t.status = TaskStatus::Failed;
4145 assert_eq!(
4146 t.superseded_attempts(true),
4147 &[] as &[String],
4148 "a task still retrying has no attempt yet that a later one made moot"
4149 );
4150
4151 t.status = TaskStatus::Running;
4152 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
4153 }
4154
4155 #[test]
4156 fn superseded_attempts_names_every_run_before_the_one_that_succeeded() {
4157 let mut t = task("retried");
4158 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4159 t.status = TaskStatus::Done;
4160 assert_eq!(
4161 t.superseded_attempts(true),
4162 &["aaaa".to_owned(), "bbbb".to_owned()],
4163 "cccc is the attempt whose success made the task done, and stays out"
4164 );
4165 }
4166
4167 #[test]
4168 fn superseded_attempts_is_empty_for_a_done_task_with_only_one_attempt() {
4169 let mut t = task("first try landed");
4170 t.runs = vec!["aaaa".to_owned()];
4171 t.status = TaskStatus::Done;
4172 assert_eq!(t.superseded_attempts(true), &[] as &[String]);
4173 }
4174
4175 #[test]
4176 fn superseded_attempts_is_empty_when_the_last_run_never_actually_succeeded() {
4177 let mut t = task("closed by hand after a manual merge");
4184 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned()];
4185 t.status = TaskStatus::Done;
4186 assert_eq!(
4187 t.superseded_attempts(false),
4188 &[] as &[String],
4189 "nothing here is provably why the task is done, so nothing is superseded"
4190 );
4191 }
4192
4193 #[test]
4194 fn latest_attempt_names_the_chain_s_current_head_not_just_the_next_one() {
4195 let (_dir, q) = queue();
4196 let mut t = task("retried twice");
4197 t.runs = vec!["aaaa".to_owned(), "bbbb".to_owned(), "cccc".to_owned()];
4198 q.put(&mut t).unwrap();
4199
4200 assert_eq!(
4201 q.latest_attempt("aaaa"),
4202 Some("cccc".to_owned()),
4203 "an old attempt points straight at the chain's current head, not the \
4204 next attempt in the middle of it"
4205 );
4206 assert_eq!(q.latest_attempt("bbbb"), Some("cccc".to_owned()));
4207 assert_eq!(
4208 q.latest_attempt("cccc"),
4209 None,
4210 "the latest attempt is not superseded by anything"
4211 );
4212 assert_eq!(
4213 q.latest_attempt("never-heard-of-it"),
4214 None,
4215 "a run belonging to no task on this queue is not superseded"
4216 );
4217 }
4218
4219 #[test]
4220 fn a_markdown_heading_is_the_title_not_decoration() {
4221 assert_eq!(
4226 title_from("# Rework the config loader\n\nIt re-reads it.\n", 40),
4227 "Rework the config loader"
4228 );
4229 assert_eq!(title_from("- fix the thing", 40), "fix the thing");
4230 assert_eq!(title_from("> quoted task", 40), "quoted task");
4231 assert_eq!(title_from(" \n\n", 40), "(empty task)");
4233 assert_eq!(title_from("###\n", 40), "(empty task)");
4234 }
4235
4236 #[test]
4237 fn a_long_title_is_elided_by_characters_not_bytes() {
4238 let long = "課題".repeat(30);
4240 let title = title_from(&long, 10);
4241 assert_eq!(title.chars().count(), 10);
4242 assert!(title.ends_with('…'));
4243 }
4244
4245 #[test]
4246 fn priority_wins_and_ties_break_oldest_first() {
4247 let (_dir, q) = queue();
4248 let mut a = task("first");
4249 let mut b = task("second");
4250 let mut c = task("urgent");
4251 a.id = "20260101-000001-aaaa".to_owned();
4253 b.id = "20260101-000002-bbbb".to_owned();
4254 c.id = "20260101-000003-cccc".to_owned();
4255 c.priority = 5;
4256 for t in [&mut a, &mut b, &mut c] {
4257 q.put(t).unwrap();
4258 }
4259
4260 assert_eq!(q.next_runnable().unwrap().id, c.id);
4262 c.hold_machine(None);
4263 q.put(&mut c).unwrap();
4264 assert_eq!(q.next_runnable().unwrap().id, a.id);
4266 assert_eq!(q.list().len(), 3, "b is still waiting its turn");
4267 }
4268
4269 #[test]
4270 fn a_blocked_task_never_starves_another_runnable_one() {
4271 let (_dir, q) = queue();
4272 let mut blocked = task("blocked");
4273 blocked.block(vec!["something".to_owned()], None);
4274 q.put(&mut blocked).unwrap();
4275
4276 let mut runnable = task("free to go");
4277 q.put(&mut runnable).unwrap();
4278
4279 let next = q.next_runnable().expect("a runnable task is still offered");
4280 assert_eq!(next.id, runnable.id);
4281 }
4282
4283 #[test]
4284 fn a_held_task_is_never_offered_to_the_loop() {
4285 let (_dir, q) = queue();
4286 let mut t = task("held");
4287 q.put(&mut t).unwrap();
4288 assert!(q.next_runnable().is_some());
4289
4290 t.hold_machine(None);
4291 q.put(&mut t).unwrap();
4292 assert!(
4293 q.next_runnable().is_none(),
4294 "a held task must wait for a human"
4295 );
4296
4297 t.status = TaskStatus::Failed;
4299 q.put(&mut t).unwrap();
4300 assert!(q.next_runnable().is_some());
4301 }
4302
4303 #[test]
4304 fn attempts_are_capped_and_then_the_task_is_held() {
4305 let mut t = task("doomed");
4306
4307 t.start("run-1".to_owned());
4308 t.fail("gate red", 2);
4309 assert_eq!(t.status, TaskStatus::Failed, "one attempt of two: retry");
4310
4311 t.start("run-2".to_owned());
4312 t.fail("gate red", 2);
4313 assert_eq!(
4314 t.status,
4315 TaskStatus::Held,
4316 "out of attempts: stop spending money on it"
4317 );
4318 assert_eq!(t.runs, ["run-1", "run-2"]);
4319 assert_eq!(t.last_error.as_deref(), Some("gate red"));
4320 assert_eq!(
4321 t.hold_reason.as_deref(),
4322 Some("gate red"),
4323 "the hold must say why, not leave hold_reason null next to a \
4324 populated last_error"
4325 );
4326 }
4327
4328 #[test]
4329 fn handing_off_a_task_records_a_hold_reason_too() {
4330 let mut t = task("left a pull request");
4331 t.start("run-1".to_owned());
4332 t.handed_off("run ended with a pull request open [run run-1]");
4333 assert_eq!(t.status, TaskStatus::Held);
4334 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4335 assert_eq!(
4336 t.hold_reason.as_deref(),
4337 Some("run ended with a pull request open [run run-1]")
4338 );
4339 assert_eq!(t.hold_reason, t.last_error);
4340 }
4341
4342 #[test]
4343 fn a_quota_stall_is_refunded_so_the_backlog_survives_the_night() {
4344 let mut t = task("stalled by quota");
4345
4346 t.start("run-1".to_owned());
4347 assert_eq!(t.attempts, 1);
4348 t.stall("judge-1, judge-2 out of quota");
4349 assert_eq!(
4350 t.attempts, 0,
4351 "a closed quota window must not spend the task's retry budget"
4352 );
4353 assert_eq!(t.status, TaskStatus::Failed, "the loop should retry it");
4354 assert_eq!(
4355 t.last_error.as_deref(),
4356 Some("judge-1, judge-2 out of quota")
4357 );
4358
4359 for _ in 0..20 {
4362 t.start("run-n".to_owned());
4363 t.stall("still out of quota");
4364 }
4365 t.start("run-real".to_owned());
4366 t.fail("gate red", 2);
4367 assert_eq!(
4368 t.status,
4369 TaskStatus::Failed,
4370 "the first attempt that was really judged is attempt one"
4371 );
4372 }
4373
4374 #[test]
4375 fn releasing_a_held_task_gives_it_a_real_second_chance() {
4376 let mut t = task("retry me");
4377 t.start("run-1".to_owned());
4378 t.fail("gate red", 1);
4379 assert_eq!(t.status, TaskStatus::Held);
4380
4381 t.release();
4382 assert_eq!(t.status, TaskStatus::Queued);
4383 assert_eq!(t.attempts, 0);
4386 assert!(t.last_error.is_none());
4387 assert_eq!(
4388 t.runs.len(),
4389 1,
4390 "history is kept: attempts reset, evidence does not"
4391 );
4392 }
4393
4394 #[test]
4395 fn a_hold_reason_survives_and_a_release_clears_it() {
4396 let mut t = task("waiting on something else");
4397 t.hold_manual(Some(
4398 "waiting for 20260101-000000-aaaa to land first".to_owned(),
4399 ));
4400 assert_eq!(t.status, TaskStatus::Held);
4401 assert_eq!(
4402 t.hold_reason.as_deref(),
4403 Some("waiting for 20260101-000000-aaaa to land first")
4404 );
4405
4406 t.hold_manual(None);
4408 assert_eq!(
4409 t.hold_reason.as_deref(),
4410 Some("waiting for 20260101-000000-aaaa to land first"),
4411 "a bare re-hold keeps whatever a human already wrote down"
4412 );
4413
4414 let mut plain = task("no reason given");
4416 plain.hold_manual(None);
4417 assert_eq!(plain.status, TaskStatus::Held);
4418 assert!(plain.hold_reason.is_none());
4419
4420 t.release();
4421 assert_eq!(t.status, TaskStatus::Queued);
4422 assert!(
4423 t.hold_reason.is_none(),
4424 "a stale reason must not greet the next person who holds this task"
4425 );
4426 }
4427
4428 #[test]
4429 fn closing_a_held_task_as_done_clears_its_hold_reason_too() {
4430 let mut t = task("landed by hand while held");
4435 t.hold_manual(Some("waiting on 3ed9".to_owned()));
4436 assert_eq!(t.hold_reason.as_deref(), Some("waiting on 3ed9"));
4437
4438 t.succeed();
4439 assert_eq!(t.status, TaskStatus::Done);
4440 assert!(
4441 t.hold_reason.is_none(),
4442 "a done task cannot still be waiting on something"
4443 );
4444 }
4445
4446 #[test]
4447 fn holding_or_closing_a_blocked_task_clears_its_dependency_too() {
4448 let mut held = task("held straight out of blocked");
4455 held.block(
4456 vec!["20260101-000000-dead".to_owned()],
4457 Some("waiting on the migration script".to_owned()),
4458 );
4459 assert_eq!(held.status, TaskStatus::Blocked);
4460
4461 held.hold_manual(None);
4462 assert_eq!(held.status, TaskStatus::Held);
4463 assert!(
4464 held.blocked_by.is_empty(),
4465 "hold overrides the wait, same as release"
4466 );
4467 assert!(held.block_reason.is_none());
4468
4469 let mut done = task("closed straight out of blocked");
4470 done.block(
4471 vec!["20260101-000000-dead".to_owned()],
4472 Some("waiting on the migration script".to_owned()),
4473 );
4474 done.succeed();
4475 assert_eq!(done.status, TaskStatus::Done);
4476 assert!(
4477 done.blocked_by.is_empty(),
4478 "a done task cannot still be waiting on a dependency"
4479 );
4480 assert!(done.block_reason.is_none());
4481 }
4482
4483 #[test]
4484 fn a_blocked_task_is_never_offered_to_the_loop() {
4485 let mut t = task("blocked");
4486 assert!(t.status.runnable());
4487 t.block(
4488 vec!["dep-id".to_owned()],
4489 Some("waits on dep-id".to_owned()),
4490 );
4491 assert_eq!(t.status, TaskStatus::Blocked);
4492 assert!(!t.status.runnable());
4493 assert_eq!(TaskStatus::Blocked.as_str(), "blocked");
4494 }
4495
4496 #[test]
4497 fn unblocking_the_last_dependency_returns_the_task_to_queued() {
4498 let mut t = task("blocked on two");
4499 t.block(
4500 vec!["a".to_owned(), "b".to_owned()],
4501 Some("waits on a and b".to_owned()),
4502 );
4503
4504 t.unblock("a");
4505 assert_eq!(t.status, TaskStatus::Blocked, "b is still outstanding");
4506 assert_eq!(t.blocked_by, ["b"]);
4507
4508 t.unblock("b");
4509 assert_eq!(t.status, TaskStatus::Queued);
4510 assert!(t.blocked_by.is_empty());
4511 assert!(t.block_reason.is_none());
4512 }
4513
4514 #[test]
4515 fn unblocking_an_id_on_a_task_that_is_not_blocked_is_a_no_op() {
4516 let mut t = task("never blocked");
4517 t.unblock("whatever");
4518 assert_eq!(t.status, TaskStatus::Queued);
4519 }
4520
4521 #[test]
4522 fn a_held_task_blocked_on_a_question_returns_to_held_not_queued() {
4523 let mut t = task("held, then asked about");
4528 t.hold_machine(Some("out of attempts".to_owned()));
4529 assert_eq!(t.status, TaskStatus::Held);
4530
4531 t.block(vec!["q1".to_owned()], Some("what now?".to_owned()));
4532 assert_eq!(t.status, TaskStatus::Blocked);
4533
4534 t.record_answer("what now?".to_owned(), "leave it held".to_owned());
4535 t.unblock("q1");
4536 assert_eq!(t.status, TaskStatus::Held, "must restore, not requeue");
4537 assert_eq!(t.hold_reason.as_deref(), Some("out of attempts"));
4538 assert_eq!(t.hold_source, Some(HoldSource::Machine));
4539 assert!(t.blocked_from.is_none(), "consumed once restored");
4540 }
4541
4542 #[test]
4543 fn a_manually_held_task_blocked_on_a_question_returns_to_held() {
4544 let mut t = task("manually held, then asked about");
4545 t.hold_manual(Some("waiting on a dependency".to_owned()));
4546
4547 t.block(vec!["q1".to_owned()], None);
4548 t.unblock("q1");
4549
4550 assert_eq!(t.status, TaskStatus::Held);
4551 assert_eq!(t.hold_source, Some(HoldSource::Manual));
4552 }
4553
4554 #[test]
4555 fn re_blocking_an_already_blocked_task_keeps_the_original_blocked_from() {
4556 let mut t = task("held, blocked twice");
4560 t.hold_machine(None);
4561 t.block(vec!["q1".to_owned()], Some("first".to_owned()));
4562 t.block(
4563 vec!["q1".to_owned(), "q2".to_owned()],
4564 Some("second".to_owned()),
4565 );
4566
4567 t.unblock("q1");
4568 assert_eq!(t.status, TaskStatus::Blocked, "q2 still outstanding");
4569 t.unblock("q2");
4570 assert_eq!(t.status, TaskStatus::Held);
4571 }
4572
4573 #[test]
4574 fn unblocking_a_task_blocked_while_running_lands_on_queued_not_running() {
4575 let mut t = task("blocked mid-run");
4579 t.start("run-1".to_owned());
4580 assert_eq!(t.status, TaskStatus::Running);
4581
4582 t.block(vec!["q1".to_owned()], None);
4583 t.unblock("q1");
4584 assert_eq!(t.status, TaskStatus::Queued);
4585 }
4586
4587 #[test]
4588 fn a_pre_schema_4_blocked_record_with_hold_evidence_restores_to_held() {
4589 let mut t = task("legacy record, held before it was blocked");
4595 t.hold_source = Some(HoldSource::Machine);
4596 t.hold_reason = Some("legacy hold reason".to_owned());
4597 t.status = TaskStatus::Blocked;
4598 t.blocked_by = vec!["q1".to_owned()];
4599 t.blocked_from = None;
4600
4601 t.unblock("q1");
4602 assert_eq!(t.status, TaskStatus::Held);
4603 }
4604
4605 #[test]
4606 fn a_pre_schema_4_blocked_record_with_no_hold_evidence_restores_to_queued() {
4607 let mut t = task("legacy record, ordinary dependency block");
4608 t.status = TaskStatus::Blocked;
4609 t.blocked_by = vec!["dep".to_owned()];
4610 t.blocked_from = None;
4611
4612 t.unblock("dep");
4613 assert_eq!(t.status, TaskStatus::Queued);
4614 }
4615
4616 #[test]
4617 fn answering_a_question_is_recorded_and_survives_a_release() {
4618 let mut t = task("asked something");
4619 t.block(vec!["q1".to_owned()], Some("which backend?".to_owned()));
4620 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
4621 t.unblock("q1");
4622 assert_eq!(t.status, TaskStatus::Queued);
4623 assert_eq!(t.answers.len(), 1);
4624 assert_eq!(t.answers[0].answer, "SQLite");
4625
4626 t.release();
4630 assert_eq!(t.answers.len(), 1, "the answer is not lost on release");
4631 }
4632
4633 #[test]
4634 fn resumption_review_branch_requires_unchanged_authored_criteria() {
4635 for source in ["cli", "web"] {
4636 let mut t = task("review old candidate");
4637 t.request_review("candidate-A".into());
4638 t.hold_for_handover(Some("candidate-A".into()), "handover refused".into());
4639 let mut unchanged = t.clone();
4640 unchanged.release_operator(source, None);
4641 assert_eq!(unchanged.review_branch.as_deref(), Some("candidate-A"));
4642 t.edit(
4643 "review approved head".into(),
4644 "Reuse measured PR head B; fix Clippy; review".into(),
4645 )
4646 .unwrap();
4647 t.release_operator(source, None);
4648 assert!(t.review_branch.is_none());
4649 assert!(t.review_instruction.is_none());
4650 let mut legacy = unchanged;
4652 legacy.review_instruction = None;
4653 legacy.hold_for_handover(None, "still checked out".into());
4654 legacy
4655 .edit("new plan".into(), "Reuse measured PR head B".into())
4656 .unwrap();
4657 legacy.release_operator(source, None);
4658 assert!(legacy.review_branch.is_none());
4659 }
4660 }
4661
4662 #[test]
4663 fn a_refused_handover_keeps_the_review_branch_across_release() {
4664 let mut t = task("refused takeover");
4665 t.start("run-1".to_owned());
4666 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
4667 assert_eq!(t.status, TaskStatus::Held);
4668 assert_eq!(t.attempts, 1);
4669 t.release();
4670 assert_eq!(t.status, TaskStatus::Queued);
4671 assert_eq!(t.attempts, 0);
4672 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4673
4674 t.hold_for_handover(None, "again".to_owned());
4676 t.requeue();
4677 assert!(t.review_branch.is_none());
4678
4679 let mut m = task("manual");
4681 m.review_branch = Some("magi/x/A".to_owned());
4682 m.hold_manual(None);
4683 m.release();
4684 assert!(m.review_branch.is_none());
4685 }
4686
4687 #[test]
4688 fn requesting_review_requeues_the_task_and_remembers_the_branch() {
4689 let mut t = task("blocked run with a surviving branch");
4690 t.start("run-1".to_owned());
4691 t.fail("blocked with major findings", 5);
4692 assert_eq!(t.status, TaskStatus::Failed);
4693
4694 t.request_review("magi/eba2/A".to_owned());
4695 assert_eq!(t.status, TaskStatus::Queued);
4696 assert_eq!(t.attempts, 0);
4697 assert_eq!(t.review_branch.as_deref(), Some("magi/eba2/A"));
4698
4699 t.release();
4701 assert!(t.review_branch.is_none());
4702 }
4703
4704 #[test]
4705 fn conductor_requeue_but_not_an_ordinary_release_forces_a_fresh_start() {
4706 let mut t = task("retry");
4707 t.start("run-1".to_owned());
4708 t.requeue();
4709 assert!(t.fresh_start);
4710
4711 t.release();
4712 assert!(!t.fresh_start);
4713 }
4714
4715 #[test]
4716 fn urgent_only_changes_queued_or_held_and_preserves_settings() {
4717 for status in [
4718 TaskStatus::Queued,
4719 TaskStatus::Held,
4720 TaskStatus::Running,
4721 TaskStatus::Done,
4722 TaskStatus::Failed,
4723 TaskStatus::Blocked,
4724 TaskStatus::Parked,
4725 ] {
4726 let mut t = task("urgent settings");
4727 t.status = status;
4728 t.attempts = 2;
4729 t.hold_reason = Some("waiting".into());
4730 t.priority = 7;
4731 t.interrupt = true;
4732 let before = serde_json::to_value(&t).unwrap();
4733 for urgent in [true, true, false, false] {
4734 let result = t.set_urgent(urgent);
4735 let mut expected = before.clone();
4736 if matches!(status, TaskStatus::Queued | TaskStatus::Held) {
4737 result.unwrap();
4738 expected["urgent"] = urgent.into();
4739 } else {
4740 let error = result.unwrap_err().to_string();
4741 assert!(error.contains(status.as_str()), "{error}");
4742 assert!(error.contains("queued or held"), "{error}");
4743 }
4744 assert_eq!(serde_json::to_value(&t).unwrap(), expected);
4745 }
4746 }
4747 }
4748
4749 #[test]
4750 fn priority_can_be_changed_while_queued_but_not_while_running() {
4751 let mut t = task("reprioritise me");
4752 t.set_priority(5).unwrap();
4753 assert_eq!(t.priority, 5);
4754
4755 t.start("run-1".to_owned());
4756 let err = t.set_priority(9).unwrap_err().to_string();
4757 assert!(err.contains("running"), "{err}");
4758 assert_eq!(t.priority, 5, "the rejected write must not partially apply");
4759 }
4760
4761 #[test]
4762 fn interrupt_can_be_marked_while_queued_but_not_while_running() {
4763 let mut t = task("interrupt me");
4764 assert!(!t.interrupt, "off unless asked, same as any other task");
4765
4766 t.set_interrupt(true).unwrap();
4767 assert!(t.interrupt);
4768
4769 t.start("run-1".to_owned());
4770 assert!(
4771 !t.interrupt,
4772 "the mark is one-shot: dispatching the task fulfils it, \
4773 whatever the run that follows ends up doing"
4774 );
4775 let err = t.set_interrupt(true).unwrap_err().to_string();
4776 assert!(err.contains("running"), "{err}");
4777 t.set_interrupt(false).unwrap();
4780 assert!(!t.interrupt);
4781 }
4782
4783 #[test]
4787 fn a_failed_run_does_not_leave_the_task_still_marked_to_interrupt() {
4788 let mut t = task("interrupt me");
4789 t.set_interrupt(true).unwrap();
4790 t.start("run-1".to_owned());
4791 t.fail("mock failure", 5);
4792 assert_eq!(t.status, TaskStatus::Failed);
4793 assert!(
4794 !t.interrupt,
4795 "one attempt already spent the mark; a retry is an ordinary \
4796 requeue, not a fresh interrupt request"
4797 );
4798 }
4799
4800 #[test]
4801 fn changing_priority_moves_a_task_ahead_in_the_real_queue_order() {
4802 let (_dir, q) = queue();
4803 let mut a = task("first filed");
4804 let mut b = task("second filed");
4805 a.id = "20260101-000001-aaaa".to_owned();
4806 b.id = "20260101-000002-bbbb".to_owned();
4807 q.put(&mut a).unwrap();
4808 q.put(&mut b).unwrap();
4809
4810 assert_eq!(
4811 q.next_runnable().unwrap().id,
4812 a.id,
4813 "with equal priority the older task goes first, so a burst of \
4814 new work cannot starve it"
4815 );
4816 assert_eq!(
4817 q.list()[0].id,
4818 b.id,
4819 "but the list an operator reads is newest first, the same as \
4820 before priority existed - a's turn to run does not make it the \
4821 newest task"
4822 );
4823
4824 let mut a = q.get(&a.id).unwrap();
4825 a.set_priority(10).unwrap();
4826 q.put(&mut a).unwrap();
4827
4828 assert_eq!(
4829 q.next_runnable().unwrap().id,
4830 a.id,
4831 "a raised priority must be reflected the moment it is saved"
4832 );
4833 assert_eq!(
4837 q.list()[0].id,
4838 a.id,
4839 "the raised task must sort first in the list an operator reads, \
4840 not only in next_runnable's own ordering"
4841 );
4842 }
4843
4844 #[test]
4845 fn editing_replaces_title_and_instruction_but_keeps_identity_and_history() {
4846 let mut t = Task::new(
4847 "old title".to_owned(),
4848 "old instruction".to_owned(),
4849 PathBuf::from("/repo"),
4850 Source::Agent {
4851 run: "20260101-000000-beef".to_owned(),
4852 node: "implement".to_owned(),
4853 },
4854 );
4855 let id = t.id.clone();
4856 let created_at = t.created_at;
4857 t.runs.push("20260101-000000-beef".to_owned());
4858
4859 t.edit("new title".to_owned(), "new instruction".to_owned())
4860 .unwrap();
4861
4862 assert_eq!(t.title, "new title");
4863 assert_eq!(t.instruction, "new instruction");
4864 assert_eq!(t.id, id, "editing must not mint a new id");
4865 assert_eq!(t.created_at, created_at);
4866 assert_eq!(
4867 t.source,
4868 Source::Agent {
4869 run: "20260101-000000-beef".to_owned(),
4870 node: "implement".to_owned(),
4871 },
4872 "editing must not turn agent attribution into human"
4873 );
4874 assert_eq!(t.runs, ["20260101-000000-beef"]);
4875 }
4876
4877 #[test]
4878 fn editing_is_refused_once_a_task_is_running_or_finished() {
4879 let mut running = task("in flight");
4880 running.start("run-1".to_owned());
4881 let err = running
4882 .edit("x".to_owned(), "y".to_owned())
4883 .unwrap_err()
4884 .to_string();
4885 assert!(err.contains("running"), "{err}");
4886
4887 let mut done = task("finished");
4888 done.succeed();
4889 let err = done
4890 .edit("x".to_owned(), "y".to_owned())
4891 .unwrap_err()
4892 .to_string();
4893 assert!(err.contains("done"), "{err}");
4894
4895 let mut queued = task("waiting");
4897 queued.edit("x".to_owned(), "y".to_owned()).unwrap();
4898 let mut held = task("parked");
4899 held.hold_machine(None);
4900 held.edit("x".to_owned(), "y".to_owned()).unwrap();
4901 }
4902
4903 #[test]
4904 fn a_task_recorded_without_a_hold_reason_still_reads_as_none() {
4905 let (_dir, q) = queue();
4906 let path = q.path_of("20260101-000000-aaaa");
4907 std::fs::create_dir_all(q.root()).unwrap();
4908 std::fs::write(
4909 &path,
4910 serde_json::json!({
4911 "schema": SCHEMA,
4912 "id": "20260101-000000-aaaa",
4913 "title": "from before hold reasons existed",
4914 "instruction": "from before hold reasons existed",
4915 "repo": ".",
4916 "source": { "kind": "human" },
4917 "status": "held",
4918 "created_at": Timestamp::now().to_string(),
4919 "updated_at": Timestamp::now().to_string(),
4920 })
4921 .to_string(),
4922 )
4923 .unwrap();
4924
4925 let task = q.get("20260101-000000-aaaa").expect("must still read");
4926 assert!(task.hold_reason.is_none());
4927 assert!(task.operator_held());
4928 }
4929
4930 #[test]
4931 fn a_legacy_reasoned_hold_defaults_to_operator_protection() {
4932 let (_dir, q) = queue();
4933 let path = q.path_of("20260101-000000-bbbb");
4934 std::fs::create_dir_all(q.root()).unwrap();
4935 std::fs::write(
4936 &path,
4937 serde_json::json!({
4938 "schema": 2,
4939 "id": "20260101-000000-bbbb",
4940 "title": "old manual recovery",
4941 "instruction": "old manual recovery",
4942 "repo": ".",
4943 "source": { "kind": "human" },
4944 "status": "held",
4945 "hold_reason": "active manual recovery run20260912-224242-daf5",
4946 "created_at": Timestamp::now().to_string(),
4947 "updated_at": Timestamp::now().to_string(),
4948 })
4949 .to_string(),
4950 )
4951 .unwrap();
4952
4953 let task = q.get("20260101-000000-bbbb").expect("must still read");
4954 assert_eq!(task.hold_source, None);
4955 assert!(task.operator_held());
4956 }
4957
4958 #[test]
4959 fn a_task_recorded_without_a_diagnostic_still_reads_as_none() {
4960 let (_dir, q) = queue();
4961 let path = q.path_of("20260101-000000-aaaa");
4962 std::fs::create_dir_all(q.root()).unwrap();
4963 std::fs::write(
4964 &path,
4965 serde_json::json!({
4966 "schema": SCHEMA,
4967 "id": "20260101-000000-aaaa",
4968 "title": "from before diagnostics existed",
4969 "instruction": "from before diagnostics existed",
4970 "repo": ".",
4971 "source": { "kind": "human" },
4972 "status": "held",
4973 "created_at": Timestamp::now().to_string(),
4974 "updated_at": Timestamp::now().to_string(),
4975 })
4976 .to_string(),
4977 )
4978 .unwrap();
4979
4980 let task = q.get("20260101-000000-aaaa").expect("must still read");
4981 assert!(task.diagnostic.is_none());
4982 }
4983
4984 #[test]
4985 fn a_schema_1_task_with_no_blocking_fields_still_reads() {
4986 let (_dir, q) = queue();
4990 let path = q.path_of("20260101-000000-aaaa");
4991 std::fs::create_dir_all(q.root()).unwrap();
4992 std::fs::write(
4993 &path,
4994 serde_json::json!({
4995 "schema": 1,
4996 "id": "20260101-000000-aaaa",
4997 "title": "from before blocking existed",
4998 "instruction": "from before blocking existed",
4999 "repo": ".",
5000 "source": { "kind": "human" },
5001 "status": "queued",
5002 "created_at": Timestamp::now().to_string(),
5003 "updated_at": Timestamp::now().to_string(),
5004 })
5005 .to_string(),
5006 )
5007 .unwrap();
5008
5009 let task = q.get("20260101-000000-aaaa").expect("must still read");
5010 assert!(task.blocked_by.is_empty());
5011 assert!(task.block_reason.is_none());
5012 assert!(task.answers.is_empty());
5013 assert!(task.review_branch.is_none());
5014 }
5015
5016 #[test]
5017 fn releasing_or_finishing_a_task_clears_its_stale_diagnostic() {
5018 let mut held = task("diagnosed");
5023 held.start("run-1".to_owned());
5024 held.fail("gate red", 1);
5025 held.diagnostic = Some("cargo test failed: ...".to_owned());
5026 assert_eq!(held.status, TaskStatus::Held);
5027
5028 held.release();
5029 assert!(held.diagnostic.is_none());
5030
5031 held.diagnostic = Some("cargo test failed: ...".to_owned());
5032 held.succeed();
5033 assert!(held.diagnostic.is_none());
5034 }
5035
5036 #[test]
5037 fn failing_a_task_always_clears_whatever_diagnostic_it_carried() {
5038 let mut t = task("retried");
5039 t.start("run-1".to_owned());
5040 t.diagnostic = Some("stale evidence from a previous hold".to_owned());
5041 t.fail("unrelated config error", 5);
5042 assert_eq!(t.status, TaskStatus::Failed);
5043 assert!(
5044 t.diagnostic.is_none(),
5045 "fail() must not let an old diagnostic outlive the run that produced it"
5046 );
5047 }
5048
5049 #[test]
5050 fn a_claim_is_exclusive_and_releases_on_drop() {
5051 let (_dir, q) = queue();
5052 let mut t = task("contended");
5053 q.put(&mut t).unwrap();
5054
5055 let held = q.claim(&t.id).unwrap();
5056 assert!(
5057 q.claim(&t.id).is_err(),
5058 "two daemons must not drive one task into two runs"
5059 );
5060 drop(held);
5061 assert!(q.claim(&t.id).is_ok(), "a released claim is reclaimable");
5062 }
5063
5064 #[test]
5065 fn a_round_trip_survives_disk() {
5066 let (_dir, q) = queue();
5067 let mut t = Task::new(
5068 "titled".to_owned(),
5069 "body".to_owned(),
5070 PathBuf::from("/repo"),
5071 Source::Agent {
5072 run: "20260101-000000-beef".to_owned(),
5073 node: "implement".to_owned(),
5074 },
5075 );
5076 t.priority = 3;
5077 q.put(&mut t).unwrap();
5078
5079 let back = q.get(&t.id).unwrap();
5080 assert_eq!(back.id, t.id);
5081 assert_eq!(back.priority, 3);
5082 assert_eq!(back.source.label(), "implement@beef");
5083 assert_eq!(q.get(t.short()).unwrap().id, t.id);
5085 }
5086
5087 #[test]
5088 fn an_unreadable_task_does_not_take_the_queue_down() {
5089 let (_dir, q) = queue();
5090 let mut t = task("fine");
5091 q.put(&mut t).unwrap();
5092 std::fs::write(q.root().join("broken.json"), "{ not json").unwrap();
5093
5094 let listed = q.list();
5095 assert_eq!(listed.len(), 1, "the readable task still lists");
5096 assert_eq!(listed[0].id, t.id);
5097 }
5098
5099 #[test]
5100 fn a_task_recorded_without_a_solo_field_still_reads_as_not_solo() {
5101 let (_dir, q) = queue();
5102 let path = q.path_of("20260101-000000-aaaa");
5103 std::fs::create_dir_all(q.root()).unwrap();
5104 std::fs::write(
5105 &path,
5106 serde_json::json!({
5107 "schema": SCHEMA,
5108 "id": "20260101-000000-aaaa",
5109 "title": "from before solo existed",
5110 "instruction": "from before solo existed",
5111 "repo": ".",
5112 "source": { "kind": "human" },
5113 "status": "queued",
5114 "created_at": Timestamp::now().to_string(),
5115 "updated_at": Timestamp::now().to_string(),
5116 })
5117 .to_string(),
5118 )
5119 .unwrap();
5120
5121 let task = q.get("20260101-000000-aaaa").expect("must still read");
5122 assert!(!task.solo, "a queue file with no `solo` field means false");
5123 }
5124
5125 #[test]
5126 fn a_task_recorded_without_an_urgent_field_still_reads_as_not_urgent() {
5127 let (_dir, q) = queue();
5128 let path = q.path_of("20260101-000000-bbbb");
5129 std::fs::create_dir_all(q.root()).unwrap();
5130 std::fs::write(
5131 &path,
5132 serde_json::json!({
5133 "schema": SCHEMA,
5134 "id": "20260101-000000-bbbb",
5135 "title": "from before urgent existed",
5136 "instruction": "from before urgent existed",
5137 "repo": ".",
5138 "source": { "kind": "human" },
5139 "status": "queued",
5140 "created_at": Timestamp::now().to_string(),
5141 "updated_at": Timestamp::now().to_string(),
5142 })
5143 .to_string(),
5144 )
5145 .unwrap();
5146
5147 let task = q.get("20260101-000000-bbbb").expect("must still read");
5148 assert!(
5149 !task.urgent,
5150 "a queue file with no `urgent` field means false, same as `solo`"
5151 );
5152 }
5153
5154 #[test]
5155 fn a_task_from_a_future_schema_is_refused_rather_than_guessed_at() {
5156 let (_dir, q) = queue();
5157 let mut t = task("from the future");
5158 q.put(&mut t).unwrap();
5159 let path = q.path_of(&t.id);
5160 let body = std::fs::read_to_string(&path)
5161 .unwrap()
5162 .replace(&format!("\"schema\": {SCHEMA}"), "\"schema\": 99");
5163 std::fs::write(&path, body).unwrap();
5164
5165 let err = q.get(&t.id).unwrap_err().to_string();
5166 assert!(err.contains("schema 99"), "{err}");
5167 }
5168
5169 #[test]
5170 fn revision_moves_when_the_queue_changes() {
5171 let (_dir, q) = queue();
5172 assert_eq!(q.revision(), 0, "an empty queue has no revision");
5173 let mut t = task("first");
5174 q.put(&mut t).unwrap();
5175 assert!(q.revision() > 0, "a written task moves the revision");
5176 }
5177
5178 #[test]
5179 fn revision_moves_when_deleting_an_older_task() {
5180 let (dir, q) = queue();
5181 let questions = Questions::at(dir.path().join("questions"));
5182 let mut t1 = task("older");
5183 q.put(&mut t1).unwrap();
5184 std::thread::sleep(std::time::Duration::from_millis(10));
5186 let mut t2 = task("newer");
5187 q.put(&mut t2).unwrap();
5188
5189 let rev_before = q.revision();
5190 q.remove(&t1.id, false, &questions).unwrap();
5191 let rev_after = q.revision();
5192
5193 assert_ne!(
5194 rev_before, rev_after,
5195 "deleting an older task must change the revision so other clients see the deletion"
5196 );
5197 }
5198
5199 #[test]
5200 fn removing_a_task_takes_it_out_of_the_listing() {
5201 let (dir, q) = queue();
5202 let questions = Questions::at(dir.path().join("questions"));
5203 let mut t = task("delete me");
5204 q.put(&mut t).unwrap();
5205 let removed = q.remove(t.short(), false, &questions).unwrap();
5206 assert_eq!(removed.id, t.id, "a prefix resolves before deleting");
5207 assert!(removed.released.is_empty() && removed.still_blocked.is_empty());
5208 assert!(q.list().is_empty());
5209 assert!(
5210 q.remove(&t.id, false, &questions).is_err(),
5211 "removing twice is an error"
5212 );
5213 }
5214
5215 #[test]
5216 fn removing_a_task_takes_its_stale_lock_with_it() {
5217 let (dir, q) = queue();
5218 let questions = Questions::at(dir.path().join("questions"));
5219 let mut t = task("interrupted");
5220 q.put(&mut t).unwrap();
5221
5222 let claim = q.claim(&t.id).unwrap();
5225 std::mem::forget(claim);
5226 assert!(
5227 q.claim(&t.id).is_err(),
5228 "the orphaned lock is what makes the task look claimed"
5229 );
5230
5231 let err = q.remove(&t.id, true, &questions).unwrap_err().to_string();
5233 assert!(err.contains("live daemon"), "{err}");
5234 assert!(q.get(&t.id).is_ok(), "a refused delete keeps the task");
5235
5236 q.remove(&t.id, false, &questions).unwrap();
5238 assert!(q.list().is_empty());
5239 let mut again = task("interrupted");
5240 again.id = t.id.clone();
5241 q.put(&mut again).unwrap();
5242 assert!(
5243 q.claim(&t.id).is_ok(),
5244 "a task that comes back must be claimable, which a left-behind lock would prevent"
5245 );
5246 }
5247
5248 fn notices_of(dir: &Path) -> Vec<crate::notices::Notice> {
5249 crate::notices::Notices::at(dir.join("notifications")).list()
5250 }
5251
5252 #[test]
5253 fn removing_a_sole_dependency_releases_the_dependent_without_a_hold() {
5254 let (dir, q) = queue();
5255 let questions = Questions::at(dir.path().join("questions"));
5256 let mut dep = task("dependency");
5257 q.put(&mut dep).unwrap();
5258 let mut blocked = task("waiting");
5259 blocked.block(vec![dep.id.clone()], Some("waits".to_owned()));
5260 q.put(&mut blocked).unwrap();
5261
5262 let removed = q.remove(&dep.id, false, &questions).unwrap();
5263 assert_eq!(removed.released, [blocked.id.clone()]);
5264 assert!(removed.still_blocked.is_empty());
5265
5266 let after = q.get(&blocked.id).unwrap();
5267 assert_eq!(after.status, TaskStatus::Queued);
5268 assert!(after.blocked_by.is_empty());
5269 assert!(after.block_reason.is_none());
5270 let notes = notices_of(dir.path());
5271 assert_eq!(notes.len(), 1, "{notes:?}");
5272 assert_eq!(notes[0].severity, crate::notices::Severity::Info);
5273 assert_eq!(
5274 notes[0].link,
5275 Some(crate::notices::Link::Task {
5276 id: blocked.id.clone(),
5277 })
5278 );
5279 }
5280
5281 #[test]
5282 fn removing_one_of_two_dependencies_keeps_the_other() {
5283 let (dir, q) = queue();
5284 let questions = Questions::at(dir.path().join("questions"));
5285 let mut dep = task("dependency");
5286 q.put(&mut dep).unwrap();
5287 let mut other = task("other");
5288 q.put(&mut other).unwrap();
5289 let mut blocked = task("waiting");
5290 blocked.block(
5291 vec![dep.id.clone(), other.id.clone()],
5292 Some("waits on both".to_owned()),
5293 );
5294 q.put(&mut blocked).unwrap();
5295
5296 let removed = q.remove(&dep.id, false, &questions).unwrap();
5297 assert!(removed.released.is_empty());
5298 assert_eq!(removed.still_blocked, [blocked.id.clone()]);
5299
5300 let after = q.get(&blocked.id).unwrap();
5301 assert_eq!(after.status, TaskStatus::Blocked);
5302 assert_eq!(after.blocked_by, [other.id.clone()]);
5303 assert!(missing_blockers(&q, &questions, &after.blocked_by).is_empty());
5306 }
5307
5308 #[test]
5309 fn a_dependency_deleted_before_its_dependents_were_rewritten_is_released_later() {
5310 let (dir, q) = queue();
5311 let questions = Questions::at(dir.path().join("questions"));
5312 let mut dep = task("dependency");
5313 q.put(&mut dep).unwrap();
5314 let mut blocked = task("waiting");
5315 blocked.block(vec![dep.id.clone()], None);
5316 q.put(&mut blocked).unwrap();
5317
5318 let claim = q.claim(&blocked.id).unwrap();
5320 let removed = q.remove(&dep.id, false, &questions).unwrap();
5321 assert!(removed.released.is_empty());
5322 drop(claim);
5323
5324 let mut task = q.get(&blocked.id).unwrap();
5325 assert_eq!(task.status, TaskStatus::Blocked);
5326 assert!(missing_blockers(&q, &questions, &task.blocked_by).is_empty());
5327 assert_eq!(
5328 q.apply_deleted_blockers(&mut task, &questions),
5329 [dep.id.clone()]
5330 );
5331 assert_eq!(task.status, TaskStatus::Queued);
5332 }
5333
5334 #[test]
5335 fn dependency_resolution_short_deleted_reference_releases_only_its_waiters() {
5336 let (dir, q) = queue();
5337 let questions = Questions::at(dir.path().join("questions"));
5338 let mut dep = task("dependency");
5339 dep.id = "20261013-193611-ca5b".into();
5340 q.put(&mut dep).unwrap();
5341 let mut waiting = task("waiting");
5342 waiting.block(vec!["ca5b".into()], None);
5343 q.put(&mut waiting).unwrap();
5344 q.remove(&dep.id, false, &questions).unwrap();
5345 assert_eq!(q.get(&waiting.id).unwrap().status, TaskStatus::Queued);
5346 assert_eq!(
5347 dependency(&q, &questions, "ca5b"),
5348 Dependency::Deleted(dep.id)
5349 );
5350 }
5351
5352 #[test]
5353 fn dependency_resolution_preserves_cross_store_ambiguity_and_exact_ids() {
5354 let (dir, queue) = queue();
5355 let questions = Questions::at(dir.path().join("questions"));
5356 let mut dep = task("dependency");
5357 dep.id = "20261013-193611-ca5b".into();
5358 queue.put(&mut dep).unwrap();
5359 std::fs::create_dir_all(questions.root()).unwrap();
5360 let question_id = "20261014-193611-ca5b";
5361 std::fs::write(questions.path_of(question_id), "unreadable question").unwrap();
5362 assert_eq!(questions.resolve_id("ca5b").unwrap(), question_id);
5363 assert_eq!(
5364 dependency(&queue, &questions, "ca5b"),
5365 Dependency::Ambiguous
5366 );
5367 assert_eq!(
5368 dependency(&queue, &questions, &dep.id),
5369 Dependency::Task(dep.id)
5370 );
5371 assert_eq!(
5372 dependency(&queue, &questions, question_id),
5373 Dependency::Question(question_id.into())
5374 );
5375 assert_eq!(
5376 dependency(&queue, &questions, "missing"),
5377 Dependency::Missing
5378 );
5379 }
5380
5381 #[test]
5382 fn dependency_resolution_directory_errors_are_unknown_not_missing() {
5383 let dir = tempfile::tempdir().unwrap();
5384 let q = Queue::at(dir.path().join("not-a-directory"));
5385 std::fs::write(q.root(), "file").unwrap();
5386 let questions = Questions::at(dir.path().join("questions"));
5387 assert_eq!(dependency(&q, &questions, "ca5b"), Dependency::Unknown);
5388 assert!(missing_blockers(&q, &questions, &["ca5b".into()]).is_empty());
5389 }
5390
5391 #[test]
5392 fn a_dependency_without_a_tombstone_is_still_missing() {
5393 let (dir, q) = queue();
5394 let questions = Questions::at(dir.path().join("questions"));
5395 let mut dep = task("dependency");
5396 q.put(&mut dep).unwrap();
5397 std::fs::remove_file(q.path_of(&dep.id)).unwrap();
5398 assert_eq!(
5399 missing_blockers(&q, &questions, std::slice::from_ref(&dep.id)),
5400 [dep.id.clone()]
5401 );
5402 assert!(deleted_blockers(&q, &questions, std::slice::from_ref(&dep.id)).is_empty());
5403 }
5404
5405 #[test]
5406 fn a_held_dependent_is_left_alone_by_a_dependency_deletion() {
5407 let mut t = task("held");
5408 t.hold_machine(Some("because".to_owned()));
5409 t.blocked_by = vec!["gone".to_owned()];
5410 assert!(!t.dependency_deleted("gone"));
5411 assert_eq!(t.status, TaskStatus::Held);
5412 }
5413
5414 #[test]
5415 fn a_failed_record_removal_takes_the_tombstone_back() {
5416 let (_dir, q) = queue();
5417 let mut t = task("stays");
5418 q.put(&mut t).unwrap();
5419 q.write_tombstone(&t.id).unwrap();
5420 let err = q.remove_record_with_attachments(&t.id, |_| Err(std::io::Error::other("nope")));
5421 assert!(err.is_err());
5422 assert!(!q.was_deleted(&t.id));
5425 }
5426
5427 fn source_file(dir: &Path, name: &str, body: &str) -> PathBuf {
5428 let p = dir.join(name);
5429 std::fs::write(&p, body).unwrap();
5430 p
5431 }
5432
5433 #[test]
5434 fn an_attachment_copy_survives_deleting_its_source() {
5435 let (dir, q) = queue();
5436 let src = source_file(dir.path(), "shot.png", "pixels");
5437 let mut t = task("with a picture");
5438 let names = q.attach(&mut t, std::slice::from_ref(&src)).unwrap();
5439 q.put(&mut t).unwrap();
5440 std::fs::remove_file(&src).unwrap();
5441 assert_eq!(names, ["shot.png"]);
5442 let loaded = q.get(&t.id).unwrap();
5443 let paths = q.attachment_paths(&loaded);
5444 assert_eq!(paths.len(), 1);
5445 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "pixels");
5446 }
5447
5448 #[test]
5449 fn attachment_names_that_could_traverse_or_are_odd_are_refused() {
5450 let (dir, q) = queue();
5451 let mut t = task("bad names");
5452 for name in ["a..b.png", ".hidden", "with space.png", "-x.png"] {
5453 let src = source_file(dir.path(), name, "x");
5454 assert!(
5455 q.attach(&mut t, &[src]).is_err(),
5456 "`{name}` must be refused"
5457 );
5458 }
5459 assert!(!crate::ask::valid_asset_name("C:foo.png"));
5463 #[cfg(not(windows))]
5464 {
5465 let src = source_file(dir.path(), "C:foo.png", "x");
5466 assert!(q.attach(&mut t, &[src]).is_err());
5467 }
5468 let long = format!("{}.png", "a".repeat(70));
5469 let src = source_file(dir.path(), &long, "x");
5470 assert!(q.attach(&mut t, &[src]).is_err());
5471 assert!(t.attachments.is_empty());
5472 assert!(!q.attachments_dir(&t.id).exists());
5473 }
5474
5475 #[test]
5476 fn a_taken_attachment_name_is_numbered_not_overwritten() {
5477 let (dir, q) = queue();
5478 let a = source_file(dir.path(), "shot.png", "one");
5479 let sub = dir.path().join("other");
5480 std::fs::create_dir_all(&sub).unwrap();
5481 let b = source_file(&sub, "shot.png", "two");
5482 let mut t = task("collision");
5483 q.attach(&mut t, &[a]).unwrap();
5484 q.attach(&mut t, &[b]).unwrap();
5485 assert_eq!(t.attachments, ["shot.png", "shot-2.png"]);
5486 let paths = q.attachment_paths(&t);
5487 assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "one");
5488 assert_eq!(std::fs::read_to_string(&paths[1]).unwrap(), "two");
5489 }
5490
5491 #[test]
5492 fn a_renumbered_name_stays_inside_the_length_bound() {
5493 let (dir, q) = queue();
5494 let name = format!("{}.png", "a".repeat(60));
5495 assert_eq!(name.len(), 64);
5496 let a = source_file(dir.path(), &name, "one");
5497 let sub = dir.path().join("other");
5498 std::fs::create_dir_all(&sub).unwrap();
5499 let b = source_file(&sub, &name, "two");
5500 let mut t = task("long");
5501 q.attach(&mut t, &[a, b]).unwrap();
5502 assert_eq!(t.attachments.len(), 2);
5503 assert!(
5504 t.attachments
5505 .iter()
5506 .all(|n| crate::ask::valid_asset_name(n))
5507 );
5508 assert!(t.attachments[1].ends_with("-2.png"));
5509 }
5510
5511 #[test]
5512 fn a_failed_attach_keeps_existing_attachments_and_leaves_no_partial_copy() {
5513 let (dir, q) = queue();
5514 let good = source_file(dir.path(), "good.png", "ok");
5515 let mut t = task("partial");
5516 q.attach(&mut t, &[good]).unwrap();
5517 let more = source_file(dir.path(), "more.png", "ok");
5518 let missing = dir.path().join("missing.png");
5519 assert!(q.attach(&mut t, &[more, missing]).is_err());
5520 assert_eq!(t.attachments, ["good.png"]);
5521 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5522 .unwrap()
5523 .flatten()
5524 .collect();
5525 assert_eq!(on_disk.len(), 1);
5526 }
5527
5528 fn block_put(q: &Queue, t: &Task) -> PathBuf {
5530 let tmp = q.path_of(&t.id).with_extension("json.tmp");
5531 std::fs::create_dir_all(&tmp).unwrap();
5532 tmp
5533 }
5534
5535 #[test]
5536 fn a_failed_put_leaves_no_new_attachment_directory() {
5537 let (dir, q) = queue();
5538 let mut t = task("fresh");
5539 let tmp = block_put(&q, &t);
5540 let src = source_file(dir.path(), "shot.png", "x");
5541 assert!(q.attach_and_put(&mut t, &[src]).is_err());
5542 assert!(t.attachments.is_empty());
5543 assert!(!q.attachments_dir(&t.id).exists());
5544 assert!(!q.path_of(&t.id).exists());
5545 std::fs::remove_dir(tmp).unwrap();
5546 }
5547
5548 #[test]
5549 fn a_failed_put_removes_only_the_copy_it_just_made() {
5550 let (dir, q) = queue();
5551 let mut t = task("edited");
5552 let first = source_file(dir.path(), "first.png", "1");
5553 q.attach_and_put(&mut t, &[first]).unwrap();
5554 block_put(&q, &t);
5555 let second = source_file(dir.path(), "second.png", "2");
5556 assert!(q.attach_and_put(&mut t, &[second]).is_err());
5557 assert_eq!(t.attachments, ["first.png"]);
5558 let on_disk: Vec<_> = std::fs::read_dir(q.attachments_dir(&t.id))
5559 .unwrap()
5560 .flatten()
5561 .map(|e| e.file_name().to_string_lossy().into_owned())
5562 .collect();
5563 assert_eq!(on_disk, ["first.png"]);
5564 assert_eq!(q.get(&t.id).unwrap().attachments, ["first.png"]);
5565 }
5566
5567 #[test]
5568 fn a_leftover_removing_directory_is_swept_by_the_next_removal() {
5569 let (dir, q) = queue();
5570 let questions = Questions::at(dir.path().join("questions"));
5571 let gone = task("gone");
5572 let mut other = task("other");
5573 let mut live = task("live");
5574 q.put(&mut other).unwrap();
5575 q.put(&mut live).unwrap();
5576 let orphan = q.root.join(format!("{}.attachments.removing", gone.id));
5579 std::fs::create_dir_all(&orphan).unwrap();
5580 std::fs::write(orphan.join("shot.png"), "x").unwrap();
5581 let busy = q.root.join(format!("{}.attachments.removing", live.id));
5584 std::fs::create_dir_all(&busy).unwrap();
5585
5586 q.remove(&other.id, false, &questions).unwrap();
5587 assert!(!orphan.exists(), "an orphan is swept");
5588 assert!(busy.exists(), "a removal in progress is left alone");
5589 }
5590
5591 #[test]
5592 fn a_blocked_aside_rename_fails_the_removal_and_loses_nothing() {
5593 let (dir, q) = queue();
5594 let questions = Questions::at(dir.path().join("questions"));
5595 let mut t = task("stuck");
5596 let src = source_file(dir.path(), "shot.png", "x");
5597 q.attach_and_put(&mut t, &[src]).unwrap();
5598 let aside = q.root.join(format!("{}.attachments.removing", t.id));
5601 std::fs::create_dir_all(&aside).unwrap();
5602 std::fs::write(aside.join("old.png"), "o").unwrap();
5603 assert!(q.remove(&t.id, false, &questions).is_err());
5604 assert!(q.path_of(&t.id).exists());
5605 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5606 }
5607
5608 #[test]
5609 fn a_failed_record_removal_puts_the_attachments_back() {
5610 let (dir, q) = queue();
5611 let mut t = task("rollback");
5612 let src = source_file(dir.path(), "shot.png", "x");
5613 q.attach_and_put(&mut t, &[src]).unwrap();
5614 let err = q
5615 .remove_record_with_attachments(&t.id, |_| {
5616 Err(std::io::Error::other("injected failure"))
5617 })
5618 .unwrap_err();
5619 assert!(format!("{err:#}").contains("injected failure"));
5620 assert!(q.path_of(&t.id).exists());
5621 assert!(q.attachments_dir(&t.id).join("shot.png").is_file());
5622 assert!(
5623 !q.root
5624 .join(format!("{}.attachments.removing", t.id))
5625 .exists()
5626 );
5627 }
5628
5629 #[test]
5630 fn editing_a_task_keeps_its_attachments() {
5631 let (dir, q) = queue();
5632 let src = source_file(dir.path(), "shot.png", "x");
5633 let mut t = task("editable");
5634 q.attach(&mut t, &[src]).unwrap();
5635 t.edit("new".to_owned(), "new text".to_owned()).unwrap();
5636 q.put(&mut t).unwrap();
5637 assert_eq!(q.get(&t.id).unwrap().attachments, ["shot.png"]);
5638 }
5639
5640 #[test]
5641 fn removing_a_task_deletes_its_attachments() {
5642 let (dir, q) = queue();
5643 let questions = Questions::at(dir.path().join("questions"));
5644 let src = source_file(dir.path(), "shot.png", "x");
5645 let mut t = task("doomed");
5646 q.attach(&mut t, &[src]).unwrap();
5647 q.put(&mut t).unwrap();
5648 assert!(q.attachments_dir(&t.id).is_dir());
5649 q.remove(&t.id, false, &questions).unwrap();
5650 assert!(!q.attachments_dir(&t.id).exists());
5651 assert!(q.list().is_empty());
5652 }
5653
5654 #[test]
5655 fn a_task_written_before_attachments_still_reads() {
5656 let (_dir, q) = queue();
5657 let mut t = task("old");
5658 q.put(&mut t).unwrap();
5659 let path = q.path_of(&t.id);
5660 let mut v: serde_json::Value =
5661 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
5662 v.as_object_mut().unwrap().remove("attachments");
5663 std::fs::write(&path, v.to_string()).unwrap();
5664 assert!(q.get(&t.id).unwrap().attachments.is_empty());
5665 }
5666
5667 #[test]
5668 fn attachment_paths_are_absolute_even_when_the_root_is_relative() {
5669 let q = Queue::at(PathBuf::from("relative-queue"));
5670 let mut t = task("rel");
5671 t.attachments.push("shot.png".to_owned());
5672 let paths = q.attachment_paths(&t);
5673 assert!(paths[0].is_absolute(), "{}", paths[0].display());
5674 assert!(paths[0].ends_with(format!("{}.attachments/shot.png", t.id)));
5675 }
5676
5677 fn upgrade_parked(id: &str, started: &str) -> Task {
5678 let mut t = task(id);
5679 t.id = id.into();
5680 t.start(format!("run-{id}"));
5681 t.park("upgrade");
5682 t.record_pass_at(
5683 &format!("run-{id}"),
5684 0,
5685 Some(started.parse().unwrap()),
5686 1,
5687 Some(crate::run::ParkCause::Upgrade {
5688 from: "old".into(),
5689 to: None,
5690 }),
5691 );
5692 t
5693 }
5694
5695 #[test]
5696 fn upgrade_resume_order_follows_dispatch_claim_not_graph_start() {
5697 let stamp = |id: &str, dispatched: &str, started: &str| {
5699 let mut t = task(id);
5700 t.id = id.into();
5701 t.start(format!("run-{id}"));
5702 t.park("upgrade");
5703 t.dispatched_at = Some(dispatched.parse().unwrap());
5704 t.record_pass_at(
5705 &format!("run-{id}"),
5706 0,
5707 Some(started.parse().unwrap()),
5708 1,
5709 Some(crate::run::ParkCause::Upgrade {
5710 from: "old".into(),
5711 to: None,
5712 }),
5713 );
5714 t
5715 };
5716 let a = stamp("a", "2026-10-22T00:00:01Z", "2026-10-22T00:00:09Z");
5717 let b = stamp("b", "2026-10-22T00:00:02Z", "2026-10-22T00:00:03Z");
5718 assert!(dispatch_order(&a, &b).is_lt());
5719 }
5720
5721 #[test]
5722 fn upgrade_resume_order_survives_queue_reload_and_reversed_settlement() {
5723 let (_dir, q) = queue();
5724 let mut later = upgrade_parked("later", "2026-10-22T00:00:02Z");
5725 let mut earlier = upgrade_parked("earlier", "2026-10-22T00:00:01Z");
5726 earlier.priority = -10;
5727 let mut fresh = task("fresh");
5728 fresh.id = "newer".into();
5729 fresh.priority = 100;
5730 for t in [&mut fresh, &mut later, &mut earlier] {
5731 q.put(t).unwrap();
5732 }
5733 let reloaded = Queue::at(q.root().to_owned());
5734 let mut tasks = reloaded.list();
5735 tasks.sort_unstable_by(dispatch_order);
5736 assert_eq!(
5737 tasks.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
5738 vec!["earlier", "later", "newer"]
5739 );
5740 assert_eq!(reloaded.next_runnable().unwrap().id, "earlier");
5741 assert_eq!(reloaded.get("earlier").unwrap().attempts, 0);
5742 }
5743
5744 #[test]
5745 fn upgrade_resume_legacy_order_uses_settlement_and_id() {
5746 let mut a = upgrade_parked("a", "2026-10-22T00:00:02Z");
5747 let mut b = upgrade_parked("b", "2026-10-22T00:00:01Z");
5748 for t in [&mut a, &mut b] {
5749 let mut value = serde_json::to_value(&*t).unwrap();
5750 value["passes"][0]
5751 .as_object_mut()
5752 .unwrap()
5753 .remove("started_at");
5754 *t = serde_json::from_value(value).unwrap();
5755 }
5756 assert!(dispatch_order(&a, &b).is_lt());
5757 b.passes[0].settled_at = a.passes[0].settled_at;
5758 assert!(dispatch_order(&a, &b).is_lt());
5759 }
5760
5761 #[test]
5762 fn upgrade_resume_key_excludes_other_parks_and_explicit_task_changes() {
5763 let original = upgrade_parked("a", "2026-10-22T00:00:01Z");
5764 assert!(original.upgrade_resume_key().is_some());
5765 for cause in [
5766 crate::run::ParkCause::Operator,
5767 crate::run::ParkCause::Question,
5768 crate::run::ParkCause::Interrupt {
5769 reason: "interrupt".into(),
5770 },
5771 crate::run::ParkCause::Unspecified,
5772 ] {
5773 let mut t = original.clone();
5774 t.passes[0].parked = Some(cause);
5775 assert!(t.upgrade_resume_key().is_none());
5776 let mut fresh = task("fresh");
5777 fresh.priority = 1;
5778 assert!(dispatch_order(&fresh, &t).is_lt());
5779 }
5780 let mut t = original.clone();
5781 t.release();
5782 assert!(t.upgrade_resume_key().is_none());
5783 t.edit("edited".into(), "new instruction".into()).unwrap();
5784 assert!(t.upgrade_resume_key().is_none());
5785 let mut t = original.clone();
5786 t.requeue();
5787 t.status = TaskStatus::Parked;
5788 assert!(t.upgrade_resume_key().is_none());
5789 let mut t = original.clone();
5790 t.start("different-run".into());
5791 t.park("no upgrade");
5792 assert!(t.upgrade_resume_key().is_none());
5793 }
5794
5795 #[test]
5796 fn explicitly_resumed_older_run_becomes_current_without_duplicate_history() {
5797 let dir = tempfile::tempdir().unwrap();
5798 let q = Queue::at(dir.path().join("queue"));
5799 let mut t = task("resume");
5800 t.start("A".into());
5801 t.record_pass("A", 0, 1, None);
5802 t.start("B".into());
5803 q.put(&mut t).unwrap();
5804 assert_eq!(t.runs, vec!["A", "B"]);
5805 t.start("A".into());
5806 t.record_pass("A", 1, 3, None);
5807 q.put(&mut t).unwrap();
5808 let saved = q.get(&t.id).unwrap();
5809 assert_eq!(saved.runs, vec!["B", "A"]);
5810 assert_eq!(saved.runs.last().map(String::as_str), Some("A"));
5811 assert_eq!(saved.attempts, 3);
5812 assert_eq!(saved.passes.len(), 2);
5813 assert_eq!(saved.passes[0].run, "A");
5814 assert_eq!(saved.passes[1].run, "A");
5815 }
5816
5817 #[test]
5818 fn repeated_start_records_one_run_and_separate_passes() {
5819 let mut t = task("resume");
5820 t.start("same-run".into());
5821 t.record_pass("same-run", 0, 1, None);
5822 t.start("same-run".into());
5823 t.record_pass("same-run", 1, 2, None);
5824 assert_eq!(t.runs, vec!["same-run"]);
5825 assert_eq!(t.attempts, 2);
5826 assert_eq!(t.passes.len(), 2);
5827 }
5828
5829 #[test]
5830 fn link_run_adds_a_run_once_and_touches_nothing_else() {
5831 let dir = tempfile::tempdir().unwrap();
5832 let queue = Queue::at(dir.path().join("queue"));
5833 let mut t = Task::new(
5834 "t".to_owned(),
5835 "do it".to_owned(),
5836 PathBuf::from("."),
5837 Source::Human,
5838 );
5839 queue.put(&mut t).unwrap();
5840 let before = queue.get(&t.id).unwrap();
5841
5842 let linked = queue.link_run(&t.id[..4], "20260930-092817-ec34").unwrap();
5843 assert_eq!(linked.runs, vec!["20260930-092817-ec34".to_owned()]);
5844 assert_eq!(linked.status, before.status);
5845 assert_eq!(linked.attempts, before.attempts);
5846 assert_eq!(linked.interrupt, before.interrupt);
5847
5848 let again = queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
5849 assert_eq!(again.runs.len(), 1, "linking twice must not duplicate");
5850 assert_eq!(queue.get(&t.id).unwrap().runs.len(), 1);
5851 assert!(queue.link_run("no-such-task", "r").is_err());
5852 }
5853
5854 #[test]
5855 fn put_keeps_a_run_linked_after_the_writer_took_its_snapshot() {
5856 let dir = tempfile::tempdir().unwrap();
5857 let queue = Queue::at(dir.path().join("queue"));
5858 let mut t = Task::new(
5859 "t".to_owned(),
5860 "do it".to_owned(),
5861 PathBuf::from("."),
5862 Source::Human,
5863 );
5864 queue.put(&mut t).unwrap();
5865 let mut snapshot = queue.get(&t.id).unwrap();
5867 queue.link_run(&t.id, "20260930-092817-ec34").unwrap();
5868
5869 snapshot.start("20260930-000000-aaaa".to_owned());
5870 queue.put(&mut snapshot).unwrap();
5871
5872 let stored = queue.get(&t.id).unwrap();
5873 assert!(stored.runs.contains(&"20260930-092817-ec34".to_owned()));
5874 assert!(stored.runs.contains(&"20260930-000000-aaaa".to_owned()));
5875 assert_eq!(stored.attempts, 1);
5876 }
5877
5878 #[test]
5879 fn concurrent_links_and_daemon_saves_lose_nothing() {
5880 let dir = tempfile::tempdir().unwrap();
5881 let queue = Queue::at(dir.path().join("queue"));
5882 let mut t = Task::new(
5883 "t".to_owned(),
5884 "do it".to_owned(),
5885 PathBuf::from("."),
5886 Source::Human,
5887 );
5888 queue.put(&mut t).unwrap();
5889 let id = t.id.clone();
5890
5891 let linkers: Vec<_> = (0..4)
5892 .map(|n| {
5893 let (queue, id) = (queue.clone(), id.clone());
5894 std::thread::spawn(move || {
5895 for k in 0..10 {
5896 queue
5897 .link_run(&id, &format!("20260930-00000{n}-l{k:03}"))
5898 .unwrap();
5899 }
5900 })
5901 })
5902 .collect();
5903 let mut mine = queue.get(&id).unwrap();
5906 for k in 0..10 {
5907 mine.start(format!("20260930-000009-d{k:03}"));
5908 queue.put(&mut mine).unwrap();
5909 }
5910 for l in linkers {
5911 l.join().unwrap();
5912 }
5913
5914 let stored = queue.get(&id).unwrap();
5915 assert_eq!(stored.runs.len(), 50, "{:?}", stored.runs);
5916 assert_eq!(
5917 stored.attempts, 10,
5918 "linking never rewinds the daemon's work"
5919 );
5920 }
5921
5922 fn age_lock(path: &Path) {
5923 let f = std::fs::OpenOptions::new().write(true).open(path).unwrap();
5924 f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(60))
5925 .unwrap();
5926 }
5927
5928 #[test]
5929 fn concurrent_stale_takeover_yields_one_holder() {
5930 use std::sync::atomic::{AtomicUsize, Ordering};
5931 use std::sync::{Arc, Barrier};
5932 for _ in 0..5 {
5933 let dir = tempfile::tempdir().unwrap();
5934 let q = Queue::at(dir.path().to_path_buf());
5935 let lock = dir.path().join("t.write-lock");
5936 std::fs::write(&lock, "dead-0000").unwrap();
5937 age_lock(&lock);
5938 let n = 6;
5939 let barrier = Arc::new(Barrier::new(n));
5940 let (now, max) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
5941 let handles: Vec<_> = (0..n)
5942 .map(|_| {
5943 let (q, b, now, max) = (q.clone(), barrier.clone(), now.clone(), max.clone());
5944 std::thread::spawn(move || {
5945 b.wait();
5946 let g = q.lock_task("t").unwrap();
5947 let held = now.fetch_add(1, Ordering::SeqCst) + 1;
5948 max.fetch_max(held, Ordering::SeqCst);
5949 std::thread::sleep(std::time::Duration::from_millis(20));
5950 now.fetch_sub(1, Ordering::SeqCst);
5951 drop(g);
5952 })
5953 })
5954 .collect();
5955 for h in handles {
5956 h.join().unwrap();
5957 }
5958 assert_eq!(max.load(Ordering::SeqCst), 1);
5959 assert!(!lock.exists());
5960 }
5961 }
5962
5963 #[test]
5964 fn dropping_a_replaced_lock_leaves_the_new_holders_token() {
5965 let dir = tempfile::tempdir().unwrap();
5966 let q = Queue::at(dir.path().to_path_buf());
5967 let lock = dir.path().join("t.write-lock");
5968 let a = q.lock_task("t").unwrap();
5969 let replacement = owner_token();
5972 std::fs::write(&lock, &replacement).unwrap();
5973 drop(a);
5974 assert_eq!(std::fs::read_to_string(&lock).unwrap(), replacement);
5975 }
5976
5977 #[test]
5978 fn aged_live_task_lock_excludes_updates_until_retirement_releases_it() {
5979 let dir = tempfile::tempdir().unwrap();
5980 let q = Queue::at(dir.path().to_path_buf());
5981 let held = q.lock_task("t").unwrap();
5982 let path = dir.path().join("t.write-lock");
5983 age_lock(&path);
5984 assert!(!break_stale(&path, &held.token));
5985 let (started_tx, started_rx) = std::sync::mpsc::channel();
5986 let (acquired_tx, acquired_rx) = std::sync::mpsc::channel();
5987 let updater = std::thread::spawn(move || {
5988 started_tx.send(()).unwrap();
5989 let _write = q.lock_task("t").unwrap();
5990 acquired_tx.send(()).unwrap();
5991 });
5992 started_rx.recv().unwrap();
5993 assert!(
5994 acquired_rx
5995 .recv_timeout(std::time::Duration::from_millis(100))
5996 .is_err()
5997 );
5998 assert_eq!(std::fs::read_to_string(&path).unwrap(), held.token);
5999 drop(held);
6000 acquired_rx
6001 .recv_timeout(std::time::Duration::from_secs(2))
6002 .unwrap();
6003 updater.join().unwrap();
6004 assert!(!path.exists());
6005 }
6006
6007 #[test]
6008 fn break_stale_leaves_a_lock_that_replaced_the_one_judged() {
6009 let dir = tempfile::tempdir().unwrap();
6010 let lock = dir.path().join("t.write-lock");
6011 std::fs::write(&lock, "new-token").unwrap();
6012 assert!(!break_stale(&lock, "old-token"));
6013 assert_eq!(std::fs::read_to_string(&lock).unwrap(), "new-token");
6014 assert!(!break_stale(&lock, "new-token"));
6016 assert!(lock.exists());
6017 age_lock(&lock);
6018 assert!(break_stale(&lock, "new-token"));
6019 assert!(!lock.exists());
6020 }
6021
6022 #[test]
6023 fn a_stale_break_marker_is_recovered_and_a_fresh_one_is_respected() {
6024 let dir = tempfile::tempdir().unwrap();
6025 let lock = dir.path().join("t.write-lock");
6026 std::fs::write(&lock, "dead-1").unwrap();
6027 age_lock(&lock);
6028 let marker = break_marker(&lock, "dead-1");
6029 std::fs::write(&marker, "crashed-remover").unwrap();
6030 assert!(!break_stale(&lock, "dead-1"));
6032 assert!(lock.exists());
6033 age_lock(&marker);
6035 assert!(break_stale(&lock, "dead-1"));
6036 assert!(!lock.exists());
6037 assert!(!marker.exists());
6038 }
6039
6040 fn chat_task() -> Task {
6041 Task::new(
6042 "t".to_owned(),
6043 "do t".to_owned(),
6044 PathBuf::from("."),
6045 Source::Agent {
6046 run: "20260902-000000-beef".to_owned(),
6047 node: CHAT_NODE.to_owned(),
6048 },
6049 )
6050 }
6051
6052 #[test]
6053 fn a_blocked_hold_keeps_its_anchor_but_a_new_run_starts_a_new_hold() {
6054 let mut t = chat_task();
6055 t.hold_machine(Some("waiting".into()));
6056 let old: Timestamp = "2020-01-01T00:00:00Z".parse().unwrap();
6057 t.held_at = Some(old);
6058 t.block(vec!["decision".into()], None);
6059 t.hold_machine(Some("same waiting decision".into()));
6060 assert_eq!(t.held_at, Some(old));
6061 t.start("real-new-run".into());
6062 t.fail("new failure", 1);
6063 assert_ne!(
6064 t.held_at,
6065 Some(old),
6066 "old answers cannot approve a new hold"
6067 );
6068 }
6069
6070 #[test]
6071 fn legacy_delivery_survives_upgrade_and_a_no_progress_run_link() {
6072 let (_dir, queue) = queue();
6073 let mut t = chat_task();
6074 t.start("old-run".into());
6075 t.hold_machine(Some("operator is considering recovery".into()));
6076 t.chat_notice = Some(ChatNotice {
6077 key: format!("held#{}", t.held_at.unwrap()),
6078 at: Timestamp::now(),
6079 outcome: ChatNoticeOutcome::Posted,
6080 });
6081 let mut raw = serde_json::to_value(&t).unwrap();
6082 raw.as_object_mut().unwrap().remove("chat_notices");
6083 raw.as_object_mut().unwrap().remove("hold_notice_cause");
6084 std::fs::create_dir_all(queue.path_of(&t.id).parent().unwrap()).unwrap();
6085 std::fs::write(queue.path_of(&t.id), serde_json::to_string(&raw).unwrap()).unwrap();
6086 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
6087 queue
6088 .modify(&t.id, |task| {
6089 task.requeue();
6090 task.link_run("unprepared-run");
6091 task.startup_notice_cause("startup-source");
6092 task.fail("same unresolved preflight", 0);
6093 true
6094 })
6095 .unwrap();
6096 assert_eq!(queue.get(&t.id).unwrap().chat_notice_due(), None);
6097 queue
6098 .modify(&t.id, |task| {
6099 task.succeed();
6100 true
6101 })
6102 .unwrap();
6103 assert_eq!(
6104 queue.get(&t.id).unwrap().chat_notice_due().as_deref(),
6105 Some("done")
6106 );
6107 }
6108
6109 #[test]
6110 fn a_dependency_wait_does_not_change_when_released_and_blocked_again() {
6111 let mut t = chat_task();
6112 t.block(vec!["a".into(), "b".into()], Some("waiting".into()));
6113 let key = t.chat_notice_key().unwrap();
6114 t.chat_notices.push(key);
6115 t.release();
6116 t.block(
6117 vec!["b".into(), "a".into(), "a".into()],
6118 Some("still waiting on prerequisites".into()),
6119 );
6120 assert_eq!(t.chat_notice_due(), None);
6121 t.block(
6122 vec!["new-question".into()],
6123 Some("different decision".into()),
6124 );
6125 assert!(t.chat_notice_due().is_some());
6126 }
6127
6128 #[test]
6129 fn running_startup_failures_preserve_stage_and_report_distinct_blockers() {
6130 let mut t = chat_task();
6131 t.start("existing-run".into());
6132 t.fail("implementation failed", 1);
6133 let failed = t.chat_notice_key().unwrap();
6134 t.chat_notices.push(failed.clone());
6135 t.release_operator("test", None);
6136 t.start("existing-run".into());
6137 t.fail_startup("startup-source", "same unresolved recovery source", 1);
6138 assert_eq!(
6139 t.chat_notice_due(),
6140 None,
6141 "no-progress recovery stays quiet"
6142 );
6143 t.release_operator("test", None);
6144 t.start("existing-run".into());
6145 t.fail_startup("startup-attachments", "attachment missing", 1);
6146 let attachments = t.chat_notice_due().expect("new attachment blocker");
6147 assert_ne!(attachments, failed);
6148 assert_eq!(
6149 t.hold_notice_cause.as_ref().unwrap().stage,
6150 "startup-attachments"
6151 );
6152 t.chat_notices.push(attachments);
6153 t.release_operator("test", None);
6154 t.start("existing-run".into());
6155 t.fail_startup(
6156 "startup-attachments",
6157 "same missing attachment, reworded",
6158 1,
6159 );
6160 assert_eq!(
6161 t.chat_notice_due(),
6162 None,
6163 "same stage after attempt counter reset"
6164 );
6165 t.release_operator("test", None);
6166 t.start("existing-run".into());
6167 t.fail_startup("startup-config", "different preflight stage", 1);
6168 assert!(t.chat_notice_due().is_some());
6169 assert_eq!(
6170 t.hold_notice_cause.as_ref().unwrap().stage,
6171 "startup-config"
6172 );
6173 }
6174
6175 #[test]
6176 fn structured_causes_preserve_no_progress_retries_and_report_new_passes() {
6177 let mut t = chat_task();
6178 t.hold_machine(Some("startup source missing".into()));
6179 t.startup_notice_cause("startup-source");
6180 let first = t.chat_notice_key().unwrap();
6181 t.chat_notices.push(first.clone());
6182 t.release_operator("test", None);
6183 t.startup_notice_cause("startup-source");
6184 t.hold_machine(Some("same source cannot be read".into()));
6185 assert_eq!(t.chat_notice_due(), None);
6186 t.startup_notice_cause("startup-config");
6187 assert!(t.chat_notice_due().is_some(), "different startup stage");
6188 t.notice_cause("failed", Some("existing-run"), Some(0));
6189 let run_failure = t.chat_notice_key().unwrap();
6190 t.chat_notices.push(run_failure.clone());
6191 t.release_operator("test", None);
6192 t.startup_notice_cause("startup-source");
6193 t.hold_machine(Some("no new work happened".into()));
6194 assert_eq!(t.chat_notice_due(), None);
6195 t.notice_cause("failed", Some("existing-run"), Some(1));
6196 assert!(t.chat_notice_due().is_some(), "a real resumed pass failed");
6197 assert_ne!(t.chat_notice_key().unwrap(), run_failure);
6198 }
6199
6200 #[test]
6201 fn legacy_hold_baseline_does_not_replay_or_hide_new_failures_and_completion() {
6202 let mut t = chat_task();
6203 t.hold_machine(Some("still waiting".into()));
6204 t.chat_notice = Some(ChatNotice {
6205 key: "held#2026-10-14T00:00:00Z".into(),
6206 at: Timestamp::now(),
6207 outcome: ChatNoticeOutcome::Posted,
6208 });
6209 assert_eq!(t.chat_notice_due(), None, "old hold baseline");
6210 t.import_chat_notice();
6211 t.notice_cause("failed", Some("new-run"), Some(0));
6212 assert!(t.chat_notice_due().is_some(), "new producer evidence");
6213 t.succeed();
6214 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
6215 }
6216
6217 #[test]
6218 fn a_chat_notice_is_due_only_for_a_final_state_and_only_once() {
6219 let mut t = chat_task();
6220 assert_eq!(t.chat_notice_due(), None, "queued");
6221 t.start("r1".to_owned());
6222 assert_eq!(t.chat_notice_due(), None, "running");
6223 t.fail("boom", 3);
6224 assert_eq!(t.status, TaskStatus::Failed);
6225 assert_eq!(t.chat_notice_due(), None, "a retried failure is not final");
6226 t.start("r2".to_owned());
6227 t.park("upgrade");
6228 assert_eq!(t.chat_notice_due(), None, "parked");
6229 t.fail("boom", 1);
6230 assert_eq!(t.status, TaskStatus::Held);
6231 let key = t
6232 .chat_notice_due()
6233 .expect("held on the last attempt is final");
6234 t.chat_notice = Some(ChatNotice {
6235 key,
6236 at: Timestamp::now(),
6237 outcome: ChatNoticeOutcome::Posted,
6238 });
6239 assert_eq!(t.chat_notice_due(), None, "same hold, no second notice");
6240 t.release();
6241 assert_eq!(t.chat_notice_due(), None, "requeued after a release");
6242 t.start("r3".to_owned());
6243 t.succeed();
6244 assert!(t.chat_notice_due().is_some(), "done is a new final state");
6245 }
6246
6247 #[test]
6248 fn completed_chat_notices_keep_current_and_legacy_keys_after_requeue() {
6249 for key in ["done", "done#0", "done#1", "done#12"] {
6250 let mut t = chat_task();
6251 t.start("r1".to_owned());
6252 t.succeed();
6253 t.chat_notice = Some(ChatNotice {
6254 key: key.to_owned(),
6255 at: Timestamp::now(),
6256 outcome: ChatNoticeOutcome::Posted,
6257 });
6258 t.requeue();
6259 t.start("r2".to_owned());
6260 t.succeed();
6261 assert_eq!(t.chat_notice_key().as_deref(), Some("done"));
6262 assert_eq!(t.chat_notice_due(), None, "recorded key: {key}");
6263 }
6264 }
6265
6266 #[test]
6267 fn a_release_without_progress_keeps_the_unresolved_notice_cause() {
6268 let mut t = chat_task();
6269 t.hold_manual(Some("wait".to_owned()));
6270 let first = t.chat_notice_due().expect("held");
6271 t.chat_notice = Some(ChatNotice {
6272 key: first.clone(),
6273 at: Timestamp::now(),
6274 outcome: ChatNoticeOutcome::Posted,
6275 });
6276 t.hold_manual(Some("still waiting".to_owned()));
6277 assert_eq!(t.chat_notice_due(), None, "a re-hold keeps its start");
6278 t.release();
6279 std::thread::sleep(std::time::Duration::from_millis(5));
6280 t.hold_manual(None);
6281 assert_eq!(t.chat_notice_due(), None, "no progress, same cause");
6282 }
6283
6284 #[test]
6285 fn a_chat_notice_needs_a_chat_source_and_a_current_schema() {
6286 let mut human = task("h");
6287 human.succeed();
6288 assert_eq!(human.chat_notice_due(), None);
6289 let mut inherited = task("i");
6291 inherited.origin_chat = Some("20260902-000000-beef".to_owned());
6292 inherited.succeed();
6293 assert_eq!(inherited.chat_notice_due(), None);
6294 let mut old = chat_task();
6295 old.schema = 12;
6296 old.status = TaskStatus::Done;
6297 assert_eq!(old.chat_notice_due(), None, "history is not replayed");
6298 let mut live = chat_task();
6299 live.schema = 12;
6300 live.start("r1".to_owned());
6301 live.succeed();
6302 assert!(live.chat_notice_due().is_some(), "started by this build");
6303 let mut held = chat_task();
6304 held.schema = 12;
6305 held.hold_machine(Some("config".to_owned()));
6306 assert!(held.chat_notice_due().is_some(), "held before any start");
6307 }
6308
6309 #[test]
6310 fn a_stale_snapshot_does_not_rewind_a_recorded_chat_notice() {
6311 let tmp = tempfile::tempdir().expect("tempdir");
6312 let q = Queue::at(tmp.path().join("queue"));
6313 let mut t = chat_task();
6314 t.succeed();
6315 q.put(&mut t).expect("put");
6316 let mut stale = q.get(&t.id).expect("get");
6317 let key = t.chat_notice_due().expect("due");
6318 q.modify(&t.id, |x| {
6319 x.chat_notice = Some(ChatNotice {
6320 key: key.clone(),
6321 at: Timestamp::now(),
6322 outcome: ChatNoticeOutcome::Posted,
6323 });
6324 true
6325 })
6326 .expect("modify");
6327 q.put(&mut stale).expect("stale put");
6328 assert_eq!(q.get(&t.id).expect("get").chat_notice_due(), None);
6329 }
6330
6331 #[test]
6332 fn empty_sources_are_omitted_so_older_builds_still_read_the_task() {
6333 let mut t = task("plain");
6336 let value = serde_json::to_value(&t).unwrap();
6337 assert!(value.get("sources").is_none(), "{value}");
6338 t.sources.pick_commits.push(crate::refs::Input {
6339 token: "abc1234".into(),
6340 sha: "a".repeat(40),
6341 provenance: crate::refs::Provenance::ExplicitCli,
6342 });
6343 let value = serde_json::to_value(&t).unwrap();
6344 assert_eq!(value["sources"]["pick_commits"][0]["sha"], "a".repeat(40));
6345 let restored: Task = serde_json::from_value(value).unwrap();
6346 assert_eq!(restored.sources, t.sources);
6347 }
6348
6349 fn queue_with_ids(dir: &Path, ids: &[&str]) -> Queue {
6351 let ids: Vec<String> = ids.iter().map(|s| (*s).to_owned()).collect();
6352 let n = std::sync::atomic::AtomicUsize::new(0);
6353 Queue::at(dir.join("queue")).with_id_source(move || {
6354 let i = n.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6355 ids.get(i)
6356 .cloned()
6357 .unwrap_or_else(|| format!("20260101-000000-f{i:03x}"))
6358 })
6359 }
6360
6361 const TAKEN: &str = "20260101-000000-aaaa";
6362
6363 #[test]
6364 fn a_new_task_never_merges_into_a_record_with_its_candidate_id() {
6365 let dir = tempfile::tempdir().unwrap();
6366 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6367 let mut first = task("first");
6368 first.start("run-1".into());
6369 first.succeed();
6370 q.put(&mut first).unwrap();
6371 assert_eq!(first.id, TAKEN);
6372 let stored_before = q.get(&first.id).unwrap();
6373 assert!(!stored_before.runs.is_empty());
6374
6375 let mut second = task("second");
6376 q.put(&mut second).unwrap();
6377
6378 assert_ne!(second.id, first.id);
6379 assert!(second.runs.is_empty());
6380 assert!(second.passes.is_empty());
6381 assert!(second.chat_notices.is_empty());
6382 assert!(second.chat_notice.is_none());
6383 assert_eq!(second.chat_notice_due(), task("fresh").chat_notice_due());
6384 let a = q.get(&first.id).unwrap();
6385 let b = q.get(&second.id).unwrap();
6386 assert_eq!(a.title, "first");
6387 assert_eq!(a.runs, stored_before.runs);
6388 assert_eq!(a.passes, stored_before.passes);
6389 assert_eq!(a.chat_notices, stored_before.chat_notices);
6390 assert_eq!(a.chat_notice, stored_before.chat_notice);
6391 assert_eq!(a.chat_notice_due(), stored_before.chat_notice_due());
6392 assert_eq!(b.title, "second");
6393 assert!(b.runs.is_empty());
6394 assert!(b.passes.is_empty());
6395 assert!(b.chat_notice.is_none());
6396
6397 let mut again = q.get(&second.id).unwrap();
6399 again.title = "second, edited".into();
6400 q.put(&mut again).unwrap();
6401 assert_eq!(q.get(&second.id).unwrap().title, "second, edited");
6402 assert_eq!(q.list().len(), 2);
6403 }
6404
6405 #[test]
6406 fn a_new_task_skips_an_id_with_an_orphaned_claim_or_attachments() {
6407 let dir = tempfile::tempdir().unwrap();
6408 let q = queue_with_ids(dir.path(), &["orphan-lock", "orphan-dir", "free"]);
6409 std::fs::create_dir_all(q.root()).unwrap();
6410 std::fs::write(q.root().join("orphan-lock.lock"), "x").unwrap();
6411 std::fs::create_dir_all(q.attachments_dir("orphan-dir")).unwrap();
6412 let mut t = task("t");
6413 q.put(&mut t).unwrap();
6414 assert_eq!(t.id, "free");
6415 }
6416
6417 #[test]
6418 fn a_claimed_new_task_holds_the_claim_on_the_reserved_id() {
6419 let dir = tempfile::tempdir().unwrap();
6420 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6421 let mut first = task("first");
6422 q.put(&mut first).unwrap();
6423 let mut second = task("second");
6424 let claim = q.file_claimed(&mut second).unwrap();
6425 assert_ne!(second.id, first.id);
6426 assert!(q.claim(&second.id).is_err());
6427 assert!(q.claim(&first.id).is_ok(), "the first task was not claimed");
6428 drop(claim);
6429 }
6430
6431 #[test]
6432 fn an_attached_new_task_skips_a_taken_id() {
6433 let dir = tempfile::tempdir().unwrap();
6434 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6435 let mut first = task("first");
6436 q.put(&mut first).unwrap();
6437 let src = source_file(dir.path(), "shot.png", "x");
6438 let mut second = task("second");
6439 q.attach_and_put(&mut second, &[src]).unwrap();
6440 assert_ne!(second.id, first.id);
6441 assert!(!q.attachments_dir(&first.id).exists());
6442 assert_eq!(q.attachment_paths(&second).len(), 1);
6443 assert!(q.attachment_paths(&second)[0].is_file());
6444 }
6445
6446 #[test]
6447 fn attaching_before_putting_a_new_task_reserves_the_id() {
6448 let dir = tempfile::tempdir().unwrap();
6449 let q = queue_with_ids(dir.path(), &[TAKEN, TAKEN]);
6450 let mut first = task("first");
6451 q.put(&mut first).unwrap();
6452 let src = source_file(dir.path(), "shot.png", "x");
6453 let mut second = task("second");
6454 q.attach(&mut second, &[src]).unwrap();
6455 assert_ne!(second.id, first.id);
6456 assert!(q.attachments_dir(&second.id).is_dir());
6457 q.put(&mut second).unwrap();
6458 assert_eq!(q.get(&second.id).unwrap().attachments.len(), 1);
6459 assert_eq!(q.get(&first.id).unwrap().title, "first");
6460 assert!(q.get(&first.id).unwrap().attachments.is_empty());
6461 }
6462
6463 #[test]
6464 fn create_new_keeps_its_already_exists_meaning() {
6465 let dir = tempfile::tempdir().unwrap();
6466 let q = Queue::at(dir.path().join("queue"));
6467 let mut a = task("a");
6468 a.id = "fixed-id".into();
6469 assert!(q.create_new(&mut a).unwrap());
6470 let mut b = task("b");
6471 b.id = "fixed-id".into();
6472 assert!(!q.create_new(&mut b).unwrap());
6473 assert_eq!(q.get("fixed-id").unwrap().title, "a");
6474 }
6475
6476 #[test]
6477 fn filing_gives_up_when_no_id_is_ever_free() {
6478 let dir = tempfile::tempdir().unwrap();
6479 let q = Queue::at(dir.path().join("queue")).with_id_source(|| TAKEN.to_owned());
6480 let mut first = task("first");
6481 q.put(&mut first).unwrap();
6482 let mut second = task("second");
6483 assert!(q.put(&mut second).is_err());
6484 assert_eq!(q.get(&first.id).unwrap().title, "first");
6485 }
6486}