1use std::path::{Path, PathBuf};
84use std::time::Duration;
85
86use jiff::Timestamp;
87
88use crate::ask::{Question, QuestionStatus, Questions};
89use crate::config::Config;
90use crate::queue::{HoldSource, OperatorResume, Queue, Task, TaskStatus};
91
92pub const NODE: &str = "triage";
95
96pub const DEPS_NODE: &str = "triage-deps";
100
101const SEAT: &str = "triage";
106
107const MANUAL_STALE_AFTER: Duration = Duration::from_secs(7 * 24 * 60 * 60);
119
120struct Wording {
124 lang: &'static str,
125 resume: &'static str,
126 wait: &'static str,
127 discard: &'static str,
128 resume_now: &'static str,
129 keep_held: &'static str,
130}
131
132const EN: Wording = Wording {
133 lang: "en",
134 resume: "resume it",
135 wait: "not yet",
136 discard: "discard it",
137 resume_now: "resume it",
138 keep_held: "keep it held",
139};
140
141const JA: Wording = Wording {
142 lang: "ja",
143 resume: "再開してよい",
144 wait: "まだ待って",
145 discard: "捨ててよい",
146 resume_now: "再開する",
147 keep_held: "まだ止めておく",
148};
149
150fn wording(language: &str) -> &'static Wording {
153 if crate::lang::is_japanese(language) {
154 &JA
155 } else {
156 &EN
157 }
158}
159
160impl Wording {
161 fn choices3(&self) -> Vec<String> {
162 vec![
163 self.resume.to_owned(),
164 self.wait.to_owned(),
165 self.discard.to_owned(),
166 ]
167 }
168
169 fn choices2(&self) -> Vec<String> {
170 vec![self.resume_now.to_owned(), self.keep_held.to_owned()]
171 }
172
173 fn source_label(&self, source: Option<HoldSource>) -> &'static str {
174 match (self.lang, source) {
175 ("ja", Some(HoldSource::Machine)) => "machine(機械による自動保留)",
176 ("ja", Some(HoldSource::Manual)) => "manual(操作者による手動保留)",
177 ("ja", None) => "unknown(schema 3 未満の旧レコード、または理由未記録)",
178 (_, Some(HoldSource::Machine)) => "machine (automatic recovery hold)",
179 (_, Some(HoldSource::Manual)) => "manual (an operator held this)",
180 (_, None) => "unknown (pre-schema-3 record, or never recorded)",
181 }
182 }
183
184 fn detail(&self, task: &Task, why: &str) -> String {
193 let none = if self.lang == "ja" {
194 "(記録なし)"
195 } else {
196 "(none recorded)"
197 };
198 let reason = task
199 .hold_reason
200 .as_deref()
201 .or(task.last_error.as_deref())
202 .unwrap_or(none);
203 format!(
204 "task: {} ({})\ntitle: {}\nhold source: {}\nhold reason: {reason}\n\n{why}",
205 task.id,
206 task.short(),
207 task.title,
208 self.source_label(task.hold_source),
209 )
210 }
211
212 fn summary_machine_unknown(&self, task: &Task) -> String {
213 if self.lang == "ja" {
214 format!("保留タスク {} の再開可否を判断してください", task.short())
215 } else {
216 format!("decide whether to resume held task {}", task.short())
217 }
218 }
219
220 fn why_machine(&self) -> &'static str {
221 if self.lang == "ja" {
222 "機械的な保留(machine hold)ですが、原因がすでに解消しているかを自動では判断できませんでした。"
223 } else {
224 "This is a machine hold, but whether its cause has resolved could not be \
225 checked automatically."
226 }
227 }
228
229 fn summary_conductor_override(&self, task: &Task) -> String {
230 if self.lang == "ja" {
231 format!(
232 "再開と回答済みのタスク {} を conductor が再び保留しました",
233 task.short()
234 )
235 } else {
236 format!(
237 "task {} was resumed at your word, but the conductor held it again",
238 task.short()
239 )
240 }
241 }
242
243 fn why_conductor_override(&self, o: &OperatorResume) -> String {
244 let reason = o.conductor_rehold.as_deref().unwrap_or_default();
245 if self.lang == "ja" {
246 format!(
247 "{} に再開と回答済みですが、conductor が再び hold しました。conductor の理由: \
248 {reason}\n\n強制再キューを選ぶと、以後 conductor はこのタスクを hold できません。",
249 o.at
250 )
251 } else {
252 format!(
253 "You answered \"resume\" at {}, but the conductor held the task again. \
254 Its reason: {reason}\n\nForcing a requeue stops the conductor from \
255 holding this task again.",
256 o.at
257 )
258 }
259 }
260
261 fn choices_conductor_override(&self) -> Vec<String> {
263 if self.lang == "ja" {
264 vec![
265 "強制再キュー(conductor は再 hold 不可)".to_owned(),
266 "手動 hold のまま".to_owned(),
267 "捨ててよい".to_owned(),
268 ]
269 } else {
270 vec![
271 "force requeue (conductor must not hold again)".to_owned(),
272 "keep held (manual)".to_owned(),
273 "discard".to_owned(),
274 ]
275 }
276 }
277
278 fn summary_legacy(&self, task: &Task) -> String {
279 if self.lang == "ja" {
280 format!(
281 "hold_source が不明な保留タスク {} を確認してください",
282 task.short()
283 )
284 } else {
285 format!(
286 "held task {} has no recorded hold source - please take a look",
287 task.short()
288 )
289 }
290 }
291
292 fn why_legacy(&self) -> &'static str {
293 if self.lang == "ja" {
294 "hold_source が記録されていません。schema 3 より前のレコードか、理由が記録されなかった \
295 holdです。人が意図して止めたのか、クラッシュや強制再起動で宙に浮いただけなのか、\
296 このデータからは区別できません。"
297 } else {
298 "No hold_source was recorded - either a pre-schema-3 record, or a hold whose \
299 reason was never written down. Whether this was a deliberate hold or the \
300 leftover of a crash cannot be told from the data alone."
301 }
302 }
303
304 fn summary_manual_stale(&self, task: &Task, days: i64) -> String {
305 if self.lang == "ja" {
306 format!(
307 "{days}日間 保留されたままの手動保留タスク {} を確認してください",
308 task.short()
309 )
310 } else {
311 format!(
312 "held task {} has been on a manual hold for {days} day(s)",
313 task.short()
314 )
315 }
316 }
317
318 fn why_manual(&self) -> &'static str {
319 if self.lang == "ja" {
320 "操作者が明示的に止めた保留ですが、長期間そのままになっています。まだ止めておくか、\
321 再開するか教えてください。"
322 } else {
323 "An operator held this on purpose, but it has sat untouched for a while. Say \
324 whether to keep holding it or resume it."
325 }
326 }
327}
328
329#[derive(Debug, Clone, Copy, PartialEq, Eq)]
331enum Bucket {
332 MachineUnknown,
334 ConductorOverride,
337 Legacy,
339 ManualStale,
341}
342
343#[derive(Debug, Clone, Default)]
345pub struct Report {
346 pub resumed: Vec<String>,
349 pub asked: Vec<String>,
351 pub answered: Vec<String>,
353 pub quarantined: Vec<String>,
358}
359
360impl Report {
361 pub fn is_empty(&self) -> bool {
364 self.resumed.is_empty()
365 && self.asked.is_empty()
366 && self.answered.is_empty()
367 && self.quarantined.is_empty()
368 }
369}
370
371fn repo_for(task: &Task) -> PathBuf {
375 if task.repo.as_os_str().is_empty() {
376 PathBuf::from(".")
377 } else {
378 task.repo.clone()
379 }
380}
381
382fn is_disk_hold(task: &Task) -> bool {
393 task.hold_reason
394 .as_deref()
395 .is_some_and(crate::disk::is_gate_reason)
396}
397
398fn machine_cause_resolved(task: &Task, cfg: &Config) -> Option<bool> {
403 machine_cause_resolved_under(task, cfg, &crate::run::home())
404}
405
406fn machine_cause_resolved_under(task: &Task, cfg: &Config, home: &Path) -> Option<bool> {
407 machine_cause_resolved_with_admission(task, cfg, home, |paths| {
408 crate::retention::OutputLease::admit(home, paths, cfg.disk.min_free_bytes).is_ok()
409 })
410}
411
412fn machine_cause_resolved_with_admission(
413 task: &Task,
414 cfg: &Config,
415 home: &Path,
416 admission_available: impl Fn(&[PathBuf]) -> bool,
417) -> Option<bool> {
418 if task
419 .hold_reason
420 .as_deref()
421 .is_some_and(|r| r.starts_with(crate::cache::RESOURCE_HOLD))
422 {
423 let id = task.execution_runs().last()?;
425 let state = crate::run::RunState::load_under(id, home).ok()?;
426 let block = state.resource_block.as_ref()?;
427 if state.id != *id {
428 return None;
429 }
430 if block.kind == "disk admission"
431 && matches!(block.cause, crate::run::ResourceBlockCause::Other)
432 {
433 let mut paths = vec![repo_for(task), state.repo.clone(), state.worktree_root()];
437 paths.extend(state.candidates.iter().map(|c| c.worktree.clone()));
438 paths.extend(state.config.cache_dir());
439 paths.extend(cfg.cache_dir());
440 return Some(admission_available(&paths));
441 }
442 if block.kind != "shared build cache"
445 || !matches!(block.cause, crate::run::ResourceBlockCause::Timeout { .. })
446 {
447 return None;
448 }
449 return Some(
450 cfg.cache_dir()
451 .is_none_or(|dir| !crate::cache::in_use(home, &dir)),
452 );
453 }
454 if !is_disk_hold(task) {
455 return None;
456 }
457 let min = cfg.disk.min_free_bytes;
458 if min == 0 {
459 return Some(true);
462 }
463 Some(crate::daemon::disk_gate(&repo_for(task), cfg).is_none())
464}
465
466fn manual_is_stale(task: &Task, now: Timestamp) -> bool {
470 now.as_second() - task.updated_at.as_second() > MANUAL_STALE_AFTER.as_secs() as i64
471}
472
473fn marker_for(q: &Question) -> String {
476 format!("[triage:{}]", q.short())
477}
478
479fn already_applied(task: &Task, q: &Question) -> bool {
482 if task.triage_applied(&q.id) {
483 return true;
484 }
485 let marker = marker_for(q);
488 task.hold_reason
489 .as_deref()
490 .is_some_and(|r| r.contains(marker.as_str()))
491}
492
493fn latest_triage_question(questions: &Questions, task_id: &str) -> Option<Question> {
498 latest_question(questions, NODE, task_id)
499}
500
501pub(crate) struct RecoveryIndex {
504 latest: std::collections::HashMap<(String, String), Question>,
505 open: std::collections::BTreeSet<String>,
506}
507
508impl RecoveryIndex {
509 pub(crate) fn load(questions: &Questions) -> Self {
510 let queue = Queue::at(
511 questions
512 .root()
513 .parent()
514 .unwrap_or(Path::new(""))
515 .join("queue"),
516 );
517 let tasks = queue.list();
518 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
519 Self::from_scan(&queue, &tasks, questions.list_for_effects_in(&ownership))
520 }
521
522 fn from_scan(queue: &Queue, tasks: &[Task], all: Vec<Question>) -> Self {
523 let ids = crate::queue::record_ids(queue.root(), "json").unwrap_or_default();
524 let by_id: std::collections::HashMap<_, _> =
525 tasks.iter().map(|task| (task.id.as_str(), task)).collect();
526 let exact: std::collections::HashSet<_> = ids.iter().map(String::as_str).collect();
527 let mut aliases: std::collections::HashMap<&str, Option<&str>> = Default::default();
528 for id in &ids {
529 for boundary in id
530 .char_indices()
531 .map(|(i, _)| i)
532 .chain(std::iter::once(id.len()))
533 {
534 for alias in [&id[..boundary], &id[boundary..]] {
535 let entry = aliases.entry(alias).or_insert(Some(id));
536 if *entry != Some(id.as_str()) {
537 *entry = None;
538 }
539 }
540 }
541 }
542 let mut index = Self {
543 latest: Default::default(),
544 open: Default::default(),
545 };
546 for q in all {
547 if q.node != NODE && q.node != DEPS_NODE {
548 continue;
549 }
550 let resolved = if exact.contains(q.run.as_str()) {
551 Some(q.run.as_str())
552 } else {
553 aliases.get(q.run.as_str()).copied().flatten()
554 };
555 if let Some(id) = resolved
556 && by_id.get(id).is_some_and(|task| {
557 crate::repair_budget::reconcile::superseded_question(task, &q)
558 })
559 {
560 continue;
561 }
562 if q.status.open() {
563 index.open.insert(q.run.clone());
564 }
565 let key = (q.node.clone(), q.run.clone());
566 let entry = index.latest.entry(key).or_insert_with(|| q.clone());
567 if (q.asked_at, &q.id) > (entry.asked_at, &entry.id) {
568 *entry = q;
569 }
570 }
571 index
572 }
573
574 fn latest(&self, node: &str, task_id: &str) -> Option<Question> {
575 self.latest
576 .get(&(node.to_owned(), task_id.to_owned()))
577 .cloned()
578 }
579
580 pub(crate) fn pending(&self, task: &Task) -> Vec<Question> {
581 [NODE, DEPS_NODE]
582 .into_iter()
583 .filter_map(|node| {
584 self.latest(node, &task.id).filter(|q| {
585 q.status.open()
586 || awaiting_task_chat(q, task)
587 || (q.status == QuestionStatus::Answered
588 && if node == NODE {
589 !already_applied(task, q)
590 } else {
591 !task.triage_applied(&q.id)
592 })
593 })
594 })
595 .collect()
596 }
597}
598
599fn latest_question(questions: &Questions, node: &str, task_id: &str) -> Option<Question> {
601 RecoveryIndex::load(questions).latest(node, task_id)
602}
603
604#[derive(Debug, Clone, Copy, PartialEq, Eq)]
607enum AnswerAction {
608 Resume,
612 Discard,
615 KeepHeld,
620}
621
622fn interpret_answer(q: &Question) -> AnswerAction {
635 let resolution = q.resolution().unwrap_or_default();
636 match q.choices.iter().position(|c| *c == resolution) {
637 Some(0) => AnswerAction::Resume,
638 Some(2) => AnswerAction::Discard,
639 _ => AnswerAction::KeepHeld,
640 }
641}
642
643pub(crate) fn contextual_keep_held(q: &Question, label: &str) -> bool {
646 q.node == NODE
647 && q.seat == SEAT
648 && q.actions.is_empty()
649 && q.choices.get(1).is_some_and(|c| c == label)
650}
651
652fn keep_held_note(task: &Task, q: &Question, resolution: &str) -> String {
657 let marker = format!("{} operator: {resolution}", marker_for(q));
658 match task.hold_reason.as_deref() {
659 Some(existing) if !existing.is_empty() => format!("{existing}\n{marker}"),
660 _ => marker,
661 }
662}
663
664fn file_question(
668 questions: &Questions,
669 ownership: &crate::task_chat::InquiryOwnership,
670 task: &Task,
671 bucket: Bucket,
672 w: &Wording,
673 now: Timestamp,
674) -> Option<Question> {
675 let (summary, why, choices) = match bucket {
676 Bucket::MachineUnknown => (
677 w.summary_machine_unknown(task),
678 w.why_machine().to_owned(),
679 w.choices3(),
680 ),
681 Bucket::ConductorOverride => (
682 w.summary_conductor_override(task),
683 task.resume_override
684 .as_ref()
685 .map(|o| w.why_conductor_override(o))
686 .unwrap_or_default(),
687 w.choices_conductor_override(),
688 ),
689 Bucket::Legacy => (
690 w.summary_legacy(task),
691 w.why_legacy().to_owned(),
692 w.choices3(),
693 ),
694 Bucket::ManualStale => {
695 let days = (now.as_second() - task.updated_at.as_second()) / (24 * 60 * 60);
696 (
697 w.summary_manual_stale(task, days),
698 w.why_manual().to_owned(),
699 w.choices2(),
700 )
701 }
702 };
703 let mut q = Question::new(
704 task.id.clone(),
705 NODE.to_owned(),
706 SEAT.to_owned(),
707 summary,
708 w.detail(task, &why),
709 choices,
710 );
711 q.triage_generation = Some(task.resumption.as_ref().map_or(0, |r| r.generation));
712 questions.put_executable_in(&mut q, ownership).ok()?;
713 Some(q)
714}
715
716pub(crate) fn deputy_brief(q: &Question, queue: &Queue) -> String {
721 let task = queue.get(&q.run).ok();
722 let mut s = match &task {
723 Some(t) => format!(
724 "Task {} (`magi task show {}`), \"{}\", is {}. Hold source: {}. Reason: {}.",
725 t.id,
726 t.id,
727 t.title,
728 t.status.as_str(),
729 match t.hold_source {
730 Some(HoldSource::Machine) => "machine (an automatic hold)",
731 Some(HoldSource::Manual) => "manual (an operator held it)",
732 None => "unknown (an old record, treated as the operator's)",
733 },
734 t.hold_reason
735 .as_deref()
736 .or(t.last_error.as_deref())
737 .unwrap_or("(none recorded)")
738 ),
739 None => format!("Task {} no longer exists (its record is gone).", q.run),
740 };
741 s.push_str("\n\nWhat each option does, by position:");
742 let n = q.choices.len();
743 for (i, c) in q.choices.iter().enumerate() {
744 let effect = if q.node == DEPS_NODE {
745 match i {
746 0 => "the dependency is released so the loop runs it",
747 1 => {
748 "the dependants are detached and the dependency task is DELETED; this cannot be undone"
749 }
750 2 => "the dependants are detached; the dependency stays as it is",
751 _ => "nothing is applied",
752 }
753 } else {
754 match i {
755 0 => "the task is released and requeued",
756 2 if n > 2 => "the task is DELETED; this cannot be undone",
757 _ => "the task stays held, with the answer noted",
758 }
759 };
760 s.push_str(&format!("\n- `{c}`: {effect}"));
761 }
762 s.push_str(
763 "\n\nThe daemon applies the answer on its next pass; you apply nothing. When \
764 the owner's words pick an option - including releasing a task they held \
765 themselves, or deleting it - settle it.",
766 );
767 s
768}
769
770fn quarantine_orphaned_blocked(queue: &Queue, questions: &Questions) -> Vec<String> {
788 let mut quarantined = Vec::new();
789 for listed in queue.list() {
790 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
791 continue;
792 }
793 let Ok(_claim) = queue.claim(&listed.id) else {
794 continue;
795 };
796 let Ok(mut task) = queue.get(&listed.id) else {
797 continue;
798 };
799 if task.status != TaskStatus::Blocked {
800 continue;
801 }
802 let deleted = queue.apply_deleted_blockers(&mut task, questions);
803 if !deleted.is_empty() && queue.put(&mut task).is_ok() {
804 for id in &deleted {
805 queue.note_dependency_deleted(&task, id);
806 }
807 }
808 if task.status != TaskStatus::Blocked {
809 continue;
810 }
811 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
812 if missing.is_empty() {
813 continue;
814 }
815 let language = crate::lang::of_repo(&repo_for(&task));
816 let mut cause = missing.clone();
817 cause.sort();
818 cause.dedup();
819 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
820 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
821 &task.blocked_by,
822 &missing,
823 &language,
824 )));
825 if queue.put(&mut task).is_ok() {
826 quarantined.push(task.id.clone());
827 }
828 }
829 quarantined
830}
831
832#[derive(Debug, Clone, Copy, PartialEq, Eq)]
835enum DepsAction {
836 Release,
837 Discard,
838 Detach,
839 Nothing,
841}
842
843fn interpret_deps_answer(q: &Question) -> DepsAction {
844 let resolution = q.resolution().unwrap_or_default();
845 match q.choices.iter().position(|c| *c == resolution) {
846 Some(0) => DepsAction::Release,
847 Some(1) => DepsAction::Discard,
848 Some(2) => DepsAction::Detach,
849 _ => DepsAction::Nothing,
850 }
851}
852
853fn deps_texts(w: &Wording, root: &Task, dependants: &[&Task]) -> (String, String, Vec<String>) {
855 let ja = w.lang == "ja";
856 let reason = root
857 .hold_reason
858 .as_deref()
859 .or(root.last_error.as_deref())
860 .unwrap_or(if ja {
861 "(記録なし)"
862 } else {
863 "(none recorded)"
864 });
865 let list: String = dependants
866 .iter()
867 .map(|t| format!("- {} {}\n", t.short(), t.title))
868 .collect();
869 if ja {
870 (
871 format!(
872 "{} ({}) が {} 件のタスクを止めています",
873 root.short(),
874 root.status.as_str(),
875 dependants.len()
876 ),
877 format!(
878 "task: {} ({})\ntitle: {}\n状態: {}\n理由: {reason}\n\n\
879 このタスクは自動では実行されないため、依存している次のタスクは永遠に待ち続けます:\n{list}",
880 root.id,
881 root.short(),
882 root.title,
883 root.status.as_str()
884 ),
885 vec![
886 "依存先を再開する".to_owned(),
887 "依存先を捨てる(依存タスクは切り離して実行)".to_owned(),
888 "依存タスクを切り離す(依存先はそのまま)".to_owned(),
889 ],
890 )
891 } else {
892 (
893 format!(
894 "{} ({}) is freezing {} blocked task(s)",
895 root.short(),
896 root.status.as_str(),
897 dependants.len()
898 ),
899 format!(
900 "task: {} ({})\ntitle: {}\nstatus: {}\nreason: {reason}\n\n\
901 Nothing in the loop will ever run this task, so these dependants wait \
902 forever:\n{list}",
903 root.id,
904 root.short(),
905 root.title,
906 root.status.as_str()
907 ),
908 vec![
909 "release the dependency".to_owned(),
910 "discard the dependency (dependants are detached and run)".to_owned(),
911 "detach the dependants (the dependency stays as it is)".to_owned(),
912 ],
913 )
914 }
915}
916
917fn detach_dependants(queue: &Queue, questions: &Questions, root: &str) {
921 for listed in queue.list() {
922 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| matches!(crate::queue::dependency(queue, questions, b), crate::queue::Dependency::Task(id) if id == root)) {
923 continue;
924 }
925 let Ok(_claim) = queue.claim(&listed.id) else {
926 continue;
927 };
928 let Ok(mut t) = queue.get(&listed.id) else {
929 continue;
930 };
931 if t.status != TaskStatus::Blocked {
932 continue;
933 }
934 for id in t.blocked_by.clone() {
935 if matches!(crate::queue::dependency(queue, questions, &id), crate::queue::Dependency::Task(full) if full == root)
936 {
937 t.unblock(&id);
938 }
939 }
940 let _ = queue.put(&mut t);
941 }
942}
943
944fn apply_deps_answer(
948 queue: &Queue,
949 questions: &Questions,
950 q: &Question,
951 now: Timestamp,
952 ownership: &crate::task_chat::InquiryOwnership,
953) -> bool {
954 let Ok(_claim) = queue.claim(&q.run) else {
955 return false;
956 };
957 let Ok(current) = questions.get_for_effects_in(&q.id, ownership) else {
958 return false;
959 };
960 if current.status != QuestionStatus::Answered {
961 return false;
962 }
963 let q = ¤t;
964 let Ok(mut root) = queue.get(&q.run) else {
965 return false;
966 };
967 if root.triage_applied(&q.id) {
968 return false;
969 }
970 if root.operator_decided_at.is_some_and(|at| at > q.asked_at)
971 || q.triage_generation
972 .is_some_and(|g| g != root.resumption.as_ref().map_or(0, |r| r.generation))
973 {
974 root.mark_triage_applied(&q.id);
975 return queue.put(&mut root).is_ok();
976 }
977 match interpret_deps_answer(q) {
978 DepsAction::Release => {
979 if root.status == TaskStatus::Running {
980 return false;
981 }
982 if crate::repair_budget::reconcile::blocker(queue, &root).is_some() {
983 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(q) else {
984 return false;
985 };
986 root.decision_effects.insert(q.id.clone(), receipt);
987 root.mark_triage_applied(&q.id);
988 return queue.put(&mut root).is_ok();
989 }
990 root.answers.push(crate::queue::AnsweredQuestion {
991 question: q.summary.clone(),
992 answer: q.resolution().unwrap_or_default(),
993 });
994 if crate::base_sync_recovery::held_task(queue, &root) {
995 root.mark_triage_applied(&q.id);
996 return queue.put(&mut root).is_ok();
997 }
998 let before = root.operator_decided_at;
999 let contract =
1000 crate::repair_budget::contract(&root, crate::repair_budget::Kind::Review).ok();
1001 root.release_operator("triage-dependencies", Some(&q.id));
1002 if let Some(contract) = contract
1003 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1004 q,
1005 before,
1006 &contract,
1007 root.resumption.as_ref().expect("release"),
1008 )
1009 {
1010 root.decision_effects.insert(q.id.clone(), evidence);
1011 }
1012 root.resume_override = Some(OperatorResume {
1013 question_id: q.id.clone(),
1014 at: now,
1015 conductor_rehold: None,
1016 forced: false,
1017 pinned_run: None,
1018 });
1019 }
1020 DepsAction::Discard => {
1021 detach_dependants(queue, questions, &root.id);
1022 return queue.discard_triage(&root.id, questions, &q.id).is_ok();
1023 }
1024 DepsAction::Detach => detach_dependants(queue, questions, &root.id),
1025 DepsAction::Nothing => return false,
1030 }
1031 root.mark_triage_applied(&q.id);
1032 queue.put(&mut root).is_ok()
1033}
1034
1035fn repair_receipt_notice(queue: &Queue, task: &Task) -> bool {
1036 let Some(detail) = crate::repair_budget::reconcile::blocker(queue, task) else {
1037 return false;
1038 };
1039 if let Some(home) = queue.root().parent() {
1040 let _ = crate::task_inquiry::Outbox::at(home).file_for_occurrence(
1041 task,
1042 task.runs.last().cloned(),
1043 "repair-receipt-recovery".into(),
1044 &task
1045 .repair_grants
1046 .last()
1047 .expect("diagnosed receipt")
1048 .request
1049 .request,
1050 "Retained repair receipt needs reconciliation",
1051 &detail,
1052 );
1053 }
1054 true
1055}
1056
1057fn ask_about_stuck_roots(
1066 queue: &Queue,
1067 questions: &Questions,
1068 config_override: Option<&Path>,
1069 report: &mut Report,
1070) {
1071 let all = questions.list_for_effects();
1072 let tasks = queue.list();
1073 let recovery = RecoveryIndex::from_scan(queue, &tasks, all.clone());
1074 let inv = crate::blockers::Inventory::load_with_tasks(queue, questions, tasks);
1075 let mut frozen: std::collections::BTreeMap<String, Vec<String>> = Default::default();
1076 for (id, roots) in inv.stuck() {
1077 for root in roots {
1078 if root != id {
1079 frozen.entry(root).or_default().push(id.clone());
1080 }
1081 }
1082 }
1083 let ownership = crate::task_chat::InquiryOwnership::default();
1087 for mut q in all
1088 .iter()
1089 .filter(|q| q.node == DEPS_NODE && q.status.open())
1090 .cloned()
1091 {
1092 if !frozen.contains_key(&q.run) {
1093 q.abandon("nothing waits on this task anymore");
1094 let _ = questions.put_executable_in(&mut q, &ownership);
1095 }
1096 }
1097
1098 for (root_id, dependant_ids) in &frozen {
1099 let Ok(_claim) = queue.claim(root_id) else {
1100 continue;
1101 };
1102 let seeds: Vec<_> = std::iter::once(root_id)
1105 .chain(dependant_ids.iter())
1106 .filter_map(|id| queue.get(id).ok())
1107 .collect();
1108 let current = crate::blockers::Inventory::load_for_tasks(queue, questions, &seeds);
1109 let Some(root) = current.task(root_id) else {
1110 continue;
1111 };
1112 if crate::base_sync_recovery::held_task(queue, root) {
1113 continue;
1114 }
1115 let dependants: Vec<&Task> = dependant_ids
1116 .iter()
1117 .filter_map(|d| current.task(d))
1118 .filter(|t| current.stuck_roots(t).contains(root_id))
1119 .collect();
1120 if dependants.is_empty() {
1121 continue;
1122 }
1123 if repair_receipt_notice(queue, root) {
1124 continue;
1125 }
1126 let latest = recovery.latest(DEPS_NODE, root_id);
1127 match latest.as_ref() {
1128 Some(q) if q.status.open() || awaiting_task_chat(q, root) => continue,
1129 Some(q) if q.status == QuestionStatus::Answered && !root.triage_applied(&q.id) => {
1130 continue;
1131 }
1132 Some(q) if q.status == QuestionStatus::Abandoned => {
1133 let changed = root.updated_at > q.asked_at
1134 || dependants.iter().any(|t| t.updated_at > q.asked_at);
1135 if !changed {
1136 continue;
1137 }
1138 }
1139 _ => {}
1140 }
1141 if !recovery.pending(root).is_empty() {
1142 continue;
1143 }
1144 let cfg = Config::discover(&repo_for(root), config_override)
1145 .ok()
1146 .map(|(c, _)| c);
1147 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1148 let (summary, detail, choices) = deps_texts(w, root, &dependants);
1149 let mut question = Question::new(
1150 root.id.clone(),
1151 DEPS_NODE.to_owned(),
1152 SEAT.to_owned(),
1153 summary,
1154 detail,
1155 choices,
1156 );
1157 question.triage_generation = Some(root.resumption.as_ref().map_or(0, |r| r.generation));
1158 if questions
1159 .put_executable_in(&mut question, &ownership)
1160 .is_ok()
1161 {
1162 report.asked.push(root.id.clone());
1163 }
1164 }
1165}
1166
1167pub fn apply_answers(queue: &Queue, questions: &Questions) -> Vec<String> {
1170 let mut applied = Vec::new();
1171 let ownership = crate::task_chat::InquiryOwnership::default();
1172 for q in questions.list_for_effects_in(&ownership) {
1173 if q.node == DEPS_NODE
1174 && q.status == QuestionStatus::Answered
1175 && apply_deps_answer(queue, questions, &q, Timestamp::now(), &ownership)
1176 {
1177 applied.push(q.run.clone());
1178 }
1179 }
1180 let tasks = queue.list();
1182 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
1183 let recovery =
1184 RecoveryIndex::from_scan(queue, &tasks, questions.list_for_effects_in(&ownership));
1185 for listed in tasks {
1186 let Some(q) = recovery.latest(NODE, &listed.id) else {
1187 continue;
1188 };
1189 if q.status != QuestionStatus::Answered || already_applied(&listed, &q) {
1190 continue;
1191 }
1192 let Ok(_claim) = queue.claim(&listed.id) else {
1193 continue;
1194 };
1195 let Ok(mut task) = queue.get(&listed.id) else {
1196 continue;
1197 };
1198 let Ok(q) = questions.get_for_effects_in(&q.id, &ownership) else {
1201 continue;
1202 };
1203 if q.status != QuestionStatus::Answered {
1204 continue;
1205 }
1206 if already_applied(&task, &q)
1207 || !matches!(task.status, TaskStatus::Held | TaskStatus::Queued)
1208 {
1209 continue;
1210 }
1211 let generation = task.resumption.as_ref().map_or(0, |r| r.generation);
1212 let action = interpret_answer(&q);
1213 let mut newer_decision = q.triage_generation.is_some_and(|g| g != generation)
1214 || task.resumption.as_ref().is_some_and(|r| r.at > q.asked_at)
1215 || task.held_at.is_some_and(|at| at > q.asked_at)
1216 || task.operator_decided_at.is_some_and(|at| at > q.asked_at);
1217 if matches!(action, AnswerAction::KeepHeld)
1221 && q.answered_at.is_some_and(|at| {
1222 task.operator_decided_at
1223 .is_none_or(|decision| at > decision)
1224 && task.resumption.as_ref().is_none_or(|r| at > r.at)
1225 })
1226 {
1227 newer_decision = false;
1228 }
1229 let resolution = q.resolution().unwrap_or_default();
1231 task.answers.push(crate::queue::AnsweredQuestion {
1232 question: q.summary.clone(),
1233 answer: if newer_decision {
1234 format!("{resolution} (superseded by a later operator decision)")
1235 } else {
1236 resolution
1237 },
1238 });
1239 if !newer_decision {
1240 match action {
1241 AnswerAction::Resume if crate::base_sync_recovery::held_task(queue, &task) => {
1242 }
1245 AnswerAction::Resume
1246 if crate::repair_budget::reconcile::blocker(queue, &task).is_some() =>
1247 {
1248 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(&q) else {
1249 continue;
1250 };
1251 task.decision_effects.insert(q.id.clone(), receipt);
1252 }
1253 AnswerAction::Resume => {
1254 let before = task.operator_decided_at;
1255 let contract =
1256 crate::repair_budget::contract(&task, crate::repair_budget::Kind::Review)
1257 .ok();
1258 task.release_operator("triage", Some(&q.id));
1259 if let Some(contract) = contract
1260 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1261 &q,
1262 before,
1263 &contract,
1264 task.resumption.as_ref().expect("release"),
1265 )
1266 {
1267 task.decision_effects.insert(q.id.clone(), evidence);
1268 }
1269 task.resume_override = Some(OperatorResume {
1270 question_id: q.id.clone(),
1271 at: Timestamp::now(),
1272 conductor_rehold: None,
1273 forced: true,
1274 pinned_run: None,
1275 });
1276 }
1277 AnswerAction::KeepHeld => {
1278 let note = keep_held_note(&task, &q, &q.resolution().unwrap_or_default());
1279 task.hold_manual(Some(note));
1280 }
1281 AnswerAction::Discard => {
1282 if queue.discard_triage(&task.id, questions, &q.id).is_ok() {
1283 applied.push(task.id);
1284 }
1285 continue;
1286 }
1287 }
1288 }
1289 task.mark_triage_applied(&q.id);
1290 if queue.put(&mut task).is_ok() {
1291 applied.push(task.id);
1292 }
1293 }
1294 applied
1295}
1296
1297pub fn run_once(
1315 queue: &Queue,
1316 questions: &Questions,
1317 config_override: Option<&Path>,
1318 now: Timestamp,
1319) -> Report {
1320 let mut report = Report {
1321 answered: apply_answers(queue, questions),
1322 quarantined: quarantine_orphaned_blocked(queue, questions),
1323 ..Report::default()
1324 };
1325 ask_about_stuck_roots(queue, questions, config_override, &mut report);
1326 let recovery = RecoveryIndex::load(questions);
1327 let ownership = crate::task_chat::InquiryOwnership::default();
1328 for listed in queue.list() {
1329 if listed.status != TaskStatus::Held {
1330 continue;
1331 }
1332 let Ok(_claim) = queue.claim(&listed.id) else {
1333 continue;
1334 };
1335 let Ok(mut task) = queue.get(&listed.id) else {
1336 continue;
1337 };
1338 if task.status != TaskStatus::Held {
1342 continue;
1343 }
1344 if crate::base_sync_recovery::held_task(queue, &task) {
1345 continue;
1346 }
1347 if repair_receipt_notice(queue, &task) {
1348 continue;
1349 }
1350 if deps_pending_in(&recovery, &task) {
1354 continue;
1355 }
1356
1357 let cfg = Config::discover(&repo_for(&task), config_override)
1358 .ok()
1359 .map(|(c, _)| c);
1360 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1361
1362 if let Some(q) = recovery.latest(NODE, &task.id) {
1363 if q.status.open() || awaiting_task_chat(&q, &task) {
1364 continue;
1366 }
1367 }
1373
1374 match task.hold_source {
1375 Some(HoldSource::Machine) => {
1376 let overridden = task
1377 .resume_override
1378 .as_ref()
1379 .is_some_and(|o| o.conductor_rehold.is_some() && !o.forced);
1380 if overridden {
1381 if file_question(
1382 questions,
1383 &ownership,
1384 &task,
1385 Bucket::ConductorOverride,
1386 w,
1387 now,
1388 )
1389 .is_some()
1390 {
1391 report.asked.push(task.id.clone());
1392 }
1393 } else if cfg.as_ref().and_then(|c| machine_cause_resolved(&task, c)) == Some(true)
1394 {
1395 task.release();
1396 if queue.put(&mut task).is_ok() {
1397 report.resumed.push(task.id.clone());
1398 }
1399 } else if file_question(
1400 questions,
1401 &ownership,
1402 &task,
1403 Bucket::MachineUnknown,
1404 w,
1405 now,
1406 )
1407 .is_some()
1408 {
1409 report.asked.push(task.id.clone());
1410 }
1411 }
1412 None => {
1413 if file_question(questions, &ownership, &task, Bucket::Legacy, w, now).is_some() {
1414 report.asked.push(task.id.clone());
1415 }
1416 }
1417 Some(HoldSource::Manual) => {
1418 if manual_is_stale(&task, now)
1419 && file_question(questions, &ownership, &task, Bucket::ManualStale, w, now)
1420 .is_some()
1421 {
1422 report.asked.push(task.id.clone());
1423 }
1424 }
1425 }
1426 }
1427 report
1428}
1429
1430pub fn open_question_for(questions: &Questions, task_id: &str) -> Option<Question> {
1435 latest_triage_question(questions, task_id).filter(|q| q.status.open())
1436}
1437
1438pub fn pending_for(questions: &Questions, task: &Task) -> bool {
1452 !pending_recovery_questions(questions, task).is_empty()
1453}
1454
1455pub(crate) fn pending_recovery_questions(questions: &Questions, task: &Task) -> Vec<Question> {
1458 RecoveryIndex::load(questions).pending(task)
1459}
1460
1461fn awaiting_task_chat(q: &Question, task: &Task) -> bool {
1462 q.status != QuestionStatus::Abandoned
1463 && q.routed_chat.is_some()
1464 && !already_applied(task, q)
1465 && task.resumption.as_ref().map_or(0, |r| r.generation) == q.triage_generation.unwrap_or(0)
1466 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1467}
1468
1469fn deps_pending_in(recovery: &RecoveryIndex, task: &Task) -> bool {
1473 match recovery.latest(DEPS_NODE, &task.id) {
1474 Some(q) if q.status.open() || awaiting_task_chat(&q, task) => true,
1475 Some(q) if q.status == QuestionStatus::Answered => !task.triage_applied(&q.id),
1476 _ => false,
1477 }
1478}
1479
1480pub fn open_task_ids(questions: &Questions) -> std::collections::BTreeSet<String> {
1485 RecoveryIndex::load(questions).open
1486}
1487
1488#[cfg(test)]
1489mod tests {
1490 use super::*;
1491 use crate::ask::Answer;
1492 use crate::queue::Source;
1493 use jiff::SignedDuration;
1494
1495 fn store() -> (tempfile::TempDir, Queue, Questions) {
1496 let dir = tempfile::tempdir().unwrap();
1497 let q = Queue::at(dir.path().join("queue"));
1498 let s = Questions::at(dir.path().join("questions"));
1499 (dir, q, s)
1500 }
1501
1502 fn task(title: &str, repo: PathBuf) -> Task {
1503 Task::new(title.to_owned(), format!("do {title}"), repo, Source::Human)
1504 }
1505
1506 #[test]
1507 fn recovery_index_preserves_supersession_and_ambiguous_file_references() {
1508 let (dir, queue, questions) = store();
1509 let mut t = task("supersession", dir.path().join("repo"));
1510 t.id = "task-alpha-end".into();
1511 queue.put(&mut t).unwrap();
1512 let at: Timestamp = "2026-11-01T00:00:00Z".parse().unwrap();
1513 let mut all = Vec::new();
1514 for (i, run) in ["task-alpha-end", "task-alpha", "end"]
1515 .into_iter()
1516 .enumerate()
1517 {
1518 let mut q = Question::new(
1519 run.into(),
1520 NODE.into(),
1521 SEAT.into(),
1522 "superseded".into(),
1523 String::new(),
1524 EN.choices3(),
1525 );
1526 q.id = format!("superseded-{i}");
1527 q.asked_at = at;
1528 all.push(q);
1529 }
1530 let evidence: Vec<_> = all
1531 .iter()
1532 .map(|q| {
1533 (
1534 q.id.clone(),
1535 crate::repair_budget::reconcile::source(q).unwrap(),
1536 )
1537 })
1538 .collect();
1539 t.repair_reconciliations.push(
1540 serde_json::from_value(serde_json::json!({
1541 "request": {
1542 "task": t.id, "run": "run", "head": "head", "expect_pass": 0,
1543 "receipt": "grant", "question": "original", "answer_at": at,
1544 "request": "reconcile", "reason": "fixture"
1545 },
1546 "grant": {
1547 "request": {
1548 "task": t.id, "run": "run", "kind": "review", "rounds": 1,
1549 "head": "head", "expect_pass": 0, "request": "grant", "reason": "fixture"
1550 },
1551 "at": at, "decision": null, "contract": "fixture"
1552 },
1553 "source": "fixture", "application": "fixture", "decision": at,
1554 "superseded_questions": evidence
1555 }))
1556 .unwrap(),
1557 );
1558 queue.put(&mut t).unwrap();
1559 let index = RecoveryIndex::from_scan(&queue, &[t.clone()], all.clone());
1560 assert!(index.open.is_empty());
1561 std::fs::write(queue.path_of("task-alphabet-other"), "{").unwrap();
1563 std::fs::write(queue.path_of("broken-end"), "{").unwrap();
1564 let index = RecoveryIndex::from_scan(&queue, &[t.clone()], all.clone());
1565 assert_eq!(index.open, ["end".into(), "task-alpha".into()].into());
1566 all[0].detail.push_str("changed evidence");
1567 let index = RecoveryIndex::from_scan(&queue, &[t], all);
1568 assert!(index.open.contains("task-alpha-end"));
1569 assert_eq!(questions.list().len(), 0);
1570 }
1571
1572 #[test]
1573 fn ownership_index_requires_positive_foreground_provenance() {
1574 let (dir, queue, _) = store();
1575 let mut run = crate::run::RunState::new(
1576 dir.path().join("repo"),
1577 "main".into(),
1578 "origin".into(),
1579 "fixture".into(),
1580 Config::default(),
1581 );
1582 run.origin = Some(crate::run::Origin::operator());
1583 run.save_under(dir.path()).unwrap();
1584 let foreground = run.id.clone();
1585 run.id = "owned-run".into();
1586 run.origin = Some(crate::run::Origin::queue("missing-task"));
1587 run.save_under(dir.path()).unwrap();
1588 let mut task = task("owner", dir.path().join("repo"));
1589 task.runs = vec!["linked-run".into()];
1590 queue.put(&mut task).unwrap();
1591 let ownership = crate::task_chat::InquiryOwnership::default();
1592 for (run, cwd, node, seat, expected) in [
1593 (foreground.as_str(), true, "node", "seat", true),
1594 (foreground.as_str(), false, "node", "seat", false),
1595 ("owned-run", true, "ask", "operator", false),
1596 ("linked-run", true, "ask", "operator", false),
1597 ("missing-run", true, "node", "seat", false),
1598 ("missing-run", false, "ask", "operator", true),
1599 ] {
1600 let mut q = Question::new(
1601 run.into(),
1602 node.into(),
1603 seat.into(),
1604 "provenance".into(),
1605 String::new(),
1606 Vec::new(),
1607 );
1608 q.cwd = cwd.then(|| dir.path().join("repo").display().to_string());
1609 assert_eq!(standalone_by_scanning(dir.path(), &q), expected);
1610 assert_eq!(
1611 crate::task_chat::standalone_inquiry_in(dir.path(), &q, &ownership),
1612 expected
1613 );
1614 }
1615 }
1616
1617 fn standalone_by_scanning(home: &Path, q: &Question) -> bool {
1619 if q.routed_task.is_some()
1620 || (q.node == crate::bump::NOTICE_NODE && q.seat == "release-watch")
1621 {
1622 return false;
1623 }
1624 if q.legacy_foreground || q.run.is_empty() {
1625 return true;
1626 }
1627 if Queue::at(home.join("queue"))
1628 .list()
1629 .iter()
1630 .any(|task| task.id == q.run || task.runs.contains(&q.run))
1631 {
1632 return false;
1633 }
1634 let run = crate::run::RunState::load_under(&q.run, home).ok();
1635 let origin = run.as_ref().and_then(|run| run.origin.as_ref());
1636 if origin.is_some_and(|origin| origin.task.is_some()) {
1637 return false;
1638 }
1639 (q.node == "ask" && q.seat == "operator")
1640 || (q.cwd.is_some() && origin.is_some_and(|origin| origin.task.is_none()))
1641 }
1642
1643 fn scan_counts() -> (usize, usize, usize) {
1644 (
1645 crate::queue::LIST_READS.with(|c| c.replace(0)),
1646 crate::queue::TASK_READS.with(|c| c.replace(0)),
1647 crate::ask::QUESTION_SCANS.with(|c| c.replace(0)),
1648 )
1649 }
1650
1651 #[test]
1652 fn creating_and_updating_questions_share_one_queue_read() {
1653 let (dir, queue, questions) = store();
1654 let size = 200;
1655 let mut ids = Vec::new();
1656 for i in 0..size {
1657 let mut t = task("scan fixture", dir.path().join("repo"));
1658 t.id = format!("task-{i:04}");
1659 t.runs = vec![format!("run-{i:04}")];
1660 t.hold_manual(Some("held for scan fixture".into()));
1661 queue.put(&mut t).unwrap();
1662 ids.push(t.id);
1663 }
1664 let ownership = crate::task_chat::InquiryOwnership::default();
1665 scan_counts();
1666 let mut created = Vec::new();
1667 for id in &ids {
1668 let mut q = Question::new(
1669 id.clone(),
1670 NODE.into(),
1671 SEAT.into(),
1672 "scan fixture".into(),
1673 String::new(),
1674 EN.choices3(),
1675 );
1676 questions.put_executable_in(&mut q, &ownership).unwrap();
1677 assert_eq!(q.routed_task.as_deref(), Some(id.as_str()));
1678 created.push(q.id);
1679 }
1680 for id in &created {
1681 questions
1682 .update_in(id, &ownership, |q| {
1683 q.answer_delivered = true;
1684 Ok(())
1685 })
1686 .unwrap();
1687 }
1688 let (lists, _, _) = scan_counts();
1689 assert_eq!(lists, 1, "one queue read for {size} creations and updates");
1690 }
1691
1692 #[test]
1693 fn recovery_scans_are_linear_and_match_legacy_decisions() {
1694 let mut previous = None;
1695 for size in [150, 300] {
1696 let (dir, queue, questions) = store();
1697 let at: Timestamp = "2026-11-01T00:00:00Z".parse().unwrap();
1698 let mut sources = Vec::new();
1699 let mut expected_applied = std::collections::BTreeSet::new();
1700 for i in 0..size {
1701 let mut t = task("scan fixture", dir.path().join("repo"));
1702 t.id = format!("task-{i:04}");
1703 t.runs = vec![format!("run-{i:04}")];
1704 t.hold_manual(Some("held for scan fixture".into()));
1705 queue.put(&mut t).unwrap();
1706 let mut q = Question::new(
1707 t.id.clone(),
1708 NODE.into(),
1709 SEAT.into(),
1710 "scan fixture".into(),
1711 String::new(),
1712 EN.choices3(),
1713 );
1714 q.id = format!("question-{i:04}");
1715 q.asked_at = at;
1716 match i % 10 {
1717 0 => {
1718 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
1719 q.accepted_answer_at = q.answered_at;
1720 expected_applied.insert(t.id.clone());
1721 }
1722 1 => q.node = DEPS_NODE.into(),
1723 2 => {
1724 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
1725 }
1726 3 => {
1727 q.run = format!("missing-{i}");
1728 q.legacy_foreground = true;
1729 }
1730 4 => {
1731 q.run.clear();
1732 q.node = crate::bump::NOTICE_NODE.into();
1733 q.seat = "release-watch".into();
1734 }
1735 5 => {
1736 q.run = format!("missing-{i}");
1737 q.node = "ask".into();
1738 q.seat = "operator".into();
1739 }
1740 6 => q.run = t.runs[0].clone(),
1741 7 => q.run = format!("missing-{i}"),
1742 8 => q.abandon("fixture"),
1743 _ => {}
1744 }
1745 questions.put(&mut q).unwrap();
1746 sources.push(q.id.clone());
1747 if i % 10 == 9 {
1748 q.id = format!("question-{i:04}-tie");
1749 questions.put(&mut q).unwrap();
1750 sources.push(q.id.clone());
1751 }
1752 }
1753 let migration = dir.path().join("notifications/task-chat-migration-v1");
1754 std::fs::create_dir_all(&migration).unwrap();
1755 std::fs::write(
1756 migration.join("state.json"),
1757 serde_json::to_vec(&serde_json::json!({
1758 "schema": 1, "classification_revision": 1, "pending": [], "sources": sources
1759 }))
1760 .unwrap(),
1761 )
1762 .unwrap();
1763
1764 scan_counts();
1765 let all = questions.list_for_effects();
1766 let (lists, reads, scans) = scan_counts();
1767 assert_eq!((lists, reads, scans), (1, size, 1));
1768 let tasks = queue.list();
1769 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
1770 for q in &all {
1771 assert_eq!(
1772 crate::task_chat::standalone_inquiry_in(dir.path(), q, &ownership),
1773 standalone_by_scanning(dir.path(), q),
1774 "{}",
1775 q.id,
1776 );
1777 if q.id.ends_with("0002") {
1778 assert!(q.answer.is_none(), "saved consent remains fenced");
1779 }
1780 }
1781 let recovery = RecoveryIndex::from_scan(&queue, &tasks, all.clone());
1782 for task in &tasks {
1783 for node in [NODE, DEPS_NODE] {
1784 let expected = all
1785 .iter()
1786 .filter(|q| {
1787 q.node == node
1788 && q.run == task.id
1789 && !queue.get(&q.run).ok().is_some_and(|t| {
1790 crate::repair_budget::reconcile::superseded_question(&t, q)
1791 })
1792 })
1793 .max_by(|a, b| a.asked_at.cmp(&b.asked_at).then_with(|| a.id.cmp(&b.id)));
1794 assert_eq!(
1795 recovery.latest(node, &task.id).map(|q| q.id),
1796 expected.map(|q| q.id.clone())
1797 );
1798 }
1799 }
1800
1801 scan_counts();
1802 crate::daemon::apply_task_choice_actions(&queue, &questions, dir.path());
1803 assert_eq!(
1804 scan_counts(),
1805 (1, size, 1),
1806 "choice-action rechecks share ownership"
1807 );
1808
1809 scan_counts();
1810 let applied: std::collections::BTreeSet<_> =
1811 apply_answers(&queue, &questions).into_iter().collect();
1812 let counts = scan_counts();
1813 assert_eq!(applied, expected_applied);
1814 assert!(counts.0 <= 3, "queue scans: {counts:?}");
1815 assert!(counts.1 <= 4 * size, "task reads: {counts:?}");
1816 assert_eq!(counts.2, 2, "question scans: {counts:?}");
1817 if let Some((_, reads, scans)) = previous {
1818 assert!(counts.1 <= 2 * reads);
1819 assert_eq!(counts.2, scans);
1820 }
1821 previous = Some(counts);
1822
1823 let config = gate_disabled_config(dir.path());
1824 scan_counts();
1825 run_once(&queue, &questions, Some(&config), Timestamp::now());
1826 let lap = scan_counts();
1827 assert!(lap.0 <= 12, "triage queue scans: {lap:?}");
1828 assert!(lap.1 <= 14 * size, "triage task reads: {lap:?}");
1829 assert!(lap.2 <= 6, "triage question scans: {lap:?}");
1830 }
1831 }
1832
1833 #[test]
1834 fn run_once_filing_many_new_questions_reads_the_queue_a_constant_number_of_times() {
1835 let (dir, queue, questions) = store();
1836 let size = 200;
1837 for i in 0..size {
1838 let mut t = task("scan fixture", dir.path().join("repo"));
1839 t.id = format!("task-{i:04}");
1840 t.hold_machine(Some("gate red".to_owned()));
1841 queue.put(&mut t).unwrap();
1842 }
1843 scan_counts();
1844 let report = run_once(&queue, &questions, None, Timestamp::now());
1845 let (lists, _, _) = scan_counts();
1846 assert_eq!(report.asked.len(), size, "every held task is asked about");
1847 assert!(
1848 lists <= 12,
1849 "queue scans while filing {size} questions: {lists}"
1850 );
1851 }
1852
1853 #[test]
1854 fn base_recovery_stops_suppress_generic_and_dependency_resume_offers() {
1855 let (dir, queue, questions) = store();
1856 let mut root = task("base stop", dir.path().to_path_buf());
1857 let mut run = crate::run::RunState::new(
1858 dir.path().to_path_buf(),
1859 "main".into(),
1860 "base".into(),
1861 "retained implementation".into(),
1862 Config::default(),
1863 );
1864 run.status = crate::run::RunStatus::Voting;
1865 run.base_sync = Some(crate::run::BaseSync {
1866 tip: "tip".into(),
1867 behind: 3,
1868 attempts: 4,
1869 conflict: Some("persisted exhaustion".into()),
1870 already_in: None,
1871 });
1872 run.save_under(dir.path()).unwrap();
1873 root.link_run(&run.id);
1874 root.hold_machine(Some("base stop".into()));
1875 queue.put(&mut root).unwrap();
1876 let mut dependent = task("dependent", dir.path().to_path_buf());
1877 dependent.blocked_by = vec![root.id.clone()];
1878 dependent.status = TaskStatus::Blocked;
1879 queue.put(&mut dependent).unwrap();
1880 for _ in 0..2 {
1881 let report = run_once(&queue, &questions, None, Timestamp::now());
1882 assert!(report.asked.is_empty());
1883 assert!(report.resumed.is_empty());
1884 }
1885 let q = file_question(
1888 &questions,
1889 &crate::task_chat::InquiryOwnership::default(),
1890 &root,
1891 Bucket::MachineUnknown,
1892 &EN,
1893 Timestamp::now(),
1894 )
1895 .unwrap();
1896 let mut q = q;
1897 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
1898 questions.put_executable(&mut q).unwrap();
1899 apply_answers(&queue, &questions);
1900 let saved = queue.get(&root.id).unwrap();
1901 assert_eq!(saved.status, TaskStatus::Held);
1902 assert!(saved.triage_applied(&q.id));
1903 assert_eq!(saved.runs, vec![run.id]);
1904 }
1905
1906 #[test]
1907 fn resource_holds_only_auto_release_a_proven_cache_wait_timeout() {
1908 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
1909 let dir = tempfile::tempdir().unwrap();
1910 let cfg = Config::default(); assert!(cfg.cache_dir().is_none());
1912 let mut state = RunState::new(
1913 dir.path().into(),
1914 "main".into(),
1915 "base".into(),
1916 "verify".into(),
1917 cfg.clone(),
1918 );
1919 let mut task = task("verify", dir.path().into());
1920 task.runs = vec![state.id.clone()];
1921 task.hold_machine(Some("[resource] cache operation stopped".into()));
1922 for cause in [
1923 ResourceBlockCause::Other,
1924 ResourceBlockCause::Unrecorded,
1925 ResourceBlockCause::Cancelled { park: None },
1926 ] {
1927 state.resource_block = Some(ResourceBlock {
1928 node: "gate".into(),
1929 kind: "shared build cache".into(),
1930 detail: "persistent metadata or manifest error".into(),
1931 cause,
1932 at: Timestamp::now(),
1933 });
1934 state.save_under(dir.path()).unwrap();
1935 for _ in 0..3 {
1937 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1938 }
1939 }
1940 let block = state.resource_block.as_mut().unwrap();
1941 block.cause = ResourceBlockCause::Timeout {
1942 waited_secs: 10,
1943 limit_secs: 10,
1944 };
1945 state.save_under(dir.path()).unwrap();
1946 assert_eq!(
1947 machine_cause_resolved_under(&task, &cfg, dir.path()),
1948 Some(true)
1949 );
1950 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
1951 state.save_under(dir.path()).unwrap();
1952 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1953 task.runs = vec!["unreadable-run".into()];
1954 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1955 }
1956
1957 #[cfg(unix)]
1958 #[test]
1959 fn disk_admission_holds_recheck_live_reservations_and_the_floor() {
1960 use crate::retention::OutputLease;
1961 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
1962 let temp = tempfile::tempdir().unwrap();
1963 let home = temp.path().canonicalize().unwrap();
1964 let mut cfg = Config::default();
1965 cfg.graph.worktree_root = Some(home.join("worktrees"));
1966 cfg.disk.min_free_bytes = 100;
1967 let cache = home.join("cache");
1968 cfg.verify.gate = vec![format!(
1969 "CARGO_TARGET_DIR='{}' cargo build",
1970 cache.display()
1971 )];
1972 let mut state = RunState::new(
1973 home.clone(),
1974 "main".into(),
1975 "base".into(),
1976 "verify".into(),
1977 cfg.clone(),
1978 );
1979 state.resource_block = Some(ResourceBlock {
1980 node: "gate".into(),
1981 kind: "disk admission".into(),
1982 detail: "writer headroom unavailable".into(),
1983 cause: ResourceBlockCause::Other,
1984 at: Timestamp::now(),
1985 });
1986 state.save_under(&home).unwrap();
1987 let mut task = task("verify", home.clone());
1988 task.runs = vec![state.id.clone()];
1989 task.hold_machine(Some("[resource] gate: disk admission: blocked".into()));
1990 let gib = 1024 * 1024 * 1024;
1991 let free = std::cell::Cell::new(100 + gib - 1);
1992 let check = || {
1993 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths| {
1994 assert!(
1995 paths.contains(&cache),
1996 "the cache volume must also be checked"
1997 );
1998 assert!(paths.contains(&state.worktree_root()));
1999 OutputLease::admit_with(&home, paths, cfg.disk.min_free_bytes, |_| Ok(free.get()))
2000 .is_ok()
2001 })
2002 };
2003 assert_eq!(check(), Some(false), "the unchanged floor still applies");
2004 free.set(100 + gib);
2005 let writer = OutputLease::admit_with(
2006 &home,
2007 std::slice::from_ref(&home),
2008 cfg.disk.min_free_bytes,
2009 |_| Ok(free.get()),
2010 )
2011 .unwrap();
2012 assert_eq!(
2013 check(),
2014 Some(false),
2015 "df alone cannot resolve a live reservation"
2016 );
2017 drop(writer);
2018 assert_eq!(
2019 check(),
2020 Some(true),
2021 "releasing headroom resolves the machine hold"
2022 );
2023 assert_eq!(
2024 check(),
2025 Some(true),
2026 "the probe must release its temporary reservation"
2027 );
2028
2029 let namespace = home.join("disk-admission");
2031 let saved = home.join("saved-admission");
2032 std::fs::rename(&namespace, &saved).unwrap();
2033 std::os::unix::fs::symlink(&saved, &namespace).unwrap();
2034 assert_eq!(check(), Some(false));
2035 std::fs::remove_file(&namespace).unwrap();
2036 std::fs::rename(saved, namespace).unwrap();
2037 assert_eq!(check(), Some(true));
2038
2039 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Cancelled { park: None };
2040 state.save_under(&home).unwrap();
2041 assert_eq!(
2042 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
2043 "a cancelled operation must remain held"
2044 )),
2045 None
2046 );
2047 state.resource_block.as_mut().unwrap().kind = "incremental cache admission".into();
2048 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Other;
2049 state.save_under(&home).unwrap();
2050 assert_eq!(
2051 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
2052 "unproven incremental caches must remain held"
2053 )),
2054 None
2055 );
2056 task.runs.push("unreadable-run".into());
2057 assert_eq!(
2058 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
2059 "missing evidence must not release a hold"
2060 )),
2061 None
2062 );
2063 }
2064
2065 fn gate_disabled_config(dir: &std::path::Path) -> PathBuf {
2069 let config = dir.join("magi.toml");
2070 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2071 config
2072 }
2073
2074 #[test]
2075 fn resumption_later_keep_held_answer_wins_but_an_older_answer_cannot_undo_release() {
2076 for source in ["cli", "web"] {
2077 for answer_after_release in [true, false] {
2078 let (dir, queue, questions) = store();
2079 let mut t = task("recovery", dir.path().join("repo"));
2080 t.hold_machine(Some("inspect remote first".into()));
2081 queue.put(&mut t).unwrap();
2082 let mut q = file_question(
2083 &questions,
2084 &crate::task_chat::InquiryOwnership::default(),
2085 &t,
2086 Bucket::MachineUnknown,
2087 &JA,
2088 Timestamp::now(),
2089 )
2090 .unwrap();
2091 if !answer_after_release {
2092 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2093 questions.put(&mut q).unwrap();
2094 }
2095 t.release_operator(source, None);
2096 queue.put(&mut t).unwrap();
2097 if answer_after_release {
2098 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2099 questions.put(&mut q).unwrap();
2100 }
2101 apply_answers(&queue, &questions);
2102 let after = queue.get(&t.id).unwrap();
2103 assert!(after.triage_applied(&q.id));
2104 assert_eq!(after.operator_held(), answer_after_release);
2105 assert_eq!(
2106 after.status,
2107 if answer_after_release {
2108 TaskStatus::Held
2109 } else {
2110 TaskStatus::Queued
2111 }
2112 );
2113 assert!(apply_answers(&queue, &questions).is_empty());
2114 }
2115 }
2116 }
2117
2118 #[test]
2119 fn explicit_resumption_applies_legacy_queued_triage_once_and_preserves_later_holds() {
2120 let (dir, queue, questions) = store();
2121 let mut t = task("recover", dir.path().join("repo"));
2122 t.hold_machine(Some("inspect first".into()));
2123 queue.put(&mut t).unwrap();
2124 let mut q = file_question(
2125 &questions,
2126 &crate::task_chat::InquiryOwnership::default(),
2127 &t,
2128 Bucket::MachineUnknown,
2129 &JA,
2130 Timestamp::now(),
2131 )
2132 .unwrap();
2133 assert!(q.actions.is_empty());
2134 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
2135 questions.put(&mut q).unwrap();
2136 t.release();
2138 t.instruction = "Reuse approved PR head".into();
2139 queue.put(&mut t).unwrap();
2140 assert_eq!(apply_answers(&queue, &questions), [t.id.clone()]);
2141 let mut after = queue.get(&t.id).unwrap();
2142 assert_eq!(
2143 after.resumption.as_ref().unwrap().question_id.as_deref(),
2144 Some(q.id.as_str())
2145 );
2146 assert_eq!(
2147 after.resumption.as_ref().unwrap().instruction,
2148 t.instruction
2149 );
2150 assert_eq!(after.answers[0].answer, JA.resume);
2151 assert!(after.triage_applied(&q.id));
2152 assert!(apply_answers(&Queue::at(dir.path().join("queue")), &questions).is_empty());
2153 after.hold_machine(Some("new blocker".into()));
2154 queue.put(&mut after).unwrap();
2155 let mut late = file_question(
2156 &questions,
2157 &crate::task_chat::InquiryOwnership::default(),
2158 &after,
2159 Bucket::MachineUnknown,
2160 &EN,
2161 Timestamp::now(),
2162 )
2163 .unwrap();
2164 after.hold_manual(Some("keep held now".into()));
2165 queue.put(&mut after).unwrap();
2166 late.answer(Answer::Choice(late.choices[0].clone()))
2167 .unwrap();
2168 questions.put(&mut late).unwrap();
2169 apply_answers(&queue, &questions);
2170 let held = queue.get(&t.id).unwrap();
2171 assert!(held.operator_held());
2172 assert_eq!(held.hold_reason.as_deref(), Some("keep held now"));
2173 assert_eq!(held.resumption.as_ref().unwrap().generation, 1);
2174 }
2175
2176 #[test]
2177 fn a_resolved_machine_hold_is_requeued_automatically() {
2178 let (dir, q, questions) = store();
2179 let config = gate_disabled_config(dir.path());
2180 let mut t = task("disk pressure", dir.path().join("repo"));
2181 t.hold_machine(Some(
2182 "not enough free space to start a run: 10 bytes free, 100 required by \
2183 `[disk] min_free_bytes`"
2184 .to_owned(),
2185 ));
2186 q.put(&mut t).unwrap();
2187
2188 let report = run_once(&q, &questions, Some(&config), Timestamp::now());
2189 assert_eq!(report.resumed, [t.id.clone()]);
2190 assert!(report.asked.is_empty());
2191
2192 let back = q.get(&t.id).unwrap();
2193 assert_eq!(back.status, TaskStatus::Queued);
2194 assert!(back.hold_source.is_none());
2195 assert!(questions.list().is_empty(), "nothing needed asking");
2196 }
2197
2198 #[test]
2199 fn a_machine_hold_with_no_recognised_cause_gets_one_question_not_two() {
2200 let (dir, q, questions) = store();
2201 let mut t = task("gate went red", dir.path().join("repo"));
2202 t.hold_machine(Some("gate red".to_owned()));
2203 q.put(&mut t).unwrap();
2204
2205 let first = run_once(&q, &questions, None, Timestamp::now());
2206 assert_eq!(first.asked, [t.id.clone()]);
2207 assert!(first.resumed.is_empty());
2208
2209 let open: Vec<_> = questions
2210 .list()
2211 .into_iter()
2212 .filter(|q| q.status.open())
2213 .collect();
2214 assert_eq!(open.len(), 1);
2215 assert_eq!(open[0].run, t.id);
2216 assert_eq!(open[0].node, NODE);
2217 assert_eq!(open[0].choices.len(), 3);
2218
2219 let second = run_once(&q, &questions, None, Timestamp::now());
2221 assert!(second.asked.is_empty());
2222 assert_eq!(
2223 questions
2224 .list()
2225 .into_iter()
2226 .filter(|q| q.status.open())
2227 .count(),
2228 1
2229 );
2230 }
2231
2232 #[test]
2233 fn a_legacy_hold_with_no_recorded_source_gets_exactly_one_question() {
2234 let (dir, q, questions) = store();
2235 let mut t = task("schema 1 record", dir.path().join("repo"));
2236 t.status = TaskStatus::Held;
2237 assert!(t.hold_source.is_none(), "the case this test is about");
2238 q.put(&mut t).unwrap();
2239
2240 let first = run_once(&q, &questions, None, Timestamp::now());
2241 assert_eq!(first.asked, [t.id.clone()]);
2242
2243 let second = run_once(&q, &questions, None, Timestamp::now());
2244 assert!(
2245 second.asked.is_empty(),
2246 "the same legacy hold must not be asked about twice"
2247 );
2248 assert_eq!(
2249 questions
2250 .list()
2251 .into_iter()
2252 .filter(|q| q.status.open())
2253 .count(),
2254 1
2255 );
2256 }
2257
2258 #[test]
2259 fn dependency_resolution_detaches_legacy_short_refs_without_dropping_other_waits() {
2260 let (dir, queue, questions) = store();
2261 let mut root = task("root", dir.path().join("repo"));
2262 root.id = "20261013-193611-ca5b".into();
2263 root.hold_machine(Some("needs owner decision".into()));
2264 queue.put(&mut root).unwrap();
2265 let mut other = task("other", dir.path().join("repo"));
2266 other.id = "20261013-192213-feb2".into();
2267 queue.put(&mut other).unwrap();
2268 let mut waiting = task("waiting", dir.path().join("repo"));
2269 waiting.block(vec!["ca5b".into(), other.id.clone()], None);
2270 queue.put(&mut waiting).unwrap();
2271 detach_dependants(&queue, &questions, &root.id);
2272 let after = queue.get(&waiting.id).unwrap();
2273 assert_eq!(after.status, TaskStatus::Blocked);
2274 assert_eq!(after.blocked_by, [other.id]);
2275 }
2276
2277 #[test]
2278 fn a_manual_hold_is_never_auto_resumed() {
2279 let (dir, q, questions) = store();
2280 let mut t = task("operator stopped this", dir.path().join("repo"));
2281 t.hold_manual(Some("waiting on a decision".to_owned()));
2282 q.put(&mut t).unwrap();
2283
2284 let report = run_once(&q, &questions, None, Timestamp::now());
2285 assert!(report.resumed.is_empty());
2286 assert!(report.asked.is_empty());
2288
2289 let back = q.get(&t.id).unwrap();
2290 assert_eq!(back.status, TaskStatus::Held);
2291 assert_eq!(back.hold_source, Some(HoldSource::Manual));
2292 assert!(questions.list().is_empty());
2293 }
2294
2295 #[test]
2296 fn a_blocked_task_on_a_deleted_dependency_is_held_and_asked_about_in_one_pass() {
2297 let (dir, q, questions) = store();
2302 let mut still_going = task("still valid", dir.path().join("repo"));
2303 q.put(&mut still_going).unwrap();
2304
2305 let mut t = task("orphaned", dir.path().join("repo"));
2306 t.block(
2307 vec!["20260101-000000-gone".to_owned(), still_going.id.clone()],
2308 Some("waits on both".to_owned()),
2309 );
2310 q.put(&mut t).unwrap();
2311
2312 let report = run_once(&q, &questions, None, Timestamp::now());
2313 assert_eq!(report.quarantined, [t.id.clone()]);
2314 assert_eq!(
2315 report.asked,
2316 [t.id.clone()],
2317 "the fresh machine hold must earn a question in the same pass"
2318 );
2319
2320 let after = q.get(&t.id).unwrap();
2321 assert_eq!(after.status, TaskStatus::Held);
2322 assert_eq!(after.hold_source, Some(HoldSource::Machine));
2323 assert!(after.blocked_by.is_empty());
2324
2325 assert!(!is_disk_hold(&after));
2328
2329 let open: Vec<_> = questions
2330 .list()
2331 .into_iter()
2332 .filter(|q| q.status.open())
2333 .collect();
2334 assert_eq!(open.len(), 1);
2335 assert_eq!(open[0].run, t.id);
2336
2337 let second = run_once(&q, &questions, None, Timestamp::now());
2339 assert!(second.quarantined.is_empty());
2340 assert!(second.asked.is_empty());
2341 }
2342
2343 fn stuck_chain(q: &Queue, dir: &std::path::Path) -> (Task, Task, Task) {
2346 let mut root = task("root", dir.join("repo"));
2347 root.hold_manual(Some("waiting".to_owned()));
2348 q.put(&mut root).unwrap();
2349 let mut mid = task("mid", dir.join("repo"));
2350 mid.block(vec![root.id.clone()], None);
2351 q.put(&mut mid).unwrap();
2352 let mut leaf = task("leaf", dir.join("repo"));
2353 leaf.block(vec![mid.id.clone()], None);
2354 q.put(&mut leaf).unwrap();
2355 (root, mid, leaf)
2356 }
2357
2358 fn deps_questions(questions: &Questions) -> Vec<Question> {
2359 questions
2360 .list()
2361 .into_iter()
2362 .filter(|q| q.node == DEPS_NODE)
2363 .collect()
2364 }
2365
2366 fn answer_deps(questions: &Questions, choice: usize) {
2367 let mut asked = deps_questions(questions).remove(0);
2368 let c = asked.choices[choice].clone();
2369 asked.answer(Answer::Choice(c)).unwrap();
2370 questions.put(&mut asked).unwrap();
2371 }
2372
2373 #[test]
2374 fn a_stuck_chain_earns_one_question_naming_every_dependant_and_is_not_reasked() {
2375 let (dir, q, questions) = store();
2376 let (root, mid, leaf) = stuck_chain(&q, dir.path());
2377
2378 let first = run_once(&q, &questions, None, Timestamp::now());
2379 assert_eq!(first.asked, std::slice::from_ref(&root.id));
2380 let asked = deps_questions(&questions);
2381 assert_eq!(asked.len(), 1, "one question per root, not per dependant");
2382 assert_eq!(asked[0].run, root.id);
2383 assert_eq!(asked[0].choices.len(), 3);
2384 assert!(asked[0].detail.contains(mid.short()), "{}", asked[0].detail);
2385 assert!(
2386 asked[0].detail.contains(leaf.short()),
2387 "{}",
2388 asked[0].detail
2389 );
2390
2391 let second = run_once(&q, &questions, None, Timestamp::now());
2392 assert!(second.asked.is_empty());
2393 assert_eq!(deps_questions(&questions).len(), 1);
2394 assert!(pending_for(&questions, &q.get(&root.id).unwrap()));
2395 }
2396
2397 #[test]
2398 fn a_stuck_root_with_a_machine_hold_gets_only_the_dependency_question() {
2399 let (dir, q, questions) = store();
2400 let mut root = task("root", dir.path().join("repo"));
2401 root.hold_machine(Some("gate red".to_owned()));
2402 q.put(&mut root).unwrap();
2403 let mut dep = task("dep", dir.path().join("repo"));
2404 dep.block(vec![root.id.clone()], None);
2405 q.put(&mut dep).unwrap();
2406
2407 run_once(&q, &questions, None, Timestamp::now());
2408 run_once(&q, &questions, None, Timestamp::now());
2409 assert_eq!(deps_questions(&questions).len(), 1);
2410 assert_eq!(
2411 questions.list().len(),
2412 1,
2413 "no second question about the hold"
2414 );
2415 }
2416
2417 #[test]
2418 fn a_dependency_that_can_still_run_asks_nothing() {
2419 let (dir, q, questions) = store();
2420 let mut live = task("live", dir.path().join("repo"));
2421 q.put(&mut live).unwrap();
2422 let mut dep = task("dep", dir.path().join("repo"));
2423 dep.block(vec![live.id.clone()], None);
2424 q.put(&mut dep).unwrap();
2425 let report = run_once(&q, &questions, None, Timestamp::now());
2426 assert!(report.asked.is_empty());
2427 assert!(questions.list().is_empty());
2428 }
2429
2430 #[test]
2431 fn answering_release_requeues_the_root_and_is_not_reasked() {
2432 let (dir, q, questions) = store();
2433 let (root, _mid, _leaf) = stuck_chain(&q, dir.path());
2434 run_once(&q, &questions, None, Timestamp::now());
2435 answer_deps(&questions, 0);
2436
2437 let report = run_once(&q, &questions, None, Timestamp::now());
2438 assert_eq!(report.answered, std::slice::from_ref(&root.id));
2439 assert!(report.asked.is_empty(), "applying must not re-ask");
2440 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Queued);
2441 let again = run_once(&q, &questions, None, Timestamp::now());
2442 assert!(again.asked.is_empty() && again.answered.is_empty());
2443 assert_eq!(deps_questions(&questions).len(), 1);
2444 }
2445
2446 #[test]
2447 fn answering_detach_frees_the_direct_dependant_and_keeps_the_root() {
2448 let (dir, q, questions) = store();
2449 let (root, mid, leaf) = stuck_chain(&q, dir.path());
2450 run_once(&q, &questions, None, Timestamp::now());
2451 answer_deps(&questions, 2);
2452
2453 let report = run_once(&q, &questions, None, Timestamp::now());
2454 assert!(report.asked.is_empty());
2455 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
2456 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
2457 let leaf = q.get(&leaf.id).unwrap();
2458 assert_eq!(leaf.status, TaskStatus::Blocked, "still waits on mid");
2459 assert_eq!(leaf.blocked_by, std::slice::from_ref(&mid.id));
2460 }
2461
2462 #[test]
2463 fn a_dependant_that_can_progress_through_another_blocker_is_not_stuck() {
2464 let (dir, q, questions) = store();
2465 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
2466 let mut live = task("live", dir.path().join("repo"));
2467 q.put(&mut live).unwrap();
2468 let mut both = q.get(&mid.id).unwrap();
2469 both.block(vec![root.id.clone(), live.id.clone()], None);
2470 q.put(&mut both).unwrap();
2471 assert!(
2473 run_once(&q, &questions, None, Timestamp::now())
2474 .asked
2475 .is_empty()
2476 );
2477 }
2478
2479 #[test]
2480 fn answering_discard_detaches_then_removes_the_root() {
2481 let (dir, q, questions) = store();
2482 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
2483 run_once(&q, &questions, None, Timestamp::now());
2484 answer_deps(&questions, 1);
2485
2486 let report = run_once(&q, &questions, None, Timestamp::now());
2487 assert!(q.get(&root.id).is_err());
2488 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
2489 assert!(
2490 report.asked.is_empty(),
2491 "no per-dependant quarantine question"
2492 );
2493 assert!(report.quarantined.is_empty());
2494 }
2495
2496 #[test]
2497 fn an_unmatched_deps_answer_changes_nothing_and_is_not_reasked() {
2498 let (dir, q, questions) = store();
2499 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
2500 run_once(&q, &questions, None, Timestamp::now());
2501 let mut asked = deps_questions(&questions).remove(0);
2502 asked.choices.clear();
2503 asked.answer(Answer::Text("dunno".to_owned())).unwrap();
2504 questions.put(&mut asked).unwrap();
2505
2506 for _ in 0..2 {
2507 let report = run_once(&q, &questions, None, Timestamp::now());
2508 assert!(report.asked.is_empty() && report.answered.is_empty());
2509 }
2510 assert_eq!(deps_questions(&questions).len(), 1);
2511 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
2512 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Blocked);
2513 }
2514
2515 #[test]
2516 fn a_dependency_cycle_terminates_and_is_asked_about_once() {
2517 let (dir, q, questions) = store();
2518 let mut a = task("a", dir.path().join("repo"));
2519 let mut b = task("b", dir.path().join("repo"));
2520 a.block(vec![b.id.clone()], None);
2521 b.block(vec![a.id.clone()], None);
2522 q.put(&mut a).unwrap();
2523 q.put(&mut b).unwrap();
2524
2525 let first = run_once(&q, &questions, None, Timestamp::now());
2526 assert_eq!(first.asked.len(), 1);
2527 let second = run_once(&q, &questions, None, Timestamp::now());
2528 assert!(second.asked.is_empty());
2529 assert_eq!(deps_questions(&questions).len(), 1);
2530 }
2531
2532 #[test]
2533 fn a_missing_dependency_is_still_quarantined_not_asked_about_as_stuck() {
2534 let (dir, q, questions) = store();
2535 let mut t = task("orphan", dir.path().join("repo"));
2536 t.block(vec!["20260101-000000-gone".to_owned()], None);
2537 q.put(&mut t).unwrap();
2538 let report = run_once(&q, &questions, None, Timestamp::now());
2539 assert_eq!(report.quarantined, [t.id.clone()]);
2540 assert!(deps_questions(&questions).is_empty());
2541 }
2542
2543 #[test]
2544 fn a_stale_manual_hold_earns_a_two_choice_question() {
2545 let (dir, q, questions) = store();
2546 let mut t = task("been sitting a while", dir.path().join("repo"));
2547 t.hold_manual(Some("waiting on a decision".to_owned()));
2548 q.put(&mut t).unwrap();
2549 let mut back = q.get(&t.id).unwrap();
2551 back.updated_at = Timestamp::now() - SignedDuration::new(8 * 24 * 60 * 60, 0);
2552 std::fs::write(
2553 q.path_of(&back.id),
2554 serde_json::to_string_pretty(&back).unwrap(),
2555 )
2556 .unwrap();
2557
2558 let report = run_once(&q, &questions, None, Timestamp::now());
2559 assert_eq!(report.asked, [t.id.clone()]);
2560 let open: Vec<_> = questions
2561 .list()
2562 .into_iter()
2563 .filter(|q| q.status.open())
2564 .collect();
2565 assert_eq!(open.len(), 1);
2566 assert_eq!(open[0].choices.len(), 2);
2567 }
2568
2569 #[test]
2570 fn answering_resume_releases_the_task() {
2571 let (dir, q, questions) = store();
2572 let mut t = task("gate went red", dir.path().join("repo"));
2573 t.hold_machine(Some("gate red".to_owned()));
2574 q.put(&mut t).unwrap();
2575 run_once(&q, &questions, None, Timestamp::now());
2576
2577 let mut asked = questions
2578 .list()
2579 .into_iter()
2580 .find(|q| q.run == t.id)
2581 .unwrap();
2582 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2583 questions.put(&mut asked).unwrap();
2584
2585 let report = run_once(&q, &questions, None, Timestamp::now());
2586 assert_eq!(report.answered, [t.id.clone()]);
2587 let back = q.get(&t.id).unwrap();
2588 assert_eq!(back.status, TaskStatus::Queued);
2589 assert!(back.hold_source.is_none());
2590 }
2591
2592 fn resumed_then_failed(q: &Queue, questions: &Questions, dir: &std::path::Path) -> Task {
2594 let mut t = task("gate went red", dir.join("repo"));
2595 t.hold_machine(Some("gate red".to_owned()));
2596 q.put(&mut t).unwrap();
2597 run_once(q, questions, None, Timestamp::now());
2598 let mut asked = questions
2599 .list()
2600 .into_iter()
2601 .find(|q| q.run == t.id)
2602 .unwrap();
2603 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2604 questions.put(&mut asked).unwrap();
2605
2606 let report = run_once(q, questions, None, Timestamp::now());
2607 assert_eq!(report.answered, [t.id.clone()]);
2608 let mut back = q.get(&t.id).unwrap();
2609 assert_eq!(back.status, TaskStatus::Queued);
2610 assert_eq!(back.attempts, 0);
2611
2612 back.start("run-1".to_owned());
2613 back.fail("rebase conflict", 1);
2614 assert_eq!(back.status, TaskStatus::Held);
2615 q.put(&mut back).unwrap();
2616 back
2617 }
2618
2619 #[test]
2620 fn a_resume_answer_is_applied_once_and_a_new_machine_hold_is_asked_about() {
2621 let (dir, q, questions) = store();
2622 let t = resumed_then_failed(&q, &questions, dir.path());
2623
2624 let report = run_once(&q, &questions, None, Timestamp::now());
2625 assert!(report.answered.is_empty(), "the old answer must not replay");
2626 assert_eq!(report.asked, std::slice::from_ref(&t.id));
2627 let back = q.get(&t.id).unwrap();
2628 assert_eq!(back.status, TaskStatus::Held);
2629 assert_eq!(
2630 questions
2631 .list()
2632 .into_iter()
2633 .filter(|q| q.status.open())
2634 .count(),
2635 1
2636 );
2637 }
2638
2639 #[test]
2640 fn a_manual_hold_placed_after_a_resume_is_not_undone_by_the_old_answer() {
2641 let (dir, q, questions) = store();
2642 let mut t = resumed_then_failed(&q, &questions, dir.path());
2643 t.hold_manual(Some("operator stopped this".to_owned()));
2644 q.put(&mut t).unwrap();
2645 let before = questions.list().len();
2646
2647 let report = run_once(&q, &questions, None, Timestamp::now());
2648 assert!(report.answered.is_empty());
2649 assert!(report.asked.is_empty());
2650 let back = q.get(&t.id).unwrap();
2651 assert_eq!(back.status, TaskStatus::Held);
2652 assert_eq!(back.hold_source, Some(HoldSource::Manual));
2653 assert_eq!(questions.list().len(), before);
2654 }
2655
2656 #[test]
2657 fn answering_not_yet_keeps_it_held_as_a_manual_hold_and_does_not_reapply() {
2658 let (dir, q, questions) = store();
2659 let mut t = task("gate went red", dir.path().join("repo"));
2660 t.hold_machine(Some("gate red".to_owned()));
2661 q.put(&mut t).unwrap();
2662 run_once(&q, &questions, None, Timestamp::now());
2663
2664 let mut asked = questions
2665 .list()
2666 .into_iter()
2667 .find(|q| q.run == t.id)
2668 .unwrap();
2669 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
2670 questions.put(&mut asked).unwrap();
2671
2672 let report = run_once(&q, &questions, None, Timestamp::now());
2673 assert_eq!(report.answered, [t.id.clone()]);
2674 let back = q.get(&t.id).unwrap();
2675 assert_eq!(back.status, TaskStatus::Held);
2676 assert_eq!(back.hold_source, Some(HoldSource::Manual));
2677 assert!(
2678 back.hold_reason
2679 .as_deref()
2680 .is_some_and(|r| r.contains("gate red")),
2681 "the original cause must survive a \"not yet\" answer, not just the \
2682 triage marker: {:?}",
2683 back.hold_reason
2684 );
2685
2686 let third = run_once(&q, &questions, None, Timestamp::now());
2690 assert!(third.answered.is_empty());
2691 assert!(third.asked.is_empty());
2692 }
2693
2694 #[test]
2695 fn answering_discard_removes_the_task_entirely() {
2696 let (dir, q, questions) = store();
2697 let mut t = task("gate went red", dir.path().join("repo"));
2698 t.hold_machine(Some("gate red".to_owned()));
2699 q.put(&mut t).unwrap();
2700 run_once(&q, &questions, None, Timestamp::now());
2701
2702 let mut asked = questions
2703 .list()
2704 .into_iter()
2705 .find(|q| q.run == t.id)
2706 .unwrap();
2707 asked.answer(Answer::Choice(EN.discard.to_owned())).unwrap();
2708 questions.put(&mut asked).unwrap();
2709
2710 let report = run_once(&q, &questions, None, Timestamp::now());
2711 assert_eq!(report.answered, [t.id.clone()]);
2712 assert!(
2713 q.get(&t.id).is_err(),
2714 "\"discard it\" (捨ててよい) must actually discard the task, not \
2715 just leave it sitting held forever"
2716 );
2717 }
2718
2719 #[test]
2720 fn an_answer_is_read_by_its_position_in_choices_not_by_the_callers_current_language() {
2721 let (dir, q, questions) = store();
2723 let ja_config = dir.path().join("ja.toml");
2724 std::fs::write(&ja_config, "[graph]\nlanguage = \"ja\"\n").unwrap();
2725 let mut t = task("gate went red", dir.path().join("repo"));
2726 t.hold_machine(Some("gate red".to_owned()));
2727 q.put(&mut t).unwrap();
2728 run_once(&q, &questions, Some(&ja_config), Timestamp::now());
2729
2730 let mut asked = questions
2731 .list()
2732 .into_iter()
2733 .find(|q| q.run == t.id)
2734 .unwrap();
2735 assert_eq!(asked.choices[0], JA.resume, "filed in Japanese");
2736 asked.answer(Answer::Choice(JA.resume.to_owned())).unwrap();
2737 questions.put(&mut asked).unwrap();
2738
2739 let en_config = dir.path().join("en.toml");
2744 std::fs::write(&en_config, "[graph]\nlanguage = \"en\"\n").unwrap();
2745 let report = run_once(&q, &questions, Some(&en_config), Timestamp::now());
2746 assert_eq!(report.answered, [t.id.clone()]);
2747 let back = q.get(&t.id).unwrap();
2748 assert_eq!(
2749 back.status,
2750 TaskStatus::Queued,
2751 "a resume answer must resume the task regardless of which \
2752 language it is read back in"
2753 );
2754 }
2755
2756 #[test]
2757 fn a_question_falls_back_to_last_error_when_hold_reason_was_never_set() {
2758 let (dir, q, questions) = store();
2763 let mut t = task("kept failing the gate", dir.path().join("repo"));
2764 t.start("run-1".to_owned());
2765 t.fail("gate red three times running", 1);
2766 assert_eq!(t.status, TaskStatus::Held);
2767 t.hold_reason = None; q.put(&mut t).unwrap();
2769
2770 run_once(&q, &questions, None, Timestamp::now());
2771 let asked = questions
2772 .list()
2773 .into_iter()
2774 .find(|q| q.run == t.id)
2775 .unwrap();
2776 assert!(
2777 asked.detail.contains("gate red three times running"),
2778 "the question must surface `last_error` when there is no \
2779 `hold_reason` to show instead: {}",
2780 asked.detail
2781 );
2782 }
2783
2784 #[test]
2785 fn open_question_for_and_open_task_ids_reflect_only_what_is_still_waiting() {
2786 let (dir, q, questions) = store();
2787 let mut t = task("gate went red", dir.path().join("repo"));
2788 t.hold_machine(Some("gate red".to_owned()));
2789 q.put(&mut t).unwrap();
2790
2791 assert!(open_question_for(&questions, &t.id).is_none());
2792 assert!(!open_task_ids(&questions).contains(&t.id));
2793
2794 run_once(&q, &questions, None, Timestamp::now());
2795 assert!(open_question_for(&questions, &t.id).is_some());
2796 assert!(open_task_ids(&questions).contains(&t.id));
2797
2798 let mut asked = questions
2799 .list()
2800 .into_iter()
2801 .find(|q| q.run == t.id)
2802 .unwrap();
2803 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2804 questions.put(&mut asked).unwrap();
2805
2806 assert!(
2807 open_question_for(&questions, &t.id).is_none(),
2808 "an answered question is no longer open"
2809 );
2810 assert!(!open_task_ids(&questions).contains(&t.id));
2811 }
2812
2813 #[test]
2814 fn pending_for_stays_true_between_an_answer_and_the_next_run_once_pass() {
2815 let (dir, q, questions) = store();
2821 let mut t = task("gate went red", dir.path().join("repo"));
2822 t.hold_machine(Some("gate red".to_owned()));
2823 q.put(&mut t).unwrap();
2824
2825 assert!(!pending_for(&questions, &t));
2826
2827 run_once(&q, &questions, None, Timestamp::now());
2828 let held = q.get(&t.id).unwrap();
2829 assert!(pending_for(&questions, &held), "still waiting on an answer");
2830
2831 let mut asked = questions
2832 .list()
2833 .into_iter()
2834 .find(|q| q.run == t.id)
2835 .unwrap();
2836 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
2837 questions.put(&mut asked).unwrap();
2838 assert!(!asked.status.open());
2839
2840 let still_held = q.get(&t.id).unwrap();
2842 assert!(
2843 pending_for(&questions, &still_held),
2844 "answered but not yet applied is still pending"
2845 );
2846
2847 run_once(&q, &questions, None, Timestamp::now());
2848 let after = q.get(&t.id).unwrap();
2849 assert!(
2850 !pending_for(&questions, &after),
2851 "the answer is applied now, nothing left pending"
2852 );
2853 }
2854}