1use std::path::{Path, PathBuf};
84use std::time::Duration;
85
86use jiff::Timestamp;
87
88use crate::ask::{Question, QuestionStatus, Questions};
89use crate::config::Config;
90use crate::queue::{HoldSource, OperatorResume, Queue, Task, TaskStatus};
91
92pub const NODE: &str = "triage";
95
96pub const DEPS_NODE: &str = "triage-deps";
100
101const SEAT: &str = "triage";
106
107const MANUAL_STALE_AFTER: Duration = Duration::from_secs(7 * 24 * 60 * 60);
119
120struct Wording {
124 lang: &'static str,
125 resume: &'static str,
126 wait: &'static str,
127 discard: &'static str,
128 resume_now: &'static str,
129 keep_held: &'static str,
130}
131
132const EN: Wording = Wording {
133 lang: "en",
134 resume: "resume it",
135 wait: "not yet",
136 discard: "discard it",
137 resume_now: "resume it",
138 keep_held: "keep it held",
139};
140
141const JA: Wording = Wording {
142 lang: "ja",
143 resume: "再開してよい",
144 wait: "まだ待って",
145 discard: "捨ててよい",
146 resume_now: "再開する",
147 keep_held: "まだ止めておく",
148};
149
150fn wording(language: &str) -> &'static Wording {
153 if crate::lang::is_japanese(language) {
154 &JA
155 } else {
156 &EN
157 }
158}
159
160impl Wording {
161 fn choices3(&self) -> Vec<String> {
162 vec![
163 self.resume.to_owned(),
164 self.wait.to_owned(),
165 self.discard.to_owned(),
166 ]
167 }
168
169 fn choices2(&self) -> Vec<String> {
170 vec![self.resume_now.to_owned(), self.keep_held.to_owned()]
171 }
172
173 fn source_label(&self, source: Option<HoldSource>) -> &'static str {
174 match (self.lang, source) {
175 ("ja", Some(HoldSource::Machine)) => "machine(機械による自動保留)",
176 ("ja", Some(HoldSource::Manual)) => "manual(操作者による手動保留)",
177 ("ja", None) => "unknown(schema 3 未満の旧レコード、または理由未記録)",
178 (_, Some(HoldSource::Machine)) => "machine (automatic recovery hold)",
179 (_, Some(HoldSource::Manual)) => "manual (an operator held this)",
180 (_, None) => "unknown (pre-schema-3 record, or never recorded)",
181 }
182 }
183
184 fn detail(&self, task: &Task, why: &str) -> String {
193 let none = if self.lang == "ja" {
194 "(記録なし)"
195 } else {
196 "(none recorded)"
197 };
198 let reason = task
199 .hold_reason
200 .as_deref()
201 .or(task.last_error.as_deref())
202 .unwrap_or(none);
203 format!(
204 "task: {} ({})\ntitle: {}\nhold source: {}\nhold reason: {reason}\n\n{why}",
205 task.id,
206 task.short(),
207 task.title,
208 self.source_label(task.hold_source),
209 )
210 }
211
212 fn summary_machine_unknown(&self, task: &Task) -> String {
213 if self.lang == "ja" {
214 format!("保留タスク {} の再開可否を判断してください", task.short())
215 } else {
216 format!("decide whether to resume held task {}", task.short())
217 }
218 }
219
220 fn why_machine(&self) -> &'static str {
221 if self.lang == "ja" {
222 "機械的な保留(machine hold)ですが、原因がすでに解消しているかを自動では判断できませんでした。"
223 } else {
224 "This is a machine hold, but whether its cause has resolved could not be \
225 checked automatically."
226 }
227 }
228
229 fn summary_conductor_override(&self, task: &Task) -> String {
230 if self.lang == "ja" {
231 format!(
232 "再開と回答済みのタスク {} を conductor が再び保留しました",
233 task.short()
234 )
235 } else {
236 format!(
237 "task {} was resumed at your word, but the conductor held it again",
238 task.short()
239 )
240 }
241 }
242
243 fn why_conductor_override(&self, o: &OperatorResume) -> String {
244 let reason = o.conductor_rehold.as_deref().unwrap_or_default();
245 if self.lang == "ja" {
246 format!(
247 "{} に再開と回答済みですが、conductor が再び hold しました。conductor の理由: \
248 {reason}\n\n強制再キューを選ぶと、以後 conductor はこのタスクを hold できません。",
249 o.at
250 )
251 } else {
252 format!(
253 "You answered \"resume\" at {}, but the conductor held the task again. \
254 Its reason: {reason}\n\nForcing a requeue stops the conductor from \
255 holding this task again.",
256 o.at
257 )
258 }
259 }
260
261 fn choices_conductor_override(&self) -> Vec<String> {
263 if self.lang == "ja" {
264 vec![
265 "強制再キュー(conductor は再 hold 不可)".to_owned(),
266 "手動 hold のまま".to_owned(),
267 "捨ててよい".to_owned(),
268 ]
269 } else {
270 vec![
271 "force requeue (conductor must not hold again)".to_owned(),
272 "keep held (manual)".to_owned(),
273 "discard".to_owned(),
274 ]
275 }
276 }
277
278 fn summary_legacy(&self, task: &Task) -> String {
279 if self.lang == "ja" {
280 format!(
281 "hold_source が不明な保留タスク {} を確認してください",
282 task.short()
283 )
284 } else {
285 format!(
286 "held task {} has no recorded hold source - please take a look",
287 task.short()
288 )
289 }
290 }
291
292 fn why_legacy(&self) -> &'static str {
293 if self.lang == "ja" {
294 "hold_source が記録されていません。schema 3 より前のレコードか、理由が記録されなかった \
295 holdです。人が意図して止めたのか、クラッシュや強制再起動で宙に浮いただけなのか、\
296 このデータからは区別できません。"
297 } else {
298 "No hold_source was recorded - either a pre-schema-3 record, or a hold whose \
299 reason was never written down. Whether this was a deliberate hold or the \
300 leftover of a crash cannot be told from the data alone."
301 }
302 }
303
304 fn summary_manual_stale(&self, task: &Task, days: i64) -> String {
305 if self.lang == "ja" {
306 format!(
307 "{days}日間 保留されたままの手動保留タスク {} を確認してください",
308 task.short()
309 )
310 } else {
311 format!(
312 "held task {} has been on a manual hold for {days} day(s)",
313 task.short()
314 )
315 }
316 }
317
318 fn why_manual(&self) -> &'static str {
319 if self.lang == "ja" {
320 "操作者が明示的に止めた保留ですが、長期間そのままになっています。まだ止めておくか、\
321 再開するか教えてください。"
322 } else {
323 "An operator held this on purpose, but it has sat untouched for a while. Say \
324 whether to keep holding it or resume it."
325 }
326 }
327}
328
329#[derive(Debug, Clone, Copy, PartialEq, Eq)]
331enum Bucket {
332 MachineUnknown,
334 ConductorOverride,
337 Legacy,
339 ManualStale,
341}
342
343#[derive(Debug, Clone, Default)]
345pub struct Report {
346 pub resumed: Vec<String>,
349 pub asked: Vec<String>,
351 pub answered: Vec<String>,
353 pub quarantined: Vec<String>,
358}
359
360impl Report {
361 pub fn is_empty(&self) -> bool {
364 self.resumed.is_empty()
365 && self.asked.is_empty()
366 && self.answered.is_empty()
367 && self.quarantined.is_empty()
368 }
369}
370
371fn repo_for(task: &Task) -> PathBuf {
375 if task.repo.as_os_str().is_empty() {
376 PathBuf::from(".")
377 } else {
378 task.repo.clone()
379 }
380}
381
382fn is_disk_hold(task: &Task) -> bool {
393 task.hold_reason
394 .as_deref()
395 .is_some_and(crate::disk::is_gate_reason)
396}
397
398fn machine_cause_resolved(task: &Task, cfg: &Config) -> Option<bool> {
403 machine_cause_resolved_under(task, cfg, &crate::run::home())
404}
405
406fn machine_cause_resolved_under(task: &Task, cfg: &Config, home: &Path) -> Option<bool> {
407 machine_cause_resolved_with_admission(task, cfg, home, |paths| {
408 crate::retention::OutputLease::admit(home, paths, cfg.disk.min_free_bytes).is_ok()
409 })
410}
411
412fn machine_cause_resolved_with_admission(
413 task: &Task,
414 cfg: &Config,
415 home: &Path,
416 admission_available: impl Fn(&[PathBuf]) -> bool,
417) -> Option<bool> {
418 if task
419 .hold_reason
420 .as_deref()
421 .is_some_and(|r| r.starts_with(crate::cache::RESOURCE_HOLD))
422 {
423 let id = task.execution_runs().last()?;
425 let state = crate::run::RunState::load_under(id, home).ok()?;
426 let block = state.resource_block.as_ref()?;
427 if state.id != *id {
428 return None;
429 }
430 if block.kind == "disk admission"
431 && matches!(block.cause, crate::run::ResourceBlockCause::Other)
432 {
433 let mut paths = vec![repo_for(task), state.repo.clone(), state.worktree_root()];
437 paths.extend(state.candidates.iter().map(|c| c.worktree.clone()));
438 paths.extend(state.config.cache_dir());
439 paths.extend(cfg.cache_dir());
440 return Some(admission_available(&paths));
441 }
442 if block.kind != "shared build cache"
445 || !matches!(block.cause, crate::run::ResourceBlockCause::Timeout { .. })
446 {
447 return None;
448 }
449 return Some(
450 cfg.cache_dir()
451 .is_none_or(|dir| !crate::cache::in_use(home, &dir)),
452 );
453 }
454 if !is_disk_hold(task) {
455 return None;
456 }
457 let min = cfg.disk.min_free_bytes;
458 if min == 0 {
459 return Some(true);
462 }
463 Some(crate::daemon::disk_gate(&repo_for(task), cfg).is_none())
464}
465
466fn manual_is_stale(task: &Task, now: Timestamp) -> bool {
470 now.as_second() - task.updated_at.as_second() > MANUAL_STALE_AFTER.as_secs() as i64
471}
472
473fn marker_for(q: &Question) -> String {
476 format!("[triage:{}]", q.short())
477}
478
479fn already_applied(task: &Task, q: &Question) -> bool {
482 if task.triage_applied(&q.id) {
483 return true;
484 }
485 let marker = marker_for(q);
488 task.hold_reason
489 .as_deref()
490 .is_some_and(|r| r.contains(marker.as_str()))
491}
492
493fn latest_triage_question(questions: &Questions, task_id: &str) -> Option<Question> {
498 latest_question(questions, NODE, task_id)
499}
500
501fn latest_question(questions: &Questions, node: &str, task_id: &str) -> Option<Question> {
503 questions
504 .list()
505 .into_iter()
506 .filter(|q| q.node == node && q.run == task_id)
507 .max_by(|a, b| a.asked_at.cmp(&b.asked_at).then_with(|| a.id.cmp(&b.id)))
510}
511
512#[derive(Debug, Clone, Copy, PartialEq, Eq)]
515enum AnswerAction {
516 Resume,
520 Discard,
523 KeepHeld,
528}
529
530fn interpret_answer(q: &Question) -> AnswerAction {
543 let resolution = q.resolution().unwrap_or_default();
544 match q.choices.iter().position(|c| *c == resolution) {
545 Some(0) => AnswerAction::Resume,
546 Some(2) => AnswerAction::Discard,
547 _ => AnswerAction::KeepHeld,
548 }
549}
550
551pub(crate) fn contextual_keep_held(q: &Question, label: &str) -> bool {
554 q.node == NODE
555 && q.seat == SEAT
556 && q.actions.is_empty()
557 && q.choices.get(1).is_some_and(|c| c == label)
558}
559
560fn keep_held_note(task: &Task, q: &Question, resolution: &str) -> String {
565 let marker = format!("{} operator: {resolution}", marker_for(q));
566 match task.hold_reason.as_deref() {
567 Some(existing) if !existing.is_empty() => format!("{existing}\n{marker}"),
568 _ => marker,
569 }
570}
571
572fn file_question(
576 questions: &Questions,
577 task: &Task,
578 bucket: Bucket,
579 w: &Wording,
580 now: Timestamp,
581) -> Option<Question> {
582 let (summary, why, choices) = match bucket {
583 Bucket::MachineUnknown => (
584 w.summary_machine_unknown(task),
585 w.why_machine().to_owned(),
586 w.choices3(),
587 ),
588 Bucket::ConductorOverride => (
589 w.summary_conductor_override(task),
590 task.resume_override
591 .as_ref()
592 .map(|o| w.why_conductor_override(o))
593 .unwrap_or_default(),
594 w.choices_conductor_override(),
595 ),
596 Bucket::Legacy => (
597 w.summary_legacy(task),
598 w.why_legacy().to_owned(),
599 w.choices3(),
600 ),
601 Bucket::ManualStale => {
602 let days = (now.as_second() - task.updated_at.as_second()) / (24 * 60 * 60);
603 (
604 w.summary_manual_stale(task, days),
605 w.why_manual().to_owned(),
606 w.choices2(),
607 )
608 }
609 };
610 let mut q = Question::new(
611 task.id.clone(),
612 NODE.to_owned(),
613 SEAT.to_owned(),
614 summary,
615 w.detail(task, &why),
616 choices,
617 );
618 q.triage_generation = Some(task.resumption.as_ref().map_or(0, |r| r.generation));
619 questions.put_executable(&mut q).ok()?;
620 Some(q)
621}
622
623pub(crate) fn deputy_brief(q: &Question, queue: &Queue) -> String {
628 let task = queue.get(&q.run).ok();
629 let mut s = match &task {
630 Some(t) => format!(
631 "Task {} (`magi task show {}`), \"{}\", is {}. Hold source: {}. Reason: {}.",
632 t.id,
633 t.id,
634 t.title,
635 t.status.as_str(),
636 match t.hold_source {
637 Some(HoldSource::Machine) => "machine (an automatic hold)",
638 Some(HoldSource::Manual) => "manual (an operator held it)",
639 None => "unknown (an old record, treated as the operator's)",
640 },
641 t.hold_reason
642 .as_deref()
643 .or(t.last_error.as_deref())
644 .unwrap_or("(none recorded)")
645 ),
646 None => format!("Task {} no longer exists (its record is gone).", q.run),
647 };
648 s.push_str("\n\nWhat each option does, by position:");
649 let n = q.choices.len();
650 for (i, c) in q.choices.iter().enumerate() {
651 let effect = if q.node == DEPS_NODE {
652 match i {
653 0 => "the dependency is released so the loop runs it",
654 1 => {
655 "the dependants are detached and the dependency task is DELETED; this cannot be undone"
656 }
657 2 => "the dependants are detached; the dependency stays as it is",
658 _ => "nothing is applied",
659 }
660 } else {
661 match i {
662 0 => "the task is released and requeued",
663 2 if n > 2 => "the task is DELETED; this cannot be undone",
664 _ => "the task stays held, with the answer noted",
665 }
666 };
667 s.push_str(&format!("\n- `{c}`: {effect}"));
668 }
669 s.push_str(
670 "\n\nThe daemon applies the answer on its next pass; you apply nothing. When \
671 the owner's words pick an option - including releasing a task they held \
672 themselves, or deleting it - settle it.",
673 );
674 s
675}
676
677fn quarantine_orphaned_blocked(queue: &Queue, questions: &Questions) -> Vec<String> {
695 let mut quarantined = Vec::new();
696 for listed in queue.list() {
697 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
698 continue;
699 }
700 let Ok(_claim) = queue.claim(&listed.id) else {
701 continue;
702 };
703 let Ok(mut task) = queue.get(&listed.id) else {
704 continue;
705 };
706 if task.status != TaskStatus::Blocked {
707 continue;
708 }
709 let deleted = queue.apply_deleted_blockers(&mut task, questions);
710 if !deleted.is_empty() && queue.put(&mut task).is_ok() {
711 for id in &deleted {
712 queue.note_dependency_deleted(&task, id);
713 }
714 }
715 if task.status != TaskStatus::Blocked {
716 continue;
717 }
718 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
719 if missing.is_empty() {
720 continue;
721 }
722 let language = crate::lang::of_repo(&repo_for(&task));
723 let mut cause = missing.clone();
724 cause.sort();
725 cause.dedup();
726 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
727 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
728 &task.blocked_by,
729 &missing,
730 &language,
731 )));
732 if queue.put(&mut task).is_ok() {
733 quarantined.push(task.id.clone());
734 }
735 }
736 quarantined
737}
738
739#[derive(Debug, Clone, Copy, PartialEq, Eq)]
742enum DepsAction {
743 Release,
744 Discard,
745 Detach,
746 Nothing,
748}
749
750fn interpret_deps_answer(q: &Question) -> DepsAction {
751 let resolution = q.resolution().unwrap_or_default();
752 match q.choices.iter().position(|c| *c == resolution) {
753 Some(0) => DepsAction::Release,
754 Some(1) => DepsAction::Discard,
755 Some(2) => DepsAction::Detach,
756 _ => DepsAction::Nothing,
757 }
758}
759
760fn deps_texts(w: &Wording, root: &Task, dependants: &[&Task]) -> (String, String, Vec<String>) {
762 let ja = w.lang == "ja";
763 let reason = root
764 .hold_reason
765 .as_deref()
766 .or(root.last_error.as_deref())
767 .unwrap_or(if ja {
768 "(記録なし)"
769 } else {
770 "(none recorded)"
771 });
772 let list: String = dependants
773 .iter()
774 .map(|t| format!("- {} {}\n", t.short(), t.title))
775 .collect();
776 if ja {
777 (
778 format!(
779 "{} ({}) が {} 件のタスクを止めています",
780 root.short(),
781 root.status.as_str(),
782 dependants.len()
783 ),
784 format!(
785 "task: {} ({})\ntitle: {}\n状態: {}\n理由: {reason}\n\n\
786 このタスクは自動では実行されないため、依存している次のタスクは永遠に待ち続けます:\n{list}",
787 root.id,
788 root.short(),
789 root.title,
790 root.status.as_str()
791 ),
792 vec![
793 "依存先を再開する".to_owned(),
794 "依存先を捨てる(依存タスクは切り離して実行)".to_owned(),
795 "依存タスクを切り離す(依存先はそのまま)".to_owned(),
796 ],
797 )
798 } else {
799 (
800 format!(
801 "{} ({}) is freezing {} blocked task(s)",
802 root.short(),
803 root.status.as_str(),
804 dependants.len()
805 ),
806 format!(
807 "task: {} ({})\ntitle: {}\nstatus: {}\nreason: {reason}\n\n\
808 Nothing in the loop will ever run this task, so these dependants wait \
809 forever:\n{list}",
810 root.id,
811 root.short(),
812 root.title,
813 root.status.as_str()
814 ),
815 vec![
816 "release the dependency".to_owned(),
817 "discard the dependency (dependants are detached and run)".to_owned(),
818 "detach the dependants (the dependency stays as it is)".to_owned(),
819 ],
820 )
821 }
822}
823
824fn detach_dependants(queue: &Queue, questions: &Questions, root: &str) {
828 for listed in queue.list() {
829 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| matches!(crate::queue::dependency(queue, questions, b), crate::queue::Dependency::Task(id) if id == root)) {
830 continue;
831 }
832 let Ok(_claim) = queue.claim(&listed.id) else {
833 continue;
834 };
835 let Ok(mut t) = queue.get(&listed.id) else {
836 continue;
837 };
838 if t.status != TaskStatus::Blocked {
839 continue;
840 }
841 for id in t.blocked_by.clone() {
842 if matches!(crate::queue::dependency(queue, questions, &id), crate::queue::Dependency::Task(full) if full == root)
843 {
844 t.unblock(&id);
845 }
846 }
847 let _ = queue.put(&mut t);
848 }
849}
850
851fn apply_deps_answer(queue: &Queue, questions: &Questions, q: &Question, now: Timestamp) -> bool {
855 let Ok(_claim) = queue.claim(&q.run) else {
856 return false;
857 };
858 let Ok(current) = questions.get(&q.id) else {
859 return false;
860 };
861 if current.status != QuestionStatus::Answered {
862 return false;
863 }
864 let q = ¤t;
865 let Ok(mut root) = queue.get(&q.run) else {
866 return false;
867 };
868 if root.triage_applied(&q.id) {
869 return false;
870 }
871 if root.operator_decided_at.is_some_and(|at| at > q.asked_at)
872 || q.triage_generation
873 .is_some_and(|g| g != root.resumption.as_ref().map_or(0, |r| r.generation))
874 {
875 root.mark_triage_applied(&q.id);
876 return queue.put(&mut root).is_ok();
877 }
878 match interpret_deps_answer(q) {
879 DepsAction::Release => {
880 if root.status == TaskStatus::Running {
881 return false;
882 }
883 root.answers.push(crate::queue::AnsweredQuestion {
884 question: q.summary.clone(),
885 answer: q.resolution().unwrap_or_default(),
886 });
887 if crate::base_sync_recovery::held_task(queue, &root) {
888 root.mark_triage_applied(&q.id);
889 return queue.put(&mut root).is_ok();
890 }
891 root.release_operator("triage-dependencies", Some(&q.id));
892 root.resume_override = Some(OperatorResume {
893 question_id: q.id.clone(),
894 at: now,
895 conductor_rehold: None,
896 forced: false,
897 pinned_run: None,
898 });
899 }
900 DepsAction::Discard => {
901 detach_dependants(queue, questions, &root.id);
902 return queue.discard_triage(&root.id, questions, &q.id).is_ok();
903 }
904 DepsAction::Detach => detach_dependants(queue, questions, &root.id),
905 DepsAction::Nothing => return false,
910 }
911 root.mark_triage_applied(&q.id);
912 queue.put(&mut root).is_ok()
913}
914
915fn ask_about_stuck_roots(
924 queue: &Queue,
925 questions: &Questions,
926 config_override: Option<&Path>,
927 report: &mut Report,
928) {
929 let all = questions.list();
930 let inv = crate::blockers::Inventory::load(queue, questions);
931 let mut frozen: std::collections::BTreeMap<String, Vec<String>> = Default::default();
932 for (id, roots) in inv.stuck() {
933 for root in roots {
934 if root != id {
935 frozen.entry(root).or_default().push(id.clone());
936 }
937 }
938 }
939 for mut q in all
943 .iter()
944 .filter(|q| q.node == DEPS_NODE && q.status.open())
945 .cloned()
946 {
947 if !frozen.contains_key(&q.run) {
948 q.abandon("nothing waits on this task anymore");
949 let _ = questions.put_executable(&mut q);
950 }
951 }
952
953 for (root_id, dependant_ids) in &frozen {
954 let Ok(_claim) = queue.claim(root_id) else {
955 continue;
956 };
957 let current = crate::blockers::Inventory::load(queue, questions);
960 let Some(root) = current.task(root_id) else {
961 continue;
962 };
963 if crate::base_sync_recovery::held_task(queue, root) {
964 continue;
965 }
966 let dependants: Vec<&Task> = dependant_ids
967 .iter()
968 .filter_map(|d| current.task(d))
969 .filter(|t| current.stuck_roots(t).contains(root_id))
970 .collect();
971 if dependants.is_empty() {
972 continue;
973 }
974 let latest = all
975 .iter()
976 .filter(|q| q.node == DEPS_NODE && q.run == *root_id)
977 .max_by(|a, b| a.asked_at.cmp(&b.asked_at).then_with(|| a.id.cmp(&b.id)));
978 match latest {
979 Some(q) if q.status.open() || awaiting_task_chat(q, root) => continue,
980 Some(q) if q.status == QuestionStatus::Answered && !root.triage_applied(&q.id) => {
981 continue;
982 }
983 Some(q) if q.status == QuestionStatus::Abandoned => {
984 let changed = root.updated_at > q.asked_at
985 || dependants.iter().any(|t| t.updated_at > q.asked_at);
986 if !changed {
987 continue;
988 }
989 }
990 _ => {}
991 }
992 if pending_for(questions, root) {
993 continue;
994 }
995 let cfg = Config::discover(&repo_for(root), config_override)
996 .ok()
997 .map(|(c, _)| c);
998 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
999 let (summary, detail, choices) = deps_texts(w, root, &dependants);
1000 let mut question = Question::new(
1001 root.id.clone(),
1002 DEPS_NODE.to_owned(),
1003 SEAT.to_owned(),
1004 summary,
1005 detail,
1006 choices,
1007 );
1008 question.triage_generation = Some(root.resumption.as_ref().map_or(0, |r| r.generation));
1009 if questions.put_executable(&mut question).is_ok() {
1010 report.asked.push(root.id.clone());
1011 }
1012 }
1013}
1014
1015pub fn apply_answers(queue: &Queue, questions: &Questions) -> Vec<String> {
1018 let mut applied = Vec::new();
1019 for q in questions.list() {
1020 if q.node == DEPS_NODE
1021 && q.status == QuestionStatus::Answered
1022 && apply_deps_answer(queue, questions, &q, Timestamp::now())
1023 {
1024 applied.push(q.run.clone());
1025 }
1026 }
1027 for listed in queue.list() {
1028 let Some(q) = latest_triage_question(questions, &listed.id) else {
1029 continue;
1030 };
1031 if q.status != QuestionStatus::Answered || already_applied(&listed, &q) {
1032 continue;
1033 }
1034 let Ok(_claim) = queue.claim(&listed.id) else {
1035 continue;
1036 };
1037 let Ok(mut task) = queue.get(&listed.id) else {
1038 continue;
1039 };
1040 let Ok(q) = questions.get(&q.id) else {
1043 continue;
1044 };
1045 if q.status != QuestionStatus::Answered {
1046 continue;
1047 }
1048 if already_applied(&task, &q)
1049 || !matches!(task.status, TaskStatus::Held | TaskStatus::Queued)
1050 {
1051 continue;
1052 }
1053 let generation = task.resumption.as_ref().map_or(0, |r| r.generation);
1054 let action = interpret_answer(&q);
1055 let mut newer_decision = q.triage_generation.is_some_and(|g| g != generation)
1056 || task.resumption.as_ref().is_some_and(|r| r.at > q.asked_at)
1057 || task.held_at.is_some_and(|at| at > q.asked_at)
1058 || task.operator_decided_at.is_some_and(|at| at > q.asked_at);
1059 if matches!(action, AnswerAction::KeepHeld)
1063 && q.answered_at.is_some_and(|at| {
1064 task.operator_decided_at
1065 .is_none_or(|decision| at > decision)
1066 && task.resumption.as_ref().is_none_or(|r| at > r.at)
1067 })
1068 {
1069 newer_decision = false;
1070 }
1071 let resolution = q.resolution().unwrap_or_default();
1073 task.answers.push(crate::queue::AnsweredQuestion {
1074 question: q.summary.clone(),
1075 answer: if newer_decision {
1076 format!("{resolution} (superseded by a later operator decision)")
1077 } else {
1078 resolution
1079 },
1080 });
1081 if !newer_decision {
1082 match action {
1083 AnswerAction::Resume if crate::base_sync_recovery::held_task(queue, &task) => {
1084 }
1087 AnswerAction::Resume => {
1088 task.release_operator("triage", Some(&q.id));
1089 task.resume_override = Some(OperatorResume {
1090 question_id: q.id.clone(),
1091 at: Timestamp::now(),
1092 conductor_rehold: None,
1093 forced: true,
1094 pinned_run: None,
1095 });
1096 }
1097 AnswerAction::KeepHeld => {
1098 let note = keep_held_note(&task, &q, &q.resolution().unwrap_or_default());
1099 task.hold_manual(Some(note));
1100 }
1101 AnswerAction::Discard => {
1102 if queue.discard_triage(&task.id, questions, &q.id).is_ok() {
1103 applied.push(task.id);
1104 }
1105 continue;
1106 }
1107 }
1108 }
1109 task.mark_triage_applied(&q.id);
1110 if queue.put(&mut task).is_ok() {
1111 applied.push(task.id);
1112 }
1113 }
1114 applied
1115}
1116
1117pub fn run_once(
1135 queue: &Queue,
1136 questions: &Questions,
1137 config_override: Option<&Path>,
1138 now: Timestamp,
1139) -> Report {
1140 let mut report = Report {
1141 answered: apply_answers(queue, questions),
1142 quarantined: quarantine_orphaned_blocked(queue, questions),
1143 ..Report::default()
1144 };
1145 ask_about_stuck_roots(queue, questions, config_override, &mut report);
1146 for listed in queue.list() {
1147 if listed.status != TaskStatus::Held {
1148 continue;
1149 }
1150 let Ok(_claim) = queue.claim(&listed.id) else {
1151 continue;
1152 };
1153 let Ok(mut task) = queue.get(&listed.id) else {
1154 continue;
1155 };
1156 if task.status != TaskStatus::Held {
1160 continue;
1161 }
1162 if crate::base_sync_recovery::held_task(queue, &task) {
1163 continue;
1164 }
1165 if deps_pending(questions, &task) {
1169 continue;
1170 }
1171
1172 let cfg = Config::discover(&repo_for(&task), config_override)
1173 .ok()
1174 .map(|(c, _)| c);
1175 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1176
1177 if let Some(q) = latest_triage_question(questions, &task.id) {
1178 if q.status.open() || awaiting_task_chat(&q, &task) {
1179 continue;
1181 }
1182 }
1188
1189 match task.hold_source {
1190 Some(HoldSource::Machine) => {
1191 let overridden = task
1192 .resume_override
1193 .as_ref()
1194 .is_some_and(|o| o.conductor_rehold.is_some() && !o.forced);
1195 if overridden {
1196 if file_question(questions, &task, Bucket::ConductorOverride, w, now).is_some()
1197 {
1198 report.asked.push(task.id.clone());
1199 }
1200 } else if cfg.as_ref().and_then(|c| machine_cause_resolved(&task, c)) == Some(true)
1201 {
1202 task.release();
1203 if queue.put(&mut task).is_ok() {
1204 report.resumed.push(task.id.clone());
1205 }
1206 } else if file_question(questions, &task, Bucket::MachineUnknown, w, now).is_some()
1207 {
1208 report.asked.push(task.id.clone());
1209 }
1210 }
1211 None => {
1212 if file_question(questions, &task, Bucket::Legacy, w, now).is_some() {
1213 report.asked.push(task.id.clone());
1214 }
1215 }
1216 Some(HoldSource::Manual) => {
1217 if manual_is_stale(&task, now)
1218 && file_question(questions, &task, Bucket::ManualStale, w, now).is_some()
1219 {
1220 report.asked.push(task.id.clone());
1221 }
1222 }
1223 }
1224 }
1225 report
1226}
1227
1228pub fn open_question_for(questions: &Questions, task_id: &str) -> Option<Question> {
1233 latest_triage_question(questions, task_id).filter(|q| q.status.open())
1234}
1235
1236pub fn pending_for(questions: &Questions, task: &Task) -> bool {
1250 !pending_recovery_questions(questions, task).is_empty()
1251}
1252
1253pub(crate) fn pending_recovery_questions(questions: &Questions, task: &Task) -> Vec<Question> {
1256 [NODE, DEPS_NODE]
1257 .into_iter()
1258 .filter_map(|node| {
1259 latest_question(questions, node, &task.id).filter(|q| {
1260 q.status.open()
1261 || awaiting_task_chat(q, task)
1262 || (q.status == QuestionStatus::Answered
1263 && if node == NODE {
1264 !already_applied(task, q)
1265 } else {
1266 !task.triage_applied(&q.id)
1267 })
1268 })
1269 })
1270 .collect()
1271}
1272
1273fn awaiting_task_chat(q: &Question, task: &Task) -> bool {
1274 q.status != QuestionStatus::Abandoned
1275 && q.routed_chat.is_some()
1276 && !already_applied(task, q)
1277 && task.resumption.as_ref().map_or(0, |r| r.generation) == q.triage_generation.unwrap_or(0)
1278 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1279}
1280
1281fn deps_pending(questions: &Questions, task: &Task) -> bool {
1285 match latest_question(questions, DEPS_NODE, &task.id) {
1286 Some(q) if q.status.open() || awaiting_task_chat(&q, task) => true,
1287 Some(q) if q.status == QuestionStatus::Answered => !task.triage_applied(&q.id),
1288 _ => false,
1289 }
1290}
1291
1292pub fn open_task_ids(questions: &Questions) -> std::collections::BTreeSet<String> {
1297 questions
1298 .list()
1299 .into_iter()
1300 .filter(|q| (q.node == NODE || q.node == DEPS_NODE) && q.status.open())
1301 .map(|q| q.run)
1302 .collect()
1303}
1304
1305#[cfg(test)]
1306mod tests {
1307 use super::*;
1308 use crate::ask::Answer;
1309 use crate::queue::Source;
1310 use jiff::SignedDuration;
1311
1312 fn store() -> (tempfile::TempDir, Queue, Questions) {
1313 let dir = tempfile::tempdir().unwrap();
1314 let q = Queue::at(dir.path().join("queue"));
1315 let s = Questions::at(dir.path().join("questions"));
1316 (dir, q, s)
1317 }
1318
1319 fn task(title: &str, repo: PathBuf) -> Task {
1320 Task::new(title.to_owned(), format!("do {title}"), repo, Source::Human)
1321 }
1322
1323 #[test]
1324 fn base_recovery_stops_suppress_generic_and_dependency_resume_offers() {
1325 let (dir, queue, questions) = store();
1326 let mut root = task("base stop", dir.path().to_path_buf());
1327 let mut run = crate::run::RunState::new(
1328 dir.path().to_path_buf(),
1329 "main".into(),
1330 "base".into(),
1331 "retained implementation".into(),
1332 Config::default(),
1333 );
1334 run.status = crate::run::RunStatus::Voting;
1335 run.base_sync = Some(crate::run::BaseSync {
1336 tip: "tip".into(),
1337 behind: 3,
1338 attempts: 4,
1339 conflict: Some("persisted exhaustion".into()),
1340 already_in: None,
1341 });
1342 run.save_under(dir.path()).unwrap();
1343 root.link_run(&run.id);
1344 root.hold_machine(Some("base stop".into()));
1345 queue.put(&mut root).unwrap();
1346 let mut dependent = task("dependent", dir.path().to_path_buf());
1347 dependent.blocked_by = vec![root.id.clone()];
1348 dependent.status = TaskStatus::Blocked;
1349 queue.put(&mut dependent).unwrap();
1350 for _ in 0..2 {
1351 let report = run_once(&queue, &questions, None, Timestamp::now());
1352 assert!(report.asked.is_empty());
1353 assert!(report.resumed.is_empty());
1354 }
1355 let q = file_question(
1358 &questions,
1359 &root,
1360 Bucket::MachineUnknown,
1361 &EN,
1362 Timestamp::now(),
1363 )
1364 .unwrap();
1365 let mut q = q;
1366 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
1367 questions.put_executable(&mut q).unwrap();
1368 apply_answers(&queue, &questions);
1369 let saved = queue.get(&root.id).unwrap();
1370 assert_eq!(saved.status, TaskStatus::Held);
1371 assert!(saved.triage_applied(&q.id));
1372 assert_eq!(saved.runs, vec![run.id]);
1373 }
1374
1375 #[test]
1376 fn resource_holds_only_auto_release_a_proven_cache_wait_timeout() {
1377 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
1378 let dir = tempfile::tempdir().unwrap();
1379 let cfg = Config::default(); assert!(cfg.cache_dir().is_none());
1381 let mut state = RunState::new(
1382 dir.path().into(),
1383 "main".into(),
1384 "base".into(),
1385 "verify".into(),
1386 cfg.clone(),
1387 );
1388 let mut task = task("verify", dir.path().into());
1389 task.runs = vec![state.id.clone()];
1390 task.hold_machine(Some("[resource] cache operation stopped".into()));
1391 for cause in [
1392 ResourceBlockCause::Other,
1393 ResourceBlockCause::Unrecorded,
1394 ResourceBlockCause::Cancelled { park: None },
1395 ] {
1396 state.resource_block = Some(ResourceBlock {
1397 node: "gate".into(),
1398 kind: "shared build cache".into(),
1399 detail: "persistent metadata or manifest error".into(),
1400 cause,
1401 at: Timestamp::now(),
1402 });
1403 state.save_under(dir.path()).unwrap();
1404 for _ in 0..3 {
1406 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1407 }
1408 }
1409 let block = state.resource_block.as_mut().unwrap();
1410 block.cause = ResourceBlockCause::Timeout {
1411 waited_secs: 10,
1412 limit_secs: 10,
1413 };
1414 state.save_under(dir.path()).unwrap();
1415 assert_eq!(
1416 machine_cause_resolved_under(&task, &cfg, dir.path()),
1417 Some(true)
1418 );
1419 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
1420 state.save_under(dir.path()).unwrap();
1421 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1422 task.runs = vec!["unreadable-run".into()];
1423 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1424 }
1425
1426 #[cfg(unix)]
1427 #[test]
1428 fn disk_admission_holds_recheck_live_reservations_and_the_floor() {
1429 use crate::retention::OutputLease;
1430 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
1431 let temp = tempfile::tempdir().unwrap();
1432 let home = temp.path().canonicalize().unwrap();
1433 let mut cfg = Config::default();
1434 cfg.graph.worktree_root = Some(home.join("worktrees"));
1435 cfg.disk.min_free_bytes = 100;
1436 let cache = home.join("cache");
1437 cfg.verify.gate = vec![format!(
1438 "CARGO_TARGET_DIR='{}' cargo build",
1439 cache.display()
1440 )];
1441 let mut state = RunState::new(
1442 home.clone(),
1443 "main".into(),
1444 "base".into(),
1445 "verify".into(),
1446 cfg.clone(),
1447 );
1448 state.resource_block = Some(ResourceBlock {
1449 node: "gate".into(),
1450 kind: "disk admission".into(),
1451 detail: "writer headroom unavailable".into(),
1452 cause: ResourceBlockCause::Other,
1453 at: Timestamp::now(),
1454 });
1455 state.save_under(&home).unwrap();
1456 let mut task = task("verify", home.clone());
1457 task.runs = vec![state.id.clone()];
1458 task.hold_machine(Some("[resource] gate: disk admission: blocked".into()));
1459 let gib = 1024 * 1024 * 1024;
1460 let free = std::cell::Cell::new(100 + gib - 1);
1461 let check = || {
1462 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths| {
1463 assert!(
1464 paths.contains(&cache),
1465 "the cache volume must also be checked"
1466 );
1467 assert!(paths.contains(&state.worktree_root()));
1468 OutputLease::admit_with(&home, paths, cfg.disk.min_free_bytes, |_| Ok(free.get()))
1469 .is_ok()
1470 })
1471 };
1472 assert_eq!(check(), Some(false), "the unchanged floor still applies");
1473 free.set(100 + gib);
1474 let writer = OutputLease::admit_with(
1475 &home,
1476 std::slice::from_ref(&home),
1477 cfg.disk.min_free_bytes,
1478 |_| Ok(free.get()),
1479 )
1480 .unwrap();
1481 assert_eq!(
1482 check(),
1483 Some(false),
1484 "df alone cannot resolve a live reservation"
1485 );
1486 drop(writer);
1487 assert_eq!(
1488 check(),
1489 Some(true),
1490 "releasing headroom resolves the machine hold"
1491 );
1492 assert_eq!(
1493 check(),
1494 Some(true),
1495 "the probe must release its temporary reservation"
1496 );
1497
1498 let namespace = home.join("disk-admission");
1500 let saved = home.join("saved-admission");
1501 std::fs::rename(&namespace, &saved).unwrap();
1502 std::os::unix::fs::symlink(&saved, &namespace).unwrap();
1503 assert_eq!(check(), Some(false));
1504 std::fs::remove_file(&namespace).unwrap();
1505 std::fs::rename(saved, namespace).unwrap();
1506 assert_eq!(check(), Some(true));
1507
1508 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Cancelled { park: None };
1509 state.save_under(&home).unwrap();
1510 assert_eq!(
1511 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
1512 "a cancelled operation must remain held"
1513 )),
1514 None
1515 );
1516 state.resource_block.as_mut().unwrap().kind = "incremental cache admission".into();
1517 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Other;
1518 state.save_under(&home).unwrap();
1519 assert_eq!(
1520 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
1521 "unproven incremental caches must remain held"
1522 )),
1523 None
1524 );
1525 task.runs.push("unreadable-run".into());
1526 assert_eq!(
1527 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
1528 "missing evidence must not release a hold"
1529 )),
1530 None
1531 );
1532 }
1533
1534 fn gate_disabled_config(dir: &std::path::Path) -> PathBuf {
1538 let config = dir.join("magi.toml");
1539 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1540 config
1541 }
1542
1543 #[test]
1544 fn resumption_later_keep_held_answer_wins_but_an_older_answer_cannot_undo_release() {
1545 for source in ["cli", "web"] {
1546 for answer_after_release in [true, false] {
1547 let (dir, queue, questions) = store();
1548 let mut t = task("recovery", dir.path().join("repo"));
1549 t.hold_machine(Some("inspect remote first".into()));
1550 queue.put(&mut t).unwrap();
1551 let mut q = file_question(
1552 &questions,
1553 &t,
1554 Bucket::MachineUnknown,
1555 &JA,
1556 Timestamp::now(),
1557 )
1558 .unwrap();
1559 if !answer_after_release {
1560 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
1561 questions.put(&mut q).unwrap();
1562 }
1563 t.release_operator(source, None);
1564 queue.put(&mut t).unwrap();
1565 if answer_after_release {
1566 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
1567 questions.put(&mut q).unwrap();
1568 }
1569 apply_answers(&queue, &questions);
1570 let after = queue.get(&t.id).unwrap();
1571 assert!(after.triage_applied(&q.id));
1572 assert_eq!(after.operator_held(), answer_after_release);
1573 assert_eq!(
1574 after.status,
1575 if answer_after_release {
1576 TaskStatus::Held
1577 } else {
1578 TaskStatus::Queued
1579 }
1580 );
1581 assert!(apply_answers(&queue, &questions).is_empty());
1582 }
1583 }
1584 }
1585
1586 #[test]
1587 fn explicit_resumption_applies_legacy_queued_triage_once_and_preserves_later_holds() {
1588 let (dir, queue, questions) = store();
1589 let mut t = task("recover", dir.path().join("repo"));
1590 t.hold_machine(Some("inspect first".into()));
1591 queue.put(&mut t).unwrap();
1592 let mut q = file_question(
1593 &questions,
1594 &t,
1595 Bucket::MachineUnknown,
1596 &JA,
1597 Timestamp::now(),
1598 )
1599 .unwrap();
1600 assert!(q.actions.is_empty());
1601 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
1602 questions.put(&mut q).unwrap();
1603 t.release();
1605 t.instruction = "Reuse approved PR head".into();
1606 queue.put(&mut t).unwrap();
1607 assert_eq!(apply_answers(&queue, &questions), [t.id.clone()]);
1608 let mut after = queue.get(&t.id).unwrap();
1609 assert_eq!(
1610 after.resumption.as_ref().unwrap().question_id.as_deref(),
1611 Some(q.id.as_str())
1612 );
1613 assert_eq!(
1614 after.resumption.as_ref().unwrap().instruction,
1615 t.instruction
1616 );
1617 assert_eq!(after.answers[0].answer, JA.resume);
1618 assert!(after.triage_applied(&q.id));
1619 assert!(apply_answers(&Queue::at(dir.path().join("queue")), &questions).is_empty());
1620 after.hold_machine(Some("new blocker".into()));
1621 queue.put(&mut after).unwrap();
1622 let mut late = file_question(
1623 &questions,
1624 &after,
1625 Bucket::MachineUnknown,
1626 &EN,
1627 Timestamp::now(),
1628 )
1629 .unwrap();
1630 after.hold_manual(Some("keep held now".into()));
1631 queue.put(&mut after).unwrap();
1632 late.answer(Answer::Choice(late.choices[0].clone()))
1633 .unwrap();
1634 questions.put(&mut late).unwrap();
1635 apply_answers(&queue, &questions);
1636 let held = queue.get(&t.id).unwrap();
1637 assert!(held.operator_held());
1638 assert_eq!(held.hold_reason.as_deref(), Some("keep held now"));
1639 assert_eq!(held.resumption.as_ref().unwrap().generation, 1);
1640 }
1641
1642 #[test]
1643 fn a_resolved_machine_hold_is_requeued_automatically() {
1644 let (dir, q, questions) = store();
1645 let config = gate_disabled_config(dir.path());
1646 let mut t = task("disk pressure", dir.path().join("repo"));
1647 t.hold_machine(Some(
1648 "not enough free space to start a run: 10 bytes free, 100 required by \
1649 `[disk] min_free_bytes`"
1650 .to_owned(),
1651 ));
1652 q.put(&mut t).unwrap();
1653
1654 let report = run_once(&q, &questions, Some(&config), Timestamp::now());
1655 assert_eq!(report.resumed, [t.id.clone()]);
1656 assert!(report.asked.is_empty());
1657
1658 let back = q.get(&t.id).unwrap();
1659 assert_eq!(back.status, TaskStatus::Queued);
1660 assert!(back.hold_source.is_none());
1661 assert!(questions.list().is_empty(), "nothing needed asking");
1662 }
1663
1664 #[test]
1665 fn a_machine_hold_with_no_recognised_cause_gets_one_question_not_two() {
1666 let (dir, q, questions) = store();
1667 let mut t = task("gate went red", dir.path().join("repo"));
1668 t.hold_machine(Some("gate red".to_owned()));
1669 q.put(&mut t).unwrap();
1670
1671 let first = run_once(&q, &questions, None, Timestamp::now());
1672 assert_eq!(first.asked, [t.id.clone()]);
1673 assert!(first.resumed.is_empty());
1674
1675 let open: Vec<_> = questions
1676 .list()
1677 .into_iter()
1678 .filter(|q| q.status.open())
1679 .collect();
1680 assert_eq!(open.len(), 1);
1681 assert_eq!(open[0].run, t.id);
1682 assert_eq!(open[0].node, NODE);
1683 assert_eq!(open[0].choices.len(), 3);
1684
1685 let second = run_once(&q, &questions, None, Timestamp::now());
1687 assert!(second.asked.is_empty());
1688 assert_eq!(
1689 questions
1690 .list()
1691 .into_iter()
1692 .filter(|q| q.status.open())
1693 .count(),
1694 1
1695 );
1696 }
1697
1698 #[test]
1699 fn a_legacy_hold_with_no_recorded_source_gets_exactly_one_question() {
1700 let (dir, q, questions) = store();
1701 let mut t = task("schema 1 record", dir.path().join("repo"));
1702 t.status = TaskStatus::Held;
1703 assert!(t.hold_source.is_none(), "the case this test is about");
1704 q.put(&mut t).unwrap();
1705
1706 let first = run_once(&q, &questions, None, Timestamp::now());
1707 assert_eq!(first.asked, [t.id.clone()]);
1708
1709 let second = run_once(&q, &questions, None, Timestamp::now());
1710 assert!(
1711 second.asked.is_empty(),
1712 "the same legacy hold must not be asked about twice"
1713 );
1714 assert_eq!(
1715 questions
1716 .list()
1717 .into_iter()
1718 .filter(|q| q.status.open())
1719 .count(),
1720 1
1721 );
1722 }
1723
1724 #[test]
1725 fn dependency_resolution_detaches_legacy_short_refs_without_dropping_other_waits() {
1726 let (dir, queue, questions) = store();
1727 let mut root = task("root", dir.path().join("repo"));
1728 root.id = "20261013-193611-ca5b".into();
1729 root.hold_machine(Some("needs owner decision".into()));
1730 queue.put(&mut root).unwrap();
1731 let mut other = task("other", dir.path().join("repo"));
1732 other.id = "20261013-192213-feb2".into();
1733 queue.put(&mut other).unwrap();
1734 let mut waiting = task("waiting", dir.path().join("repo"));
1735 waiting.block(vec!["ca5b".into(), other.id.clone()], None);
1736 queue.put(&mut waiting).unwrap();
1737 detach_dependants(&queue, &questions, &root.id);
1738 let after = queue.get(&waiting.id).unwrap();
1739 assert_eq!(after.status, TaskStatus::Blocked);
1740 assert_eq!(after.blocked_by, [other.id]);
1741 }
1742
1743 #[test]
1744 fn a_manual_hold_is_never_auto_resumed() {
1745 let (dir, q, questions) = store();
1746 let mut t = task("operator stopped this", dir.path().join("repo"));
1747 t.hold_manual(Some("waiting on a decision".to_owned()));
1748 q.put(&mut t).unwrap();
1749
1750 let report = run_once(&q, &questions, None, Timestamp::now());
1751 assert!(report.resumed.is_empty());
1752 assert!(report.asked.is_empty());
1754
1755 let back = q.get(&t.id).unwrap();
1756 assert_eq!(back.status, TaskStatus::Held);
1757 assert_eq!(back.hold_source, Some(HoldSource::Manual));
1758 assert!(questions.list().is_empty());
1759 }
1760
1761 #[test]
1762 fn a_blocked_task_on_a_deleted_dependency_is_held_and_asked_about_in_one_pass() {
1763 let (dir, q, questions) = store();
1768 let mut still_going = task("still valid", dir.path().join("repo"));
1769 q.put(&mut still_going).unwrap();
1770
1771 let mut t = task("orphaned", dir.path().join("repo"));
1772 t.block(
1773 vec!["20260101-000000-gone".to_owned(), still_going.id.clone()],
1774 Some("waits on both".to_owned()),
1775 );
1776 q.put(&mut t).unwrap();
1777
1778 let report = run_once(&q, &questions, None, Timestamp::now());
1779 assert_eq!(report.quarantined, [t.id.clone()]);
1780 assert_eq!(
1781 report.asked,
1782 [t.id.clone()],
1783 "the fresh machine hold must earn a question in the same pass"
1784 );
1785
1786 let after = q.get(&t.id).unwrap();
1787 assert_eq!(after.status, TaskStatus::Held);
1788 assert_eq!(after.hold_source, Some(HoldSource::Machine));
1789 assert!(after.blocked_by.is_empty());
1790
1791 assert!(!is_disk_hold(&after));
1794
1795 let open: Vec<_> = questions
1796 .list()
1797 .into_iter()
1798 .filter(|q| q.status.open())
1799 .collect();
1800 assert_eq!(open.len(), 1);
1801 assert_eq!(open[0].run, t.id);
1802
1803 let second = run_once(&q, &questions, None, Timestamp::now());
1805 assert!(second.quarantined.is_empty());
1806 assert!(second.asked.is_empty());
1807 }
1808
1809 fn stuck_chain(q: &Queue, dir: &std::path::Path) -> (Task, Task, Task) {
1812 let mut root = task("root", dir.join("repo"));
1813 root.hold_manual(Some("waiting".to_owned()));
1814 q.put(&mut root).unwrap();
1815 let mut mid = task("mid", dir.join("repo"));
1816 mid.block(vec![root.id.clone()], None);
1817 q.put(&mut mid).unwrap();
1818 let mut leaf = task("leaf", dir.join("repo"));
1819 leaf.block(vec![mid.id.clone()], None);
1820 q.put(&mut leaf).unwrap();
1821 (root, mid, leaf)
1822 }
1823
1824 fn deps_questions(questions: &Questions) -> Vec<Question> {
1825 questions
1826 .list()
1827 .into_iter()
1828 .filter(|q| q.node == DEPS_NODE)
1829 .collect()
1830 }
1831
1832 fn answer_deps(questions: &Questions, choice: usize) {
1833 let mut asked = deps_questions(questions).remove(0);
1834 let c = asked.choices[choice].clone();
1835 asked.answer(Answer::Choice(c)).unwrap();
1836 questions.put(&mut asked).unwrap();
1837 }
1838
1839 #[test]
1840 fn a_stuck_chain_earns_one_question_naming_every_dependant_and_is_not_reasked() {
1841 let (dir, q, questions) = store();
1842 let (root, mid, leaf) = stuck_chain(&q, dir.path());
1843
1844 let first = run_once(&q, &questions, None, Timestamp::now());
1845 assert_eq!(first.asked, std::slice::from_ref(&root.id));
1846 let asked = deps_questions(&questions);
1847 assert_eq!(asked.len(), 1, "one question per root, not per dependant");
1848 assert_eq!(asked[0].run, root.id);
1849 assert_eq!(asked[0].choices.len(), 3);
1850 assert!(asked[0].detail.contains(mid.short()), "{}", asked[0].detail);
1851 assert!(
1852 asked[0].detail.contains(leaf.short()),
1853 "{}",
1854 asked[0].detail
1855 );
1856
1857 let second = run_once(&q, &questions, None, Timestamp::now());
1858 assert!(second.asked.is_empty());
1859 assert_eq!(deps_questions(&questions).len(), 1);
1860 assert!(pending_for(&questions, &q.get(&root.id).unwrap()));
1861 }
1862
1863 #[test]
1864 fn a_stuck_root_with_a_machine_hold_gets_only_the_dependency_question() {
1865 let (dir, q, questions) = store();
1866 let mut root = task("root", dir.path().join("repo"));
1867 root.hold_machine(Some("gate red".to_owned()));
1868 q.put(&mut root).unwrap();
1869 let mut dep = task("dep", dir.path().join("repo"));
1870 dep.block(vec![root.id.clone()], None);
1871 q.put(&mut dep).unwrap();
1872
1873 run_once(&q, &questions, None, Timestamp::now());
1874 run_once(&q, &questions, None, Timestamp::now());
1875 assert_eq!(deps_questions(&questions).len(), 1);
1876 assert_eq!(
1877 questions.list().len(),
1878 1,
1879 "no second question about the hold"
1880 );
1881 }
1882
1883 #[test]
1884 fn a_dependency_that_can_still_run_asks_nothing() {
1885 let (dir, q, questions) = store();
1886 let mut live = task("live", dir.path().join("repo"));
1887 q.put(&mut live).unwrap();
1888 let mut dep = task("dep", dir.path().join("repo"));
1889 dep.block(vec![live.id.clone()], None);
1890 q.put(&mut dep).unwrap();
1891 let report = run_once(&q, &questions, None, Timestamp::now());
1892 assert!(report.asked.is_empty());
1893 assert!(questions.list().is_empty());
1894 }
1895
1896 #[test]
1897 fn answering_release_requeues_the_root_and_is_not_reasked() {
1898 let (dir, q, questions) = store();
1899 let (root, _mid, _leaf) = stuck_chain(&q, dir.path());
1900 run_once(&q, &questions, None, Timestamp::now());
1901 answer_deps(&questions, 0);
1902
1903 let report = run_once(&q, &questions, None, Timestamp::now());
1904 assert_eq!(report.answered, std::slice::from_ref(&root.id));
1905 assert!(report.asked.is_empty(), "applying must not re-ask");
1906 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Queued);
1907 let again = run_once(&q, &questions, None, Timestamp::now());
1908 assert!(again.asked.is_empty() && again.answered.is_empty());
1909 assert_eq!(deps_questions(&questions).len(), 1);
1910 }
1911
1912 #[test]
1913 fn answering_detach_frees_the_direct_dependant_and_keeps_the_root() {
1914 let (dir, q, questions) = store();
1915 let (root, mid, leaf) = stuck_chain(&q, dir.path());
1916 run_once(&q, &questions, None, Timestamp::now());
1917 answer_deps(&questions, 2);
1918
1919 let report = run_once(&q, &questions, None, Timestamp::now());
1920 assert!(report.asked.is_empty());
1921 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
1922 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
1923 let leaf = q.get(&leaf.id).unwrap();
1924 assert_eq!(leaf.status, TaskStatus::Blocked, "still waits on mid");
1925 assert_eq!(leaf.blocked_by, std::slice::from_ref(&mid.id));
1926 }
1927
1928 #[test]
1929 fn a_dependant_that_can_progress_through_another_blocker_is_not_stuck() {
1930 let (dir, q, questions) = store();
1931 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
1932 let mut live = task("live", dir.path().join("repo"));
1933 q.put(&mut live).unwrap();
1934 let mut both = q.get(&mid.id).unwrap();
1935 both.block(vec![root.id.clone(), live.id.clone()], None);
1936 q.put(&mut both).unwrap();
1937 assert!(
1939 run_once(&q, &questions, None, Timestamp::now())
1940 .asked
1941 .is_empty()
1942 );
1943 }
1944
1945 #[test]
1946 fn answering_discard_detaches_then_removes_the_root() {
1947 let (dir, q, questions) = store();
1948 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
1949 run_once(&q, &questions, None, Timestamp::now());
1950 answer_deps(&questions, 1);
1951
1952 let report = run_once(&q, &questions, None, Timestamp::now());
1953 assert!(q.get(&root.id).is_err());
1954 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
1955 assert!(
1956 report.asked.is_empty(),
1957 "no per-dependant quarantine question"
1958 );
1959 assert!(report.quarantined.is_empty());
1960 }
1961
1962 #[test]
1963 fn an_unmatched_deps_answer_changes_nothing_and_is_not_reasked() {
1964 let (dir, q, questions) = store();
1965 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
1966 run_once(&q, &questions, None, Timestamp::now());
1967 let mut asked = deps_questions(&questions).remove(0);
1968 asked.choices.clear();
1969 asked.answer(Answer::Text("dunno".to_owned())).unwrap();
1970 questions.put(&mut asked).unwrap();
1971
1972 for _ in 0..2 {
1973 let report = run_once(&q, &questions, None, Timestamp::now());
1974 assert!(report.asked.is_empty() && report.answered.is_empty());
1975 }
1976 assert_eq!(deps_questions(&questions).len(), 1);
1977 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
1978 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Blocked);
1979 }
1980
1981 #[test]
1982 fn a_dependency_cycle_terminates_and_is_asked_about_once() {
1983 let (dir, q, questions) = store();
1984 let mut a = task("a", dir.path().join("repo"));
1985 let mut b = task("b", dir.path().join("repo"));
1986 a.block(vec![b.id.clone()], None);
1987 b.block(vec![a.id.clone()], None);
1988 q.put(&mut a).unwrap();
1989 q.put(&mut b).unwrap();
1990
1991 let first = run_once(&q, &questions, None, Timestamp::now());
1992 assert_eq!(first.asked.len(), 1);
1993 let second = run_once(&q, &questions, None, Timestamp::now());
1994 assert!(second.asked.is_empty());
1995 assert_eq!(deps_questions(&questions).len(), 1);
1996 }
1997
1998 #[test]
1999 fn a_missing_dependency_is_still_quarantined_not_asked_about_as_stuck() {
2000 let (dir, q, questions) = store();
2001 let mut t = task("orphan", dir.path().join("repo"));
2002 t.block(vec!["20260101-000000-gone".to_owned()], None);
2003 q.put(&mut t).unwrap();
2004 let report = run_once(&q, &questions, None, Timestamp::now());
2005 assert_eq!(report.quarantined, [t.id.clone()]);
2006 assert!(deps_questions(&questions).is_empty());
2007 }
2008
2009 #[test]
2010 fn a_stale_manual_hold_earns_a_two_choice_question() {
2011 let (dir, q, questions) = store();
2012 let mut t = task("been sitting a while", dir.path().join("repo"));
2013 t.hold_manual(Some("waiting on a decision".to_owned()));
2014 q.put(&mut t).unwrap();
2015 let mut back = q.get(&t.id).unwrap();
2017 back.updated_at = Timestamp::now() - SignedDuration::new(8 * 24 * 60 * 60, 0);
2018 std::fs::write(
2019 q.path_of(&back.id),
2020 serde_json::to_string_pretty(&back).unwrap(),
2021 )
2022 .unwrap();
2023
2024 let report = run_once(&q, &questions, None, Timestamp::now());
2025 assert_eq!(report.asked, [t.id.clone()]);
2026 let open: Vec<_> = questions
2027 .list()
2028 .into_iter()
2029 .filter(|q| q.status.open())
2030 .collect();
2031 assert_eq!(open.len(), 1);
2032 assert_eq!(open[0].choices.len(), 2);
2033 }
2034
2035 #[test]
2036 fn answering_resume_releases_the_task() {
2037 let (dir, q, questions) = store();
2038 let mut t = task("gate went red", dir.path().join("repo"));
2039 t.hold_machine(Some("gate red".to_owned()));
2040 q.put(&mut t).unwrap();
2041 run_once(&q, &questions, None, Timestamp::now());
2042
2043 let mut asked = questions
2044 .list()
2045 .into_iter()
2046 .find(|q| q.run == t.id)
2047 .unwrap();
2048 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2049 questions.put(&mut asked).unwrap();
2050
2051 let report = run_once(&q, &questions, None, Timestamp::now());
2052 assert_eq!(report.answered, [t.id.clone()]);
2053 let back = q.get(&t.id).unwrap();
2054 assert_eq!(back.status, TaskStatus::Queued);
2055 assert!(back.hold_source.is_none());
2056 }
2057
2058 fn resumed_then_failed(q: &Queue, questions: &Questions, dir: &std::path::Path) -> Task {
2060 let mut t = task("gate went red", dir.join("repo"));
2061 t.hold_machine(Some("gate red".to_owned()));
2062 q.put(&mut t).unwrap();
2063 run_once(q, questions, None, Timestamp::now());
2064 let mut asked = questions
2065 .list()
2066 .into_iter()
2067 .find(|q| q.run == t.id)
2068 .unwrap();
2069 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2070 questions.put(&mut asked).unwrap();
2071
2072 let report = run_once(q, questions, None, Timestamp::now());
2073 assert_eq!(report.answered, [t.id.clone()]);
2074 let mut back = q.get(&t.id).unwrap();
2075 assert_eq!(back.status, TaskStatus::Queued);
2076 assert_eq!(back.attempts, 0);
2077
2078 back.start("run-1".to_owned());
2079 back.fail("rebase conflict", 1);
2080 assert_eq!(back.status, TaskStatus::Held);
2081 q.put(&mut back).unwrap();
2082 back
2083 }
2084
2085 #[test]
2086 fn a_resume_answer_is_applied_once_and_a_new_machine_hold_is_asked_about() {
2087 let (dir, q, questions) = store();
2088 let t = resumed_then_failed(&q, &questions, dir.path());
2089
2090 let report = run_once(&q, &questions, None, Timestamp::now());
2091 assert!(report.answered.is_empty(), "the old answer must not replay");
2092 assert_eq!(report.asked, std::slice::from_ref(&t.id));
2093 let back = q.get(&t.id).unwrap();
2094 assert_eq!(back.status, TaskStatus::Held);
2095 assert_eq!(
2096 questions
2097 .list()
2098 .into_iter()
2099 .filter(|q| q.status.open())
2100 .count(),
2101 1
2102 );
2103 }
2104
2105 #[test]
2106 fn a_manual_hold_placed_after_a_resume_is_not_undone_by_the_old_answer() {
2107 let (dir, q, questions) = store();
2108 let mut t = resumed_then_failed(&q, &questions, dir.path());
2109 t.hold_manual(Some("operator stopped this".to_owned()));
2110 q.put(&mut t).unwrap();
2111 let before = questions.list().len();
2112
2113 let report = run_once(&q, &questions, None, Timestamp::now());
2114 assert!(report.answered.is_empty());
2115 assert!(report.asked.is_empty());
2116 let back = q.get(&t.id).unwrap();
2117 assert_eq!(back.status, TaskStatus::Held);
2118 assert_eq!(back.hold_source, Some(HoldSource::Manual));
2119 assert_eq!(questions.list().len(), before);
2120 }
2121
2122 #[test]
2123 fn answering_not_yet_keeps_it_held_as_a_manual_hold_and_does_not_reapply() {
2124 let (dir, q, questions) = store();
2125 let mut t = task("gate went red", dir.path().join("repo"));
2126 t.hold_machine(Some("gate red".to_owned()));
2127 q.put(&mut t).unwrap();
2128 run_once(&q, &questions, None, Timestamp::now());
2129
2130 let mut asked = questions
2131 .list()
2132 .into_iter()
2133 .find(|q| q.run == t.id)
2134 .unwrap();
2135 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
2136 questions.put(&mut asked).unwrap();
2137
2138 let report = run_once(&q, &questions, None, Timestamp::now());
2139 assert_eq!(report.answered, [t.id.clone()]);
2140 let back = q.get(&t.id).unwrap();
2141 assert_eq!(back.status, TaskStatus::Held);
2142 assert_eq!(back.hold_source, Some(HoldSource::Manual));
2143 assert!(
2144 back.hold_reason
2145 .as_deref()
2146 .is_some_and(|r| r.contains("gate red")),
2147 "the original cause must survive a \"not yet\" answer, not just the \
2148 triage marker: {:?}",
2149 back.hold_reason
2150 );
2151
2152 let third = run_once(&q, &questions, None, Timestamp::now());
2156 assert!(third.answered.is_empty());
2157 assert!(third.asked.is_empty());
2158 }
2159
2160 #[test]
2161 fn answering_discard_removes_the_task_entirely() {
2162 let (dir, q, questions) = store();
2163 let mut t = task("gate went red", dir.path().join("repo"));
2164 t.hold_machine(Some("gate red".to_owned()));
2165 q.put(&mut t).unwrap();
2166 run_once(&q, &questions, None, Timestamp::now());
2167
2168 let mut asked = questions
2169 .list()
2170 .into_iter()
2171 .find(|q| q.run == t.id)
2172 .unwrap();
2173 asked.answer(Answer::Choice(EN.discard.to_owned())).unwrap();
2174 questions.put(&mut asked).unwrap();
2175
2176 let report = run_once(&q, &questions, None, Timestamp::now());
2177 assert_eq!(report.answered, [t.id.clone()]);
2178 assert!(
2179 q.get(&t.id).is_err(),
2180 "\"discard it\" (捨ててよい) must actually discard the task, not \
2181 just leave it sitting held forever"
2182 );
2183 }
2184
2185 #[test]
2186 fn an_answer_is_read_by_its_position_in_choices_not_by_the_callers_current_language() {
2187 let (dir, q, questions) = store();
2189 let ja_config = dir.path().join("ja.toml");
2190 std::fs::write(&ja_config, "[graph]\nlanguage = \"ja\"\n").unwrap();
2191 let mut t = task("gate went red", dir.path().join("repo"));
2192 t.hold_machine(Some("gate red".to_owned()));
2193 q.put(&mut t).unwrap();
2194 run_once(&q, &questions, Some(&ja_config), Timestamp::now());
2195
2196 let mut asked = questions
2197 .list()
2198 .into_iter()
2199 .find(|q| q.run == t.id)
2200 .unwrap();
2201 assert_eq!(asked.choices[0], JA.resume, "filed in Japanese");
2202 asked.answer(Answer::Choice(JA.resume.to_owned())).unwrap();
2203 questions.put(&mut asked).unwrap();
2204
2205 let en_config = dir.path().join("en.toml");
2210 std::fs::write(&en_config, "[graph]\nlanguage = \"en\"\n").unwrap();
2211 let report = run_once(&q, &questions, Some(&en_config), Timestamp::now());
2212 assert_eq!(report.answered, [t.id.clone()]);
2213 let back = q.get(&t.id).unwrap();
2214 assert_eq!(
2215 back.status,
2216 TaskStatus::Queued,
2217 "a resume answer must resume the task regardless of which \
2218 language it is read back in"
2219 );
2220 }
2221
2222 #[test]
2223 fn a_question_falls_back_to_last_error_when_hold_reason_was_never_set() {
2224 let (dir, q, questions) = store();
2229 let mut t = task("kept failing the gate", dir.path().join("repo"));
2230 t.start("run-1".to_owned());
2231 t.fail("gate red three times running", 1);
2232 assert_eq!(t.status, TaskStatus::Held);
2233 t.hold_reason = None; q.put(&mut t).unwrap();
2235
2236 run_once(&q, &questions, None, Timestamp::now());
2237 let asked = questions
2238 .list()
2239 .into_iter()
2240 .find(|q| q.run == t.id)
2241 .unwrap();
2242 assert!(
2243 asked.detail.contains("gate red three times running"),
2244 "the question must surface `last_error` when there is no \
2245 `hold_reason` to show instead: {}",
2246 asked.detail
2247 );
2248 }
2249
2250 #[test]
2251 fn open_question_for_and_open_task_ids_reflect_only_what_is_still_waiting() {
2252 let (dir, q, questions) = store();
2253 let mut t = task("gate went red", dir.path().join("repo"));
2254 t.hold_machine(Some("gate red".to_owned()));
2255 q.put(&mut t).unwrap();
2256
2257 assert!(open_question_for(&questions, &t.id).is_none());
2258 assert!(!open_task_ids(&questions).contains(&t.id));
2259
2260 run_once(&q, &questions, None, Timestamp::now());
2261 assert!(open_question_for(&questions, &t.id).is_some());
2262 assert!(open_task_ids(&questions).contains(&t.id));
2263
2264 let mut asked = questions
2265 .list()
2266 .into_iter()
2267 .find(|q| q.run == t.id)
2268 .unwrap();
2269 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2270 questions.put(&mut asked).unwrap();
2271
2272 assert!(
2273 open_question_for(&questions, &t.id).is_none(),
2274 "an answered question is no longer open"
2275 );
2276 assert!(!open_task_ids(&questions).contains(&t.id));
2277 }
2278
2279 #[test]
2280 fn pending_for_stays_true_between_an_answer_and_the_next_run_once_pass() {
2281 let (dir, q, questions) = store();
2287 let mut t = task("gate went red", dir.path().join("repo"));
2288 t.hold_machine(Some("gate red".to_owned()));
2289 q.put(&mut t).unwrap();
2290
2291 assert!(!pending_for(&questions, &t));
2292
2293 run_once(&q, &questions, None, Timestamp::now());
2294 let held = q.get(&t.id).unwrap();
2295 assert!(pending_for(&questions, &held), "still waiting on an answer");
2296
2297 let mut asked = questions
2298 .list()
2299 .into_iter()
2300 .find(|q| q.run == t.id)
2301 .unwrap();
2302 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
2303 questions.put(&mut asked).unwrap();
2304 assert!(!asked.status.open());
2305
2306 let still_held = q.get(&t.id).unwrap();
2308 assert!(
2309 pending_for(&questions, &still_held),
2310 "answered but not yet applied is still pending"
2311 );
2312
2313 run_once(&q, &questions, None, Timestamp::now());
2314 let after = q.get(&t.id).unwrap();
2315 assert!(
2316 !pending_for(&questions, &after),
2317 "the answer is applied now, nothing left pending"
2318 );
2319 }
2320}