1use std::path::{Path, PathBuf};
84use std::time::Duration;
85
86use jiff::Timestamp;
87
88use crate::ask::{Question, QuestionStatus, Questions};
89use crate::config::Config;
90use crate::queue::{HoldSource, OperatorResume, Queue, Task, TaskStatus};
91
92pub const NODE: &str = "triage";
95
96pub const DEPS_NODE: &str = "triage-deps";
100
101const SEAT: &str = "triage";
106
107const MANUAL_STALE_AFTER: Duration = Duration::from_secs(7 * 24 * 60 * 60);
119
120struct Wording {
124 lang: &'static str,
125 resume: &'static str,
126 wait: &'static str,
127 discard: &'static str,
128 resume_now: &'static str,
129 keep_held: &'static str,
130}
131
132const EN: Wording = Wording {
133 lang: "en",
134 resume: "resume it",
135 wait: "not yet",
136 discard: "discard it",
137 resume_now: "resume it",
138 keep_held: "keep it held",
139};
140
141const JA: Wording = Wording {
142 lang: "ja",
143 resume: "再開してよい",
144 wait: "まだ待って",
145 discard: "捨ててよい",
146 resume_now: "再開する",
147 keep_held: "まだ止めておく",
148};
149
150fn wording(language: &str) -> &'static Wording {
153 if crate::lang::is_japanese(language) {
154 &JA
155 } else {
156 &EN
157 }
158}
159
160impl Wording {
161 fn choices3(&self) -> Vec<String> {
162 vec![
163 self.resume.to_owned(),
164 self.wait.to_owned(),
165 self.discard.to_owned(),
166 ]
167 }
168
169 fn choices2(&self) -> Vec<String> {
170 vec![self.resume_now.to_owned(), self.keep_held.to_owned()]
171 }
172
173 fn source_label(&self, source: Option<HoldSource>) -> &'static str {
174 match (self.lang, source) {
175 ("ja", Some(HoldSource::Machine)) => "machine(機械による自動保留)",
176 ("ja", Some(HoldSource::Manual)) => "manual(操作者による手動保留)",
177 ("ja", None) => "unknown(schema 3 未満の旧レコード、または理由未記録)",
178 (_, Some(HoldSource::Machine)) => "machine (automatic recovery hold)",
179 (_, Some(HoldSource::Manual)) => "manual (an operator held this)",
180 (_, None) => "unknown (pre-schema-3 record, or never recorded)",
181 }
182 }
183
184 fn detail(&self, task: &Task, why: &str) -> String {
193 let none = if self.lang == "ja" {
194 "(記録なし)"
195 } else {
196 "(none recorded)"
197 };
198 let reason = task
199 .hold_reason
200 .as_deref()
201 .or(task.last_error.as_deref())
202 .unwrap_or(none);
203 format!(
204 "task: {} ({})\ntitle: {}\nhold source: {}\nhold reason: {reason}\n\n{why}",
205 task.id,
206 task.short(),
207 task.title,
208 self.source_label(task.hold_source),
209 )
210 }
211
212 fn summary_machine_unknown(&self, task: &Task) -> String {
213 if self.lang == "ja" {
214 format!("保留タスク {} の再開可否を判断してください", task.short())
215 } else {
216 format!("decide whether to resume held task {}", task.short())
217 }
218 }
219
220 fn why_machine(&self) -> &'static str {
221 if self.lang == "ja" {
222 "機械的な保留(machine hold)ですが、原因がすでに解消しているかを自動では判断できませんでした。"
223 } else {
224 "This is a machine hold, but whether its cause has resolved could not be \
225 checked automatically."
226 }
227 }
228
229 fn summary_conductor_override(&self, task: &Task) -> String {
230 if self.lang == "ja" {
231 format!(
232 "再開と回答済みのタスク {} を conductor が再び保留しました",
233 task.short()
234 )
235 } else {
236 format!(
237 "task {} was resumed at your word, but the conductor held it again",
238 task.short()
239 )
240 }
241 }
242
243 fn why_conductor_override(&self, o: &OperatorResume) -> String {
244 let reason = o.conductor_rehold.as_deref().unwrap_or_default();
245 if self.lang == "ja" {
246 format!(
247 "{} に再開と回答済みですが、conductor が再び hold しました。conductor の理由: \
248 {reason}\n\n強制再キューを選ぶと、以後 conductor はこのタスクを hold できません。",
249 o.at
250 )
251 } else {
252 format!(
253 "You answered \"resume\" at {}, but the conductor held the task again. \
254 Its reason: {reason}\n\nForcing a requeue stops the conductor from \
255 holding this task again.",
256 o.at
257 )
258 }
259 }
260
261 fn choices_conductor_override(&self) -> Vec<String> {
263 if self.lang == "ja" {
264 vec![
265 "強制再キュー(conductor は再 hold 不可)".to_owned(),
266 "手動 hold のまま".to_owned(),
267 "捨ててよい".to_owned(),
268 ]
269 } else {
270 vec![
271 "force requeue (conductor must not hold again)".to_owned(),
272 "keep held (manual)".to_owned(),
273 "discard".to_owned(),
274 ]
275 }
276 }
277
278 fn summary_legacy(&self, task: &Task) -> String {
279 if self.lang == "ja" {
280 format!(
281 "hold_source が不明な保留タスク {} を確認してください",
282 task.short()
283 )
284 } else {
285 format!(
286 "held task {} has no recorded hold source - please take a look",
287 task.short()
288 )
289 }
290 }
291
292 fn why_legacy(&self) -> &'static str {
293 if self.lang == "ja" {
294 "hold_source が記録されていません。schema 3 より前のレコードか、理由が記録されなかった \
295 holdです。人が意図して止めたのか、クラッシュや強制再起動で宙に浮いただけなのか、\
296 このデータからは区別できません。"
297 } else {
298 "No hold_source was recorded - either a pre-schema-3 record, or a hold whose \
299 reason was never written down. Whether this was a deliberate hold or the \
300 leftover of a crash cannot be told from the data alone."
301 }
302 }
303
304 fn summary_manual_stale(&self, task: &Task, days: i64) -> String {
305 if self.lang == "ja" {
306 format!(
307 "{days}日間 保留されたままの手動保留タスク {} を確認してください",
308 task.short()
309 )
310 } else {
311 format!(
312 "held task {} has been on a manual hold for {days} day(s)",
313 task.short()
314 )
315 }
316 }
317
318 fn why_manual(&self) -> &'static str {
319 if self.lang == "ja" {
320 "操作者が明示的に止めた保留ですが、長期間そのままになっています。まだ止めておくか、\
321 再開するか教えてください。"
322 } else {
323 "An operator held this on purpose, but it has sat untouched for a while. Say \
324 whether to keep holding it or resume it."
325 }
326 }
327}
328
329#[derive(Debug, Clone, Copy, PartialEq, Eq)]
331enum Bucket {
332 MachineUnknown,
334 ConductorOverride,
337 Legacy,
339 ManualStale,
341}
342
343#[derive(Debug, Clone, Default)]
345pub struct Report {
346 pub resumed: Vec<String>,
349 pub asked: Vec<String>,
351 pub answered: Vec<String>,
353 pub quarantined: Vec<String>,
358}
359
360impl Report {
361 pub fn is_empty(&self) -> bool {
364 self.resumed.is_empty()
365 && self.asked.is_empty()
366 && self.answered.is_empty()
367 && self.quarantined.is_empty()
368 }
369}
370
371fn repo_for(task: &Task) -> PathBuf {
375 if task.repo.as_os_str().is_empty() {
376 PathBuf::from(".")
377 } else {
378 task.repo.clone()
379 }
380}
381
382fn is_disk_hold(task: &Task) -> bool {
393 task.hold_reason
394 .as_deref()
395 .is_some_and(crate::disk::is_gate_reason)
396}
397
398fn machine_cause_resolved(task: &Task, cfg: &Config) -> Option<bool> {
403 machine_cause_resolved_under(task, cfg, &crate::run::home())
404}
405
406fn machine_cause_resolved_under(task: &Task, cfg: &Config, home: &Path) -> Option<bool> {
407 machine_cause_resolved_with_admission(task, cfg, home, |paths| {
408 crate::retention::OutputLease::admit(home, paths, cfg.disk.min_free_bytes).is_ok()
409 })
410}
411
412fn machine_cause_resolved_with_admission(
413 task: &Task,
414 cfg: &Config,
415 home: &Path,
416 admission_available: impl Fn(&[PathBuf]) -> bool,
417) -> Option<bool> {
418 if task
419 .hold_reason
420 .as_deref()
421 .is_some_and(|r| r.starts_with(crate::cache::RESOURCE_HOLD))
422 {
423 let id = task.execution_runs().last()?;
425 let state = crate::run::RunState::load_under(id, home).ok()?;
426 let block = state.resource_block.as_ref()?;
427 if state.id != *id {
428 return None;
429 }
430 if block.kind == "disk admission"
431 && matches!(block.cause, crate::run::ResourceBlockCause::Other)
432 {
433 let mut paths = vec![repo_for(task), state.repo.clone(), state.worktree_root()];
437 paths.extend(state.candidates.iter().map(|c| c.worktree.clone()));
438 paths.extend(state.config.cache_dir());
439 paths.extend(cfg.cache_dir());
440 return Some(admission_available(&paths));
441 }
442 if block.kind != "shared build cache"
445 || !matches!(block.cause, crate::run::ResourceBlockCause::Timeout { .. })
446 {
447 return None;
448 }
449 return Some(
450 cfg.cache_dir()
451 .is_none_or(|dir| !crate::cache::in_use(home, &dir)),
452 );
453 }
454 if !is_disk_hold(task) {
455 return None;
456 }
457 let min = cfg.disk.min_free_bytes;
458 if min == 0 {
459 return Some(true);
462 }
463 Some(crate::daemon::disk_gate(&repo_for(task), cfg).is_none())
464}
465
466fn manual_is_stale(task: &Task, now: Timestamp) -> bool {
470 now.as_second() - task.updated_at.as_second() > MANUAL_STALE_AFTER.as_secs() as i64
471}
472
473fn marker_for(q: &Question) -> String {
476 format!("[triage:{}]", q.short())
477}
478
479fn already_applied(task: &Task, q: &Question) -> bool {
482 if task.triage_applied(&q.id) {
483 return true;
484 }
485 let marker = marker_for(q);
488 task.hold_reason
489 .as_deref()
490 .is_some_and(|r| r.contains(marker.as_str()))
491}
492
493fn latest_triage_question(questions: &Questions, task_id: &str) -> Option<Question> {
498 latest_question(questions, NODE, task_id)
499}
500
501fn superseded_recovery_question(questions: &Questions, q: &Question) -> bool {
505 questions
506 .root()
507 .parent()
508 .and_then(|home| Queue::at(home.join("queue")).get(&q.run).ok())
509 .is_some_and(|task| crate::repair_budget::reconcile::superseded_question(&task, q))
510}
511
512fn latest_question(questions: &Questions, node: &str, task_id: &str) -> Option<Question> {
514 questions
515 .list()
516 .into_iter()
517 .filter(|q| {
518 q.node == node && q.run == task_id && !superseded_recovery_question(questions, q)
519 })
520 .max_by(|a, b| a.asked_at.cmp(&b.asked_at).then_with(|| a.id.cmp(&b.id)))
523}
524
525#[derive(Debug, Clone, Copy, PartialEq, Eq)]
528enum AnswerAction {
529 Resume,
533 Discard,
536 KeepHeld,
541}
542
543fn interpret_answer(q: &Question) -> AnswerAction {
556 let resolution = q.resolution().unwrap_or_default();
557 match q.choices.iter().position(|c| *c == resolution) {
558 Some(0) => AnswerAction::Resume,
559 Some(2) => AnswerAction::Discard,
560 _ => AnswerAction::KeepHeld,
561 }
562}
563
564pub(crate) fn contextual_keep_held(q: &Question, label: &str) -> bool {
567 q.node == NODE
568 && q.seat == SEAT
569 && q.actions.is_empty()
570 && q.choices.get(1).is_some_and(|c| c == label)
571}
572
573fn keep_held_note(task: &Task, q: &Question, resolution: &str) -> String {
578 let marker = format!("{} operator: {resolution}", marker_for(q));
579 match task.hold_reason.as_deref() {
580 Some(existing) if !existing.is_empty() => format!("{existing}\n{marker}"),
581 _ => marker,
582 }
583}
584
585fn file_question(
589 questions: &Questions,
590 task: &Task,
591 bucket: Bucket,
592 w: &Wording,
593 now: Timestamp,
594) -> Option<Question> {
595 let (summary, why, choices) = match bucket {
596 Bucket::MachineUnknown => (
597 w.summary_machine_unknown(task),
598 w.why_machine().to_owned(),
599 w.choices3(),
600 ),
601 Bucket::ConductorOverride => (
602 w.summary_conductor_override(task),
603 task.resume_override
604 .as_ref()
605 .map(|o| w.why_conductor_override(o))
606 .unwrap_or_default(),
607 w.choices_conductor_override(),
608 ),
609 Bucket::Legacy => (
610 w.summary_legacy(task),
611 w.why_legacy().to_owned(),
612 w.choices3(),
613 ),
614 Bucket::ManualStale => {
615 let days = (now.as_second() - task.updated_at.as_second()) / (24 * 60 * 60);
616 (
617 w.summary_manual_stale(task, days),
618 w.why_manual().to_owned(),
619 w.choices2(),
620 )
621 }
622 };
623 let mut q = Question::new(
624 task.id.clone(),
625 NODE.to_owned(),
626 SEAT.to_owned(),
627 summary,
628 w.detail(task, &why),
629 choices,
630 );
631 q.triage_generation = Some(task.resumption.as_ref().map_or(0, |r| r.generation));
632 questions.put_executable(&mut q).ok()?;
633 Some(q)
634}
635
636pub(crate) fn deputy_brief(q: &Question, queue: &Queue) -> String {
641 let task = queue.get(&q.run).ok();
642 let mut s = match &task {
643 Some(t) => format!(
644 "Task {} (`magi task show {}`), \"{}\", is {}. Hold source: {}. Reason: {}.",
645 t.id,
646 t.id,
647 t.title,
648 t.status.as_str(),
649 match t.hold_source {
650 Some(HoldSource::Machine) => "machine (an automatic hold)",
651 Some(HoldSource::Manual) => "manual (an operator held it)",
652 None => "unknown (an old record, treated as the operator's)",
653 },
654 t.hold_reason
655 .as_deref()
656 .or(t.last_error.as_deref())
657 .unwrap_or("(none recorded)")
658 ),
659 None => format!("Task {} no longer exists (its record is gone).", q.run),
660 };
661 s.push_str("\n\nWhat each option does, by position:");
662 let n = q.choices.len();
663 for (i, c) in q.choices.iter().enumerate() {
664 let effect = if q.node == DEPS_NODE {
665 match i {
666 0 => "the dependency is released so the loop runs it",
667 1 => {
668 "the dependants are detached and the dependency task is DELETED; this cannot be undone"
669 }
670 2 => "the dependants are detached; the dependency stays as it is",
671 _ => "nothing is applied",
672 }
673 } else {
674 match i {
675 0 => "the task is released and requeued",
676 2 if n > 2 => "the task is DELETED; this cannot be undone",
677 _ => "the task stays held, with the answer noted",
678 }
679 };
680 s.push_str(&format!("\n- `{c}`: {effect}"));
681 }
682 s.push_str(
683 "\n\nThe daemon applies the answer on its next pass; you apply nothing. When \
684 the owner's words pick an option - including releasing a task they held \
685 themselves, or deleting it - settle it.",
686 );
687 s
688}
689
690fn quarantine_orphaned_blocked(queue: &Queue, questions: &Questions) -> Vec<String> {
708 let mut quarantined = Vec::new();
709 for listed in queue.list() {
710 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
711 continue;
712 }
713 let Ok(_claim) = queue.claim(&listed.id) else {
714 continue;
715 };
716 let Ok(mut task) = queue.get(&listed.id) else {
717 continue;
718 };
719 if task.status != TaskStatus::Blocked {
720 continue;
721 }
722 let deleted = queue.apply_deleted_blockers(&mut task, questions);
723 if !deleted.is_empty() && queue.put(&mut task).is_ok() {
724 for id in &deleted {
725 queue.note_dependency_deleted(&task, id);
726 }
727 }
728 if task.status != TaskStatus::Blocked {
729 continue;
730 }
731 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
732 if missing.is_empty() {
733 continue;
734 }
735 let language = crate::lang::of_repo(&repo_for(&task));
736 let mut cause = missing.clone();
737 cause.sort();
738 cause.dedup();
739 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
740 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
741 &task.blocked_by,
742 &missing,
743 &language,
744 )));
745 if queue.put(&mut task).is_ok() {
746 quarantined.push(task.id.clone());
747 }
748 }
749 quarantined
750}
751
752#[derive(Debug, Clone, Copy, PartialEq, Eq)]
755enum DepsAction {
756 Release,
757 Discard,
758 Detach,
759 Nothing,
761}
762
763fn interpret_deps_answer(q: &Question) -> DepsAction {
764 let resolution = q.resolution().unwrap_or_default();
765 match q.choices.iter().position(|c| *c == resolution) {
766 Some(0) => DepsAction::Release,
767 Some(1) => DepsAction::Discard,
768 Some(2) => DepsAction::Detach,
769 _ => DepsAction::Nothing,
770 }
771}
772
773fn deps_texts(w: &Wording, root: &Task, dependants: &[&Task]) -> (String, String, Vec<String>) {
775 let ja = w.lang == "ja";
776 let reason = root
777 .hold_reason
778 .as_deref()
779 .or(root.last_error.as_deref())
780 .unwrap_or(if ja {
781 "(記録なし)"
782 } else {
783 "(none recorded)"
784 });
785 let list: String = dependants
786 .iter()
787 .map(|t| format!("- {} {}\n", t.short(), t.title))
788 .collect();
789 if ja {
790 (
791 format!(
792 "{} ({}) が {} 件のタスクを止めています",
793 root.short(),
794 root.status.as_str(),
795 dependants.len()
796 ),
797 format!(
798 "task: {} ({})\ntitle: {}\n状態: {}\n理由: {reason}\n\n\
799 このタスクは自動では実行されないため、依存している次のタスクは永遠に待ち続けます:\n{list}",
800 root.id,
801 root.short(),
802 root.title,
803 root.status.as_str()
804 ),
805 vec![
806 "依存先を再開する".to_owned(),
807 "依存先を捨てる(依存タスクは切り離して実行)".to_owned(),
808 "依存タスクを切り離す(依存先はそのまま)".to_owned(),
809 ],
810 )
811 } else {
812 (
813 format!(
814 "{} ({}) is freezing {} blocked task(s)",
815 root.short(),
816 root.status.as_str(),
817 dependants.len()
818 ),
819 format!(
820 "task: {} ({})\ntitle: {}\nstatus: {}\nreason: {reason}\n\n\
821 Nothing in the loop will ever run this task, so these dependants wait \
822 forever:\n{list}",
823 root.id,
824 root.short(),
825 root.title,
826 root.status.as_str()
827 ),
828 vec![
829 "release the dependency".to_owned(),
830 "discard the dependency (dependants are detached and run)".to_owned(),
831 "detach the dependants (the dependency stays as it is)".to_owned(),
832 ],
833 )
834 }
835}
836
837fn detach_dependants(queue: &Queue, questions: &Questions, root: &str) {
841 for listed in queue.list() {
842 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| matches!(crate::queue::dependency(queue, questions, b), crate::queue::Dependency::Task(id) if id == root)) {
843 continue;
844 }
845 let Ok(_claim) = queue.claim(&listed.id) else {
846 continue;
847 };
848 let Ok(mut t) = queue.get(&listed.id) else {
849 continue;
850 };
851 if t.status != TaskStatus::Blocked {
852 continue;
853 }
854 for id in t.blocked_by.clone() {
855 if matches!(crate::queue::dependency(queue, questions, &id), crate::queue::Dependency::Task(full) if full == root)
856 {
857 t.unblock(&id);
858 }
859 }
860 let _ = queue.put(&mut t);
861 }
862}
863
864fn apply_deps_answer(queue: &Queue, questions: &Questions, q: &Question, now: Timestamp) -> bool {
868 let Ok(_claim) = queue.claim(&q.run) else {
869 return false;
870 };
871 let Ok(current) = questions.get(&q.id) else {
872 return false;
873 };
874 if current.status != QuestionStatus::Answered {
875 return false;
876 }
877 let q = ¤t;
878 let Ok(mut root) = queue.get(&q.run) else {
879 return false;
880 };
881 if root.triage_applied(&q.id) {
882 return false;
883 }
884 if root.operator_decided_at.is_some_and(|at| at > q.asked_at)
885 || q.triage_generation
886 .is_some_and(|g| g != root.resumption.as_ref().map_or(0, |r| r.generation))
887 {
888 root.mark_triage_applied(&q.id);
889 return queue.put(&mut root).is_ok();
890 }
891 match interpret_deps_answer(q) {
892 DepsAction::Release => {
893 if root.status == TaskStatus::Running {
894 return false;
895 }
896 if crate::repair_budget::reconcile::blocker(queue, &root).is_some() {
897 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(q) else {
898 return false;
899 };
900 root.decision_effects.insert(q.id.clone(), receipt);
901 root.mark_triage_applied(&q.id);
902 return queue.put(&mut root).is_ok();
903 }
904 root.answers.push(crate::queue::AnsweredQuestion {
905 question: q.summary.clone(),
906 answer: q.resolution().unwrap_or_default(),
907 });
908 if crate::base_sync_recovery::held_task(queue, &root) {
909 root.mark_triage_applied(&q.id);
910 return queue.put(&mut root).is_ok();
911 }
912 let before = root.operator_decided_at;
913 let contract =
914 crate::repair_budget::contract(&root, crate::repair_budget::Kind::Review).ok();
915 root.release_operator("triage-dependencies", Some(&q.id));
916 if let Some(contract) = contract
917 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
918 q,
919 before,
920 &contract,
921 root.resumption.as_ref().expect("release"),
922 )
923 {
924 root.decision_effects.insert(q.id.clone(), evidence);
925 }
926 root.resume_override = Some(OperatorResume {
927 question_id: q.id.clone(),
928 at: now,
929 conductor_rehold: None,
930 forced: false,
931 pinned_run: None,
932 });
933 }
934 DepsAction::Discard => {
935 detach_dependants(queue, questions, &root.id);
936 return queue.discard_triage(&root.id, questions, &q.id).is_ok();
937 }
938 DepsAction::Detach => detach_dependants(queue, questions, &root.id),
939 DepsAction::Nothing => return false,
944 }
945 root.mark_triage_applied(&q.id);
946 queue.put(&mut root).is_ok()
947}
948
949fn repair_receipt_notice(queue: &Queue, task: &Task) -> bool {
950 let Some(detail) = crate::repair_budget::reconcile::blocker(queue, task) else {
951 return false;
952 };
953 if let Some(home) = queue.root().parent() {
954 let _ = crate::task_inquiry::Outbox::at(home).file_for_occurrence(
955 task,
956 task.runs.last().cloned(),
957 "repair-receipt-recovery".into(),
958 &task
959 .repair_grants
960 .last()
961 .expect("diagnosed receipt")
962 .request
963 .request,
964 "Retained repair receipt needs reconciliation",
965 &detail,
966 );
967 }
968 true
969}
970
971fn ask_about_stuck_roots(
980 queue: &Queue,
981 questions: &Questions,
982 config_override: Option<&Path>,
983 report: &mut Report,
984) {
985 let all = questions.list();
986 let inv = crate::blockers::Inventory::load(queue, questions);
987 let mut frozen: std::collections::BTreeMap<String, Vec<String>> = Default::default();
988 for (id, roots) in inv.stuck() {
989 for root in roots {
990 if root != id {
991 frozen.entry(root).or_default().push(id.clone());
992 }
993 }
994 }
995 for mut q in all
999 .iter()
1000 .filter(|q| q.node == DEPS_NODE && q.status.open())
1001 .cloned()
1002 {
1003 if !frozen.contains_key(&q.run) {
1004 q.abandon("nothing waits on this task anymore");
1005 let _ = questions.put_executable(&mut q);
1006 }
1007 }
1008
1009 for (root_id, dependant_ids) in &frozen {
1010 let Ok(_claim) = queue.claim(root_id) else {
1011 continue;
1012 };
1013 let current = crate::blockers::Inventory::load(queue, questions);
1016 let Some(root) = current.task(root_id) else {
1017 continue;
1018 };
1019 if crate::base_sync_recovery::held_task(queue, root) {
1020 continue;
1021 }
1022 let dependants: Vec<&Task> = dependant_ids
1023 .iter()
1024 .filter_map(|d| current.task(d))
1025 .filter(|t| current.stuck_roots(t).contains(root_id))
1026 .collect();
1027 if dependants.is_empty() {
1028 continue;
1029 }
1030 if repair_receipt_notice(queue, root) {
1031 continue;
1032 }
1033 let latest = all
1034 .iter()
1035 .filter(|q| {
1036 q.node == DEPS_NODE
1037 && q.run == *root_id
1038 && !crate::repair_budget::reconcile::superseded_question(root, q)
1039 })
1040 .max_by(|a, b| a.asked_at.cmp(&b.asked_at).then_with(|| a.id.cmp(&b.id)));
1041 match latest {
1042 Some(q) if q.status.open() || awaiting_task_chat(q, root) => continue,
1043 Some(q) if q.status == QuestionStatus::Answered && !root.triage_applied(&q.id) => {
1044 continue;
1045 }
1046 Some(q) if q.status == QuestionStatus::Abandoned => {
1047 let changed = root.updated_at > q.asked_at
1048 || dependants.iter().any(|t| t.updated_at > q.asked_at);
1049 if !changed {
1050 continue;
1051 }
1052 }
1053 _ => {}
1054 }
1055 if pending_for(questions, root) {
1056 continue;
1057 }
1058 let cfg = Config::discover(&repo_for(root), config_override)
1059 .ok()
1060 .map(|(c, _)| c);
1061 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1062 let (summary, detail, choices) = deps_texts(w, root, &dependants);
1063 let mut question = Question::new(
1064 root.id.clone(),
1065 DEPS_NODE.to_owned(),
1066 SEAT.to_owned(),
1067 summary,
1068 detail,
1069 choices,
1070 );
1071 question.triage_generation = Some(root.resumption.as_ref().map_or(0, |r| r.generation));
1072 if questions.put_executable(&mut question).is_ok() {
1073 report.asked.push(root.id.clone());
1074 }
1075 }
1076}
1077
1078pub fn apply_answers(queue: &Queue, questions: &Questions) -> Vec<String> {
1081 let mut applied = Vec::new();
1082 for q in questions.list() {
1083 if q.node == DEPS_NODE
1084 && q.status == QuestionStatus::Answered
1085 && apply_deps_answer(queue, questions, &q, Timestamp::now())
1086 {
1087 applied.push(q.run.clone());
1088 }
1089 }
1090 for listed in queue.list() {
1091 let Some(q) = latest_triage_question(questions, &listed.id) else {
1092 continue;
1093 };
1094 if q.status != QuestionStatus::Answered || already_applied(&listed, &q) {
1095 continue;
1096 }
1097 let Ok(_claim) = queue.claim(&listed.id) else {
1098 continue;
1099 };
1100 let Ok(mut task) = queue.get(&listed.id) else {
1101 continue;
1102 };
1103 let Ok(q) = questions.get(&q.id) else {
1106 continue;
1107 };
1108 if q.status != QuestionStatus::Answered {
1109 continue;
1110 }
1111 if already_applied(&task, &q)
1112 || !matches!(task.status, TaskStatus::Held | TaskStatus::Queued)
1113 {
1114 continue;
1115 }
1116 let generation = task.resumption.as_ref().map_or(0, |r| r.generation);
1117 let action = interpret_answer(&q);
1118 let mut newer_decision = q.triage_generation.is_some_and(|g| g != generation)
1119 || task.resumption.as_ref().is_some_and(|r| r.at > q.asked_at)
1120 || task.held_at.is_some_and(|at| at > q.asked_at)
1121 || task.operator_decided_at.is_some_and(|at| at > q.asked_at);
1122 if matches!(action, AnswerAction::KeepHeld)
1126 && q.answered_at.is_some_and(|at| {
1127 task.operator_decided_at
1128 .is_none_or(|decision| at > decision)
1129 && task.resumption.as_ref().is_none_or(|r| at > r.at)
1130 })
1131 {
1132 newer_decision = false;
1133 }
1134 let resolution = q.resolution().unwrap_or_default();
1136 task.answers.push(crate::queue::AnsweredQuestion {
1137 question: q.summary.clone(),
1138 answer: if newer_decision {
1139 format!("{resolution} (superseded by a later operator decision)")
1140 } else {
1141 resolution
1142 },
1143 });
1144 if !newer_decision {
1145 match action {
1146 AnswerAction::Resume if crate::base_sync_recovery::held_task(queue, &task) => {
1147 }
1150 AnswerAction::Resume
1151 if crate::repair_budget::reconcile::blocker(queue, &task).is_some() =>
1152 {
1153 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(&q) else {
1154 continue;
1155 };
1156 task.decision_effects.insert(q.id.clone(), receipt);
1157 }
1158 AnswerAction::Resume => {
1159 let before = task.operator_decided_at;
1160 let contract =
1161 crate::repair_budget::contract(&task, crate::repair_budget::Kind::Review)
1162 .ok();
1163 task.release_operator("triage", Some(&q.id));
1164 if let Some(contract) = contract
1165 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1166 &q,
1167 before,
1168 &contract,
1169 task.resumption.as_ref().expect("release"),
1170 )
1171 {
1172 task.decision_effects.insert(q.id.clone(), evidence);
1173 }
1174 task.resume_override = Some(OperatorResume {
1175 question_id: q.id.clone(),
1176 at: Timestamp::now(),
1177 conductor_rehold: None,
1178 forced: true,
1179 pinned_run: None,
1180 });
1181 }
1182 AnswerAction::KeepHeld => {
1183 let note = keep_held_note(&task, &q, &q.resolution().unwrap_or_default());
1184 task.hold_manual(Some(note));
1185 }
1186 AnswerAction::Discard => {
1187 if queue.discard_triage(&task.id, questions, &q.id).is_ok() {
1188 applied.push(task.id);
1189 }
1190 continue;
1191 }
1192 }
1193 }
1194 task.mark_triage_applied(&q.id);
1195 if queue.put(&mut task).is_ok() {
1196 applied.push(task.id);
1197 }
1198 }
1199 applied
1200}
1201
1202pub fn run_once(
1220 queue: &Queue,
1221 questions: &Questions,
1222 config_override: Option<&Path>,
1223 now: Timestamp,
1224) -> Report {
1225 let mut report = Report {
1226 answered: apply_answers(queue, questions),
1227 quarantined: quarantine_orphaned_blocked(queue, questions),
1228 ..Report::default()
1229 };
1230 ask_about_stuck_roots(queue, questions, config_override, &mut report);
1231 for listed in queue.list() {
1232 if listed.status != TaskStatus::Held {
1233 continue;
1234 }
1235 let Ok(_claim) = queue.claim(&listed.id) else {
1236 continue;
1237 };
1238 let Ok(mut task) = queue.get(&listed.id) else {
1239 continue;
1240 };
1241 if task.status != TaskStatus::Held {
1245 continue;
1246 }
1247 if crate::base_sync_recovery::held_task(queue, &task) {
1248 continue;
1249 }
1250 if repair_receipt_notice(queue, &task) {
1251 continue;
1252 }
1253 if deps_pending(questions, &task) {
1257 continue;
1258 }
1259
1260 let cfg = Config::discover(&repo_for(&task), config_override)
1261 .ok()
1262 .map(|(c, _)| c);
1263 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1264
1265 if let Some(q) = latest_triage_question(questions, &task.id) {
1266 if q.status.open() || awaiting_task_chat(&q, &task) {
1267 continue;
1269 }
1270 }
1276
1277 match task.hold_source {
1278 Some(HoldSource::Machine) => {
1279 let overridden = task
1280 .resume_override
1281 .as_ref()
1282 .is_some_and(|o| o.conductor_rehold.is_some() && !o.forced);
1283 if overridden {
1284 if file_question(questions, &task, Bucket::ConductorOverride, w, now).is_some()
1285 {
1286 report.asked.push(task.id.clone());
1287 }
1288 } else if cfg.as_ref().and_then(|c| machine_cause_resolved(&task, c)) == Some(true)
1289 {
1290 task.release();
1291 if queue.put(&mut task).is_ok() {
1292 report.resumed.push(task.id.clone());
1293 }
1294 } else if file_question(questions, &task, Bucket::MachineUnknown, w, now).is_some()
1295 {
1296 report.asked.push(task.id.clone());
1297 }
1298 }
1299 None => {
1300 if file_question(questions, &task, Bucket::Legacy, w, now).is_some() {
1301 report.asked.push(task.id.clone());
1302 }
1303 }
1304 Some(HoldSource::Manual) => {
1305 if manual_is_stale(&task, now)
1306 && file_question(questions, &task, Bucket::ManualStale, w, now).is_some()
1307 {
1308 report.asked.push(task.id.clone());
1309 }
1310 }
1311 }
1312 }
1313 report
1314}
1315
1316pub fn open_question_for(questions: &Questions, task_id: &str) -> Option<Question> {
1321 latest_triage_question(questions, task_id).filter(|q| q.status.open())
1322}
1323
1324pub fn pending_for(questions: &Questions, task: &Task) -> bool {
1338 !pending_recovery_questions(questions, task).is_empty()
1339}
1340
1341pub(crate) fn pending_recovery_questions(questions: &Questions, task: &Task) -> Vec<Question> {
1344 [NODE, DEPS_NODE]
1345 .into_iter()
1346 .filter_map(|node| {
1347 latest_question(questions, node, &task.id).filter(|q| {
1348 q.status.open()
1349 || awaiting_task_chat(q, task)
1350 || (q.status == QuestionStatus::Answered
1351 && if node == NODE {
1352 !already_applied(task, q)
1353 } else {
1354 !task.triage_applied(&q.id)
1355 })
1356 })
1357 })
1358 .collect()
1359}
1360
1361fn awaiting_task_chat(q: &Question, task: &Task) -> bool {
1362 q.status != QuestionStatus::Abandoned
1363 && q.routed_chat.is_some()
1364 && !already_applied(task, q)
1365 && task.resumption.as_ref().map_or(0, |r| r.generation) == q.triage_generation.unwrap_or(0)
1366 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1367}
1368
1369fn deps_pending(questions: &Questions, task: &Task) -> bool {
1373 match latest_question(questions, DEPS_NODE, &task.id) {
1374 Some(q) if q.status.open() || awaiting_task_chat(&q, task) => true,
1375 Some(q) if q.status == QuestionStatus::Answered => !task.triage_applied(&q.id),
1376 _ => false,
1377 }
1378}
1379
1380pub fn open_task_ids(questions: &Questions) -> std::collections::BTreeSet<String> {
1385 questions
1386 .list()
1387 .into_iter()
1388 .filter(|q| {
1389 (q.node == NODE || q.node == DEPS_NODE)
1390 && q.status.open()
1391 && !superseded_recovery_question(questions, q)
1392 })
1393 .map(|q| q.run)
1394 .collect()
1395}
1396
1397#[cfg(test)]
1398mod tests {
1399 use super::*;
1400 use crate::ask::Answer;
1401 use crate::queue::Source;
1402 use jiff::SignedDuration;
1403
1404 fn store() -> (tempfile::TempDir, Queue, Questions) {
1405 let dir = tempfile::tempdir().unwrap();
1406 let q = Queue::at(dir.path().join("queue"));
1407 let s = Questions::at(dir.path().join("questions"));
1408 (dir, q, s)
1409 }
1410
1411 fn task(title: &str, repo: PathBuf) -> Task {
1412 Task::new(title.to_owned(), format!("do {title}"), repo, Source::Human)
1413 }
1414
1415 #[test]
1416 fn base_recovery_stops_suppress_generic_and_dependency_resume_offers() {
1417 let (dir, queue, questions) = store();
1418 let mut root = task("base stop", dir.path().to_path_buf());
1419 let mut run = crate::run::RunState::new(
1420 dir.path().to_path_buf(),
1421 "main".into(),
1422 "base".into(),
1423 "retained implementation".into(),
1424 Config::default(),
1425 );
1426 run.status = crate::run::RunStatus::Voting;
1427 run.base_sync = Some(crate::run::BaseSync {
1428 tip: "tip".into(),
1429 behind: 3,
1430 attempts: 4,
1431 conflict: Some("persisted exhaustion".into()),
1432 already_in: None,
1433 });
1434 run.save_under(dir.path()).unwrap();
1435 root.link_run(&run.id);
1436 root.hold_machine(Some("base stop".into()));
1437 queue.put(&mut root).unwrap();
1438 let mut dependent = task("dependent", dir.path().to_path_buf());
1439 dependent.blocked_by = vec![root.id.clone()];
1440 dependent.status = TaskStatus::Blocked;
1441 queue.put(&mut dependent).unwrap();
1442 for _ in 0..2 {
1443 let report = run_once(&queue, &questions, None, Timestamp::now());
1444 assert!(report.asked.is_empty());
1445 assert!(report.resumed.is_empty());
1446 }
1447 let q = file_question(
1450 &questions,
1451 &root,
1452 Bucket::MachineUnknown,
1453 &EN,
1454 Timestamp::now(),
1455 )
1456 .unwrap();
1457 let mut q = q;
1458 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
1459 questions.put_executable(&mut q).unwrap();
1460 apply_answers(&queue, &questions);
1461 let saved = queue.get(&root.id).unwrap();
1462 assert_eq!(saved.status, TaskStatus::Held);
1463 assert!(saved.triage_applied(&q.id));
1464 assert_eq!(saved.runs, vec![run.id]);
1465 }
1466
1467 #[test]
1468 fn resource_holds_only_auto_release_a_proven_cache_wait_timeout() {
1469 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
1470 let dir = tempfile::tempdir().unwrap();
1471 let cfg = Config::default(); assert!(cfg.cache_dir().is_none());
1473 let mut state = RunState::new(
1474 dir.path().into(),
1475 "main".into(),
1476 "base".into(),
1477 "verify".into(),
1478 cfg.clone(),
1479 );
1480 let mut task = task("verify", dir.path().into());
1481 task.runs = vec![state.id.clone()];
1482 task.hold_machine(Some("[resource] cache operation stopped".into()));
1483 for cause in [
1484 ResourceBlockCause::Other,
1485 ResourceBlockCause::Unrecorded,
1486 ResourceBlockCause::Cancelled { park: None },
1487 ] {
1488 state.resource_block = Some(ResourceBlock {
1489 node: "gate".into(),
1490 kind: "shared build cache".into(),
1491 detail: "persistent metadata or manifest error".into(),
1492 cause,
1493 at: Timestamp::now(),
1494 });
1495 state.save_under(dir.path()).unwrap();
1496 for _ in 0..3 {
1498 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1499 }
1500 }
1501 let block = state.resource_block.as_mut().unwrap();
1502 block.cause = ResourceBlockCause::Timeout {
1503 waited_secs: 10,
1504 limit_secs: 10,
1505 };
1506 state.save_under(dir.path()).unwrap();
1507 assert_eq!(
1508 machine_cause_resolved_under(&task, &cfg, dir.path()),
1509 Some(true)
1510 );
1511 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
1512 state.save_under(dir.path()).unwrap();
1513 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1514 task.runs = vec!["unreadable-run".into()];
1515 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
1516 }
1517
1518 #[cfg(unix)]
1519 #[test]
1520 fn disk_admission_holds_recheck_live_reservations_and_the_floor() {
1521 use crate::retention::OutputLease;
1522 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
1523 let temp = tempfile::tempdir().unwrap();
1524 let home = temp.path().canonicalize().unwrap();
1525 let mut cfg = Config::default();
1526 cfg.graph.worktree_root = Some(home.join("worktrees"));
1527 cfg.disk.min_free_bytes = 100;
1528 let cache = home.join("cache");
1529 cfg.verify.gate = vec![format!(
1530 "CARGO_TARGET_DIR='{}' cargo build",
1531 cache.display()
1532 )];
1533 let mut state = RunState::new(
1534 home.clone(),
1535 "main".into(),
1536 "base".into(),
1537 "verify".into(),
1538 cfg.clone(),
1539 );
1540 state.resource_block = Some(ResourceBlock {
1541 node: "gate".into(),
1542 kind: "disk admission".into(),
1543 detail: "writer headroom unavailable".into(),
1544 cause: ResourceBlockCause::Other,
1545 at: Timestamp::now(),
1546 });
1547 state.save_under(&home).unwrap();
1548 let mut task = task("verify", home.clone());
1549 task.runs = vec![state.id.clone()];
1550 task.hold_machine(Some("[resource] gate: disk admission: blocked".into()));
1551 let gib = 1024 * 1024 * 1024;
1552 let free = std::cell::Cell::new(100 + gib - 1);
1553 let check = || {
1554 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths| {
1555 assert!(
1556 paths.contains(&cache),
1557 "the cache volume must also be checked"
1558 );
1559 assert!(paths.contains(&state.worktree_root()));
1560 OutputLease::admit_with(&home, paths, cfg.disk.min_free_bytes, |_| Ok(free.get()))
1561 .is_ok()
1562 })
1563 };
1564 assert_eq!(check(), Some(false), "the unchanged floor still applies");
1565 free.set(100 + gib);
1566 let writer = OutputLease::admit_with(
1567 &home,
1568 std::slice::from_ref(&home),
1569 cfg.disk.min_free_bytes,
1570 |_| Ok(free.get()),
1571 )
1572 .unwrap();
1573 assert_eq!(
1574 check(),
1575 Some(false),
1576 "df alone cannot resolve a live reservation"
1577 );
1578 drop(writer);
1579 assert_eq!(
1580 check(),
1581 Some(true),
1582 "releasing headroom resolves the machine hold"
1583 );
1584 assert_eq!(
1585 check(),
1586 Some(true),
1587 "the probe must release its temporary reservation"
1588 );
1589
1590 let namespace = home.join("disk-admission");
1592 let saved = home.join("saved-admission");
1593 std::fs::rename(&namespace, &saved).unwrap();
1594 std::os::unix::fs::symlink(&saved, &namespace).unwrap();
1595 assert_eq!(check(), Some(false));
1596 std::fs::remove_file(&namespace).unwrap();
1597 std::fs::rename(saved, namespace).unwrap();
1598 assert_eq!(check(), Some(true));
1599
1600 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Cancelled { park: None };
1601 state.save_under(&home).unwrap();
1602 assert_eq!(
1603 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
1604 "a cancelled operation must remain held"
1605 )),
1606 None
1607 );
1608 state.resource_block.as_mut().unwrap().kind = "incremental cache admission".into();
1609 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Other;
1610 state.save_under(&home).unwrap();
1611 assert_eq!(
1612 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
1613 "unproven incremental caches must remain held"
1614 )),
1615 None
1616 );
1617 task.runs.push("unreadable-run".into());
1618 assert_eq!(
1619 machine_cause_resolved_with_admission(&task, &cfg, &home, |_| panic!(
1620 "missing evidence must not release a hold"
1621 )),
1622 None
1623 );
1624 }
1625
1626 fn gate_disabled_config(dir: &std::path::Path) -> PathBuf {
1630 let config = dir.join("magi.toml");
1631 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1632 config
1633 }
1634
1635 #[test]
1636 fn resumption_later_keep_held_answer_wins_but_an_older_answer_cannot_undo_release() {
1637 for source in ["cli", "web"] {
1638 for answer_after_release in [true, false] {
1639 let (dir, queue, questions) = store();
1640 let mut t = task("recovery", dir.path().join("repo"));
1641 t.hold_machine(Some("inspect remote first".into()));
1642 queue.put(&mut t).unwrap();
1643 let mut q = file_question(
1644 &questions,
1645 &t,
1646 Bucket::MachineUnknown,
1647 &JA,
1648 Timestamp::now(),
1649 )
1650 .unwrap();
1651 if !answer_after_release {
1652 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
1653 questions.put(&mut q).unwrap();
1654 }
1655 t.release_operator(source, None);
1656 queue.put(&mut t).unwrap();
1657 if answer_after_release {
1658 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
1659 questions.put(&mut q).unwrap();
1660 }
1661 apply_answers(&queue, &questions);
1662 let after = queue.get(&t.id).unwrap();
1663 assert!(after.triage_applied(&q.id));
1664 assert_eq!(after.operator_held(), answer_after_release);
1665 assert_eq!(
1666 after.status,
1667 if answer_after_release {
1668 TaskStatus::Held
1669 } else {
1670 TaskStatus::Queued
1671 }
1672 );
1673 assert!(apply_answers(&queue, &questions).is_empty());
1674 }
1675 }
1676 }
1677
1678 #[test]
1679 fn explicit_resumption_applies_legacy_queued_triage_once_and_preserves_later_holds() {
1680 let (dir, queue, questions) = store();
1681 let mut t = task("recover", dir.path().join("repo"));
1682 t.hold_machine(Some("inspect first".into()));
1683 queue.put(&mut t).unwrap();
1684 let mut q = file_question(
1685 &questions,
1686 &t,
1687 Bucket::MachineUnknown,
1688 &JA,
1689 Timestamp::now(),
1690 )
1691 .unwrap();
1692 assert!(q.actions.is_empty());
1693 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
1694 questions.put(&mut q).unwrap();
1695 t.release();
1697 t.instruction = "Reuse approved PR head".into();
1698 queue.put(&mut t).unwrap();
1699 assert_eq!(apply_answers(&queue, &questions), [t.id.clone()]);
1700 let mut after = queue.get(&t.id).unwrap();
1701 assert_eq!(
1702 after.resumption.as_ref().unwrap().question_id.as_deref(),
1703 Some(q.id.as_str())
1704 );
1705 assert_eq!(
1706 after.resumption.as_ref().unwrap().instruction,
1707 t.instruction
1708 );
1709 assert_eq!(after.answers[0].answer, JA.resume);
1710 assert!(after.triage_applied(&q.id));
1711 assert!(apply_answers(&Queue::at(dir.path().join("queue")), &questions).is_empty());
1712 after.hold_machine(Some("new blocker".into()));
1713 queue.put(&mut after).unwrap();
1714 let mut late = file_question(
1715 &questions,
1716 &after,
1717 Bucket::MachineUnknown,
1718 &EN,
1719 Timestamp::now(),
1720 )
1721 .unwrap();
1722 after.hold_manual(Some("keep held now".into()));
1723 queue.put(&mut after).unwrap();
1724 late.answer(Answer::Choice(late.choices[0].clone()))
1725 .unwrap();
1726 questions.put(&mut late).unwrap();
1727 apply_answers(&queue, &questions);
1728 let held = queue.get(&t.id).unwrap();
1729 assert!(held.operator_held());
1730 assert_eq!(held.hold_reason.as_deref(), Some("keep held now"));
1731 assert_eq!(held.resumption.as_ref().unwrap().generation, 1);
1732 }
1733
1734 #[test]
1735 fn a_resolved_machine_hold_is_requeued_automatically() {
1736 let (dir, q, questions) = store();
1737 let config = gate_disabled_config(dir.path());
1738 let mut t = task("disk pressure", dir.path().join("repo"));
1739 t.hold_machine(Some(
1740 "not enough free space to start a run: 10 bytes free, 100 required by \
1741 `[disk] min_free_bytes`"
1742 .to_owned(),
1743 ));
1744 q.put(&mut t).unwrap();
1745
1746 let report = run_once(&q, &questions, Some(&config), Timestamp::now());
1747 assert_eq!(report.resumed, [t.id.clone()]);
1748 assert!(report.asked.is_empty());
1749
1750 let back = q.get(&t.id).unwrap();
1751 assert_eq!(back.status, TaskStatus::Queued);
1752 assert!(back.hold_source.is_none());
1753 assert!(questions.list().is_empty(), "nothing needed asking");
1754 }
1755
1756 #[test]
1757 fn a_machine_hold_with_no_recognised_cause_gets_one_question_not_two() {
1758 let (dir, q, questions) = store();
1759 let mut t = task("gate went red", dir.path().join("repo"));
1760 t.hold_machine(Some("gate red".to_owned()));
1761 q.put(&mut t).unwrap();
1762
1763 let first = run_once(&q, &questions, None, Timestamp::now());
1764 assert_eq!(first.asked, [t.id.clone()]);
1765 assert!(first.resumed.is_empty());
1766
1767 let open: Vec<_> = questions
1768 .list()
1769 .into_iter()
1770 .filter(|q| q.status.open())
1771 .collect();
1772 assert_eq!(open.len(), 1);
1773 assert_eq!(open[0].run, t.id);
1774 assert_eq!(open[0].node, NODE);
1775 assert_eq!(open[0].choices.len(), 3);
1776
1777 let second = run_once(&q, &questions, None, Timestamp::now());
1779 assert!(second.asked.is_empty());
1780 assert_eq!(
1781 questions
1782 .list()
1783 .into_iter()
1784 .filter(|q| q.status.open())
1785 .count(),
1786 1
1787 );
1788 }
1789
1790 #[test]
1791 fn a_legacy_hold_with_no_recorded_source_gets_exactly_one_question() {
1792 let (dir, q, questions) = store();
1793 let mut t = task("schema 1 record", dir.path().join("repo"));
1794 t.status = TaskStatus::Held;
1795 assert!(t.hold_source.is_none(), "the case this test is about");
1796 q.put(&mut t).unwrap();
1797
1798 let first = run_once(&q, &questions, None, Timestamp::now());
1799 assert_eq!(first.asked, [t.id.clone()]);
1800
1801 let second = run_once(&q, &questions, None, Timestamp::now());
1802 assert!(
1803 second.asked.is_empty(),
1804 "the same legacy hold must not be asked about twice"
1805 );
1806 assert_eq!(
1807 questions
1808 .list()
1809 .into_iter()
1810 .filter(|q| q.status.open())
1811 .count(),
1812 1
1813 );
1814 }
1815
1816 #[test]
1817 fn dependency_resolution_detaches_legacy_short_refs_without_dropping_other_waits() {
1818 let (dir, queue, questions) = store();
1819 let mut root = task("root", dir.path().join("repo"));
1820 root.id = "20261013-193611-ca5b".into();
1821 root.hold_machine(Some("needs owner decision".into()));
1822 queue.put(&mut root).unwrap();
1823 let mut other = task("other", dir.path().join("repo"));
1824 other.id = "20261013-192213-feb2".into();
1825 queue.put(&mut other).unwrap();
1826 let mut waiting = task("waiting", dir.path().join("repo"));
1827 waiting.block(vec!["ca5b".into(), other.id.clone()], None);
1828 queue.put(&mut waiting).unwrap();
1829 detach_dependants(&queue, &questions, &root.id);
1830 let after = queue.get(&waiting.id).unwrap();
1831 assert_eq!(after.status, TaskStatus::Blocked);
1832 assert_eq!(after.blocked_by, [other.id]);
1833 }
1834
1835 #[test]
1836 fn a_manual_hold_is_never_auto_resumed() {
1837 let (dir, q, questions) = store();
1838 let mut t = task("operator stopped this", dir.path().join("repo"));
1839 t.hold_manual(Some("waiting on a decision".to_owned()));
1840 q.put(&mut t).unwrap();
1841
1842 let report = run_once(&q, &questions, None, Timestamp::now());
1843 assert!(report.resumed.is_empty());
1844 assert!(report.asked.is_empty());
1846
1847 let back = q.get(&t.id).unwrap();
1848 assert_eq!(back.status, TaskStatus::Held);
1849 assert_eq!(back.hold_source, Some(HoldSource::Manual));
1850 assert!(questions.list().is_empty());
1851 }
1852
1853 #[test]
1854 fn a_blocked_task_on_a_deleted_dependency_is_held_and_asked_about_in_one_pass() {
1855 let (dir, q, questions) = store();
1860 let mut still_going = task("still valid", dir.path().join("repo"));
1861 q.put(&mut still_going).unwrap();
1862
1863 let mut t = task("orphaned", dir.path().join("repo"));
1864 t.block(
1865 vec!["20260101-000000-gone".to_owned(), still_going.id.clone()],
1866 Some("waits on both".to_owned()),
1867 );
1868 q.put(&mut t).unwrap();
1869
1870 let report = run_once(&q, &questions, None, Timestamp::now());
1871 assert_eq!(report.quarantined, [t.id.clone()]);
1872 assert_eq!(
1873 report.asked,
1874 [t.id.clone()],
1875 "the fresh machine hold must earn a question in the same pass"
1876 );
1877
1878 let after = q.get(&t.id).unwrap();
1879 assert_eq!(after.status, TaskStatus::Held);
1880 assert_eq!(after.hold_source, Some(HoldSource::Machine));
1881 assert!(after.blocked_by.is_empty());
1882
1883 assert!(!is_disk_hold(&after));
1886
1887 let open: Vec<_> = questions
1888 .list()
1889 .into_iter()
1890 .filter(|q| q.status.open())
1891 .collect();
1892 assert_eq!(open.len(), 1);
1893 assert_eq!(open[0].run, t.id);
1894
1895 let second = run_once(&q, &questions, None, Timestamp::now());
1897 assert!(second.quarantined.is_empty());
1898 assert!(second.asked.is_empty());
1899 }
1900
1901 fn stuck_chain(q: &Queue, dir: &std::path::Path) -> (Task, Task, Task) {
1904 let mut root = task("root", dir.join("repo"));
1905 root.hold_manual(Some("waiting".to_owned()));
1906 q.put(&mut root).unwrap();
1907 let mut mid = task("mid", dir.join("repo"));
1908 mid.block(vec![root.id.clone()], None);
1909 q.put(&mut mid).unwrap();
1910 let mut leaf = task("leaf", dir.join("repo"));
1911 leaf.block(vec![mid.id.clone()], None);
1912 q.put(&mut leaf).unwrap();
1913 (root, mid, leaf)
1914 }
1915
1916 fn deps_questions(questions: &Questions) -> Vec<Question> {
1917 questions
1918 .list()
1919 .into_iter()
1920 .filter(|q| q.node == DEPS_NODE)
1921 .collect()
1922 }
1923
1924 fn answer_deps(questions: &Questions, choice: usize) {
1925 let mut asked = deps_questions(questions).remove(0);
1926 let c = asked.choices[choice].clone();
1927 asked.answer(Answer::Choice(c)).unwrap();
1928 questions.put(&mut asked).unwrap();
1929 }
1930
1931 #[test]
1932 fn a_stuck_chain_earns_one_question_naming_every_dependant_and_is_not_reasked() {
1933 let (dir, q, questions) = store();
1934 let (root, mid, leaf) = stuck_chain(&q, dir.path());
1935
1936 let first = run_once(&q, &questions, None, Timestamp::now());
1937 assert_eq!(first.asked, std::slice::from_ref(&root.id));
1938 let asked = deps_questions(&questions);
1939 assert_eq!(asked.len(), 1, "one question per root, not per dependant");
1940 assert_eq!(asked[0].run, root.id);
1941 assert_eq!(asked[0].choices.len(), 3);
1942 assert!(asked[0].detail.contains(mid.short()), "{}", asked[0].detail);
1943 assert!(
1944 asked[0].detail.contains(leaf.short()),
1945 "{}",
1946 asked[0].detail
1947 );
1948
1949 let second = run_once(&q, &questions, None, Timestamp::now());
1950 assert!(second.asked.is_empty());
1951 assert_eq!(deps_questions(&questions).len(), 1);
1952 assert!(pending_for(&questions, &q.get(&root.id).unwrap()));
1953 }
1954
1955 #[test]
1956 fn a_stuck_root_with_a_machine_hold_gets_only_the_dependency_question() {
1957 let (dir, q, questions) = store();
1958 let mut root = task("root", dir.path().join("repo"));
1959 root.hold_machine(Some("gate red".to_owned()));
1960 q.put(&mut root).unwrap();
1961 let mut dep = task("dep", dir.path().join("repo"));
1962 dep.block(vec![root.id.clone()], None);
1963 q.put(&mut dep).unwrap();
1964
1965 run_once(&q, &questions, None, Timestamp::now());
1966 run_once(&q, &questions, None, Timestamp::now());
1967 assert_eq!(deps_questions(&questions).len(), 1);
1968 assert_eq!(
1969 questions.list().len(),
1970 1,
1971 "no second question about the hold"
1972 );
1973 }
1974
1975 #[test]
1976 fn a_dependency_that_can_still_run_asks_nothing() {
1977 let (dir, q, questions) = store();
1978 let mut live = task("live", dir.path().join("repo"));
1979 q.put(&mut live).unwrap();
1980 let mut dep = task("dep", dir.path().join("repo"));
1981 dep.block(vec![live.id.clone()], None);
1982 q.put(&mut dep).unwrap();
1983 let report = run_once(&q, &questions, None, Timestamp::now());
1984 assert!(report.asked.is_empty());
1985 assert!(questions.list().is_empty());
1986 }
1987
1988 #[test]
1989 fn answering_release_requeues_the_root_and_is_not_reasked() {
1990 let (dir, q, questions) = store();
1991 let (root, _mid, _leaf) = stuck_chain(&q, dir.path());
1992 run_once(&q, &questions, None, Timestamp::now());
1993 answer_deps(&questions, 0);
1994
1995 let report = run_once(&q, &questions, None, Timestamp::now());
1996 assert_eq!(report.answered, std::slice::from_ref(&root.id));
1997 assert!(report.asked.is_empty(), "applying must not re-ask");
1998 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Queued);
1999 let again = run_once(&q, &questions, None, Timestamp::now());
2000 assert!(again.asked.is_empty() && again.answered.is_empty());
2001 assert_eq!(deps_questions(&questions).len(), 1);
2002 }
2003
2004 #[test]
2005 fn answering_detach_frees_the_direct_dependant_and_keeps_the_root() {
2006 let (dir, q, questions) = store();
2007 let (root, mid, leaf) = stuck_chain(&q, dir.path());
2008 run_once(&q, &questions, None, Timestamp::now());
2009 answer_deps(&questions, 2);
2010
2011 let report = run_once(&q, &questions, None, Timestamp::now());
2012 assert!(report.asked.is_empty());
2013 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
2014 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
2015 let leaf = q.get(&leaf.id).unwrap();
2016 assert_eq!(leaf.status, TaskStatus::Blocked, "still waits on mid");
2017 assert_eq!(leaf.blocked_by, std::slice::from_ref(&mid.id));
2018 }
2019
2020 #[test]
2021 fn a_dependant_that_can_progress_through_another_blocker_is_not_stuck() {
2022 let (dir, q, questions) = store();
2023 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
2024 let mut live = task("live", dir.path().join("repo"));
2025 q.put(&mut live).unwrap();
2026 let mut both = q.get(&mid.id).unwrap();
2027 both.block(vec![root.id.clone(), live.id.clone()], None);
2028 q.put(&mut both).unwrap();
2029 assert!(
2031 run_once(&q, &questions, None, Timestamp::now())
2032 .asked
2033 .is_empty()
2034 );
2035 }
2036
2037 #[test]
2038 fn answering_discard_detaches_then_removes_the_root() {
2039 let (dir, q, questions) = store();
2040 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
2041 run_once(&q, &questions, None, Timestamp::now());
2042 answer_deps(&questions, 1);
2043
2044 let report = run_once(&q, &questions, None, Timestamp::now());
2045 assert!(q.get(&root.id).is_err());
2046 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
2047 assert!(
2048 report.asked.is_empty(),
2049 "no per-dependant quarantine question"
2050 );
2051 assert!(report.quarantined.is_empty());
2052 }
2053
2054 #[test]
2055 fn an_unmatched_deps_answer_changes_nothing_and_is_not_reasked() {
2056 let (dir, q, questions) = store();
2057 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
2058 run_once(&q, &questions, None, Timestamp::now());
2059 let mut asked = deps_questions(&questions).remove(0);
2060 asked.choices.clear();
2061 asked.answer(Answer::Text("dunno".to_owned())).unwrap();
2062 questions.put(&mut asked).unwrap();
2063
2064 for _ in 0..2 {
2065 let report = run_once(&q, &questions, None, Timestamp::now());
2066 assert!(report.asked.is_empty() && report.answered.is_empty());
2067 }
2068 assert_eq!(deps_questions(&questions).len(), 1);
2069 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
2070 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Blocked);
2071 }
2072
2073 #[test]
2074 fn a_dependency_cycle_terminates_and_is_asked_about_once() {
2075 let (dir, q, questions) = store();
2076 let mut a = task("a", dir.path().join("repo"));
2077 let mut b = task("b", dir.path().join("repo"));
2078 a.block(vec![b.id.clone()], None);
2079 b.block(vec![a.id.clone()], None);
2080 q.put(&mut a).unwrap();
2081 q.put(&mut b).unwrap();
2082
2083 let first = run_once(&q, &questions, None, Timestamp::now());
2084 assert_eq!(first.asked.len(), 1);
2085 let second = run_once(&q, &questions, None, Timestamp::now());
2086 assert!(second.asked.is_empty());
2087 assert_eq!(deps_questions(&questions).len(), 1);
2088 }
2089
2090 #[test]
2091 fn a_missing_dependency_is_still_quarantined_not_asked_about_as_stuck() {
2092 let (dir, q, questions) = store();
2093 let mut t = task("orphan", dir.path().join("repo"));
2094 t.block(vec!["20260101-000000-gone".to_owned()], None);
2095 q.put(&mut t).unwrap();
2096 let report = run_once(&q, &questions, None, Timestamp::now());
2097 assert_eq!(report.quarantined, [t.id.clone()]);
2098 assert!(deps_questions(&questions).is_empty());
2099 }
2100
2101 #[test]
2102 fn a_stale_manual_hold_earns_a_two_choice_question() {
2103 let (dir, q, questions) = store();
2104 let mut t = task("been sitting a while", dir.path().join("repo"));
2105 t.hold_manual(Some("waiting on a decision".to_owned()));
2106 q.put(&mut t).unwrap();
2107 let mut back = q.get(&t.id).unwrap();
2109 back.updated_at = Timestamp::now() - SignedDuration::new(8 * 24 * 60 * 60, 0);
2110 std::fs::write(
2111 q.path_of(&back.id),
2112 serde_json::to_string_pretty(&back).unwrap(),
2113 )
2114 .unwrap();
2115
2116 let report = run_once(&q, &questions, None, Timestamp::now());
2117 assert_eq!(report.asked, [t.id.clone()]);
2118 let open: Vec<_> = questions
2119 .list()
2120 .into_iter()
2121 .filter(|q| q.status.open())
2122 .collect();
2123 assert_eq!(open.len(), 1);
2124 assert_eq!(open[0].choices.len(), 2);
2125 }
2126
2127 #[test]
2128 fn answering_resume_releases_the_task() {
2129 let (dir, q, questions) = store();
2130 let mut t = task("gate went red", dir.path().join("repo"));
2131 t.hold_machine(Some("gate red".to_owned()));
2132 q.put(&mut t).unwrap();
2133 run_once(&q, &questions, None, Timestamp::now());
2134
2135 let mut asked = questions
2136 .list()
2137 .into_iter()
2138 .find(|q| q.run == t.id)
2139 .unwrap();
2140 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2141 questions.put(&mut asked).unwrap();
2142
2143 let report = run_once(&q, &questions, None, Timestamp::now());
2144 assert_eq!(report.answered, [t.id.clone()]);
2145 let back = q.get(&t.id).unwrap();
2146 assert_eq!(back.status, TaskStatus::Queued);
2147 assert!(back.hold_source.is_none());
2148 }
2149
2150 fn resumed_then_failed(q: &Queue, questions: &Questions, dir: &std::path::Path) -> Task {
2152 let mut t = task("gate went red", dir.join("repo"));
2153 t.hold_machine(Some("gate red".to_owned()));
2154 q.put(&mut t).unwrap();
2155 run_once(q, questions, None, Timestamp::now());
2156 let mut asked = questions
2157 .list()
2158 .into_iter()
2159 .find(|q| q.run == t.id)
2160 .unwrap();
2161 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2162 questions.put(&mut asked).unwrap();
2163
2164 let report = run_once(q, questions, None, Timestamp::now());
2165 assert_eq!(report.answered, [t.id.clone()]);
2166 let mut back = q.get(&t.id).unwrap();
2167 assert_eq!(back.status, TaskStatus::Queued);
2168 assert_eq!(back.attempts, 0);
2169
2170 back.start("run-1".to_owned());
2171 back.fail("rebase conflict", 1);
2172 assert_eq!(back.status, TaskStatus::Held);
2173 q.put(&mut back).unwrap();
2174 back
2175 }
2176
2177 #[test]
2178 fn a_resume_answer_is_applied_once_and_a_new_machine_hold_is_asked_about() {
2179 let (dir, q, questions) = store();
2180 let t = resumed_then_failed(&q, &questions, dir.path());
2181
2182 let report = run_once(&q, &questions, None, Timestamp::now());
2183 assert!(report.answered.is_empty(), "the old answer must not replay");
2184 assert_eq!(report.asked, std::slice::from_ref(&t.id));
2185 let back = q.get(&t.id).unwrap();
2186 assert_eq!(back.status, TaskStatus::Held);
2187 assert_eq!(
2188 questions
2189 .list()
2190 .into_iter()
2191 .filter(|q| q.status.open())
2192 .count(),
2193 1
2194 );
2195 }
2196
2197 #[test]
2198 fn a_manual_hold_placed_after_a_resume_is_not_undone_by_the_old_answer() {
2199 let (dir, q, questions) = store();
2200 let mut t = resumed_then_failed(&q, &questions, dir.path());
2201 t.hold_manual(Some("operator stopped this".to_owned()));
2202 q.put(&mut t).unwrap();
2203 let before = questions.list().len();
2204
2205 let report = run_once(&q, &questions, None, Timestamp::now());
2206 assert!(report.answered.is_empty());
2207 assert!(report.asked.is_empty());
2208 let back = q.get(&t.id).unwrap();
2209 assert_eq!(back.status, TaskStatus::Held);
2210 assert_eq!(back.hold_source, Some(HoldSource::Manual));
2211 assert_eq!(questions.list().len(), before);
2212 }
2213
2214 #[test]
2215 fn answering_not_yet_keeps_it_held_as_a_manual_hold_and_does_not_reapply() {
2216 let (dir, q, questions) = store();
2217 let mut t = task("gate went red", dir.path().join("repo"));
2218 t.hold_machine(Some("gate red".to_owned()));
2219 q.put(&mut t).unwrap();
2220 run_once(&q, &questions, None, Timestamp::now());
2221
2222 let mut asked = questions
2223 .list()
2224 .into_iter()
2225 .find(|q| q.run == t.id)
2226 .unwrap();
2227 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
2228 questions.put(&mut asked).unwrap();
2229
2230 let report = run_once(&q, &questions, None, Timestamp::now());
2231 assert_eq!(report.answered, [t.id.clone()]);
2232 let back = q.get(&t.id).unwrap();
2233 assert_eq!(back.status, TaskStatus::Held);
2234 assert_eq!(back.hold_source, Some(HoldSource::Manual));
2235 assert!(
2236 back.hold_reason
2237 .as_deref()
2238 .is_some_and(|r| r.contains("gate red")),
2239 "the original cause must survive a \"not yet\" answer, not just the \
2240 triage marker: {:?}",
2241 back.hold_reason
2242 );
2243
2244 let third = run_once(&q, &questions, None, Timestamp::now());
2248 assert!(third.answered.is_empty());
2249 assert!(third.asked.is_empty());
2250 }
2251
2252 #[test]
2253 fn answering_discard_removes_the_task_entirely() {
2254 let (dir, q, questions) = store();
2255 let mut t = task("gate went red", dir.path().join("repo"));
2256 t.hold_machine(Some("gate red".to_owned()));
2257 q.put(&mut t).unwrap();
2258 run_once(&q, &questions, None, Timestamp::now());
2259
2260 let mut asked = questions
2261 .list()
2262 .into_iter()
2263 .find(|q| q.run == t.id)
2264 .unwrap();
2265 asked.answer(Answer::Choice(EN.discard.to_owned())).unwrap();
2266 questions.put(&mut asked).unwrap();
2267
2268 let report = run_once(&q, &questions, None, Timestamp::now());
2269 assert_eq!(report.answered, [t.id.clone()]);
2270 assert!(
2271 q.get(&t.id).is_err(),
2272 "\"discard it\" (捨ててよい) must actually discard the task, not \
2273 just leave it sitting held forever"
2274 );
2275 }
2276
2277 #[test]
2278 fn an_answer_is_read_by_its_position_in_choices_not_by_the_callers_current_language() {
2279 let (dir, q, questions) = store();
2281 let ja_config = dir.path().join("ja.toml");
2282 std::fs::write(&ja_config, "[graph]\nlanguage = \"ja\"\n").unwrap();
2283 let mut t = task("gate went red", dir.path().join("repo"));
2284 t.hold_machine(Some("gate red".to_owned()));
2285 q.put(&mut t).unwrap();
2286 run_once(&q, &questions, Some(&ja_config), Timestamp::now());
2287
2288 let mut asked = questions
2289 .list()
2290 .into_iter()
2291 .find(|q| q.run == t.id)
2292 .unwrap();
2293 assert_eq!(asked.choices[0], JA.resume, "filed in Japanese");
2294 asked.answer(Answer::Choice(JA.resume.to_owned())).unwrap();
2295 questions.put(&mut asked).unwrap();
2296
2297 let en_config = dir.path().join("en.toml");
2302 std::fs::write(&en_config, "[graph]\nlanguage = \"en\"\n").unwrap();
2303 let report = run_once(&q, &questions, Some(&en_config), Timestamp::now());
2304 assert_eq!(report.answered, [t.id.clone()]);
2305 let back = q.get(&t.id).unwrap();
2306 assert_eq!(
2307 back.status,
2308 TaskStatus::Queued,
2309 "a resume answer must resume the task regardless of which \
2310 language it is read back in"
2311 );
2312 }
2313
2314 #[test]
2315 fn a_question_falls_back_to_last_error_when_hold_reason_was_never_set() {
2316 let (dir, q, questions) = store();
2321 let mut t = task("kept failing the gate", dir.path().join("repo"));
2322 t.start("run-1".to_owned());
2323 t.fail("gate red three times running", 1);
2324 assert_eq!(t.status, TaskStatus::Held);
2325 t.hold_reason = None; q.put(&mut t).unwrap();
2327
2328 run_once(&q, &questions, None, Timestamp::now());
2329 let asked = questions
2330 .list()
2331 .into_iter()
2332 .find(|q| q.run == t.id)
2333 .unwrap();
2334 assert!(
2335 asked.detail.contains("gate red three times running"),
2336 "the question must surface `last_error` when there is no \
2337 `hold_reason` to show instead: {}",
2338 asked.detail
2339 );
2340 }
2341
2342 #[test]
2343 fn open_question_for_and_open_task_ids_reflect_only_what_is_still_waiting() {
2344 let (dir, q, questions) = store();
2345 let mut t = task("gate went red", dir.path().join("repo"));
2346 t.hold_machine(Some("gate red".to_owned()));
2347 q.put(&mut t).unwrap();
2348
2349 assert!(open_question_for(&questions, &t.id).is_none());
2350 assert!(!open_task_ids(&questions).contains(&t.id));
2351
2352 run_once(&q, &questions, None, Timestamp::now());
2353 assert!(open_question_for(&questions, &t.id).is_some());
2354 assert!(open_task_ids(&questions).contains(&t.id));
2355
2356 let mut asked = questions
2357 .list()
2358 .into_iter()
2359 .find(|q| q.run == t.id)
2360 .unwrap();
2361 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
2362 questions.put(&mut asked).unwrap();
2363
2364 assert!(
2365 open_question_for(&questions, &t.id).is_none(),
2366 "an answered question is no longer open"
2367 );
2368 assert!(!open_task_ids(&questions).contains(&t.id));
2369 }
2370
2371 #[test]
2372 fn pending_for_stays_true_between_an_answer_and_the_next_run_once_pass() {
2373 let (dir, q, questions) = store();
2379 let mut t = task("gate went red", dir.path().join("repo"));
2380 t.hold_machine(Some("gate red".to_owned()));
2381 q.put(&mut t).unwrap();
2382
2383 assert!(!pending_for(&questions, &t));
2384
2385 run_once(&q, &questions, None, Timestamp::now());
2386 let held = q.get(&t.id).unwrap();
2387 assert!(pending_for(&questions, &held), "still waiting on an answer");
2388
2389 let mut asked = questions
2390 .list()
2391 .into_iter()
2392 .find(|q| q.run == t.id)
2393 .unwrap();
2394 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
2395 questions.put(&mut asked).unwrap();
2396 assert!(!asked.status.open());
2397
2398 let still_held = q.get(&t.id).unwrap();
2400 assert!(
2401 pending_for(&questions, &still_held),
2402 "answered but not yet applied is still pending"
2403 );
2404
2405 run_once(&q, &questions, None, Timestamp::now());
2406 let after = q.get(&t.id).unwrap();
2407 assert!(
2408 !pending_for(&questions, &after),
2409 "the answer is applied now, nothing left pending"
2410 );
2411 }
2412}