1use std::path::{Path, PathBuf};
86use std::time::Duration;
87
88use jiff::Timestamp;
89
90use crate::ask::{Question, QuestionStatus, Questions};
91use crate::config::Config;
92use crate::queue::{HoldSource, OperatorResume, Queue, Task, TaskStatus};
93
94pub const NODE: &str = "triage";
97
98pub const DEPS_NODE: &str = "triage-deps";
102
103const SEAT: &str = "triage";
108
109const MANUAL_STALE_AFTER: Duration = Duration::from_secs(7 * 24 * 60 * 60);
121
122struct Wording {
126 lang: &'static str,
127 resume: &'static str,
128 wait: &'static str,
129 discard: &'static str,
130}
131
132const EN: Wording = Wording {
133 lang: "en",
134 resume: "resume it",
135 wait: "not yet",
136 discard: "discard it",
137};
138
139const JA: Wording = Wording {
140 lang: "ja",
141 resume: "再開してよい",
142 wait: "まだ待って",
143 discard: "捨ててよい",
144};
145
146fn wording(language: &str) -> &'static Wording {
149 if crate::lang::is_japanese(language) {
150 &JA
151 } else {
152 &EN
153 }
154}
155
156impl Wording {
157 fn choices3(&self) -> Vec<String> {
158 vec![
159 self.resume.to_owned(),
160 self.wait.to_owned(),
161 self.discard.to_owned(),
162 ]
163 }
164
165 fn choices_manual(&self) -> Vec<String> {
166 let (resume, wait) = if self.lang == "ja" {
167 ("再開する", "まだ止めておく")
168 } else {
169 ("resume it", "keep it held")
170 };
171 vec![resume.to_owned(), wait.to_owned(), self.discard.to_owned()]
172 }
173
174 fn source_label(&self, source: Option<HoldSource>) -> &'static str {
175 match (self.lang, source) {
176 ("ja", Some(HoldSource::Machine)) => "machine(機械による自動保留)",
177 ("ja", Some(HoldSource::Manual)) => "manual(操作者による手動保留)",
178 ("ja", None) => "unknown(schema 3 未満の旧レコード、または理由未記録)",
179 (_, Some(HoldSource::Machine)) => "machine (automatic recovery hold)",
180 (_, Some(HoldSource::Manual)) => "manual (an operator held this)",
181 (_, None) => "unknown (pre-schema-3 record, or never recorded)",
182 }
183 }
184
185 fn detail(&self, task: &Task, why: &str) -> String {
194 let none = if self.lang == "ja" {
195 "(記録なし)"
196 } else {
197 "(none recorded)"
198 };
199 let reason = task
200 .hold_reason
201 .as_deref()
202 .or(task.last_error.as_deref())
203 .unwrap_or(none);
204 format!(
205 "task: {} ({})\ntitle: {}\nhold source: {}\nhold reason: {reason}\n\n{why}",
206 task.id,
207 task.short(),
208 task.title,
209 self.source_label(task.hold_source),
210 )
211 }
212
213 fn summary_machine_unknown(&self, task: &Task) -> String {
214 if self.lang == "ja" {
215 format!("保留タスク {} の再開可否を判断してください", task.short())
216 } else {
217 format!("decide whether to resume held task {}", task.short())
218 }
219 }
220
221 fn why_machine(&self) -> &'static str {
222 if self.lang == "ja" {
223 "機械的な保留(machine hold)ですが、原因がすでに解消しているかを自動では判断できませんでした。"
224 } else {
225 "This is a machine hold, but whether its cause has resolved could not be \
226 checked automatically."
227 }
228 }
229
230 fn summary_conductor_override(&self, task: &Task) -> String {
231 if self.lang == "ja" {
232 format!(
233 "再開と回答済みのタスク {} を conductor が再び保留しました",
234 task.short()
235 )
236 } else {
237 format!(
238 "task {} was resumed at your word, but the conductor held it again",
239 task.short()
240 )
241 }
242 }
243
244 fn why_conductor_override(&self, o: &OperatorResume) -> String {
245 let reason = o.conductor_rehold.as_deref().unwrap_or_default();
246 if self.lang == "ja" {
247 format!(
248 "{} に再開と回答済みですが、conductor が再び hold しました。conductor の理由: \
249 {reason}\n\n強制再キューを選ぶと、以後 conductor はこのタスクを hold できません。",
250 o.at
251 )
252 } else {
253 format!(
254 "You answered \"resume\" at {}, but the conductor held the task again. \
255 Its reason: {reason}\n\nForcing a requeue stops the conductor from \
256 holding this task again.",
257 o.at
258 )
259 }
260 }
261
262 fn choices_conductor_override(&self) -> Vec<String> {
264 if self.lang == "ja" {
265 vec![
266 "強制再キュー(conductor は再 hold 不可)".to_owned(),
267 "手動 hold のまま".to_owned(),
268 "捨ててよい".to_owned(),
269 ]
270 } else {
271 vec![
272 "force requeue (conductor must not hold again)".to_owned(),
273 "keep held (manual)".to_owned(),
274 "discard".to_owned(),
275 ]
276 }
277 }
278
279 fn summary_legacy(&self, task: &Task) -> String {
280 if self.lang == "ja" {
281 format!(
282 "hold_source が不明な保留タスク {} を確認してください",
283 task.short()
284 )
285 } else {
286 format!(
287 "held task {} has no recorded hold source - please take a look",
288 task.short()
289 )
290 }
291 }
292
293 fn why_legacy(&self) -> &'static str {
294 if self.lang == "ja" {
295 "hold_source が記録されていません。schema 3 より前のレコードか、理由が記録されなかった \
296 holdです。人が意図して止めたのか、クラッシュや強制再起動で宙に浮いただけなのか、\
297 このデータからは区別できません。"
298 } else {
299 "No hold_source was recorded - either a pre-schema-3 record, or a hold whose \
300 reason was never written down. Whether this was a deliberate hold or the \
301 leftover of a crash cannot be told from the data alone."
302 }
303 }
304
305 fn summary_manual_stale(&self, task: &Task, days: i64) -> String {
306 if self.lang == "ja" {
307 format!(
308 "{days}日間 保留されたままの手動保留タスク {} を確認してください",
309 task.short()
310 )
311 } else {
312 format!(
313 "held task {} has been on a manual hold for {days} day(s)",
314 task.short()
315 )
316 }
317 }
318
319 fn why_manual(&self) -> &'static str {
320 if self.lang == "ja" {
321 "操作者が明示的に止めた保留ですが、長期間そのままになっています。まだ止めておくか、\
322 再開するか教えてください。"
323 } else {
324 "An operator held this on purpose, but it has sat untouched for a while. Say \
325 whether to keep holding it or resume it."
326 }
327 }
328}
329
330#[derive(Debug, Clone, Copy, PartialEq, Eq)]
332enum Bucket {
333 MachineUnknown,
335 ConductorOverride,
338 Legacy,
340 ManualStale,
342}
343
344#[derive(Debug, Clone, Default)]
346pub struct Report {
347 pub resumed: Vec<String>,
350 pub asked: Vec<String>,
352 pub answered: Vec<String>,
354 pub quarantined: Vec<String>,
359 pub superseded: Vec<String>,
362 pub needs_owner: Vec<String>,
365}
366
367impl Report {
368 pub fn is_empty(&self) -> bool {
371 self.resumed.is_empty()
372 && self.asked.is_empty()
373 && self.answered.is_empty()
374 && self.quarantined.is_empty()
375 && self.superseded.is_empty()
376 && self.needs_owner.is_empty()
377 }
378}
379
380fn repo_for(task: &Task) -> PathBuf {
384 if task.repo.as_os_str().is_empty() {
385 PathBuf::from(".")
386 } else {
387 task.repo.clone()
388 }
389}
390
391fn is_disk_hold(task: &Task) -> bool {
393 task.has_disk_capacity_cause()
394 || (task.schema < 22
395 && task
396 .hold_reason
397 .as_deref()
398 .is_some_and(crate::disk::is_gate_reason))
399}
400
401pub(crate) fn disk_wait_eligible(task: &Task) -> bool {
402 disk_wait_known(task, &crate::run::home())
403}
404
405fn disk_wait_known(task: &Task, home: &Path) -> bool {
407 if task.status != TaskStatus::Held
408 || task.hold_source != Some(HoldSource::Machine)
409 || task
410 .resume_override
411 .as_ref()
412 .is_some_and(|o| o.conductor_rehold.is_some())
413 {
414 return false;
415 }
416 if task.has_disk_capacity_cause() {
417 return true;
418 }
419 if task.resume_override.is_some() {
421 return false;
422 }
423 if is_disk_hold(task) {
424 return true;
425 }
426 if task.schema >= 22 {
427 return false;
428 }
429 if !task.hold_reason.as_deref().is_some_and(|r| {
430 r.starts_with(crate::disk::RUNNING_PRESSURE) || r.starts_with(crate::cache::RESOURCE_HOLD)
431 }) {
432 return false;
433 }
434 let Some(id) = task.execution_runs().last() else {
435 return false;
436 };
437 let Ok(state) = crate::run::RunState::load_under(id, home) else {
438 return false;
439 };
440 state.id == *id
441 && state.resource_block.as_ref().is_some_and(|b| {
442 let expected = match b.kind.as_str() {
443 "disk pressure" => format!("{}{}", crate::disk::RUNNING_PRESSURE, b.detail),
444 "disk admission" => format!(
445 "{}{}; no attempt was spent; resume to retry the affected operation",
446 crate::cache::RESOURCE_HOLD,
447 b.description()
448 ),
449 _ => return false,
450 };
451 matches!(b.cause, crate::run::ResourceBlockCause::Other)
452 && task.hold_reason.as_deref() == Some(expected.as_str())
453 })
454}
455
456fn machine_cause_resolved(task: &Task, cfg: &Config) -> Option<bool> {
461 machine_cause_resolved_under(task, cfg, &crate::run::home())
462}
463
464fn machine_cause_resolved_under(task: &Task, cfg: &Config, home: &Path) -> Option<bool> {
465 machine_cause_resolved_with_admission(task, cfg, home, |paths, floor| {
466 crate::retention::OutputLease::admit(home, paths, floor).is_ok()
467 })
468}
469
470fn machine_cause_resolved_with_admission(
471 task: &Task,
472 cfg: &Config,
473 home: &Path,
474 admission_available: impl Fn(&[PathBuf], u64) -> bool,
475) -> Option<bool> {
476 if disk_wait_known(task, home) {
477 if crate::daemon::disk_gate(&repo_for(task), cfg).is_some() {
478 return Some(false);
479 }
480 let mut floor = cfg.disk.min_free_bytes;
481 let mut paths = vec![repo_for(task)];
482 paths.extend(cfg.cache_dir());
483 paths.push(
484 cfg.graph
485 .worktree_root
486 .clone()
487 .unwrap_or_else(crate::run::default_worktree_root),
488 );
489 if let Some(id) = task.execution_runs().last() {
490 let Ok(state) = crate::run::RunState::load_under(id, home) else {
491 return Some(false);
492 };
493 if state.id != *id || !state.status.resumable() {
494 return Some(false);
495 }
496 if crate::daemon::disk_gate(&state.repo, &state.config).is_some() {
497 return Some(false);
498 }
499 floor = floor.max(state.config.disk.min_free_bytes);
500 paths.extend([state.repo.clone(), state.worktree_root()]);
501 paths.extend(state.candidates.iter().map(|c| c.worktree.clone()));
502 paths.extend(state.config.cache_dir());
503 }
504 return Some(admission_available(&paths, floor));
505 }
506 if task
507 .hold_reason
508 .as_deref()
509 .is_some_and(|r| r.starts_with(crate::cache::RESOURCE_HOLD))
510 {
511 let id = task.execution_runs().last()?;
513 let state = crate::run::RunState::load_under(id, home).ok()?;
514 let block = state.resource_block.as_ref()?;
515 if state.id != *id {
516 return None;
517 }
518 if block.kind != "shared build cache"
521 || !matches!(block.cause, crate::run::ResourceBlockCause::Timeout { .. })
522 {
523 return None;
524 }
525 return Some(
526 cfg.cache_dir()
527 .is_none_or(|dir| !crate::cache::in_use(home, &dir)),
528 );
529 }
530 None
531}
532
533fn manual_is_stale(task: &Task, now: Timestamp) -> bool {
537 now.as_second() - task.updated_at.as_second() > MANUAL_STALE_AFTER.as_secs() as i64
538}
539
540fn marker_for(q: &Question) -> String {
543 format!("[triage:{}]", q.short())
544}
545
546fn already_applied(task: &Task, q: &Question) -> bool {
549 if task.triage_applied(&q.id) {
550 return true;
551 }
552 let marker = marker_for(q);
555 task.hold_reason
556 .as_deref()
557 .is_some_and(|r| r.contains(marker.as_str()))
558}
559
560fn consumed(task: &Task, q: &Question) -> bool {
562 if q.node == NODE {
563 already_applied(task, q)
564 } else {
565 task.triage_applied(&q.id)
566 }
567}
568
569pub(crate) struct RecoveryIndex {
572 latest: std::collections::HashMap<(String, String), Question>,
573}
574
575impl RecoveryIndex {
576 pub(crate) fn load(questions: &Questions) -> Self {
577 let queue = Queue::at(
578 questions
579 .root()
580 .parent()
581 .unwrap_or(Path::new(""))
582 .join("queue"),
583 );
584 let tasks = queue.list();
585 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
586 Self::from_scan(&queue, &tasks, questions.list_for_effects_in(&ownership))
587 }
588
589 fn from_scan(queue: &Queue, tasks: &[Task], all: Vec<Question>) -> Self {
590 let ids = crate::queue::record_ids(queue.root(), "json").unwrap_or_default();
591 let by_id: std::collections::HashMap<_, _> =
592 tasks.iter().map(|task| (task.id.as_str(), task)).collect();
593 let exact: std::collections::HashSet<_> = ids.iter().map(String::as_str).collect();
594 let mut aliases: std::collections::HashMap<&str, Option<&str>> = Default::default();
595 for id in &ids {
596 for boundary in id
597 .char_indices()
598 .map(|(i, _)| i)
599 .chain(std::iter::once(id.len()))
600 {
601 for alias in [&id[..boundary], &id[boundary..]] {
602 let entry = aliases.entry(alias).or_insert(Some(id));
603 if *entry != Some(id.as_str()) {
604 *entry = None;
605 }
606 }
607 }
608 }
609 let mut index = Self {
610 latest: Default::default(),
611 };
612 for q in all {
613 if q.node != NODE && q.node != DEPS_NODE {
614 continue;
615 }
616 let resolved = if exact.contains(q.run.as_str()) {
617 Some(q.run.as_str())
618 } else {
619 aliases.get(q.run.as_str()).copied().flatten()
620 };
621 if let Some(id) = resolved
622 && by_id.get(id).is_some_and(|task| {
623 crate::repair_budget::reconcile::superseded_question(task, &q)
624 })
625 {
626 continue;
627 }
628 let key = (q.node.clone(), q.run.clone());
629 let entry = index.latest.entry(key).or_insert_with(|| q.clone());
630 if (q.asked_at, &q.id) > (entry.asked_at, &entry.id) {
631 *entry = q;
632 }
633 }
634 index
635 }
636
637 fn latest(&self, node: &str, task_id: &str) -> Option<Question> {
638 self.latest
639 .get(&(node.to_owned(), task_id.to_owned()))
640 .cloned()
641 }
642
643 pub(crate) fn pending(&self, task: &Task) -> Vec<Question> {
644 [NODE, DEPS_NODE]
645 .into_iter()
646 .filter_map(|node| {
647 self.latest(node, &task.id).filter(|q| {
648 !consumed(task, q)
649 && (q.status.open()
650 || awaiting_task_chat(q, task)
651 || q.status == QuestionStatus::Answered)
652 })
653 })
654 .collect()
655 }
656
657 fn displayed_open(&self, task: &Task) -> Option<Question> {
658 [NODE, DEPS_NODE]
659 .into_iter()
660 .filter(|node| {
661 task.status == TaskStatus::Held
662 || (*node == DEPS_NODE && task.status == TaskStatus::Blocked)
663 })
664 .filter_map(|node| self.latest(node, &task.id))
665 .find(|q| q.status.open() && !consumed(task, q))
666 }
667}
668
669#[derive(Debug, Clone, Copy, PartialEq, Eq)]
672enum AnswerAction {
673 Resume,
677 Discard,
680 KeepHeld,
685}
686
687fn interpret_answer(q: &Question) -> AnswerAction {
700 let resolution = q.resolution().unwrap_or_default();
701 match q.choices.iter().position(|c| *c == resolution) {
702 Some(0) => AnswerAction::Resume,
703 Some(2) => AnswerAction::Discard,
704 _ => AnswerAction::KeepHeld,
705 }
706}
707
708pub(crate) fn contextual_keep_held(q: &Question, label: &str) -> bool {
711 q.node == NODE
712 && q.seat == SEAT
713 && q.actions.is_empty()
714 && q.choices.get(1).is_some_and(|c| c == label)
715}
716
717fn keep_held_note(task: &Task, q: &Question, resolution: &str) -> String {
722 let marker = format!("{} operator: {resolution}", marker_for(q));
723 match task.hold_reason.as_deref() {
724 Some(existing) if !existing.is_empty() => format!("{existing}\n{marker}"),
725 _ => marker,
726 }
727}
728
729fn file_question(
733 questions: &Questions,
734 ownership: &crate::task_chat::InquiryOwnership,
735 task: &Task,
736 bucket: Bucket,
737 w: &Wording,
738 now: Timestamp,
739) -> Option<Question> {
740 let (summary, why, choices) = match bucket {
741 Bucket::MachineUnknown => (
742 w.summary_machine_unknown(task),
743 w.why_machine().to_owned(),
744 w.choices3(),
745 ),
746 Bucket::ConductorOverride => (
747 w.summary_conductor_override(task),
748 task.resume_override
749 .as_ref()
750 .map(|o| w.why_conductor_override(o))
751 .unwrap_or_default(),
752 w.choices_conductor_override(),
753 ),
754 Bucket::Legacy => (
755 w.summary_legacy(task),
756 w.why_legacy().to_owned(),
757 w.choices3(),
758 ),
759 Bucket::ManualStale => {
760 let days = (now.as_second() - task.updated_at.as_second()) / (24 * 60 * 60);
761 (
762 w.summary_manual_stale(task, days),
763 w.why_manual().to_owned(),
764 w.choices_manual(),
765 )
766 }
767 };
768 let mut q = Question::new(
769 task.id.clone(),
770 NODE.to_owned(),
771 SEAT.to_owned(),
772 summary,
773 w.detail(task, &why),
774 choices,
775 );
776 q.triage_generation = Some(task.resumption.as_ref().map_or(0, |r| r.generation));
777 questions.put_executable_in(&mut q, ownership).ok()?;
778 Some(q)
779}
780
781pub(crate) fn deputy_brief(q: &Question, queue: &Queue) -> String {
786 let task = queue.get(&q.run).ok();
787 let mut s = match &task {
788 Some(t) => format!(
789 "Task {} (`magi task show {}`), \"{}\", is {}. Hold source: {}. Reason: {}.",
790 t.id,
791 t.id,
792 t.title,
793 t.status.as_str(),
794 match t.hold_source {
795 Some(HoldSource::Machine) => "machine (an automatic hold)",
796 Some(HoldSource::Manual) => "manual (an operator held it)",
797 None => "unknown (an old record, treated as the operator's)",
798 },
799 t.hold_reason
800 .as_deref()
801 .or(t.last_error.as_deref())
802 .unwrap_or("(none recorded)")
803 ),
804 None => format!("Task {} no longer exists (its record is gone).", q.run),
805 };
806 if task.as_ref().is_some_and(disk_wait_eligible) {
807 s.push_str("\n\nDisk capacity waits are monitored automatically. No owner answer is needed for an untouched automatic disk triage question. Explicit holds, discussion and other approvals still require owner resolution.");
808 }
809 s.push_str("\n\nWhat each option does, by position:");
810 let n = q.choices.len();
811 for (i, c) in q.choices.iter().enumerate() {
812 let effect = if q.node == DEPS_NODE {
813 match i {
814 0 => "the dependency is released so the loop runs it",
815 1 => {
816 "the dependants are detached and the dependency task is DELETED; this cannot be undone"
817 }
818 2 => "the dependants are detached; the dependency stays as it is",
819 _ => "nothing is applied",
820 }
821 } else {
822 match i {
823 0 => "the task is released and requeued",
824 2 if n > 2 => "the task is DELETED; this cannot be undone",
825 _ => "the task stays held, with the answer noted",
826 }
827 };
828 s.push_str(&format!("\n- `{c}`: {effect}"));
829 }
830 s.push_str(
831 "\n\nThe daemon applies the answer on its next pass; you apply nothing. When \
832 the owner's words pick an option - including releasing a task they held \
833 themselves, or deleting it - settle it.",
834 );
835 s
836}
837
838fn quarantine_orphaned_blocked(queue: &Queue, questions: &Questions) -> Vec<String> {
856 let mut quarantined = Vec::new();
857 for listed in queue.list() {
858 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
859 continue;
860 }
861 let Ok(_claim) = queue.claim(&listed.id) else {
862 continue;
863 };
864 let Ok(mut task) = queue.get(&listed.id) else {
865 continue;
866 };
867 if task.status != TaskStatus::Blocked {
868 continue;
869 }
870 let deleted = queue.apply_deleted_blockers(&mut task, questions);
871 if !deleted.is_empty() && queue.put(&mut task).is_ok() {
872 for id in &deleted {
873 queue.note_dependency_deleted(&task, id);
874 }
875 }
876 if task.status != TaskStatus::Blocked {
877 continue;
878 }
879 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
880 if missing.is_empty() {
881 continue;
882 }
883 let language = crate::lang::of_repo(&repo_for(&task));
884 let mut cause = missing.clone();
885 cause.sort();
886 cause.dedup();
887 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
888 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
889 &task.blocked_by,
890 &missing,
891 &language,
892 )));
893 if queue.put(&mut task).is_ok() {
894 quarantined.push(task.id.clone());
895 }
896 }
897 quarantined
898}
899
900#[derive(Debug, Clone, Copy, PartialEq, Eq)]
903enum DepsAction {
904 Release,
905 Discard,
906 Detach,
907 Nothing,
909}
910
911fn interpret_deps_answer(q: &Question) -> DepsAction {
912 let resolution = q.resolution().unwrap_or_default();
913 match q.choices.iter().position(|c| *c == resolution) {
914 Some(0) => DepsAction::Release,
915 Some(1) => DepsAction::Discard,
916 Some(2) => DepsAction::Detach,
917 _ => DepsAction::Nothing,
918 }
919}
920
921fn deps_texts(w: &Wording, root: &Task, dependants: &[&Task]) -> (String, String, Vec<String>) {
923 let ja = w.lang == "ja";
924 let reason = root
925 .hold_reason
926 .as_deref()
927 .or(root.last_error.as_deref())
928 .unwrap_or(if ja {
929 "(記録なし)"
930 } else {
931 "(none recorded)"
932 });
933 let list: String = dependants
934 .iter()
935 .map(|t| format!("- {} {}\n", t.short(), t.title))
936 .collect();
937 if ja {
938 (
939 format!(
940 "{} ({}) が {} 件のタスクを止めています",
941 root.short(),
942 root.status.as_str(),
943 dependants.len()
944 ),
945 format!(
946 "task: {} ({})\ntitle: {}\n状態: {}\n理由: {reason}\n\n\
947 このタスクは自動では実行されないため、依存している次のタスクは永遠に待ち続けます:\n{list}",
948 root.id,
949 root.short(),
950 root.title,
951 root.status.as_str()
952 ),
953 vec![
954 "依存先を再開する".to_owned(),
955 "依存先を捨てる(依存タスクは切り離して実行)".to_owned(),
956 "依存タスクを切り離す(依存先はそのまま)".to_owned(),
957 ],
958 )
959 } else {
960 (
961 format!(
962 "{} ({}) is freezing {} blocked task(s)",
963 root.short(),
964 root.status.as_str(),
965 dependants.len()
966 ),
967 format!(
968 "task: {} ({})\ntitle: {}\nstatus: {}\nreason: {reason}\n\n\
969 Nothing in the loop will ever run this task, so these dependants wait \
970 forever:\n{list}",
971 root.id,
972 root.short(),
973 root.title,
974 root.status.as_str()
975 ),
976 vec![
977 "release the dependency".to_owned(),
978 "discard the dependency (dependants are detached and run)".to_owned(),
979 "detach the dependants (the dependency stays as it is)".to_owned(),
980 ],
981 )
982 }
983}
984
985fn detach_dependants(queue: &Queue, questions: &Questions, root: &str) {
989 for listed in queue.list() {
990 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| matches!(crate::queue::dependency(queue, questions, b), crate::queue::Dependency::Task(id) if id == root)) {
991 continue;
992 }
993 let Ok(_claim) = queue.claim(&listed.id) else {
994 continue;
995 };
996 let Ok(mut t) = queue.get(&listed.id) else {
997 continue;
998 };
999 if t.status != TaskStatus::Blocked {
1000 continue;
1001 }
1002 for id in t.blocked_by.clone() {
1003 if matches!(crate::queue::dependency(queue, questions, &id), crate::queue::Dependency::Task(full) if full == root)
1004 {
1005 t.unblock(&id);
1006 }
1007 }
1008 let _ = queue.put(&mut t);
1009 }
1010}
1011
1012fn apply_deps_answer(
1016 queue: &Queue,
1017 questions: &Questions,
1018 q: &Question,
1019 now: Timestamp,
1020 ownership: &crate::task_chat::InquiryOwnership,
1021) -> bool {
1022 let Ok(_claim) = queue.claim(&q.run) else {
1023 return false;
1024 };
1025 let Ok(current) = questions.get_for_effects_in(&q.id, ownership) else {
1026 return false;
1027 };
1028 if current.status != QuestionStatus::Answered {
1029 return false;
1030 }
1031 let q = ¤t;
1032 let Ok(mut root) = queue.get(&q.run) else {
1033 return false;
1034 };
1035 if root.triage_applied(&q.id) {
1036 return false;
1037 }
1038 if root.operator_decided_at.is_some_and(|at| at > q.asked_at)
1039 || q.triage_generation
1040 .is_some_and(|g| g != root.resumption.as_ref().map_or(0, |r| r.generation))
1041 {
1042 root.mark_triage_applied(&q.id);
1043 return queue.put(&mut root).is_ok();
1044 }
1045 match interpret_deps_answer(q) {
1046 DepsAction::Release => {
1047 if root.status == TaskStatus::Running {
1048 return false;
1049 }
1050 if crate::repair_budget::reconcile::blocker(queue, &root).is_some() {
1051 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(q) else {
1052 return false;
1053 };
1054 root.decision_effects.insert(q.id.clone(), receipt);
1055 root.mark_triage_applied(&q.id);
1056 return queue.put(&mut root).is_ok();
1057 }
1058 root.answers.push(crate::queue::AnsweredQuestion {
1059 question: q.summary.clone(),
1060 answer: q.resolution().unwrap_or_default(),
1061 });
1062 if crate::base_sync_recovery::held_task(queue, &root) {
1063 root.mark_triage_applied(&q.id);
1064 return queue.put(&mut root).is_ok();
1065 }
1066 let before = root.operator_decided_at;
1067 let contract =
1068 crate::repair_budget::contract(&root, crate::repair_budget::Kind::Review).ok();
1069 root.release_operator("triage-dependencies", Some(&q.id));
1070 if let Some(contract) = contract
1071 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1072 q,
1073 before,
1074 &contract,
1075 root.resumption.as_ref().expect("release"),
1076 )
1077 {
1078 root.decision_effects.insert(q.id.clone(), evidence);
1079 }
1080 root.resume_override = Some(OperatorResume {
1081 question_id: q.id.clone(),
1082 at: now,
1083 conductor_rehold: None,
1084 forced: false,
1085 pinned_run: None,
1086 });
1087 }
1088 DepsAction::Discard => {
1089 detach_dependants(queue, questions, &root.id);
1090 return queue.discard_triage(&root.id, questions, &q.id).is_ok();
1091 }
1092 DepsAction::Detach => detach_dependants(queue, questions, &root.id),
1093 DepsAction::Nothing => return false,
1098 }
1099 root.mark_triage_applied(&q.id);
1100 queue.put(&mut root).is_ok()
1101}
1102
1103fn repair_receipt_notice(queue: &Queue, task: &Task) -> bool {
1104 let Some(detail) = crate::repair_budget::reconcile::blocker(queue, task) else {
1105 return false;
1106 };
1107 if let Some(home) = queue.root().parent() {
1108 let _ = crate::task_inquiry::Outbox::at(home).file_for_occurrence(
1109 task,
1110 task.runs.last().cloned(),
1111 "repair-receipt-recovery".into(),
1112 &task
1113 .repair_grants
1114 .last()
1115 .expect("diagnosed receipt")
1116 .request
1117 .request,
1118 "Retained repair receipt needs reconciliation",
1119 &detail,
1120 );
1121 }
1122 true
1123}
1124
1125fn ask_about_stuck_roots(
1134 queue: &Queue,
1135 questions: &Questions,
1136 config_override: Option<&Path>,
1137 report: &mut Report,
1138) {
1139 let all = questions.list_for_effects();
1140 let tasks = queue.list();
1141 let recovery = RecoveryIndex::from_scan(queue, &tasks, all.clone());
1142 let inv = crate::blockers::Inventory::load_with_tasks(queue, questions, tasks);
1143 let mut frozen: std::collections::BTreeMap<String, Vec<String>> = Default::default();
1144 for (id, roots) in inv.stuck() {
1145 for root in roots {
1146 if root != id {
1147 frozen.entry(root).or_default().push(id.clone());
1148 }
1149 }
1150 }
1151 let ownership = crate::task_chat::InquiryOwnership::default();
1155 for mut q in all
1156 .iter()
1157 .filter(|q| q.node == DEPS_NODE && q.status.open())
1158 .cloned()
1159 {
1160 if inv.task(&q.run).is_some_and(|task| consumed(task, &q)) {
1161 continue;
1162 }
1163 if !frozen.contains_key(&q.run) {
1164 q.abandon("nothing waits on this task anymore");
1165 let _ = questions.put_executable_in(&mut q, &ownership);
1166 }
1167 }
1168
1169 for (root_id, dependant_ids) in &frozen {
1170 let Ok(_claim) = queue.claim(root_id) else {
1171 continue;
1172 };
1173 let seeds: Vec<_> = std::iter::once(root_id)
1176 .chain(dependant_ids.iter())
1177 .filter_map(|id| queue.get(id).ok())
1178 .collect();
1179 let current = crate::blockers::Inventory::load_for_tasks(queue, questions, &seeds);
1180 let Some(root) = current.task(root_id) else {
1181 continue;
1182 };
1183 if crate::base_sync_recovery::held_task(queue, root) {
1184 continue;
1185 }
1186 if disk_wait_known(root, questions.root().parent().unwrap_or(Path::new(""))) {
1189 continue;
1190 }
1191 let dependants: Vec<&Task> = dependant_ids
1192 .iter()
1193 .filter_map(|d| current.task(d))
1194 .filter(|t| current.stuck_roots(t).contains(root_id))
1195 .collect();
1196 if dependants.is_empty() {
1197 continue;
1198 }
1199 if repair_receipt_notice(queue, root) {
1200 continue;
1201 }
1202 let latest = recovery.latest(DEPS_NODE, root_id);
1203 match latest.as_ref() {
1204 Some(q) if !consumed(root, q) && (q.status.open() || awaiting_task_chat(q, root)) => {
1205 continue;
1206 }
1207 Some(q) if q.status == QuestionStatus::Answered && !consumed(root, q) => {
1208 continue;
1209 }
1210 Some(q) if q.status == QuestionStatus::Abandoned => {
1211 let changed = root.updated_at > q.asked_at
1212 || dependants.iter().any(|t| t.updated_at > q.asked_at);
1213 if !changed {
1214 continue;
1215 }
1216 }
1217 _ => {}
1218 }
1219 if !recovery.pending(root).is_empty() {
1220 continue;
1221 }
1222 let cfg = Config::discover(&repo_for(root), config_override)
1223 .ok()
1224 .map(|(c, _)| c);
1225 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1226 let (summary, detail, choices) = deps_texts(w, root, &dependants);
1227 let mut question = Question::new(
1228 root.id.clone(),
1229 DEPS_NODE.to_owned(),
1230 SEAT.to_owned(),
1231 summary,
1232 detail,
1233 choices,
1234 );
1235 question.triage_generation = Some(root.resumption.as_ref().map_or(0, |r| r.generation));
1236 if questions
1237 .put_executable_in(&mut question, &ownership)
1238 .is_ok()
1239 {
1240 report.asked.push(root.id.clone());
1241 }
1242 }
1243}
1244
1245pub fn apply_answers(queue: &Queue, questions: &Questions) -> Vec<String> {
1248 let mut applied = Vec::new();
1249 let ownership = crate::task_chat::InquiryOwnership::default();
1250 for q in questions.list_for_effects_in(&ownership) {
1251 if q.node == DEPS_NODE
1252 && q.status == QuestionStatus::Answered
1253 && apply_deps_answer(queue, questions, &q, Timestamp::now(), &ownership)
1254 {
1255 applied.push(q.run.clone());
1256 }
1257 }
1258 let tasks = queue.list();
1260 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
1261 let recovery =
1262 RecoveryIndex::from_scan(queue, &tasks, questions.list_for_effects_in(&ownership));
1263 for listed in tasks {
1264 let Some(q) = recovery.latest(NODE, &listed.id) else {
1265 continue;
1266 };
1267 if q.status != QuestionStatus::Answered || already_applied(&listed, &q) {
1268 continue;
1269 }
1270 let Ok(_claim) = queue.claim(&listed.id) else {
1271 continue;
1272 };
1273 let Ok(mut task) = queue.get(&listed.id) else {
1274 continue;
1275 };
1276 let Ok(q) = questions.get_for_effects_in(&q.id, &ownership) else {
1279 continue;
1280 };
1281 if q.status != QuestionStatus::Answered {
1282 continue;
1283 }
1284 if already_applied(&task, &q)
1285 || !matches!(task.status, TaskStatus::Held | TaskStatus::Queued)
1286 {
1287 continue;
1288 }
1289 let generation = task.resumption.as_ref().map_or(0, |r| r.generation);
1290 let action = interpret_answer(&q);
1291 let mut newer_decision = q.triage_generation.is_some_and(|g| g != generation)
1292 || task.resumption.as_ref().is_some_and(|r| r.at > q.asked_at)
1293 || task.held_at.is_some_and(|at| at > q.asked_at)
1294 || task.operator_decided_at.is_some_and(|at| at > q.asked_at);
1295 if matches!(action, AnswerAction::KeepHeld)
1299 && q.answered_at.is_some_and(|at| {
1300 task.operator_decided_at
1301 .is_none_or(|decision| at > decision)
1302 && task.resumption.as_ref().is_none_or(|r| at > r.at)
1303 })
1304 {
1305 newer_decision = false;
1306 }
1307 let resolution = q.resolution().unwrap_or_default();
1309 task.answers.push(crate::queue::AnsweredQuestion {
1310 question: q.summary.clone(),
1311 answer: if newer_decision {
1312 format!("{resolution} (superseded by a later operator decision)")
1313 } else {
1314 resolution
1315 },
1316 });
1317 if !newer_decision {
1318 match action {
1319 AnswerAction::Resume if crate::base_sync_recovery::held_task(queue, &task) => {
1320 }
1323 AnswerAction::Resume
1324 if crate::repair_budget::reconcile::blocker(queue, &task).is_some() =>
1325 {
1326 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(&q) else {
1327 continue;
1328 };
1329 task.decision_effects.insert(q.id.clone(), receipt);
1330 }
1331 AnswerAction::Resume => {
1332 let before = task.operator_decided_at;
1333 let contract =
1334 crate::repair_budget::contract(&task, crate::repair_budget::Kind::Review)
1335 .ok();
1336 task.release_operator("triage", Some(&q.id));
1337 if let Some(contract) = contract
1338 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1339 &q,
1340 before,
1341 &contract,
1342 task.resumption.as_ref().expect("release"),
1343 )
1344 {
1345 task.decision_effects.insert(q.id.clone(), evidence);
1346 }
1347 task.resume_override = Some(OperatorResume {
1348 question_id: q.id.clone(),
1349 at: Timestamp::now(),
1350 conductor_rehold: None,
1351 forced: true,
1352 pinned_run: None,
1353 });
1354 }
1355 AnswerAction::KeepHeld => {
1356 let note = keep_held_note(&task, &q, &q.resolution().unwrap_or_default());
1357 task.hold_manual(Some(note));
1358 }
1359 AnswerAction::Discard => {
1360 if queue.discard_triage(&task.id, questions, &q.id).is_ok() {
1361 applied.push(task.id);
1362 }
1363 continue;
1364 }
1365 }
1366 }
1367 task.mark_triage_applied(&q.id);
1368 if queue.put(&mut task).is_ok() {
1369 applied.push(task.id);
1370 }
1371 }
1372 applied
1373}
1374const DISK_SUPERSEDED: &str =
1375 "Automatic disk wait superseded unanswered triage; no owner answer or approval was created";
1376
1377fn untouched_disk_question(q: &Question, task: &Task) -> bool {
1378 q.status.open()
1379 && q.node == NODE
1380 && q.seat == SEAT
1381 && q.run == task.id
1382 && q.routed_task.as_deref().is_none_or(|id| id == task.id)
1383 && q.triage_generation == Some(task.resumption.as_ref().map_or(0, |r| r.generation))
1384 && task.held_at.is_some_and(|at| q.asked_at >= at)
1385 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1386 && q.answer.is_none()
1387 && q.answered_at.is_none()
1388 && q.accepted_answer_at.is_none()
1389 && q.thread.is_empty()
1390 && q.actions.is_empty()
1391 && q.merge_target.is_none()
1392 && [&EN, &JA].iter().any(|w| {
1393 q.summary == w.summary_machine_unknown(task)
1394 && q.choices == w.choices3()
1395 && q.detail == w.detail(task, w.why_machine())
1396 })
1397}
1398
1399fn refresh_disk_notices(home: &Path, task: &Task) {
1401 let notices = crate::notices::Notices::at(home.join("notifications"));
1402 let Some(task_notice) = crate::notices::task_held(task) else {
1403 return;
1404 };
1405 for (key, message) in [
1406 (task_notice.key, task_notice.message),
1407 (
1408 format!("disk:{}", repo_for(task).display()),
1409 format!(
1410 "Task {} is waiting for disk space. It will resume automatically when disk and output admission checks succeed.",
1411 task.id
1412 ),
1413 ),
1414 ] {
1415 let id = crate::notices::id_of(&key);
1416 if notices
1417 .get(&id)
1418 .is_ok_and(|n| n.key == key && n.message != message)
1419 {
1420 let _ = notices.update(&id, |n| {
1421 if n.key == key {
1422 n.message = message;
1423 }
1424 });
1425 }
1426 }
1427}
1428
1429fn resume_disk_task(
1433 queue: &Queue,
1434 questions: &Questions,
1435 task: &mut Task,
1436 ownership: &crate::task_chat::InquiryOwnership,
1437) -> anyhow::Result<bool> {
1438 let expected = task.clone();
1439 let home = questions
1440 .root()
1441 .parent()
1442 .ok_or_else(|| anyhow::anyhow!("missing question home"))?;
1443 let _lease = if let Some(id) = task.execution_runs().last() {
1444 anyhow::ensure!(
1445 task.resume_override
1446 .as_ref()
1447 .and_then(|o| o.pinned_run.as_ref())
1448 .is_none_or(|pin| pin == id),
1449 "selected recovery run differs from disk wait evidence"
1450 );
1451 let state = crate::run::RunState::load_under(id, home)?;
1452 anyhow::ensure!(
1453 state.id == *id && state.status.resumable() && !task.fresh_start,
1454 "retained execution cannot resume"
1455 );
1456 anyhow::ensure!(
1457 state.liveness_under(
1458 home,
1459 crate::daemon::is_working_on(home, id, Timestamp::now())
1460 ) == crate::run::Liveness::Dead,
1461 "retained execution is not proven idle"
1462 );
1463 Some(crate::driver::maintenance(home, id)?)
1464 } else {
1465 None
1466 };
1467 let outbox = crate::task_inquiry::Outbox::at(home);
1468 let all = questions.list_for_effects_in(ownership);
1469 let pending: Vec<_> = all
1470 .iter()
1471 .filter(|q| {
1472 (!q.run.is_empty()
1474 && std::iter::once(task.id.as_str())
1475 .chain(task.execution_runs().iter().map(String::as_str))
1476 .any(|id| id == q.run || id.starts_with(&q.run) || id.ends_with(&q.run)))
1477 && (q.status.open()
1478 || ((q.node == NODE || q.node == DEPS_NODE)
1479 && (awaiting_task_chat(q, task)
1480 || (q.status == QuestionStatus::Answered
1481 && !already_applied(task, q)))))
1482 })
1483 .collect();
1484 anyhow::ensure!(pending.len() <= 1, "multiple decisions remain");
1486 let candidate = pending.first().copied();
1487 if let Some(q) = candidate {
1488 anyhow::ensure!(untouched_disk_question(q, task), "owner decision remains");
1489 }
1490 for inquiry in outbox.pending()? {
1491 if inquiry.task == task.id && inquiry.decision.is_some() {
1492 anyhow::ensure!(
1493 candidate.is_some_and(|q| q.id == inquiry.id),
1494 "unrelated decision remains"
1495 );
1496 }
1497 }
1498 if let Some(q) = candidate {
1499 let _execution = outbox.execution_claim(&q.id)?;
1500 questions.update_in(&q.id, ownership, |current| {
1501 anyhow::ensure!(untouched_disk_question(current, task), "question changed");
1502 let inquiry = match outbox.get(&q.id) {
1503 Ok(inquiry) => inquiry,
1504 Err(error)
1505 if error
1506 .downcast_ref::<std::io::Error>()
1507 .is_some_and(|e| e.kind() == std::io::ErrorKind::NotFound) =>
1508 {
1509 anyhow::ensure!(
1510 current.routed_chat.is_none(),
1511 "delivered inquiry is missing; owner reconciliation required"
1512 );
1513 outbox.enqueue(crate::task_chat::live_record(home, task, current))?
1516 }
1517 Err(error) => return Err(error),
1518 };
1519 anyhow::ensure!(
1520 inquiry.task == task.id
1521 && inquiry.repo == task.repo
1522 && inquiry.source == format!("{}:{}:{}", NODE, SEAT, task.id)
1523 && current
1524 .routed_chat
1525 .as_ref()
1526 .is_none_or(|chat| inquiry.delivered_to.as_ref() == Some(chat)),
1527 "inquiry binding changed"
1528 );
1529 outbox.update_decision(&q.id, |d| {
1530 if d.status == crate::task_decision::Status::Resolved {
1531 anyhow::ensure!(
1532 d.request.is_none()
1533 && d.history
1534 .last()
1535 .is_some_and(|e| e.evidence == DISK_SUPERSEDED),
1536 "decision already settled by owner"
1537 );
1538 return Ok(false);
1539 }
1540 anyhow::ensure!(
1541 d.status == crate::task_decision::Status::AwaitingOwner
1542 && d.request.is_none()
1543 && d.source_answer_at.is_none(),
1544 "owner request remains"
1545 );
1546 d.settle(DISK_SUPERSEDED)
1547 })?;
1548 current.abandon(DISK_SUPERSEDED);
1549 Ok(())
1550 })?;
1551 }
1552 if let Some(q) = candidate {
1553 task.mark_triage_applied(&q.id);
1554 }
1555 task.resume_disk_wait();
1556 queue.commit_disk_wait(task, &expected)?;
1557 let notices = crate::notices::Notices::at(home.join("notifications"));
1558 let keys = [
1559 format!("disk-recovered:{}", task.id),
1560 format!("disk:{}", repo_for(task).display()),
1561 format!("task:{}", task.id),
1562 ];
1563 let repo_waiting = queue
1564 .list()
1565 .iter()
1566 .any(|other| other.repo == task.repo && disk_wait_eligible(other));
1567 for notice in notices.list() {
1568 if notice.key == keys[0]
1569 || notice.key == keys[2]
1570 || (!repo_waiting && notice.key == keys[1])
1571 {
1572 let _ = notices.dismiss(¬ice.id);
1573 }
1574 }
1575 Ok(candidate.is_some())
1576}
1577
1578pub fn run_once(
1597 queue: &Queue,
1598 questions: &Questions,
1599 config_override: Option<&Path>,
1600 now: Timestamp,
1601) -> Report {
1602 run_once_inner(queue, questions, config_override, now, false)
1603}
1604
1605pub(crate) fn monitor_disk_waits(
1607 queue: &Queue,
1608 questions: &Questions,
1609 config: Option<&Path>,
1610 now: Timestamp,
1611) -> Report {
1612 run_once_inner(queue, questions, config, now, true)
1613}
1614
1615fn run_once_inner(
1616 queue: &Queue,
1617 questions: &Questions,
1618 config_override: Option<&Path>,
1619 now: Timestamp,
1620 disk_only: bool,
1621) -> Report {
1622 run_once_with_probe(
1623 queue,
1624 questions,
1625 config_override,
1626 now,
1627 disk_only,
1628 machine_cause_resolved,
1629 )
1630}
1631
1632fn run_once_with_probe(
1633 queue: &Queue,
1634 questions: &Questions,
1635 config_override: Option<&Path>,
1636 now: Timestamp,
1637 disk_only: bool,
1638 resolved: impl Fn(&Task, &Config) -> Option<bool>,
1639) -> Report {
1640 let mut report = Report {
1641 answered: if disk_only {
1642 Vec::new()
1643 } else {
1644 apply_answers(queue, questions)
1645 },
1646 quarantined: if disk_only {
1647 Vec::new()
1648 } else {
1649 quarantine_orphaned_blocked(queue, questions)
1650 },
1651 ..Report::default()
1652 };
1653 if !disk_only {
1654 ask_about_stuck_roots(queue, questions, config_override, &mut report);
1655 }
1656 let recovery = RecoveryIndex::load(questions);
1657 let ownership = crate::task_chat::InquiryOwnership::default();
1658 for listed in queue.list() {
1659 if listed.status != TaskStatus::Held {
1660 continue;
1661 }
1662 let Ok(_claim) = queue.claim(&listed.id) else {
1663 continue;
1664 };
1665 let Ok(mut task) = queue.get(&listed.id) else {
1666 continue;
1667 };
1668 if task.status != TaskStatus::Held {
1672 continue;
1673 }
1674 if crate::base_sync_recovery::held_task(queue, &task) {
1675 continue;
1676 }
1677 if repair_receipt_notice(queue, &task) {
1678 continue;
1679 }
1680 if !disk_wait_known(&task, questions.root().parent().unwrap_or(Path::new("")))
1687 && deps_pending_in(&recovery, &task)
1688 {
1689 continue;
1690 }
1691
1692 let explicit = match &task.overrides {
1693 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
1694 _ => config_override,
1695 };
1696 let cfg = Config::discover(&repo_for(&task), explicit)
1697 .ok()
1698 .map(|(c, _)| c);
1699 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1700
1701 if disk_wait_known(&task, questions.root().parent().unwrap_or(Path::new(""))) {
1702 if task
1703 .disk_checked_at
1704 .is_some_and(|at| now.as_second() - at.as_second() < 60)
1705 {
1706 continue;
1707 }
1708 let expected = task.clone();
1709 task.disk_checked_at = Some(now);
1710 if !task.has_disk_capacity_cause() {
1711 task.mark_disk_capacity();
1713 }
1714 if queue.commit_disk_wait(&mut task, &expected).is_err() {
1715 continue;
1716 }
1717 refresh_disk_notices(questions.root().parent().unwrap_or(Path::new("")), &task);
1718 if cfg.as_ref().and_then(|c| resolved(&task, c)) == Some(true) {
1721 match resume_disk_task(queue, questions, &mut task, &ownership) {
1722 Ok(superseded) => {
1723 if superseded {
1724 report.superseded.push(task.id.clone());
1725 }
1726 report.resumed.push(task.id.clone());
1727 }
1728 Err(error) => {
1729 if !report.needs_owner.contains(&task.id) {
1730 report.needs_owner.push(task.id.clone());
1731 }
1732 crate::notices::raise_in(
1733 questions.root().parent().unwrap_or(Path::new("")),
1734 crate::notices::Notice::warn(
1735 &format!("disk-recovered:{}", task.id),
1736 format!(
1737 "Task {}: disk capacity recovered; automatic recovery is still blocked ({error}). Existing owner decisions and unrelated approvals require explicit resolution. Disk monitoring continues.",
1738 task.id
1739 ),
1740 ).link(crate::notices::Link::Task { id: task.id.clone() }).about([task.id.clone()]),
1741 );
1742 }
1743 }
1744 }
1745 continue;
1746 }
1747 if disk_only {
1748 continue;
1749 }
1750
1751 if let Some(q) = recovery.latest(NODE, &task.id) {
1752 if !consumed(&task, &q) && (q.status.open() || awaiting_task_chat(&q, &task)) {
1753 continue;
1755 }
1756 }
1762
1763 match task.hold_source {
1764 Some(HoldSource::Machine) => {
1765 let overridden = task
1766 .resume_override
1767 .as_ref()
1768 .is_some_and(|o| o.conductor_rehold.is_some() && !o.forced);
1769 if overridden {
1770 if file_question(
1771 questions,
1772 &ownership,
1773 &task,
1774 Bucket::ConductorOverride,
1775 w,
1776 now,
1777 )
1778 .is_some()
1779 {
1780 report.asked.push(task.id.clone());
1781 }
1782 } else if cfg.as_ref().and_then(|c| resolved(&task, c)) == Some(true) {
1783 task.release();
1784 if queue.put(&mut task).is_ok() {
1785 report.resumed.push(task.id.clone());
1786 }
1787 } else if file_question(
1788 questions,
1789 &ownership,
1790 &task,
1791 Bucket::MachineUnknown,
1792 w,
1793 now,
1794 )
1795 .is_some()
1796 {
1797 report.asked.push(task.id.clone());
1798 }
1799 }
1800 None => {
1801 if file_question(questions, &ownership, &task, Bucket::Legacy, w, now).is_some() {
1802 report.asked.push(task.id.clone());
1803 }
1804 }
1805 Some(HoldSource::Manual) => {
1806 if manual_is_stale(&task, now)
1807 && file_question(questions, &ownership, &task, Bucket::ManualStale, w, now)
1808 .is_some()
1809 {
1810 report.asked.push(task.id.clone());
1811 }
1812 }
1813 }
1814 }
1815 report
1816}
1817
1818pub fn open_question_for(questions: &Questions, task_id: &str) -> Option<Question> {
1823 let queue = question_queue(questions);
1824 let task = queue.get(task_id).ok()?;
1825 display_index(questions, &queue, std::slice::from_ref(&task)).displayed_open(&task)
1826}
1827
1828pub fn pending_for(questions: &Questions, task: &Task) -> bool {
1842 !pending_recovery_questions(questions, task).is_empty()
1843}
1844
1845pub(crate) fn pending_recovery_questions(questions: &Questions, task: &Task) -> Vec<Question> {
1848 RecoveryIndex::load(questions).pending(task)
1849}
1850
1851fn awaiting_task_chat(q: &Question, task: &Task) -> bool {
1852 q.status != QuestionStatus::Abandoned
1853 && q.routed_chat.is_some()
1854 && !already_applied(task, q)
1855 && task.resumption.as_ref().map_or(0, |r| r.generation) == q.triage_generation.unwrap_or(0)
1856 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1857}
1858
1859fn deps_pending_in(recovery: &RecoveryIndex, task: &Task) -> bool {
1863 match recovery.latest(DEPS_NODE, &task.id) {
1864 Some(q) if consumed(task, &q) => false,
1865 Some(q) if q.status.open() || awaiting_task_chat(&q, task) => true,
1866 Some(q) if q.status == QuestionStatus::Answered => true,
1867 _ => false,
1868 }
1869}
1870
1871pub fn open_task_ids(questions: &Questions) -> std::collections::BTreeSet<String> {
1876 let queue = question_queue(questions);
1877 let tasks = queue.list();
1878 let index = display_index(questions, &queue, &tasks);
1879 tasks
1880 .iter()
1881 .filter(|task| index.displayed_open(task).is_some())
1882 .map(|task| task.id.clone())
1883 .collect()
1884}
1885
1886fn question_queue(questions: &Questions) -> Queue {
1887 Queue::at(
1888 questions
1889 .root()
1890 .parent()
1891 .unwrap_or(Path::new(""))
1892 .join("queue"),
1893 )
1894}
1895
1896fn display_index(questions: &Questions, queue: &Queue, tasks: &[Task]) -> RecoveryIndex {
1898 RecoveryIndex::from_scan(queue, tasks, questions.list())
1899}
1900
1901#[cfg(test)]
1902mod tests {
1903 use super::*;
1904 use crate::ask::Answer;
1905 use crate::queue::Source;
1906 use jiff::SignedDuration;
1907
1908 fn store() -> (tempfile::TempDir, Queue, Questions) {
1909 let dir = tempfile::tempdir().unwrap();
1910 let q = Queue::at(dir.path().join("queue"));
1911 let s = Questions::at(dir.path().join("questions"));
1912 (dir, q, s)
1913 }
1914
1915 fn task(title: &str, repo: PathBuf) -> Task {
1916 Task::new(title.to_owned(), format!("do {title}"), repo, Source::Human)
1917 }
1918
1919 fn disk_task(repo: PathBuf) -> Task {
1920 let mut t = task("capacity", repo);
1921 t.hold_machine(crate::disk::gate_in(1_477_050_368, 3_221_225_472, "ja"));
1922 t.mark_disk_capacity();
1923 t.attempts = 3;
1924 t
1925 }
1926
1927 #[test]
1928 fn disk_wait_repeated_pressure_measurement_failure_and_recovery_cost_nothing() {
1929 let (dir, queue, questions) = store();
1930 let cfg = gate_disabled_config(dir.path());
1931 let mut t = disk_task(dir.path().to_owned());
1932 queue.put(&mut t).unwrap();
1933 let notices = crate::notices::Notices::at(dir.path().join("notifications"));
1934 let before = serde_json::to_value(notices.list()).unwrap();
1935 let start = Timestamp::now();
1936 let probes = std::cell::Cell::new(0);
1937 for seconds in [0, 1, 10, 60, 61, 120] {
1938 let report = run_once_with_probe(
1939 &queue,
1940 &questions,
1941 Some(&cfg),
1942 start + SignedDuration::from_secs(seconds),
1943 true,
1944 |_, _| {
1945 probes.set(probes.get() + 1);
1946 Some(false) },
1948 );
1949 assert!(report.is_empty());
1950 assert_eq!(queue.get(&t.id).unwrap().attempts, 3);
1951 }
1952 assert_eq!(
1953 probes.get(),
1954 3,
1955 "resource probes are bounded to one per minute"
1956 );
1957 assert!(questions.list().is_empty());
1958 assert_eq!(serde_json::to_value(notices.list()).unwrap(), before);
1959 let recovered = run_once_with_probe(
1960 &queue,
1961 &questions,
1962 Some(&cfg),
1963 start + SignedDuration::from_secs(180),
1964 true,
1965 |_, _| Some(true),
1966 );
1967 assert_eq!(recovered.resumed, [t.id.clone()]);
1968 let saved = queue.get(&t.id).unwrap();
1969 assert_eq!(saved.status, TaskStatus::Queued);
1970 assert_eq!(saved.attempts, 3);
1971 let again = run_once_with_probe(
1972 &queue,
1973 &questions,
1974 Some(&cfg),
1975 start + SignedDuration::from_secs(240),
1976 true,
1977 |_, _| panic!("queued tasks are not relaunched"),
1978 );
1979 assert!(again.is_empty());
1980 }
1981
1982 #[test]
1983 fn disk_wait_supersedes_only_untouched_automatic_questions_in_both_stores() {
1984 for routed in [false, true] {
1985 let (dir, queue, questions) = store();
1986 let cfg = gate_disabled_config(dir.path());
1987 let mut t = disk_task(dir.path().to_owned());
1988 queue.put(&mut t).unwrap();
1989 let q = file_question(
1990 &questions,
1991 &crate::task_chat::InquiryOwnership::default(),
1992 &t,
1993 Bucket::MachineUnknown,
1994 &JA,
1995 Timestamp::now(),
1996 )
1997 .unwrap();
1998 if routed {
1999 crate::task_chat::sweep(
2000 &queue,
2001 &crate::talk::Talks::at(dir.path().join("talks")),
2002 &questions,
2003 dir.path(),
2004 );
2005 assert!(
2006 crate::task_inquiry::Outbox::at(dir.path())
2007 .get(&q.id)
2008 .is_ok()
2009 );
2010 }
2011 let start = Timestamp::now();
2012 for seconds in [0, 60, 120] {
2013 assert!(
2014 run_once_with_probe(
2015 &queue,
2016 &questions,
2017 Some(&cfg),
2018 start + SignedDuration::from_secs(seconds),
2019 true,
2020 |_, _| Some(false)
2021 )
2022 .is_empty()
2023 );
2024 assert!(questions.read_history(&q.id).unwrap().status.open());
2025 assert_eq!(questions.list().len(), 1);
2026 }
2027 let report = monitor_disk_waits(
2028 &queue,
2029 &questions,
2030 Some(&cfg),
2031 start + SignedDuration::from_secs(180),
2032 );
2033 assert_eq!(report.resumed, [t.id.clone()], "routed={routed}");
2034 let source = questions.read_history(&q.id).unwrap();
2035 assert_eq!(source.status, QuestionStatus::Abandoned);
2036 assert!(source.answer.is_none());
2037 assert!(source.answered_at.is_none());
2038 assert!(!pending_for(&questions, &queue.get(&t.id).unwrap()));
2039 if routed {
2040 let d = crate::task_inquiry::Outbox::at(dir.path())
2041 .get(&q.id)
2042 .unwrap()
2043 .decision
2044 .unwrap();
2045 assert_eq!(d.status, crate::task_decision::Status::Resolved);
2046 assert!(
2047 d.request.is_none(),
2048 "system supersession fabricates no owner request"
2049 );
2050 }
2051 }
2052 }
2053
2054 #[test]
2055 fn disk_wait_respects_owner_hold_discussion_and_unrelated_approvals() {
2056 for kind in [
2057 "manual",
2058 "answered",
2059 "discussion",
2060 "unrelated",
2061 "override",
2062 "native-hold",
2063 "missing-native",
2064 ] {
2065 let (dir, queue, questions) = store();
2066 let cfg = gate_disabled_config(dir.path());
2067 let mut t = disk_task(dir.path().to_owned());
2068 queue.put(&mut t).unwrap();
2069 let q = file_question(
2070 &questions,
2071 &crate::task_chat::InquiryOwnership::default(),
2072 &t,
2073 Bucket::MachineUnknown,
2074 &EN,
2075 Timestamp::now(),
2076 )
2077 .unwrap();
2078 match kind {
2079 "manual" => {
2080 t.hold_manual(Some("owner hold".into()));
2081 queue.put(&mut t).unwrap();
2082 }
2083 "answered" => {
2084 questions
2085 .update(&q.id, |q| q.answer(Answer::Choice(q.choices[1].clone())))
2086 .unwrap();
2087 }
2088 "discussion" => {
2089 questions
2090 .update(&q.id, |q| q.say("please wait".to_owned()))
2091 .unwrap();
2092 }
2093 "unrelated" => {
2094 questions
2095 .update(&q.id, |q| {
2096 q.node = "land".into();
2097 Ok(())
2098 })
2099 .unwrap();
2100 }
2101 "override" => {
2102 t.resume_override = Some(OperatorResume {
2103 question_id: "owner".into(),
2104 at: Timestamp::now(),
2105 conductor_rehold: Some("pending conductor decision".into()),
2106 forced: true,
2107 pinned_run: None,
2108 });
2109 queue.put(&mut t).unwrap();
2110 }
2111 "native-hold" | "missing-native" => {
2112 crate::task_chat::sweep(
2113 &queue,
2114 &crate::talk::Talks::at(dir.path().join("talks")),
2115 &questions,
2116 dir.path(),
2117 );
2118 if kind == "native-hold" {
2119 crate::task_inquiry::Outbox::at(dir.path())
2120 .update_decision(&q.id, |d| d.accept("owner requested hold"))
2121 .unwrap();
2122 } else {
2123 questions
2127 .update(&q.id, |q| {
2128 q.routed_chat = Some("delivered-chat".into());
2129 Ok(())
2130 })
2131 .unwrap();
2132 std::fs::remove_file(
2133 dir.path()
2134 .join("notifications/task-inquiries")
2135 .join(format!("{}.json", q.id)),
2136 )
2137 .unwrap();
2138 }
2139 }
2140 _ => unreachable!(),
2141 }
2142 let report = run_once(&queue, &questions, Some(&cfg), Timestamp::now());
2143 assert!(report.resumed.is_empty(), "{kind}");
2144 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held, "{kind}");
2145 assert!(
2146 questions.read_history(&q.id).unwrap().status != QuestionStatus::Abandoned,
2147 "{kind}"
2148 );
2149 }
2150 }
2151
2152 #[test]
2153 fn disk_wait_reuses_retained_execution_and_preserves_attempt_and_approval_evidence() {
2154 let (dir, queue, questions) = store();
2155 let cfg = gate_disabled_config(dir.path());
2156 let mut run = crate::run::RunState::new(
2157 dir.path().to_owned(),
2158 "main".into(),
2159 "base".into(),
2160 "capacity".into(),
2161 Config::default(),
2162 );
2163 run.status = crate::run::RunStatus::Blocked;
2164 run.driver_exited = true;
2165 run.save_under(dir.path()).unwrap();
2166 let mut t = disk_task(dir.path().to_owned());
2167 t.runs.push(run.id.clone());
2168 t.resume_override = Some(OperatorResume {
2169 question_id: "approved-recovery".into(),
2170 at: Timestamp::now(),
2171 conductor_rehold: None,
2172 forced: true,
2173 pinned_run: Some(run.id.clone()),
2174 });
2175 queue.put(&mut t).unwrap();
2176 let before = std::fs::read(dir.path().join("runs").join(&run.id).join("run.json")).unwrap();
2177 let report = run_once_with_probe(
2178 &queue,
2179 &questions,
2180 Some(&cfg),
2181 Timestamp::now(),
2182 true,
2183 |_, _| Some(true),
2184 );
2185 assert_eq!(report.resumed, [t.id.clone()]);
2186 let resumed = queue.get(&t.id).unwrap();
2187 assert_eq!(resumed.runs, t.runs);
2188 assert_eq!(resumed.disk_resume_run.as_deref(), Some(run.id.as_str()));
2189 assert_eq!(resumed.attempts, t.attempts);
2190 assert_eq!(resumed.resumption, t.resumption);
2191 assert_eq!(
2192 serde_json::to_value(&resumed.resume_override).unwrap(),
2193 serde_json::to_value(&t.resume_override).unwrap()
2194 );
2195 assert_eq!(
2196 std::fs::read(dir.path().join("runs").join(&run.id).join("run.json")).unwrap(),
2197 before
2198 );
2199 assert!(
2200 run_once_with_probe(
2201 &queue,
2202 &questions,
2203 Some(&cfg),
2204 Timestamp::now(),
2205 true,
2206 |_, _| panic!("no duplicate launch")
2207 )
2208 .is_empty()
2209 );
2210 }
2211
2212 #[test]
2213 fn disk_wait_interrupted_question_supersession_is_idempotent() {
2214 for abandoned in [false, true] {
2215 let (dir, queue, questions) = store();
2216 let cfg = gate_disabled_config(dir.path());
2217 let mut t = disk_task(dir.path().to_owned());
2218 queue.put(&mut t).unwrap();
2219 let q = file_question(
2220 &questions,
2221 &crate::task_chat::InquiryOwnership::default(),
2222 &t,
2223 Bucket::MachineUnknown,
2224 &EN,
2225 Timestamp::now(),
2226 )
2227 .unwrap();
2228 let outbox = crate::task_inquiry::Outbox::at(dir.path());
2229 outbox
2230 .enqueue(crate::task_chat::live_record(dir.path(), &t, &q))
2231 .unwrap();
2232 outbox
2233 .update_decision(&q.id, |d| d.settle(DISK_SUPERSEDED))
2234 .unwrap();
2235 if abandoned {
2236 questions
2237 .update(&q.id, |q| {
2238 q.abandon(DISK_SUPERSEDED);
2239 Ok(())
2240 })
2241 .unwrap();
2242 }
2243 let report = monitor_disk_waits(&queue, &questions, Some(&cfg), Timestamp::now());
2244 assert_eq!(report.resumed, [t.id.clone()]);
2245 assert_eq!(queue.get(&t.id).unwrap().attempts, 3);
2246 assert!(questions.read_history(&q.id).unwrap().answer.is_none());
2247 assert_eq!(
2248 outbox.get(&q.id).unwrap().decision.unwrap().history.len(),
2249 1
2250 );
2251 }
2252 }
2253
2254 #[test]
2255 fn disk_wait_legacy_migration_requires_machine_provenance_and_exact_localized_format() {
2256 let (dir, queue, _) = store();
2257 for (language, source, suffix, expected) in [
2258 ("en", Some(HoldSource::Machine), "", true),
2259 ("ja", Some(HoldSource::Machine), "", true),
2260 ("ja", Some(HoldSource::Manual), "", false),
2261 ("ja", None, "", false),
2262 ("ja", Some(HoldSource::Machine), " owner hold", false),
2263 ] {
2264 let mut t = task("legacy", dir.path().to_owned());
2265 t.hold_machine(Some(format!(
2266 "{}{suffix}",
2267 crate::disk::gate_in(1477050368, 3221225472, language).unwrap()
2268 )));
2269 t.schema = 21;
2270 t.hold_source = source;
2271 queue.put(&mut t).unwrap();
2272 let loaded = queue.get(&t.id).unwrap();
2273 assert_eq!(loaded.machine_hold_cause.is_some(), expected);
2274 assert_eq!(disk_wait_eligible(&loaded), expected);
2275 }
2276 }
2277
2278 #[test]
2279 fn disk_wait_refreshes_stale_notice_wording_without_redelivery() {
2280 let (dir, queue, questions) = store();
2281 let cfg = gate_disabled_config(dir.path());
2282 let mut t = disk_task(dir.path().to_owned());
2283 queue.put(&mut t).unwrap();
2284 let notices = crate::notices::Notices::at(dir.path().join("notifications"));
2285 let key = format!("disk:{}", t.repo.display());
2286 let n = notices
2287 .raise(crate::notices::Notice::warn(
2288 &key,
2289 "Free some, then release it from the queue.",
2290 ))
2291 .unwrap();
2292 notices.mark_read(&n.id).unwrap();
2293 let before = notices.get(&n.id).unwrap();
2294 let report = run_once_with_probe(
2295 &queue,
2296 &questions,
2297 Some(&cfg),
2298 Timestamp::now(),
2299 true,
2300 |_, _| Some(false),
2301 );
2302 assert!(report.is_empty());
2303 let after = notices.get(&n.id).unwrap();
2304 assert!(after.message.contains("automatically"));
2305 assert_eq!(before.generation, after.generation);
2306 assert_eq!(before.count, after.count);
2307 assert_eq!(before.read_at, after.read_at);
2308 }
2309
2310 #[test]
2311 fn disk_wait_owner_discard_is_applied_without_automatic_resumption() {
2312 let (dir, queue, questions) = store();
2313 let cfg = gate_disabled_config(dir.path());
2314 let mut t = disk_task(dir.path().to_owned());
2315 queue.put(&mut t).unwrap();
2316 let q = file_question(
2317 &questions,
2318 &crate::task_chat::InquiryOwnership::default(),
2319 &t,
2320 Bucket::MachineUnknown,
2321 &EN,
2322 Timestamp::now(),
2323 )
2324 .unwrap();
2325 questions
2326 .update(&q.id, |q| q.answer(Answer::Choice(q.choices[2].clone())))
2327 .unwrap();
2328 let report = run_once(&queue, &questions, Some(&cfg), Timestamp::now());
2329 assert!(report.resumed.is_empty());
2330 assert!(queue.get(&t.id).is_err());
2331 assert!(questions.read_history(&q.id).unwrap().answer.is_some());
2332 }
2333
2334 #[test]
2335 fn disk_wait_commit_cannot_overwrite_a_concurrent_owner_hold() {
2336 let (dir, queue, questions) = store();
2337 let mut stale = disk_task(dir.path().to_owned());
2338 queue.put(&mut stale).unwrap();
2339 let mut current = queue.get(&stale.id).unwrap();
2340 current.hold_manual(Some("wait for owner".into()));
2341 queue.put(&mut current).unwrap();
2342 assert!(
2343 resume_disk_task(
2344 &queue,
2345 &questions,
2346 &mut stale,
2347 &crate::task_chat::InquiryOwnership::default()
2348 )
2349 .is_err()
2350 );
2351 assert_eq!(
2352 queue.get(¤t.id).unwrap().hold_source,
2353 Some(HoldSource::Manual)
2354 );
2355 }
2356
2357 #[test]
2358 fn recovery_index_preserves_supersession_and_ambiguous_file_references() {
2359 let (dir, queue, questions) = store();
2360 let mut t = task("supersession", dir.path().join("repo"));
2361 t.id = "task-alpha-end".into();
2362 queue.put(&mut t).unwrap();
2363 let at: Timestamp = "2026-11-01T00:00:00Z".parse().unwrap();
2364 let mut all = Vec::new();
2365 for (i, run) in ["task-alpha-end", "task-alpha", "end"]
2366 .into_iter()
2367 .enumerate()
2368 {
2369 let mut q = Question::new(
2370 run.into(),
2371 NODE.into(),
2372 SEAT.into(),
2373 "superseded".into(),
2374 String::new(),
2375 EN.choices3(),
2376 );
2377 q.id = format!("superseded-{i}");
2378 q.asked_at = at;
2379 all.push(q);
2380 }
2381 let evidence: Vec<_> = all
2382 .iter()
2383 .map(|q| {
2384 (
2385 q.id.clone(),
2386 crate::repair_budget::reconcile::source(q).unwrap(),
2387 )
2388 })
2389 .collect();
2390 t.repair_reconciliations.push(
2391 serde_json::from_value(serde_json::json!({
2392 "request": {
2393 "task": t.id, "run": "run", "head": "head", "expect_pass": 0,
2394 "receipt": "grant", "question": "original", "answer_at": at,
2395 "request": "reconcile", "reason": "fixture"
2396 },
2397 "grant": {
2398 "request": {
2399 "task": t.id, "run": "run", "kind": "review", "rounds": 1,
2400 "head": "head", "expect_pass": 0, "request": "grant", "reason": "fixture"
2401 },
2402 "at": at, "decision": null, "contract": "fixture"
2403 },
2404 "source": "fixture", "application": "fixture", "decision": at,
2405 "superseded_questions": evidence
2406 }))
2407 .unwrap(),
2408 );
2409 queue.put(&mut t).unwrap();
2410 let index = RecoveryIndex::from_scan(&queue, &[t.clone()], all.clone());
2411 assert!(index.latest.is_empty());
2412 std::fs::write(queue.path_of("task-alphabet-other"), "{").unwrap();
2414 std::fs::write(queue.path_of("broken-end"), "{").unwrap();
2415 let index = RecoveryIndex::from_scan(&queue, &[t.clone()], all.clone());
2416 assert!(index.latest(NODE, "end").is_some());
2417 assert!(index.latest(NODE, "task-alpha").is_some());
2418 assert!(index.latest(NODE, &t.id).is_none());
2419 all[0].detail.push_str("changed evidence");
2420 let index = RecoveryIndex::from_scan(&queue, &[t], all);
2421 assert!(index.latest(NODE, "task-alpha-end").is_some());
2422 assert_eq!(questions.list().len(), 0);
2423 }
2424
2425 #[test]
2426 fn ownership_index_requires_positive_foreground_provenance() {
2427 let (dir, queue, _) = store();
2428 let mut run = crate::run::RunState::new(
2429 dir.path().join("repo"),
2430 "main".into(),
2431 "origin".into(),
2432 "fixture".into(),
2433 Config::default(),
2434 );
2435 run.origin = Some(crate::run::Origin::operator());
2436 run.save_under(dir.path()).unwrap();
2437 let foreground = run.id.clone();
2438 run.id = "owned-run".into();
2439 run.origin = Some(crate::run::Origin::queue("missing-task"));
2440 run.save_under(dir.path()).unwrap();
2441 let mut task = task("owner", dir.path().join("repo"));
2442 task.runs = vec!["linked-run".into()];
2443 queue.put(&mut task).unwrap();
2444 let ownership = crate::task_chat::InquiryOwnership::default();
2445 for (run, cwd, node, seat, expected) in [
2446 (foreground.as_str(), true, "node", "seat", true),
2447 (foreground.as_str(), false, "node", "seat", false),
2448 ("owned-run", true, "ask", "operator", false),
2449 ("linked-run", true, "ask", "operator", false),
2450 ("missing-run", true, "node", "seat", false),
2451 ("missing-run", false, "ask", "operator", true),
2452 ] {
2453 let mut q = Question::new(
2454 run.into(),
2455 node.into(),
2456 seat.into(),
2457 "provenance".into(),
2458 String::new(),
2459 Vec::new(),
2460 );
2461 q.cwd = cwd.then(|| dir.path().join("repo").display().to_string());
2462 assert_eq!(standalone_by_scanning(dir.path(), &q), expected);
2463 assert_eq!(
2464 crate::task_chat::standalone_inquiry_in(dir.path(), &q, &ownership),
2465 expected
2466 );
2467 }
2468 }
2469
2470 fn standalone_by_scanning(home: &Path, q: &Question) -> bool {
2472 if q.routed_task.is_some()
2473 || (q.node == crate::bump::NOTICE_NODE && q.seat == "release-watch")
2474 {
2475 return false;
2476 }
2477 if q.legacy_foreground || q.run.is_empty() {
2478 return true;
2479 }
2480 if Queue::at(home.join("queue"))
2481 .list()
2482 .iter()
2483 .any(|task| task.id == q.run || task.runs.contains(&q.run))
2484 {
2485 return false;
2486 }
2487 let run = crate::run::RunState::load_under(&q.run, home).ok();
2488 let origin = run.as_ref().and_then(|run| run.origin.as_ref());
2489 if origin.is_some_and(|origin| origin.task.is_some()) {
2490 return false;
2491 }
2492 (q.node == "ask" && q.seat == "operator")
2493 || (q.cwd.is_some() && origin.is_some_and(|origin| origin.task.is_none()))
2494 }
2495
2496 fn scan_counts() -> (usize, usize, usize) {
2497 (
2498 crate::queue::LIST_READS.with(|c| c.replace(0)),
2499 crate::queue::TASK_READS.with(|c| c.replace(0)),
2500 crate::ask::QUESTION_SCANS.with(|c| c.replace(0)),
2501 )
2502 }
2503
2504 #[test]
2505 fn creating_and_updating_questions_share_one_queue_read() {
2506 let (dir, queue, questions) = store();
2507 let size = 200;
2508 let mut ids = Vec::new();
2509 for i in 0..size {
2510 let mut t = task("scan fixture", dir.path().join("repo"));
2511 t.id = format!("task-{i:04}");
2512 t.runs = vec![format!("run-{i:04}")];
2513 t.hold_manual(Some("held for scan fixture".into()));
2514 queue.put(&mut t).unwrap();
2515 ids.push(t.id);
2516 }
2517 let ownership = crate::task_chat::InquiryOwnership::default();
2518 scan_counts();
2519 let mut created = Vec::new();
2520 for id in &ids {
2521 let mut q = Question::new(
2522 id.clone(),
2523 NODE.into(),
2524 SEAT.into(),
2525 "scan fixture".into(),
2526 String::new(),
2527 EN.choices3(),
2528 );
2529 questions.put_executable_in(&mut q, &ownership).unwrap();
2530 assert_eq!(q.routed_task.as_deref(), Some(id.as_str()));
2531 created.push(q.id);
2532 }
2533 for id in &created {
2534 questions
2535 .update_in(id, &ownership, |q| {
2536 q.answer_delivered = true;
2537 Ok(())
2538 })
2539 .unwrap();
2540 }
2541 let (lists, _, _) = scan_counts();
2542 assert_eq!(lists, 1, "one queue read for {size} creations and updates");
2543 }
2544
2545 #[test]
2546 fn recovery_scans_are_linear_and_match_legacy_decisions() {
2547 let mut previous = None;
2548 for size in [150, 300] {
2549 let (dir, queue, questions) = store();
2550 let at: Timestamp = "2026-11-01T00:00:00Z".parse().unwrap();
2551 let mut sources = Vec::new();
2552 let mut expected_applied = std::collections::BTreeSet::new();
2553 for i in 0..size {
2554 let mut t = task("scan fixture", dir.path().join("repo"));
2555 t.id = format!("task-{i:04}");
2556 t.runs = vec![format!("run-{i:04}")];
2557 t.hold_manual(Some("held for scan fixture".into()));
2558 queue.put(&mut t).unwrap();
2559 let mut q = Question::new(
2560 t.id.clone(),
2561 NODE.into(),
2562 SEAT.into(),
2563 "scan fixture".into(),
2564 String::new(),
2565 EN.choices3(),
2566 );
2567 q.id = format!("question-{i:04}");
2568 q.asked_at = at;
2569 match i % 10 {
2570 0 => {
2571 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2572 q.accepted_answer_at = q.answered_at;
2573 expected_applied.insert(t.id.clone());
2574 }
2575 1 => q.node = DEPS_NODE.into(),
2576 2 => {
2577 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
2578 }
2579 3 => {
2580 q.run = format!("missing-{i}");
2581 q.legacy_foreground = true;
2582 }
2583 4 => {
2584 q.run.clear();
2585 q.node = crate::bump::NOTICE_NODE.into();
2586 q.seat = "release-watch".into();
2587 }
2588 5 => {
2589 q.run = format!("missing-{i}");
2590 q.node = "ask".into();
2591 q.seat = "operator".into();
2592 }
2593 6 => q.run = t.runs[0].clone(),
2594 7 => q.run = format!("missing-{i}"),
2595 8 => q.abandon("fixture"),
2596 _ => {}
2597 }
2598 questions.put(&mut q).unwrap();
2599 sources.push(q.id.clone());
2600 if i % 10 == 9 {
2601 q.id = format!("question-{i:04}-tie");
2602 questions.put(&mut q).unwrap();
2603 sources.push(q.id.clone());
2604 }
2605 }
2606 let migration = dir.path().join("notifications/task-chat-migration-v1");
2607 std::fs::create_dir_all(&migration).unwrap();
2608 std::fs::write(
2609 migration.join("state.json"),
2610 serde_json::to_vec(&serde_json::json!({
2611 "schema": 1, "classification_revision": 1, "pending": [], "sources": sources
2612 }))
2613 .unwrap(),
2614 )
2615 .unwrap();
2616
2617 scan_counts();
2618 let all = questions.list_for_effects();
2619 let (lists, reads, scans) = scan_counts();
2620 assert_eq!((lists, reads, scans), (1, size, 1));
2621 let tasks = queue.list();
2622 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
2623 for q in &all {
2624 assert_eq!(
2625 crate::task_chat::standalone_inquiry_in(dir.path(), q, &ownership),
2626 standalone_by_scanning(dir.path(), q),
2627 "{}",
2628 q.id,
2629 );
2630 if q.id.ends_with("0002") {
2631 assert!(q.answer.is_none(), "saved consent remains fenced");
2632 }
2633 }
2634 let recovery = RecoveryIndex::from_scan(&queue, &tasks, all.clone());
2635 for task in &tasks {
2636 for node in [NODE, DEPS_NODE] {
2637 let expected = all
2638 .iter()
2639 .filter(|q| {
2640 q.node == node
2641 && q.run == task.id
2642 && !queue.get(&q.run).ok().is_some_and(|t| {
2643 crate::repair_budget::reconcile::superseded_question(&t, q)
2644 })
2645 })
2646 .max_by(|a, b| a.asked_at.cmp(&b.asked_at).then_with(|| a.id.cmp(&b.id)));
2647 assert_eq!(
2648 recovery.latest(node, &task.id).map(|q| q.id),
2649 expected.map(|q| q.id.clone())
2650 );
2651 }
2652 }
2653
2654 scan_counts();
2655 crate::daemon::apply_task_choice_actions(&queue, &questions, dir.path());
2656 assert_eq!(
2657 scan_counts(),
2658 (1, size, 1),
2659 "choice-action rechecks share ownership"
2660 );
2661
2662 scan_counts();
2663 let applied: std::collections::BTreeSet<_> =
2664 apply_answers(&queue, &questions).into_iter().collect();
2665 let counts = scan_counts();
2666 assert_eq!(applied, expected_applied);
2667 assert!(counts.0 <= 3, "queue scans: {counts:?}");
2668 assert!(counts.1 <= 4 * size, "task reads: {counts:?}");
2669 assert_eq!(counts.2, 2, "question scans: {counts:?}");
2670 if let Some((_, reads, scans)) = previous {
2671 assert!(counts.1 <= 2 * reads);
2672 assert_eq!(counts.2, scans);
2673 }
2674 previous = Some(counts);
2675
2676 let config = gate_disabled_config(dir.path());
2677 scan_counts();
2678 run_once(&queue, &questions, Some(&config), Timestamp::now());
2679 let lap = scan_counts();
2680 assert!(lap.0 <= 12, "triage queue scans: {lap:?}");
2681 assert!(lap.1 <= 14 * size, "triage task reads: {lap:?}");
2682 assert!(lap.2 <= 6, "triage question scans: {lap:?}");
2683 }
2684 }
2685
2686 #[test]
2687 fn run_once_filing_many_new_questions_reads_the_queue_a_constant_number_of_times() {
2688 let (dir, queue, questions) = store();
2689 let size = 200;
2690 for i in 0..size {
2691 let mut t = task("scan fixture", dir.path().join("repo"));
2692 t.id = format!("task-{i:04}");
2693 t.hold_machine(Some("gate red".to_owned()));
2694 queue.put(&mut t).unwrap();
2695 }
2696 scan_counts();
2697 let report = run_once(&queue, &questions, None, Timestamp::now());
2698 let (lists, _, _) = scan_counts();
2699 assert_eq!(report.asked.len(), size, "every held task is asked about");
2700 assert!(
2701 lists <= 12,
2702 "queue scans while filing {size} questions: {lists}"
2703 );
2704 }
2705
2706 #[test]
2707 fn base_recovery_stops_suppress_generic_and_dependency_resume_offers() {
2708 let (dir, queue, questions) = store();
2709 let mut root = task("base stop", dir.path().to_path_buf());
2710 let mut run = crate::run::RunState::new(
2711 dir.path().to_path_buf(),
2712 "main".into(),
2713 "base".into(),
2714 "retained implementation".into(),
2715 Config::default(),
2716 );
2717 run.status = crate::run::RunStatus::Voting;
2718 run.base_sync = Some(crate::run::BaseSync {
2719 tip: "tip".into(),
2720 behind: 3,
2721 attempts: 4,
2722 conflict: Some("persisted exhaustion".into()),
2723 already_in: None,
2724 });
2725 run.save_under(dir.path()).unwrap();
2726 root.link_run(&run.id);
2727 root.hold_machine(Some("base stop".into()));
2728 queue.put(&mut root).unwrap();
2729 let mut dependent = task("dependent", dir.path().to_path_buf());
2730 dependent.blocked_by = vec![root.id.clone()];
2731 dependent.status = TaskStatus::Blocked;
2732 queue.put(&mut dependent).unwrap();
2733 for _ in 0..2 {
2734 let report = run_once(&queue, &questions, None, Timestamp::now());
2735 assert!(report.asked.is_empty());
2736 assert!(report.resumed.is_empty());
2737 }
2738 let q = file_question(
2741 &questions,
2742 &crate::task_chat::InquiryOwnership::default(),
2743 &root,
2744 Bucket::MachineUnknown,
2745 &EN,
2746 Timestamp::now(),
2747 )
2748 .unwrap();
2749 let mut q = q;
2750 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
2751 questions.put_executable(&mut q).unwrap();
2752 apply_answers(&queue, &questions);
2753 let saved = queue.get(&root.id).unwrap();
2754 assert_eq!(saved.status, TaskStatus::Held);
2755 assert!(saved.triage_applied(&q.id));
2756 assert_eq!(saved.runs, vec![run.id]);
2757 }
2758
2759 #[test]
2760 fn resource_holds_only_auto_release_a_proven_cache_wait_timeout() {
2761 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
2762 let dir = tempfile::tempdir().unwrap();
2763 let cfg = Config::default(); assert!(cfg.cache_dir().is_none());
2765 let mut state = RunState::new(
2766 dir.path().into(),
2767 "main".into(),
2768 "base".into(),
2769 "verify".into(),
2770 cfg.clone(),
2771 );
2772 let mut task = task("verify", dir.path().into());
2773 task.runs = vec![state.id.clone()];
2774 task.hold_machine(Some("[resource] cache operation stopped".into()));
2775 for cause in [
2776 ResourceBlockCause::Other,
2777 ResourceBlockCause::Unrecorded,
2778 ResourceBlockCause::Cancelled { park: None },
2779 ] {
2780 state.resource_block = Some(ResourceBlock {
2781 node: "gate".into(),
2782 kind: "shared build cache".into(),
2783 detail: "persistent metadata or manifest error".into(),
2784 cause,
2785 at: Timestamp::now(),
2786 });
2787 state.save_under(dir.path()).unwrap();
2788 for _ in 0..3 {
2790 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2791 }
2792 }
2793 let block = state.resource_block.as_mut().unwrap();
2794 block.cause = ResourceBlockCause::Timeout {
2795 waited_secs: 10,
2796 limit_secs: 10,
2797 };
2798 state.save_under(dir.path()).unwrap();
2799 assert_eq!(
2800 machine_cause_resolved_under(&task, &cfg, dir.path()),
2801 Some(true)
2802 );
2803 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
2804 state.save_under(dir.path()).unwrap();
2805 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2806 task.runs = vec!["unreadable-run".into()];
2807 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2808 }
2809
2810 #[cfg(unix)]
2811 #[test]
2812 fn disk_admission_holds_recheck_live_reservations_and_the_floor() {
2813 use crate::retention::OutputLease;
2814 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
2815 let temp = tempfile::tempdir().unwrap();
2816 let home = temp.path().canonicalize().unwrap();
2817 let mut cfg = Config::default();
2818 cfg.graph.worktree_root = Some(home.join("worktrees"));
2819 cfg.disk.min_free_bytes = 100;
2820 let cache = home.join("cache");
2821 cfg.verify.gate = vec![format!(
2822 "CARGO_TARGET_DIR='{}' cargo build",
2823 cache.display()
2824 )];
2825 let mut state = RunState::new(
2826 home.clone(),
2827 "main".into(),
2828 "base".into(),
2829 "verify".into(),
2830 cfg.clone(),
2831 );
2832 let saved_cache = home.join("saved-cache");
2833 state.config.verify.gate = vec![format!(
2834 "CARGO_TARGET_DIR='{}' cargo build",
2835 saved_cache.display()
2836 )];
2837 state.resource_block = Some(ResourceBlock {
2838 node: "gate".into(),
2839 kind: "disk admission".into(),
2840 detail: "writer headroom unavailable".into(),
2841 cause: ResourceBlockCause::Other,
2842 at: Timestamp::now(),
2843 });
2844 state.save_under(&home).unwrap();
2845 let mut task = task("verify", home.clone());
2846 task.runs = vec![state.id.clone()];
2847 task.hold_machine(Some(format!(
2848 "{}{}; no attempt was spent; resume to retry the affected operation",
2849 crate::cache::RESOURCE_HOLD,
2850 state.resource_block.as_ref().unwrap().description()
2851 )));
2852 task.schema = 21; let gib = 1024 * 1024 * 1024;
2854 let free = std::cell::Cell::new(100 + gib - 1);
2855 let check = || {
2856 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths, floor| {
2857 assert!(
2858 paths.contains(&cache),
2859 "the cache volume must also be checked"
2860 );
2861 assert!(
2862 paths.contains(&saved_cache),
2863 "saved and current cache locations both matter"
2864 );
2865 assert!(paths.contains(&state.worktree_root()));
2866 OutputLease::admit_with(&home, paths, floor, |_| Ok(free.get())).is_ok()
2867 })
2868 };
2869 assert_eq!(check(), Some(false), "the unchanged floor still applies");
2870 assert_eq!(
2871 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths, floor| {
2872 OutputLease::admit_with(&home, paths, floor, |_| {
2873 Err(anyhow::anyhow!("measurement unavailable"))
2874 })
2875 .is_ok()
2876 }),
2877 Some(false),
2878 "failed output-volume measurements must remain closed"
2879 );
2880 free.set(100 + gib);
2881 let writer = OutputLease::admit_with(
2882 &home,
2883 std::slice::from_ref(&home),
2884 cfg.disk.min_free_bytes,
2885 |_| Ok(free.get()),
2886 )
2887 .unwrap();
2888 assert_eq!(
2889 check(),
2890 Some(false),
2891 "df alone cannot resolve a live reservation"
2892 );
2893 drop(writer);
2894 assert_eq!(
2895 check(),
2896 Some(true),
2897 "releasing headroom resolves the machine hold"
2898 );
2899 assert_eq!(
2900 check(),
2901 Some(true),
2902 "the probe must release its temporary reservation"
2903 );
2904
2905 let namespace = home.join("disk-admission");
2907 let saved = home.join("saved-admission");
2908 std::fs::rename(&namespace, &saved).unwrap();
2909 std::os::unix::fs::symlink(&saved, &namespace).unwrap();
2910 assert_eq!(check(), Some(false));
2911 std::fs::remove_file(&namespace).unwrap();
2912 std::fs::rename(saved, namespace).unwrap();
2913 assert_eq!(check(), Some(true));
2914
2915 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Cancelled { park: None };
2916 state.save_under(&home).unwrap();
2917 assert_eq!(
2918 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
2919 "a cancelled operation must remain held"
2920 )),
2921 None
2922 );
2923 state.resource_block.as_mut().unwrap().kind = "incremental cache admission".into();
2924 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Other;
2925 state.save_under(&home).unwrap();
2926 assert_eq!(
2927 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
2928 "unproven incremental caches must remain held"
2929 )),
2930 None
2931 );
2932 task.runs.push("unreadable-run".into());
2933 assert_eq!(
2934 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
2935 "missing evidence must not release a hold"
2936 )),
2937 None
2938 );
2939 }
2940
2941 fn gate_disabled_config(dir: &std::path::Path) -> PathBuf {
2945 let config = dir.join("magi.toml");
2946 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2947 config
2948 }
2949
2950 #[test]
2951 fn resumption_later_keep_held_answer_wins_but_an_older_answer_cannot_undo_release() {
2952 for source in ["cli", "web"] {
2953 for answer_after_release in [true, false] {
2954 let (dir, queue, questions) = store();
2955 let mut t = task("recovery", dir.path().join("repo"));
2956 t.hold_machine(Some("inspect remote first".into()));
2957 queue.put(&mut t).unwrap();
2958 let mut q = file_question(
2959 &questions,
2960 &crate::task_chat::InquiryOwnership::default(),
2961 &t,
2962 Bucket::MachineUnknown,
2963 &JA,
2964 Timestamp::now(),
2965 )
2966 .unwrap();
2967 if !answer_after_release {
2968 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2969 questions.put(&mut q).unwrap();
2970 }
2971 t.release_operator(source, None);
2972 queue.put(&mut t).unwrap();
2973 if answer_after_release {
2974 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2975 questions.put(&mut q).unwrap();
2976 }
2977 apply_answers(&queue, &questions);
2978 let after = queue.get(&t.id).unwrap();
2979 assert!(after.triage_applied(&q.id));
2980 assert_eq!(after.operator_held(), answer_after_release);
2981 assert_eq!(
2982 after.status,
2983 if answer_after_release {
2984 TaskStatus::Held
2985 } else {
2986 TaskStatus::Queued
2987 }
2988 );
2989 assert!(apply_answers(&queue, &questions).is_empty());
2990 }
2991 }
2992 }
2993
2994 #[test]
2995 fn explicit_resumption_applies_legacy_queued_triage_once_and_preserves_later_holds() {
2996 let (dir, queue, questions) = store();
2997 let mut t = task("recover", dir.path().join("repo"));
2998 t.hold_machine(Some("inspect first".into()));
2999 queue.put(&mut t).unwrap();
3000 let mut q = file_question(
3001 &questions,
3002 &crate::task_chat::InquiryOwnership::default(),
3003 &t,
3004 Bucket::MachineUnknown,
3005 &JA,
3006 Timestamp::now(),
3007 )
3008 .unwrap();
3009 assert!(q.actions.is_empty());
3010 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
3011 questions.put(&mut q).unwrap();
3012 t.release();
3014 t.instruction = "Reuse approved PR head".into();
3015 queue.put(&mut t).unwrap();
3016 assert_eq!(apply_answers(&queue, &questions), [t.id.clone()]);
3017 let mut after = queue.get(&t.id).unwrap();
3018 assert_eq!(
3019 after.resumption.as_ref().unwrap().question_id.as_deref(),
3020 Some(q.id.as_str())
3021 );
3022 assert_eq!(
3023 after.resumption.as_ref().unwrap().instruction,
3024 t.instruction
3025 );
3026 assert_eq!(after.answers[0].answer, JA.resume);
3027 assert!(after.triage_applied(&q.id));
3028 assert!(apply_answers(&Queue::at(dir.path().join("queue")), &questions).is_empty());
3029 after.hold_machine(Some("new blocker".into()));
3030 queue.put(&mut after).unwrap();
3031 let mut late = file_question(
3032 &questions,
3033 &crate::task_chat::InquiryOwnership::default(),
3034 &after,
3035 Bucket::MachineUnknown,
3036 &EN,
3037 Timestamp::now(),
3038 )
3039 .unwrap();
3040 after.hold_manual(Some("keep held now".into()));
3041 queue.put(&mut after).unwrap();
3042 late.answer(Answer::Choice(late.choices[0].clone()))
3043 .unwrap();
3044 questions.put(&mut late).unwrap();
3045 apply_answers(&queue, &questions);
3046 let held = queue.get(&t.id).unwrap();
3047 assert!(held.operator_held());
3048 assert_eq!(held.hold_reason.as_deref(), Some("keep held now"));
3049 assert_eq!(held.resumption.as_ref().unwrap().generation, 1);
3050 }
3051
3052 #[test]
3053 fn a_resolved_machine_hold_is_requeued_automatically() {
3054 let (dir, q, questions) = store();
3055 let config = gate_disabled_config(dir.path());
3056 let mut t = task("disk pressure", dir.path().join("repo"));
3057 t.hold_machine(Some(
3058 "not enough free space to start a run: 10 bytes free, 100 required by \
3059 `[disk] min_free_bytes`"
3060 .to_owned(),
3061 ));
3062 t.schema = 21; q.put(&mut t).unwrap();
3064
3065 let report = run_once(&q, &questions, Some(&config), Timestamp::now());
3066 assert_eq!(report.resumed, [t.id.clone()]);
3067 assert!(report.asked.is_empty());
3068
3069 let back = q.get(&t.id).unwrap();
3070 assert_eq!(back.status, TaskStatus::Queued);
3071 assert!(back.hold_source.is_none());
3072 assert!(questions.list().is_empty(), "nothing needed asking");
3073 }
3074
3075 #[test]
3076 fn a_machine_hold_with_no_recognised_cause_gets_one_question_not_two() {
3077 let (dir, q, questions) = store();
3078 let mut t = task("gate went red", dir.path().join("repo"));
3079 t.hold_machine(Some("gate red".to_owned()));
3080 q.put(&mut t).unwrap();
3081
3082 let first = run_once(&q, &questions, None, Timestamp::now());
3083 assert_eq!(first.asked, [t.id.clone()]);
3084 assert!(first.resumed.is_empty());
3085
3086 let open: Vec<_> = questions
3087 .list()
3088 .into_iter()
3089 .filter(|q| q.status.open())
3090 .collect();
3091 assert_eq!(open.len(), 1);
3092 assert_eq!(open[0].run, t.id);
3093 assert_eq!(open[0].node, NODE);
3094 assert_eq!(open[0].choices.len(), 3);
3095
3096 let second = run_once(&q, &questions, None, Timestamp::now());
3098 assert!(second.asked.is_empty());
3099 assert_eq!(
3100 questions
3101 .list()
3102 .into_iter()
3103 .filter(|q| q.status.open())
3104 .count(),
3105 1
3106 );
3107 }
3108
3109 #[test]
3110 fn a_legacy_hold_with_no_recorded_source_gets_exactly_one_question() {
3111 let (dir, q, questions) = store();
3112 let mut t = task("schema 1 record", dir.path().join("repo"));
3113 t.status = TaskStatus::Held;
3114 assert!(t.hold_source.is_none(), "the case this test is about");
3115 q.put(&mut t).unwrap();
3116
3117 let first = run_once(&q, &questions, None, Timestamp::now());
3118 assert_eq!(first.asked, [t.id.clone()]);
3119
3120 let second = run_once(&q, &questions, None, Timestamp::now());
3121 assert!(
3122 second.asked.is_empty(),
3123 "the same legacy hold must not be asked about twice"
3124 );
3125 assert_eq!(
3126 questions
3127 .list()
3128 .into_iter()
3129 .filter(|q| q.status.open())
3130 .count(),
3131 1
3132 );
3133 }
3134
3135 #[test]
3136 fn dependency_resolution_detaches_legacy_short_refs_without_dropping_other_waits() {
3137 let (dir, queue, questions) = store();
3138 let mut root = task("root", dir.path().join("repo"));
3139 root.id = "20261013-193611-ca5b".into();
3140 root.hold_machine(Some("needs owner decision".into()));
3141 queue.put(&mut root).unwrap();
3142 let mut other = task("other", dir.path().join("repo"));
3143 other.id = "20261013-192213-feb2".into();
3144 queue.put(&mut other).unwrap();
3145 let mut waiting = task("waiting", dir.path().join("repo"));
3146 waiting.block(vec!["ca5b".into(), other.id.clone()], None);
3147 queue.put(&mut waiting).unwrap();
3148 detach_dependants(&queue, &questions, &root.id);
3149 let after = queue.get(&waiting.id).unwrap();
3150 assert_eq!(after.status, TaskStatus::Blocked);
3151 assert_eq!(after.blocked_by, [other.id]);
3152 }
3153
3154 #[test]
3155 fn a_manual_hold_is_never_auto_resumed() {
3156 let (dir, q, questions) = store();
3157 let mut t = task("operator stopped this", dir.path().join("repo"));
3158 t.hold_manual(Some("waiting on a decision".to_owned()));
3159 q.put(&mut t).unwrap();
3160
3161 let report = run_once(&q, &questions, None, Timestamp::now());
3162 assert!(report.resumed.is_empty());
3163 assert!(report.asked.is_empty());
3165
3166 let back = q.get(&t.id).unwrap();
3167 assert_eq!(back.status, TaskStatus::Held);
3168 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3169 assert!(questions.list().is_empty());
3170 }
3171
3172 #[test]
3173 fn a_blocked_task_on_a_deleted_dependency_is_held_and_asked_about_in_one_pass() {
3174 let (dir, q, questions) = store();
3179 let mut still_going = task("still valid", dir.path().join("repo"));
3180 q.put(&mut still_going).unwrap();
3181
3182 let mut t = task("orphaned", dir.path().join("repo"));
3183 t.block(
3184 vec!["20260101-000000-gone".to_owned(), still_going.id.clone()],
3185 Some("waits on both".to_owned()),
3186 );
3187 q.put(&mut t).unwrap();
3188
3189 let report = run_once(&q, &questions, None, Timestamp::now());
3190 assert_eq!(report.quarantined, [t.id.clone()]);
3191 assert_eq!(
3192 report.asked,
3193 [t.id.clone()],
3194 "the fresh machine hold must earn a question in the same pass"
3195 );
3196
3197 let after = q.get(&t.id).unwrap();
3198 assert_eq!(after.status, TaskStatus::Held);
3199 assert_eq!(after.hold_source, Some(HoldSource::Machine));
3200 assert!(after.blocked_by.is_empty());
3201
3202 assert!(!is_disk_hold(&after));
3205
3206 let open: Vec<_> = questions
3207 .list()
3208 .into_iter()
3209 .filter(|q| q.status.open())
3210 .collect();
3211 assert_eq!(open.len(), 1);
3212 assert_eq!(open[0].run, t.id);
3213
3214 let second = run_once(&q, &questions, None, Timestamp::now());
3216 assert!(second.quarantined.is_empty());
3217 assert!(second.asked.is_empty());
3218 }
3219
3220 fn stuck_chain(q: &Queue, dir: &std::path::Path) -> (Task, Task, Task) {
3223 let mut root = task("root", dir.join("repo"));
3224 root.hold_manual(Some("waiting".to_owned()));
3225 q.put(&mut root).unwrap();
3226 let mut mid = task("mid", dir.join("repo"));
3227 mid.block(vec![root.id.clone()], None);
3228 q.put(&mut mid).unwrap();
3229 let mut leaf = task("leaf", dir.join("repo"));
3230 leaf.block(vec![mid.id.clone()], None);
3231 q.put(&mut leaf).unwrap();
3232 (root, mid, leaf)
3233 }
3234
3235 fn deps_questions(questions: &Questions) -> Vec<Question> {
3236 questions
3237 .list()
3238 .into_iter()
3239 .filter(|q| q.node == DEPS_NODE)
3240 .collect()
3241 }
3242
3243 fn answer_deps(questions: &Questions, choice: usize) {
3244 let mut asked = deps_questions(questions).remove(0);
3245 let c = asked.choices[choice].clone();
3246 asked.answer(Answer::Choice(c)).unwrap();
3247 questions.put(&mut asked).unwrap();
3248 }
3249
3250 #[test]
3251 fn a_stuck_chain_earns_one_question_naming_every_dependant_and_is_not_reasked() {
3252 let (dir, q, questions) = store();
3253 let (root, mid, leaf) = stuck_chain(&q, dir.path());
3254
3255 let first = run_once(&q, &questions, None, Timestamp::now());
3256 assert_eq!(first.asked, std::slice::from_ref(&root.id));
3257 let asked = deps_questions(&questions);
3258 assert_eq!(asked.len(), 1, "one question per root, not per dependant");
3259 assert_eq!(asked[0].run, root.id);
3260 assert_eq!(asked[0].choices.len(), 3);
3261 assert!(asked[0].detail.contains(mid.short()), "{}", asked[0].detail);
3262 assert!(
3263 asked[0].detail.contains(leaf.short()),
3264 "{}",
3265 asked[0].detail
3266 );
3267
3268 let second = run_once(&q, &questions, None, Timestamp::now());
3269 assert!(second.asked.is_empty());
3270 assert_eq!(deps_questions(&questions).len(), 1);
3271 assert!(pending_for(&questions, &q.get(&root.id).unwrap()));
3272 }
3273
3274 #[test]
3275 fn a_stuck_root_with_a_machine_hold_gets_only_the_dependency_question() {
3276 let (dir, q, questions) = store();
3277 let mut root = task("root", dir.path().join("repo"));
3278 root.hold_machine(Some("gate red".to_owned()));
3279 q.put(&mut root).unwrap();
3280 let mut dep = task("dep", dir.path().join("repo"));
3281 dep.block(vec![root.id.clone()], None);
3282 q.put(&mut dep).unwrap();
3283
3284 run_once(&q, &questions, None, Timestamp::now());
3285 run_once(&q, &questions, None, Timestamp::now());
3286 assert_eq!(deps_questions(&questions).len(), 1);
3287 assert_eq!(
3288 questions.list().len(),
3289 1,
3290 "no second question about the hold"
3291 );
3292 }
3293
3294 #[test]
3295 fn a_dependency_that_can_still_run_asks_nothing() {
3296 let (dir, q, questions) = store();
3297 let mut live = task("live", dir.path().join("repo"));
3298 q.put(&mut live).unwrap();
3299 let mut dep = task("dep", dir.path().join("repo"));
3300 dep.block(vec![live.id.clone()], None);
3301 q.put(&mut dep).unwrap();
3302 let report = run_once(&q, &questions, None, Timestamp::now());
3303 assert!(report.asked.is_empty());
3304 assert!(questions.list().is_empty());
3305 }
3306
3307 #[test]
3308 fn answering_release_requeues_the_root_and_is_not_reasked() {
3309 let (dir, q, questions) = store();
3310 let (root, _mid, _leaf) = stuck_chain(&q, dir.path());
3311 run_once(&q, &questions, None, Timestamp::now());
3312 answer_deps(&questions, 0);
3313
3314 let report = run_once(&q, &questions, None, Timestamp::now());
3315 assert_eq!(report.answered, std::slice::from_ref(&root.id));
3316 assert!(report.asked.is_empty(), "applying must not re-ask");
3317 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Queued);
3318 let again = run_once(&q, &questions, None, Timestamp::now());
3319 assert!(again.asked.is_empty() && again.answered.is_empty());
3320 assert_eq!(deps_questions(&questions).len(), 1);
3321 }
3322
3323 #[test]
3324 fn answering_detach_frees_the_direct_dependant_and_keeps_the_root() {
3325 let (dir, q, questions) = store();
3326 let (root, mid, leaf) = stuck_chain(&q, dir.path());
3327 run_once(&q, &questions, None, Timestamp::now());
3328 answer_deps(&questions, 2);
3329
3330 let report = run_once(&q, &questions, None, Timestamp::now());
3331 assert!(report.asked.is_empty());
3332 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
3333 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
3334 let leaf = q.get(&leaf.id).unwrap();
3335 assert_eq!(leaf.status, TaskStatus::Blocked, "still waits on mid");
3336 assert_eq!(leaf.blocked_by, std::slice::from_ref(&mid.id));
3337 }
3338
3339 #[test]
3340 fn a_dependant_that_can_progress_through_another_blocker_is_not_stuck() {
3341 let (dir, q, questions) = store();
3342 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3343 let mut live = task("live", dir.path().join("repo"));
3344 q.put(&mut live).unwrap();
3345 let mut both = q.get(&mid.id).unwrap();
3346 both.block(vec![root.id.clone(), live.id.clone()], None);
3347 q.put(&mut both).unwrap();
3348 assert!(
3350 run_once(&q, &questions, None, Timestamp::now())
3351 .asked
3352 .is_empty()
3353 );
3354 }
3355
3356 #[test]
3357 fn answering_discard_detaches_then_removes_the_root() {
3358 let (dir, q, questions) = store();
3359 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3360 run_once(&q, &questions, None, Timestamp::now());
3361 answer_deps(&questions, 1);
3362
3363 let report = run_once(&q, &questions, None, Timestamp::now());
3364 assert!(q.get(&root.id).is_err());
3365 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
3366 assert!(
3367 report.asked.is_empty(),
3368 "no per-dependant quarantine question"
3369 );
3370 assert!(report.quarantined.is_empty());
3371 }
3372
3373 #[test]
3374 fn an_unmatched_deps_answer_changes_nothing_and_is_not_reasked() {
3375 let (dir, q, questions) = store();
3376 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3377 run_once(&q, &questions, None, Timestamp::now());
3378 let mut asked = deps_questions(&questions).remove(0);
3379 asked.choices.clear();
3380 asked.answer(Answer::Text("dunno".to_owned())).unwrap();
3381 questions.put(&mut asked).unwrap();
3382
3383 for _ in 0..2 {
3384 let report = run_once(&q, &questions, None, Timestamp::now());
3385 assert!(report.asked.is_empty() && report.answered.is_empty());
3386 }
3387 assert_eq!(deps_questions(&questions).len(), 1);
3388 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
3389 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Blocked);
3390 }
3391
3392 #[test]
3393 fn a_dependency_cycle_terminates_and_is_asked_about_once() {
3394 let (dir, q, questions) = store();
3395 let mut a = task("a", dir.path().join("repo"));
3396 let mut b = task("b", dir.path().join("repo"));
3397 a.block(vec![b.id.clone()], None);
3398 b.block(vec![a.id.clone()], None);
3399 q.put(&mut a).unwrap();
3400 q.put(&mut b).unwrap();
3401
3402 let first = run_once(&q, &questions, None, Timestamp::now());
3403 assert_eq!(first.asked.len(), 1);
3404 let second = run_once(&q, &questions, None, Timestamp::now());
3405 assert!(second.asked.is_empty());
3406 assert_eq!(deps_questions(&questions).len(), 1);
3407 }
3408
3409 #[test]
3410 fn a_missing_dependency_is_still_quarantined_not_asked_about_as_stuck() {
3411 let (dir, q, questions) = store();
3412 let mut t = task("orphan", dir.path().join("repo"));
3413 t.block(vec!["20260101-000000-gone".to_owned()], None);
3414 q.put(&mut t).unwrap();
3415 let report = run_once(&q, &questions, None, Timestamp::now());
3416 assert_eq!(report.quarantined, [t.id.clone()]);
3417 assert!(deps_questions(&questions).is_empty());
3418 }
3419
3420 #[test]
3421 fn a_stale_manual_hold_can_be_discarded() {
3422 let (dir, q, questions) = store();
3423 let mut t = task("been sitting a while", dir.path().join("repo"));
3424 t.hold_manual(Some("waiting on a decision".to_owned()));
3425 q.put(&mut t).unwrap();
3426 let mut back = q.get(&t.id).unwrap();
3428 back.updated_at = Timestamp::now() - SignedDuration::new(8 * 24 * 60 * 60, 0);
3429 std::fs::write(
3430 q.path_of(&back.id),
3431 serde_json::to_string_pretty(&back).unwrap(),
3432 )
3433 .unwrap();
3434
3435 let report = run_once(&q, &questions, None, Timestamp::now());
3436 assert_eq!(report.asked, [t.id.clone()]);
3437 let open: Vec<_> = questions
3438 .list()
3439 .into_iter()
3440 .filter(|q| q.status.open())
3441 .collect();
3442 assert_eq!(open.len(), 1);
3443 let mut asked = open[0].clone();
3444 assert_eq!(asked.choices, ["resume it", "keep it held", "discard it"]);
3445 assert_eq!(
3446 JA.choices_manual(),
3447 ["再開する", "まだ止めておく", "捨ててよい"]
3448 );
3449 asked
3450 .answer(Answer::Choice(asked.choices[2].clone()))
3451 .unwrap();
3452 questions.put(&mut asked).unwrap();
3453 run_once(&q, &questions, None, Timestamp::now());
3454 assert!(q.get(&t.id).is_err());
3455 }
3456
3457 #[test]
3458 fn answering_resume_releases_the_task() {
3459 let (dir, q, questions) = store();
3460 let mut t = task("gate went red", dir.path().join("repo"));
3461 t.hold_machine(Some("gate red".to_owned()));
3462 q.put(&mut t).unwrap();
3463 run_once(&q, &questions, None, Timestamp::now());
3464
3465 let mut asked = questions
3466 .list()
3467 .into_iter()
3468 .find(|q| q.run == t.id)
3469 .unwrap();
3470 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3471 questions.put(&mut asked).unwrap();
3472
3473 let report = run_once(&q, &questions, None, Timestamp::now());
3474 assert_eq!(report.answered, [t.id.clone()]);
3475 let back = q.get(&t.id).unwrap();
3476 assert_eq!(back.status, TaskStatus::Queued);
3477 assert!(back.hold_source.is_none());
3478 }
3479
3480 fn resumed_then_failed(q: &Queue, questions: &Questions, dir: &std::path::Path) -> Task {
3482 let mut t = task("gate went red", dir.join("repo"));
3483 t.hold_machine(Some("gate red".to_owned()));
3484 q.put(&mut t).unwrap();
3485 run_once(q, questions, None, Timestamp::now());
3486 let mut asked = questions
3487 .list()
3488 .into_iter()
3489 .find(|q| q.run == t.id)
3490 .unwrap();
3491 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3492 questions.put(&mut asked).unwrap();
3493
3494 let report = run_once(q, questions, None, Timestamp::now());
3495 assert_eq!(report.answered, [t.id.clone()]);
3496 let mut back = q.get(&t.id).unwrap();
3497 assert_eq!(back.status, TaskStatus::Queued);
3498 assert_eq!(back.attempts, 0);
3499
3500 back.start("run-1".to_owned());
3501 back.fail("rebase conflict", 1);
3502 assert_eq!(back.status, TaskStatus::Held);
3503 q.put(&mut back).unwrap();
3504 back
3505 }
3506
3507 #[test]
3508 fn a_resume_answer_is_applied_once_and_a_new_machine_hold_is_asked_about() {
3509 let (dir, q, questions) = store();
3510 let t = resumed_then_failed(&q, &questions, dir.path());
3511
3512 let report = run_once(&q, &questions, None, Timestamp::now());
3513 assert!(report.answered.is_empty(), "the old answer must not replay");
3514 assert_eq!(report.asked, std::slice::from_ref(&t.id));
3515 let back = q.get(&t.id).unwrap();
3516 assert_eq!(back.status, TaskStatus::Held);
3517 assert_eq!(
3518 questions
3519 .list()
3520 .into_iter()
3521 .filter(|q| q.status.open())
3522 .count(),
3523 1
3524 );
3525 }
3526
3527 #[test]
3528 fn a_manual_hold_placed_after_a_resume_is_not_undone_by_the_old_answer() {
3529 let (dir, q, questions) = store();
3530 let mut t = resumed_then_failed(&q, &questions, dir.path());
3531 t.hold_manual(Some("operator stopped this".to_owned()));
3532 q.put(&mut t).unwrap();
3533 let before = questions.list().len();
3534
3535 let report = run_once(&q, &questions, None, Timestamp::now());
3536 assert!(report.answered.is_empty());
3537 assert!(report.asked.is_empty());
3538 let back = q.get(&t.id).unwrap();
3539 assert_eq!(back.status, TaskStatus::Held);
3540 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3541 assert_eq!(questions.list().len(), before);
3542 }
3543
3544 #[test]
3545 fn answering_not_yet_keeps_it_held_as_a_manual_hold_and_does_not_reapply() {
3546 let (dir, q, questions) = store();
3547 let mut t = task("gate went red", dir.path().join("repo"));
3548 t.hold_machine(Some("gate red".to_owned()));
3549 q.put(&mut t).unwrap();
3550 run_once(&q, &questions, None, Timestamp::now());
3551
3552 let mut asked = questions
3553 .list()
3554 .into_iter()
3555 .find(|q| q.run == t.id)
3556 .unwrap();
3557 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
3558 questions.put(&mut asked).unwrap();
3559
3560 let report = run_once(&q, &questions, None, Timestamp::now());
3561 assert_eq!(report.answered, [t.id.clone()]);
3562 let back = q.get(&t.id).unwrap();
3563 assert_eq!(back.status, TaskStatus::Held);
3564 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3565 assert!(
3566 back.hold_reason
3567 .as_deref()
3568 .is_some_and(|r| r.contains("gate red")),
3569 "the original cause must survive a \"not yet\" answer, not just the \
3570 triage marker: {:?}",
3571 back.hold_reason
3572 );
3573
3574 let third = run_once(&q, &questions, None, Timestamp::now());
3578 assert!(third.answered.is_empty());
3579 assert!(third.asked.is_empty());
3580 }
3581
3582 #[test]
3583 fn answering_discard_removes_the_task_entirely() {
3584 let (dir, q, questions) = store();
3585 let mut t = task("gate went red", dir.path().join("repo"));
3586 t.hold_machine(Some("gate red".to_owned()));
3587 q.put(&mut t).unwrap();
3588 run_once(&q, &questions, None, Timestamp::now());
3589
3590 let mut asked = questions
3591 .list()
3592 .into_iter()
3593 .find(|q| q.run == t.id)
3594 .unwrap();
3595 asked.answer(Answer::Choice(EN.discard.to_owned())).unwrap();
3596 questions.put(&mut asked).unwrap();
3597
3598 let report = run_once(&q, &questions, None, Timestamp::now());
3599 assert_eq!(report.answered, [t.id.clone()]);
3600 assert!(
3601 q.get(&t.id).is_err(),
3602 "\"discard it\" (捨ててよい) must actually discard the task, not \
3603 just leave it sitting held forever"
3604 );
3605 }
3606
3607 #[test]
3608 fn an_answer_is_read_by_its_position_in_choices_not_by_the_callers_current_language() {
3609 let (dir, q, questions) = store();
3611 let ja_config = dir.path().join("ja.toml");
3612 std::fs::write(&ja_config, "[graph]\nlanguage = \"ja\"\n").unwrap();
3613 let mut t = task("gate went red", dir.path().join("repo"));
3614 t.hold_machine(Some("gate red".to_owned()));
3615 q.put(&mut t).unwrap();
3616 run_once(&q, &questions, Some(&ja_config), Timestamp::now());
3617
3618 let mut asked = questions
3619 .list()
3620 .into_iter()
3621 .find(|q| q.run == t.id)
3622 .unwrap();
3623 assert_eq!(asked.choices[0], JA.resume, "filed in Japanese");
3624 asked.answer(Answer::Choice(JA.resume.to_owned())).unwrap();
3625 questions.put(&mut asked).unwrap();
3626
3627 let en_config = dir.path().join("en.toml");
3632 std::fs::write(&en_config, "[graph]\nlanguage = \"en\"\n").unwrap();
3633 let report = run_once(&q, &questions, Some(&en_config), Timestamp::now());
3634 assert_eq!(report.answered, [t.id.clone()]);
3635 let back = q.get(&t.id).unwrap();
3636 assert_eq!(
3637 back.status,
3638 TaskStatus::Queued,
3639 "a resume answer must resume the task regardless of which \
3640 language it is read back in"
3641 );
3642 }
3643
3644 #[test]
3645 fn a_question_falls_back_to_last_error_when_hold_reason_was_never_set() {
3646 let (dir, q, questions) = store();
3651 let mut t = task("kept failing the gate", dir.path().join("repo"));
3652 t.start("run-1".to_owned());
3653 t.fail("gate red three times running", 1);
3654 assert_eq!(t.status, TaskStatus::Held);
3655 t.hold_reason = None; q.put(&mut t).unwrap();
3657
3658 run_once(&q, &questions, None, Timestamp::now());
3659 let asked = questions
3660 .list()
3661 .into_iter()
3662 .find(|q| q.run == t.id)
3663 .unwrap();
3664 assert!(
3665 asked.detail.contains("gate red three times running"),
3666 "the question must surface `last_error` when there is no \
3667 `hold_reason` to show instead: {}",
3668 asked.detail
3669 );
3670 }
3671
3672 #[test]
3673 fn open_question_for_and_open_task_ids_reflect_only_what_is_still_waiting() {
3674 let (dir, q, questions) = store();
3675 let mut t = task("gate went red", dir.path().join("repo"));
3676 t.hold_machine(Some("gate red".to_owned()));
3677 q.put(&mut t).unwrap();
3678
3679 assert!(open_question_for(&questions, &t.id).is_none());
3680 assert!(!open_task_ids(&questions).contains(&t.id));
3681
3682 run_once(&q, &questions, None, Timestamp::now());
3683 assert!(open_question_for(&questions, &t.id).is_some());
3684 assert!(open_task_ids(&questions).contains(&t.id));
3685
3686 let mut asked = questions
3687 .list()
3688 .into_iter()
3689 .find(|q| q.run == t.id)
3690 .unwrap();
3691 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3692 questions.put(&mut asked).unwrap();
3693
3694 assert!(
3695 open_question_for(&questions, &t.id).is_none(),
3696 "an answered question is no longer open"
3697 );
3698 assert!(!open_task_ids(&questions).contains(&t.id));
3699 }
3700
3701 #[test]
3702 fn pending_for_stays_true_between_an_answer_and_the_next_run_once_pass() {
3703 let (dir, q, questions) = store();
3709 let mut t = task("gate went red", dir.path().join("repo"));
3710 t.hold_machine(Some("gate red".to_owned()));
3711 q.put(&mut t).unwrap();
3712
3713 assert!(!pending_for(&questions, &t));
3714
3715 run_once(&q, &questions, None, Timestamp::now());
3716 let held = q.get(&t.id).unwrap();
3717 assert!(pending_for(&questions, &held), "still waiting on an answer");
3718
3719 let mut asked = questions
3720 .list()
3721 .into_iter()
3722 .find(|q| q.run == t.id)
3723 .unwrap();
3724 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
3725 questions.put(&mut asked).unwrap();
3726 assert!(!asked.status.open());
3727
3728 let still_held = q.get(&t.id).unwrap();
3730 assert!(
3731 pending_for(&questions, &still_held),
3732 "answered but not yet applied is still pending"
3733 );
3734
3735 run_once(&q, &questions, None, Timestamp::now());
3736 let after = q.get(&t.id).unwrap();
3737 assert!(
3738 !pending_for(&questions, &after),
3739 "the answer is applied now, nothing left pending"
3740 );
3741 }
3742
3743 #[test]
3744 fn migrated_keep_held_is_consumed_but_can_be_reviewed_and_discarded_again() {
3745 let (dir, queue, questions) = store();
3746 let mut t = task("keep held", dir.path().join("repo"));
3747 t.hold_machine(Some("unresolved cause".into()));
3748 queue.put(&mut t).unwrap();
3749 run_once(&queue, &questions, None, Timestamp::now());
3750 let mut old = questions.list().remove(0);
3751 old.answer(Answer::Choice(old.choices[1].clone())).unwrap();
3752 questions.put(&mut old).unwrap();
3753 run_once(&queue, &questions, None, Timestamp::now());
3754 let held = queue.get(&t.id).unwrap();
3755 assert!(held.triage_applied(&old.id));
3756
3757 let migration = dir.path().join("notifications/task-chat-migration-v1");
3758 std::fs::create_dir_all(&migration).unwrap();
3759 std::fs::write(
3760 migration.join("state.json"),
3761 serde_json::to_vec(&serde_json::json!({
3762 "schema": 1, "classification_revision": 1, "pending": [], "sources": [old.id]
3763 }))
3764 .unwrap(),
3765 )
3766 .unwrap();
3767 assert!(questions.list_for_effects()[0].status.open());
3768 assert!(!pending_for(&questions, &held));
3769 assert!(open_question_for(&questions, &t.id).is_none());
3770 assert!(!open_task_ids(&questions).contains(&t.id));
3771 assert!(old.answer(Answer::Choice(old.choices[2].clone())).is_err());
3772 assert!(
3773 run_once(&queue, &questions, None, Timestamp::now())
3774 .answered
3775 .is_empty()
3776 );
3777
3778 let later = held.updated_at + SignedDuration::from_hours(24 * 8);
3779 assert_eq!(
3780 run_once(&queue, &questions, None, later).asked,
3781 [t.id.clone()]
3782 );
3783 let mut fresh = open_question_for(&questions, &t.id).unwrap();
3784 assert_ne!(fresh.id, old.id);
3785 assert_eq!(fresh.choices[2], EN.discard);
3786 fresh
3787 .answer(Answer::Choice(fresh.choices[2].clone()))
3788 .unwrap();
3789 questions.put(&mut fresh).unwrap();
3790 assert_eq!(
3791 run_once(&queue, &questions, None, later).answered,
3792 [t.id.clone()]
3793 );
3794 assert!(queue.get(&t.id).is_err());
3795 assert_eq!(
3796 questions.read_history(&old.id).unwrap().resolution(),
3797 Some(EN.wait.into())
3798 );
3799 }
3800
3801 #[test]
3802 fn migrated_unapplied_answers_remain_fenced_but_are_not_displayed_open() {
3803 for node in [NODE, DEPS_NODE] {
3804 let (dir, queue, questions) = store();
3805 let mut t = task("unapplied", dir.path().join("repo"));
3806 t.hold_manual(Some("waiting".into()));
3807 queue.put(&mut t).unwrap();
3808 let mut q = Question::new(
3809 t.id.clone(),
3810 node.into(),
3811 SEAT.into(),
3812 "unapplied".into(),
3813 String::new(),
3814 EN.choices3(),
3815 );
3816 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
3817 questions.put(&mut q).unwrap();
3818 let migration = dir.path().join("notifications/task-chat-migration-v1");
3819 std::fs::create_dir_all(&migration).unwrap();
3820 std::fs::write(
3821 migration.join("state.json"),
3822 serde_json::to_vec(&serde_json::json!({
3823 "schema": 1, "classification_revision": 1, "pending": [], "sources": [q.id]
3824 }))
3825 .unwrap(),
3826 )
3827 .unwrap();
3828 assert!(pending_for(&questions, &t));
3829 assert!(open_question_for(&questions, &t.id).is_none());
3830 assert!(!open_task_ids(&questions).contains(&t.id));
3831 assert!(apply_answers(&queue, &questions).is_empty());
3832 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
3833 t.mark_triage_applied(&q.id);
3834 queue.put(&mut t).unwrap();
3835 assert!(!pending_for(&questions, &t));
3836 assert!(!deps_pending_in(&RecoveryIndex::load(&questions), &t));
3837 }
3838 }
3839}