1use std::path::{Path, PathBuf};
86use std::time::Duration;
87
88use jiff::Timestamp;
89
90use crate::ask::{Question, QuestionStatus, Questions};
91use crate::config::Config;
92use crate::queue::{HoldSource, OperatorResume, Queue, Task, TaskStatus};
93
94pub const NODE: &str = "triage";
97
98pub const DEPS_NODE: &str = "triage-deps";
102
103const SEAT: &str = "triage";
108
109const MANUAL_STALE_AFTER: Duration = Duration::from_secs(7 * 24 * 60 * 60);
121
122struct Wording {
126 lang: &'static str,
127 resume: &'static str,
128 wait: &'static str,
129 discard: &'static str,
130 resume_now: &'static str,
131 keep_held: &'static str,
132}
133
134const EN: Wording = Wording {
135 lang: "en",
136 resume: "resume it",
137 wait: "not yet",
138 discard: "discard it",
139 resume_now: "resume it",
140 keep_held: "keep it held",
141};
142
143const JA: Wording = Wording {
144 lang: "ja",
145 resume: "再開してよい",
146 wait: "まだ待って",
147 discard: "捨ててよい",
148 resume_now: "再開する",
149 keep_held: "まだ止めておく",
150};
151
152fn wording(language: &str) -> &'static Wording {
155 if crate::lang::is_japanese(language) {
156 &JA
157 } else {
158 &EN
159 }
160}
161
162impl Wording {
163 fn choices3(&self) -> Vec<String> {
164 vec![
165 self.resume.to_owned(),
166 self.wait.to_owned(),
167 self.discard.to_owned(),
168 ]
169 }
170
171 fn choices2(&self) -> Vec<String> {
172 vec![self.resume_now.to_owned(), self.keep_held.to_owned()]
173 }
174
175 fn source_label(&self, source: Option<HoldSource>) -> &'static str {
176 match (self.lang, source) {
177 ("ja", Some(HoldSource::Machine)) => "machine(機械による自動保留)",
178 ("ja", Some(HoldSource::Manual)) => "manual(操作者による手動保留)",
179 ("ja", None) => "unknown(schema 3 未満の旧レコード、または理由未記録)",
180 (_, Some(HoldSource::Machine)) => "machine (automatic recovery hold)",
181 (_, Some(HoldSource::Manual)) => "manual (an operator held this)",
182 (_, None) => "unknown (pre-schema-3 record, or never recorded)",
183 }
184 }
185
186 fn detail(&self, task: &Task, why: &str) -> String {
195 let none = if self.lang == "ja" {
196 "(記録なし)"
197 } else {
198 "(none recorded)"
199 };
200 let reason = task
201 .hold_reason
202 .as_deref()
203 .or(task.last_error.as_deref())
204 .unwrap_or(none);
205 format!(
206 "task: {} ({})\ntitle: {}\nhold source: {}\nhold reason: {reason}\n\n{why}",
207 task.id,
208 task.short(),
209 task.title,
210 self.source_label(task.hold_source),
211 )
212 }
213
214 fn summary_machine_unknown(&self, task: &Task) -> String {
215 if self.lang == "ja" {
216 format!("保留タスク {} の再開可否を判断してください", task.short())
217 } else {
218 format!("decide whether to resume held task {}", task.short())
219 }
220 }
221
222 fn why_machine(&self) -> &'static str {
223 if self.lang == "ja" {
224 "機械的な保留(machine hold)ですが、原因がすでに解消しているかを自動では判断できませんでした。"
225 } else {
226 "This is a machine hold, but whether its cause has resolved could not be \
227 checked automatically."
228 }
229 }
230
231 fn summary_conductor_override(&self, task: &Task) -> String {
232 if self.lang == "ja" {
233 format!(
234 "再開と回答済みのタスク {} を conductor が再び保留しました",
235 task.short()
236 )
237 } else {
238 format!(
239 "task {} was resumed at your word, but the conductor held it again",
240 task.short()
241 )
242 }
243 }
244
245 fn why_conductor_override(&self, o: &OperatorResume) -> String {
246 let reason = o.conductor_rehold.as_deref().unwrap_or_default();
247 if self.lang == "ja" {
248 format!(
249 "{} に再開と回答済みですが、conductor が再び hold しました。conductor の理由: \
250 {reason}\n\n強制再キューを選ぶと、以後 conductor はこのタスクを hold できません。",
251 o.at
252 )
253 } else {
254 format!(
255 "You answered \"resume\" at {}, but the conductor held the task again. \
256 Its reason: {reason}\n\nForcing a requeue stops the conductor from \
257 holding this task again.",
258 o.at
259 )
260 }
261 }
262
263 fn choices_conductor_override(&self) -> Vec<String> {
265 if self.lang == "ja" {
266 vec![
267 "強制再キュー(conductor は再 hold 不可)".to_owned(),
268 "手動 hold のまま".to_owned(),
269 "捨ててよい".to_owned(),
270 ]
271 } else {
272 vec![
273 "force requeue (conductor must not hold again)".to_owned(),
274 "keep held (manual)".to_owned(),
275 "discard".to_owned(),
276 ]
277 }
278 }
279
280 fn summary_legacy(&self, task: &Task) -> String {
281 if self.lang == "ja" {
282 format!(
283 "hold_source が不明な保留タスク {} を確認してください",
284 task.short()
285 )
286 } else {
287 format!(
288 "held task {} has no recorded hold source - please take a look",
289 task.short()
290 )
291 }
292 }
293
294 fn why_legacy(&self) -> &'static str {
295 if self.lang == "ja" {
296 "hold_source が記録されていません。schema 3 より前のレコードか、理由が記録されなかった \
297 holdです。人が意図して止めたのか、クラッシュや強制再起動で宙に浮いただけなのか、\
298 このデータからは区別できません。"
299 } else {
300 "No hold_source was recorded - either a pre-schema-3 record, or a hold whose \
301 reason was never written down. Whether this was a deliberate hold or the \
302 leftover of a crash cannot be told from the data alone."
303 }
304 }
305
306 fn summary_manual_stale(&self, task: &Task, days: i64) -> String {
307 if self.lang == "ja" {
308 format!(
309 "{days}日間 保留されたままの手動保留タスク {} を確認してください",
310 task.short()
311 )
312 } else {
313 format!(
314 "held task {} has been on a manual hold for {days} day(s)",
315 task.short()
316 )
317 }
318 }
319
320 fn why_manual(&self) -> &'static str {
321 if self.lang == "ja" {
322 "操作者が明示的に止めた保留ですが、長期間そのままになっています。まだ止めておくか、\
323 再開するか教えてください。"
324 } else {
325 "An operator held this on purpose, but it has sat untouched for a while. Say \
326 whether to keep holding it or resume it."
327 }
328 }
329}
330
331#[derive(Debug, Clone, Copy, PartialEq, Eq)]
333enum Bucket {
334 MachineUnknown,
336 ConductorOverride,
339 Legacy,
341 ManualStale,
343}
344
345#[derive(Debug, Clone, Default)]
347pub struct Report {
348 pub resumed: Vec<String>,
351 pub asked: Vec<String>,
353 pub answered: Vec<String>,
355 pub quarantined: Vec<String>,
360 pub superseded: Vec<String>,
363 pub needs_owner: Vec<String>,
366}
367
368impl Report {
369 pub fn is_empty(&self) -> bool {
372 self.resumed.is_empty()
373 && self.asked.is_empty()
374 && self.answered.is_empty()
375 && self.quarantined.is_empty()
376 && self.superseded.is_empty()
377 && self.needs_owner.is_empty()
378 }
379}
380
381fn repo_for(task: &Task) -> PathBuf {
385 if task.repo.as_os_str().is_empty() {
386 PathBuf::from(".")
387 } else {
388 task.repo.clone()
389 }
390}
391
392fn is_disk_hold(task: &Task) -> bool {
394 task.has_disk_capacity_cause()
395 || (task.schema < 22
396 && task
397 .hold_reason
398 .as_deref()
399 .is_some_and(crate::disk::is_gate_reason))
400}
401
402pub(crate) fn disk_wait_eligible(task: &Task) -> bool {
403 disk_wait_known(task, &crate::run::home())
404}
405
406fn disk_wait_known(task: &Task, home: &Path) -> bool {
408 if task.status != TaskStatus::Held
409 || task.hold_source != Some(HoldSource::Machine)
410 || task
411 .resume_override
412 .as_ref()
413 .is_some_and(|o| o.conductor_rehold.is_some())
414 {
415 return false;
416 }
417 if task.has_disk_capacity_cause() {
418 return true;
419 }
420 if task.resume_override.is_some() {
422 return false;
423 }
424 if is_disk_hold(task) {
425 return true;
426 }
427 if task.schema >= 22 {
428 return false;
429 }
430 if !task.hold_reason.as_deref().is_some_and(|r| {
431 r.starts_with(crate::disk::RUNNING_PRESSURE) || r.starts_with(crate::cache::RESOURCE_HOLD)
432 }) {
433 return false;
434 }
435 let Some(id) = task.execution_runs().last() else {
436 return false;
437 };
438 let Ok(state) = crate::run::RunState::load_under(id, home) else {
439 return false;
440 };
441 state.id == *id
442 && state.resource_block.as_ref().is_some_and(|b| {
443 let expected = match b.kind.as_str() {
444 "disk pressure" => format!("{}{}", crate::disk::RUNNING_PRESSURE, b.detail),
445 "disk admission" => format!(
446 "{}{}; no attempt was spent; resume to retry the affected operation",
447 crate::cache::RESOURCE_HOLD,
448 b.description()
449 ),
450 _ => return false,
451 };
452 matches!(b.cause, crate::run::ResourceBlockCause::Other)
453 && task.hold_reason.as_deref() == Some(expected.as_str())
454 })
455}
456
457fn machine_cause_resolved(task: &Task, cfg: &Config) -> Option<bool> {
462 machine_cause_resolved_under(task, cfg, &crate::run::home())
463}
464
465fn machine_cause_resolved_under(task: &Task, cfg: &Config, home: &Path) -> Option<bool> {
466 machine_cause_resolved_with_admission(task, cfg, home, |paths, floor| {
467 crate::retention::OutputLease::admit(home, paths, floor).is_ok()
468 })
469}
470
471fn machine_cause_resolved_with_admission(
472 task: &Task,
473 cfg: &Config,
474 home: &Path,
475 admission_available: impl Fn(&[PathBuf], u64) -> bool,
476) -> Option<bool> {
477 if disk_wait_known(task, home) {
478 if crate::daemon::disk_gate(&repo_for(task), cfg).is_some() {
479 return Some(false);
480 }
481 let mut floor = cfg.disk.min_free_bytes;
482 let mut paths = vec![repo_for(task)];
483 paths.extend(cfg.cache_dir());
484 paths.push(
485 cfg.graph
486 .worktree_root
487 .clone()
488 .unwrap_or_else(crate::run::default_worktree_root),
489 );
490 if let Some(id) = task.execution_runs().last() {
491 let Ok(state) = crate::run::RunState::load_under(id, home) else {
492 return Some(false);
493 };
494 if state.id != *id || !state.status.resumable() {
495 return Some(false);
496 }
497 if crate::daemon::disk_gate(&state.repo, &state.config).is_some() {
498 return Some(false);
499 }
500 floor = floor.max(state.config.disk.min_free_bytes);
501 paths.extend([state.repo.clone(), state.worktree_root()]);
502 paths.extend(state.candidates.iter().map(|c| c.worktree.clone()));
503 paths.extend(state.config.cache_dir());
504 }
505 return Some(admission_available(&paths, floor));
506 }
507 if task
508 .hold_reason
509 .as_deref()
510 .is_some_and(|r| r.starts_with(crate::cache::RESOURCE_HOLD))
511 {
512 let id = task.execution_runs().last()?;
514 let state = crate::run::RunState::load_under(id, home).ok()?;
515 let block = state.resource_block.as_ref()?;
516 if state.id != *id {
517 return None;
518 }
519 if block.kind != "shared build cache"
522 || !matches!(block.cause, crate::run::ResourceBlockCause::Timeout { .. })
523 {
524 return None;
525 }
526 return Some(
527 cfg.cache_dir()
528 .is_none_or(|dir| !crate::cache::in_use(home, &dir)),
529 );
530 }
531 None
532}
533
534fn manual_is_stale(task: &Task, now: Timestamp) -> bool {
538 now.as_second() - task.updated_at.as_second() > MANUAL_STALE_AFTER.as_secs() as i64
539}
540
541fn marker_for(q: &Question) -> String {
544 format!("[triage:{}]", q.short())
545}
546
547fn already_applied(task: &Task, q: &Question) -> bool {
550 if task.triage_applied(&q.id) {
551 return true;
552 }
553 let marker = marker_for(q);
556 task.hold_reason
557 .as_deref()
558 .is_some_and(|r| r.contains(marker.as_str()))
559}
560
561fn latest_triage_question(questions: &Questions, task_id: &str) -> Option<Question> {
566 latest_question(questions, NODE, task_id)
567}
568
569pub(crate) struct RecoveryIndex {
572 latest: std::collections::HashMap<(String, String), Question>,
573 open: std::collections::BTreeSet<String>,
574}
575
576impl RecoveryIndex {
577 pub(crate) fn load(questions: &Questions) -> Self {
578 let queue = Queue::at(
579 questions
580 .root()
581 .parent()
582 .unwrap_or(Path::new(""))
583 .join("queue"),
584 );
585 let tasks = queue.list();
586 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
587 Self::from_scan(&queue, &tasks, questions.list_for_effects_in(&ownership))
588 }
589
590 fn from_scan(queue: &Queue, tasks: &[Task], all: Vec<Question>) -> Self {
591 let ids = crate::queue::record_ids(queue.root(), "json").unwrap_or_default();
592 let by_id: std::collections::HashMap<_, _> =
593 tasks.iter().map(|task| (task.id.as_str(), task)).collect();
594 let exact: std::collections::HashSet<_> = ids.iter().map(String::as_str).collect();
595 let mut aliases: std::collections::HashMap<&str, Option<&str>> = Default::default();
596 for id in &ids {
597 for boundary in id
598 .char_indices()
599 .map(|(i, _)| i)
600 .chain(std::iter::once(id.len()))
601 {
602 for alias in [&id[..boundary], &id[boundary..]] {
603 let entry = aliases.entry(alias).or_insert(Some(id));
604 if *entry != Some(id.as_str()) {
605 *entry = None;
606 }
607 }
608 }
609 }
610 let mut index = Self {
611 latest: Default::default(),
612 open: Default::default(),
613 };
614 for q in all {
615 if q.node != NODE && q.node != DEPS_NODE {
616 continue;
617 }
618 let resolved = if exact.contains(q.run.as_str()) {
619 Some(q.run.as_str())
620 } else {
621 aliases.get(q.run.as_str()).copied().flatten()
622 };
623 if let Some(id) = resolved
624 && by_id.get(id).is_some_and(|task| {
625 crate::repair_budget::reconcile::superseded_question(task, &q)
626 })
627 {
628 continue;
629 }
630 if q.status.open() {
631 index.open.insert(q.run.clone());
632 }
633 let key = (q.node.clone(), q.run.clone());
634 let entry = index.latest.entry(key).or_insert_with(|| q.clone());
635 if (q.asked_at, &q.id) > (entry.asked_at, &entry.id) {
636 *entry = q;
637 }
638 }
639 index
640 }
641
642 fn latest(&self, node: &str, task_id: &str) -> Option<Question> {
643 self.latest
644 .get(&(node.to_owned(), task_id.to_owned()))
645 .cloned()
646 }
647
648 pub(crate) fn pending(&self, task: &Task) -> Vec<Question> {
649 [NODE, DEPS_NODE]
650 .into_iter()
651 .filter_map(|node| {
652 self.latest(node, &task.id).filter(|q| {
653 q.status.open()
654 || awaiting_task_chat(q, task)
655 || (q.status == QuestionStatus::Answered
656 && if node == NODE {
657 !already_applied(task, q)
658 } else {
659 !task.triage_applied(&q.id)
660 })
661 })
662 })
663 .collect()
664 }
665}
666
667fn latest_question(questions: &Questions, node: &str, task_id: &str) -> Option<Question> {
669 RecoveryIndex::load(questions).latest(node, task_id)
670}
671
672#[derive(Debug, Clone, Copy, PartialEq, Eq)]
675enum AnswerAction {
676 Resume,
680 Discard,
683 KeepHeld,
688}
689
690fn interpret_answer(q: &Question) -> AnswerAction {
703 let resolution = q.resolution().unwrap_or_default();
704 match q.choices.iter().position(|c| *c == resolution) {
705 Some(0) => AnswerAction::Resume,
706 Some(2) => AnswerAction::Discard,
707 _ => AnswerAction::KeepHeld,
708 }
709}
710
711pub(crate) fn contextual_keep_held(q: &Question, label: &str) -> bool {
714 q.node == NODE
715 && q.seat == SEAT
716 && q.actions.is_empty()
717 && q.choices.get(1).is_some_and(|c| c == label)
718}
719
720fn keep_held_note(task: &Task, q: &Question, resolution: &str) -> String {
725 let marker = format!("{} operator: {resolution}", marker_for(q));
726 match task.hold_reason.as_deref() {
727 Some(existing) if !existing.is_empty() => format!("{existing}\n{marker}"),
728 _ => marker,
729 }
730}
731
732fn file_question(
736 questions: &Questions,
737 ownership: &crate::task_chat::InquiryOwnership,
738 task: &Task,
739 bucket: Bucket,
740 w: &Wording,
741 now: Timestamp,
742) -> Option<Question> {
743 let (summary, why, choices) = match bucket {
744 Bucket::MachineUnknown => (
745 w.summary_machine_unknown(task),
746 w.why_machine().to_owned(),
747 w.choices3(),
748 ),
749 Bucket::ConductorOverride => (
750 w.summary_conductor_override(task),
751 task.resume_override
752 .as_ref()
753 .map(|o| w.why_conductor_override(o))
754 .unwrap_or_default(),
755 w.choices_conductor_override(),
756 ),
757 Bucket::Legacy => (
758 w.summary_legacy(task),
759 w.why_legacy().to_owned(),
760 w.choices3(),
761 ),
762 Bucket::ManualStale => {
763 let days = (now.as_second() - task.updated_at.as_second()) / (24 * 60 * 60);
764 (
765 w.summary_manual_stale(task, days),
766 w.why_manual().to_owned(),
767 w.choices2(),
768 )
769 }
770 };
771 let mut q = Question::new(
772 task.id.clone(),
773 NODE.to_owned(),
774 SEAT.to_owned(),
775 summary,
776 w.detail(task, &why),
777 choices,
778 );
779 q.triage_generation = Some(task.resumption.as_ref().map_or(0, |r| r.generation));
780 questions.put_executable_in(&mut q, ownership).ok()?;
781 Some(q)
782}
783
784pub(crate) fn deputy_brief(q: &Question, queue: &Queue) -> String {
789 let task = queue.get(&q.run).ok();
790 let mut s = match &task {
791 Some(t) => format!(
792 "Task {} (`magi task show {}`), \"{}\", is {}. Hold source: {}. Reason: {}.",
793 t.id,
794 t.id,
795 t.title,
796 t.status.as_str(),
797 match t.hold_source {
798 Some(HoldSource::Machine) => "machine (an automatic hold)",
799 Some(HoldSource::Manual) => "manual (an operator held it)",
800 None => "unknown (an old record, treated as the operator's)",
801 },
802 t.hold_reason
803 .as_deref()
804 .or(t.last_error.as_deref())
805 .unwrap_or("(none recorded)")
806 ),
807 None => format!("Task {} no longer exists (its record is gone).", q.run),
808 };
809 if task.as_ref().is_some_and(disk_wait_eligible) {
810 s.push_str("\n\nDisk capacity waits are monitored automatically. No owner answer is needed for an untouched automatic disk triage question. Explicit holds, discussion and other approvals still require owner resolution.");
811 }
812 s.push_str("\n\nWhat each option does, by position:");
813 let n = q.choices.len();
814 for (i, c) in q.choices.iter().enumerate() {
815 let effect = if q.node == DEPS_NODE {
816 match i {
817 0 => "the dependency is released so the loop runs it",
818 1 => {
819 "the dependants are detached and the dependency task is DELETED; this cannot be undone"
820 }
821 2 => "the dependants are detached; the dependency stays as it is",
822 _ => "nothing is applied",
823 }
824 } else {
825 match i {
826 0 => "the task is released and requeued",
827 2 if n > 2 => "the task is DELETED; this cannot be undone",
828 _ => "the task stays held, with the answer noted",
829 }
830 };
831 s.push_str(&format!("\n- `{c}`: {effect}"));
832 }
833 s.push_str(
834 "\n\nThe daemon applies the answer on its next pass; you apply nothing. When \
835 the owner's words pick an option - including releasing a task they held \
836 themselves, or deleting it - settle it.",
837 );
838 s
839}
840
841fn quarantine_orphaned_blocked(queue: &Queue, questions: &Questions) -> Vec<String> {
859 let mut quarantined = Vec::new();
860 for listed in queue.list() {
861 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
862 continue;
863 }
864 let Ok(_claim) = queue.claim(&listed.id) else {
865 continue;
866 };
867 let Ok(mut task) = queue.get(&listed.id) else {
868 continue;
869 };
870 if task.status != TaskStatus::Blocked {
871 continue;
872 }
873 let deleted = queue.apply_deleted_blockers(&mut task, questions);
874 if !deleted.is_empty() && queue.put(&mut task).is_ok() {
875 for id in &deleted {
876 queue.note_dependency_deleted(&task, id);
877 }
878 }
879 if task.status != TaskStatus::Blocked {
880 continue;
881 }
882 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
883 if missing.is_empty() {
884 continue;
885 }
886 let language = crate::lang::of_repo(&repo_for(&task));
887 let mut cause = missing.clone();
888 cause.sort();
889 cause.dedup();
890 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
891 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
892 &task.blocked_by,
893 &missing,
894 &language,
895 )));
896 if queue.put(&mut task).is_ok() {
897 quarantined.push(task.id.clone());
898 }
899 }
900 quarantined
901}
902
903#[derive(Debug, Clone, Copy, PartialEq, Eq)]
906enum DepsAction {
907 Release,
908 Discard,
909 Detach,
910 Nothing,
912}
913
914fn interpret_deps_answer(q: &Question) -> DepsAction {
915 let resolution = q.resolution().unwrap_or_default();
916 match q.choices.iter().position(|c| *c == resolution) {
917 Some(0) => DepsAction::Release,
918 Some(1) => DepsAction::Discard,
919 Some(2) => DepsAction::Detach,
920 _ => DepsAction::Nothing,
921 }
922}
923
924fn deps_texts(w: &Wording, root: &Task, dependants: &[&Task]) -> (String, String, Vec<String>) {
926 let ja = w.lang == "ja";
927 let reason = root
928 .hold_reason
929 .as_deref()
930 .or(root.last_error.as_deref())
931 .unwrap_or(if ja {
932 "(記録なし)"
933 } else {
934 "(none recorded)"
935 });
936 let list: String = dependants
937 .iter()
938 .map(|t| format!("- {} {}\n", t.short(), t.title))
939 .collect();
940 if ja {
941 (
942 format!(
943 "{} ({}) が {} 件のタスクを止めています",
944 root.short(),
945 root.status.as_str(),
946 dependants.len()
947 ),
948 format!(
949 "task: {} ({})\ntitle: {}\n状態: {}\n理由: {reason}\n\n\
950 このタスクは自動では実行されないため、依存している次のタスクは永遠に待ち続けます:\n{list}",
951 root.id,
952 root.short(),
953 root.title,
954 root.status.as_str()
955 ),
956 vec![
957 "依存先を再開する".to_owned(),
958 "依存先を捨てる(依存タスクは切り離して実行)".to_owned(),
959 "依存タスクを切り離す(依存先はそのまま)".to_owned(),
960 ],
961 )
962 } else {
963 (
964 format!(
965 "{} ({}) is freezing {} blocked task(s)",
966 root.short(),
967 root.status.as_str(),
968 dependants.len()
969 ),
970 format!(
971 "task: {} ({})\ntitle: {}\nstatus: {}\nreason: {reason}\n\n\
972 Nothing in the loop will ever run this task, so these dependants wait \
973 forever:\n{list}",
974 root.id,
975 root.short(),
976 root.title,
977 root.status.as_str()
978 ),
979 vec![
980 "release the dependency".to_owned(),
981 "discard the dependency (dependants are detached and run)".to_owned(),
982 "detach the dependants (the dependency stays as it is)".to_owned(),
983 ],
984 )
985 }
986}
987
988fn detach_dependants(queue: &Queue, questions: &Questions, root: &str) {
992 for listed in queue.list() {
993 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| matches!(crate::queue::dependency(queue, questions, b), crate::queue::Dependency::Task(id) if id == root)) {
994 continue;
995 }
996 let Ok(_claim) = queue.claim(&listed.id) else {
997 continue;
998 };
999 let Ok(mut t) = queue.get(&listed.id) else {
1000 continue;
1001 };
1002 if t.status != TaskStatus::Blocked {
1003 continue;
1004 }
1005 for id in t.blocked_by.clone() {
1006 if matches!(crate::queue::dependency(queue, questions, &id), crate::queue::Dependency::Task(full) if full == root)
1007 {
1008 t.unblock(&id);
1009 }
1010 }
1011 let _ = queue.put(&mut t);
1012 }
1013}
1014
1015fn apply_deps_answer(
1019 queue: &Queue,
1020 questions: &Questions,
1021 q: &Question,
1022 now: Timestamp,
1023 ownership: &crate::task_chat::InquiryOwnership,
1024) -> bool {
1025 let Ok(_claim) = queue.claim(&q.run) else {
1026 return false;
1027 };
1028 let Ok(current) = questions.get_for_effects_in(&q.id, ownership) else {
1029 return false;
1030 };
1031 if current.status != QuestionStatus::Answered {
1032 return false;
1033 }
1034 let q = ¤t;
1035 let Ok(mut root) = queue.get(&q.run) else {
1036 return false;
1037 };
1038 if root.triage_applied(&q.id) {
1039 return false;
1040 }
1041 if root.operator_decided_at.is_some_and(|at| at > q.asked_at)
1042 || q.triage_generation
1043 .is_some_and(|g| g != root.resumption.as_ref().map_or(0, |r| r.generation))
1044 {
1045 root.mark_triage_applied(&q.id);
1046 return queue.put(&mut root).is_ok();
1047 }
1048 match interpret_deps_answer(q) {
1049 DepsAction::Release => {
1050 if root.status == TaskStatus::Running {
1051 return false;
1052 }
1053 if crate::repair_budget::reconcile::blocker(queue, &root).is_some() {
1054 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(q) else {
1055 return false;
1056 };
1057 root.decision_effects.insert(q.id.clone(), receipt);
1058 root.mark_triage_applied(&q.id);
1059 return queue.put(&mut root).is_ok();
1060 }
1061 root.answers.push(crate::queue::AnsweredQuestion {
1062 question: q.summary.clone(),
1063 answer: q.resolution().unwrap_or_default(),
1064 });
1065 if crate::base_sync_recovery::held_task(queue, &root) {
1066 root.mark_triage_applied(&q.id);
1067 return queue.put(&mut root).is_ok();
1068 }
1069 let before = root.operator_decided_at;
1070 let contract =
1071 crate::repair_budget::contract(&root, crate::repair_budget::Kind::Review).ok();
1072 root.release_operator("triage-dependencies", Some(&q.id));
1073 if let Some(contract) = contract
1074 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1075 q,
1076 before,
1077 &contract,
1078 root.resumption.as_ref().expect("release"),
1079 )
1080 {
1081 root.decision_effects.insert(q.id.clone(), evidence);
1082 }
1083 root.resume_override = Some(OperatorResume {
1084 question_id: q.id.clone(),
1085 at: now,
1086 conductor_rehold: None,
1087 forced: false,
1088 pinned_run: None,
1089 });
1090 }
1091 DepsAction::Discard => {
1092 detach_dependants(queue, questions, &root.id);
1093 return queue.discard_triage(&root.id, questions, &q.id).is_ok();
1094 }
1095 DepsAction::Detach => detach_dependants(queue, questions, &root.id),
1096 DepsAction::Nothing => return false,
1101 }
1102 root.mark_triage_applied(&q.id);
1103 queue.put(&mut root).is_ok()
1104}
1105
1106fn repair_receipt_notice(queue: &Queue, task: &Task) -> bool {
1107 let Some(detail) = crate::repair_budget::reconcile::blocker(queue, task) else {
1108 return false;
1109 };
1110 if let Some(home) = queue.root().parent() {
1111 let _ = crate::task_inquiry::Outbox::at(home).file_for_occurrence(
1112 task,
1113 task.runs.last().cloned(),
1114 "repair-receipt-recovery".into(),
1115 &task
1116 .repair_grants
1117 .last()
1118 .expect("diagnosed receipt")
1119 .request
1120 .request,
1121 "Retained repair receipt needs reconciliation",
1122 &detail,
1123 );
1124 }
1125 true
1126}
1127
1128fn ask_about_stuck_roots(
1137 queue: &Queue,
1138 questions: &Questions,
1139 config_override: Option<&Path>,
1140 report: &mut Report,
1141) {
1142 let all = questions.list_for_effects();
1143 let tasks = queue.list();
1144 let recovery = RecoveryIndex::from_scan(queue, &tasks, all.clone());
1145 let inv = crate::blockers::Inventory::load_with_tasks(queue, questions, tasks);
1146 let mut frozen: std::collections::BTreeMap<String, Vec<String>> = Default::default();
1147 for (id, roots) in inv.stuck() {
1148 for root in roots {
1149 if root != id {
1150 frozen.entry(root).or_default().push(id.clone());
1151 }
1152 }
1153 }
1154 let ownership = crate::task_chat::InquiryOwnership::default();
1158 for mut q in all
1159 .iter()
1160 .filter(|q| q.node == DEPS_NODE && q.status.open())
1161 .cloned()
1162 {
1163 if !frozen.contains_key(&q.run) {
1164 q.abandon("nothing waits on this task anymore");
1165 let _ = questions.put_executable_in(&mut q, &ownership);
1166 }
1167 }
1168
1169 for (root_id, dependant_ids) in &frozen {
1170 let Ok(_claim) = queue.claim(root_id) else {
1171 continue;
1172 };
1173 let seeds: Vec<_> = std::iter::once(root_id)
1176 .chain(dependant_ids.iter())
1177 .filter_map(|id| queue.get(id).ok())
1178 .collect();
1179 let current = crate::blockers::Inventory::load_for_tasks(queue, questions, &seeds);
1180 let Some(root) = current.task(root_id) else {
1181 continue;
1182 };
1183 if crate::base_sync_recovery::held_task(queue, root) {
1184 continue;
1185 }
1186 if disk_wait_known(root, questions.root().parent().unwrap_or(Path::new(""))) {
1189 continue;
1190 }
1191 let dependants: Vec<&Task> = dependant_ids
1192 .iter()
1193 .filter_map(|d| current.task(d))
1194 .filter(|t| current.stuck_roots(t).contains(root_id))
1195 .collect();
1196 if dependants.is_empty() {
1197 continue;
1198 }
1199 if repair_receipt_notice(queue, root) {
1200 continue;
1201 }
1202 let latest = recovery.latest(DEPS_NODE, root_id);
1203 match latest.as_ref() {
1204 Some(q) if q.status.open() || awaiting_task_chat(q, root) => continue,
1205 Some(q) if q.status == QuestionStatus::Answered && !root.triage_applied(&q.id) => {
1206 continue;
1207 }
1208 Some(q) if q.status == QuestionStatus::Abandoned => {
1209 let changed = root.updated_at > q.asked_at
1210 || dependants.iter().any(|t| t.updated_at > q.asked_at);
1211 if !changed {
1212 continue;
1213 }
1214 }
1215 _ => {}
1216 }
1217 if !recovery.pending(root).is_empty() {
1218 continue;
1219 }
1220 let cfg = Config::discover(&repo_for(root), config_override)
1221 .ok()
1222 .map(|(c, _)| c);
1223 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1224 let (summary, detail, choices) = deps_texts(w, root, &dependants);
1225 let mut question = Question::new(
1226 root.id.clone(),
1227 DEPS_NODE.to_owned(),
1228 SEAT.to_owned(),
1229 summary,
1230 detail,
1231 choices,
1232 );
1233 question.triage_generation = Some(root.resumption.as_ref().map_or(0, |r| r.generation));
1234 if questions
1235 .put_executable_in(&mut question, &ownership)
1236 .is_ok()
1237 {
1238 report.asked.push(root.id.clone());
1239 }
1240 }
1241}
1242
1243pub fn apply_answers(queue: &Queue, questions: &Questions) -> Vec<String> {
1246 let mut applied = Vec::new();
1247 let ownership = crate::task_chat::InquiryOwnership::default();
1248 for q in questions.list_for_effects_in(&ownership) {
1249 if q.node == DEPS_NODE
1250 && q.status == QuestionStatus::Answered
1251 && apply_deps_answer(queue, questions, &q, Timestamp::now(), &ownership)
1252 {
1253 applied.push(q.run.clone());
1254 }
1255 }
1256 let tasks = queue.list();
1258 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
1259 let recovery =
1260 RecoveryIndex::from_scan(queue, &tasks, questions.list_for_effects_in(&ownership));
1261 for listed in tasks {
1262 let Some(q) = recovery.latest(NODE, &listed.id) else {
1263 continue;
1264 };
1265 if q.status != QuestionStatus::Answered || already_applied(&listed, &q) {
1266 continue;
1267 }
1268 let Ok(_claim) = queue.claim(&listed.id) else {
1269 continue;
1270 };
1271 let Ok(mut task) = queue.get(&listed.id) else {
1272 continue;
1273 };
1274 let Ok(q) = questions.get_for_effects_in(&q.id, &ownership) else {
1277 continue;
1278 };
1279 if q.status != QuestionStatus::Answered {
1280 continue;
1281 }
1282 if already_applied(&task, &q)
1283 || !matches!(task.status, TaskStatus::Held | TaskStatus::Queued)
1284 {
1285 continue;
1286 }
1287 let generation = task.resumption.as_ref().map_or(0, |r| r.generation);
1288 let action = interpret_answer(&q);
1289 let mut newer_decision = q.triage_generation.is_some_and(|g| g != generation)
1290 || task.resumption.as_ref().is_some_and(|r| r.at > q.asked_at)
1291 || task.held_at.is_some_and(|at| at > q.asked_at)
1292 || task.operator_decided_at.is_some_and(|at| at > q.asked_at);
1293 if matches!(action, AnswerAction::KeepHeld)
1297 && q.answered_at.is_some_and(|at| {
1298 task.operator_decided_at
1299 .is_none_or(|decision| at > decision)
1300 && task.resumption.as_ref().is_none_or(|r| at > r.at)
1301 })
1302 {
1303 newer_decision = false;
1304 }
1305 let resolution = q.resolution().unwrap_or_default();
1307 task.answers.push(crate::queue::AnsweredQuestion {
1308 question: q.summary.clone(),
1309 answer: if newer_decision {
1310 format!("{resolution} (superseded by a later operator decision)")
1311 } else {
1312 resolution
1313 },
1314 });
1315 if !newer_decision {
1316 match action {
1317 AnswerAction::Resume if crate::base_sync_recovery::held_task(queue, &task) => {
1318 }
1321 AnswerAction::Resume
1322 if crate::repair_budget::reconcile::blocker(queue, &task).is_some() =>
1323 {
1324 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(&q) else {
1325 continue;
1326 };
1327 task.decision_effects.insert(q.id.clone(), receipt);
1328 }
1329 AnswerAction::Resume => {
1330 let before = task.operator_decided_at;
1331 let contract =
1332 crate::repair_budget::contract(&task, crate::repair_budget::Kind::Review)
1333 .ok();
1334 task.release_operator("triage", Some(&q.id));
1335 if let Some(contract) = contract
1336 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1337 &q,
1338 before,
1339 &contract,
1340 task.resumption.as_ref().expect("release"),
1341 )
1342 {
1343 task.decision_effects.insert(q.id.clone(), evidence);
1344 }
1345 task.resume_override = Some(OperatorResume {
1346 question_id: q.id.clone(),
1347 at: Timestamp::now(),
1348 conductor_rehold: None,
1349 forced: true,
1350 pinned_run: None,
1351 });
1352 }
1353 AnswerAction::KeepHeld => {
1354 let note = keep_held_note(&task, &q, &q.resolution().unwrap_or_default());
1355 task.hold_manual(Some(note));
1356 }
1357 AnswerAction::Discard => {
1358 if queue.discard_triage(&task.id, questions, &q.id).is_ok() {
1359 applied.push(task.id);
1360 }
1361 continue;
1362 }
1363 }
1364 }
1365 task.mark_triage_applied(&q.id);
1366 if queue.put(&mut task).is_ok() {
1367 applied.push(task.id);
1368 }
1369 }
1370 applied
1371}
1372const DISK_SUPERSEDED: &str =
1373 "Automatic disk wait superseded unanswered triage; no owner answer or approval was created";
1374
1375fn untouched_disk_question(q: &Question, task: &Task) -> bool {
1376 q.status.open()
1377 && q.node == NODE
1378 && q.seat == SEAT
1379 && q.run == task.id
1380 && q.routed_task.as_deref().is_none_or(|id| id == task.id)
1381 && q.triage_generation == Some(task.resumption.as_ref().map_or(0, |r| r.generation))
1382 && task.held_at.is_some_and(|at| q.asked_at >= at)
1383 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1384 && q.answer.is_none()
1385 && q.answered_at.is_none()
1386 && q.accepted_answer_at.is_none()
1387 && q.thread.is_empty()
1388 && q.actions.is_empty()
1389 && q.merge_target.is_none()
1390 && [&EN, &JA].iter().any(|w| {
1391 q.summary == w.summary_machine_unknown(task)
1392 && q.choices == w.choices3()
1393 && q.detail == w.detail(task, w.why_machine())
1394 })
1395}
1396
1397fn refresh_disk_notices(home: &Path, task: &Task) {
1399 let notices = crate::notices::Notices::at(home.join("notifications"));
1400 let Some(task_notice) = crate::notices::task_held(task) else {
1401 return;
1402 };
1403 for (key, message) in [
1404 (task_notice.key, task_notice.message),
1405 (
1406 format!("disk:{}", repo_for(task).display()),
1407 format!(
1408 "Task {} is waiting for disk space. It will resume automatically when disk and output admission checks succeed.",
1409 task.id
1410 ),
1411 ),
1412 ] {
1413 let id = crate::notices::id_of(&key);
1414 if notices
1415 .get(&id)
1416 .is_ok_and(|n| n.key == key && n.message != message)
1417 {
1418 let _ = notices.update(&id, |n| {
1419 if n.key == key {
1420 n.message = message;
1421 }
1422 });
1423 }
1424 }
1425}
1426
1427fn resume_disk_task(
1431 queue: &Queue,
1432 questions: &Questions,
1433 task: &mut Task,
1434 ownership: &crate::task_chat::InquiryOwnership,
1435) -> anyhow::Result<bool> {
1436 let expected = task.clone();
1437 let home = questions
1438 .root()
1439 .parent()
1440 .ok_or_else(|| anyhow::anyhow!("missing question home"))?;
1441 let _lease = if let Some(id) = task.execution_runs().last() {
1442 anyhow::ensure!(
1443 task.resume_override
1444 .as_ref()
1445 .and_then(|o| o.pinned_run.as_ref())
1446 .is_none_or(|pin| pin == id),
1447 "selected recovery run differs from disk wait evidence"
1448 );
1449 let state = crate::run::RunState::load_under(id, home)?;
1450 anyhow::ensure!(
1451 state.id == *id && state.status.resumable() && !task.fresh_start,
1452 "retained execution cannot resume"
1453 );
1454 anyhow::ensure!(
1455 state.liveness_under(
1456 home,
1457 crate::daemon::is_working_on(home, id, Timestamp::now())
1458 ) == crate::run::Liveness::Dead,
1459 "retained execution is not proven idle"
1460 );
1461 Some(crate::driver::maintenance(home, id)?)
1462 } else {
1463 None
1464 };
1465 let outbox = crate::task_inquiry::Outbox::at(home);
1466 let all = questions.list_for_effects_in(ownership);
1467 let pending: Vec<_> = all
1468 .iter()
1469 .filter(|q| {
1470 (!q.run.is_empty()
1472 && std::iter::once(task.id.as_str())
1473 .chain(task.execution_runs().iter().map(String::as_str))
1474 .any(|id| id == q.run || id.starts_with(&q.run) || id.ends_with(&q.run)))
1475 && (q.status.open()
1476 || ((q.node == NODE || q.node == DEPS_NODE)
1477 && (awaiting_task_chat(q, task)
1478 || (q.status == QuestionStatus::Answered
1479 && !already_applied(task, q)))))
1480 })
1481 .collect();
1482 anyhow::ensure!(pending.len() <= 1, "multiple decisions remain");
1484 let candidate = pending.first().copied();
1485 if let Some(q) = candidate {
1486 anyhow::ensure!(untouched_disk_question(q, task), "owner decision remains");
1487 }
1488 for inquiry in outbox.pending()? {
1489 if inquiry.task == task.id && inquiry.decision.is_some() {
1490 anyhow::ensure!(
1491 candidate.is_some_and(|q| q.id == inquiry.id),
1492 "unrelated decision remains"
1493 );
1494 }
1495 }
1496 if let Some(q) = candidate {
1497 let _execution = outbox.execution_claim(&q.id)?;
1498 questions.update_in(&q.id, ownership, |current| {
1499 anyhow::ensure!(untouched_disk_question(current, task), "question changed");
1500 let inquiry = match outbox.get(&q.id) {
1501 Ok(inquiry) => inquiry,
1502 Err(error)
1503 if error
1504 .downcast_ref::<std::io::Error>()
1505 .is_some_and(|e| e.kind() == std::io::ErrorKind::NotFound) =>
1506 {
1507 anyhow::ensure!(
1508 current.routed_chat.is_none(),
1509 "delivered inquiry is missing; owner reconciliation required"
1510 );
1511 outbox.enqueue(crate::task_chat::live_record(home, task, current))?
1514 }
1515 Err(error) => return Err(error),
1516 };
1517 anyhow::ensure!(
1518 inquiry.task == task.id
1519 && inquiry.repo == task.repo
1520 && inquiry.source == format!("{}:{}:{}", NODE, SEAT, task.id)
1521 && current
1522 .routed_chat
1523 .as_ref()
1524 .is_none_or(|chat| inquiry.delivered_to.as_ref() == Some(chat)),
1525 "inquiry binding changed"
1526 );
1527 outbox.update_decision(&q.id, |d| {
1528 if d.status == crate::task_decision::Status::Resolved {
1529 anyhow::ensure!(
1530 d.request.is_none()
1531 && d.history
1532 .last()
1533 .is_some_and(|e| e.evidence == DISK_SUPERSEDED),
1534 "decision already settled by owner"
1535 );
1536 return Ok(false);
1537 }
1538 anyhow::ensure!(
1539 d.status == crate::task_decision::Status::AwaitingOwner
1540 && d.request.is_none()
1541 && d.source_answer_at.is_none(),
1542 "owner request remains"
1543 );
1544 d.settle(DISK_SUPERSEDED)
1545 })?;
1546 current.abandon(DISK_SUPERSEDED);
1547 Ok(())
1548 })?;
1549 }
1550 if let Some(q) = candidate {
1551 task.mark_triage_applied(&q.id);
1552 }
1553 task.resume_disk_wait();
1554 queue.commit_disk_wait(task, &expected)?;
1555 let notices = crate::notices::Notices::at(home.join("notifications"));
1556 let keys = [
1557 format!("disk-recovered:{}", task.id),
1558 format!("disk:{}", repo_for(task).display()),
1559 format!("task:{}", task.id),
1560 ];
1561 let repo_waiting = queue
1562 .list()
1563 .iter()
1564 .any(|other| other.repo == task.repo && disk_wait_eligible(other));
1565 for notice in notices.list() {
1566 if notice.key == keys[0]
1567 || notice.key == keys[2]
1568 || (!repo_waiting && notice.key == keys[1])
1569 {
1570 let _ = notices.dismiss(¬ice.id);
1571 }
1572 }
1573 Ok(candidate.is_some())
1574}
1575
1576pub fn run_once(
1595 queue: &Queue,
1596 questions: &Questions,
1597 config_override: Option<&Path>,
1598 now: Timestamp,
1599) -> Report {
1600 run_once_inner(queue, questions, config_override, now, false)
1601}
1602
1603pub(crate) fn monitor_disk_waits(
1605 queue: &Queue,
1606 questions: &Questions,
1607 config: Option<&Path>,
1608 now: Timestamp,
1609) -> Report {
1610 run_once_inner(queue, questions, config, now, true)
1611}
1612
1613fn run_once_inner(
1614 queue: &Queue,
1615 questions: &Questions,
1616 config_override: Option<&Path>,
1617 now: Timestamp,
1618 disk_only: bool,
1619) -> Report {
1620 run_once_with_probe(
1621 queue,
1622 questions,
1623 config_override,
1624 now,
1625 disk_only,
1626 machine_cause_resolved,
1627 )
1628}
1629
1630fn run_once_with_probe(
1631 queue: &Queue,
1632 questions: &Questions,
1633 config_override: Option<&Path>,
1634 now: Timestamp,
1635 disk_only: bool,
1636 resolved: impl Fn(&Task, &Config) -> Option<bool>,
1637) -> Report {
1638 let mut report = Report {
1639 answered: if disk_only {
1640 Vec::new()
1641 } else {
1642 apply_answers(queue, questions)
1643 },
1644 quarantined: if disk_only {
1645 Vec::new()
1646 } else {
1647 quarantine_orphaned_blocked(queue, questions)
1648 },
1649 ..Report::default()
1650 };
1651 if !disk_only {
1652 ask_about_stuck_roots(queue, questions, config_override, &mut report);
1653 }
1654 let recovery = RecoveryIndex::load(questions);
1655 let ownership = crate::task_chat::InquiryOwnership::default();
1656 for listed in queue.list() {
1657 if listed.status != TaskStatus::Held {
1658 continue;
1659 }
1660 let Ok(_claim) = queue.claim(&listed.id) else {
1661 continue;
1662 };
1663 let Ok(mut task) = queue.get(&listed.id) else {
1664 continue;
1665 };
1666 if task.status != TaskStatus::Held {
1670 continue;
1671 }
1672 if crate::base_sync_recovery::held_task(queue, &task) {
1673 continue;
1674 }
1675 if repair_receipt_notice(queue, &task) {
1676 continue;
1677 }
1678 if !disk_wait_known(&task, questions.root().parent().unwrap_or(Path::new("")))
1685 && deps_pending_in(&recovery, &task)
1686 {
1687 continue;
1688 }
1689
1690 let explicit = match &task.overrides {
1691 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
1692 _ => config_override,
1693 };
1694 let cfg = Config::discover(&repo_for(&task), explicit)
1695 .ok()
1696 .map(|(c, _)| c);
1697 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1698
1699 if disk_wait_known(&task, questions.root().parent().unwrap_or(Path::new(""))) {
1700 if task
1701 .disk_checked_at
1702 .is_some_and(|at| now.as_second() - at.as_second() < 60)
1703 {
1704 continue;
1705 }
1706 let expected = task.clone();
1707 task.disk_checked_at = Some(now);
1708 if !task.has_disk_capacity_cause() {
1709 task.mark_disk_capacity();
1711 }
1712 if queue.commit_disk_wait(&mut task, &expected).is_err() {
1713 continue;
1714 }
1715 refresh_disk_notices(questions.root().parent().unwrap_or(Path::new("")), &task);
1716 if cfg.as_ref().and_then(|c| resolved(&task, c)) == Some(true) {
1719 match resume_disk_task(queue, questions, &mut task, &ownership) {
1720 Ok(superseded) => {
1721 if superseded {
1722 report.superseded.push(task.id.clone());
1723 }
1724 report.resumed.push(task.id.clone());
1725 }
1726 Err(error) => {
1727 if !report.needs_owner.contains(&task.id) {
1728 report.needs_owner.push(task.id.clone());
1729 }
1730 crate::notices::raise_in(
1731 questions.root().parent().unwrap_or(Path::new("")),
1732 crate::notices::Notice::warn(
1733 &format!("disk-recovered:{}", task.id),
1734 format!(
1735 "Task {}: disk capacity recovered; automatic recovery is still blocked ({error}). Existing owner decisions and unrelated approvals require explicit resolution. Disk monitoring continues.",
1736 task.id
1737 ),
1738 ).link(crate::notices::Link::Task { id: task.id.clone() }).about([task.id.clone()]),
1739 );
1740 }
1741 }
1742 }
1743 continue;
1744 }
1745 if disk_only {
1746 continue;
1747 }
1748
1749 if let Some(q) = recovery.latest(NODE, &task.id) {
1750 if q.status.open() || awaiting_task_chat(&q, &task) {
1751 continue;
1753 }
1754 }
1760
1761 match task.hold_source {
1762 Some(HoldSource::Machine) => {
1763 let overridden = task
1764 .resume_override
1765 .as_ref()
1766 .is_some_and(|o| o.conductor_rehold.is_some() && !o.forced);
1767 if overridden {
1768 if file_question(
1769 questions,
1770 &ownership,
1771 &task,
1772 Bucket::ConductorOverride,
1773 w,
1774 now,
1775 )
1776 .is_some()
1777 {
1778 report.asked.push(task.id.clone());
1779 }
1780 } else if cfg.as_ref().and_then(|c| resolved(&task, c)) == Some(true) {
1781 task.release();
1782 if queue.put(&mut task).is_ok() {
1783 report.resumed.push(task.id.clone());
1784 }
1785 } else if file_question(
1786 questions,
1787 &ownership,
1788 &task,
1789 Bucket::MachineUnknown,
1790 w,
1791 now,
1792 )
1793 .is_some()
1794 {
1795 report.asked.push(task.id.clone());
1796 }
1797 }
1798 None => {
1799 if file_question(questions, &ownership, &task, Bucket::Legacy, w, now).is_some() {
1800 report.asked.push(task.id.clone());
1801 }
1802 }
1803 Some(HoldSource::Manual) => {
1804 if manual_is_stale(&task, now)
1805 && file_question(questions, &ownership, &task, Bucket::ManualStale, w, now)
1806 .is_some()
1807 {
1808 report.asked.push(task.id.clone());
1809 }
1810 }
1811 }
1812 }
1813 report
1814}
1815
1816pub fn open_question_for(questions: &Questions, task_id: &str) -> Option<Question> {
1821 latest_triage_question(questions, task_id).filter(|q| q.status.open())
1822}
1823
1824pub fn pending_for(questions: &Questions, task: &Task) -> bool {
1838 !pending_recovery_questions(questions, task).is_empty()
1839}
1840
1841pub(crate) fn pending_recovery_questions(questions: &Questions, task: &Task) -> Vec<Question> {
1844 RecoveryIndex::load(questions).pending(task)
1845}
1846
1847fn awaiting_task_chat(q: &Question, task: &Task) -> bool {
1848 q.status != QuestionStatus::Abandoned
1849 && q.routed_chat.is_some()
1850 && !already_applied(task, q)
1851 && task.resumption.as_ref().map_or(0, |r| r.generation) == q.triage_generation.unwrap_or(0)
1852 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1853}
1854
1855fn deps_pending_in(recovery: &RecoveryIndex, task: &Task) -> bool {
1859 match recovery.latest(DEPS_NODE, &task.id) {
1860 Some(q) if q.status.open() || awaiting_task_chat(&q, task) => true,
1861 Some(q) if q.status == QuestionStatus::Answered => !task.triage_applied(&q.id),
1862 _ => false,
1863 }
1864}
1865
1866pub fn open_task_ids(questions: &Questions) -> std::collections::BTreeSet<String> {
1871 RecoveryIndex::load(questions).open
1872}
1873
1874#[cfg(test)]
1875mod tests {
1876 use super::*;
1877 use crate::ask::Answer;
1878 use crate::queue::Source;
1879 use jiff::SignedDuration;
1880
1881 fn store() -> (tempfile::TempDir, Queue, Questions) {
1882 let dir = tempfile::tempdir().unwrap();
1883 let q = Queue::at(dir.path().join("queue"));
1884 let s = Questions::at(dir.path().join("questions"));
1885 (dir, q, s)
1886 }
1887
1888 fn task(title: &str, repo: PathBuf) -> Task {
1889 Task::new(title.to_owned(), format!("do {title}"), repo, Source::Human)
1890 }
1891
1892 fn disk_task(repo: PathBuf) -> Task {
1893 let mut t = task("capacity", repo);
1894 t.hold_machine(crate::disk::gate_in(1_477_050_368, 3_221_225_472, "ja"));
1895 t.mark_disk_capacity();
1896 t.attempts = 3;
1897 t
1898 }
1899
1900 #[test]
1901 fn disk_wait_repeated_pressure_measurement_failure_and_recovery_cost_nothing() {
1902 let (dir, queue, questions) = store();
1903 let cfg = gate_disabled_config(dir.path());
1904 let mut t = disk_task(dir.path().to_owned());
1905 queue.put(&mut t).unwrap();
1906 let notices = crate::notices::Notices::at(dir.path().join("notifications"));
1907 let before = serde_json::to_value(notices.list()).unwrap();
1908 let start = Timestamp::now();
1909 let probes = std::cell::Cell::new(0);
1910 for seconds in [0, 1, 10, 60, 61, 120] {
1911 let report = run_once_with_probe(
1912 &queue,
1913 &questions,
1914 Some(&cfg),
1915 start + SignedDuration::from_secs(seconds),
1916 true,
1917 |_, _| {
1918 probes.set(probes.get() + 1);
1919 Some(false) },
1921 );
1922 assert!(report.is_empty());
1923 assert_eq!(queue.get(&t.id).unwrap().attempts, 3);
1924 }
1925 assert_eq!(
1926 probes.get(),
1927 3,
1928 "resource probes are bounded to one per minute"
1929 );
1930 assert!(questions.list().is_empty());
1931 assert_eq!(serde_json::to_value(notices.list()).unwrap(), before);
1932 let recovered = run_once_with_probe(
1933 &queue,
1934 &questions,
1935 Some(&cfg),
1936 start + SignedDuration::from_secs(180),
1937 true,
1938 |_, _| Some(true),
1939 );
1940 assert_eq!(recovered.resumed, [t.id.clone()]);
1941 let saved = queue.get(&t.id).unwrap();
1942 assert_eq!(saved.status, TaskStatus::Queued);
1943 assert_eq!(saved.attempts, 3);
1944 let again = run_once_with_probe(
1945 &queue,
1946 &questions,
1947 Some(&cfg),
1948 start + SignedDuration::from_secs(240),
1949 true,
1950 |_, _| panic!("queued tasks are not relaunched"),
1951 );
1952 assert!(again.is_empty());
1953 }
1954
1955 #[test]
1956 fn disk_wait_supersedes_only_untouched_automatic_questions_in_both_stores() {
1957 for routed in [false, true] {
1958 let (dir, queue, questions) = store();
1959 let cfg = gate_disabled_config(dir.path());
1960 let mut t = disk_task(dir.path().to_owned());
1961 queue.put(&mut t).unwrap();
1962 let q = file_question(
1963 &questions,
1964 &crate::task_chat::InquiryOwnership::default(),
1965 &t,
1966 Bucket::MachineUnknown,
1967 &JA,
1968 Timestamp::now(),
1969 )
1970 .unwrap();
1971 if routed {
1972 crate::task_chat::sweep(
1973 &queue,
1974 &crate::talk::Talks::at(dir.path().join("talks")),
1975 &questions,
1976 dir.path(),
1977 );
1978 assert!(
1979 crate::task_inquiry::Outbox::at(dir.path())
1980 .get(&q.id)
1981 .is_ok()
1982 );
1983 }
1984 let start = Timestamp::now();
1985 for seconds in [0, 60, 120] {
1986 assert!(
1987 run_once_with_probe(
1988 &queue,
1989 &questions,
1990 Some(&cfg),
1991 start + SignedDuration::from_secs(seconds),
1992 true,
1993 |_, _| Some(false)
1994 )
1995 .is_empty()
1996 );
1997 assert!(questions.read_history(&q.id).unwrap().status.open());
1998 assert_eq!(questions.list().len(), 1);
1999 }
2000 let report = monitor_disk_waits(
2001 &queue,
2002 &questions,
2003 Some(&cfg),
2004 start + SignedDuration::from_secs(180),
2005 );
2006 assert_eq!(report.resumed, [t.id.clone()], "routed={routed}");
2007 let source = questions.read_history(&q.id).unwrap();
2008 assert_eq!(source.status, QuestionStatus::Abandoned);
2009 assert!(source.answer.is_none());
2010 assert!(source.answered_at.is_none());
2011 assert!(!pending_for(&questions, &queue.get(&t.id).unwrap()));
2012 if routed {
2013 let d = crate::task_inquiry::Outbox::at(dir.path())
2014 .get(&q.id)
2015 .unwrap()
2016 .decision
2017 .unwrap();
2018 assert_eq!(d.status, crate::task_decision::Status::Resolved);
2019 assert!(
2020 d.request.is_none(),
2021 "system supersession fabricates no owner request"
2022 );
2023 }
2024 }
2025 }
2026
2027 #[test]
2028 fn disk_wait_respects_owner_hold_discussion_and_unrelated_approvals() {
2029 for kind in [
2030 "manual",
2031 "answered",
2032 "discussion",
2033 "unrelated",
2034 "override",
2035 "native-hold",
2036 "missing-native",
2037 ] {
2038 let (dir, queue, questions) = store();
2039 let cfg = gate_disabled_config(dir.path());
2040 let mut t = disk_task(dir.path().to_owned());
2041 queue.put(&mut t).unwrap();
2042 let q = file_question(
2043 &questions,
2044 &crate::task_chat::InquiryOwnership::default(),
2045 &t,
2046 Bucket::MachineUnknown,
2047 &EN,
2048 Timestamp::now(),
2049 )
2050 .unwrap();
2051 match kind {
2052 "manual" => {
2053 t.hold_manual(Some("owner hold".into()));
2054 queue.put(&mut t).unwrap();
2055 }
2056 "answered" => {
2057 questions
2058 .update(&q.id, |q| q.answer(Answer::Choice(q.choices[1].clone())))
2059 .unwrap();
2060 }
2061 "discussion" => {
2062 questions
2063 .update(&q.id, |q| q.say("please wait".to_owned()))
2064 .unwrap();
2065 }
2066 "unrelated" => {
2067 questions
2068 .update(&q.id, |q| {
2069 q.node = "land".into();
2070 Ok(())
2071 })
2072 .unwrap();
2073 }
2074 "override" => {
2075 t.resume_override = Some(OperatorResume {
2076 question_id: "owner".into(),
2077 at: Timestamp::now(),
2078 conductor_rehold: Some("pending conductor decision".into()),
2079 forced: true,
2080 pinned_run: None,
2081 });
2082 queue.put(&mut t).unwrap();
2083 }
2084 "native-hold" | "missing-native" => {
2085 crate::task_chat::sweep(
2086 &queue,
2087 &crate::talk::Talks::at(dir.path().join("talks")),
2088 &questions,
2089 dir.path(),
2090 );
2091 if kind == "native-hold" {
2092 crate::task_inquiry::Outbox::at(dir.path())
2093 .update_decision(&q.id, |d| d.accept("owner requested hold"))
2094 .unwrap();
2095 } else {
2096 questions
2100 .update(&q.id, |q| {
2101 q.routed_chat = Some("delivered-chat".into());
2102 Ok(())
2103 })
2104 .unwrap();
2105 std::fs::remove_file(
2106 dir.path()
2107 .join("notifications/task-inquiries")
2108 .join(format!("{}.json", q.id)),
2109 )
2110 .unwrap();
2111 }
2112 }
2113 _ => unreachable!(),
2114 }
2115 let report = run_once(&queue, &questions, Some(&cfg), Timestamp::now());
2116 assert!(report.resumed.is_empty(), "{kind}");
2117 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held, "{kind}");
2118 assert!(
2119 questions.read_history(&q.id).unwrap().status != QuestionStatus::Abandoned,
2120 "{kind}"
2121 );
2122 }
2123 }
2124
2125 #[test]
2126 fn disk_wait_reuses_retained_execution_and_preserves_attempt_and_approval_evidence() {
2127 let (dir, queue, questions) = store();
2128 let cfg = gate_disabled_config(dir.path());
2129 let mut run = crate::run::RunState::new(
2130 dir.path().to_owned(),
2131 "main".into(),
2132 "base".into(),
2133 "capacity".into(),
2134 Config::default(),
2135 );
2136 run.status = crate::run::RunStatus::Blocked;
2137 run.driver_exited = true;
2138 run.save_under(dir.path()).unwrap();
2139 let mut t = disk_task(dir.path().to_owned());
2140 t.runs.push(run.id.clone());
2141 t.resume_override = Some(OperatorResume {
2142 question_id: "approved-recovery".into(),
2143 at: Timestamp::now(),
2144 conductor_rehold: None,
2145 forced: true,
2146 pinned_run: Some(run.id.clone()),
2147 });
2148 queue.put(&mut t).unwrap();
2149 let before = std::fs::read(dir.path().join("runs").join(&run.id).join("run.json")).unwrap();
2150 let report = run_once_with_probe(
2151 &queue,
2152 &questions,
2153 Some(&cfg),
2154 Timestamp::now(),
2155 true,
2156 |_, _| Some(true),
2157 );
2158 assert_eq!(report.resumed, [t.id.clone()]);
2159 let resumed = queue.get(&t.id).unwrap();
2160 assert_eq!(resumed.runs, t.runs);
2161 assert_eq!(resumed.disk_resume_run.as_deref(), Some(run.id.as_str()));
2162 assert_eq!(resumed.attempts, t.attempts);
2163 assert_eq!(resumed.resumption, t.resumption);
2164 assert_eq!(
2165 serde_json::to_value(&resumed.resume_override).unwrap(),
2166 serde_json::to_value(&t.resume_override).unwrap()
2167 );
2168 assert_eq!(
2169 std::fs::read(dir.path().join("runs").join(&run.id).join("run.json")).unwrap(),
2170 before
2171 );
2172 assert!(
2173 run_once_with_probe(
2174 &queue,
2175 &questions,
2176 Some(&cfg),
2177 Timestamp::now(),
2178 true,
2179 |_, _| panic!("no duplicate launch")
2180 )
2181 .is_empty()
2182 );
2183 }
2184
2185 #[test]
2186 fn disk_wait_interrupted_question_supersession_is_idempotent() {
2187 for abandoned in [false, true] {
2188 let (dir, queue, questions) = store();
2189 let cfg = gate_disabled_config(dir.path());
2190 let mut t = disk_task(dir.path().to_owned());
2191 queue.put(&mut t).unwrap();
2192 let q = file_question(
2193 &questions,
2194 &crate::task_chat::InquiryOwnership::default(),
2195 &t,
2196 Bucket::MachineUnknown,
2197 &EN,
2198 Timestamp::now(),
2199 )
2200 .unwrap();
2201 let outbox = crate::task_inquiry::Outbox::at(dir.path());
2202 outbox
2203 .enqueue(crate::task_chat::live_record(dir.path(), &t, &q))
2204 .unwrap();
2205 outbox
2206 .update_decision(&q.id, |d| d.settle(DISK_SUPERSEDED))
2207 .unwrap();
2208 if abandoned {
2209 questions
2210 .update(&q.id, |q| {
2211 q.abandon(DISK_SUPERSEDED);
2212 Ok(())
2213 })
2214 .unwrap();
2215 }
2216 let report = monitor_disk_waits(&queue, &questions, Some(&cfg), Timestamp::now());
2217 assert_eq!(report.resumed, [t.id.clone()]);
2218 assert_eq!(queue.get(&t.id).unwrap().attempts, 3);
2219 assert!(questions.read_history(&q.id).unwrap().answer.is_none());
2220 assert_eq!(
2221 outbox.get(&q.id).unwrap().decision.unwrap().history.len(),
2222 1
2223 );
2224 }
2225 }
2226
2227 #[test]
2228 fn disk_wait_legacy_migration_requires_machine_provenance_and_exact_localized_format() {
2229 let (dir, queue, _) = store();
2230 for (language, source, suffix, expected) in [
2231 ("en", Some(HoldSource::Machine), "", true),
2232 ("ja", Some(HoldSource::Machine), "", true),
2233 ("ja", Some(HoldSource::Manual), "", false),
2234 ("ja", None, "", false),
2235 ("ja", Some(HoldSource::Machine), " owner hold", false),
2236 ] {
2237 let mut t = task("legacy", dir.path().to_owned());
2238 t.hold_machine(Some(format!(
2239 "{}{suffix}",
2240 crate::disk::gate_in(1477050368, 3221225472, language).unwrap()
2241 )));
2242 t.schema = 21;
2243 t.hold_source = source;
2244 queue.put(&mut t).unwrap();
2245 let loaded = queue.get(&t.id).unwrap();
2246 assert_eq!(loaded.machine_hold_cause.is_some(), expected);
2247 assert_eq!(disk_wait_eligible(&loaded), expected);
2248 }
2249 }
2250
2251 #[test]
2252 fn disk_wait_refreshes_stale_notice_wording_without_redelivery() {
2253 let (dir, queue, questions) = store();
2254 let cfg = gate_disabled_config(dir.path());
2255 let mut t = disk_task(dir.path().to_owned());
2256 queue.put(&mut t).unwrap();
2257 let notices = crate::notices::Notices::at(dir.path().join("notifications"));
2258 let key = format!("disk:{}", t.repo.display());
2259 let n = notices
2260 .raise(crate::notices::Notice::warn(
2261 &key,
2262 "Free some, then release it from the queue.",
2263 ))
2264 .unwrap();
2265 notices.mark_read(&n.id).unwrap();
2266 let before = notices.get(&n.id).unwrap();
2267 let report = run_once_with_probe(
2268 &queue,
2269 &questions,
2270 Some(&cfg),
2271 Timestamp::now(),
2272 true,
2273 |_, _| Some(false),
2274 );
2275 assert!(report.is_empty());
2276 let after = notices.get(&n.id).unwrap();
2277 assert!(after.message.contains("automatically"));
2278 assert_eq!(before.generation, after.generation);
2279 assert_eq!(before.count, after.count);
2280 assert_eq!(before.read_at, after.read_at);
2281 }
2282
2283 #[test]
2284 fn disk_wait_owner_discard_is_applied_without_automatic_resumption() {
2285 let (dir, queue, questions) = store();
2286 let cfg = gate_disabled_config(dir.path());
2287 let mut t = disk_task(dir.path().to_owned());
2288 queue.put(&mut t).unwrap();
2289 let q = file_question(
2290 &questions,
2291 &crate::task_chat::InquiryOwnership::default(),
2292 &t,
2293 Bucket::MachineUnknown,
2294 &EN,
2295 Timestamp::now(),
2296 )
2297 .unwrap();
2298 questions
2299 .update(&q.id, |q| q.answer(Answer::Choice(q.choices[2].clone())))
2300 .unwrap();
2301 let report = run_once(&queue, &questions, Some(&cfg), Timestamp::now());
2302 assert!(report.resumed.is_empty());
2303 assert!(queue.get(&t.id).is_err());
2304 assert!(questions.read_history(&q.id).unwrap().answer.is_some());
2305 }
2306
2307 #[test]
2308 fn disk_wait_commit_cannot_overwrite_a_concurrent_owner_hold() {
2309 let (dir, queue, questions) = store();
2310 let mut stale = disk_task(dir.path().to_owned());
2311 queue.put(&mut stale).unwrap();
2312 let mut current = queue.get(&stale.id).unwrap();
2313 current.hold_manual(Some("wait for owner".into()));
2314 queue.put(&mut current).unwrap();
2315 assert!(
2316 resume_disk_task(
2317 &queue,
2318 &questions,
2319 &mut stale,
2320 &crate::task_chat::InquiryOwnership::default()
2321 )
2322 .is_err()
2323 );
2324 assert_eq!(
2325 queue.get(¤t.id).unwrap().hold_source,
2326 Some(HoldSource::Manual)
2327 );
2328 }
2329
2330 #[test]
2331 fn recovery_index_preserves_supersession_and_ambiguous_file_references() {
2332 let (dir, queue, questions) = store();
2333 let mut t = task("supersession", dir.path().join("repo"));
2334 t.id = "task-alpha-end".into();
2335 queue.put(&mut t).unwrap();
2336 let at: Timestamp = "2026-11-01T00:00:00Z".parse().unwrap();
2337 let mut all = Vec::new();
2338 for (i, run) in ["task-alpha-end", "task-alpha", "end"]
2339 .into_iter()
2340 .enumerate()
2341 {
2342 let mut q = Question::new(
2343 run.into(),
2344 NODE.into(),
2345 SEAT.into(),
2346 "superseded".into(),
2347 String::new(),
2348 EN.choices3(),
2349 );
2350 q.id = format!("superseded-{i}");
2351 q.asked_at = at;
2352 all.push(q);
2353 }
2354 let evidence: Vec<_> = all
2355 .iter()
2356 .map(|q| {
2357 (
2358 q.id.clone(),
2359 crate::repair_budget::reconcile::source(q).unwrap(),
2360 )
2361 })
2362 .collect();
2363 t.repair_reconciliations.push(
2364 serde_json::from_value(serde_json::json!({
2365 "request": {
2366 "task": t.id, "run": "run", "head": "head", "expect_pass": 0,
2367 "receipt": "grant", "question": "original", "answer_at": at,
2368 "request": "reconcile", "reason": "fixture"
2369 },
2370 "grant": {
2371 "request": {
2372 "task": t.id, "run": "run", "kind": "review", "rounds": 1,
2373 "head": "head", "expect_pass": 0, "request": "grant", "reason": "fixture"
2374 },
2375 "at": at, "decision": null, "contract": "fixture"
2376 },
2377 "source": "fixture", "application": "fixture", "decision": at,
2378 "superseded_questions": evidence
2379 }))
2380 .unwrap(),
2381 );
2382 queue.put(&mut t).unwrap();
2383 let index = RecoveryIndex::from_scan(&queue, &[t.clone()], all.clone());
2384 assert!(index.open.is_empty());
2385 std::fs::write(queue.path_of("task-alphabet-other"), "{").unwrap();
2387 std::fs::write(queue.path_of("broken-end"), "{").unwrap();
2388 let index = RecoveryIndex::from_scan(&queue, &[t.clone()], all.clone());
2389 assert_eq!(index.open, ["end".into(), "task-alpha".into()].into());
2390 all[0].detail.push_str("changed evidence");
2391 let index = RecoveryIndex::from_scan(&queue, &[t], all);
2392 assert!(index.open.contains("task-alpha-end"));
2393 assert_eq!(questions.list().len(), 0);
2394 }
2395
2396 #[test]
2397 fn ownership_index_requires_positive_foreground_provenance() {
2398 let (dir, queue, _) = store();
2399 let mut run = crate::run::RunState::new(
2400 dir.path().join("repo"),
2401 "main".into(),
2402 "origin".into(),
2403 "fixture".into(),
2404 Config::default(),
2405 );
2406 run.origin = Some(crate::run::Origin::operator());
2407 run.save_under(dir.path()).unwrap();
2408 let foreground = run.id.clone();
2409 run.id = "owned-run".into();
2410 run.origin = Some(crate::run::Origin::queue("missing-task"));
2411 run.save_under(dir.path()).unwrap();
2412 let mut task = task("owner", dir.path().join("repo"));
2413 task.runs = vec!["linked-run".into()];
2414 queue.put(&mut task).unwrap();
2415 let ownership = crate::task_chat::InquiryOwnership::default();
2416 for (run, cwd, node, seat, expected) in [
2417 (foreground.as_str(), true, "node", "seat", true),
2418 (foreground.as_str(), false, "node", "seat", false),
2419 ("owned-run", true, "ask", "operator", false),
2420 ("linked-run", true, "ask", "operator", false),
2421 ("missing-run", true, "node", "seat", false),
2422 ("missing-run", false, "ask", "operator", true),
2423 ] {
2424 let mut q = Question::new(
2425 run.into(),
2426 node.into(),
2427 seat.into(),
2428 "provenance".into(),
2429 String::new(),
2430 Vec::new(),
2431 );
2432 q.cwd = cwd.then(|| dir.path().join("repo").display().to_string());
2433 assert_eq!(standalone_by_scanning(dir.path(), &q), expected);
2434 assert_eq!(
2435 crate::task_chat::standalone_inquiry_in(dir.path(), &q, &ownership),
2436 expected
2437 );
2438 }
2439 }
2440
2441 fn standalone_by_scanning(home: &Path, q: &Question) -> bool {
2443 if q.routed_task.is_some()
2444 || (q.node == crate::bump::NOTICE_NODE && q.seat == "release-watch")
2445 {
2446 return false;
2447 }
2448 if q.legacy_foreground || q.run.is_empty() {
2449 return true;
2450 }
2451 if Queue::at(home.join("queue"))
2452 .list()
2453 .iter()
2454 .any(|task| task.id == q.run || task.runs.contains(&q.run))
2455 {
2456 return false;
2457 }
2458 let run = crate::run::RunState::load_under(&q.run, home).ok();
2459 let origin = run.as_ref().and_then(|run| run.origin.as_ref());
2460 if origin.is_some_and(|origin| origin.task.is_some()) {
2461 return false;
2462 }
2463 (q.node == "ask" && q.seat == "operator")
2464 || (q.cwd.is_some() && origin.is_some_and(|origin| origin.task.is_none()))
2465 }
2466
2467 fn scan_counts() -> (usize, usize, usize) {
2468 (
2469 crate::queue::LIST_READS.with(|c| c.replace(0)),
2470 crate::queue::TASK_READS.with(|c| c.replace(0)),
2471 crate::ask::QUESTION_SCANS.with(|c| c.replace(0)),
2472 )
2473 }
2474
2475 #[test]
2476 fn creating_and_updating_questions_share_one_queue_read() {
2477 let (dir, queue, questions) = store();
2478 let size = 200;
2479 let mut ids = Vec::new();
2480 for i in 0..size {
2481 let mut t = task("scan fixture", dir.path().join("repo"));
2482 t.id = format!("task-{i:04}");
2483 t.runs = vec![format!("run-{i:04}")];
2484 t.hold_manual(Some("held for scan fixture".into()));
2485 queue.put(&mut t).unwrap();
2486 ids.push(t.id);
2487 }
2488 let ownership = crate::task_chat::InquiryOwnership::default();
2489 scan_counts();
2490 let mut created = Vec::new();
2491 for id in &ids {
2492 let mut q = Question::new(
2493 id.clone(),
2494 NODE.into(),
2495 SEAT.into(),
2496 "scan fixture".into(),
2497 String::new(),
2498 EN.choices3(),
2499 );
2500 questions.put_executable_in(&mut q, &ownership).unwrap();
2501 assert_eq!(q.routed_task.as_deref(), Some(id.as_str()));
2502 created.push(q.id);
2503 }
2504 for id in &created {
2505 questions
2506 .update_in(id, &ownership, |q| {
2507 q.answer_delivered = true;
2508 Ok(())
2509 })
2510 .unwrap();
2511 }
2512 let (lists, _, _) = scan_counts();
2513 assert_eq!(lists, 1, "one queue read for {size} creations and updates");
2514 }
2515
2516 #[test]
2517 fn recovery_scans_are_linear_and_match_legacy_decisions() {
2518 let mut previous = None;
2519 for size in [150, 300] {
2520 let (dir, queue, questions) = store();
2521 let at: Timestamp = "2026-11-01T00:00:00Z".parse().unwrap();
2522 let mut sources = Vec::new();
2523 let mut expected_applied = std::collections::BTreeSet::new();
2524 for i in 0..size {
2525 let mut t = task("scan fixture", dir.path().join("repo"));
2526 t.id = format!("task-{i:04}");
2527 t.runs = vec![format!("run-{i:04}")];
2528 t.hold_manual(Some("held for scan fixture".into()));
2529 queue.put(&mut t).unwrap();
2530 let mut q = Question::new(
2531 t.id.clone(),
2532 NODE.into(),
2533 SEAT.into(),
2534 "scan fixture".into(),
2535 String::new(),
2536 EN.choices3(),
2537 );
2538 q.id = format!("question-{i:04}");
2539 q.asked_at = at;
2540 match i % 10 {
2541 0 => {
2542 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2543 q.accepted_answer_at = q.answered_at;
2544 expected_applied.insert(t.id.clone());
2545 }
2546 1 => q.node = DEPS_NODE.into(),
2547 2 => {
2548 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
2549 }
2550 3 => {
2551 q.run = format!("missing-{i}");
2552 q.legacy_foreground = true;
2553 }
2554 4 => {
2555 q.run.clear();
2556 q.node = crate::bump::NOTICE_NODE.into();
2557 q.seat = "release-watch".into();
2558 }
2559 5 => {
2560 q.run = format!("missing-{i}");
2561 q.node = "ask".into();
2562 q.seat = "operator".into();
2563 }
2564 6 => q.run = t.runs[0].clone(),
2565 7 => q.run = format!("missing-{i}"),
2566 8 => q.abandon("fixture"),
2567 _ => {}
2568 }
2569 questions.put(&mut q).unwrap();
2570 sources.push(q.id.clone());
2571 if i % 10 == 9 {
2572 q.id = format!("question-{i:04}-tie");
2573 questions.put(&mut q).unwrap();
2574 sources.push(q.id.clone());
2575 }
2576 }
2577 let migration = dir.path().join("notifications/task-chat-migration-v1");
2578 std::fs::create_dir_all(&migration).unwrap();
2579 std::fs::write(
2580 migration.join("state.json"),
2581 serde_json::to_vec(&serde_json::json!({
2582 "schema": 1, "classification_revision": 1, "pending": [], "sources": sources
2583 }))
2584 .unwrap(),
2585 )
2586 .unwrap();
2587
2588 scan_counts();
2589 let all = questions.list_for_effects();
2590 let (lists, reads, scans) = scan_counts();
2591 assert_eq!((lists, reads, scans), (1, size, 1));
2592 let tasks = queue.list();
2593 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
2594 for q in &all {
2595 assert_eq!(
2596 crate::task_chat::standalone_inquiry_in(dir.path(), q, &ownership),
2597 standalone_by_scanning(dir.path(), q),
2598 "{}",
2599 q.id,
2600 );
2601 if q.id.ends_with("0002") {
2602 assert!(q.answer.is_none(), "saved consent remains fenced");
2603 }
2604 }
2605 let recovery = RecoveryIndex::from_scan(&queue, &tasks, all.clone());
2606 for task in &tasks {
2607 for node in [NODE, DEPS_NODE] {
2608 let expected = all
2609 .iter()
2610 .filter(|q| {
2611 q.node == node
2612 && q.run == task.id
2613 && !queue.get(&q.run).ok().is_some_and(|t| {
2614 crate::repair_budget::reconcile::superseded_question(&t, q)
2615 })
2616 })
2617 .max_by(|a, b| a.asked_at.cmp(&b.asked_at).then_with(|| a.id.cmp(&b.id)));
2618 assert_eq!(
2619 recovery.latest(node, &task.id).map(|q| q.id),
2620 expected.map(|q| q.id.clone())
2621 );
2622 }
2623 }
2624
2625 scan_counts();
2626 crate::daemon::apply_task_choice_actions(&queue, &questions, dir.path());
2627 assert_eq!(
2628 scan_counts(),
2629 (1, size, 1),
2630 "choice-action rechecks share ownership"
2631 );
2632
2633 scan_counts();
2634 let applied: std::collections::BTreeSet<_> =
2635 apply_answers(&queue, &questions).into_iter().collect();
2636 let counts = scan_counts();
2637 assert_eq!(applied, expected_applied);
2638 assert!(counts.0 <= 3, "queue scans: {counts:?}");
2639 assert!(counts.1 <= 4 * size, "task reads: {counts:?}");
2640 assert_eq!(counts.2, 2, "question scans: {counts:?}");
2641 if let Some((_, reads, scans)) = previous {
2642 assert!(counts.1 <= 2 * reads);
2643 assert_eq!(counts.2, scans);
2644 }
2645 previous = Some(counts);
2646
2647 let config = gate_disabled_config(dir.path());
2648 scan_counts();
2649 run_once(&queue, &questions, Some(&config), Timestamp::now());
2650 let lap = scan_counts();
2651 assert!(lap.0 <= 12, "triage queue scans: {lap:?}");
2652 assert!(lap.1 <= 14 * size, "triage task reads: {lap:?}");
2653 assert!(lap.2 <= 6, "triage question scans: {lap:?}");
2654 }
2655 }
2656
2657 #[test]
2658 fn run_once_filing_many_new_questions_reads_the_queue_a_constant_number_of_times() {
2659 let (dir, queue, questions) = store();
2660 let size = 200;
2661 for i in 0..size {
2662 let mut t = task("scan fixture", dir.path().join("repo"));
2663 t.id = format!("task-{i:04}");
2664 t.hold_machine(Some("gate red".to_owned()));
2665 queue.put(&mut t).unwrap();
2666 }
2667 scan_counts();
2668 let report = run_once(&queue, &questions, None, Timestamp::now());
2669 let (lists, _, _) = scan_counts();
2670 assert_eq!(report.asked.len(), size, "every held task is asked about");
2671 assert!(
2672 lists <= 12,
2673 "queue scans while filing {size} questions: {lists}"
2674 );
2675 }
2676
2677 #[test]
2678 fn base_recovery_stops_suppress_generic_and_dependency_resume_offers() {
2679 let (dir, queue, questions) = store();
2680 let mut root = task("base stop", dir.path().to_path_buf());
2681 let mut run = crate::run::RunState::new(
2682 dir.path().to_path_buf(),
2683 "main".into(),
2684 "base".into(),
2685 "retained implementation".into(),
2686 Config::default(),
2687 );
2688 run.status = crate::run::RunStatus::Voting;
2689 run.base_sync = Some(crate::run::BaseSync {
2690 tip: "tip".into(),
2691 behind: 3,
2692 attempts: 4,
2693 conflict: Some("persisted exhaustion".into()),
2694 already_in: None,
2695 });
2696 run.save_under(dir.path()).unwrap();
2697 root.link_run(&run.id);
2698 root.hold_machine(Some("base stop".into()));
2699 queue.put(&mut root).unwrap();
2700 let mut dependent = task("dependent", dir.path().to_path_buf());
2701 dependent.blocked_by = vec![root.id.clone()];
2702 dependent.status = TaskStatus::Blocked;
2703 queue.put(&mut dependent).unwrap();
2704 for _ in 0..2 {
2705 let report = run_once(&queue, &questions, None, Timestamp::now());
2706 assert!(report.asked.is_empty());
2707 assert!(report.resumed.is_empty());
2708 }
2709 let q = file_question(
2712 &questions,
2713 &crate::task_chat::InquiryOwnership::default(),
2714 &root,
2715 Bucket::MachineUnknown,
2716 &EN,
2717 Timestamp::now(),
2718 )
2719 .unwrap();
2720 let mut q = q;
2721 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
2722 questions.put_executable(&mut q).unwrap();
2723 apply_answers(&queue, &questions);
2724 let saved = queue.get(&root.id).unwrap();
2725 assert_eq!(saved.status, TaskStatus::Held);
2726 assert!(saved.triage_applied(&q.id));
2727 assert_eq!(saved.runs, vec![run.id]);
2728 }
2729
2730 #[test]
2731 fn resource_holds_only_auto_release_a_proven_cache_wait_timeout() {
2732 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
2733 let dir = tempfile::tempdir().unwrap();
2734 let cfg = Config::default(); assert!(cfg.cache_dir().is_none());
2736 let mut state = RunState::new(
2737 dir.path().into(),
2738 "main".into(),
2739 "base".into(),
2740 "verify".into(),
2741 cfg.clone(),
2742 );
2743 let mut task = task("verify", dir.path().into());
2744 task.runs = vec![state.id.clone()];
2745 task.hold_machine(Some("[resource] cache operation stopped".into()));
2746 for cause in [
2747 ResourceBlockCause::Other,
2748 ResourceBlockCause::Unrecorded,
2749 ResourceBlockCause::Cancelled { park: None },
2750 ] {
2751 state.resource_block = Some(ResourceBlock {
2752 node: "gate".into(),
2753 kind: "shared build cache".into(),
2754 detail: "persistent metadata or manifest error".into(),
2755 cause,
2756 at: Timestamp::now(),
2757 });
2758 state.save_under(dir.path()).unwrap();
2759 for _ in 0..3 {
2761 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2762 }
2763 }
2764 let block = state.resource_block.as_mut().unwrap();
2765 block.cause = ResourceBlockCause::Timeout {
2766 waited_secs: 10,
2767 limit_secs: 10,
2768 };
2769 state.save_under(dir.path()).unwrap();
2770 assert_eq!(
2771 machine_cause_resolved_under(&task, &cfg, dir.path()),
2772 Some(true)
2773 );
2774 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
2775 state.save_under(dir.path()).unwrap();
2776 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2777 task.runs = vec!["unreadable-run".into()];
2778 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2779 }
2780
2781 #[cfg(unix)]
2782 #[test]
2783 fn disk_admission_holds_recheck_live_reservations_and_the_floor() {
2784 use crate::retention::OutputLease;
2785 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
2786 let temp = tempfile::tempdir().unwrap();
2787 let home = temp.path().canonicalize().unwrap();
2788 let mut cfg = Config::default();
2789 cfg.graph.worktree_root = Some(home.join("worktrees"));
2790 cfg.disk.min_free_bytes = 100;
2791 let cache = home.join("cache");
2792 cfg.verify.gate = vec![format!(
2793 "CARGO_TARGET_DIR='{}' cargo build",
2794 cache.display()
2795 )];
2796 let mut state = RunState::new(
2797 home.clone(),
2798 "main".into(),
2799 "base".into(),
2800 "verify".into(),
2801 cfg.clone(),
2802 );
2803 let saved_cache = home.join("saved-cache");
2804 state.config.verify.gate = vec![format!(
2805 "CARGO_TARGET_DIR='{}' cargo build",
2806 saved_cache.display()
2807 )];
2808 state.resource_block = Some(ResourceBlock {
2809 node: "gate".into(),
2810 kind: "disk admission".into(),
2811 detail: "writer headroom unavailable".into(),
2812 cause: ResourceBlockCause::Other,
2813 at: Timestamp::now(),
2814 });
2815 state.save_under(&home).unwrap();
2816 let mut task = task("verify", home.clone());
2817 task.runs = vec![state.id.clone()];
2818 task.hold_machine(Some(format!(
2819 "{}{}; no attempt was spent; resume to retry the affected operation",
2820 crate::cache::RESOURCE_HOLD,
2821 state.resource_block.as_ref().unwrap().description()
2822 )));
2823 task.schema = 21; let gib = 1024 * 1024 * 1024;
2825 let free = std::cell::Cell::new(100 + gib - 1);
2826 let check = || {
2827 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths, floor| {
2828 assert!(
2829 paths.contains(&cache),
2830 "the cache volume must also be checked"
2831 );
2832 assert!(
2833 paths.contains(&saved_cache),
2834 "saved and current cache locations both matter"
2835 );
2836 assert!(paths.contains(&state.worktree_root()));
2837 OutputLease::admit_with(&home, paths, floor, |_| Ok(free.get())).is_ok()
2838 })
2839 };
2840 assert_eq!(check(), Some(false), "the unchanged floor still applies");
2841 assert_eq!(
2842 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths, floor| {
2843 OutputLease::admit_with(&home, paths, floor, |_| {
2844 Err(anyhow::anyhow!("measurement unavailable"))
2845 })
2846 .is_ok()
2847 }),
2848 Some(false),
2849 "failed output-volume measurements must remain closed"
2850 );
2851 free.set(100 + gib);
2852 let writer = OutputLease::admit_with(
2853 &home,
2854 std::slice::from_ref(&home),
2855 cfg.disk.min_free_bytes,
2856 |_| Ok(free.get()),
2857 )
2858 .unwrap();
2859 assert_eq!(
2860 check(),
2861 Some(false),
2862 "df alone cannot resolve a live reservation"
2863 );
2864 drop(writer);
2865 assert_eq!(
2866 check(),
2867 Some(true),
2868 "releasing headroom resolves the machine hold"
2869 );
2870 assert_eq!(
2871 check(),
2872 Some(true),
2873 "the probe must release its temporary reservation"
2874 );
2875
2876 let namespace = home.join("disk-admission");
2878 let saved = home.join("saved-admission");
2879 std::fs::rename(&namespace, &saved).unwrap();
2880 std::os::unix::fs::symlink(&saved, &namespace).unwrap();
2881 assert_eq!(check(), Some(false));
2882 std::fs::remove_file(&namespace).unwrap();
2883 std::fs::rename(saved, namespace).unwrap();
2884 assert_eq!(check(), Some(true));
2885
2886 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Cancelled { park: None };
2887 state.save_under(&home).unwrap();
2888 assert_eq!(
2889 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
2890 "a cancelled operation must remain held"
2891 )),
2892 None
2893 );
2894 state.resource_block.as_mut().unwrap().kind = "incremental cache admission".into();
2895 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Other;
2896 state.save_under(&home).unwrap();
2897 assert_eq!(
2898 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
2899 "unproven incremental caches must remain held"
2900 )),
2901 None
2902 );
2903 task.runs.push("unreadable-run".into());
2904 assert_eq!(
2905 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
2906 "missing evidence must not release a hold"
2907 )),
2908 None
2909 );
2910 }
2911
2912 fn gate_disabled_config(dir: &std::path::Path) -> PathBuf {
2916 let config = dir.join("magi.toml");
2917 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2918 config
2919 }
2920
2921 #[test]
2922 fn resumption_later_keep_held_answer_wins_but_an_older_answer_cannot_undo_release() {
2923 for source in ["cli", "web"] {
2924 for answer_after_release in [true, false] {
2925 let (dir, queue, questions) = store();
2926 let mut t = task("recovery", dir.path().join("repo"));
2927 t.hold_machine(Some("inspect remote first".into()));
2928 queue.put(&mut t).unwrap();
2929 let mut q = file_question(
2930 &questions,
2931 &crate::task_chat::InquiryOwnership::default(),
2932 &t,
2933 Bucket::MachineUnknown,
2934 &JA,
2935 Timestamp::now(),
2936 )
2937 .unwrap();
2938 if !answer_after_release {
2939 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2940 questions.put(&mut q).unwrap();
2941 }
2942 t.release_operator(source, None);
2943 queue.put(&mut t).unwrap();
2944 if answer_after_release {
2945 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2946 questions.put(&mut q).unwrap();
2947 }
2948 apply_answers(&queue, &questions);
2949 let after = queue.get(&t.id).unwrap();
2950 assert!(after.triage_applied(&q.id));
2951 assert_eq!(after.operator_held(), answer_after_release);
2952 assert_eq!(
2953 after.status,
2954 if answer_after_release {
2955 TaskStatus::Held
2956 } else {
2957 TaskStatus::Queued
2958 }
2959 );
2960 assert!(apply_answers(&queue, &questions).is_empty());
2961 }
2962 }
2963 }
2964
2965 #[test]
2966 fn explicit_resumption_applies_legacy_queued_triage_once_and_preserves_later_holds() {
2967 let (dir, queue, questions) = store();
2968 let mut t = task("recover", dir.path().join("repo"));
2969 t.hold_machine(Some("inspect first".into()));
2970 queue.put(&mut t).unwrap();
2971 let mut q = file_question(
2972 &questions,
2973 &crate::task_chat::InquiryOwnership::default(),
2974 &t,
2975 Bucket::MachineUnknown,
2976 &JA,
2977 Timestamp::now(),
2978 )
2979 .unwrap();
2980 assert!(q.actions.is_empty());
2981 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
2982 questions.put(&mut q).unwrap();
2983 t.release();
2985 t.instruction = "Reuse approved PR head".into();
2986 queue.put(&mut t).unwrap();
2987 assert_eq!(apply_answers(&queue, &questions), [t.id.clone()]);
2988 let mut after = queue.get(&t.id).unwrap();
2989 assert_eq!(
2990 after.resumption.as_ref().unwrap().question_id.as_deref(),
2991 Some(q.id.as_str())
2992 );
2993 assert_eq!(
2994 after.resumption.as_ref().unwrap().instruction,
2995 t.instruction
2996 );
2997 assert_eq!(after.answers[0].answer, JA.resume);
2998 assert!(after.triage_applied(&q.id));
2999 assert!(apply_answers(&Queue::at(dir.path().join("queue")), &questions).is_empty());
3000 after.hold_machine(Some("new blocker".into()));
3001 queue.put(&mut after).unwrap();
3002 let mut late = file_question(
3003 &questions,
3004 &crate::task_chat::InquiryOwnership::default(),
3005 &after,
3006 Bucket::MachineUnknown,
3007 &EN,
3008 Timestamp::now(),
3009 )
3010 .unwrap();
3011 after.hold_manual(Some("keep held now".into()));
3012 queue.put(&mut after).unwrap();
3013 late.answer(Answer::Choice(late.choices[0].clone()))
3014 .unwrap();
3015 questions.put(&mut late).unwrap();
3016 apply_answers(&queue, &questions);
3017 let held = queue.get(&t.id).unwrap();
3018 assert!(held.operator_held());
3019 assert_eq!(held.hold_reason.as_deref(), Some("keep held now"));
3020 assert_eq!(held.resumption.as_ref().unwrap().generation, 1);
3021 }
3022
3023 #[test]
3024 fn a_resolved_machine_hold_is_requeued_automatically() {
3025 let (dir, q, questions) = store();
3026 let config = gate_disabled_config(dir.path());
3027 let mut t = task("disk pressure", dir.path().join("repo"));
3028 t.hold_machine(Some(
3029 "not enough free space to start a run: 10 bytes free, 100 required by \
3030 `[disk] min_free_bytes`"
3031 .to_owned(),
3032 ));
3033 t.schema = 21; q.put(&mut t).unwrap();
3035
3036 let report = run_once(&q, &questions, Some(&config), Timestamp::now());
3037 assert_eq!(report.resumed, [t.id.clone()]);
3038 assert!(report.asked.is_empty());
3039
3040 let back = q.get(&t.id).unwrap();
3041 assert_eq!(back.status, TaskStatus::Queued);
3042 assert!(back.hold_source.is_none());
3043 assert!(questions.list().is_empty(), "nothing needed asking");
3044 }
3045
3046 #[test]
3047 fn a_machine_hold_with_no_recognised_cause_gets_one_question_not_two() {
3048 let (dir, q, questions) = store();
3049 let mut t = task("gate went red", dir.path().join("repo"));
3050 t.hold_machine(Some("gate red".to_owned()));
3051 q.put(&mut t).unwrap();
3052
3053 let first = run_once(&q, &questions, None, Timestamp::now());
3054 assert_eq!(first.asked, [t.id.clone()]);
3055 assert!(first.resumed.is_empty());
3056
3057 let open: Vec<_> = questions
3058 .list()
3059 .into_iter()
3060 .filter(|q| q.status.open())
3061 .collect();
3062 assert_eq!(open.len(), 1);
3063 assert_eq!(open[0].run, t.id);
3064 assert_eq!(open[0].node, NODE);
3065 assert_eq!(open[0].choices.len(), 3);
3066
3067 let second = run_once(&q, &questions, None, Timestamp::now());
3069 assert!(second.asked.is_empty());
3070 assert_eq!(
3071 questions
3072 .list()
3073 .into_iter()
3074 .filter(|q| q.status.open())
3075 .count(),
3076 1
3077 );
3078 }
3079
3080 #[test]
3081 fn a_legacy_hold_with_no_recorded_source_gets_exactly_one_question() {
3082 let (dir, q, questions) = store();
3083 let mut t = task("schema 1 record", dir.path().join("repo"));
3084 t.status = TaskStatus::Held;
3085 assert!(t.hold_source.is_none(), "the case this test is about");
3086 q.put(&mut t).unwrap();
3087
3088 let first = run_once(&q, &questions, None, Timestamp::now());
3089 assert_eq!(first.asked, [t.id.clone()]);
3090
3091 let second = run_once(&q, &questions, None, Timestamp::now());
3092 assert!(
3093 second.asked.is_empty(),
3094 "the same legacy hold must not be asked about twice"
3095 );
3096 assert_eq!(
3097 questions
3098 .list()
3099 .into_iter()
3100 .filter(|q| q.status.open())
3101 .count(),
3102 1
3103 );
3104 }
3105
3106 #[test]
3107 fn dependency_resolution_detaches_legacy_short_refs_without_dropping_other_waits() {
3108 let (dir, queue, questions) = store();
3109 let mut root = task("root", dir.path().join("repo"));
3110 root.id = "20261013-193611-ca5b".into();
3111 root.hold_machine(Some("needs owner decision".into()));
3112 queue.put(&mut root).unwrap();
3113 let mut other = task("other", dir.path().join("repo"));
3114 other.id = "20261013-192213-feb2".into();
3115 queue.put(&mut other).unwrap();
3116 let mut waiting = task("waiting", dir.path().join("repo"));
3117 waiting.block(vec!["ca5b".into(), other.id.clone()], None);
3118 queue.put(&mut waiting).unwrap();
3119 detach_dependants(&queue, &questions, &root.id);
3120 let after = queue.get(&waiting.id).unwrap();
3121 assert_eq!(after.status, TaskStatus::Blocked);
3122 assert_eq!(after.blocked_by, [other.id]);
3123 }
3124
3125 #[test]
3126 fn a_manual_hold_is_never_auto_resumed() {
3127 let (dir, q, questions) = store();
3128 let mut t = task("operator stopped this", dir.path().join("repo"));
3129 t.hold_manual(Some("waiting on a decision".to_owned()));
3130 q.put(&mut t).unwrap();
3131
3132 let report = run_once(&q, &questions, None, Timestamp::now());
3133 assert!(report.resumed.is_empty());
3134 assert!(report.asked.is_empty());
3136
3137 let back = q.get(&t.id).unwrap();
3138 assert_eq!(back.status, TaskStatus::Held);
3139 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3140 assert!(questions.list().is_empty());
3141 }
3142
3143 #[test]
3144 fn a_blocked_task_on_a_deleted_dependency_is_held_and_asked_about_in_one_pass() {
3145 let (dir, q, questions) = store();
3150 let mut still_going = task("still valid", dir.path().join("repo"));
3151 q.put(&mut still_going).unwrap();
3152
3153 let mut t = task("orphaned", dir.path().join("repo"));
3154 t.block(
3155 vec!["20260101-000000-gone".to_owned(), still_going.id.clone()],
3156 Some("waits on both".to_owned()),
3157 );
3158 q.put(&mut t).unwrap();
3159
3160 let report = run_once(&q, &questions, None, Timestamp::now());
3161 assert_eq!(report.quarantined, [t.id.clone()]);
3162 assert_eq!(
3163 report.asked,
3164 [t.id.clone()],
3165 "the fresh machine hold must earn a question in the same pass"
3166 );
3167
3168 let after = q.get(&t.id).unwrap();
3169 assert_eq!(after.status, TaskStatus::Held);
3170 assert_eq!(after.hold_source, Some(HoldSource::Machine));
3171 assert!(after.blocked_by.is_empty());
3172
3173 assert!(!is_disk_hold(&after));
3176
3177 let open: Vec<_> = questions
3178 .list()
3179 .into_iter()
3180 .filter(|q| q.status.open())
3181 .collect();
3182 assert_eq!(open.len(), 1);
3183 assert_eq!(open[0].run, t.id);
3184
3185 let second = run_once(&q, &questions, None, Timestamp::now());
3187 assert!(second.quarantined.is_empty());
3188 assert!(second.asked.is_empty());
3189 }
3190
3191 fn stuck_chain(q: &Queue, dir: &std::path::Path) -> (Task, Task, Task) {
3194 let mut root = task("root", dir.join("repo"));
3195 root.hold_manual(Some("waiting".to_owned()));
3196 q.put(&mut root).unwrap();
3197 let mut mid = task("mid", dir.join("repo"));
3198 mid.block(vec![root.id.clone()], None);
3199 q.put(&mut mid).unwrap();
3200 let mut leaf = task("leaf", dir.join("repo"));
3201 leaf.block(vec![mid.id.clone()], None);
3202 q.put(&mut leaf).unwrap();
3203 (root, mid, leaf)
3204 }
3205
3206 fn deps_questions(questions: &Questions) -> Vec<Question> {
3207 questions
3208 .list()
3209 .into_iter()
3210 .filter(|q| q.node == DEPS_NODE)
3211 .collect()
3212 }
3213
3214 fn answer_deps(questions: &Questions, choice: usize) {
3215 let mut asked = deps_questions(questions).remove(0);
3216 let c = asked.choices[choice].clone();
3217 asked.answer(Answer::Choice(c)).unwrap();
3218 questions.put(&mut asked).unwrap();
3219 }
3220
3221 #[test]
3222 fn a_stuck_chain_earns_one_question_naming_every_dependant_and_is_not_reasked() {
3223 let (dir, q, questions) = store();
3224 let (root, mid, leaf) = stuck_chain(&q, dir.path());
3225
3226 let first = run_once(&q, &questions, None, Timestamp::now());
3227 assert_eq!(first.asked, std::slice::from_ref(&root.id));
3228 let asked = deps_questions(&questions);
3229 assert_eq!(asked.len(), 1, "one question per root, not per dependant");
3230 assert_eq!(asked[0].run, root.id);
3231 assert_eq!(asked[0].choices.len(), 3);
3232 assert!(asked[0].detail.contains(mid.short()), "{}", asked[0].detail);
3233 assert!(
3234 asked[0].detail.contains(leaf.short()),
3235 "{}",
3236 asked[0].detail
3237 );
3238
3239 let second = run_once(&q, &questions, None, Timestamp::now());
3240 assert!(second.asked.is_empty());
3241 assert_eq!(deps_questions(&questions).len(), 1);
3242 assert!(pending_for(&questions, &q.get(&root.id).unwrap()));
3243 }
3244
3245 #[test]
3246 fn a_stuck_root_with_a_machine_hold_gets_only_the_dependency_question() {
3247 let (dir, q, questions) = store();
3248 let mut root = task("root", dir.path().join("repo"));
3249 root.hold_machine(Some("gate red".to_owned()));
3250 q.put(&mut root).unwrap();
3251 let mut dep = task("dep", dir.path().join("repo"));
3252 dep.block(vec![root.id.clone()], None);
3253 q.put(&mut dep).unwrap();
3254
3255 run_once(&q, &questions, None, Timestamp::now());
3256 run_once(&q, &questions, None, Timestamp::now());
3257 assert_eq!(deps_questions(&questions).len(), 1);
3258 assert_eq!(
3259 questions.list().len(),
3260 1,
3261 "no second question about the hold"
3262 );
3263 }
3264
3265 #[test]
3266 fn a_dependency_that_can_still_run_asks_nothing() {
3267 let (dir, q, questions) = store();
3268 let mut live = task("live", dir.path().join("repo"));
3269 q.put(&mut live).unwrap();
3270 let mut dep = task("dep", dir.path().join("repo"));
3271 dep.block(vec![live.id.clone()], None);
3272 q.put(&mut dep).unwrap();
3273 let report = run_once(&q, &questions, None, Timestamp::now());
3274 assert!(report.asked.is_empty());
3275 assert!(questions.list().is_empty());
3276 }
3277
3278 #[test]
3279 fn answering_release_requeues_the_root_and_is_not_reasked() {
3280 let (dir, q, questions) = store();
3281 let (root, _mid, _leaf) = stuck_chain(&q, dir.path());
3282 run_once(&q, &questions, None, Timestamp::now());
3283 answer_deps(&questions, 0);
3284
3285 let report = run_once(&q, &questions, None, Timestamp::now());
3286 assert_eq!(report.answered, std::slice::from_ref(&root.id));
3287 assert!(report.asked.is_empty(), "applying must not re-ask");
3288 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Queued);
3289 let again = run_once(&q, &questions, None, Timestamp::now());
3290 assert!(again.asked.is_empty() && again.answered.is_empty());
3291 assert_eq!(deps_questions(&questions).len(), 1);
3292 }
3293
3294 #[test]
3295 fn answering_detach_frees_the_direct_dependant_and_keeps_the_root() {
3296 let (dir, q, questions) = store();
3297 let (root, mid, leaf) = stuck_chain(&q, dir.path());
3298 run_once(&q, &questions, None, Timestamp::now());
3299 answer_deps(&questions, 2);
3300
3301 let report = run_once(&q, &questions, None, Timestamp::now());
3302 assert!(report.asked.is_empty());
3303 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
3304 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
3305 let leaf = q.get(&leaf.id).unwrap();
3306 assert_eq!(leaf.status, TaskStatus::Blocked, "still waits on mid");
3307 assert_eq!(leaf.blocked_by, std::slice::from_ref(&mid.id));
3308 }
3309
3310 #[test]
3311 fn a_dependant_that_can_progress_through_another_blocker_is_not_stuck() {
3312 let (dir, q, questions) = store();
3313 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3314 let mut live = task("live", dir.path().join("repo"));
3315 q.put(&mut live).unwrap();
3316 let mut both = q.get(&mid.id).unwrap();
3317 both.block(vec![root.id.clone(), live.id.clone()], None);
3318 q.put(&mut both).unwrap();
3319 assert!(
3321 run_once(&q, &questions, None, Timestamp::now())
3322 .asked
3323 .is_empty()
3324 );
3325 }
3326
3327 #[test]
3328 fn answering_discard_detaches_then_removes_the_root() {
3329 let (dir, q, questions) = store();
3330 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3331 run_once(&q, &questions, None, Timestamp::now());
3332 answer_deps(&questions, 1);
3333
3334 let report = run_once(&q, &questions, None, Timestamp::now());
3335 assert!(q.get(&root.id).is_err());
3336 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
3337 assert!(
3338 report.asked.is_empty(),
3339 "no per-dependant quarantine question"
3340 );
3341 assert!(report.quarantined.is_empty());
3342 }
3343
3344 #[test]
3345 fn an_unmatched_deps_answer_changes_nothing_and_is_not_reasked() {
3346 let (dir, q, questions) = store();
3347 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3348 run_once(&q, &questions, None, Timestamp::now());
3349 let mut asked = deps_questions(&questions).remove(0);
3350 asked.choices.clear();
3351 asked.answer(Answer::Text("dunno".to_owned())).unwrap();
3352 questions.put(&mut asked).unwrap();
3353
3354 for _ in 0..2 {
3355 let report = run_once(&q, &questions, None, Timestamp::now());
3356 assert!(report.asked.is_empty() && report.answered.is_empty());
3357 }
3358 assert_eq!(deps_questions(&questions).len(), 1);
3359 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
3360 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Blocked);
3361 }
3362
3363 #[test]
3364 fn a_dependency_cycle_terminates_and_is_asked_about_once() {
3365 let (dir, q, questions) = store();
3366 let mut a = task("a", dir.path().join("repo"));
3367 let mut b = task("b", dir.path().join("repo"));
3368 a.block(vec![b.id.clone()], None);
3369 b.block(vec![a.id.clone()], None);
3370 q.put(&mut a).unwrap();
3371 q.put(&mut b).unwrap();
3372
3373 let first = run_once(&q, &questions, None, Timestamp::now());
3374 assert_eq!(first.asked.len(), 1);
3375 let second = run_once(&q, &questions, None, Timestamp::now());
3376 assert!(second.asked.is_empty());
3377 assert_eq!(deps_questions(&questions).len(), 1);
3378 }
3379
3380 #[test]
3381 fn a_missing_dependency_is_still_quarantined_not_asked_about_as_stuck() {
3382 let (dir, q, questions) = store();
3383 let mut t = task("orphan", dir.path().join("repo"));
3384 t.block(vec!["20260101-000000-gone".to_owned()], None);
3385 q.put(&mut t).unwrap();
3386 let report = run_once(&q, &questions, None, Timestamp::now());
3387 assert_eq!(report.quarantined, [t.id.clone()]);
3388 assert!(deps_questions(&questions).is_empty());
3389 }
3390
3391 #[test]
3392 fn a_stale_manual_hold_earns_a_two_choice_question() {
3393 let (dir, q, questions) = store();
3394 let mut t = task("been sitting a while", dir.path().join("repo"));
3395 t.hold_manual(Some("waiting on a decision".to_owned()));
3396 q.put(&mut t).unwrap();
3397 let mut back = q.get(&t.id).unwrap();
3399 back.updated_at = Timestamp::now() - SignedDuration::new(8 * 24 * 60 * 60, 0);
3400 std::fs::write(
3401 q.path_of(&back.id),
3402 serde_json::to_string_pretty(&back).unwrap(),
3403 )
3404 .unwrap();
3405
3406 let report = run_once(&q, &questions, None, Timestamp::now());
3407 assert_eq!(report.asked, [t.id.clone()]);
3408 let open: Vec<_> = questions
3409 .list()
3410 .into_iter()
3411 .filter(|q| q.status.open())
3412 .collect();
3413 assert_eq!(open.len(), 1);
3414 assert_eq!(open[0].choices.len(), 2);
3415 }
3416
3417 #[test]
3418 fn answering_resume_releases_the_task() {
3419 let (dir, q, questions) = store();
3420 let mut t = task("gate went red", dir.path().join("repo"));
3421 t.hold_machine(Some("gate red".to_owned()));
3422 q.put(&mut t).unwrap();
3423 run_once(&q, &questions, None, Timestamp::now());
3424
3425 let mut asked = questions
3426 .list()
3427 .into_iter()
3428 .find(|q| q.run == t.id)
3429 .unwrap();
3430 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3431 questions.put(&mut asked).unwrap();
3432
3433 let report = run_once(&q, &questions, None, Timestamp::now());
3434 assert_eq!(report.answered, [t.id.clone()]);
3435 let back = q.get(&t.id).unwrap();
3436 assert_eq!(back.status, TaskStatus::Queued);
3437 assert!(back.hold_source.is_none());
3438 }
3439
3440 fn resumed_then_failed(q: &Queue, questions: &Questions, dir: &std::path::Path) -> Task {
3442 let mut t = task("gate went red", dir.join("repo"));
3443 t.hold_machine(Some("gate red".to_owned()));
3444 q.put(&mut t).unwrap();
3445 run_once(q, questions, None, Timestamp::now());
3446 let mut asked = questions
3447 .list()
3448 .into_iter()
3449 .find(|q| q.run == t.id)
3450 .unwrap();
3451 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3452 questions.put(&mut asked).unwrap();
3453
3454 let report = run_once(q, questions, None, Timestamp::now());
3455 assert_eq!(report.answered, [t.id.clone()]);
3456 let mut back = q.get(&t.id).unwrap();
3457 assert_eq!(back.status, TaskStatus::Queued);
3458 assert_eq!(back.attempts, 0);
3459
3460 back.start("run-1".to_owned());
3461 back.fail("rebase conflict", 1);
3462 assert_eq!(back.status, TaskStatus::Held);
3463 q.put(&mut back).unwrap();
3464 back
3465 }
3466
3467 #[test]
3468 fn a_resume_answer_is_applied_once_and_a_new_machine_hold_is_asked_about() {
3469 let (dir, q, questions) = store();
3470 let t = resumed_then_failed(&q, &questions, dir.path());
3471
3472 let report = run_once(&q, &questions, None, Timestamp::now());
3473 assert!(report.answered.is_empty(), "the old answer must not replay");
3474 assert_eq!(report.asked, std::slice::from_ref(&t.id));
3475 let back = q.get(&t.id).unwrap();
3476 assert_eq!(back.status, TaskStatus::Held);
3477 assert_eq!(
3478 questions
3479 .list()
3480 .into_iter()
3481 .filter(|q| q.status.open())
3482 .count(),
3483 1
3484 );
3485 }
3486
3487 #[test]
3488 fn a_manual_hold_placed_after_a_resume_is_not_undone_by_the_old_answer() {
3489 let (dir, q, questions) = store();
3490 let mut t = resumed_then_failed(&q, &questions, dir.path());
3491 t.hold_manual(Some("operator stopped this".to_owned()));
3492 q.put(&mut t).unwrap();
3493 let before = questions.list().len();
3494
3495 let report = run_once(&q, &questions, None, Timestamp::now());
3496 assert!(report.answered.is_empty());
3497 assert!(report.asked.is_empty());
3498 let back = q.get(&t.id).unwrap();
3499 assert_eq!(back.status, TaskStatus::Held);
3500 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3501 assert_eq!(questions.list().len(), before);
3502 }
3503
3504 #[test]
3505 fn answering_not_yet_keeps_it_held_as_a_manual_hold_and_does_not_reapply() {
3506 let (dir, q, questions) = store();
3507 let mut t = task("gate went red", dir.path().join("repo"));
3508 t.hold_machine(Some("gate red".to_owned()));
3509 q.put(&mut t).unwrap();
3510 run_once(&q, &questions, None, Timestamp::now());
3511
3512 let mut asked = questions
3513 .list()
3514 .into_iter()
3515 .find(|q| q.run == t.id)
3516 .unwrap();
3517 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
3518 questions.put(&mut asked).unwrap();
3519
3520 let report = run_once(&q, &questions, None, Timestamp::now());
3521 assert_eq!(report.answered, [t.id.clone()]);
3522 let back = q.get(&t.id).unwrap();
3523 assert_eq!(back.status, TaskStatus::Held);
3524 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3525 assert!(
3526 back.hold_reason
3527 .as_deref()
3528 .is_some_and(|r| r.contains("gate red")),
3529 "the original cause must survive a \"not yet\" answer, not just the \
3530 triage marker: {:?}",
3531 back.hold_reason
3532 );
3533
3534 let third = run_once(&q, &questions, None, Timestamp::now());
3538 assert!(third.answered.is_empty());
3539 assert!(third.asked.is_empty());
3540 }
3541
3542 #[test]
3543 fn answering_discard_removes_the_task_entirely() {
3544 let (dir, q, questions) = store();
3545 let mut t = task("gate went red", dir.path().join("repo"));
3546 t.hold_machine(Some("gate red".to_owned()));
3547 q.put(&mut t).unwrap();
3548 run_once(&q, &questions, None, Timestamp::now());
3549
3550 let mut asked = questions
3551 .list()
3552 .into_iter()
3553 .find(|q| q.run == t.id)
3554 .unwrap();
3555 asked.answer(Answer::Choice(EN.discard.to_owned())).unwrap();
3556 questions.put(&mut asked).unwrap();
3557
3558 let report = run_once(&q, &questions, None, Timestamp::now());
3559 assert_eq!(report.answered, [t.id.clone()]);
3560 assert!(
3561 q.get(&t.id).is_err(),
3562 "\"discard it\" (捨ててよい) must actually discard the task, not \
3563 just leave it sitting held forever"
3564 );
3565 }
3566
3567 #[test]
3568 fn an_answer_is_read_by_its_position_in_choices_not_by_the_callers_current_language() {
3569 let (dir, q, questions) = store();
3571 let ja_config = dir.path().join("ja.toml");
3572 std::fs::write(&ja_config, "[graph]\nlanguage = \"ja\"\n").unwrap();
3573 let mut t = task("gate went red", dir.path().join("repo"));
3574 t.hold_machine(Some("gate red".to_owned()));
3575 q.put(&mut t).unwrap();
3576 run_once(&q, &questions, Some(&ja_config), Timestamp::now());
3577
3578 let mut asked = questions
3579 .list()
3580 .into_iter()
3581 .find(|q| q.run == t.id)
3582 .unwrap();
3583 assert_eq!(asked.choices[0], JA.resume, "filed in Japanese");
3584 asked.answer(Answer::Choice(JA.resume.to_owned())).unwrap();
3585 questions.put(&mut asked).unwrap();
3586
3587 let en_config = dir.path().join("en.toml");
3592 std::fs::write(&en_config, "[graph]\nlanguage = \"en\"\n").unwrap();
3593 let report = run_once(&q, &questions, Some(&en_config), Timestamp::now());
3594 assert_eq!(report.answered, [t.id.clone()]);
3595 let back = q.get(&t.id).unwrap();
3596 assert_eq!(
3597 back.status,
3598 TaskStatus::Queued,
3599 "a resume answer must resume the task regardless of which \
3600 language it is read back in"
3601 );
3602 }
3603
3604 #[test]
3605 fn a_question_falls_back_to_last_error_when_hold_reason_was_never_set() {
3606 let (dir, q, questions) = store();
3611 let mut t = task("kept failing the gate", dir.path().join("repo"));
3612 t.start("run-1".to_owned());
3613 t.fail("gate red three times running", 1);
3614 assert_eq!(t.status, TaskStatus::Held);
3615 t.hold_reason = None; q.put(&mut t).unwrap();
3617
3618 run_once(&q, &questions, None, Timestamp::now());
3619 let asked = questions
3620 .list()
3621 .into_iter()
3622 .find(|q| q.run == t.id)
3623 .unwrap();
3624 assert!(
3625 asked.detail.contains("gate red three times running"),
3626 "the question must surface `last_error` when there is no \
3627 `hold_reason` to show instead: {}",
3628 asked.detail
3629 );
3630 }
3631
3632 #[test]
3633 fn open_question_for_and_open_task_ids_reflect_only_what_is_still_waiting() {
3634 let (dir, q, questions) = store();
3635 let mut t = task("gate went red", dir.path().join("repo"));
3636 t.hold_machine(Some("gate red".to_owned()));
3637 q.put(&mut t).unwrap();
3638
3639 assert!(open_question_for(&questions, &t.id).is_none());
3640 assert!(!open_task_ids(&questions).contains(&t.id));
3641
3642 run_once(&q, &questions, None, Timestamp::now());
3643 assert!(open_question_for(&questions, &t.id).is_some());
3644 assert!(open_task_ids(&questions).contains(&t.id));
3645
3646 let mut asked = questions
3647 .list()
3648 .into_iter()
3649 .find(|q| q.run == t.id)
3650 .unwrap();
3651 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3652 questions.put(&mut asked).unwrap();
3653
3654 assert!(
3655 open_question_for(&questions, &t.id).is_none(),
3656 "an answered question is no longer open"
3657 );
3658 assert!(!open_task_ids(&questions).contains(&t.id));
3659 }
3660
3661 #[test]
3662 fn pending_for_stays_true_between_an_answer_and_the_next_run_once_pass() {
3663 let (dir, q, questions) = store();
3669 let mut t = task("gate went red", dir.path().join("repo"));
3670 t.hold_machine(Some("gate red".to_owned()));
3671 q.put(&mut t).unwrap();
3672
3673 assert!(!pending_for(&questions, &t));
3674
3675 run_once(&q, &questions, None, Timestamp::now());
3676 let held = q.get(&t.id).unwrap();
3677 assert!(pending_for(&questions, &held), "still waiting on an answer");
3678
3679 let mut asked = questions
3680 .list()
3681 .into_iter()
3682 .find(|q| q.run == t.id)
3683 .unwrap();
3684 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
3685 questions.put(&mut asked).unwrap();
3686 assert!(!asked.status.open());
3687
3688 let still_held = q.get(&t.id).unwrap();
3690 assert!(
3691 pending_for(&questions, &still_held),
3692 "answered but not yet applied is still pending"
3693 );
3694
3695 run_once(&q, &questions, None, Timestamp::now());
3696 let after = q.get(&t.id).unwrap();
3697 assert!(
3698 !pending_for(&questions, &after),
3699 "the answer is applied now, nothing left pending"
3700 );
3701 }
3702}