1use std::path::{Path, PathBuf};
86use std::time::Duration;
87
88use jiff::Timestamp;
89
90use crate::ask::{Question, QuestionStatus, Questions};
91use crate::config::Config;
92use crate::queue::{HoldSource, OperatorResume, Queue, Task, TaskStatus};
93
94pub const NODE: &str = "triage";
97
98pub const DEPS_NODE: &str = "triage-deps";
102
103const SEAT: &str = "triage";
108
109const MANUAL_STALE_AFTER: Duration = Duration::from_secs(7 * 24 * 60 * 60);
121
122struct Wording {
126 lang: &'static str,
127 resume: &'static str,
128 wait: &'static str,
129 discard: &'static str,
130}
131
132const EN: Wording = Wording {
133 lang: "en",
134 resume: "resume it",
135 wait: "not yet",
136 discard: "discard it",
137};
138
139const JA: Wording = Wording {
140 lang: "ja",
141 resume: "再開してよい",
142 wait: "まだ待って",
143 discard: "捨ててよい",
144};
145
146fn wording(language: &str) -> &'static Wording {
149 if crate::lang::is_japanese(language) {
150 &JA
151 } else {
152 &EN
153 }
154}
155
156impl Wording {
157 fn choices3(&self) -> Vec<String> {
158 vec![
159 self.resume.to_owned(),
160 self.wait.to_owned(),
161 self.discard.to_owned(),
162 ]
163 }
164
165 fn choices_manual(&self) -> Vec<String> {
166 let (resume, wait) = if self.lang == "ja" {
167 ("再開する", "まだ止めておく")
168 } else {
169 ("resume it", "keep it held")
170 };
171 vec![resume.to_owned(), wait.to_owned(), self.discard.to_owned()]
172 }
173
174 fn source_label(&self, source: Option<HoldSource>) -> &'static str {
175 match (self.lang, source) {
176 ("ja", Some(HoldSource::Machine)) => "machine(機械による自動保留)",
177 ("ja", Some(HoldSource::Manual)) => "manual(操作者による手動保留)",
178 ("ja", None) => "unknown(schema 3 未満の旧レコード、または理由未記録)",
179 (_, Some(HoldSource::Machine)) => "machine (automatic recovery hold)",
180 (_, Some(HoldSource::Manual)) => "manual (an operator held this)",
181 (_, None) => "unknown (pre-schema-3 record, or never recorded)",
182 }
183 }
184
185 fn detail(&self, task: &Task, why: &str) -> String {
194 let none = if self.lang == "ja" {
195 "(記録なし)"
196 } else {
197 "(none recorded)"
198 };
199 let reason = task
200 .hold_reason
201 .as_deref()
202 .or(task.last_error.as_deref())
203 .unwrap_or(none);
204 format!(
205 "task: {} ({})\ntitle: {}\nhold source: {}\nhold reason: {reason}\n\n{why}",
206 task.id,
207 task.short(),
208 task.title,
209 self.source_label(task.hold_source),
210 )
211 }
212
213 fn summary_machine_unknown(&self, task: &Task) -> String {
214 if self.lang == "ja" {
215 format!("保留タスク {} の再開可否を判断してください", task.short())
216 } else {
217 format!("decide whether to resume held task {}", task.short())
218 }
219 }
220
221 fn why_machine(&self) -> &'static str {
222 if self.lang == "ja" {
223 "機械的な保留(machine hold)ですが、原因がすでに解消しているかを自動では判断できませんでした。"
224 } else {
225 "This is a machine hold, but whether its cause has resolved could not be \
226 checked automatically."
227 }
228 }
229
230 fn summary_conductor_override(&self, task: &Task) -> String {
231 if self.lang == "ja" {
232 format!(
233 "再開と回答済みのタスク {} を conductor が再び保留しました",
234 task.short()
235 )
236 } else {
237 format!(
238 "task {} was resumed at your word, but the conductor held it again",
239 task.short()
240 )
241 }
242 }
243
244 fn why_conductor_override(&self, o: &OperatorResume) -> String {
245 let reason = o.conductor_rehold.as_deref().unwrap_or_default();
246 if self.lang == "ja" {
247 format!(
248 "{} に再開と回答済みですが、conductor が再び hold しました。conductor の理由: \
249 {reason}\n\n強制再キューを選ぶと、以後 conductor はこのタスクを hold できません。",
250 o.at
251 )
252 } else {
253 format!(
254 "You answered \"resume\" at {}, but the conductor held the task again. \
255 Its reason: {reason}\n\nForcing a requeue stops the conductor from \
256 holding this task again.",
257 o.at
258 )
259 }
260 }
261
262 fn choices_conductor_override(&self) -> Vec<String> {
264 if self.lang == "ja" {
265 vec![
266 "強制再キュー(conductor は再 hold 不可)".to_owned(),
267 "手動 hold のまま".to_owned(),
268 "捨ててよい".to_owned(),
269 ]
270 } else {
271 vec![
272 "force requeue (conductor must not hold again)".to_owned(),
273 "keep held (manual)".to_owned(),
274 "discard".to_owned(),
275 ]
276 }
277 }
278
279 fn summary_legacy(&self, task: &Task) -> String {
280 if self.lang == "ja" {
281 format!(
282 "hold_source が不明な保留タスク {} を確認してください",
283 task.short()
284 )
285 } else {
286 format!(
287 "held task {} has no recorded hold source - please take a look",
288 task.short()
289 )
290 }
291 }
292
293 fn why_legacy(&self) -> &'static str {
294 if self.lang == "ja" {
295 "hold_source が記録されていません。schema 3 より前のレコードか、理由が記録されなかった \
296 holdです。人が意図して止めたのか、クラッシュや強制再起動で宙に浮いただけなのか、\
297 このデータからは区別できません。"
298 } else {
299 "No hold_source was recorded - either a pre-schema-3 record, or a hold whose \
300 reason was never written down. Whether this was a deliberate hold or the \
301 leftover of a crash cannot be told from the data alone."
302 }
303 }
304
305 fn summary_manual_stale(&self, task: &Task, days: i64) -> String {
306 if self.lang == "ja" {
307 format!(
308 "{days}日間 保留されたままの手動保留タスク {} を確認してください",
309 task.short()
310 )
311 } else {
312 format!(
313 "held task {} has been on a manual hold for {days} day(s)",
314 task.short()
315 )
316 }
317 }
318
319 fn why_manual(&self) -> &'static str {
320 if self.lang == "ja" {
321 "操作者が明示的に止めた保留ですが、長期間そのままになっています。まだ止めておくか、\
322 再開するか教えてください。"
323 } else {
324 "An operator held this on purpose, but it has sat untouched for a while. Say \
325 whether to keep holding it or resume it."
326 }
327 }
328}
329
330#[derive(Debug, Clone, Copy, PartialEq, Eq)]
332enum Bucket {
333 MachineUnknown,
335 ConductorOverride,
338 Legacy,
340 ManualStale,
342}
343
344#[derive(Debug, Clone, Default)]
346pub struct Report {
347 pub resumed: Vec<String>,
350 pub asked: Vec<String>,
352 pub answered: Vec<String>,
354 pub quarantined: Vec<String>,
359 pub superseded: Vec<String>,
362 pub needs_owner: Vec<String>,
365}
366
367impl Report {
368 pub fn is_empty(&self) -> bool {
371 self.resumed.is_empty()
372 && self.asked.is_empty()
373 && self.answered.is_empty()
374 && self.quarantined.is_empty()
375 && self.superseded.is_empty()
376 && self.needs_owner.is_empty()
377 }
378}
379
380fn repo_for(task: &Task) -> PathBuf {
384 if task.repo.as_os_str().is_empty() {
385 PathBuf::from(".")
386 } else {
387 task.repo.clone()
388 }
389}
390
391fn is_disk_hold(task: &Task) -> bool {
393 task.has_disk_capacity_cause()
394 || (task.schema < 22
395 && task
396 .hold_reason
397 .as_deref()
398 .is_some_and(crate::disk::is_gate_reason))
399}
400
401pub(crate) fn disk_wait_eligible(task: &Task) -> bool {
402 disk_wait_known(task, &crate::run::home())
403}
404
405fn disk_wait_known(task: &Task, home: &Path) -> bool {
407 if task.status != TaskStatus::Held
408 || task.hold_source != Some(HoldSource::Machine)
409 || task
410 .resume_override
411 .as_ref()
412 .is_some_and(|o| o.conductor_rehold.is_some())
413 {
414 return false;
415 }
416 if task.has_disk_capacity_cause() {
417 return true;
418 }
419 if task.resume_override.is_some() {
421 return false;
422 }
423 if is_disk_hold(task) {
424 return true;
425 }
426 if task.schema >= 22 {
427 return false;
428 }
429 if !task.hold_reason.as_deref().is_some_and(|r| {
430 r.starts_with(crate::disk::RUNNING_PRESSURE) || r.starts_with(crate::cache::RESOURCE_HOLD)
431 }) {
432 return false;
433 }
434 let Some(id) = task.execution_runs().last() else {
435 return false;
436 };
437 let Ok(state) = crate::run::RunState::load_under(id, home) else {
438 return false;
439 };
440 state.id == *id
441 && state.resource_block.as_ref().is_some_and(|b| {
442 let expected = match b.kind.as_str() {
443 "disk pressure" => format!("{}{}", crate::disk::RUNNING_PRESSURE, b.detail),
444 "disk admission" => format!(
445 "{}{}; no attempt was spent; resume to retry the affected operation",
446 crate::cache::RESOURCE_HOLD,
447 b.description()
448 ),
449 _ => return false,
450 };
451 matches!(b.cause, crate::run::ResourceBlockCause::Other)
452 && task.hold_reason.as_deref() == Some(expected.as_str())
453 })
454}
455
456fn machine_cause_resolved(task: &Task, cfg: &Config) -> Option<bool> {
461 machine_cause_resolved_under(task, cfg, &crate::run::home())
462}
463
464fn machine_cause_resolved_under(task: &Task, cfg: &Config, home: &Path) -> Option<bool> {
465 machine_cause_resolved_with_admission(task, cfg, home, |paths, floor| {
466 let admit = if paths.contains(&crate::build_target::root(home)) {
467 crate::retention::OutputLease::admit_isolated
468 } else {
469 crate::retention::OutputLease::admit
470 };
471 admit(home, paths, floor).is_ok()
472 })
473}
474
475fn cache_wait_run(task: &Task, home: &Path) -> Option<crate::run::RunState> {
476 if !task
477 .hold_reason
478 .as_deref()?
479 .starts_with(crate::cache::RESOURCE_HOLD)
480 {
481 return None;
482 }
483 let id = task.execution_runs().last()?;
484 let state = crate::run::RunState::load_under(id, home).ok()?;
485 (state.id == *id).then_some(state)
486}
487
488fn known_isolated_cache_wait(task: &Task, home: &Path) -> bool {
489 let Some(crate::run::ResourceBlockCause::IsolatedCacheTimeout { target, .. }) =
490 cache_wait_run(task, home)
491 .and_then(|state| state.resource_block)
492 .map(|block| block.cause)
493 else {
494 return false;
495 };
496 crate::build_target::check_prune(home, &target).is_ok_and(|owned| owned)
497}
498
499fn machine_cause_resolved_with_admission(
500 task: &Task,
501 cfg: &Config,
502 home: &Path,
503 admission_available: impl Fn(&[PathBuf], u64) -> bool,
504) -> Option<bool> {
505 if disk_wait_known(task, home) {
506 let saved = if let Some(id) = task.disk_wait_run() {
507 let Ok(state) = crate::run::RunState::load_under(id, home) else {
508 return Some(false);
509 };
510 if state.id != *id || !state.status.resumable() {
511 return Some(false);
512 }
513 Some(state)
514 } else {
515 None
516 };
517 let override_config = task
518 .build_cache_override(cfg)
519 .map(|(mode, _)| mode)
520 .filter(|mode| saved.is_none() && *mode != cfg.verify.build_cache)
521 .map(|mode| {
522 let mut config = cfg.clone();
523 config.verify.build_cache = mode;
524 config
525 });
526 let (repo, config) = saved.as_ref().map_or_else(
527 || (repo_for(task), override_config.as_ref().unwrap_or(cfg)),
528 |state| (state.repo.clone(), &state.config),
529 );
530 if crate::daemon::disk_gate_at(&repo, config, || home.to_owned()).is_some() {
531 return Some(false);
532 }
533 let mut paths = vec![repo];
534 if let Some(state) = &saved {
535 paths.push(state.worktree_root());
536 paths.extend(state.candidates.iter().map(|c| c.worktree.clone()));
537 } else {
538 paths.push(
539 config
540 .graph
541 .worktree_root
542 .clone()
543 .unwrap_or_else(crate::run::default_worktree_root),
544 );
545 }
546 if crate::build_target::isolated(config) {
547 paths.push(crate::build_target::root(home));
548 } else {
549 paths.extend(config.cache_dir());
550 }
551 let mut floor = config.disk.min_free_bytes;
552 if saved.is_some()
553 && !crate::build_target::isolated(config)
554 && !crate::build_target::isolated(cfg)
555 {
556 let repo = repo_for(task);
557 if crate::daemon::disk_gate_at(&repo, cfg, || home.to_owned()).is_some() {
558 return Some(false);
559 }
560 paths.push(repo);
561 paths.extend(cfg.cache_dir());
562 paths.push(
563 cfg.graph
564 .worktree_root
565 .clone()
566 .unwrap_or_else(crate::run::default_worktree_root),
567 );
568 floor = floor.max(cfg.disk.min_free_bytes);
569 }
570 return Some(admission_available(&paths, floor));
571 }
572 if let Some(state) = cache_wait_run(task, home) {
573 let block = state.resource_block.as_ref()?;
574 if let crate::run::ResourceBlockCause::IsolatedCacheTimeout { target, .. } = &block.cause {
575 return Some(
576 crate::build_target::check_prune(home, target).is_ok_and(|owned| owned)
577 && !crate::cache::in_use(home, target),
578 );
579 }
580 if block.kind != "shared build cache"
583 || !matches!(block.cause, crate::run::ResourceBlockCause::Timeout { .. })
584 {
585 return None;
586 }
587 return Some(
588 state
589 .config
590 .cache_dir()
591 .is_none_or(|dir| !crate::cache::in_use(home, &dir)),
592 );
593 }
594 None
595}
596
597fn manual_is_stale(task: &Task, now: Timestamp) -> bool {
601 now.as_second() - task.updated_at.as_second() > MANUAL_STALE_AFTER.as_secs() as i64
602}
603
604fn marker_for(q: &Question) -> String {
607 format!("[triage:{}]", q.short())
608}
609
610fn already_applied(task: &Task, q: &Question) -> bool {
613 if task.triage_applied(&q.id) {
614 return true;
615 }
616 let marker = marker_for(q);
619 task.hold_reason
620 .as_deref()
621 .is_some_and(|r| r.contains(marker.as_str()))
622}
623
624fn consumed(task: &Task, q: &Question) -> bool {
626 if q.node == NODE {
627 already_applied(task, q)
628 } else {
629 task.triage_applied(&q.id)
630 }
631}
632
633pub(crate) struct RecoveryIndex {
636 latest: std::collections::HashMap<(String, String), Question>,
637}
638
639impl RecoveryIndex {
640 pub(crate) fn load(questions: &Questions) -> Self {
641 let queue = Queue::at(
642 questions
643 .root()
644 .parent()
645 .unwrap_or(Path::new(""))
646 .join("queue"),
647 );
648 let tasks = queue.list();
649 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
650 Self::from_scan(&queue, &tasks, questions.list_for_effects_in(&ownership))
651 }
652
653 fn from_scan(queue: &Queue, tasks: &[Task], all: Vec<Question>) -> Self {
654 let ids = crate::queue::record_ids(queue.root(), "json").unwrap_or_default();
655 let by_id: std::collections::HashMap<_, _> =
656 tasks.iter().map(|task| (task.id.as_str(), task)).collect();
657 let exact: std::collections::HashSet<_> = ids.iter().map(String::as_str).collect();
658 let mut aliases: std::collections::HashMap<&str, Option<&str>> = Default::default();
659 for id in &ids {
660 for boundary in id
661 .char_indices()
662 .map(|(i, _)| i)
663 .chain(std::iter::once(id.len()))
664 {
665 for alias in [&id[..boundary], &id[boundary..]] {
666 let entry = aliases.entry(alias).or_insert(Some(id));
667 if *entry != Some(id.as_str()) {
668 *entry = None;
669 }
670 }
671 }
672 }
673 let mut index = Self {
674 latest: Default::default(),
675 };
676 for q in all {
677 if q.node != NODE && q.node != DEPS_NODE {
678 continue;
679 }
680 let resolved = if exact.contains(q.run.as_str()) {
681 Some(q.run.as_str())
682 } else {
683 aliases.get(q.run.as_str()).copied().flatten()
684 };
685 if let Some(id) = resolved
686 && by_id.get(id).is_some_and(|task| {
687 crate::repair_budget::reconcile::superseded_question(task, &q)
688 })
689 {
690 continue;
691 }
692 let key = (q.node.clone(), q.run.clone());
693 let entry = index.latest.entry(key).or_insert_with(|| q.clone());
694 if (q.asked_at, &q.id) > (entry.asked_at, &entry.id) {
695 *entry = q;
696 }
697 }
698 index
699 }
700
701 fn latest(&self, node: &str, task_id: &str) -> Option<Question> {
702 self.latest
703 .get(&(node.to_owned(), task_id.to_owned()))
704 .cloned()
705 }
706
707 pub(crate) fn pending(&self, task: &Task) -> Vec<Question> {
708 [NODE, DEPS_NODE]
709 .into_iter()
710 .filter_map(|node| {
711 self.latest(node, &task.id).filter(|q| {
712 !consumed(task, q)
713 && (q.status.open()
714 || awaiting_task_chat(q, task)
715 || q.status == QuestionStatus::Answered)
716 })
717 })
718 .collect()
719 }
720
721 fn displayed_open(&self, task: &Task) -> Option<Question> {
722 [NODE, DEPS_NODE]
723 .into_iter()
724 .filter(|node| {
725 task.status == TaskStatus::Held
726 || (*node == DEPS_NODE && task.status == TaskStatus::Blocked)
727 })
728 .filter_map(|node| self.latest(node, &task.id))
729 .find(|q| q.status.open() && !consumed(task, q))
730 }
731}
732
733#[derive(Debug, Clone, Copy, PartialEq, Eq)]
736enum AnswerAction {
737 Resume,
741 Discard,
744 KeepHeld,
749}
750
751fn interpret_answer(q: &Question) -> AnswerAction {
764 let resolution = q.resolution().unwrap_or_default();
765 match q.choices.iter().position(|c| *c == resolution) {
766 Some(0) => AnswerAction::Resume,
767 Some(2) => AnswerAction::Discard,
768 _ => AnswerAction::KeepHeld,
769 }
770}
771
772pub(crate) fn contextual_keep_held(q: &Question, label: &str) -> bool {
775 q.node == NODE
776 && q.seat == SEAT
777 && q.actions.is_empty()
778 && q.choices.get(1).is_some_and(|c| c == label)
779}
780
781fn keep_held_note(task: &Task, q: &Question, resolution: &str) -> String {
786 let marker = format!("{} operator: {resolution}", marker_for(q));
787 match task.hold_reason.as_deref() {
788 Some(existing) if !existing.is_empty() => format!("{existing}\n{marker}"),
789 _ => marker,
790 }
791}
792
793fn file_question(
797 questions: &Questions,
798 ownership: &crate::task_chat::InquiryOwnership,
799 task: &Task,
800 bucket: Bucket,
801 w: &Wording,
802 now: Timestamp,
803) -> Option<Question> {
804 let (summary, why, choices) = match bucket {
805 Bucket::MachineUnknown => (
806 w.summary_machine_unknown(task),
807 w.why_machine().to_owned(),
808 w.choices3(),
809 ),
810 Bucket::ConductorOverride => (
811 w.summary_conductor_override(task),
812 task.resume_override
813 .as_ref()
814 .map(|o| w.why_conductor_override(o))
815 .unwrap_or_default(),
816 w.choices_conductor_override(),
817 ),
818 Bucket::Legacy => (
819 w.summary_legacy(task),
820 w.why_legacy().to_owned(),
821 w.choices3(),
822 ),
823 Bucket::ManualStale => {
824 let days = (now.as_second() - task.updated_at.as_second()) / (24 * 60 * 60);
825 (
826 w.summary_manual_stale(task, days),
827 w.why_manual().to_owned(),
828 w.choices_manual(),
829 )
830 }
831 };
832 let mut q = Question::new(
833 task.id.clone(),
834 NODE.to_owned(),
835 SEAT.to_owned(),
836 summary,
837 w.detail(task, &why),
838 choices,
839 );
840 q.triage_generation = Some(task.resumption.as_ref().map_or(0, |r| r.generation));
841 questions.put_executable_in(&mut q, ownership).ok()?;
842 Some(q)
843}
844
845pub(crate) fn deputy_brief(q: &Question, queue: &Queue) -> String {
850 let task = queue.get(&q.run).ok();
851 let mut s = match &task {
852 Some(t) => format!(
853 "Task {} (`magi task show {}`), \"{}\", is {}. Hold source: {}. Reason: {}.",
854 t.id,
855 t.id,
856 t.title,
857 t.status.as_str(),
858 match t.hold_source {
859 Some(HoldSource::Machine) => "machine (an automatic hold)",
860 Some(HoldSource::Manual) => "manual (an operator held it)",
861 None => "unknown (an old record, treated as the operator's)",
862 },
863 t.hold_reason
864 .as_deref()
865 .or(t.last_error.as_deref())
866 .unwrap_or("(none recorded)")
867 ),
868 None => format!("Task {} no longer exists (its record is gone).", q.run),
869 };
870 if task.as_ref().is_some_and(disk_wait_eligible) {
871 s.push_str("\n\nDisk capacity waits are monitored automatically. No owner answer is needed for an untouched automatic disk triage question. Explicit holds, discussion and other approvals still require owner resolution.");
872 }
873 s.push_str("\n\nWhat each option does, by position:");
874 let n = q.choices.len();
875 for (i, c) in q.choices.iter().enumerate() {
876 let effect = if q.node == DEPS_NODE {
877 match i {
878 0 => "the dependency is released so the loop runs it",
879 1 => {
880 "the dependants are detached and the dependency task is DELETED; this cannot be undone"
881 }
882 2 => "the dependants are detached; the dependency stays as it is",
883 _ => "nothing is applied",
884 }
885 } else {
886 match i {
887 0 => "the task is released and requeued",
888 2 if n > 2 => "the task is DELETED; this cannot be undone",
889 _ => "the task stays held, with the answer noted",
890 }
891 };
892 s.push_str(&format!("\n- `{c}`: {effect}"));
893 }
894 s.push_str(
895 "\n\nThe daemon applies the answer on its next pass; you apply nothing. When \
896 the owner's words pick an option - including releasing a task they held \
897 themselves, or deleting it - settle it.",
898 );
899 s
900}
901
902fn quarantine_orphaned_blocked(queue: &Queue, questions: &Questions) -> Vec<String> {
920 let mut quarantined = Vec::new();
921 for listed in queue.list() {
922 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
923 continue;
924 }
925 let Ok(_claim) = queue.claim(&listed.id) else {
926 continue;
927 };
928 let Ok(mut task) = queue.get(&listed.id) else {
929 continue;
930 };
931 if task.status != TaskStatus::Blocked {
932 continue;
933 }
934 let deleted = queue.apply_deleted_blockers(&mut task, questions);
935 if !deleted.is_empty() && queue.put(&mut task).is_ok() {
936 for id in &deleted {
937 queue.note_dependency_deleted(&task, id);
938 }
939 }
940 if task.status != TaskStatus::Blocked {
941 continue;
942 }
943 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
944 if missing.is_empty() {
945 continue;
946 }
947 let language = crate::lang::of_repo(&repo_for(&task));
948 let mut cause = missing.clone();
949 cause.sort();
950 cause.dedup();
951 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
952 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
953 &task.blocked_by,
954 &missing,
955 &language,
956 )));
957 if queue.put(&mut task).is_ok() {
958 quarantined.push(task.id.clone());
959 }
960 }
961 quarantined
962}
963
964#[derive(Debug, Clone, Copy, PartialEq, Eq)]
967enum DepsAction {
968 Release,
969 Discard,
970 Detach,
971 Nothing,
973}
974
975fn interpret_deps_answer(q: &Question) -> DepsAction {
976 let resolution = q.resolution().unwrap_or_default();
977 match q.choices.iter().position(|c| *c == resolution) {
978 Some(0) => DepsAction::Release,
979 Some(1) => DepsAction::Discard,
980 Some(2) => DepsAction::Detach,
981 _ => DepsAction::Nothing,
982 }
983}
984
985fn deps_texts(w: &Wording, root: &Task, dependants: &[&Task]) -> (String, String, Vec<String>) {
987 let ja = w.lang == "ja";
988 let reason = root
989 .hold_reason
990 .as_deref()
991 .or(root.last_error.as_deref())
992 .unwrap_or(if ja {
993 "(記録なし)"
994 } else {
995 "(none recorded)"
996 });
997 let list: String = dependants
998 .iter()
999 .map(|t| format!("- {} {}\n", t.short(), t.title))
1000 .collect();
1001 if ja {
1002 (
1003 format!(
1004 "{} ({}) が {} 件のタスクを止めています",
1005 root.short(),
1006 root.status.as_str(),
1007 dependants.len()
1008 ),
1009 format!(
1010 "task: {} ({})\ntitle: {}\n状態: {}\n理由: {reason}\n\n\
1011 このタスクは自動では実行されないため、依存している次のタスクは永遠に待ち続けます:\n{list}",
1012 root.id,
1013 root.short(),
1014 root.title,
1015 root.status.as_str()
1016 ),
1017 vec![
1018 "依存先を再開する".to_owned(),
1019 "依存先を捨てる(依存タスクは切り離して実行)".to_owned(),
1020 "依存タスクを切り離す(依存先はそのまま)".to_owned(),
1021 ],
1022 )
1023 } else {
1024 (
1025 format!(
1026 "{} ({}) is freezing {} blocked task(s)",
1027 root.short(),
1028 root.status.as_str(),
1029 dependants.len()
1030 ),
1031 format!(
1032 "task: {} ({})\ntitle: {}\nstatus: {}\nreason: {reason}\n\n\
1033 Nothing in the loop will ever run this task, so these dependants wait \
1034 forever:\n{list}",
1035 root.id,
1036 root.short(),
1037 root.title,
1038 root.status.as_str()
1039 ),
1040 vec![
1041 "release the dependency".to_owned(),
1042 "discard the dependency (dependants are detached and run)".to_owned(),
1043 "detach the dependants (the dependency stays as it is)".to_owned(),
1044 ],
1045 )
1046 }
1047}
1048
1049fn detach_dependants(queue: &Queue, questions: &Questions, root: &str) {
1053 for listed in queue.list() {
1054 if listed.status != TaskStatus::Blocked || !listed.blocked_by.iter().any(|b| matches!(crate::queue::dependency(queue, questions, b), crate::queue::Dependency::Task(id) if id == root)) {
1055 continue;
1056 }
1057 let Ok(_claim) = queue.claim(&listed.id) else {
1058 continue;
1059 };
1060 let Ok(mut t) = queue.get(&listed.id) else {
1061 continue;
1062 };
1063 if t.status != TaskStatus::Blocked {
1064 continue;
1065 }
1066 for id in t.blocked_by.clone() {
1067 if matches!(crate::queue::dependency(queue, questions, &id), crate::queue::Dependency::Task(full) if full == root)
1068 {
1069 t.unblock(&id);
1070 }
1071 }
1072 let _ = queue.put(&mut t);
1073 }
1074}
1075
1076fn apply_deps_answer(
1080 queue: &Queue,
1081 questions: &Questions,
1082 q: &Question,
1083 now: Timestamp,
1084 ownership: &crate::task_chat::InquiryOwnership,
1085) -> bool {
1086 let Ok(_claim) = queue.claim(&q.run) else {
1087 return false;
1088 };
1089 let Ok(current) = questions.get_for_effects_in(&q.id, ownership) else {
1090 return false;
1091 };
1092 if current.status != QuestionStatus::Answered {
1093 return false;
1094 }
1095 let q = ¤t;
1096 let Ok(mut root) = queue.get(&q.run) else {
1097 return false;
1098 };
1099 if root.triage_applied(&q.id) {
1100 return false;
1101 }
1102 if root.operator_decided_at.is_some_and(|at| at > q.asked_at)
1103 || q.triage_generation
1104 .is_some_and(|g| g != root.resumption.as_ref().map_or(0, |r| r.generation))
1105 {
1106 root.mark_triage_applied(&q.id);
1107 return queue.put(&mut root).is_ok();
1108 }
1109 match interpret_deps_answer(q) {
1110 DepsAction::Release => {
1111 if root.status == TaskStatus::Running {
1112 return false;
1113 }
1114 if crate::repair_budget::reconcile::blocker(queue, &root).is_some() {
1115 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(q) else {
1116 return false;
1117 };
1118 root.decision_effects.insert(q.id.clone(), receipt);
1119 root.mark_triage_applied(&q.id);
1120 return queue.put(&mut root).is_ok();
1121 }
1122 root.answers.push(crate::queue::AnsweredQuestion {
1123 question: q.summary.clone(),
1124 answer: q.resolution().unwrap_or_default(),
1125 });
1126 if crate::base_sync_recovery::held_task(queue, &root) {
1127 root.mark_triage_applied(&q.id);
1128 return queue.put(&mut root).is_ok();
1129 }
1130 let before = root.operator_decided_at;
1131 let contract =
1132 crate::repair_budget::contract(&root, crate::repair_budget::Kind::Review).ok();
1133 root.release_operator("triage-dependencies", Some(&q.id));
1134 if let Some(contract) = contract
1135 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1136 q,
1137 before,
1138 &contract,
1139 root.resumption.as_ref().expect("release"),
1140 )
1141 {
1142 root.decision_effects.insert(q.id.clone(), evidence);
1143 }
1144 root.resume_override = Some(OperatorResume {
1145 question_id: q.id.clone(),
1146 at: now,
1147 conductor_rehold: None,
1148 forced: false,
1149 pinned_run: None,
1150 });
1151 }
1152 DepsAction::Discard => {
1153 detach_dependants(queue, questions, &root.id);
1154 return queue.discard_triage(&root.id, questions, &q.id).is_ok();
1155 }
1156 DepsAction::Detach => detach_dependants(queue, questions, &root.id),
1157 DepsAction::Nothing => return false,
1162 }
1163 root.mark_triage_applied(&q.id);
1164 queue.put(&mut root).is_ok()
1165}
1166
1167fn repair_receipt_notice(queue: &Queue, task: &Task) -> bool {
1168 let Some(detail) = crate::repair_budget::reconcile::blocker(queue, task) else {
1169 return false;
1170 };
1171 if let Some(home) = queue.root().parent() {
1172 let _ = crate::task_inquiry::Outbox::at(home).file_for_occurrence(
1173 task,
1174 task.runs.last().cloned(),
1175 "repair-receipt-recovery".into(),
1176 &task
1177 .repair_grants
1178 .last()
1179 .expect("diagnosed receipt")
1180 .request
1181 .request,
1182 "Retained repair receipt needs reconciliation",
1183 &detail,
1184 );
1185 }
1186 true
1187}
1188
1189fn ask_about_stuck_roots(
1198 queue: &Queue,
1199 questions: &Questions,
1200 config_override: Option<&Path>,
1201 report: &mut Report,
1202) {
1203 let all = questions.list_for_effects();
1204 let tasks = queue.list();
1205 let recovery = RecoveryIndex::from_scan(queue, &tasks, all.clone());
1206 let inv = crate::blockers::Inventory::load_with_tasks(queue, questions, tasks);
1207 let mut frozen: std::collections::BTreeMap<String, Vec<String>> = Default::default();
1208 for (id, roots) in inv.stuck() {
1209 for root in roots {
1210 if root != id {
1211 frozen.entry(root).or_default().push(id.clone());
1212 }
1213 }
1214 }
1215 let ownership = crate::task_chat::InquiryOwnership::default();
1219 for mut q in all
1220 .iter()
1221 .filter(|q| q.node == DEPS_NODE && q.status.open())
1222 .cloned()
1223 {
1224 if inv.task(&q.run).is_some_and(|task| consumed(task, &q)) {
1225 continue;
1226 }
1227 if !frozen.contains_key(&q.run) {
1228 q.abandon("nothing waits on this task anymore");
1229 let _ = questions.put_executable_in(&mut q, &ownership);
1230 }
1231 }
1232
1233 for (root_id, dependant_ids) in &frozen {
1234 let Ok(_claim) = queue.claim(root_id) else {
1235 continue;
1236 };
1237 let seeds: Vec<_> = std::iter::once(root_id)
1240 .chain(dependant_ids.iter())
1241 .filter_map(|id| queue.get(id).ok())
1242 .collect();
1243 let current = crate::blockers::Inventory::load_for_tasks(queue, questions, &seeds);
1244 let Some(root) = current.task(root_id) else {
1245 continue;
1246 };
1247 if crate::base_sync_recovery::held_task(queue, root) {
1248 continue;
1249 }
1250 if disk_wait_known(root, questions.root().parent().unwrap_or(Path::new(""))) {
1253 continue;
1254 }
1255 let dependants: Vec<&Task> = dependant_ids
1256 .iter()
1257 .filter_map(|d| current.task(d))
1258 .filter(|t| current.stuck_roots(t).contains(root_id))
1259 .collect();
1260 if dependants.is_empty() {
1261 continue;
1262 }
1263 if repair_receipt_notice(queue, root) {
1264 continue;
1265 }
1266 let latest = recovery.latest(DEPS_NODE, root_id);
1267 match latest.as_ref() {
1268 Some(q) if !consumed(root, q) && (q.status.open() || awaiting_task_chat(q, root)) => {
1269 continue;
1270 }
1271 Some(q) if q.status == QuestionStatus::Answered && !consumed(root, q) => {
1272 continue;
1273 }
1274 Some(q) if q.status == QuestionStatus::Abandoned => {
1275 let changed = root.updated_at > q.asked_at
1276 || dependants.iter().any(|t| t.updated_at > q.asked_at);
1277 if !changed {
1278 continue;
1279 }
1280 }
1281 _ => {}
1282 }
1283 if !recovery.pending(root).is_empty() {
1284 continue;
1285 }
1286 let cfg = Config::discover(&repo_for(root), config_override)
1287 .ok()
1288 .map(|(c, _)| c);
1289 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1290 let (summary, detail, choices) = deps_texts(w, root, &dependants);
1291 let mut question = Question::new(
1292 root.id.clone(),
1293 DEPS_NODE.to_owned(),
1294 SEAT.to_owned(),
1295 summary,
1296 detail,
1297 choices,
1298 );
1299 question.triage_generation = Some(root.resumption.as_ref().map_or(0, |r| r.generation));
1300 if questions
1301 .put_executable_in(&mut question, &ownership)
1302 .is_ok()
1303 {
1304 report.asked.push(root.id.clone());
1305 }
1306 }
1307}
1308
1309pub fn apply_answers(queue: &Queue, questions: &Questions) -> Vec<String> {
1312 let mut applied = Vec::new();
1313 let ownership = crate::task_chat::InquiryOwnership::default();
1314 for q in questions.list_for_effects_in(&ownership) {
1315 if q.node == DEPS_NODE
1316 && q.status == QuestionStatus::Answered
1317 && apply_deps_answer(queue, questions, &q, Timestamp::now(), &ownership)
1318 {
1319 applied.push(q.run.clone());
1320 }
1321 }
1322 let tasks = queue.list();
1324 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
1325 let recovery =
1326 RecoveryIndex::from_scan(queue, &tasks, questions.list_for_effects_in(&ownership));
1327 for listed in tasks {
1328 let Some(q) = recovery.latest(NODE, &listed.id) else {
1329 continue;
1330 };
1331 if q.status != QuestionStatus::Answered || already_applied(&listed, &q) {
1332 continue;
1333 }
1334 let Ok(_claim) = queue.claim(&listed.id) else {
1335 continue;
1336 };
1337 let Ok(mut task) = queue.get(&listed.id) else {
1338 continue;
1339 };
1340 let Ok(q) = questions.get_for_effects_in(&q.id, &ownership) else {
1343 continue;
1344 };
1345 if q.status != QuestionStatus::Answered {
1346 continue;
1347 }
1348 if already_applied(&task, &q)
1349 || !matches!(task.status, TaskStatus::Held | TaskStatus::Queued)
1350 {
1351 continue;
1352 }
1353 let generation = task.resumption.as_ref().map_or(0, |r| r.generation);
1354 let action = interpret_answer(&q);
1355 let mut newer_decision = q.triage_generation.is_some_and(|g| g != generation)
1356 || task.resumption.as_ref().is_some_and(|r| r.at > q.asked_at)
1357 || task.held_at.is_some_and(|at| at > q.asked_at)
1358 || task.operator_decided_at.is_some_and(|at| at > q.asked_at);
1359 if matches!(action, AnswerAction::KeepHeld)
1363 && q.answered_at.is_some_and(|at| {
1364 task.operator_decided_at
1365 .is_none_or(|decision| at > decision)
1366 && task.resumption.as_ref().is_none_or(|r| at > r.at)
1367 })
1368 {
1369 newer_decision = false;
1370 }
1371 let resolution = q.resolution().unwrap_or_default();
1373 task.answers.push(crate::queue::AnsweredQuestion {
1374 question: q.summary.clone(),
1375 answer: if newer_decision {
1376 format!("{resolution} (superseded by a later operator decision)")
1377 } else {
1378 resolution
1379 },
1380 });
1381 if !newer_decision {
1382 match action {
1383 AnswerAction::Resume if crate::base_sync_recovery::held_task(queue, &task) => {
1384 }
1387 AnswerAction::Resume
1388 if crate::repair_budget::reconcile::blocker(queue, &task).is_some() =>
1389 {
1390 let Ok(receipt) = crate::repair_budget::reconcile::blocked_resume(&q) else {
1391 continue;
1392 };
1393 task.decision_effects.insert(q.id.clone(), receipt);
1394 }
1395 AnswerAction::Resume => {
1396 let before = task.operator_decided_at;
1397 let contract =
1398 crate::repair_budget::contract(&task, crate::repair_budget::Kind::Review)
1399 .ok();
1400 task.release_operator("triage", Some(&q.id));
1401 if let Some(contract) = contract
1402 && let Ok(evidence) = crate::repair_budget::reconcile::effect_evidence(
1403 &q,
1404 before,
1405 &contract,
1406 task.resumption.as_ref().expect("release"),
1407 )
1408 {
1409 task.decision_effects.insert(q.id.clone(), evidence);
1410 }
1411 task.resume_override = Some(OperatorResume {
1412 question_id: q.id.clone(),
1413 at: Timestamp::now(),
1414 conductor_rehold: None,
1415 forced: true,
1416 pinned_run: None,
1417 });
1418 }
1419 AnswerAction::KeepHeld => {
1420 let note = keep_held_note(&task, &q, &q.resolution().unwrap_or_default());
1421 task.hold_manual(Some(note));
1422 }
1423 AnswerAction::Discard => {
1424 if queue.discard_triage(&task.id, questions, &q.id).is_ok() {
1425 applied.push(task.id);
1426 }
1427 continue;
1428 }
1429 }
1430 }
1431 task.mark_triage_applied(&q.id);
1432 if queue.put(&mut task).is_ok() {
1433 applied.push(task.id);
1434 }
1435 }
1436 applied
1437}
1438const DISK_SUPERSEDED: &str =
1439 "Automatic disk wait superseded unanswered triage; no owner answer or approval was created";
1440
1441fn untouched_disk_question(q: &Question, task: &Task) -> bool {
1442 q.status.open()
1443 && q.node == NODE
1444 && q.seat == SEAT
1445 && q.run == task.id
1446 && q.routed_task.as_deref().is_none_or(|id| id == task.id)
1447 && q.triage_generation == Some(task.resumption.as_ref().map_or(0, |r| r.generation))
1448 && task.held_at.is_some_and(|at| q.asked_at >= at)
1449 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1450 && q.answer.is_none()
1451 && q.answered_at.is_none()
1452 && q.accepted_answer_at.is_none()
1453 && q.thread.is_empty()
1454 && q.actions.is_empty()
1455 && q.merge_target.is_none()
1456 && [&EN, &JA].iter().any(|w| {
1457 q.summary == w.summary_machine_unknown(task)
1458 && q.choices == w.choices3()
1459 && q.detail == w.detail(task, w.why_machine())
1460 })
1461}
1462
1463fn refresh_disk_notices(home: &Path, task: &Task) {
1465 let notices = crate::notices::Notices::at(home.join("notifications"));
1466 let Some(task_notice) = crate::notices::task_held(task) else {
1467 return;
1468 };
1469 for (key, message) in [
1470 (task_notice.key, task_notice.message),
1471 (
1472 format!("disk:{}", repo_for(task).display()),
1473 format!(
1474 "Task {} is waiting for disk space. It will resume automatically when disk and output admission checks succeed.",
1475 task.id
1476 ),
1477 ),
1478 ] {
1479 let id = crate::notices::id_of(&key);
1480 if notices
1481 .get(&id)
1482 .is_ok_and(|n| n.key == key && n.message != message)
1483 {
1484 let _ = notices.update(&id, |n| {
1485 if n.key == key {
1486 n.message = message;
1487 }
1488 });
1489 }
1490 }
1491}
1492
1493fn resume_disk_task(
1497 queue: &Queue,
1498 questions: &Questions,
1499 task: &mut Task,
1500 ownership: &crate::task_chat::InquiryOwnership,
1501) -> anyhow::Result<bool> {
1502 let expected = task.clone();
1503 let home = questions
1504 .root()
1505 .parent()
1506 .ok_or_else(|| anyhow::anyhow!("missing question home"))?;
1507 let _lease = if let Some(id) = task.disk_wait_run() {
1508 anyhow::ensure!(
1509 task.resume_override
1510 .as_ref()
1511 .and_then(|o| o.pinned_run.as_ref())
1512 .is_none_or(|pin| pin == id),
1513 "selected recovery run differs from disk wait evidence"
1514 );
1515 let state = crate::run::RunState::load_under(id, home)?;
1516 anyhow::ensure!(
1517 state.id == *id && state.status.resumable() && !task.fresh_start,
1518 "retained execution cannot resume"
1519 );
1520 anyhow::ensure!(
1521 state.liveness_under(
1522 home,
1523 crate::daemon::is_working_on(home, id, Timestamp::now())
1524 ) == crate::run::Liveness::Dead,
1525 "retained execution is not proven idle"
1526 );
1527 Some(crate::driver::maintenance(home, id)?)
1528 } else {
1529 None
1530 };
1531 let outbox = crate::task_inquiry::Outbox::at(home);
1532 let all = questions.list_for_effects_in(ownership);
1533 let pending: Vec<_> = all
1534 .iter()
1535 .filter(|q| {
1536 (!q.run.is_empty()
1538 && std::iter::once(task.id.as_str())
1539 .chain(task.execution_runs().iter().map(String::as_str))
1540 .any(|id| id == q.run || id.starts_with(&q.run) || id.ends_with(&q.run)))
1541 && (q.status.open()
1542 || ((q.node == NODE || q.node == DEPS_NODE)
1543 && (awaiting_task_chat(q, task)
1544 || (q.status == QuestionStatus::Answered
1545 && !already_applied(task, q)))))
1546 })
1547 .collect();
1548 anyhow::ensure!(pending.len() <= 1, "multiple decisions remain");
1550 let candidate = pending.first().copied();
1551 if let Some(q) = candidate {
1552 anyhow::ensure!(untouched_disk_question(q, task), "owner decision remains");
1553 }
1554 for inquiry in outbox.pending()? {
1555 if inquiry.task == task.id && inquiry.decision.is_some() {
1556 anyhow::ensure!(
1557 candidate.is_some_and(|q| q.id == inquiry.id),
1558 "unrelated decision remains"
1559 );
1560 }
1561 }
1562 if let Some(q) = candidate {
1563 let _execution = outbox.execution_claim(&q.id)?;
1564 questions.update_in(&q.id, ownership, |current| {
1565 anyhow::ensure!(untouched_disk_question(current, task), "question changed");
1566 let inquiry = match outbox.get(&q.id) {
1567 Ok(inquiry) => inquiry,
1568 Err(error)
1569 if error
1570 .downcast_ref::<std::io::Error>()
1571 .is_some_and(|e| e.kind() == std::io::ErrorKind::NotFound) =>
1572 {
1573 anyhow::ensure!(
1574 current.routed_chat.is_none(),
1575 "delivered inquiry is missing; owner reconciliation required"
1576 );
1577 outbox.enqueue(crate::task_chat::live_record(home, task, current))?
1580 }
1581 Err(error) => return Err(error),
1582 };
1583 anyhow::ensure!(
1584 inquiry.task == task.id
1585 && inquiry.repo == task.repo
1586 && inquiry.source == format!("{}:{}:{}", NODE, SEAT, task.id)
1587 && current
1588 .routed_chat
1589 .as_ref()
1590 .is_none_or(|chat| inquiry.delivered_to.as_ref() == Some(chat)),
1591 "inquiry binding changed"
1592 );
1593 outbox.update_decision(&q.id, |d| {
1594 if d.status == crate::task_decision::Status::Resolved {
1595 anyhow::ensure!(
1596 d.request.is_none()
1597 && d.history
1598 .last()
1599 .is_some_and(|e| e.evidence == DISK_SUPERSEDED),
1600 "decision already settled by owner"
1601 );
1602 return Ok(false);
1603 }
1604 anyhow::ensure!(
1605 d.status == crate::task_decision::Status::AwaitingOwner
1606 && d.request.is_none()
1607 && d.source_answer_at.is_none(),
1608 "owner request remains"
1609 );
1610 d.settle(DISK_SUPERSEDED)
1611 })?;
1612 current.abandon(DISK_SUPERSEDED);
1613 Ok(())
1614 })?;
1615 }
1616 if let Some(q) = candidate {
1617 task.mark_triage_applied(&q.id);
1618 }
1619 task.resume_disk_wait();
1620 queue.commit_disk_wait(task, &expected)?;
1621 let notices = crate::notices::Notices::at(home.join("notifications"));
1622 let keys = [
1623 format!("disk-recovered:{}", task.id),
1624 format!("disk:{}", repo_for(task).display()),
1625 format!("task:{}", task.id),
1626 ];
1627 let repo_waiting = queue
1628 .list()
1629 .iter()
1630 .any(|other| other.repo == task.repo && disk_wait_eligible(other));
1631 for notice in notices.list() {
1632 if notice.key == keys[0]
1633 || notice.key == keys[2]
1634 || (!repo_waiting && notice.key == keys[1])
1635 {
1636 let _ = notices.dismiss(¬ice.id);
1637 }
1638 }
1639 Ok(candidate.is_some())
1640}
1641
1642pub fn run_once(
1661 queue: &Queue,
1662 questions: &Questions,
1663 config_override: Option<&Path>,
1664 now: Timestamp,
1665) -> Report {
1666 run_once_inner(queue, questions, config_override, now, false)
1667}
1668
1669pub(crate) fn monitor_disk_waits(
1671 queue: &Queue,
1672 questions: &Questions,
1673 config: Option<&Path>,
1674 now: Timestamp,
1675) -> Report {
1676 run_once_inner(queue, questions, config, now, true)
1677}
1678
1679fn run_once_inner(
1680 queue: &Queue,
1681 questions: &Questions,
1682 config_override: Option<&Path>,
1683 now: Timestamp,
1684 disk_only: bool,
1685) -> Report {
1686 run_once_with_probe(
1687 queue,
1688 questions,
1689 config_override,
1690 now,
1691 disk_only,
1692 machine_cause_resolved,
1693 )
1694}
1695
1696fn run_once_with_probe(
1697 queue: &Queue,
1698 questions: &Questions,
1699 config_override: Option<&Path>,
1700 now: Timestamp,
1701 disk_only: bool,
1702 resolved: impl Fn(&Task, &Config) -> Option<bool>,
1703) -> Report {
1704 let mut report = Report {
1705 answered: if disk_only {
1706 Vec::new()
1707 } else {
1708 apply_answers(queue, questions)
1709 },
1710 quarantined: if disk_only {
1711 Vec::new()
1712 } else {
1713 quarantine_orphaned_blocked(queue, questions)
1714 },
1715 ..Report::default()
1716 };
1717 if !disk_only {
1718 ask_about_stuck_roots(queue, questions, config_override, &mut report);
1719 }
1720 let recovery = RecoveryIndex::load(questions);
1721 let ownership = crate::task_chat::InquiryOwnership::default();
1722 for listed in queue.list() {
1723 if listed.status != TaskStatus::Held {
1724 continue;
1725 }
1726 let Ok(_claim) = queue.claim(&listed.id) else {
1727 continue;
1728 };
1729 let Ok(mut task) = queue.get(&listed.id) else {
1730 continue;
1731 };
1732 if task.status != TaskStatus::Held {
1736 continue;
1737 }
1738 if crate::base_sync_recovery::held_task(queue, &task) {
1739 continue;
1740 }
1741 if repair_receipt_notice(queue, &task) {
1742 continue;
1743 }
1744 if !disk_wait_known(&task, questions.root().parent().unwrap_or(Path::new("")))
1751 && deps_pending_in(&recovery, &task)
1752 {
1753 continue;
1754 }
1755
1756 let explicit = match &task.overrides {
1757 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
1758 _ => config_override,
1759 };
1760 let cfg = Config::discover(&repo_for(&task), explicit)
1761 .ok()
1762 .map(|(c, _)| c);
1763 let w = wording(cfg.as_ref().map_or("en", |c| c.graph.language.as_str()));
1764
1765 if disk_wait_known(&task, questions.root().parent().unwrap_or(Path::new(""))) {
1766 if task
1767 .disk_checked_at
1768 .is_some_and(|at| now.as_second() - at.as_second() < 60)
1769 {
1770 continue;
1771 }
1772 let expected = task.clone();
1773 task.disk_checked_at = Some(now);
1774 if !task.has_disk_capacity_cause() {
1775 task.mark_disk_capacity();
1777 }
1778 if queue.commit_disk_wait(&mut task, &expected).is_err() {
1779 continue;
1780 }
1781 refresh_disk_notices(questions.root().parent().unwrap_or(Path::new("")), &task);
1782 if cfg.as_ref().and_then(|c| resolved(&task, c)) == Some(true) {
1785 match resume_disk_task(queue, questions, &mut task, &ownership) {
1786 Ok(superseded) => {
1787 if superseded {
1788 report.superseded.push(task.id.clone());
1789 }
1790 report.resumed.push(task.id.clone());
1791 }
1792 Err(error) => {
1793 if !report.needs_owner.contains(&task.id) {
1794 report.needs_owner.push(task.id.clone());
1795 }
1796 crate::notices::raise_in(
1797 questions.root().parent().unwrap_or(Path::new("")),
1798 crate::notices::Notice::warn(
1799 &format!("disk-recovered:{}", task.id),
1800 format!(
1801 "Task {}: disk capacity recovered; automatic recovery is still blocked ({error}). Existing owner decisions and unrelated approvals require explicit resolution. Disk monitoring continues.",
1802 task.id
1803 ),
1804 ).link(crate::notices::Link::Task { id: task.id.clone() }).about([task.id.clone()]),
1805 );
1806 }
1807 }
1808 }
1809 continue;
1810 }
1811 if disk_only {
1812 continue;
1813 }
1814
1815 if let Some(q) = recovery.latest(NODE, &task.id) {
1816 if !consumed(&task, &q) && (q.status.open() || awaiting_task_chat(&q, &task)) {
1817 continue;
1819 }
1820 }
1826
1827 match task.hold_source {
1828 Some(HoldSource::Machine) => {
1829 let overridden = task
1830 .resume_override
1831 .as_ref()
1832 .is_some_and(|o| o.conductor_rehold.is_some() && !o.forced);
1833 if overridden {
1834 if file_question(
1835 questions,
1836 &ownership,
1837 &task,
1838 Bucket::ConductorOverride,
1839 w,
1840 now,
1841 )
1842 .is_some()
1843 {
1844 report.asked.push(task.id.clone());
1845 }
1846 } else if cfg.as_ref().and_then(|c| resolved(&task, c)) == Some(true) {
1847 task.release();
1848 if queue.put(&mut task).is_ok() {
1849 report.resumed.push(task.id.clone());
1850 }
1851 } else if !known_isolated_cache_wait(&task, &crate::run::home())
1852 && file_question(questions, &ownership, &task, Bucket::MachineUnknown, w, now)
1853 .is_some()
1854 {
1855 report.asked.push(task.id.clone());
1856 }
1857 }
1858 None => {
1859 if file_question(questions, &ownership, &task, Bucket::Legacy, w, now).is_some() {
1860 report.asked.push(task.id.clone());
1861 }
1862 }
1863 Some(HoldSource::Manual) => {
1864 if manual_is_stale(&task, now)
1865 && file_question(questions, &ownership, &task, Bucket::ManualStale, w, now)
1866 .is_some()
1867 {
1868 report.asked.push(task.id.clone());
1869 }
1870 }
1871 }
1872 }
1873 report
1874}
1875
1876pub fn open_question_for(questions: &Questions, task_id: &str) -> Option<Question> {
1881 let queue = question_queue(questions);
1882 let task = queue.get(task_id).ok()?;
1883 display_index(questions, &queue, std::slice::from_ref(&task)).displayed_open(&task)
1884}
1885
1886pub fn pending_for(questions: &Questions, task: &Task) -> bool {
1900 !pending_recovery_questions(questions, task).is_empty()
1901}
1902
1903pub(crate) fn pending_recovery_questions(questions: &Questions, task: &Task) -> Vec<Question> {
1906 RecoveryIndex::load(questions).pending(task)
1907}
1908
1909fn awaiting_task_chat(q: &Question, task: &Task) -> bool {
1910 q.status != QuestionStatus::Abandoned
1911 && q.routed_chat.is_some()
1912 && !already_applied(task, q)
1913 && task.resumption.as_ref().map_or(0, |r| r.generation) == q.triage_generation.unwrap_or(0)
1914 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1915}
1916
1917fn deps_pending_in(recovery: &RecoveryIndex, task: &Task) -> bool {
1921 match recovery.latest(DEPS_NODE, &task.id) {
1922 Some(q) if consumed(task, &q) => false,
1923 Some(q) if q.status.open() || awaiting_task_chat(&q, task) => true,
1924 Some(q) if q.status == QuestionStatus::Answered => true,
1925 _ => false,
1926 }
1927}
1928
1929pub fn open_task_ids(questions: &Questions) -> std::collections::BTreeSet<String> {
1934 let queue = question_queue(questions);
1935 let tasks = queue.list();
1936 let index = display_index(questions, &queue, &tasks);
1937 tasks
1938 .iter()
1939 .filter(|task| index.displayed_open(task).is_some())
1940 .map(|task| task.id.clone())
1941 .collect()
1942}
1943
1944fn question_queue(questions: &Questions) -> Queue {
1945 Queue::at(
1946 questions
1947 .root()
1948 .parent()
1949 .unwrap_or(Path::new(""))
1950 .join("queue"),
1951 )
1952}
1953
1954fn display_index(questions: &Questions, queue: &Queue, tasks: &[Task]) -> RecoveryIndex {
1956 RecoveryIndex::from_scan(queue, tasks, questions.list())
1957}
1958
1959#[cfg(test)]
1960mod tests {
1961 use super::*;
1962 use crate::ask::Answer;
1963 use crate::queue::Source;
1964 use jiff::SignedDuration;
1965
1966 fn store() -> (tempfile::TempDir, Queue, Questions) {
1967 let dir = tempfile::tempdir().unwrap();
1968 let q = Queue::at(dir.path().join("queue"));
1969 let s = Questions::at(dir.path().join("questions"));
1970 (dir, q, s)
1971 }
1972
1973 fn task(title: &str, repo: PathBuf) -> Task {
1974 Task::new(title.to_owned(), format!("do {title}"), repo, Source::Human)
1975 }
1976
1977 fn disk_task(repo: PathBuf) -> Task {
1978 let mut t = task("capacity", repo);
1979 t.hold_machine(crate::disk::gate_in(1_477_050_368, 3_221_225_472, "ja"));
1980 t.mark_disk_capacity();
1981 t.attempts = 3;
1982 t
1983 }
1984
1985 #[test]
1986 fn disk_wait_repeated_pressure_measurement_failure_and_recovery_cost_nothing() {
1987 let (dir, queue, questions) = store();
1988 let cfg = gate_disabled_config(dir.path());
1989 let mut t = disk_task(dir.path().to_owned());
1990 queue.put(&mut t).unwrap();
1991 let notices = crate::notices::Notices::at(dir.path().join("notifications"));
1992 let before = serde_json::to_value(notices.list()).unwrap();
1993 let start = Timestamp::now();
1994 let probes = std::cell::Cell::new(0);
1995 for seconds in [0, 1, 10, 60, 61, 120] {
1996 let report = run_once_with_probe(
1997 &queue,
1998 &questions,
1999 Some(&cfg),
2000 start + SignedDuration::from_secs(seconds),
2001 true,
2002 |_, _| {
2003 probes.set(probes.get() + 1);
2004 Some(false) },
2006 );
2007 assert!(report.is_empty());
2008 assert_eq!(queue.get(&t.id).unwrap().attempts, 3);
2009 }
2010 assert_eq!(
2011 probes.get(),
2012 3,
2013 "resource probes are bounded to one per minute"
2014 );
2015 assert!(questions.list().is_empty());
2016 assert_eq!(serde_json::to_value(notices.list()).unwrap(), before);
2017 let recovered = run_once_with_probe(
2018 &queue,
2019 &questions,
2020 Some(&cfg),
2021 start + SignedDuration::from_secs(180),
2022 true,
2023 |_, _| Some(true),
2024 );
2025 assert_eq!(recovered.resumed, [t.id.clone()]);
2026 let saved = queue.get(&t.id).unwrap();
2027 assert_eq!(saved.status, TaskStatus::Queued);
2028 assert_eq!(saved.attempts, 3);
2029 let again = run_once_with_probe(
2030 &queue,
2031 &questions,
2032 Some(&cfg),
2033 start + SignedDuration::from_secs(240),
2034 true,
2035 |_, _| panic!("queued tasks are not relaunched"),
2036 );
2037 assert!(again.is_empty());
2038 }
2039
2040 #[test]
2041 fn disk_wait_supersedes_only_untouched_automatic_questions_in_both_stores() {
2042 for routed in [false, true] {
2043 let (dir, queue, questions) = store();
2044 let cfg = gate_disabled_config(dir.path());
2045 let mut t = disk_task(dir.path().to_owned());
2046 queue.put(&mut t).unwrap();
2047 let q = file_question(
2048 &questions,
2049 &crate::task_chat::InquiryOwnership::default(),
2050 &t,
2051 Bucket::MachineUnknown,
2052 &JA,
2053 Timestamp::now(),
2054 )
2055 .unwrap();
2056 if routed {
2057 crate::task_chat::sweep(
2058 &queue,
2059 &crate::talk::Talks::at(dir.path().join("talks")),
2060 &questions,
2061 dir.path(),
2062 );
2063 assert!(
2064 crate::task_inquiry::Outbox::at(dir.path())
2065 .get(&q.id)
2066 .is_ok()
2067 );
2068 }
2069 let start = Timestamp::now();
2070 for seconds in [0, 60, 120] {
2071 assert!(
2072 run_once_with_probe(
2073 &queue,
2074 &questions,
2075 Some(&cfg),
2076 start + SignedDuration::from_secs(seconds),
2077 true,
2078 |_, _| Some(false)
2079 )
2080 .is_empty()
2081 );
2082 assert!(questions.read_history(&q.id).unwrap().status.open());
2083 assert_eq!(questions.list().len(), 1);
2084 }
2085 let report = monitor_disk_waits(
2086 &queue,
2087 &questions,
2088 Some(&cfg),
2089 start + SignedDuration::from_secs(180),
2090 );
2091 assert_eq!(report.resumed, [t.id.clone()], "routed={routed}");
2092 let source = questions.read_history(&q.id).unwrap();
2093 assert_eq!(source.status, QuestionStatus::Abandoned);
2094 assert!(source.answer.is_none());
2095 assert!(source.answered_at.is_none());
2096 assert!(!pending_for(&questions, &queue.get(&t.id).unwrap()));
2097 if routed {
2098 let d = crate::task_inquiry::Outbox::at(dir.path())
2099 .get(&q.id)
2100 .unwrap()
2101 .decision
2102 .unwrap();
2103 assert_eq!(d.status, crate::task_decision::Status::Resolved);
2104 assert!(
2105 d.request.is_none(),
2106 "system supersession fabricates no owner request"
2107 );
2108 }
2109 }
2110 }
2111
2112 #[test]
2113 fn disk_wait_respects_owner_hold_discussion_and_unrelated_approvals() {
2114 for kind in [
2115 "manual",
2116 "answered",
2117 "discussion",
2118 "unrelated",
2119 "override",
2120 "native-hold",
2121 "missing-native",
2122 ] {
2123 let (dir, queue, questions) = store();
2124 let cfg = gate_disabled_config(dir.path());
2125 let mut t = disk_task(dir.path().to_owned());
2126 queue.put(&mut t).unwrap();
2127 let q = file_question(
2128 &questions,
2129 &crate::task_chat::InquiryOwnership::default(),
2130 &t,
2131 Bucket::MachineUnknown,
2132 &EN,
2133 Timestamp::now(),
2134 )
2135 .unwrap();
2136 match kind {
2137 "manual" => {
2138 t.hold_manual(Some("owner hold".into()));
2139 queue.put(&mut t).unwrap();
2140 }
2141 "answered" => {
2142 questions
2143 .update(&q.id, |q| q.answer(Answer::Choice(q.choices[1].clone())))
2144 .unwrap();
2145 }
2146 "discussion" => {
2147 questions
2148 .update(&q.id, |q| q.say("please wait".to_owned()))
2149 .unwrap();
2150 }
2151 "unrelated" => {
2152 questions
2153 .update(&q.id, |q| {
2154 q.node = "land".into();
2155 Ok(())
2156 })
2157 .unwrap();
2158 }
2159 "override" => {
2160 t.resume_override = Some(OperatorResume {
2161 question_id: "owner".into(),
2162 at: Timestamp::now(),
2163 conductor_rehold: Some("pending conductor decision".into()),
2164 forced: true,
2165 pinned_run: None,
2166 });
2167 queue.put(&mut t).unwrap();
2168 }
2169 "native-hold" | "missing-native" => {
2170 crate::task_chat::sweep(
2171 &queue,
2172 &crate::talk::Talks::at(dir.path().join("talks")),
2173 &questions,
2174 dir.path(),
2175 );
2176 if kind == "native-hold" {
2177 crate::task_inquiry::Outbox::at(dir.path())
2178 .update_decision(&q.id, |d| d.accept("owner requested hold"))
2179 .unwrap();
2180 } else {
2181 questions
2185 .update(&q.id, |q| {
2186 q.routed_chat = Some("delivered-chat".into());
2187 Ok(())
2188 })
2189 .unwrap();
2190 std::fs::remove_file(
2191 dir.path()
2192 .join("notifications/task-inquiries")
2193 .join(format!("{}.json", q.id)),
2194 )
2195 .unwrap();
2196 }
2197 }
2198 _ => unreachable!(),
2199 }
2200 let report = run_once(&queue, &questions, Some(&cfg), Timestamp::now());
2201 assert!(report.resumed.is_empty(), "{kind}");
2202 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held, "{kind}");
2203 assert!(
2204 questions.read_history(&q.id).unwrap().status != QuestionStatus::Abandoned,
2205 "{kind}"
2206 );
2207 }
2208 }
2209
2210 #[test]
2211 fn startup_disk_wait_probes_the_task_override_and_does_not_pin_history() {
2212 use crate::config::BuildCache;
2213 crate::run::pin_test_home();
2216 let (dir, queue, questions) = store();
2217 let shared = dir.path().join("shared");
2218 let isolated = crate::build_target::root(dir.path());
2219 let mut config = Config::default();
2220 config.disk.min_free_bytes = 0;
2221 config.graph.worktree_root = Some(dir.path().to_owned());
2222 config.verify.gate = vec![format!(
2223 "CARGO_TARGET_DIR='{}' cargo check",
2224 shared.display()
2225 )];
2226 let mut task = disk_task(dir.path().to_owned());
2227 task.mark_startup_disk_capacity();
2228 task.overrides = Some(crate::queue::RunOverrides {
2229 build_cache: Some(BuildCache::Isolated),
2230 ..Default::default()
2231 });
2232 let check = |task: &Task, config: &Config, blocked: &Path| {
2233 machine_cause_resolved_with_admission(task, config, dir.path(), |paths, _| {
2234 !paths.iter().any(|path| path == blocked)
2235 })
2236 };
2237 assert_eq!(check(&task, &config, &isolated), Some(false));
2238 assert_eq!(check(&task, &config, &shared), Some(true));
2239 config.verify.build_cache = BuildCache::Isolated;
2240 task.overrides.as_mut().unwrap().build_cache = Some(BuildCache::Shared);
2241 assert_eq!(check(&task, &config, &shared), Some(false));
2242 assert_eq!(check(&task, &config, &isolated), Some(true));
2243 task.urgent = true;
2244 config.verify.build_cache = BuildCache::Shared;
2245 task.overrides.as_mut().unwrap().build_cache = None;
2246 assert_eq!(check(&task, &config, &isolated), Some(false));
2247 assert_eq!(check(&task, &config, &shared), Some(true));
2248 task.overrides.as_mut().unwrap().build_cache = Some(BuildCache::Shared);
2249
2250 task.runs.push("historical-execution".into());
2251 queue.put(&mut task).unwrap();
2252 let cfg = gate_disabled_config(dir.path());
2253 let report = run_once_with_probe(
2254 &queue,
2255 &questions,
2256 Some(&cfg),
2257 Timestamp::now(),
2258 true,
2259 |task, _| check(task, &config, &isolated),
2260 );
2261 assert_eq!(report.resumed, [task.id.clone()]);
2262 let saved = queue.get(&task.id).unwrap();
2263 assert_eq!(saved.attempts, task.attempts);
2264 assert_eq!(saved.runs, task.runs);
2265 assert!(saved.disk_resume_run.is_none());
2266 }
2267
2268 #[test]
2269 fn disk_wait_reuses_retained_execution_and_preserves_attempt_and_approval_evidence() {
2270 let (dir, queue, questions) = store();
2271 let cfg = gate_disabled_config(dir.path());
2272 let mut run = crate::run::RunState::new(
2273 dir.path().to_owned(),
2274 "main".into(),
2275 "base".into(),
2276 "capacity".into(),
2277 Config::default(),
2278 );
2279 run.status = crate::run::RunStatus::Blocked;
2280 run.driver_exited = true;
2281 run.save_under(dir.path()).unwrap();
2282 let mut t = disk_task(dir.path().to_owned());
2283 t.runs.push(run.id.clone());
2284 t.resume_override = Some(OperatorResume {
2285 question_id: "approved-recovery".into(),
2286 at: Timestamp::now(),
2287 conductor_rehold: None,
2288 forced: true,
2289 pinned_run: Some(run.id.clone()),
2290 });
2291 queue.put(&mut t).unwrap();
2292 let before = std::fs::read(dir.path().join("runs").join(&run.id).join("run.json")).unwrap();
2293 let report = run_once_with_probe(
2294 &queue,
2295 &questions,
2296 Some(&cfg),
2297 Timestamp::now(),
2298 true,
2299 |_, _| Some(true),
2300 );
2301 assert_eq!(report.resumed, [t.id.clone()]);
2302 let resumed = queue.get(&t.id).unwrap();
2303 assert_eq!(resumed.runs, t.runs);
2304 assert_eq!(resumed.disk_resume_run.as_deref(), Some(run.id.as_str()));
2305 assert_eq!(resumed.attempts, t.attempts);
2306 assert_eq!(resumed.resumption, t.resumption);
2307 assert_eq!(
2308 serde_json::to_value(&resumed.resume_override).unwrap(),
2309 serde_json::to_value(&t.resume_override).unwrap()
2310 );
2311 assert_eq!(
2312 std::fs::read(dir.path().join("runs").join(&run.id).join("run.json")).unwrap(),
2313 before
2314 );
2315 assert!(
2316 run_once_with_probe(
2317 &queue,
2318 &questions,
2319 Some(&cfg),
2320 Timestamp::now(),
2321 true,
2322 |_, _| panic!("no duplicate launch")
2323 )
2324 .is_empty()
2325 );
2326 }
2327
2328 #[test]
2329 fn disk_wait_interrupted_question_supersession_is_idempotent() {
2330 for abandoned in [false, true] {
2331 let (dir, queue, questions) = store();
2332 let cfg = gate_disabled_config(dir.path());
2333 let mut t = disk_task(dir.path().to_owned());
2334 queue.put(&mut t).unwrap();
2335 let q = file_question(
2336 &questions,
2337 &crate::task_chat::InquiryOwnership::default(),
2338 &t,
2339 Bucket::MachineUnknown,
2340 &EN,
2341 Timestamp::now(),
2342 )
2343 .unwrap();
2344 let outbox = crate::task_inquiry::Outbox::at(dir.path());
2345 outbox
2346 .enqueue(crate::task_chat::live_record(dir.path(), &t, &q))
2347 .unwrap();
2348 outbox
2349 .update_decision(&q.id, |d| d.settle(DISK_SUPERSEDED))
2350 .unwrap();
2351 if abandoned {
2352 questions
2353 .update(&q.id, |q| {
2354 q.abandon(DISK_SUPERSEDED);
2355 Ok(())
2356 })
2357 .unwrap();
2358 }
2359 let report = monitor_disk_waits(&queue, &questions, Some(&cfg), Timestamp::now());
2360 assert_eq!(report.resumed, [t.id.clone()]);
2361 assert_eq!(queue.get(&t.id).unwrap().attempts, 3);
2362 assert!(questions.read_history(&q.id).unwrap().answer.is_none());
2363 assert_eq!(
2364 outbox.get(&q.id).unwrap().decision.unwrap().history.len(),
2365 1
2366 );
2367 }
2368 }
2369
2370 #[test]
2371 fn disk_wait_legacy_migration_requires_machine_provenance_and_exact_localized_format() {
2372 let (dir, queue, _) = store();
2373 for (language, source, suffix, expected) in [
2374 ("en", Some(HoldSource::Machine), "", true),
2375 ("ja", Some(HoldSource::Machine), "", true),
2376 ("ja", Some(HoldSource::Manual), "", false),
2377 ("ja", None, "", false),
2378 ("ja", Some(HoldSource::Machine), " owner hold", false),
2379 ] {
2380 let mut t = task("legacy", dir.path().to_owned());
2381 t.hold_machine(Some(format!(
2382 "{}{suffix}",
2383 crate::disk::gate_in(1477050368, 3221225472, language).unwrap()
2384 )));
2385 t.schema = 21;
2386 t.hold_source = source;
2387 queue.put(&mut t).unwrap();
2388 let loaded = queue.get(&t.id).unwrap();
2389 assert_eq!(loaded.machine_hold_cause.is_some(), expected);
2390 assert_eq!(disk_wait_eligible(&loaded), expected);
2391 }
2392 }
2393
2394 #[test]
2395 fn disk_wait_refreshes_stale_notice_wording_without_redelivery() {
2396 let (dir, queue, questions) = store();
2397 let cfg = gate_disabled_config(dir.path());
2398 let mut t = disk_task(dir.path().to_owned());
2399 queue.put(&mut t).unwrap();
2400 let notices = crate::notices::Notices::at(dir.path().join("notifications"));
2401 let key = format!("disk:{}", t.repo.display());
2402 let n = notices
2403 .raise(crate::notices::Notice::warn(
2404 &key,
2405 "Free some, then release it from the queue.",
2406 ))
2407 .unwrap();
2408 notices.mark_read(&n.id).unwrap();
2409 let before = notices.get(&n.id).unwrap();
2410 let report = run_once_with_probe(
2411 &queue,
2412 &questions,
2413 Some(&cfg),
2414 Timestamp::now(),
2415 true,
2416 |_, _| Some(false),
2417 );
2418 assert!(report.is_empty());
2419 let after = notices.get(&n.id).unwrap();
2420 assert!(after.message.contains("automatically"));
2421 assert_eq!(before.generation, after.generation);
2422 assert_eq!(before.count, after.count);
2423 assert_eq!(before.read_at, after.read_at);
2424 }
2425
2426 #[test]
2427 fn disk_wait_owner_discard_is_applied_without_automatic_resumption() {
2428 let (dir, queue, questions) = store();
2429 let cfg = gate_disabled_config(dir.path());
2430 let mut t = disk_task(dir.path().to_owned());
2431 queue.put(&mut t).unwrap();
2432 let q = file_question(
2433 &questions,
2434 &crate::task_chat::InquiryOwnership::default(),
2435 &t,
2436 Bucket::MachineUnknown,
2437 &EN,
2438 Timestamp::now(),
2439 )
2440 .unwrap();
2441 questions
2442 .update(&q.id, |q| q.answer(Answer::Choice(q.choices[2].clone())))
2443 .unwrap();
2444 let report = run_once(&queue, &questions, Some(&cfg), Timestamp::now());
2445 assert!(report.resumed.is_empty());
2446 assert!(queue.get(&t.id).is_err());
2447 assert!(questions.read_history(&q.id).unwrap().answer.is_some());
2448 }
2449
2450 #[test]
2451 fn disk_wait_commit_cannot_overwrite_a_concurrent_owner_hold() {
2452 let (dir, queue, questions) = store();
2453 let mut stale = disk_task(dir.path().to_owned());
2454 queue.put(&mut stale).unwrap();
2455 let mut current = queue.get(&stale.id).unwrap();
2456 current.hold_manual(Some("wait for owner".into()));
2457 queue.put(&mut current).unwrap();
2458 assert!(
2459 resume_disk_task(
2460 &queue,
2461 &questions,
2462 &mut stale,
2463 &crate::task_chat::InquiryOwnership::default()
2464 )
2465 .is_err()
2466 );
2467 assert_eq!(
2468 queue.get(¤t.id).unwrap().hold_source,
2469 Some(HoldSource::Manual)
2470 );
2471 }
2472
2473 #[test]
2474 fn recovery_index_preserves_supersession_and_ambiguous_file_references() {
2475 let (dir, queue, questions) = store();
2476 let mut t = task("supersession", dir.path().join("repo"));
2477 t.id = "task-alpha-end".into();
2478 queue.put(&mut t).unwrap();
2479 let at: Timestamp = "2026-11-01T00:00:00Z".parse().unwrap();
2480 let mut all = Vec::new();
2481 for (i, run) in ["task-alpha-end", "task-alpha", "end"]
2482 .into_iter()
2483 .enumerate()
2484 {
2485 let mut q = Question::new(
2486 run.into(),
2487 NODE.into(),
2488 SEAT.into(),
2489 "superseded".into(),
2490 String::new(),
2491 EN.choices3(),
2492 );
2493 q.id = format!("superseded-{i}");
2494 q.asked_at = at;
2495 all.push(q);
2496 }
2497 let evidence: Vec<_> = all
2498 .iter()
2499 .map(|q| {
2500 (
2501 q.id.clone(),
2502 crate::repair_budget::reconcile::source(q).unwrap(),
2503 )
2504 })
2505 .collect();
2506 t.repair_reconciliations.push(
2507 serde_json::from_value(serde_json::json!({
2508 "request": {
2509 "task": t.id, "run": "run", "head": "head", "expect_pass": 0,
2510 "receipt": "grant", "question": "original", "answer_at": at,
2511 "request": "reconcile", "reason": "fixture"
2512 },
2513 "grant": {
2514 "request": {
2515 "task": t.id, "run": "run", "kind": "review", "rounds": 1,
2516 "head": "head", "expect_pass": 0, "request": "grant", "reason": "fixture"
2517 },
2518 "at": at, "decision": null, "contract": "fixture"
2519 },
2520 "source": "fixture", "application": "fixture", "decision": at,
2521 "superseded_questions": evidence
2522 }))
2523 .unwrap(),
2524 );
2525 queue.put(&mut t).unwrap();
2526 let index = RecoveryIndex::from_scan(&queue, &[t.clone()], all.clone());
2527 assert!(index.latest.is_empty());
2528 std::fs::write(queue.path_of("task-alphabet-other"), "{").unwrap();
2530 std::fs::write(queue.path_of("broken-end"), "{").unwrap();
2531 let index = RecoveryIndex::from_scan(&queue, &[t.clone()], all.clone());
2532 assert!(index.latest(NODE, "end").is_some());
2533 assert!(index.latest(NODE, "task-alpha").is_some());
2534 assert!(index.latest(NODE, &t.id).is_none());
2535 all[0].detail.push_str("changed evidence");
2536 let index = RecoveryIndex::from_scan(&queue, &[t], all);
2537 assert!(index.latest(NODE, "task-alpha-end").is_some());
2538 assert_eq!(questions.list().len(), 0);
2539 }
2540
2541 #[test]
2542 fn ownership_index_requires_positive_foreground_provenance() {
2543 let (dir, queue, _) = store();
2544 let mut run = crate::run::RunState::new(
2545 dir.path().join("repo"),
2546 "main".into(),
2547 "origin".into(),
2548 "fixture".into(),
2549 Config::default(),
2550 );
2551 run.origin = Some(crate::run::Origin::operator());
2552 run.save_under(dir.path()).unwrap();
2553 let foreground = run.id.clone();
2554 run.id = "owned-run".into();
2555 run.origin = Some(crate::run::Origin::queue("missing-task"));
2556 run.save_under(dir.path()).unwrap();
2557 let mut task = task("owner", dir.path().join("repo"));
2558 task.runs = vec!["linked-run".into()];
2559 queue.put(&mut task).unwrap();
2560 let ownership = crate::task_chat::InquiryOwnership::default();
2561 for (run, cwd, node, seat, expected) in [
2562 (foreground.as_str(), true, "node", "seat", true),
2563 (foreground.as_str(), false, "node", "seat", false),
2564 ("owned-run", true, "ask", "operator", false),
2565 ("linked-run", true, "ask", "operator", false),
2566 ("missing-run", true, "node", "seat", false),
2567 ("missing-run", false, "ask", "operator", true),
2568 ] {
2569 let mut q = Question::new(
2570 run.into(),
2571 node.into(),
2572 seat.into(),
2573 "provenance".into(),
2574 String::new(),
2575 Vec::new(),
2576 );
2577 q.cwd = cwd.then(|| dir.path().join("repo").display().to_string());
2578 assert_eq!(standalone_by_scanning(dir.path(), &q), expected);
2579 assert_eq!(
2580 crate::task_chat::standalone_inquiry_in(dir.path(), &q, &ownership),
2581 expected
2582 );
2583 }
2584 }
2585
2586 fn standalone_by_scanning(home: &Path, q: &Question) -> bool {
2588 if q.routed_task.is_some()
2589 || (q.node == crate::bump::NOTICE_NODE && q.seat == "release-watch")
2590 {
2591 return false;
2592 }
2593 if q.legacy_foreground || q.run.is_empty() {
2594 return true;
2595 }
2596 if Queue::at(home.join("queue"))
2597 .list()
2598 .iter()
2599 .any(|task| task.id == q.run || task.runs.contains(&q.run))
2600 {
2601 return false;
2602 }
2603 let run = crate::run::RunState::load_under(&q.run, home).ok();
2604 let origin = run.as_ref().and_then(|run| run.origin.as_ref());
2605 if origin.is_some_and(|origin| origin.task.is_some()) {
2606 return false;
2607 }
2608 (q.node == "ask" && q.seat == "operator")
2609 || (q.cwd.is_some() && origin.is_some_and(|origin| origin.task.is_none()))
2610 }
2611
2612 fn scan_counts() -> (usize, usize, usize) {
2613 (
2614 crate::queue::LIST_READS.with(|c| c.replace(0)),
2615 crate::queue::TASK_READS.with(|c| c.replace(0)),
2616 crate::ask::QUESTION_SCANS.with(|c| c.replace(0)),
2617 )
2618 }
2619
2620 #[test]
2621 fn creating_and_updating_questions_share_one_queue_read() {
2622 let (dir, queue, questions) = store();
2623 let size = 200;
2624 let mut ids = Vec::new();
2625 for i in 0..size {
2626 let mut t = task("scan fixture", dir.path().join("repo"));
2627 t.id = format!("task-{i:04}");
2628 t.runs = vec![format!("run-{i:04}")];
2629 t.hold_manual(Some("held for scan fixture".into()));
2630 queue.put(&mut t).unwrap();
2631 ids.push(t.id);
2632 }
2633 let ownership = crate::task_chat::InquiryOwnership::default();
2634 scan_counts();
2635 let mut created = Vec::new();
2636 for id in &ids {
2637 let mut q = Question::new(
2638 id.clone(),
2639 NODE.into(),
2640 SEAT.into(),
2641 "scan fixture".into(),
2642 String::new(),
2643 EN.choices3(),
2644 );
2645 questions.put_executable_in(&mut q, &ownership).unwrap();
2646 assert_eq!(q.routed_task.as_deref(), Some(id.as_str()));
2647 created.push(q.id);
2648 }
2649 for id in &created {
2650 questions
2651 .update_in(id, &ownership, |q| {
2652 q.answer_delivered = true;
2653 Ok(())
2654 })
2655 .unwrap();
2656 }
2657 let (lists, _, _) = scan_counts();
2658 assert_eq!(lists, 1, "one queue read for {size} creations and updates");
2659 }
2660
2661 #[test]
2662 fn recovery_scans_are_linear_and_match_legacy_decisions() {
2663 let mut previous = None;
2664 for size in [150, 300] {
2665 let (dir, queue, questions) = store();
2666 let at: Timestamp = "2026-11-01T00:00:00Z".parse().unwrap();
2667 let mut sources = Vec::new();
2668 let mut expected_applied = std::collections::BTreeSet::new();
2669 for i in 0..size {
2670 let mut t = task("scan fixture", dir.path().join("repo"));
2671 t.id = format!("task-{i:04}");
2672 t.runs = vec![format!("run-{i:04}")];
2673 t.hold_manual(Some("held for scan fixture".into()));
2674 queue.put(&mut t).unwrap();
2675 let mut q = Question::new(
2676 t.id.clone(),
2677 NODE.into(),
2678 SEAT.into(),
2679 "scan fixture".into(),
2680 String::new(),
2681 EN.choices3(),
2682 );
2683 q.id = format!("question-{i:04}");
2684 q.asked_at = at;
2685 match i % 10 {
2686 0 => {
2687 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
2688 q.accepted_answer_at = q.answered_at;
2689 expected_applied.insert(t.id.clone());
2690 }
2691 1 => q.node = DEPS_NODE.into(),
2692 2 => {
2693 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
2694 }
2695 3 => {
2696 q.run = format!("missing-{i}");
2697 q.legacy_foreground = true;
2698 }
2699 4 => {
2700 q.run.clear();
2701 q.node = crate::bump::NOTICE_NODE.into();
2702 q.seat = "release-watch".into();
2703 }
2704 5 => {
2705 q.run = format!("missing-{i}");
2706 q.node = "ask".into();
2707 q.seat = "operator".into();
2708 }
2709 6 => q.run = t.runs[0].clone(),
2710 7 => q.run = format!("missing-{i}"),
2711 8 => q.abandon("fixture"),
2712 _ => {}
2713 }
2714 questions.put(&mut q).unwrap();
2715 sources.push(q.id.clone());
2716 if i % 10 == 9 {
2717 q.id = format!("question-{i:04}-tie");
2718 questions.put(&mut q).unwrap();
2719 sources.push(q.id.clone());
2720 }
2721 }
2722 let migration = dir.path().join("notifications/task-chat-migration-v1");
2723 std::fs::create_dir_all(&migration).unwrap();
2724 std::fs::write(
2725 migration.join("state.json"),
2726 serde_json::to_vec(&serde_json::json!({
2727 "schema": 1, "classification_revision": 1, "pending": [], "sources": sources
2728 }))
2729 .unwrap(),
2730 )
2731 .unwrap();
2732
2733 scan_counts();
2734 let all = questions.list_for_effects();
2735 let (lists, reads, scans) = scan_counts();
2736 assert_eq!((lists, reads, scans), (1, size, 1));
2737 let tasks = queue.list();
2738 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
2739 for q in &all {
2740 assert_eq!(
2741 crate::task_chat::standalone_inquiry_in(dir.path(), q, &ownership),
2742 standalone_by_scanning(dir.path(), q),
2743 "{}",
2744 q.id,
2745 );
2746 if q.id.ends_with("0002") {
2747 assert!(q.answer.is_none(), "saved consent remains fenced");
2748 }
2749 }
2750 let recovery = RecoveryIndex::from_scan(&queue, &tasks, all.clone());
2751 for task in &tasks {
2752 for node in [NODE, DEPS_NODE] {
2753 let expected = all
2754 .iter()
2755 .filter(|q| {
2756 q.node == node
2757 && q.run == task.id
2758 && !queue.get(&q.run).ok().is_some_and(|t| {
2759 crate::repair_budget::reconcile::superseded_question(&t, q)
2760 })
2761 })
2762 .max_by(|a, b| a.asked_at.cmp(&b.asked_at).then_with(|| a.id.cmp(&b.id)));
2763 assert_eq!(
2764 recovery.latest(node, &task.id).map(|q| q.id),
2765 expected.map(|q| q.id.clone())
2766 );
2767 }
2768 }
2769
2770 scan_counts();
2771 crate::daemon::apply_task_choice_actions(&queue, &questions, dir.path());
2772 assert_eq!(
2773 scan_counts(),
2774 (1, size, 1),
2775 "choice-action rechecks share ownership"
2776 );
2777
2778 scan_counts();
2779 let applied: std::collections::BTreeSet<_> =
2780 apply_answers(&queue, &questions).into_iter().collect();
2781 let counts = scan_counts();
2782 assert_eq!(applied, expected_applied);
2783 assert!(counts.0 <= 3, "queue scans: {counts:?}");
2784 assert!(counts.1 <= 4 * size, "task reads: {counts:?}");
2785 assert_eq!(counts.2, 2, "question scans: {counts:?}");
2786 if let Some((_, reads, scans)) = previous {
2787 assert!(counts.1 <= 2 * reads);
2788 assert_eq!(counts.2, scans);
2789 }
2790 previous = Some(counts);
2791
2792 let config = gate_disabled_config(dir.path());
2793 scan_counts();
2794 run_once(&queue, &questions, Some(&config), Timestamp::now());
2795 let lap = scan_counts();
2796 assert!(lap.0 <= 12, "triage queue scans: {lap:?}");
2797 assert!(lap.1 <= 14 * size, "triage task reads: {lap:?}");
2798 assert!(lap.2 <= 6, "triage question scans: {lap:?}");
2799 }
2800 }
2801
2802 #[test]
2803 fn run_once_filing_many_new_questions_reads_the_queue_a_constant_number_of_times() {
2804 let (dir, queue, questions) = store();
2805 let size = 200;
2806 for i in 0..size {
2807 let mut t = task("scan fixture", dir.path().join("repo"));
2808 t.id = format!("task-{i:04}");
2809 t.hold_machine(Some("gate red".to_owned()));
2810 queue.put(&mut t).unwrap();
2811 }
2812 scan_counts();
2813 let report = run_once(&queue, &questions, None, Timestamp::now());
2814 let (lists, _, _) = scan_counts();
2815 assert_eq!(report.asked.len(), size, "every held task is asked about");
2816 assert!(
2817 lists <= 12,
2818 "queue scans while filing {size} questions: {lists}"
2819 );
2820 }
2821
2822 #[test]
2823 fn base_recovery_stops_suppress_generic_and_dependency_resume_offers() {
2824 let (dir, queue, questions) = store();
2825 let mut root = task("base stop", dir.path().to_path_buf());
2826 let mut run = crate::run::RunState::new(
2827 dir.path().to_path_buf(),
2828 "main".into(),
2829 "base".into(),
2830 "retained implementation".into(),
2831 Config::default(),
2832 );
2833 run.status = crate::run::RunStatus::Voting;
2834 run.base_sync = Some(crate::run::BaseSync {
2835 tip: "tip".into(),
2836 behind: 3,
2837 attempts: 4,
2838 conflict: Some("persisted exhaustion".into()),
2839 already_in: None,
2840 });
2841 run.save_under(dir.path()).unwrap();
2842 root.link_run(&run.id);
2843 root.hold_machine(Some("base stop".into()));
2844 queue.put(&mut root).unwrap();
2845 let mut dependent = task("dependent", dir.path().to_path_buf());
2846 dependent.blocked_by = vec![root.id.clone()];
2847 dependent.status = TaskStatus::Blocked;
2848 queue.put(&mut dependent).unwrap();
2849 for _ in 0..2 {
2850 let report = run_once(&queue, &questions, None, Timestamp::now());
2851 assert!(report.asked.is_empty());
2852 assert!(report.resumed.is_empty());
2853 }
2854 let q = file_question(
2857 &questions,
2858 &crate::task_chat::InquiryOwnership::default(),
2859 &root,
2860 Bucket::MachineUnknown,
2861 &EN,
2862 Timestamp::now(),
2863 )
2864 .unwrap();
2865 let mut q = q;
2866 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
2867 questions.put_executable(&mut q).unwrap();
2868 apply_answers(&queue, &questions);
2869 let saved = queue.get(&root.id).unwrap();
2870 assert_eq!(saved.status, TaskStatus::Held);
2871 assert!(saved.triage_applied(&q.id));
2872 assert_eq!(saved.runs, vec![run.id]);
2873 }
2874
2875 #[test]
2876 fn resource_holds_only_auto_release_a_proven_cache_wait_timeout() {
2877 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
2878 let dir = tempfile::tempdir().unwrap();
2879 let cfg = Config::default(); assert!(cfg.cache_dir().is_none());
2881 let mut state = RunState::new(
2882 dir.path().into(),
2883 "main".into(),
2884 "base".into(),
2885 "verify".into(),
2886 cfg.clone(),
2887 );
2888 let mut task = task("verify", dir.path().into());
2889 task.runs = vec![state.id.clone()];
2890 task.hold_machine(Some("[resource] cache operation stopped".into()));
2891 for cause in [
2892 ResourceBlockCause::Other,
2893 ResourceBlockCause::Unrecorded,
2894 ResourceBlockCause::Cancelled { park: None },
2895 ] {
2896 state.resource_block = Some(ResourceBlock {
2897 node: "gate".into(),
2898 kind: "shared build cache".into(),
2899 detail: "persistent metadata or manifest error".into(),
2900 cause,
2901 at: Timestamp::now(),
2902 });
2903 state.save_under(dir.path()).unwrap();
2904 for _ in 0..3 {
2906 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2907 }
2908 }
2909 let block = state.resource_block.as_mut().unwrap();
2910 block.cause = ResourceBlockCause::Timeout {
2911 waited_secs: 10,
2912 limit_secs: 10,
2913 };
2914 state.save_under(dir.path()).unwrap();
2915 assert_eq!(
2916 machine_cause_resolved_under(&task, &cfg, dir.path()),
2917 Some(true)
2918 );
2919 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
2920 state.save_under(dir.path()).unwrap();
2921 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2922 task.runs = vec!["unreadable-run".into()];
2923 assert_eq!(machine_cause_resolved_under(&task, &cfg, dir.path()), None);
2924 }
2925
2926 #[cfg(unix)]
2927 #[test]
2928 fn disk_admission_holds_recheck_live_reservations_and_the_floor() {
2929 use crate::retention::OutputLease;
2930 use crate::run::{ResourceBlock, ResourceBlockCause, RunState};
2931 let temp = tempfile::tempdir().unwrap();
2932 let home = temp.path().canonicalize().unwrap();
2933 let mut cfg = Config::default();
2934 cfg.graph.worktree_root = Some(home.join("worktrees"));
2935 cfg.disk.min_free_bytes = 100;
2936 let cache = home.join("cache");
2937 cfg.verify.gate = vec![format!(
2938 "CARGO_TARGET_DIR='{}' cargo build",
2939 cache.display()
2940 )];
2941 let mut state = RunState::new(
2942 home.clone(),
2943 "main".into(),
2944 "base".into(),
2945 "verify".into(),
2946 cfg.clone(),
2947 );
2948 let saved_cache = home.join("saved-cache");
2949 state.config.verify.gate = vec![format!(
2950 "CARGO_TARGET_DIR='{}' cargo build",
2951 saved_cache.display()
2952 )];
2953 state.resource_block = Some(ResourceBlock {
2954 node: "gate".into(),
2955 kind: "disk admission".into(),
2956 detail: "writer headroom unavailable".into(),
2957 cause: ResourceBlockCause::Other,
2958 at: Timestamp::now(),
2959 });
2960 state.save_under(&home).unwrap();
2961 let mut task = task("verify", home.clone());
2962 task.runs = vec![state.id.clone()];
2963 task.hold_machine(Some(format!(
2964 "{}{}; no attempt was spent; resume to retry the affected operation",
2965 crate::cache::RESOURCE_HOLD,
2966 state.resource_block.as_ref().unwrap().description()
2967 )));
2968 task.schema = 21; let gib = 1024 * 1024 * 1024;
2970 let free = std::cell::Cell::new(100 + gib - 1);
2971 let check = || {
2972 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths, floor| {
2973 assert!(
2974 paths.contains(&cache),
2975 "the cache volume must also be checked"
2976 );
2977 assert!(
2978 paths.contains(&saved_cache),
2979 "saved and current cache locations both matter"
2980 );
2981 assert!(paths.contains(&state.worktree_root()));
2982 OutputLease::admit_with(&home, paths, floor, |_| Ok(free.get())).is_ok()
2983 })
2984 };
2985 assert_eq!(check(), Some(false), "the unchanged floor still applies");
2986 assert_eq!(
2987 machine_cause_resolved_with_admission(&task, &cfg, &home, |paths, floor| {
2988 OutputLease::admit_with(&home, paths, floor, |_| {
2989 Err(anyhow::anyhow!("measurement unavailable"))
2990 })
2991 .is_ok()
2992 }),
2993 Some(false),
2994 "failed output-volume measurements must remain closed"
2995 );
2996 free.set(100 + gib);
2997 let writer = OutputLease::admit_with(
2998 &home,
2999 std::slice::from_ref(&home),
3000 cfg.disk.min_free_bytes,
3001 |_| Ok(free.get()),
3002 )
3003 .unwrap();
3004 assert_eq!(
3005 check(),
3006 Some(false),
3007 "df alone cannot resolve a live reservation"
3008 );
3009 drop(writer);
3010 assert_eq!(
3011 check(),
3012 Some(true),
3013 "releasing headroom resolves the machine hold"
3014 );
3015 assert_eq!(
3016 check(),
3017 Some(true),
3018 "the probe must release its temporary reservation"
3019 );
3020
3021 let namespace = home.join("disk-admission");
3023 let saved = home.join("saved-admission");
3024 std::fs::rename(&namespace, &saved).unwrap();
3025 std::os::unix::fs::symlink(&saved, &namespace).unwrap();
3026 assert_eq!(check(), Some(false));
3027 std::fs::remove_file(&namespace).unwrap();
3028 std::fs::rename(saved, namespace).unwrap();
3029 assert_eq!(check(), Some(true));
3030
3031 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Cancelled { park: None };
3032 state.save_under(&home).unwrap();
3033 assert_eq!(
3034 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
3035 "a cancelled operation must remain held"
3036 )),
3037 None
3038 );
3039 state.resource_block.as_mut().unwrap().kind = "incremental cache admission".into();
3040 state.resource_block.as_mut().unwrap().cause = ResourceBlockCause::Other;
3041 state.save_under(&home).unwrap();
3042 assert_eq!(
3043 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
3044 "unproven incremental caches must remain held"
3045 )),
3046 None
3047 );
3048 task.runs.push("unreadable-run".into());
3049 assert_eq!(
3050 machine_cause_resolved_with_admission(&task, &cfg, &home, |_, _| panic!(
3051 "missing evidence must not release a hold"
3052 )),
3053 None
3054 );
3055 }
3056
3057 fn gate_disabled_config(dir: &std::path::Path) -> PathBuf {
3061 let config = dir.join("magi.toml");
3062 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
3063 config
3064 }
3065
3066 #[test]
3067 fn resumption_later_keep_held_answer_wins_but_an_older_answer_cannot_undo_release() {
3068 for source in ["cli", "web"] {
3069 for answer_after_release in [true, false] {
3070 let (dir, queue, questions) = store();
3071 let mut t = task("recovery", dir.path().join("repo"));
3072 t.hold_machine(Some("inspect remote first".into()));
3073 queue.put(&mut t).unwrap();
3074 let mut q = file_question(
3075 &questions,
3076 &crate::task_chat::InquiryOwnership::default(),
3077 &t,
3078 Bucket::MachineUnknown,
3079 &JA,
3080 Timestamp::now(),
3081 )
3082 .unwrap();
3083 if !answer_after_release {
3084 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
3085 questions.put(&mut q).unwrap();
3086 }
3087 t.release_operator(source, None);
3088 queue.put(&mut t).unwrap();
3089 if answer_after_release {
3090 q.answer(Answer::Choice(q.choices[1].clone())).unwrap();
3091 questions.put(&mut q).unwrap();
3092 }
3093 apply_answers(&queue, &questions);
3094 let after = queue.get(&t.id).unwrap();
3095 assert!(after.triage_applied(&q.id));
3096 assert_eq!(after.operator_held(), answer_after_release);
3097 assert_eq!(
3098 after.status,
3099 if answer_after_release {
3100 TaskStatus::Held
3101 } else {
3102 TaskStatus::Queued
3103 }
3104 );
3105 assert!(apply_answers(&queue, &questions).is_empty());
3106 }
3107 }
3108 }
3109
3110 #[test]
3111 fn explicit_resumption_applies_legacy_queued_triage_once_and_preserves_later_holds() {
3112 let (dir, queue, questions) = store();
3113 let mut t = task("recover", dir.path().join("repo"));
3114 t.hold_machine(Some("inspect first".into()));
3115 queue.put(&mut t).unwrap();
3116 let mut q = file_question(
3117 &questions,
3118 &crate::task_chat::InquiryOwnership::default(),
3119 &t,
3120 Bucket::MachineUnknown,
3121 &JA,
3122 Timestamp::now(),
3123 )
3124 .unwrap();
3125 assert!(q.actions.is_empty());
3126 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
3127 questions.put(&mut q).unwrap();
3128 t.release();
3130 t.instruction = "Reuse approved PR head".into();
3131 queue.put(&mut t).unwrap();
3132 assert_eq!(apply_answers(&queue, &questions), [t.id.clone()]);
3133 let mut after = queue.get(&t.id).unwrap();
3134 assert_eq!(
3135 after.resumption.as_ref().unwrap().question_id.as_deref(),
3136 Some(q.id.as_str())
3137 );
3138 assert_eq!(
3139 after.resumption.as_ref().unwrap().instruction,
3140 t.instruction
3141 );
3142 assert_eq!(after.answers[0].answer, JA.resume);
3143 assert!(after.triage_applied(&q.id));
3144 assert!(apply_answers(&Queue::at(dir.path().join("queue")), &questions).is_empty());
3145 after.hold_machine(Some("new blocker".into()));
3146 queue.put(&mut after).unwrap();
3147 let mut late = file_question(
3148 &questions,
3149 &crate::task_chat::InquiryOwnership::default(),
3150 &after,
3151 Bucket::MachineUnknown,
3152 &EN,
3153 Timestamp::now(),
3154 )
3155 .unwrap();
3156 after.hold_manual(Some("keep held now".into()));
3157 queue.put(&mut after).unwrap();
3158 late.answer(Answer::Choice(late.choices[0].clone()))
3159 .unwrap();
3160 questions.put(&mut late).unwrap();
3161 apply_answers(&queue, &questions);
3162 let held = queue.get(&t.id).unwrap();
3163 assert!(held.operator_held());
3164 assert_eq!(held.hold_reason.as_deref(), Some("keep held now"));
3165 assert_eq!(held.resumption.as_ref().unwrap().generation, 1);
3166 }
3167
3168 #[test]
3169 fn a_resolved_machine_hold_is_requeued_automatically() {
3170 let (dir, q, questions) = store();
3171 let config = gate_disabled_config(dir.path());
3172 let mut t = task("disk pressure", dir.path().join("repo"));
3173 t.hold_machine(Some(
3174 "not enough free space to start a run: 10 bytes free, 100 required by \
3175 `[disk] min_free_bytes`"
3176 .to_owned(),
3177 ));
3178 t.schema = 21; q.put(&mut t).unwrap();
3180
3181 let report = run_once(&q, &questions, Some(&config), Timestamp::now());
3182 assert_eq!(report.resumed, [t.id.clone()]);
3183 assert!(report.asked.is_empty());
3184
3185 let back = q.get(&t.id).unwrap();
3186 assert_eq!(back.status, TaskStatus::Queued);
3187 assert!(back.hold_source.is_none());
3188 assert!(questions.list().is_empty(), "nothing needed asking");
3189 }
3190
3191 #[test]
3192 fn a_machine_hold_with_no_recognised_cause_gets_one_question_not_two() {
3193 let (dir, q, questions) = store();
3194 let mut t = task("gate went red", dir.path().join("repo"));
3195 t.hold_machine(Some("gate red".to_owned()));
3196 q.put(&mut t).unwrap();
3197
3198 let first = run_once(&q, &questions, None, Timestamp::now());
3199 assert_eq!(first.asked, [t.id.clone()]);
3200 assert!(first.resumed.is_empty());
3201
3202 let open: Vec<_> = questions
3203 .list()
3204 .into_iter()
3205 .filter(|q| q.status.open())
3206 .collect();
3207 assert_eq!(open.len(), 1);
3208 assert_eq!(open[0].run, t.id);
3209 assert_eq!(open[0].node, NODE);
3210 assert_eq!(open[0].choices.len(), 3);
3211
3212 let second = run_once(&q, &questions, None, Timestamp::now());
3214 assert!(second.asked.is_empty());
3215 assert_eq!(
3216 questions
3217 .list()
3218 .into_iter()
3219 .filter(|q| q.status.open())
3220 .count(),
3221 1
3222 );
3223 }
3224
3225 #[test]
3226 fn a_legacy_hold_with_no_recorded_source_gets_exactly_one_question() {
3227 let (dir, q, questions) = store();
3228 let mut t = task("schema 1 record", dir.path().join("repo"));
3229 t.status = TaskStatus::Held;
3230 assert!(t.hold_source.is_none(), "the case this test is about");
3231 q.put(&mut t).unwrap();
3232
3233 let first = run_once(&q, &questions, None, Timestamp::now());
3234 assert_eq!(first.asked, [t.id.clone()]);
3235
3236 let second = run_once(&q, &questions, None, Timestamp::now());
3237 assert!(
3238 second.asked.is_empty(),
3239 "the same legacy hold must not be asked about twice"
3240 );
3241 assert_eq!(
3242 questions
3243 .list()
3244 .into_iter()
3245 .filter(|q| q.status.open())
3246 .count(),
3247 1
3248 );
3249 }
3250
3251 #[test]
3252 fn dependency_resolution_detaches_legacy_short_refs_without_dropping_other_waits() {
3253 let (dir, queue, questions) = store();
3254 let mut root = task("root", dir.path().join("repo"));
3255 root.id = "20261013-193611-ca5b".into();
3256 root.hold_machine(Some("needs owner decision".into()));
3257 queue.put(&mut root).unwrap();
3258 let mut other = task("other", dir.path().join("repo"));
3259 other.id = "20261013-192213-feb2".into();
3260 queue.put(&mut other).unwrap();
3261 let mut waiting = task("waiting", dir.path().join("repo"));
3262 waiting.block(vec!["ca5b".into(), other.id.clone()], None);
3263 queue.put(&mut waiting).unwrap();
3264 detach_dependants(&queue, &questions, &root.id);
3265 let after = queue.get(&waiting.id).unwrap();
3266 assert_eq!(after.status, TaskStatus::Blocked);
3267 assert_eq!(after.blocked_by, [other.id]);
3268 }
3269
3270 #[test]
3271 fn a_manual_hold_is_never_auto_resumed() {
3272 let (dir, q, questions) = store();
3273 let mut t = task("operator stopped this", dir.path().join("repo"));
3274 t.hold_manual(Some("waiting on a decision".to_owned()));
3275 q.put(&mut t).unwrap();
3276
3277 let report = run_once(&q, &questions, None, Timestamp::now());
3278 assert!(report.resumed.is_empty());
3279 assert!(report.asked.is_empty());
3281
3282 let back = q.get(&t.id).unwrap();
3283 assert_eq!(back.status, TaskStatus::Held);
3284 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3285 assert!(questions.list().is_empty());
3286 }
3287
3288 #[test]
3289 fn a_blocked_task_on_a_deleted_dependency_is_held_and_asked_about_in_one_pass() {
3290 let (dir, q, questions) = store();
3295 let mut still_going = task("still valid", dir.path().join("repo"));
3296 q.put(&mut still_going).unwrap();
3297
3298 let mut t = task("orphaned", dir.path().join("repo"));
3299 t.block(
3300 vec!["20260101-000000-gone".to_owned(), still_going.id.clone()],
3301 Some("waits on both".to_owned()),
3302 );
3303 q.put(&mut t).unwrap();
3304
3305 let report = run_once(&q, &questions, None, Timestamp::now());
3306 assert_eq!(report.quarantined, [t.id.clone()]);
3307 assert_eq!(
3308 report.asked,
3309 [t.id.clone()],
3310 "the fresh machine hold must earn a question in the same pass"
3311 );
3312
3313 let after = q.get(&t.id).unwrap();
3314 assert_eq!(after.status, TaskStatus::Held);
3315 assert_eq!(after.hold_source, Some(HoldSource::Machine));
3316 assert!(after.blocked_by.is_empty());
3317
3318 assert!(!is_disk_hold(&after));
3321
3322 let open: Vec<_> = questions
3323 .list()
3324 .into_iter()
3325 .filter(|q| q.status.open())
3326 .collect();
3327 assert_eq!(open.len(), 1);
3328 assert_eq!(open[0].run, t.id);
3329
3330 let second = run_once(&q, &questions, None, Timestamp::now());
3332 assert!(second.quarantined.is_empty());
3333 assert!(second.asked.is_empty());
3334 }
3335
3336 fn stuck_chain(q: &Queue, dir: &std::path::Path) -> (Task, Task, Task) {
3339 let mut root = task("root", dir.join("repo"));
3340 root.hold_manual(Some("waiting".to_owned()));
3341 q.put(&mut root).unwrap();
3342 let mut mid = task("mid", dir.join("repo"));
3343 mid.block(vec![root.id.clone()], None);
3344 q.put(&mut mid).unwrap();
3345 let mut leaf = task("leaf", dir.join("repo"));
3346 leaf.block(vec![mid.id.clone()], None);
3347 q.put(&mut leaf).unwrap();
3348 (root, mid, leaf)
3349 }
3350
3351 fn deps_questions(questions: &Questions) -> Vec<Question> {
3352 questions
3353 .list()
3354 .into_iter()
3355 .filter(|q| q.node == DEPS_NODE)
3356 .collect()
3357 }
3358
3359 fn answer_deps(questions: &Questions, choice: usize) {
3360 let mut asked = deps_questions(questions).remove(0);
3361 let c = asked.choices[choice].clone();
3362 asked.answer(Answer::Choice(c)).unwrap();
3363 questions.put(&mut asked).unwrap();
3364 }
3365
3366 #[test]
3367 fn a_stuck_chain_earns_one_question_naming_every_dependant_and_is_not_reasked() {
3368 let (dir, q, questions) = store();
3369 let (root, mid, leaf) = stuck_chain(&q, dir.path());
3370
3371 let first = run_once(&q, &questions, None, Timestamp::now());
3372 assert_eq!(first.asked, std::slice::from_ref(&root.id));
3373 let asked = deps_questions(&questions);
3374 assert_eq!(asked.len(), 1, "one question per root, not per dependant");
3375 assert_eq!(asked[0].run, root.id);
3376 assert_eq!(asked[0].choices.len(), 3);
3377 assert!(asked[0].detail.contains(mid.short()), "{}", asked[0].detail);
3378 assert!(
3379 asked[0].detail.contains(leaf.short()),
3380 "{}",
3381 asked[0].detail
3382 );
3383
3384 let second = run_once(&q, &questions, None, Timestamp::now());
3385 assert!(second.asked.is_empty());
3386 assert_eq!(deps_questions(&questions).len(), 1);
3387 assert!(pending_for(&questions, &q.get(&root.id).unwrap()));
3388 }
3389
3390 #[test]
3391 fn a_stuck_root_with_a_machine_hold_gets_only_the_dependency_question() {
3392 let (dir, q, questions) = store();
3393 let mut root = task("root", dir.path().join("repo"));
3394 root.hold_machine(Some("gate red".to_owned()));
3395 q.put(&mut root).unwrap();
3396 let mut dep = task("dep", dir.path().join("repo"));
3397 dep.block(vec![root.id.clone()], None);
3398 q.put(&mut dep).unwrap();
3399
3400 run_once(&q, &questions, None, Timestamp::now());
3401 run_once(&q, &questions, None, Timestamp::now());
3402 assert_eq!(deps_questions(&questions).len(), 1);
3403 assert_eq!(
3404 questions.list().len(),
3405 1,
3406 "no second question about the hold"
3407 );
3408 }
3409
3410 #[test]
3411 fn a_dependency_that_can_still_run_asks_nothing() {
3412 let (dir, q, questions) = store();
3413 let mut live = task("live", dir.path().join("repo"));
3414 q.put(&mut live).unwrap();
3415 let mut dep = task("dep", dir.path().join("repo"));
3416 dep.block(vec![live.id.clone()], None);
3417 q.put(&mut dep).unwrap();
3418 let report = run_once(&q, &questions, None, Timestamp::now());
3419 assert!(report.asked.is_empty());
3420 assert!(questions.list().is_empty());
3421 }
3422
3423 #[test]
3424 fn answering_release_requeues_the_root_and_is_not_reasked() {
3425 let (dir, q, questions) = store();
3426 let (root, _mid, _leaf) = stuck_chain(&q, dir.path());
3427 run_once(&q, &questions, None, Timestamp::now());
3428 answer_deps(&questions, 0);
3429
3430 let report = run_once(&q, &questions, None, Timestamp::now());
3431 assert_eq!(report.answered, std::slice::from_ref(&root.id));
3432 assert!(report.asked.is_empty(), "applying must not re-ask");
3433 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Queued);
3434 let again = run_once(&q, &questions, None, Timestamp::now());
3435 assert!(again.asked.is_empty() && again.answered.is_empty());
3436 assert_eq!(deps_questions(&questions).len(), 1);
3437 }
3438
3439 #[test]
3440 fn answering_detach_frees_the_direct_dependant_and_keeps_the_root() {
3441 let (dir, q, questions) = store();
3442 let (root, mid, leaf) = stuck_chain(&q, dir.path());
3443 run_once(&q, &questions, None, Timestamp::now());
3444 answer_deps(&questions, 2);
3445
3446 let report = run_once(&q, &questions, None, Timestamp::now());
3447 assert!(report.asked.is_empty());
3448 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
3449 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
3450 let leaf = q.get(&leaf.id).unwrap();
3451 assert_eq!(leaf.status, TaskStatus::Blocked, "still waits on mid");
3452 assert_eq!(leaf.blocked_by, std::slice::from_ref(&mid.id));
3453 }
3454
3455 #[test]
3456 fn a_dependant_that_can_progress_through_another_blocker_is_not_stuck() {
3457 let (dir, q, questions) = store();
3458 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3459 let mut live = task("live", dir.path().join("repo"));
3460 q.put(&mut live).unwrap();
3461 let mut both = q.get(&mid.id).unwrap();
3462 both.block(vec![root.id.clone(), live.id.clone()], None);
3463 q.put(&mut both).unwrap();
3464 assert!(
3466 run_once(&q, &questions, None, Timestamp::now())
3467 .asked
3468 .is_empty()
3469 );
3470 }
3471
3472 #[test]
3473 fn answering_discard_detaches_then_removes_the_root() {
3474 let (dir, q, questions) = store();
3475 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3476 run_once(&q, &questions, None, Timestamp::now());
3477 answer_deps(&questions, 1);
3478
3479 let report = run_once(&q, &questions, None, Timestamp::now());
3480 assert!(q.get(&root.id).is_err());
3481 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Queued);
3482 assert!(
3483 report.asked.is_empty(),
3484 "no per-dependant quarantine question"
3485 );
3486 assert!(report.quarantined.is_empty());
3487 }
3488
3489 #[test]
3490 fn an_unmatched_deps_answer_changes_nothing_and_is_not_reasked() {
3491 let (dir, q, questions) = store();
3492 let (root, mid, _leaf) = stuck_chain(&q, dir.path());
3493 run_once(&q, &questions, None, Timestamp::now());
3494 let mut asked = deps_questions(&questions).remove(0);
3495 asked.choices.clear();
3496 asked.answer(Answer::Text("dunno".to_owned())).unwrap();
3497 questions.put(&mut asked).unwrap();
3498
3499 for _ in 0..2 {
3500 let report = run_once(&q, &questions, None, Timestamp::now());
3501 assert!(report.asked.is_empty() && report.answered.is_empty());
3502 }
3503 assert_eq!(deps_questions(&questions).len(), 1);
3504 assert_eq!(q.get(&root.id).unwrap().status, TaskStatus::Held);
3505 assert_eq!(q.get(&mid.id).unwrap().status, TaskStatus::Blocked);
3506 }
3507
3508 #[test]
3509 fn a_dependency_cycle_terminates_and_is_asked_about_once() {
3510 let (dir, q, questions) = store();
3511 let mut a = task("a", dir.path().join("repo"));
3512 let mut b = task("b", dir.path().join("repo"));
3513 a.block(vec![b.id.clone()], None);
3514 b.block(vec![a.id.clone()], None);
3515 q.put(&mut a).unwrap();
3516 q.put(&mut b).unwrap();
3517
3518 let first = run_once(&q, &questions, None, Timestamp::now());
3519 assert_eq!(first.asked.len(), 1);
3520 let second = run_once(&q, &questions, None, Timestamp::now());
3521 assert!(second.asked.is_empty());
3522 assert_eq!(deps_questions(&questions).len(), 1);
3523 }
3524
3525 #[test]
3526 fn a_missing_dependency_is_still_quarantined_not_asked_about_as_stuck() {
3527 let (dir, q, questions) = store();
3528 let mut t = task("orphan", dir.path().join("repo"));
3529 t.block(vec!["20260101-000000-gone".to_owned()], None);
3530 q.put(&mut t).unwrap();
3531 let report = run_once(&q, &questions, None, Timestamp::now());
3532 assert_eq!(report.quarantined, [t.id.clone()]);
3533 assert!(deps_questions(&questions).is_empty());
3534 }
3535
3536 #[test]
3537 fn a_stale_manual_hold_can_be_discarded() {
3538 let (dir, q, questions) = store();
3539 let mut t = task("been sitting a while", dir.path().join("repo"));
3540 t.hold_manual(Some("waiting on a decision".to_owned()));
3541 q.put(&mut t).unwrap();
3542 let mut back = q.get(&t.id).unwrap();
3544 back.updated_at = Timestamp::now() - SignedDuration::new(8 * 24 * 60 * 60, 0);
3545 std::fs::write(
3546 q.path_of(&back.id),
3547 serde_json::to_string_pretty(&back).unwrap(),
3548 )
3549 .unwrap();
3550
3551 let report = run_once(&q, &questions, None, Timestamp::now());
3552 assert_eq!(report.asked, [t.id.clone()]);
3553 let open: Vec<_> = questions
3554 .list()
3555 .into_iter()
3556 .filter(|q| q.status.open())
3557 .collect();
3558 assert_eq!(open.len(), 1);
3559 let mut asked = open[0].clone();
3560 assert_eq!(asked.choices, ["resume it", "keep it held", "discard it"]);
3561 assert_eq!(
3562 JA.choices_manual(),
3563 ["再開する", "まだ止めておく", "捨ててよい"]
3564 );
3565 asked
3566 .answer(Answer::Choice(asked.choices[2].clone()))
3567 .unwrap();
3568 questions.put(&mut asked).unwrap();
3569 run_once(&q, &questions, None, Timestamp::now());
3570 assert!(q.get(&t.id).is_err());
3571 }
3572
3573 #[test]
3574 fn answering_resume_releases_the_task() {
3575 let (dir, q, questions) = store();
3576 let mut t = task("gate went red", dir.path().join("repo"));
3577 t.hold_machine(Some("gate red".to_owned()));
3578 q.put(&mut t).unwrap();
3579 run_once(&q, &questions, None, Timestamp::now());
3580
3581 let mut asked = questions
3582 .list()
3583 .into_iter()
3584 .find(|q| q.run == t.id)
3585 .unwrap();
3586 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3587 questions.put(&mut asked).unwrap();
3588
3589 let report = run_once(&q, &questions, None, Timestamp::now());
3590 assert_eq!(report.answered, [t.id.clone()]);
3591 let back = q.get(&t.id).unwrap();
3592 assert_eq!(back.status, TaskStatus::Queued);
3593 assert!(back.hold_source.is_none());
3594 }
3595
3596 fn resumed_then_failed(q: &Queue, questions: &Questions, dir: &std::path::Path) -> Task {
3598 let mut t = task("gate went red", dir.join("repo"));
3599 t.hold_machine(Some("gate red".to_owned()));
3600 q.put(&mut t).unwrap();
3601 run_once(q, questions, None, Timestamp::now());
3602 let mut asked = questions
3603 .list()
3604 .into_iter()
3605 .find(|q| q.run == t.id)
3606 .unwrap();
3607 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3608 questions.put(&mut asked).unwrap();
3609
3610 let report = run_once(q, questions, None, Timestamp::now());
3611 assert_eq!(report.answered, [t.id.clone()]);
3612 let mut back = q.get(&t.id).unwrap();
3613 assert_eq!(back.status, TaskStatus::Queued);
3614 assert_eq!(back.attempts, 0);
3615
3616 back.start("run-1".to_owned());
3617 back.fail("rebase conflict", 1);
3618 assert_eq!(back.status, TaskStatus::Held);
3619 q.put(&mut back).unwrap();
3620 back
3621 }
3622
3623 #[test]
3624 fn a_resume_answer_is_applied_once_and_a_new_machine_hold_is_asked_about() {
3625 let (dir, q, questions) = store();
3626 let t = resumed_then_failed(&q, &questions, dir.path());
3627
3628 let report = run_once(&q, &questions, None, Timestamp::now());
3629 assert!(report.answered.is_empty(), "the old answer must not replay");
3630 assert_eq!(report.asked, std::slice::from_ref(&t.id));
3631 let back = q.get(&t.id).unwrap();
3632 assert_eq!(back.status, TaskStatus::Held);
3633 assert_eq!(
3634 questions
3635 .list()
3636 .into_iter()
3637 .filter(|q| q.status.open())
3638 .count(),
3639 1
3640 );
3641 }
3642
3643 #[test]
3644 fn a_manual_hold_placed_after_a_resume_is_not_undone_by_the_old_answer() {
3645 let (dir, q, questions) = store();
3646 let mut t = resumed_then_failed(&q, &questions, dir.path());
3647 t.hold_manual(Some("operator stopped this".to_owned()));
3648 q.put(&mut t).unwrap();
3649 let before = questions.list().len();
3650
3651 let report = run_once(&q, &questions, None, Timestamp::now());
3652 assert!(report.answered.is_empty());
3653 assert!(report.asked.is_empty());
3654 let back = q.get(&t.id).unwrap();
3655 assert_eq!(back.status, TaskStatus::Held);
3656 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3657 assert_eq!(questions.list().len(), before);
3658 }
3659
3660 #[test]
3661 fn answering_not_yet_keeps_it_held_as_a_manual_hold_and_does_not_reapply() {
3662 let (dir, q, questions) = store();
3663 let mut t = task("gate went red", dir.path().join("repo"));
3664 t.hold_machine(Some("gate red".to_owned()));
3665 q.put(&mut t).unwrap();
3666 run_once(&q, &questions, None, Timestamp::now());
3667
3668 let mut asked = questions
3669 .list()
3670 .into_iter()
3671 .find(|q| q.run == t.id)
3672 .unwrap();
3673 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
3674 questions.put(&mut asked).unwrap();
3675
3676 let report = run_once(&q, &questions, None, Timestamp::now());
3677 assert_eq!(report.answered, [t.id.clone()]);
3678 let back = q.get(&t.id).unwrap();
3679 assert_eq!(back.status, TaskStatus::Held);
3680 assert_eq!(back.hold_source, Some(HoldSource::Manual));
3681 assert!(
3682 back.hold_reason
3683 .as_deref()
3684 .is_some_and(|r| r.contains("gate red")),
3685 "the original cause must survive a \"not yet\" answer, not just the \
3686 triage marker: {:?}",
3687 back.hold_reason
3688 );
3689
3690 let third = run_once(&q, &questions, None, Timestamp::now());
3694 assert!(third.answered.is_empty());
3695 assert!(third.asked.is_empty());
3696 }
3697
3698 #[test]
3699 fn answering_discard_removes_the_task_entirely() {
3700 let (dir, q, questions) = store();
3701 let mut t = task("gate went red", dir.path().join("repo"));
3702 t.hold_machine(Some("gate red".to_owned()));
3703 q.put(&mut t).unwrap();
3704 run_once(&q, &questions, None, Timestamp::now());
3705
3706 let mut asked = questions
3707 .list()
3708 .into_iter()
3709 .find(|q| q.run == t.id)
3710 .unwrap();
3711 asked.answer(Answer::Choice(EN.discard.to_owned())).unwrap();
3712 questions.put(&mut asked).unwrap();
3713
3714 let report = run_once(&q, &questions, None, Timestamp::now());
3715 assert_eq!(report.answered, [t.id.clone()]);
3716 assert!(
3717 q.get(&t.id).is_err(),
3718 "\"discard it\" (捨ててよい) must actually discard the task, not \
3719 just leave it sitting held forever"
3720 );
3721 }
3722
3723 #[test]
3724 fn an_answer_is_read_by_its_position_in_choices_not_by_the_callers_current_language() {
3725 let (dir, q, questions) = store();
3727 let ja_config = dir.path().join("ja.toml");
3728 std::fs::write(&ja_config, "[graph]\nlanguage = \"ja\"\n").unwrap();
3729 let mut t = task("gate went red", dir.path().join("repo"));
3730 t.hold_machine(Some("gate red".to_owned()));
3731 q.put(&mut t).unwrap();
3732 run_once(&q, &questions, Some(&ja_config), Timestamp::now());
3733
3734 let mut asked = questions
3735 .list()
3736 .into_iter()
3737 .find(|q| q.run == t.id)
3738 .unwrap();
3739 assert_eq!(asked.choices[0], JA.resume, "filed in Japanese");
3740 asked.answer(Answer::Choice(JA.resume.to_owned())).unwrap();
3741 questions.put(&mut asked).unwrap();
3742
3743 let en_config = dir.path().join("en.toml");
3748 std::fs::write(&en_config, "[graph]\nlanguage = \"en\"\n").unwrap();
3749 let report = run_once(&q, &questions, Some(&en_config), Timestamp::now());
3750 assert_eq!(report.answered, [t.id.clone()]);
3751 let back = q.get(&t.id).unwrap();
3752 assert_eq!(
3753 back.status,
3754 TaskStatus::Queued,
3755 "a resume answer must resume the task regardless of which \
3756 language it is read back in"
3757 );
3758 }
3759
3760 #[test]
3761 fn a_question_falls_back_to_last_error_when_hold_reason_was_never_set() {
3762 let (dir, q, questions) = store();
3767 let mut t = task("kept failing the gate", dir.path().join("repo"));
3768 t.start("run-1".to_owned());
3769 t.fail("gate red three times running", 1);
3770 assert_eq!(t.status, TaskStatus::Held);
3771 t.hold_reason = None; q.put(&mut t).unwrap();
3773
3774 run_once(&q, &questions, None, Timestamp::now());
3775 let asked = questions
3776 .list()
3777 .into_iter()
3778 .find(|q| q.run == t.id)
3779 .unwrap();
3780 assert!(
3781 asked.detail.contains("gate red three times running"),
3782 "the question must surface `last_error` when there is no \
3783 `hold_reason` to show instead: {}",
3784 asked.detail
3785 );
3786 }
3787
3788 #[test]
3789 fn open_question_for_and_open_task_ids_reflect_only_what_is_still_waiting() {
3790 let (dir, q, questions) = store();
3791 let mut t = task("gate went red", dir.path().join("repo"));
3792 t.hold_machine(Some("gate red".to_owned()));
3793 q.put(&mut t).unwrap();
3794
3795 assert!(open_question_for(&questions, &t.id).is_none());
3796 assert!(!open_task_ids(&questions).contains(&t.id));
3797
3798 run_once(&q, &questions, None, Timestamp::now());
3799 assert!(open_question_for(&questions, &t.id).is_some());
3800 assert!(open_task_ids(&questions).contains(&t.id));
3801
3802 let mut asked = questions
3803 .list()
3804 .into_iter()
3805 .find(|q| q.run == t.id)
3806 .unwrap();
3807 asked.answer(Answer::Choice(EN.resume.to_owned())).unwrap();
3808 questions.put(&mut asked).unwrap();
3809
3810 assert!(
3811 open_question_for(&questions, &t.id).is_none(),
3812 "an answered question is no longer open"
3813 );
3814 assert!(!open_task_ids(&questions).contains(&t.id));
3815 }
3816
3817 #[test]
3818 fn pending_for_stays_true_between_an_answer_and_the_next_run_once_pass() {
3819 let (dir, q, questions) = store();
3825 let mut t = task("gate went red", dir.path().join("repo"));
3826 t.hold_machine(Some("gate red".to_owned()));
3827 q.put(&mut t).unwrap();
3828
3829 assert!(!pending_for(&questions, &t));
3830
3831 run_once(&q, &questions, None, Timestamp::now());
3832 let held = q.get(&t.id).unwrap();
3833 assert!(pending_for(&questions, &held), "still waiting on an answer");
3834
3835 let mut asked = questions
3836 .list()
3837 .into_iter()
3838 .find(|q| q.run == t.id)
3839 .unwrap();
3840 asked.answer(Answer::Choice(EN.wait.to_owned())).unwrap();
3841 questions.put(&mut asked).unwrap();
3842 assert!(!asked.status.open());
3843
3844 let still_held = q.get(&t.id).unwrap();
3846 assert!(
3847 pending_for(&questions, &still_held),
3848 "answered but not yet applied is still pending"
3849 );
3850
3851 run_once(&q, &questions, None, Timestamp::now());
3852 let after = q.get(&t.id).unwrap();
3853 assert!(
3854 !pending_for(&questions, &after),
3855 "the answer is applied now, nothing left pending"
3856 );
3857 }
3858
3859 #[test]
3860 fn migrated_keep_held_is_consumed_but_can_be_reviewed_and_discarded_again() {
3861 let (dir, queue, questions) = store();
3862 let mut t = task("keep held", dir.path().join("repo"));
3863 t.hold_machine(Some("unresolved cause".into()));
3864 queue.put(&mut t).unwrap();
3865 run_once(&queue, &questions, None, Timestamp::now());
3866 let mut old = questions.list().remove(0);
3867 old.answer(Answer::Choice(old.choices[1].clone())).unwrap();
3868 questions.put(&mut old).unwrap();
3869 run_once(&queue, &questions, None, Timestamp::now());
3870 let held = queue.get(&t.id).unwrap();
3871 assert!(held.triage_applied(&old.id));
3872
3873 let migration = dir.path().join("notifications/task-chat-migration-v1");
3874 std::fs::create_dir_all(&migration).unwrap();
3875 std::fs::write(
3876 migration.join("state.json"),
3877 serde_json::to_vec(&serde_json::json!({
3878 "schema": 1, "classification_revision": 1, "pending": [], "sources": [old.id]
3879 }))
3880 .unwrap(),
3881 )
3882 .unwrap();
3883 assert!(questions.list_for_effects()[0].status.open());
3884 assert!(!pending_for(&questions, &held));
3885 assert!(open_question_for(&questions, &t.id).is_none());
3886 assert!(!open_task_ids(&questions).contains(&t.id));
3887 assert!(old.answer(Answer::Choice(old.choices[2].clone())).is_err());
3888 assert!(
3889 run_once(&queue, &questions, None, Timestamp::now())
3890 .answered
3891 .is_empty()
3892 );
3893
3894 let later = held.updated_at + SignedDuration::from_hours(24 * 8);
3895 assert_eq!(
3896 run_once(&queue, &questions, None, later).asked,
3897 [t.id.clone()]
3898 );
3899 let mut fresh = open_question_for(&questions, &t.id).unwrap();
3900 assert_ne!(fresh.id, old.id);
3901 assert_eq!(fresh.choices[2], EN.discard);
3902 fresh
3903 .answer(Answer::Choice(fresh.choices[2].clone()))
3904 .unwrap();
3905 questions.put(&mut fresh).unwrap();
3906 assert_eq!(
3907 run_once(&queue, &questions, None, later).answered,
3908 [t.id.clone()]
3909 );
3910 assert!(queue.get(&t.id).is_err());
3911 assert_eq!(
3912 questions.read_history(&old.id).unwrap().resolution(),
3913 Some(EN.wait.into())
3914 );
3915 }
3916
3917 #[test]
3918 fn migrated_unapplied_answers_remain_fenced_but_are_not_displayed_open() {
3919 for node in [NODE, DEPS_NODE] {
3920 let (dir, queue, questions) = store();
3921 let mut t = task("unapplied", dir.path().join("repo"));
3922 t.hold_manual(Some("waiting".into()));
3923 queue.put(&mut t).unwrap();
3924 let mut q = Question::new(
3925 t.id.clone(),
3926 node.into(),
3927 SEAT.into(),
3928 "unapplied".into(),
3929 String::new(),
3930 EN.choices3(),
3931 );
3932 q.answer(Answer::Choice(q.choices[0].clone())).unwrap();
3933 questions.put(&mut q).unwrap();
3934 let migration = dir.path().join("notifications/task-chat-migration-v1");
3935 std::fs::create_dir_all(&migration).unwrap();
3936 std::fs::write(
3937 migration.join("state.json"),
3938 serde_json::to_vec(&serde_json::json!({
3939 "schema": 1, "classification_revision": 1, "pending": [], "sources": [q.id]
3940 }))
3941 .unwrap(),
3942 )
3943 .unwrap();
3944 assert!(pending_for(&questions, &t));
3945 assert!(open_question_for(&questions, &t.id).is_none());
3946 assert!(!open_task_ids(&questions).contains(&t.id));
3947 assert!(apply_answers(&queue, &questions).is_empty());
3948 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
3949 t.mark_triage_applied(&q.id);
3950 queue.put(&mut t).unwrap();
3951 assert!(!pending_for(&questions, &t));
3952 assert!(!deps_pending_in(&RecoveryIndex::load(&questions), &t));
3953 }
3954 }
3955}