1use std::fmt;
7#[cfg(any(test, feature = "test-support"))]
8use std::sync::atomic::AtomicU64;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::sync::{Arc, OnceLock};
11
12use crate::resources::ResourceProfile;
13#[cfg(any(
14 not(all(target_arch = "wasm32", target_os = "unknown")),
15 feature = "operation-deadlines",
16 feature = "system-timing",
17 test
18))]
19use std::time::Duration;
20
21#[cfg(not(all(
22 target_arch = "wasm32",
23 target_os = "unknown",
24 any(feature = "operation-deadlines", feature = "system-timing")
25)))]
26use std::time::Instant;
27#[cfg(all(
28 target_arch = "wasm32",
29 target_os = "unknown",
30 any(feature = "operation-deadlines", feature = "system-timing")
31))]
32use web_time::Instant;
33
34#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36#[non_exhaustive]
37pub enum OperationPhase {
38 Admission,
39 Parse,
40 Semantic,
41 Analysis,
42 Layout,
43 Emit,
44 Postprocess,
45 Export,
46 Unknown,
47}
48
49impl OperationPhase {
50 pub const fn as_str(self) -> &'static str {
51 match self {
52 Self::Admission => "admission",
53 Self::Parse => "parse",
54 Self::Semantic => "semantic",
55 Self::Analysis => "analysis",
56 Self::Layout => "layout",
57 Self::Emit => "emit",
58 Self::Postprocess => "postprocess",
59 Self::Export => "export",
60 Self::Unknown => "unknown",
61 }
62 }
63}
64
65impl fmt::Display for OperationPhase {
66 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
67 f.write_str(self.as_str())
68 }
69}
70
71#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
73pub enum CancelReason {
74 Requested,
75 DeadlineExceeded,
76}
77
78impl CancelReason {
79 pub const fn as_str(self) -> &'static str {
80 match self {
81 Self::Requested => "requested",
82 Self::DeadlineExceeded => "deadline_exceeded",
83 }
84 }
85}
86
87impl fmt::Display for CancelReason {
88 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
89 f.write_str(self.as_str())
90 }
91}
92
93#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
95#[error("operation cancelled during {phase}: {reason}")]
96pub struct OperationCancelled {
97 pub phase: OperationPhase,
98 pub reason: CancelReason,
99}
100
101pub type OperationControlResult<T> = std::result::Result<T, OperationCancelled>;
103
104type Clock = Arc<dyn Fn() -> Instant + Send + Sync + 'static>;
105
106struct OperationState {
107 attention_required: AtomicBool,
108 cancelled: AtomicBool,
109 terminal: OnceLock<OperationLedgerError>,
110 parent: Option<Arc<OperationState>>,
111 deadline: OnceLock<Instant>,
112 clock: Clock,
113 #[cfg(any(test, feature = "test-support"))]
114 successful_checkpoints_before_cancellation: AtomicU64,
115}
116
117impl fmt::Debug for OperationState {
118 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
119 f.debug_struct("OperationState")
120 .field("cancelled", &self.cancelled.load(Ordering::Acquire))
121 .field("terminal", &self.terminal.get())
122 .field("deadline", &self.deadline.get())
123 .finish_non_exhaustive()
124 }
125}
126
127impl OperationState {
128 fn new(parent: Option<Arc<OperationState>>, clock: Clock) -> Self {
129 let attention_required = parent.is_some();
130 Self {
131 attention_required: AtomicBool::new(attention_required),
132 cancelled: AtomicBool::new(false),
133 terminal: OnceLock::new(),
134 parent,
135 deadline: OnceLock::new(),
136 clock,
137 #[cfg(any(test, feature = "test-support"))]
138 successful_checkpoints_before_cancellation: AtomicU64::new(u64::MAX),
139 }
140 }
141
142 fn latch_terminal(&self, error: OperationLedgerError) -> OperationLedgerError {
143 self.attention_required.store(true, Ordering::Release);
144 self.terminal.get_or_init(|| error).clone()
145 }
146
147 fn attention_required(&self) -> bool {
148 self.attention_required.load(Ordering::Acquire)
149 }
150}
151
152#[derive(Clone, Debug)]
158pub struct OperationControl {
159 state: Arc<OperationState>,
160 default_phase: OperationPhase,
161}
162
163impl Default for OperationControl {
164 fn default() -> Self {
165 Self::new()
166 }
167}
168
169impl OperationControl {
170 pub fn new() -> Self {
172 Self::with_clock(default_clock())
173 }
174
175 fn with_clock(clock: Clock) -> Self {
180 Self {
181 state: Arc::new(OperationState::new(None, clock)),
182 default_phase: OperationPhase::Unknown,
183 }
184 }
185
186 pub fn child(&self) -> Self {
189 Self {
190 state: Arc::new(OperationState::new(
191 Some(Arc::clone(&self.state)),
192 Arc::clone(&self.state.clock),
193 )),
194 default_phase: self.default_phase,
195 }
196 }
197
198 pub fn for_phase(&self, phase: OperationPhase) -> Self {
200 Self {
201 state: Arc::clone(&self.state),
202 default_phase: phase,
203 }
204 }
205
206 #[cfg(any(
208 not(all(target_arch = "wasm32", target_os = "unknown")),
209 feature = "operation-deadlines",
210 feature = "system-timing"
211 ))]
212 pub fn with_deadline(self, timeout: Duration) -> Self {
213 self.set_deadline(timeout);
214 self
215 }
216
217 #[cfg(any(
219 not(all(target_arch = "wasm32", target_os = "unknown")),
220 feature = "operation-deadlines",
221 feature = "system-timing"
222 ))]
223 pub fn set_deadline(&self, timeout: Duration) -> bool {
224 let now = (self.state.clock)();
225 self.state.attention_required.store(true, Ordering::Release);
226 self.state
227 .deadline
228 .set(deadline_after(now, timeout))
229 .is_ok()
230 }
231
232 pub fn cancel(&self) {
234 self.state.attention_required.store(true, Ordering::Release);
235 self.state.cancelled.store(true, Ordering::Release);
236 }
237
238 pub fn is_cancelled(&self) -> bool {
240 let mut state = Some(self.state.as_ref());
241 while let Some(current) = state {
242 if current.cancelled.load(Ordering::Acquire) {
243 return true;
244 }
245 state = current.parent.as_deref();
246 }
247 false
248 }
249
250 pub fn checkpoint(&self) -> Result<(), OperationCancelled> {
252 self.checkpoint_at(self.default_phase)
253 }
254
255 pub(crate) fn observed_cancellation(&self) -> Option<OperationCancelled> {
257 match self.state.terminal.get() {
258 Some(OperationLedgerError::Cancelled(error)) => Some(*error),
259 _ => None,
260 }
261 }
262
263 pub fn checkpoint_at(&self, phase: OperationPhase) -> Result<(), OperationCancelled> {
270 if !self.state.attention_required() {
271 return Ok(());
272 }
273 self.observe_cancellation_at(phase).map_or(Ok(()), Err)
274 }
275
276 pub fn terminal_checkpoint_at(
282 &self,
283 phase: OperationPhase,
284 ) -> Result<(), OperationLedgerError> {
285 if !self.state.attention_required() {
286 return Ok(());
287 }
288 if let Some(error) = self.state.terminal.get() {
289 return Err(error.clone());
290 }
291 let Some(cancellation) = self.observe_cancellation_at(phase) else {
292 return self.state.terminal.get().cloned().map_or(Ok(()), Err);
293 };
294 Err(self.latch_terminal_error(OperationLedgerError::Cancelled(cancellation)))
295 }
296
297 fn observe_cancellation_at(&self, phase: OperationPhase) -> Option<OperationCancelled> {
298 #[cfg(any(test, feature = "test-support"))]
299 if self.consume_scheduled_checkpoint() {
300 self.cancel();
301 }
302
303 self.resolve_cancellation_at(phase)
304 }
305
306 fn resolve_cancellation_at(&self, phase: OperationPhase) -> Option<OperationCancelled> {
307 let mut state = Some(self.state.as_ref());
308 let mut inherited = false;
309 while let Some(current) = state {
310 let terminal = if let Some(error) = current.terminal.get() {
311 match error {
312 OperationLedgerError::Cancelled(error) if inherited => {
313 Some(OperationCancelled {
314 phase,
315 reason: error.reason,
316 })
317 }
318 OperationLedgerError::Cancelled(error) => Some(*error),
319 OperationLedgerError::ResourceLimitExceeded(_)
320 | OperationLedgerError::ArithmeticOverflow { .. }
321 if !inherited =>
322 {
323 return None;
324 }
325 OperationLedgerError::ResourceLimitExceeded(_)
326 | OperationLedgerError::ArithmeticOverflow { .. }
327 if current.cancelled.load(Ordering::Acquire) =>
328 {
329 Some(OperationCancelled {
330 phase,
331 reason: CancelReason::Requested,
332 })
333 }
334 OperationLedgerError::ResourceLimitExceeded(_)
335 | OperationLedgerError::ArithmeticOverflow { .. }
336 if current
337 .deadline
338 .get()
339 .is_some_and(|deadline| (current.clock)() >= *deadline) =>
340 {
341 Some(OperationCancelled {
342 phase,
343 reason: CancelReason::DeadlineExceeded,
344 })
345 }
346 OperationLedgerError::ResourceLimitExceeded(_)
347 | OperationLedgerError::ArithmeticOverflow { .. } => None,
348 }
349 } else if current.cancelled.load(Ordering::Acquire) {
350 let cancellation = OperationCancelled {
351 phase,
352 reason: CancelReason::Requested,
353 };
354 match Self::resolve_latched_cancellation(current, inherited, cancellation) {
355 Some(error) => Some(error),
356 None => return None,
357 }
358 } else if current
359 .deadline
360 .get()
361 .is_some_and(|deadline| (current.clock)() >= *deadline)
362 {
363 let cancellation = OperationCancelled {
364 phase,
365 reason: CancelReason::DeadlineExceeded,
366 };
367 match Self::resolve_latched_cancellation(current, inherited, cancellation) {
368 Some(error) => Some(error),
369 None => return None,
370 }
371 } else {
372 None
373 };
374
375 if let Some(error) = terminal {
376 if !inherited {
377 return Some(error);
378 }
379 return match self
380 .state
381 .latch_terminal(OperationLedgerError::Cancelled(error))
382 {
383 OperationLedgerError::Cancelled(error) => Some(error),
384 OperationLedgerError::ResourceLimitExceeded(_)
385 | OperationLedgerError::ArithmeticOverflow { .. } => None,
386 };
387 }
388
389 state = current.parent.as_deref();
390 inherited = true;
391 }
392 None
393 }
394
395 fn resolve_latched_cancellation(
396 state: &OperationState,
397 inherited: bool,
398 cancellation: OperationCancelled,
399 ) -> Option<OperationCancelled> {
400 match state.latch_terminal(OperationLedgerError::Cancelled(cancellation)) {
401 OperationLedgerError::Cancelled(error) => Some(error),
402 OperationLedgerError::ResourceLimitExceeded(_)
403 | OperationLedgerError::ArithmeticOverflow { .. }
404 if inherited =>
405 {
406 Some(cancellation)
407 }
408 OperationLedgerError::ResourceLimitExceeded(_)
409 | OperationLedgerError::ArithmeticOverflow { .. } => None,
410 }
411 }
412
413 fn latch_terminal_error(&self, error: OperationLedgerError) -> OperationLedgerError {
414 self.state.latch_terminal(error)
415 }
416
417 pub fn terminate_resource_limit(
419 &self,
420 error: OperationResourceLimitExceeded,
421 ) -> OperationLedgerError {
422 self.latch_terminal_error(OperationLedgerError::ResourceLimitExceeded(error))
423 }
424
425 pub fn terminate_resource_overflow(
427 &self,
428 id: &'static str,
429 phase: OperationPhase,
430 resource_phase: &'static str,
431 actual: u64,
432 maximum: u64,
433 provenance: OperationResourceProvenance,
434 ) -> OperationLedgerError {
435 self.latch_terminal_error(OperationLedgerError::ArithmeticOverflow {
436 id,
437 phase,
438 resource_phase,
439 actual,
440 maximum,
441 provenance,
442 })
443 }
444
445 #[cfg(any(test, feature = "test-support"))]
446 fn consume_scheduled_checkpoint(&self) -> bool {
447 self.state
448 .successful_checkpoints_before_cancellation
449 .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |remaining| {
450 (remaining != u64::MAX).then(|| remaining.saturating_sub(1))
451 })
452 .is_ok_and(|remaining| remaining == 0)
453 }
454
455 #[cfg(any(test, feature = "test-support"))]
457 pub fn cancel_after_checkpoints(&self, successful_checkpoints: usize) {
458 self.state.attention_required.store(true, Ordering::Release);
459 self.state
460 .successful_checkpoints_before_cancellation
461 .store(successful_checkpoints as u64, Ordering::Relaxed);
462 }
463}
464
465#[cfg(any(
466 not(all(target_arch = "wasm32", target_os = "unknown")),
467 feature = "operation-deadlines",
468 feature = "system-timing"
469))]
470fn deadline_after(now: Instant, mut timeout: Duration) -> Instant {
471 loop {
472 if let Some(deadline) = now.checked_add(timeout) {
473 return deadline;
474 }
475 timeout /= 2;
476 }
477}
478
479fn default_clock() -> Clock {
480 #[cfg(not(all(
481 target_arch = "wasm32",
482 target_os = "unknown",
483 not(any(feature = "operation-deadlines", feature = "system-timing"))
484 )))]
485 {
486 Arc::new(Instant::now)
487 }
488
489 #[cfg(all(
490 target_arch = "wasm32",
491 target_os = "unknown",
492 not(any(feature = "operation-deadlines", feature = "system-timing"))
493 ))]
494 {
495 Arc::new(|| unreachable!("deadline clock is unavailable in this artifact"))
496 }
497}
498
499#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
501#[non_exhaustive]
502pub enum OperationResourceDomain {
503 Input,
504 Render,
505 Ascii,
506 Export,
507}
508
509impl OperationResourceDomain {
510 pub const fn as_str(self) -> &'static str {
511 match self {
512 Self::Input => "input",
513 Self::Render => "render",
514 Self::Ascii => "ascii",
515 Self::Export => "export",
516 }
517 }
518}
519
520impl fmt::Display for OperationResourceDomain {
521 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
522 formatter.write_str(self.as_str())
523 }
524}
525
526#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
528pub struct OperationResourceOverride {
529 pub id: &'static str,
530 pub value: u64,
531}
532
533#[derive(Debug, Clone, PartialEq, Eq)]
535pub struct OperationResourceProvenance {
536 pub domain: OperationResourceDomain,
537 pub profile: Option<ResourceProfile>,
538 pub explicit_overrides: Arc<[OperationResourceOverride]>,
539}
540
541impl OperationResourceProvenance {
542 pub fn new(
543 domain: OperationResourceDomain,
544 profile: Option<ResourceProfile>,
545 explicit_overrides: impl IntoIterator<Item = OperationResourceOverride>,
546 ) -> Self {
547 Self {
548 domain,
549 profile,
550 explicit_overrides: explicit_overrides.into_iter().collect::<Vec<_>>().into(),
551 }
552 }
553}
554
555#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
557#[error(
558 "operation resource limit `{id}` exceeded during {phase}: {consumed} + {requested} > {limit}"
559)]
560pub struct OperationResourceLimitExceeded {
561 pub id: &'static str,
562 pub phase: OperationPhase,
563 pub resource_phase: &'static str,
564 pub limit: u64,
565 pub consumed: u64,
566 pub requested: u64,
567 pub provenance: OperationResourceProvenance,
568}
569
570#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
572pub enum OperationLedgerError {
573 #[error(transparent)]
574 Cancelled(#[from] OperationCancelled),
575 #[error(transparent)]
576 ResourceLimitExceeded(#[from] OperationResourceLimitExceeded),
577 #[error(
578 "operation resource `{id}` arithmetic overflow during {phase} ({resource_phase}; actual {actual}, maximum {maximum})"
579 )]
580 ArithmeticOverflow {
581 id: &'static str,
582 phase: OperationPhase,
583 resource_phase: &'static str,
584 actual: u64,
585 maximum: u64,
586 provenance: OperationResourceProvenance,
587 },
588}
589
590#[cfg(test)]
591mod tests {
592 use super::*;
593 use std::sync::Barrier;
594 use std::sync::atomic::AtomicU64;
595 use std::thread;
596
597 fn test_resource_provenance() -> OperationResourceProvenance {
598 OperationResourceProvenance::new(
599 OperationResourceDomain::Render,
600 Some(ResourceProfile::Interactive),
601 [],
602 )
603 }
604
605 #[test]
606 fn clones_and_children_observe_shared_cancellation() {
607 let control = OperationControl::new();
608 let clone = control.clone();
609 let child = control.child();
610 child.cancel();
611 assert!(child.is_cancelled());
612 assert!(!control.is_cancelled());
613 control.cancel();
614 assert!(clone.is_cancelled());
615 assert!(child.checkpoint_at(OperationPhase::Parse).is_err());
616 }
617
618 #[test]
619 fn phase_views_share_state_and_label_unnamed_checkpoints() {
620 let control = OperationControl::new();
621 let parse = control.for_phase(OperationPhase::Parse);
622
623 control.cancel();
624
625 let error = parse.checkpoint().unwrap_err();
626 assert_eq!(error.phase, OperationPhase::Parse);
627 assert_eq!(error.reason, CancelReason::Requested);
628 }
629
630 #[test]
631 fn deadline_is_sticky_and_distinct_from_requested_cancellation() {
632 let now = Arc::new(AtomicU64::new(0));
633 let source = Arc::clone(&now);
634 let base = Instant::now();
635 let clock = Arc::new(move || base + Duration::from_millis(source.load(Ordering::Relaxed)));
636 let control = OperationControl::with_clock(clock).with_deadline(Duration::from_millis(0));
637 let error = control.checkpoint_at(OperationPhase::Layout).unwrap_err();
638 assert_eq!(error.reason, CancelReason::DeadlineExceeded);
639 assert_eq!(control.checkpoint().unwrap_err(), error);
640 assert_eq!(
641 control
642 .terminal_checkpoint_at(OperationPhase::Emit)
643 .unwrap_err(),
644 OperationLedgerError::Cancelled(error)
645 );
646 }
647
648 #[test]
649 fn child_latches_the_first_reason_observed_across_its_parent_chain() {
650 let now = Arc::new(AtomicU64::new(0));
651 let source = Arc::clone(&now);
652 let base = Instant::now();
653 let clock = Arc::new(move || base + Duration::from_millis(source.load(Ordering::Relaxed)));
654 let parent = OperationControl::with_clock(clock);
655 let child = parent.child().with_deadline(Duration::from_millis(1));
656
657 parent.cancel();
658 assert_eq!(
659 child
660 .checkpoint_at(OperationPhase::Parse)
661 .unwrap_err()
662 .reason,
663 CancelReason::Requested
664 );
665 now.store(2, Ordering::Relaxed);
666 assert_eq!(
667 child
668 .checkpoint_at(OperationPhase::Layout)
669 .unwrap_err()
670 .reason,
671 CancelReason::Requested
672 );
673 }
674
675 #[test]
676 fn child_rebinds_an_observed_parent_cancellation_to_its_checkpoint_phase() {
677 let parent = OperationControl::new();
678 parent.cancel();
679 let parent_error = parent
680 .checkpoint_at(OperationPhase::Parse)
681 .expect_err("the parent cancellation must be observed");
682 let child = parent.child();
683
684 let child_error = child
685 .checkpoint_at(OperationPhase::Layout)
686 .expect_err("the child must observe the parent cancellation");
687
688 assert_eq!(parent_error.reason, CancelReason::Requested);
689 assert_eq!(parent_error.phase, OperationPhase::Parse);
690 assert_eq!(child_error.reason, parent_error.reason);
691 assert_eq!(child_error.phase, OperationPhase::Layout);
692 assert_eq!(
693 parent.checkpoint_at(OperationPhase::Emit).unwrap_err(),
694 parent_error
695 );
696 assert_eq!(
697 child.checkpoint_at(OperationPhase::Emit).unwrap_err(),
698 child_error
699 );
700 }
701
702 #[test]
703 fn child_rebinds_an_observed_parent_deadline_to_its_checkpoint_phase() {
704 let parent = OperationControl::new().with_deadline(Duration::ZERO);
705 let parent_error = parent
706 .checkpoint_at(OperationPhase::Parse)
707 .expect_err("the parent deadline must be observed");
708 let child = parent.child();
709
710 let child_error = child
711 .checkpoint_at(OperationPhase::Layout)
712 .expect_err("the child must observe the parent deadline");
713
714 assert_eq!(parent_error.reason, CancelReason::DeadlineExceeded);
715 assert_eq!(parent_error.phase, OperationPhase::Parse);
716 assert_eq!(child_error.reason, parent_error.reason);
717 assert_eq!(child_error.phase, OperationPhase::Layout);
718 assert_eq!(
719 parent.checkpoint_at(OperationPhase::Emit).unwrap_err(),
720 parent_error
721 );
722 assert_eq!(
723 child.checkpoint_at(OperationPhase::Emit).unwrap_err(),
724 child_error
725 );
726 }
727
728 #[test]
729 fn child_keeps_its_requested_reason_after_parent_deadline_expires() {
730 let now = Arc::new(AtomicU64::new(0));
731 let source = Arc::clone(&now);
732 let base = Instant::now();
733 let clock = Arc::new(move || base + Duration::from_millis(source.load(Ordering::Relaxed)));
734 let parent = OperationControl::with_clock(clock).with_deadline(Duration::from_millis(1));
735 let child = parent.child();
736
737 child.cancel();
738 assert_eq!(
739 child
740 .checkpoint_at(OperationPhase::Parse)
741 .unwrap_err()
742 .reason,
743 CancelReason::Requested
744 );
745 now.store(2, Ordering::Relaxed);
746 assert_eq!(
747 child
748 .checkpoint_at(OperationPhase::Layout)
749 .unwrap_err()
750 .reason,
751 CancelReason::Requested
752 );
753 }
754
755 #[test]
756 fn child_latches_an_expired_parent_deadline() {
757 let now = Arc::new(AtomicU64::new(0));
758 let source = Arc::clone(&now);
759 let base = Instant::now();
760 let clock = Arc::new(move || base + Duration::from_millis(source.load(Ordering::Relaxed)));
761 let parent = OperationControl::with_clock(clock).with_deadline(Duration::from_millis(1));
762 let child = parent.child();
763
764 now.store(2, Ordering::Relaxed);
765 assert_eq!(
766 child
767 .checkpoint_at(OperationPhase::Parse)
768 .unwrap_err()
769 .reason,
770 CancelReason::DeadlineExceeded
771 );
772 parent.cancel();
773 assert_eq!(
774 child
775 .checkpoint_at(OperationPhase::Layout)
776 .unwrap_err()
777 .reason,
778 CancelReason::DeadlineExceeded
779 );
780 }
781
782 #[test]
783 fn child_keeps_an_observed_parent_cancellation_when_parent_resource_wins_the_latch() {
784 let parent = OperationControl::new();
785 let child = parent.child();
786 let cancellation = OperationCancelled {
787 phase: OperationPhase::Parse,
788 reason: CancelReason::Requested,
789 };
790 let parent_terminal = OperationLedgerError::ArithmeticOverflow {
791 id: "parent_work",
792 phase: OperationPhase::Layout,
793 resource_phase: "layout",
794 actual: u64::MAX,
795 maximum: u64::MAX,
796 provenance: test_resource_provenance(),
797 };
798
799 parent.cancel();
800 assert_eq!(
801 parent.latch_terminal_error(parent_terminal.clone()),
802 parent_terminal
803 );
804 let observed = OperationControl::resolve_latched_cancellation(
805 parent.state.as_ref(),
806 true,
807 cancellation,
808 )
809 .expect("an ancestor resource terminal must not hide an observed cancellation");
810 assert_eq!(observed, cancellation);
811 assert_eq!(
812 child.latch_terminal_error(OperationLedgerError::Cancelled(observed)),
813 OperationLedgerError::Cancelled(cancellation)
814 );
815 assert_eq!(
816 child
817 .terminal_checkpoint_at(OperationPhase::Emit)
818 .expect_err("the child must replay its cancellation"),
819 OperationLedgerError::Cancelled(cancellation)
820 );
821 assert_eq!(
822 parent
823 .terminal_checkpoint_at(OperationPhase::Emit)
824 .expect_err("the parent must retain its resource terminal"),
825 parent_terminal
826 );
827 }
828
829 #[test]
830 fn child_latches_parent_deadline_when_parent_resource_wins_during_clock_read() {
831 let now = Arc::new(AtomicU64::new(0));
832 let clock_now = Arc::clone(&now);
833 let parent_slot = Arc::new(OnceLock::<OperationControl>::new());
834 let clock_parent = Arc::clone(&parent_slot);
835 let base = Instant::now();
836 let parent_terminal = OperationLedgerError::ArithmeticOverflow {
837 id: "parent_work",
838 phase: OperationPhase::Layout,
839 resource_phase: "layout",
840 actual: u64::MAX,
841 maximum: u64::MAX,
842 provenance: test_resource_provenance(),
843 };
844 let clock_terminal = parent_terminal.clone();
845 let clock = Arc::new(move || {
846 if let Some(parent) = clock_parent.get() {
847 parent.latch_terminal_error(clock_terminal.clone());
848 }
849 base + Duration::from_millis(clock_now.load(Ordering::Relaxed))
850 });
851 let parent = OperationControl::with_clock(clock).with_deadline(Duration::from_millis(1));
852 assert!(parent_slot.set(parent.clone()).is_ok());
853 let child = parent.child();
854
855 now.store(2, Ordering::Relaxed);
856 let cancellation = child
857 .terminal_checkpoint_at(OperationPhase::Parse)
858 .expect_err("the child must observe the expired parent deadline");
859 assert_eq!(
860 cancellation,
861 OperationLedgerError::Cancelled(OperationCancelled {
862 phase: OperationPhase::Parse,
863 reason: CancelReason::DeadlineExceeded,
864 })
865 );
866 assert_eq!(
867 child
868 .terminal_checkpoint_at(OperationPhase::Emit)
869 .expect_err("the child must replay its cancellation"),
870 cancellation
871 );
872 assert_eq!(
873 parent
874 .terminal_checkpoint_at(OperationPhase::Emit)
875 .expect_err("the parent must retain its resource terminal"),
876 parent_terminal
877 );
878 }
879
880 #[test]
881 fn oversized_deadline_clamps_without_panicking_or_expiring_immediately() {
882 let control = OperationControl::new().with_deadline(Duration::MAX);
883 assert!(control.state.deadline.get().is_some());
884 assert_eq!(control.checkpoint(), Ok(()));
885 }
886
887 #[test]
888 fn concurrent_checkpoints_replay_one_complete_cancellation() {
889 const THREADS: usize = 16;
890 for _ in 0..32 {
891 let control = OperationControl::new();
892 control.cancel();
893 let barrier = Arc::new(Barrier::new(THREADS));
894 let handles = (0..THREADS)
895 .map(|index| {
896 let control = control.clone();
897 let barrier = Arc::clone(&barrier);
898 thread::spawn(move || {
899 barrier.wait();
900 let phase = if index % 2 == 0 {
901 OperationPhase::Parse
902 } else {
903 OperationPhase::Layout
904 };
905 control.checkpoint_at(phase)
906 })
907 })
908 .collect::<Vec<_>>();
909
910 let mut first = None;
911 for handle in handles {
912 let cancelled = handle
913 .join()
914 .expect("checkpoint thread should not panic")
915 .expect_err("every concurrent observer must see cancellation");
916 assert_eq!(cancelled.reason, CancelReason::Requested);
917 if let Some(first) = first {
918 assert_eq!(cancelled, first);
919 } else {
920 first = Some(cancelled);
921 }
922 }
923 }
924 }
925
926 #[test]
927 fn operation_replays_the_first_resource_rejection_after_later_cancellation() {
928 let control = OperationControl::new();
929 let first = control.terminate_resource_limit(OperationResourceLimitExceeded {
930 id: "work",
931 phase: OperationPhase::Layout,
932 resource_phase: "layout",
933 limit: 2,
934 consumed: 2,
935 requested: 1,
936 provenance: test_resource_provenance(),
937 });
938 assert_eq!(
939 first,
940 OperationLedgerError::ResourceLimitExceeded(OperationResourceLimitExceeded {
941 id: "work",
942 phase: OperationPhase::Layout,
943 resource_phase: "layout",
944 limit: 2,
945 consumed: 2,
946 requested: 1,
947 provenance: test_resource_provenance(),
948 })
949 );
950
951 let clone = control.clone();
952 let child = control.child();
953 control.cancel();
954 assert_eq!(clone.checkpoint_at(OperationPhase::Emit), Ok(()));
955 assert_eq!(
956 clone
957 .terminal_checkpoint_at(OperationPhase::Emit)
958 .expect_err("the complete checkpoint must replay the resource terminal"),
959 first
960 );
961 let cancellation = child
962 .checkpoint_at(OperationPhase::Postprocess)
963 .expect_err("a child has a distinct terminal scope and observes parent cancellation");
964 assert_eq!(cancellation.reason, CancelReason::Requested);
965 assert_eq!(cancellation.phase, OperationPhase::Postprocess);
966 assert_eq!(
967 child
968 .terminal_checkpoint_at(OperationPhase::Emit)
969 .expect_err("the child's complete checkpoint replays its cancellation"),
970 OperationLedgerError::Cancelled(cancellation)
971 );
972 assert_eq!(
973 control.terminate_resource_overflow(
974 "later_work",
975 OperationPhase::Emit,
976 "emit",
977 u64::MAX,
978 u64::MAX,
979 test_resource_provenance(),
980 ),
981 first
982 );
983 }
984
985 #[test]
986 fn child_control_starts_a_distinct_terminal_scope() {
987 let parent = OperationControl::new();
988 let parent_terminal = parent.terminate_resource_limit(OperationResourceLimitExceeded {
989 id: "parent_work",
990 phase: OperationPhase::Layout,
991 resource_phase: "layout",
992 limit: 0,
993 consumed: 0,
994 requested: 1,
995 provenance: test_resource_provenance(),
996 });
997
998 let child = parent.child();
999 assert_eq!(child.terminal_checkpoint_at(OperationPhase::Emit), Ok(()));
1000 let child_terminal = child.terminate_resource_limit(OperationResourceLimitExceeded {
1001 id: "child_work",
1002 phase: OperationPhase::Emit,
1003 resource_phase: "emit",
1004 limit: 0,
1005 consumed: 0,
1006 requested: 1,
1007 provenance: test_resource_provenance(),
1008 });
1009 assert_ne!(child_terminal, parent_terminal);
1010 assert_eq!(
1011 parent
1012 .terminal_checkpoint_at(OperationPhase::Layout)
1013 .unwrap_err(),
1014 parent_terminal
1015 );
1016 assert_eq!(
1017 child
1018 .terminal_checkpoint_at(OperationPhase::Emit)
1019 .unwrap_err(),
1020 child_terminal
1021 );
1022 }
1023
1024 #[test]
1025 fn operation_replays_cancellation_before_later_resource_failure() {
1026 let control = OperationControl::new();
1027 control.cancel();
1028
1029 let first = control
1030 .terminal_checkpoint_at(OperationPhase::Parse)
1031 .unwrap_err();
1032 assert_eq!(
1033 first,
1034 OperationLedgerError::Cancelled(OperationCancelled {
1035 phase: OperationPhase::Parse,
1036 reason: CancelReason::Requested,
1037 })
1038 );
1039 assert_eq!(
1040 control.terminate_resource_limit(OperationResourceLimitExceeded {
1041 id: "work",
1042 phase: OperationPhase::Emit,
1043 resource_phase: "emit",
1044 limit: 0,
1045 consumed: 0,
1046 requested: 1,
1047 provenance: test_resource_provenance(),
1048 }),
1049 first
1050 );
1051 assert_eq!(
1052 control
1053 .terminal_checkpoint_at(OperationPhase::Postprocess)
1054 .unwrap_err(),
1055 first
1056 );
1057 }
1058
1059 #[test]
1060 fn operation_replays_the_first_arithmetic_overflow() {
1061 let control = OperationControl::new();
1062 let first = control.terminate_resource_overflow(
1063 "layout_work",
1064 OperationPhase::Layout,
1065 "layout",
1066 u64::MAX,
1067 u64::MAX,
1068 test_resource_provenance(),
1069 );
1070 assert_eq!(
1071 first,
1072 OperationLedgerError::ArithmeticOverflow {
1073 id: "layout_work",
1074 phase: OperationPhase::Layout,
1075 resource_phase: "layout",
1076 actual: u64::MAX,
1077 maximum: u64::MAX,
1078 provenance: test_resource_provenance(),
1079 }
1080 );
1081 assert_eq!(
1082 control.terminate_resource_limit(OperationResourceLimitExceeded {
1083 id: "output_bytes",
1084 phase: OperationPhase::Emit,
1085 resource_phase: "emit",
1086 limit: 0,
1087 consumed: 0,
1088 requested: 1,
1089 provenance: test_resource_provenance(),
1090 }),
1091 first
1092 );
1093 assert_eq!(
1094 control
1095 .terminal_checkpoint_at(OperationPhase::Postprocess)
1096 .unwrap_err(),
1097 first
1098 );
1099 }
1100
1101 #[test]
1102 fn concurrent_resource_and_cancellation_observers_replay_one_terminal_error() {
1103 for _ in 0..32 {
1104 let control = OperationControl::new();
1105 let barrier = Arc::new(Barrier::new(2));
1106
1107 let resource = {
1108 let control = control.clone();
1109 let barrier = Arc::clone(&barrier);
1110 thread::spawn(move || {
1111 barrier.wait();
1112 control.terminate_resource_limit(OperationResourceLimitExceeded {
1113 id: "race_work",
1114 phase: OperationPhase::Layout,
1115 resource_phase: "layout",
1116 limit: 0,
1117 consumed: 0,
1118 requested: 1,
1119 provenance: test_resource_provenance(),
1120 })
1121 })
1122 };
1123 let cancellation = {
1124 let control = control.clone();
1125 let barrier = Arc::clone(&barrier);
1126 thread::spawn(move || {
1127 barrier.wait();
1128 control.cancel();
1129 control.terminal_checkpoint_at(OperationPhase::Emit)
1130 })
1131 };
1132
1133 let resource = resource.join().expect("resource thread should not panic");
1134 let cancellation = cancellation
1135 .join()
1136 .expect("cancellation thread should not panic")
1137 .unwrap_err();
1138 assert_eq!(resource, cancellation);
1139 assert_eq!(
1140 control
1141 .terminal_checkpoint_at(OperationPhase::Postprocess)
1142 .unwrap_err(),
1143 resource
1144 );
1145 }
1146 }
1147}