1use crate::event::payloads::delivery_payload::DeliveryResult;
6use crate::event::payloads::system_payload::ContractName;
7use crate::event::{
8 types::{Count, JournalIndex, SeqNo},
9 ChainEvent, ChainPayload, EventId,
10};
11use crate::id::StageId;
12use chrono::{DateTime, Utc};
13use serde::{Deserialize, Serialize};
14use serde_json::{json, Value as JsonValue};
15use std::any::{Any, TypeId};
16use std::collections::{HashMap, HashSet};
17use std::sync::Mutex;
18use std::time::{Duration, Instant};
19
20#[derive(Debug, Clone, Serialize, Deserialize)]
22pub enum ContractResult {
23 Passed(ContractEvidence),
25 Failed(ContractViolation),
27 Pending,
29}
30
31#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct ContractEvidence {
34 pub contract_name: ContractName,
35 pub upstream_stage: StageId,
36 pub downstream_stage: StageId,
37 pub verified_at: DateTime<Utc>,
38 pub details: JsonValue,
39}
40
41#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct ContractViolation {
44 pub contract_name: ContractName,
45 pub upstream_stage: StageId,
46 pub downstream_stage: StageId,
47 pub detected_at: DateTime<Utc>,
48 pub cause: ViolationCause,
49 pub details: JsonValue,
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize)]
54pub enum ViolationCause {
55 SeqDivergence {
57 advertised: Option<SeqNo>,
58 reader: SeqNo,
59 },
60 ContentMismatch { mismatches: Vec<HashMismatch> },
62 DeliveryMismatch {
64 missing_deliveries: usize,
65 orphan_deliveries: usize,
66 },
67 AccountingMismatch {
69 inputs_observed: Count,
70 accounted_for: Count,
71 },
72 Divergence {
74 predicate: String,
76 observed: f64,
78 threshold: f64,
80 #[serde(skip_serializing_if = "Option::is_none")]
82 window_seconds: Option<u64>,
83 },
84 Other(String),
86}
87
88impl ViolationCause {
89 pub fn cause_label(&self) -> &'static str {
91 match self {
92 ViolationCause::SeqDivergence { .. } => "seq_divergence",
93 ViolationCause::ContentMismatch { .. } => "content_mismatch",
94 ViolationCause::DeliveryMismatch { .. } => "delivery_mismatch",
95 ViolationCause::AccountingMismatch { .. } => "accounting_mismatch",
96 ViolationCause::Divergence { .. } => "divergence",
97 ViolationCause::Other(_) => "other",
98 }
99 }
100}
101
102#[derive(Debug, Clone, Serialize, Deserialize)]
104pub struct HashMismatch {
105 pub index: JournalIndex,
106 pub writer_event_id: Option<EventId>,
107 pub reader_event_id: Option<EventId>,
108}
109
110#[derive(Default, Debug)]
112pub struct ContractState {
113 inner: HashMap<TypeId, Box<dyn Any + Send + Sync>>,
114}
115
116impl ContractState {
117 pub fn get<T: 'static>(&self) -> Option<&T> {
119 self.inner
120 .get(&TypeId::of::<T>())
121 .and_then(|b| b.downcast_ref::<T>())
122 }
123
124 pub fn get_mut<T: 'static>(&mut self) -> Option<&mut T> {
126 self.inner
127 .get_mut(&TypeId::of::<T>())
128 .and_then(|b| b.downcast_mut::<T>())
129 }
130
131 pub fn insert<T: 'static + Send + Sync>(&mut self, value: T) {
133 self.inner.insert(
134 TypeId::of::<T>(),
135 Box::new(value) as Box<dyn Any + Send + Sync>,
136 );
137 }
138
139 pub fn get_or_insert_with<T, F>(&mut self, default: F) -> &mut T
141 where
142 T: 'static + Send + Sync,
143 F: FnOnce() -> T,
144 {
145 if !self.inner.contains_key(&TypeId::of::<T>()) {
146 self.insert(default());
147 }
148 self.get_mut::<T>()
149 .expect("type just inserted should be present")
150 }
151}
152
153#[derive(Debug)]
155pub struct ContractWriteContext {
156 pub writer_stage: StageId,
157 pub writer_seq: SeqNo,
158 pub state: ContractState,
159}
160
161impl ContractWriteContext {
162 pub fn new(writer_stage: StageId) -> Self {
163 Self {
164 writer_stage,
165 writer_seq: SeqNo(0),
166 state: ContractState::default(),
167 }
168 }
169}
170
171#[derive(Debug)]
173pub struct ContractReadContext {
174 pub reader_stage: StageId,
175 pub reader_seq: SeqNo,
176 pub upstream_stage: StageId,
177 pub state: ContractState,
178}
179
180impl ContractReadContext {
181 pub fn new(reader_stage: StageId, upstream_stage: StageId) -> Self {
182 Self {
183 reader_stage,
184 reader_seq: SeqNo(0),
185 upstream_stage,
186 state: ContractState::default(),
187 }
188 }
189}
190
191#[derive(Debug)]
193pub struct ContractContext<'a> {
194 pub upstream_stage: StageId,
195 pub downstream_stage: StageId,
196 pub write_state: &'a ContractState,
197 pub read_state: &'a ContractState,
198}
199
200#[derive(Debug, Clone, Copy, PartialEq, Eq)]
206pub enum ContractEventScope {
207 UpstreamAuthored,
209 PhysicalEdge,
211}
212
213pub trait Contract: Send + Sync {
215 fn name(&self) -> &str;
217
218 fn contract_name(&self) -> ContractName {
220 ContractName::from(self.name())
221 }
222
223 fn event_scope(&self) -> ContractEventScope {
225 ContractEventScope::UpstreamAuthored
226 }
227
228 fn on_write(&self, event: &ChainEvent, ctx: &mut ContractWriteContext);
230
231 fn on_read(&self, event: &ChainEvent, ctx: &mut ContractReadContext);
233
234 fn verify(&self, ctx: &ContractContext<'_>) -> ContractResult;
236
237 fn check_progress(&self, _ctx: &ContractContext<'_>) -> Option<ContractViolation> {
239 None
240 }
241}
242
243#[derive(Debug, Default)]
249struct WriterCount(pub u64);
250
251#[derive(Debug, Default)]
253struct ReaderCount(pub u64);
254
255pub struct TransportContract;
258
259impl Default for TransportContract {
260 fn default() -> Self {
261 Self::new()
262 }
263}
264
265impl TransportContract {
266 pub const NAME: &'static str = "TransportContract";
267
268 pub fn new() -> Self {
269 Self
270 }
271}
272
273impl Contract for TransportContract {
274 fn name(&self) -> &str {
275 Self::NAME
276 }
277
278 fn on_write(&self, event: &ChainEvent, ctx: &mut ContractWriteContext) {
279 if let ChainPayload::FlowControl(
287 crate::event::payloads::flow_control_payload::FlowControlPayload::Eof {
288 writer_seq: Some(seq),
289 ..
290 },
291 ) = &event.payload
292 {
293 let counter = ctx
294 .state
295 .get_or_insert_with::<WriterCount, _>(WriterCount::default);
296 counter.0 = seq.0;
297 }
298 }
299
300 fn on_read(&self, event: &ChainEvent, ctx: &mut ContractReadContext) {
301 if event.consumes_data_credit() {
302 let counter = ctx
303 .state
304 .get_or_insert_with::<ReaderCount, _>(ReaderCount::default);
305 counter.0 = counter.0.saturating_add(1);
306 }
307 }
308
309 fn verify(&self, ctx: &ContractContext<'_>) -> ContractResult {
310 let writer_count = ctx
311 .write_state
312 .get::<WriterCount>()
313 .map(|c| c.0)
314 .unwrap_or(0);
315 let reader_count = ctx
316 .read_state
317 .get::<ReaderCount>()
318 .map(|c| c.0)
319 .unwrap_or(0);
320
321 if writer_count == reader_count {
322 ContractResult::Passed(ContractEvidence {
323 contract_name: self.contract_name(),
324 upstream_stage: ctx.upstream_stage,
325 downstream_stage: ctx.downstream_stage,
326 verified_at: Utc::now(),
327 details: JsonValue::Object(
328 [
329 ("writer_seq".to_string(), JsonValue::from(writer_count)),
330 ("reader_seq".to_string(), JsonValue::from(reader_count)),
331 ]
332 .into_iter()
333 .collect(),
334 ),
335 })
336 } else {
337 ContractResult::Failed(ContractViolation {
338 contract_name: self.contract_name(),
339 upstream_stage: ctx.upstream_stage,
340 downstream_stage: ctx.downstream_stage,
341 detected_at: Utc::now(),
342 cause: ViolationCause::SeqDivergence {
343 advertised: Some(SeqNo(writer_count)),
344 reader: SeqNo(reader_count),
345 },
346 details: JsonValue::Object(
347 [
348 ("writer_seq".to_string(), JsonValue::from(writer_count)),
349 ("reader_seq".to_string(), JsonValue::from(reader_count)),
350 (
351 "delta".to_string(),
352 JsonValue::from(writer_count as i64 - reader_count as i64),
353 ),
354 ]
355 .into_iter()
356 .collect(),
357 ),
358 })
359 }
360 }
361}
362
363#[derive(Debug, Default)]
369struct SourceWriterState {
370 expected_count: Option<u64>,
371 eof_writer_seq: Option<u64>,
372}
373
374pub struct SourceContract;
382
383impl Default for SourceContract {
384 fn default() -> Self {
385 Self::new()
386 }
387}
388
389impl SourceContract {
390 pub const NAME: &'static str = "SourceContract";
391
392 pub fn new() -> Self {
393 Self
394 }
395}
396
397impl Contract for SourceContract {
398 fn name(&self) -> &str {
399 Self::NAME
400 }
401
402 fn on_write(&self, event: &ChainEvent, ctx: &mut ContractWriteContext) {
403 use crate::event::payloads::flow_control_payload::FlowControlPayload;
404
405 if let ChainPayload::FlowControl(payload) = &event.payload {
406 match payload {
407 FlowControlPayload::SourceContract {
408 expected_count: Some(count),
409 ..
410 } => {
411 let state = ctx
412 .state
413 .get_or_insert_with::<SourceWriterState, _>(SourceWriterState::default);
414 state.expected_count = Some(count.0);
415 }
416 FlowControlPayload::Eof {
417 writer_seq: Some(seq),
418 ..
419 } => {
420 let state = ctx
421 .state
422 .get_or_insert_with::<SourceWriterState, _>(SourceWriterState::default);
423 state.eof_writer_seq = Some(seq.0);
424 }
425 _ => {}
426 }
427 }
428 }
429
430 fn on_read(&self, _event: &ChainEvent, _ctx: &mut ContractReadContext) {
431 }
433
434 fn verify(&self, ctx: &ContractContext<'_>) -> ContractResult {
435 let state = match ctx.write_state.get::<SourceWriterState>() {
436 Some(s) => s,
437 None => {
438 return ContractResult::Passed(ContractEvidence {
440 contract_name: self.contract_name(),
441 upstream_stage: ctx.upstream_stage,
442 downstream_stage: ctx.downstream_stage,
443 verified_at: Utc::now(),
444 details: JsonValue::String(
445 "no source_contract / EOF evidence observed".to_string(),
446 ),
447 });
448 }
449 };
450
451 match (state.expected_count, state.eof_writer_seq) {
452 (Some(expected), Some(observed)) if expected != observed => {
453 ContractResult::Failed(ContractViolation {
454 contract_name: self.contract_name(),
455 upstream_stage: ctx.upstream_stage,
456 downstream_stage: ctx.downstream_stage,
457 detected_at: Utc::now(),
458 cause: ViolationCause::Other("source_expected_count_mismatch".into()),
459 details: JsonValue::Object(
460 [
461 ("expected_count".to_string(), JsonValue::from(expected)),
462 ("observed_writer_seq".to_string(), JsonValue::from(observed)),
463 (
464 "delta".to_string(),
465 JsonValue::from(observed as i64 - expected as i64),
466 ),
467 ]
468 .into_iter()
469 .collect(),
470 ),
471 })
472 }
473 _ => ContractResult::Passed(ContractEvidence {
474 contract_name: self.contract_name(),
475 upstream_stage: ctx.upstream_stage,
476 downstream_stage: ctx.downstream_stage,
477 verified_at: Utc::now(),
478 details: JsonValue::Object(
479 [
480 (
481 "expected_count".to_string(),
482 JsonValue::from(state.expected_count.unwrap_or(0)),
483 ),
484 (
485 "observed_writer_seq".to_string(),
486 JsonValue::from(state.eof_writer_seq.unwrap_or(0)),
487 ),
488 ]
489 .into_iter()
490 .collect(),
491 ),
492 }),
493 }
494 }
495}
496
497#[derive(Debug, Default)]
502struct DeliveryState {
503 pending: HashSet<EventId>,
505 pending_peak: usize,
506
507 consumed_total: u64,
509 receipted_total: u64,
510 buffered_count: u64,
511 success_count: u64,
512 partial_count: u64,
513 failed_count: u64,
514
515 orphan_deliveries: u64,
519}
520
521pub struct DeliveryContract {
528 state: Mutex<DeliveryState>,
529}
530
531impl Default for DeliveryContract {
532 fn default() -> Self {
533 Self {
534 state: Mutex::new(DeliveryState::default()),
535 }
536 }
537}
538
539impl DeliveryContract {
540 pub const NAME: &'static str = "DeliveryContract";
541}
542
543impl Contract for DeliveryContract {
544 fn name(&self) -> &str {
545 Self::NAME
546 }
547
548 fn on_write(&self, event: &ChainEvent, _ctx: &mut ContractWriteContext) {
549 let ChainPayload::Delivery(payload) = &event.payload else {
550 return;
551 };
552
553 if event.causality.parent_ids.is_empty() {
554 return;
555 }
556
557 let mut st = self.state.lock().expect("DeliveryContract state poisoned");
558
559 if matches!(&payload.result, DeliveryResult::Buffered { .. }) {
560 st.buffered_count = st.buffered_count.saturating_add(1);
561 return;
562 }
563
564 let mut matched_any = false;
565 for parent_id in &event.causality.parent_ids {
566 if st.pending.remove(parent_id) {
567 matched_any = true;
568 st.receipted_total = st.receipted_total.saturating_add(1);
569 } else {
570 st.orphan_deliveries = st.orphan_deliveries.saturating_add(1);
571 }
572 }
573
574 if !matched_any {
575 return;
576 }
577
578 match &payload.result {
579 DeliveryResult::Buffered { .. } => {}
580 DeliveryResult::Success { .. } => {
581 st.success_count = st
582 .success_count
583 .saturating_add(event.causality.parent_ids.len() as u64);
584 }
585 DeliveryResult::Partial { .. } => {
586 st.partial_count = st
587 .partial_count
588 .saturating_add(event.causality.parent_ids.len() as u64);
589 }
590 DeliveryResult::Failed { .. } => {
591 st.failed_count = st
592 .failed_count
593 .saturating_add(event.causality.parent_ids.len() as u64);
594 }
595 }
596 }
597
598 fn on_read(&self, event: &ChainEvent, _ctx: &mut ContractReadContext) {
599 if !event.consumes_data_credit() {
601 return;
602 }
603
604 let mut st = self.state.lock().expect("DeliveryContract state poisoned");
605
606 st.consumed_total = st.consumed_total.saturating_add(1);
607 st.pending.insert(event.id);
608 st.pending_peak = st.pending_peak.max(st.pending.len());
609 }
610
611 fn verify(&self, ctx: &ContractContext<'_>) -> ContractResult {
612 let st = self.state.lock().expect("DeliveryContract state poisoned");
613
614 let missing_count = st.pending.len();
615 let orphan_count = st.orphan_deliveries as usize;
616
617 let mut missing_sample: Vec<EventId> = st.pending.iter().take(100).copied().collect();
618 missing_sample.sort();
619
620 if missing_count == 0 && orphan_count == 0 {
621 ContractResult::Passed(ContractEvidence {
622 contract_name: self.contract_name(),
623 upstream_stage: ctx.upstream_stage,
624 downstream_stage: ctx.downstream_stage,
625 verified_at: Utc::now(),
626 details: json!({
627 "consumed_total": st.consumed_total,
628 "receipted_total": st.receipted_total,
629 "pending_peak": st.pending_peak,
630 "buffered_count": st.buffered_count,
631 "has_failures": st.failed_count > 0,
632 "success_count": st.success_count,
633 "partial_count": st.partial_count,
634 "failed_count": st.failed_count,
635 }),
636 })
637 } else {
638 ContractResult::Failed(ContractViolation {
639 contract_name: self.contract_name(),
640 upstream_stage: ctx.upstream_stage,
641 downstream_stage: ctx.downstream_stage,
642 detected_at: Utc::now(),
643 cause: ViolationCause::DeliveryMismatch {
644 missing_deliveries: missing_count,
645 orphan_deliveries: orphan_count,
646 },
647 details: json!({
648 "consumed_total": st.consumed_total,
649 "receipted_total": st.receipted_total,
650 "pending_peak": st.pending_peak,
651 "buffered_count": st.buffered_count,
652 "missing_count": missing_count,
653 "orphan_count": orphan_count,
654 "missing_event_ids": missing_sample
655 .iter()
656 .map(|id| id.to_string())
657 .collect::<Vec<_>>(),
658 }),
659 })
660 }
661 }
662}
663
664#[derive(Debug, Clone)]
676pub struct DivergenceThresholds {
677 pub window: Duration,
679
680 pub signal_to_data_ratio: f64,
682
683 pub max_signals_when_no_data: u64,
685
686 pub max_cycle_depth: u16,
688
689 pub state_ttl: Duration,
694}
695
696impl Default for DivergenceThresholds {
697 fn default() -> Self {
698 Self {
699 window: Duration::from_secs(60),
700 signal_to_data_ratio: 10.0,
701 max_signals_when_no_data: 1_000,
702 max_cycle_depth: 30,
704 state_ttl: Duration::from_secs(300),
705 }
706 }
707}
708
709#[derive(Debug, Default)]
710struct DivergenceState {
711 window_start: Option<Instant>,
712 data_events: u64,
713 flow_control_signals: u64,
714 max_cycle_depth_observed: u16,
715}
716
717pub struct DivergenceContract {
722 scc_id: crate::SccId,
723 thresholds: DivergenceThresholds,
724 state: Mutex<DivergenceState>,
725}
726
727impl DivergenceContract {
728 pub const NAME: &'static str = "DivergenceContract";
729
730 pub fn new(scc_id: crate::SccId) -> Self {
732 Self::with_thresholds(scc_id, DivergenceThresholds::default())
733 }
734
735 pub fn with_thresholds(scc_id: crate::SccId, thresholds: DivergenceThresholds) -> Self {
737 Self {
738 scc_id,
739 thresholds,
740 state: Mutex::new(DivergenceState::default()),
741 }
742 }
743
744 fn check_signal_to_data_ratio(
745 &self,
746 ctx: &ContractContext<'_>,
747 st: &DivergenceState,
748 ) -> Option<ContractViolation> {
749 let window_seconds = Some(self.thresholds.window.as_secs());
750
751 if st.data_events == 0 {
752 if st.flow_control_signals > self.thresholds.max_signals_when_no_data {
753 return Some(ContractViolation {
754 contract_name: self.contract_name(),
755 upstream_stage: ctx.upstream_stage,
756 downstream_stage: ctx.downstream_stage,
757 detected_at: Utc::now(),
758 cause: ViolationCause::Divergence {
759 predicate: "signals_when_no_data".to_string(),
760 observed: st.flow_control_signals as f64,
761 threshold: self.thresholds.max_signals_when_no_data as f64,
762 window_seconds,
763 },
764 details: json!({
765 "window_seconds": self.thresholds.window.as_secs(),
766 "flow_control_signals": st.flow_control_signals,
767 "data_events": st.data_events,
768 "max_signals_when_no_data": self.thresholds.max_signals_when_no_data,
769 }),
770 });
771 }
772 return None;
773 }
774
775 let observed_ratio = st.flow_control_signals as f64 / st.data_events as f64;
776 if observed_ratio > self.thresholds.signal_to_data_ratio {
777 return Some(ContractViolation {
778 contract_name: self.contract_name(),
779 upstream_stage: ctx.upstream_stage,
780 downstream_stage: ctx.downstream_stage,
781 detected_at: Utc::now(),
782 cause: ViolationCause::Divergence {
783 predicate: "signal_to_data_ratio".to_string(),
784 observed: observed_ratio,
785 threshold: self.thresholds.signal_to_data_ratio,
786 window_seconds,
787 },
788 details: json!({
789 "window_seconds": self.thresholds.window.as_secs(),
790 "flow_control_signals": st.flow_control_signals,
791 "data_events": st.data_events,
792 "observed_ratio": observed_ratio,
793 "threshold_ratio": self.thresholds.signal_to_data_ratio,
794 }),
795 });
796 }
797
798 None
799 }
800
801 fn check_cycle_depth(
802 &self,
803 ctx: &ContractContext<'_>,
804 st: &DivergenceState,
805 ) -> Option<ContractViolation> {
806 if st.max_cycle_depth_observed > self.thresholds.max_cycle_depth {
807 return Some(ContractViolation {
808 contract_name: self.contract_name(),
809 upstream_stage: ctx.upstream_stage,
810 downstream_stage: ctx.downstream_stage,
811 detected_at: Utc::now(),
812 cause: ViolationCause::Divergence {
813 predicate: "cycle_depth".to_string(),
814 observed: st.max_cycle_depth_observed as f64,
815 threshold: self.thresholds.max_cycle_depth as f64,
816 window_seconds: None,
817 },
818 details: json!({
819 "scc_id": self.scc_id.to_string(),
820 "max_cycle_depth_observed": st.max_cycle_depth_observed,
821 "max_cycle_depth": self.thresholds.max_cycle_depth,
822 }),
823 });
824 }
825 None
826 }
827}
828
829impl Contract for DivergenceContract {
830 fn name(&self) -> &str {
831 DivergenceContract::NAME
832 }
833
834 fn event_scope(&self) -> ContractEventScope {
835 ContractEventScope::PhysicalEdge
836 }
837
838 fn on_write(&self, _event: &ChainEvent, _ctx: &mut ContractWriteContext) {
839 }
841
842 fn on_read(&self, event: &ChainEvent, _ctx: &mut ContractReadContext) {
843 let mut st = self
844 .state
845 .lock()
846 .expect("DivergenceContract state poisoned");
847
848 if st.window_start.is_none() {
849 st.window_start = Some(Instant::now());
850 }
851
852 match &event.payload {
853 _ if event.consumes_data_credit() => {
854 st.data_events = st.data_events.saturating_add(1);
855 }
856 ChainPayload::FlowControl(_) => {
857 st.flow_control_signals = st.flow_control_signals.saturating_add(1);
858 }
859 _ => {}
860 }
861
862 if event.consumes_data_credit() && event.cycle_scc_id == Some(self.scc_id) {
865 if let Some(depth) = event.cycle_depth {
866 st.max_cycle_depth_observed = st.max_cycle_depth_observed.max(depth.as_u16());
867 }
868 }
869 }
870
871 fn verify(&self, ctx: &ContractContext<'_>) -> ContractResult {
872 let st = self
873 .state
874 .lock()
875 .expect("DivergenceContract state poisoned");
876 let observed_ratio = if st.data_events == 0 {
877 None
878 } else {
879 Some(st.flow_control_signals as f64 / st.data_events as f64)
880 };
881
882 ContractResult::Passed(ContractEvidence {
883 contract_name: self.contract_name(),
884 upstream_stage: ctx.upstream_stage,
885 downstream_stage: ctx.downstream_stage,
886 verified_at: Utc::now(),
887 details: json!({
888 "scc_id": self.scc_id.to_string(),
889 "window_seconds": self.thresholds.window.as_secs(),
890 "signal_to_data_ratio_threshold": self.thresholds.signal_to_data_ratio,
891 "max_signals_when_no_data": self.thresholds.max_signals_when_no_data,
892 "max_cycle_depth": self.thresholds.max_cycle_depth,
893 "data_events_observed_in_window": st.data_events,
894 "flow_control_signals_observed_in_window": st.flow_control_signals,
895 "signal_to_data_ratio_observed": observed_ratio,
896 "max_cycle_depth_observed": st.max_cycle_depth_observed,
897 }),
898 })
899 }
900
901 fn check_progress(&self, ctx: &ContractContext<'_>) -> Option<ContractViolation> {
902 let now = Instant::now();
903 let mut st = self
904 .state
905 .lock()
906 .expect("DivergenceContract state poisoned");
907
908 let Some(window_start) = st.window_start else {
909 st.window_start = Some(now);
910 return None;
911 };
912
913 let window_elapsed = now.duration_since(window_start) >= self.thresholds.window;
914
915 if let Some(v) = self.check_cycle_depth(ctx, &st) {
916 return Some(v);
917 }
918 if let Some(v) = self.check_signal_to_data_ratio(ctx, &st) {
919 return Some(v);
920 }
921
922 if window_elapsed {
923 st.window_start = Some(now);
924 st.data_events = 0;
925 st.flow_control_signals = 0;
926 st.max_cycle_depth_observed = 0;
927 }
928
929 None
930 }
931}
932
933#[cfg(test)]
934mod tests {
935 use super::*;
936 use crate::event::payloads::delivery_payload::{DeliveryMethod, DeliveryPayload};
937 use crate::event::provenance::causality_context::CausalityContext;
938 use crate::event::types::SeqNo;
939 use crate::event::{ChainEventFactory, ConsumptionProgressEventParams};
940 use crate::{CycleDepth, WriterId};
941
942 fn dummy_ctx() -> (ContractWriteContext, ContractReadContext, StageId, StageId) {
943 let upstream_stage = StageId::new();
944 let downstream_stage = StageId::new();
945 let write_ctx = ContractWriteContext::new(upstream_stage);
946 let read_ctx = ContractReadContext::new(downstream_stage, upstream_stage);
947 (write_ctx, read_ctx, upstream_stage, downstream_stage)
948 }
949
950 #[test]
951 fn delivery_contract_empty_passes() {
952 let contract = DeliveryContract::default();
953 let (write_ctx, read_ctx, upstream, downstream) = dummy_ctx();
954 let ctx = ContractContext {
955 upstream_stage: upstream,
956 downstream_stage: downstream,
957 write_state: &write_ctx.state,
958 read_state: &read_ctx.state,
959 };
960 assert!(matches!(contract.verify(&ctx), ContractResult::Passed(_)));
961 }
962
963 #[test]
964 fn delivery_contract_missing_receipt_fails() {
965 let contract = DeliveryContract::default();
966 let (write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
967
968 let consumed =
969 ChainEventFactory::data_event(WriterId::from(upstream), "test.event", json!({"a": 1}));
970 contract.on_read(&consumed, &mut read_ctx);
971
972 let ctx = ContractContext {
973 upstream_stage: upstream,
974 downstream_stage: downstream,
975 write_state: &write_ctx.state,
976 read_state: &read_ctx.state,
977 };
978
979 match contract.verify(&ctx) {
980 ContractResult::Failed(v) => match v.cause {
981 ViolationCause::DeliveryMismatch {
982 missing_deliveries,
983 orphan_deliveries,
984 } => {
985 assert_eq!(missing_deliveries, 1);
986 assert_eq!(orphan_deliveries, 0);
987 }
988 other => panic!("unexpected cause: {other:?}"),
989 },
990 other => panic!("expected failure, got: {other:?}"),
991 }
992 }
993
994 #[test]
995 fn delivery_contract_failed_receipt_is_accounted_for() {
996 let contract = DeliveryContract::default();
997 let (mut write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
998
999 let consumed =
1000 ChainEventFactory::data_event(WriterId::from(upstream), "test.event", json!({"a": 1}));
1001 let parent_id = consumed.id;
1002 contract.on_read(&consumed, &mut read_ctx);
1003
1004 let receipt_payload = DeliveryPayload::failed(DeliveryMethod::Noop, "sink_error", "boom");
1005 let receipt =
1006 ChainEventFactory::delivery_event(WriterId::from(downstream), receipt_payload)
1007 .with_causality(CausalityContext::with_parent(parent_id));
1008
1009 contract.on_write(&receipt, &mut write_ctx);
1010
1011 let ctx = ContractContext {
1012 upstream_stage: upstream,
1013 downstream_stage: downstream,
1014 write_state: &write_ctx.state,
1015 read_state: &read_ctx.state,
1016 };
1017 assert!(matches!(contract.verify(&ctx), ContractResult::Passed(_)));
1018 }
1019
1020 #[test]
1021 fn delivery_contract_buffered_receipt_does_not_clear_pending() {
1022 let contract = DeliveryContract::default();
1023 let (mut write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1024
1025 let consumed =
1026 ChainEventFactory::data_event(WriterId::from(upstream), "test.event", json!({"a": 1}));
1027 let parent_id = consumed.id;
1028 contract.on_read(&consumed, &mut read_ctx);
1029
1030 let receipt_payload =
1031 DeliveryPayload::buffered(DeliveryMethod::Noop, None);
1032 let receipt =
1033 ChainEventFactory::delivery_event(WriterId::from(downstream), receipt_payload)
1034 .with_causality(CausalityContext::with_parent(parent_id));
1035
1036 contract.on_write(&receipt, &mut write_ctx);
1037
1038 let ctx = ContractContext {
1039 upstream_stage: upstream,
1040 downstream_stage: downstream,
1041 write_state: &write_ctx.state,
1042 read_state: &read_ctx.state,
1043 };
1044
1045 match contract.verify(&ctx) {
1046 ContractResult::Failed(v) => match v.cause {
1047 ViolationCause::DeliveryMismatch {
1048 missing_deliveries,
1049 orphan_deliveries,
1050 } => {
1051 assert_eq!(missing_deliveries, 1);
1052 assert_eq!(orphan_deliveries, 0);
1053 }
1054 other => panic!("unexpected cause: {other:?}"),
1055 },
1056 other => panic!("expected failure, got: {other:?}"),
1057 }
1058 }
1059
1060 #[test]
1061 fn delivery_contract_multi_parent_receipt_clears_all_parents() {
1062 let contract = DeliveryContract::default();
1063 let (mut write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1064
1065 let consumed_a =
1066 ChainEventFactory::data_event(WriterId::from(upstream), "test.event", json!({"a": 1}));
1067 let consumed_b =
1068 ChainEventFactory::data_event(WriterId::from(upstream), "test.event", json!({"b": 2}));
1069 contract.on_read(&consumed_a, &mut read_ctx);
1070 contract.on_read(&consumed_b, &mut read_ctx);
1071
1072 let receipt_payload = DeliveryPayload::success(DeliveryMethod::Noop, None);
1073 let receipt =
1074 ChainEventFactory::delivery_event(WriterId::from(downstream), receipt_payload)
1075 .with_causality(
1076 CausalityContext::with_parent(consumed_a.id).add_parent(consumed_b.id),
1077 );
1078
1079 contract.on_write(&receipt, &mut write_ctx);
1080
1081 let ctx = ContractContext {
1082 upstream_stage: upstream,
1083 downstream_stage: downstream,
1084 write_state: &write_ctx.state,
1085 read_state: &read_ctx.state,
1086 };
1087 assert!(matches!(contract.verify(&ctx), ContractResult::Passed(_)));
1088 }
1089
1090 #[test]
1091 fn delivery_contract_orphan_receipt_fails() {
1092 let contract = DeliveryContract::default();
1093 let (mut write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1094
1095 let receipt_payload = DeliveryPayload::success(DeliveryMethod::Noop, None);
1097 let receipt =
1098 ChainEventFactory::delivery_event(WriterId::from(downstream), receipt_payload)
1099 .with_causality(CausalityContext::with_parent(EventId::new()));
1100
1101 contract.on_write(&receipt, &mut write_ctx);
1102
1103 let ctx = ContractContext {
1104 upstream_stage: upstream,
1105 downstream_stage: downstream,
1106 write_state: &write_ctx.state,
1107 read_state: &read_ctx.state,
1108 };
1109
1110 match contract.verify(&ctx) {
1111 ContractResult::Failed(v) => match v.cause {
1112 ViolationCause::DeliveryMismatch {
1113 missing_deliveries,
1114 orphan_deliveries,
1115 } => {
1116 assert_eq!(missing_deliveries, 0);
1117 assert_eq!(orphan_deliveries, 1);
1118 }
1119 other => panic!("unexpected cause: {other:?}"),
1120 },
1121 other => panic!("expected failure, got: {other:?}"),
1122 }
1123
1124 let _ = &mut read_ctx;
1126 }
1127
1128 #[test]
1129 fn divergence_contract_signal_ratio_violation_emits_progress_violation() {
1130 let scc_id = crate::SccId::from(crate::Ulid::new());
1131 let thresholds = DivergenceThresholds {
1132 window: Duration::from_secs(60),
1133 signal_to_data_ratio: 2.0,
1134 max_signals_when_no_data: 10,
1135 max_cycle_depth: 30,
1136 state_ttl: Duration::from_secs(300),
1137 };
1138 let contract = DivergenceContract::with_thresholds(scc_id, thresholds);
1139 let (write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1140
1141 let data = ChainEventFactory::data_event(
1143 crate::WriterId::from(upstream),
1144 "test.event",
1145 json!({"a": 1}),
1146 );
1147 contract.on_read(&data, &mut read_ctx);
1148
1149 let progress = ChainEventFactory::consumption_progress_event(
1150 crate::WriterId::from(upstream),
1151 ConsumptionProgressEventParams {
1152 reader_seq: SeqNo(1),
1153 last_event_id: None,
1154 vector_clock: None,
1155 eof_seen: false,
1156 reader_path: crate::event::types::JournalPath("x".to_string()),
1157 reader_index: crate::event::types::JournalIndex(0),
1158 advertised_writer_seq: None,
1159 advertised_vector_clock: None,
1160 stalled_since: None,
1161 },
1162 );
1163 contract.on_read(&progress, &mut read_ctx);
1164 contract.on_read(&progress, &mut read_ctx);
1165 contract.on_read(&progress, &mut read_ctx);
1166
1167 let ctx = ContractContext {
1168 upstream_stage: upstream,
1169 downstream_stage: downstream,
1170 write_state: &write_ctx.state,
1171 read_state: &read_ctx.state,
1172 };
1173
1174 let Some(v) = contract.check_progress(&ctx) else {
1175 panic!("expected divergence violation, got None");
1176 };
1177
1178 match v.cause {
1179 ViolationCause::Divergence {
1180 predicate,
1181 observed,
1182 threshold,
1183 window_seconds,
1184 } => {
1185 assert_eq!(predicate, "signal_to_data_ratio");
1186 assert!(observed > threshold);
1187 assert_eq!(window_seconds, Some(60));
1188 }
1189 other => panic!("unexpected cause: {other:?}"),
1190 }
1191 }
1192
1193 #[test]
1194 fn divergence_contract_does_not_apply_cycle_depth_to_flow_control_signals() {
1195 let scc_id = crate::SccId::from(crate::Ulid::new());
1196 let thresholds = DivergenceThresholds {
1197 window: Duration::from_secs(60),
1198 signal_to_data_ratio: 10.0,
1199 max_signals_when_no_data: 1_000,
1200 max_cycle_depth: 1,
1201 state_ttl: Duration::from_secs(300),
1202 };
1203 let contract = DivergenceContract::with_thresholds(scc_id, thresholds);
1204 let (write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1205
1206 let progress = ChainEventFactory::consumption_progress_event(
1207 crate::WriterId::from(upstream),
1208 ConsumptionProgressEventParams {
1209 reader_seq: SeqNo(0),
1210 last_event_id: None,
1211 vector_clock: None,
1212 eof_seen: false,
1213 reader_path: crate::event::types::JournalPath("x".to_string()),
1214 reader_index: crate::event::types::JournalIndex(0),
1215 advertised_writer_seq: None,
1216 advertised_vector_clock: None,
1217 stalled_since: None,
1218 },
1219 );
1220 contract.on_read(&progress, &mut read_ctx);
1221
1222 let ctx = ContractContext {
1223 upstream_stage: upstream,
1224 downstream_stage: downstream,
1225 write_state: &write_ctx.state,
1226 read_state: &read_ctx.state,
1227 };
1228
1229 assert!(contract.check_progress(&ctx).is_none());
1231 }
1232
1233 #[test]
1234 fn divergence_contract_signals_when_no_data_violation_emits_progress_violation() {
1235 let scc_id = crate::SccId::from(crate::Ulid::new());
1236 let thresholds = DivergenceThresholds {
1237 window: Duration::from_secs(60),
1238 signal_to_data_ratio: 10.0,
1239 max_signals_when_no_data: 2,
1240 max_cycle_depth: 30,
1241 state_ttl: Duration::from_secs(300),
1242 };
1243 let contract = DivergenceContract::with_thresholds(scc_id, thresholds);
1244 let (write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1245
1246 let progress = ChainEventFactory::consumption_progress_event(
1247 crate::WriterId::from(upstream),
1248 ConsumptionProgressEventParams {
1249 reader_seq: SeqNo(0),
1250 last_event_id: None,
1251 vector_clock: None,
1252 eof_seen: false,
1253 reader_path: crate::event::types::JournalPath("x".to_string()),
1254 reader_index: crate::event::types::JournalIndex(0),
1255 advertised_writer_seq: None,
1256 advertised_vector_clock: None,
1257 stalled_since: None,
1258 },
1259 );
1260 contract.on_read(&progress, &mut read_ctx);
1261 contract.on_read(&progress, &mut read_ctx);
1262 contract.on_read(&progress, &mut read_ctx);
1263
1264 let ctx = ContractContext {
1265 upstream_stage: upstream,
1266 downstream_stage: downstream,
1267 write_state: &write_ctx.state,
1268 read_state: &read_ctx.state,
1269 };
1270
1271 let Some(v) = contract.check_progress(&ctx) else {
1272 panic!("expected divergence violation, got None");
1273 };
1274
1275 match v.cause {
1276 ViolationCause::Divergence {
1277 predicate,
1278 observed,
1279 threshold,
1280 window_seconds,
1281 } => {
1282 assert_eq!(predicate, "signals_when_no_data");
1283 assert!(observed > threshold);
1284 assert_eq!(window_seconds, Some(60));
1285 }
1286 other => panic!("unexpected cause: {other:?}"),
1287 }
1288 }
1289
1290 #[test]
1291 fn divergence_contract_cycle_depth_violation_emits_progress_violation() {
1292 let scc_id = crate::SccId::from(crate::Ulid::new());
1293 let thresholds = DivergenceThresholds {
1294 window: Duration::from_secs(60),
1295 signal_to_data_ratio: 10.0,
1296 max_signals_when_no_data: 1_000,
1297 max_cycle_depth: 3,
1298 state_ttl: Duration::from_secs(300),
1299 };
1300 let contract = DivergenceContract::with_thresholds(scc_id, thresholds);
1301 let (write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1302
1303 let mut data =
1304 ChainEventFactory::data_event(crate::WriterId::from(upstream), "test.event", json!({}));
1305 data.cycle_scc_id = Some(scc_id);
1306 data.cycle_depth = Some(CycleDepth::new(4));
1307
1308 contract.on_read(&data, &mut read_ctx);
1309
1310 let ctx = ContractContext {
1311 upstream_stage: upstream,
1312 downstream_stage: downstream,
1313 write_state: &write_ctx.state,
1314 read_state: &read_ctx.state,
1315 };
1316
1317 let Some(v) = contract.check_progress(&ctx) else {
1318 panic!("expected divergence violation, got None");
1319 };
1320
1321 match v.cause {
1322 ViolationCause::Divergence {
1323 predicate,
1324 observed,
1325 threshold,
1326 window_seconds,
1327 } => {
1328 assert_eq!(predicate, "cycle_depth");
1329 assert_eq!(observed, 4.0);
1330 assert_eq!(threshold, 3.0);
1331 assert_eq!(window_seconds, None);
1332 }
1333 other => panic!("unexpected cause: {other:?}"),
1334 }
1335 }
1336
1337 #[test]
1338 fn divergence_contract_within_bounds_returns_none() {
1339 let scc_id = crate::SccId::from(crate::Ulid::new());
1340 let thresholds = DivergenceThresholds {
1341 window: Duration::from_secs(60),
1342 signal_to_data_ratio: 10.0,
1343 max_signals_when_no_data: 1_000,
1344 max_cycle_depth: 30,
1345 state_ttl: Duration::from_secs(300),
1346 };
1347 let contract = DivergenceContract::with_thresholds(scc_id, thresholds);
1348 let (write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1349
1350 for _ in 0..10 {
1351 let data = ChainEventFactory::data_event(
1352 crate::WriterId::from(upstream),
1353 "test.event",
1354 json!({"a": 1}),
1355 );
1356 contract.on_read(&data, &mut read_ctx);
1357 }
1358
1359 let signal = ChainEventFactory::watermark_event(crate::WriterId::from(upstream), 0, None);
1360 for _ in 0..50 {
1361 contract.on_read(&signal, &mut read_ctx);
1362 }
1363
1364 let ctx = ContractContext {
1365 upstream_stage: upstream,
1366 downstream_stage: downstream,
1367 write_state: &write_ctx.state,
1368 read_state: &read_ctx.state,
1369 };
1370
1371 assert!(contract.check_progress(&ctx).is_none());
1372 }
1373
1374 #[test]
1375 fn divergence_contract_window_rollover_resets_counters() {
1376 let scc_id = crate::SccId::from(crate::Ulid::new());
1377 let thresholds = DivergenceThresholds {
1378 window: Duration::from_secs(60),
1379 signal_to_data_ratio: 10.0,
1380 max_signals_when_no_data: 1_000,
1381 max_cycle_depth: 30,
1382 state_ttl: Duration::from_secs(300),
1383 };
1384 let contract = DivergenceContract::with_thresholds(scc_id, thresholds.clone());
1385 let (write_ctx, mut read_ctx, upstream, downstream) = dummy_ctx();
1386
1387 let data = ChainEventFactory::data_event(
1388 crate::WriterId::from(upstream),
1389 "test.event",
1390 json!({"a": 1}),
1391 );
1392 contract.on_read(&data, &mut read_ctx);
1393
1394 let signal = ChainEventFactory::watermark_event(crate::WriterId::from(upstream), 0, None);
1395 contract.on_read(&signal, &mut read_ctx);
1396
1397 {
1398 let mut st = contract.state.lock().expect("state poisoned");
1399 if let Some(backdated) =
1400 Instant::now().checked_sub(thresholds.window + Duration::from_secs(1))
1401 {
1402 st.window_start = Some(backdated);
1403 } else {
1404 st.window_start = Some(Instant::now());
1405 }
1406 }
1407
1408 let ctx = ContractContext {
1409 upstream_stage: upstream,
1410 downstream_stage: downstream,
1411 write_state: &write_ctx.state,
1412 read_state: &read_ctx.state,
1413 };
1414
1415 assert!(contract.check_progress(&ctx).is_none());
1416
1417 let st = contract.state.lock().expect("state poisoned");
1418 assert_eq!(st.data_events, 0);
1419 assert_eq!(st.flow_control_signals, 0);
1420 assert_eq!(st.max_cycle_depth_observed, 0);
1421 }
1422}