1mod codec;
4mod transcript;
5
6use std::{
7 collections::{BTreeMap, HashMap, HashSet, VecDeque},
8 sync::atomic::{AtomicU64, Ordering},
9 time::{SystemTime, UNIX_EPOCH},
10};
11
12use alloy_network::{Ethereum, Network};
13use alloy_primitives::{B256, Keccak256, keccak256};
14use evm_fork_cache::reactive::{
15 BlockRef, CanonicalSequenceError, CanonicalSequenceMutation, CanonicalSequenceState,
16 ChainControl, ChainStatus, DeliveryAudience, DeliveryScope, EventSubscriber, HandlerId,
17 InputRef, InputSource, InterestOwnerSubscriber, ReactiveContext, ReactiveInput,
18 ReactiveInputBatch, ReactiveInputDelivery, ReactiveInputIdentity, ReactiveInputKind,
19 ReactiveInputRecord, ReactiveInterest, SubscriberBackfill, SubscriberCapabilities,
20 SubscriberCapability, SubscriberCheckpoint, SubscriberDeliveryToken, SubscriberError,
21 SubscriberNextBatch, SubscriberOperation, SubscriberPayloadCommitment,
22 SubscriberResumePosition, normalize_and_validate_canonical_sequence_diagnostic,
23};
24use evm_fork_cache_event_protocol::v1::{BlockMode, portable_interest};
25use serde::{Deserialize, Serialize};
26
27use self::transcript::CanonicalHasher;
28use crate::compile_portable_interests;
29
30const TOKEN_MAGIC: &[u8; 8] = b"EFCHYB2\0";
31#[cfg(test)]
32const CHECKPOINT_MAGIC: &[u8; 8] = codec::CHECKPOINT_MAGIC_V5;
33#[cfg(test)]
34const CHECKPOINT_VERSION: u16 = codec::CHECKPOINT_VERSION_V5;
35#[cfg(test)]
36const MAX_CHECKPOINT_BYTES: usize = codec::MAX_CHECKPOINT_PAYLOAD_BYTES;
37const MAX_HEADER_CANONICALIZATION_BYTES: usize = 256 * 1024;
38const BATCH_ACCOUNTING_OVERHEAD: usize = 256;
39const MIN_CONTROL_ACCOUNTING_BYTES: usize = 256;
40pub const HYBRID_MAX_RECENT_INPUTS: usize = codec::MAX_RECENT_INPUTS;
42pub const HYBRID_MAX_CANONICAL_HISTORY: usize = codec::MAX_CANONICAL_HISTORY;
44pub const HYBRID_MAX_RECENT_OWNER_ENTRIES: usize = 65_536;
48pub const HYBRID_MAX_HANDLER_ID_BYTES: usize = codec::MAX_HANDLER_ID_BYTES;
50pub const HYBRID_MAX_SOURCE_DELIVERY_TOKEN_BYTES: usize = codec::MAX_CHECKPOINT_PAYLOAD_BYTES;
56pub const HYBRID_MAX_SOURCE_CHECKPOINT_BYTES: usize = codec::MAX_CHECKPOINT_PAYLOAD_BYTES;
62const LIFECYCLE_FINGERPRINT_DOMAIN: &[u8] = b"EFCHY-LIFECYCLE-V2";
63const EMPTY_LIFECYCLE_BARRIER_DOMAIN: &[u8] = b"EFCHY-EMPTY-LIFECYCLE-BARRIER-V1";
64const HEADER_WITNESS_DOMAIN: &[u8] = b"EFCHY-HEADER-CANONICAL-JSON-V1";
65const EXCLUSIVE_TOPOLOGY_ERROR: &str = "hybrid base/unowned interests and owner-managed interests are mutually exclusive; after coverage, either mode may be cleared to effective-empty and allowed to reach Live, then base-to-owner activation requires replace_interest_owners_with_global_backfill; owner-to-base activation has no EventSubscriber global-backfill primitive and requires a fresh coordinator restored at an authoritative checkpoint";
66static NEXT_EPOCH_NONCE: AtomicU64 = AtomicU64::new(1);
67
68#[derive(Clone, Copy, Debug, PartialEq, Eq)]
70#[non_exhaustive]
71pub struct HybridConfig {
72 pub max_buffered_live_batches: usize,
76 pub max_buffered_live_records: usize,
80 pub max_buffered_live_bytes: usize,
87 pub max_source_delivery_token_bytes: usize,
97 pub max_source_checkpoint_bytes: usize,
105 pub recent_input_capacity: usize,
113 pub max_recent_owner_entries: usize,
125 pub canonical_history_capacity: usize,
133}
134
135impl Default for HybridConfig {
136 fn default() -> Self {
137 Self {
138 max_buffered_live_batches: 2_048,
139 max_buffered_live_records: 262_144,
140 max_buffered_live_bytes: 64 * 1024 * 1024,
141 max_source_delivery_token_bytes: 64 * 1024,
142 max_source_checkpoint_bytes: 1024 * 1024,
143 recent_input_capacity: 32_768,
146 max_recent_owner_entries: HYBRID_MAX_RECENT_OWNER_ENTRIES,
147 canonical_history_capacity: 512,
148 }
149 }
150}
151
152#[derive(Clone, Copy, Debug, PartialEq, Eq)]
154#[non_exhaustive]
155pub enum HybridPhase {
156 CatchingUp,
158 DrainingLive,
161 Live,
163 Recovering,
166 Poisoned,
170}
171
172#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
174#[non_exhaustive]
175pub enum HybridSource {
176 Historical,
178 Live,
180}
181
182#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
183enum HybridTokenKind {
184 Forwarded,
185 Synthetic,
186}
187
188impl HybridTokenKind {
189 const fn tag(self) -> u8 {
190 match self {
191 Self::Forwarded => 1,
192 Self::Synthetic => 2,
193 }
194 }
195
196 fn from_tag(tag: u8) -> Result<Self, SubscriberError> {
197 match tag {
198 1 => Ok(Self::Forwarded),
199 2 => Ok(Self::Synthetic),
200 _ => Err(SubscriberError::Provider(
201 "invalid hybrid delivery-token kind".into(),
202 )),
203 }
204 }
205}
206
207impl HybridSource {
208 const fn tag(self) -> u8 {
209 match self {
210 Self::Historical => 1,
211 Self::Live => 2,
212 }
213 }
214
215 fn from_tag(tag: u8) -> Result<Self, SubscriberError> {
216 match tag {
217 1 => Ok(Self::Historical),
218 2 => Ok(Self::Live),
219 _ => Err(SubscriberError::Provider(
220 "invalid hybrid delivery-token source".into(),
221 )),
222 }
223 }
224}
225
226pub struct HybridSubscriber<H, L, N: Network = Ethereum> {
313 historical: H,
314 live: L,
315 chain_id: u64,
316 config: HybridConfig,
317 phase: HybridPhase,
318 fence: Option<u64>,
319 drain_through: Option<u64>,
320 pending_cutover: Option<PendingCutover>,
321 acknowledged_historical_through: Option<u64>,
322 live_buffer: VecDeque<BufferedBatch<N>>,
323 buffered_live_records: usize,
324 buffered_live_bytes: usize,
325 recent_inputs: HashMap<ReactiveInputIdentity, AudienceCoverage>,
326 recent_order: VecDeque<ReactiveInputIdentity>,
327 pending_inputs: HashMap<(HybridSource, HybridTokenKind, Vec<u8>), PendingCoordinatorCommit>,
328 pending_output: Option<ReactiveInputBatch<N>>,
329 pending_live_rollback: Option<LiveRollback<N>>,
330 epoch: [u8; 16],
331 next_synthetic_token: u64,
332 lifecycle_generation: u64,
333 owner_generations: BTreeMap<HandlerId, u64>,
334 restored_source_replays: HashSet<HybridSource>,
335 canonical_history: VecDeque<BlockRef>,
336 coverage_head: Option<BlockRef>,
337 safe_head: Option<BlockRef>,
338 finalized_head: Option<BlockRef>,
339 certified_historical: Option<CertifiedHistoricalCoverage>,
340 recovery_anchor: Option<BlockRef>,
341 historical_position: SourcePosition,
342 live_position: SourcePosition,
343 last_committed_token: Option<StoredCommittedToken>,
344 pending_restore: Option<PendingRestore>,
345 pending_restore_preparation: Option<PendingRestorePreparation<N>>,
346 prepared_restore_position: Option<SubscriberResumePosition>,
347 poisoned: Option<String>,
348 base_interests: Vec<ReactiveInterest<N>>,
349 owners: HashMap<HandlerId, Vec<ReactiveInterest<N>>>,
350 lifecycle_intent: LifecycleIntent,
351}
352
353struct PendingCutover {
354 historical_token: Vec<u8>,
355 through: u64,
356}
357
358#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
359struct AudienceCoverage {
360 base: bool,
361 owners: BTreeMap<HandlerId, u64>,
362 block: Option<BlockRef>,
363 witness: Option<RecordWitness>,
364}
365
366#[derive(Clone, Debug)]
367struct AudienceCommit {
368 identity: ReactiveInputIdentity,
369 audience: DeliveryAudience,
370 block: Option<BlockRef>,
371 witness: RecordWitness,
372}
373
374#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
375enum WitnessLifecycle {
376 Included,
377 Safe,
378 Finalized,
379 Reorg,
380 Pending,
381}
382
383#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
387struct RecordWitness {
388 payload_digest: [u8; 32],
389 chain_id: u64,
390 lifecycle: WitnessLifecycle,
391 block: Option<BlockRef>,
392 transaction_index: Option<u64>,
393 log_index: Option<u64>,
394 log_block_timestamp: Option<u64>,
395}
396
397#[derive(Clone, Debug)]
398enum CanonicalMutation {
399 Rewind(BlockRef),
400 Reset,
401 Advance(BlockRef),
402}
403
404#[derive(Clone, Debug)]
405struct PendingCoordinatorCommit {
406 audiences: Vec<AudienceCommit>,
407 canonical: Vec<CanonicalMutation>,
408 source: HybridSource,
409 source_token: Option<Vec<u8>>,
410 source_checkpoint: Option<Vec<u8>>,
411 source_progress: Option<BlockRef>,
412 source_observed_through: Option<BlockRef>,
413 token_kind: HybridTokenKind,
414 token_bytes: Vec<u8>,
415 source_delivery_digest: [u8; 32],
416 next_safe_head: Option<BlockRef>,
417 next_finalized_head: Option<BlockRef>,
418 next_canonical_history: Vec<BlockRef>,
419 next_coverage_head: Option<BlockRef>,
420}
421
422struct PreparedEmptyLifecycleBarrier<N: Network> {
427 base_state: HybridCheckpointV5,
428 pending: PendingCoordinatorCommit,
429 output: ReactiveInputBatch<N>,
430}
431
432#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
433struct SourcePosition {
434 delivery_token: Option<Vec<u8>>,
435 checkpoint: Option<Vec<u8>>,
436 coverage_head: Option<BlockRef>,
437 canonical_history: Vec<BlockRef>,
438 delivery_digest: Option<[u8; 32]>,
439}
440
441#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
442struct StoredRecentInput {
443 identity: ReactiveInputIdentity,
444 coverage: AudienceCoverage,
445}
446
447#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
448struct StoredCommittedToken {
449 source: HybridSource,
450 kind: HybridTokenKind,
451 inner: Vec<u8>,
452}
453
454#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
455struct CertifiedHistoricalCoverage {
456 lifecycle_generation: u64,
457 through: BlockRef,
458}
459
460#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
461struct HybridCheckpointV5 {
462 chain_id: u64,
463 epoch: [u8; 16],
464 next_synthetic_token: u64,
465 lifecycle_generation: u64,
466 owner_generations: BTreeMap<HandlerId, u64>,
467 lifecycle_intent: LifecycleIntent,
468 recent_inputs: Vec<StoredRecentInput>,
469 canonical_history: Vec<BlockRef>,
470 coverage_head: Option<BlockRef>,
471 safe_head: Option<BlockRef>,
472 finalized_head: Option<BlockRef>,
473 certified_historical: Option<CertifiedHistoricalCoverage>,
474 historical_position: SourcePosition,
475 live_position: SourcePosition,
476 last_committed_token: Option<StoredCommittedToken>,
477}
478
479#[derive(Clone, Debug)]
480struct PendingRestore {
481 position: SubscriberResumePosition,
482 state: HybridCheckpointV5,
483 historical_resume: Option<SubscriberResumePosition>,
484 live_resume: Option<SubscriberResumePosition>,
485 historical_restored: bool,
486 live_restored: bool,
487}
488
489#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
490struct LifecycleIntent {
491 base: [u8; 32],
492 owners: BTreeMap<HandlerId, [u8; 32]>,
493}
494
495impl LifecycleIntent {
496 fn has_active_interests(&self) -> bool {
497 self.base != empty_interest_fingerprint()
498 || self
499 .owners
500 .values()
501 .any(|fingerprint| *fingerprint != empty_interest_fingerprint())
502 }
503
504 fn requires_restore_preparation(&self) -> bool {
505 self.has_active_interests() || !self.owners.is_empty()
506 }
507}
508
509#[derive(Clone)]
510struct PendingRestorePreparation<N: Network> {
511 position: SubscriberResumePosition,
512 intent: LifecycleIntent,
513 base_interests: Vec<ReactiveInterest<N>>,
514 owners: Vec<(HandlerId, Vec<ReactiveInterest<N>>)>,
515 live_topology_installed: bool,
516}
517
518enum OwnerTopologyTransition {
519 DestructiveReset,
522 Incremental { removed_owners: Vec<HandlerId> },
525}
526
527#[derive(Clone)]
528enum LiveRollback<N: Network> {
529 Base {
530 previous: Vec<ReactiveInterest<N>>,
531 gap_anchor: Option<BlockRef>,
532 },
533 Topology {
534 base: Vec<ReactiveInterest<N>>,
535 owners: Vec<(HandlerId, Vec<ReactiveInterest<N>>)>,
536 recovery_anchor: Option<BlockRef>,
537 gap_anchor: Option<BlockRef>,
538 },
539 Owner {
540 owner: HandlerId,
541 previous: Option<Vec<ReactiveInterest<N>>>,
542 gap_anchor: Option<BlockRef>,
543 },
544 Bulk {
545 previous: Vec<(HandlerId, Option<Vec<ReactiveInterest<N>>>)>,
546 gap_anchor: Option<BlockRef>,
547 },
548 Recovery {
549 anchor: BlockRef,
550 },
551}
552
553impl<N: Network> LiveRollback<N> {
554 const fn gap_anchor(&self) -> Option<BlockRef> {
555 match self {
556 Self::Base { gap_anchor, .. }
557 | Self::Topology { gap_anchor, .. }
558 | Self::Owner { gap_anchor, .. }
559 | Self::Bulk { gap_anchor, .. } => *gap_anchor,
560 Self::Recovery { .. } => None,
561 }
562 }
563
564 const fn gap_scope(&self) -> &'static str {
565 match self {
566 Self::Base { .. } => "base subscription",
567 Self::Topology { .. } => "owner topology",
568 Self::Owner { .. } => "owner subscription",
569 Self::Bulk { .. } => "bulk owner subscription",
570 Self::Recovery { .. } => "topology recovery",
571 }
572 }
573}
574
575struct BufferedBatch<N: Network> {
576 records: Vec<RoutedRecord<N>>,
577 controls: Vec<ChainControl>,
578 token: Option<SubscriberDeliveryToken>,
579 checkpoint: Option<SubscriberCheckpoint>,
580 payload_commitment: Option<SubscriberPayloadCommitment>,
581 max_canonical_block: Option<u64>,
582 source_progress: Option<BlockRef>,
583 accounted_bytes: usize,
584 delivery_digest: [u8; 32],
585 canonical: Vec<CanonicalMutation>,
586 next_safe_head: Option<BlockRef>,
587 next_finalized_head: Option<BlockRef>,
588 next_canonical_history: Vec<BlockRef>,
589 next_coverage_head: Option<BlockRef>,
590}
591
592#[derive(Clone, Copy)]
593struct HybridIngressLimits {
594 max_records: usize,
595 max_accounted_bytes: usize,
596 max_delivery_token_bytes: usize,
597 max_checkpoint_bytes: usize,
598 max_projected_owner_associations: usize,
599}
600
601#[derive(Clone, Copy)]
602struct CapacityProbeLimits {
603 history_capacity: usize,
604 max_delivery_token_bytes: usize,
605 max_checkpoint_bytes: usize,
606}
607
608impl From<&HybridConfig> for HybridIngressLimits {
609 fn from(config: &HybridConfig) -> Self {
610 Self {
611 max_records: config.max_buffered_live_records,
612 max_accounted_bytes: config.max_buffered_live_bytes,
613 max_delivery_token_bytes: config.max_source_delivery_token_bytes,
614 max_checkpoint_bytes: config.max_source_checkpoint_bytes,
615 max_projected_owner_associations: config.max_recent_owner_entries,
616 }
617 }
618}
619
620const fn maximum_ingress_control_count(max_accounted_bytes: usize) -> usize {
621 max_accounted_bytes.saturating_sub(BATCH_ACCOUNTING_OVERHEAD) / MIN_CONTROL_ACCOUNTING_BYTES
622}
623
624#[cfg(test)]
625fn maximum_admitted_canonical_advances(config: &HybridConfig) -> usize {
626 maximum_ingress_control_count(config.max_buffered_live_bytes)
627 .saturating_add(1)
628 .min(config.canonical_history_capacity)
629}
630
631type RoutedRecord<N> = (ReactiveInputRecord<N>, DeliveryAudience, DeliveryScope);
632
633impl<N: Network> BufferedBatch<N> {
634 fn from_batch(
635 batch: ReactiveInputBatch<N>,
636 expected_chain_id: u64,
637 limits: HybridIngressLimits,
638 owner_generations: &BTreeMap<HandlerId, u64>,
639 ) -> Result<Self, SubscriberError> {
640 let HybridIngressLimits {
641 max_records,
642 max_accounted_bytes,
643 max_delivery_token_bytes,
644 max_checkpoint_bytes,
645 max_projected_owner_associations,
646 } = limits;
647 let parts = batch.into_parts();
648 if parts.chain_id != Some(expected_chain_id) {
649 return Err(SubscriberError::Provider(format!(
650 "hybrid child batch has chain {:?}; expected {expected_chain_id}",
651 parts.chain_id
652 )));
653 }
654 if parts.deliveries.is_empty()
655 && parts.chain_controls.is_empty()
656 && parts.delivery_token.is_none()
657 {
658 return Err(SubscriberError::Provider(
659 "hybrid source emitted an empty child batch; return None for idle/keepalive state or emit an explicit chain control"
660 .into(),
661 ));
662 }
663 if let Some(token) = parts.delivery_token.as_ref()
664 && token.as_bytes().len() > max_delivery_token_bytes
665 {
666 return Err(SubscriberError::Provider(format!(
667 "hybrid child delivery token exceeds the configured opaque cursor bound ({}/{max_delivery_token_bytes} bytes)",
668 token.as_bytes().len()
669 )));
670 }
671 if let Some(checkpoint) = parts.subscriber_checkpoint.as_ref()
672 && checkpoint.as_bytes().len() > max_checkpoint_bytes
673 {
674 return Err(SubscriberError::Provider(format!(
675 "hybrid child checkpoint exceeds the configured opaque cursor bound ({}/{max_checkpoint_bytes} bytes)",
676 checkpoint.as_bytes().len()
677 )));
678 }
679 if parts.deliveries.len() > max_records {
680 return Err(SubscriberError::Provider(format!(
681 "hybrid child batch exceeds configured per-batch ingress record bound ({}/{max_records})",
682 parts.deliveries.len()
683 )));
684 }
685 let max_controls = maximum_ingress_control_count(max_accounted_bytes);
686 if parts.chain_controls.len() > max_controls {
687 return Err(SubscriberError::Provider(format!(
688 "hybrid child batch exceeds the derived per-batch ingress control bound ({}/{max_controls})",
689 parts.chain_controls.len()
690 )));
691 }
692 preflight_ingress_routing(
693 &parts.deliveries,
694 owner_generations,
695 max_projected_owner_associations,
696 )?;
697 if parts.deliveries.iter().any(|delivery| {
698 matches!(
699 delivery.record().input,
700 ReactiveInput::FullBlock(_) | ReactiveInput::PendingTx(_)
701 )
702 }) {
703 return Err(SubscriberError::Unsupported(
704 "hybrid delivery does not accept full blocks or hydrated pending transactions until their complete bodies can be verified generically",
705 ));
706 }
707 if parts.payload_commitment.is_none()
708 && parts
709 .deliveries
710 .iter()
711 .any(|delivery| matches!(delivery.record().input, ReactiveInput::BlockHeader(_)))
712 {
713 return Err(SubscriberError::Unsupported(
714 "hybrid block-header delivery requires a subscriber payload commitment covering the exact canonical wire body",
715 ));
716 }
717 let mut accounted_bytes = BATCH_ACCOUNTING_OVERHEAD;
718 for delivery in &parts.deliveries {
719 let remaining = max_accounted_bytes.saturating_sub(accounted_bytes);
720 let record_bytes =
721 accounted_record_bytes(delivery.record(), delivery.audience(), remaining)?;
722 accounted_bytes = accounted_bytes.checked_add(record_bytes).ok_or_else(|| {
723 SubscriberError::Provider(
724 "hybrid child batch ingress byte accounting overflowed".into(),
725 )
726 })?;
727 if accounted_bytes > max_accounted_bytes {
728 return Err(SubscriberError::Provider(format!(
729 "hybrid child batch exceeds configured per-batch ingress byte bound ({accounted_bytes}/{max_accounted_bytes})"
730 )));
731 }
732 }
733 for control in &parts.chain_controls {
734 accounted_bytes = accounted_bytes
735 .checked_add(accounted_control_bytes(control))
736 .ok_or_else(|| {
737 SubscriberError::Provider(
738 "hybrid child batch ingress byte accounting overflowed".into(),
739 )
740 })?;
741 if accounted_bytes > max_accounted_bytes {
742 return Err(SubscriberError::Provider(format!(
743 "hybrid child batch exceeds configured per-batch ingress byte bound ({accounted_bytes}/{max_accounted_bytes})"
744 )));
745 }
746 }
747 if let Some(token) = parts.delivery_token.as_ref() {
748 accounted_bytes = accounted_bytes
749 .checked_add(token.as_bytes().len())
750 .ok_or_else(|| {
751 SubscriberError::Provider(
752 "hybrid child batch ingress byte accounting overflowed".into(),
753 )
754 })?;
755 }
756 if let Some(checkpoint) = parts.subscriber_checkpoint.as_ref() {
757 accounted_bytes = accounted_bytes
758 .checked_add(checkpoint.as_bytes().len())
759 .ok_or_else(|| {
760 SubscriberError::Provider(
761 "hybrid child batch ingress byte accounting overflowed".into(),
762 )
763 })?;
764 }
765 if parts.payload_commitment.is_some() {
766 accounted_bytes = accounted_bytes.checked_add(32).ok_or_else(|| {
767 SubscriberError::Provider(
768 "hybrid child batch ingress byte accounting overflowed".into(),
769 )
770 })?;
771 }
772 if accounted_bytes > max_accounted_bytes {
773 return Err(SubscriberError::Provider(format!(
774 "hybrid child batch exceeds configured per-batch ingress byte bound ({accounted_bytes}/{max_accounted_bytes})"
775 )));
776 }
777 let token = parts.delivery_token;
778 let checkpoint = parts.subscriber_checkpoint;
779 let payload_commitment = parts.payload_commitment;
780 let controls = parts.chain_controls;
781 let records = parts
782 .deliveries
783 .into_iter()
784 .map(ReactiveInputDelivery::into_parts)
785 .collect::<Vec<_>>();
786 for (record, audience, scope) in &records {
787 validated_record(record, expected_chain_id)?;
788 debug_assert!(validate_routing(audience, *scope).is_ok());
789 }
790 validate_chain_controls(&controls)?;
791 let delivery_digest = source_delivery_digest(
792 &records,
793 &controls,
794 checkpoint.as_ref(),
795 payload_commitment.as_ref(),
796 expected_chain_id,
797 )?;
798 let max_canonical_block = records
799 .iter()
800 .filter(|(_, _, scope)| scope_advances_canonical(*scope))
801 .filter_map(|(record, _, _)| coverage_record_block_number(record))
802 .chain(controls.iter().filter_map(coverage_control_block_number))
803 .max();
804 Ok(Self {
805 records,
806 controls,
807 token,
808 checkpoint,
809 payload_commitment,
810 max_canonical_block,
811 source_progress: None,
812 accounted_bytes,
813 delivery_digest,
814 canonical: Vec::new(),
815 next_safe_head: None,
816 next_finalized_head: None,
817 next_canonical_history: Vec::new(),
818 next_coverage_head: None,
819 })
820 }
821}
822
823#[derive(Debug)]
824enum CanonicalPlanError {
825 NeedsHistoricalRecovery(String),
829 Invalid(SubscriberError),
832}
833
834impl CanonicalPlanError {
835 fn into_subscriber_error(self) -> SubscriberError {
836 match self {
837 Self::NeedsHistoricalRecovery(message) => SubscriberError::Provider(message),
838 Self::Invalid(error) => error,
839 }
840 }
841}
842
843impl<H, L, N> HybridSubscriber<H, L, N>
844where
845 H: EventSubscriber<N>,
846 L: EventSubscriber<N>,
847 N: Network,
848{
849 pub fn new(historical: H, live: L, config: HybridConfig) -> Result<Self, SubscriberError>
861 where
862 H: EventSubscriber<N>,
863 L: EventSubscriber<N>,
864 {
865 if config.max_buffered_live_batches == 0 {
866 return Err(SubscriberError::InvalidConfig(
867 "hybrid live buffer capacity must be non-zero",
868 ));
869 }
870 if config.recent_input_capacity == 0 {
871 return Err(SubscriberError::InvalidConfig(
872 "hybrid deduplication capacity must be non-zero",
873 ));
874 }
875 if config.recent_input_capacity > HYBRID_MAX_RECENT_INPUTS {
876 return Err(SubscriberError::InvalidConfig(
877 "hybrid deduplication capacity exceeds the v5 durable checkpoint limit",
878 ));
879 }
880 if config.max_recent_owner_entries > HYBRID_MAX_RECENT_OWNER_ENTRIES {
881 return Err(SubscriberError::InvalidConfig(
882 "hybrid recent owner-entry budget exceeds the v5 durable checkpoint limit",
883 ));
884 }
885 if config.max_buffered_live_records == 0 {
886 return Err(SubscriberError::InvalidConfig(
887 "hybrid live record capacity must be non-zero",
888 ));
889 }
890 if config.max_buffered_live_bytes == 0 {
891 return Err(SubscriberError::InvalidConfig(
892 "hybrid live byte capacity must be non-zero",
893 ));
894 }
895 if config.max_source_delivery_token_bytes == 0 {
896 return Err(SubscriberError::InvalidConfig(
897 "hybrid source delivery-token byte budget must be non-zero",
898 ));
899 }
900 if config.max_source_delivery_token_bytes > HYBRID_MAX_SOURCE_DELIVERY_TOKEN_BYTES {
901 return Err(SubscriberError::InvalidConfig(
902 "hybrid source delivery-token byte budget exceeds the v5 durable checkpoint limit",
903 ));
904 }
905 if config.max_source_checkpoint_bytes == 0 {
906 return Err(SubscriberError::InvalidConfig(
907 "hybrid source checkpoint byte budget must be non-zero",
908 ));
909 }
910 if config.max_source_checkpoint_bytes > HYBRID_MAX_SOURCE_CHECKPOINT_BYTES {
911 return Err(SubscriberError::InvalidConfig(
912 "hybrid source checkpoint byte budget exceeds the v5 durable checkpoint limit",
913 ));
914 }
915 if config.canonical_history_capacity == 0 {
916 return Err(SubscriberError::InvalidConfig(
917 "hybrid canonical history capacity must be non-zero",
918 ));
919 }
920 if config.canonical_history_capacity > HYBRID_MAX_CANONICAL_HISTORY {
921 return Err(SubscriberError::InvalidConfig(
922 "hybrid canonical history capacity exceeds the v5 durable checkpoint limit",
923 ));
924 }
925 let chain_id = historical.chain_id().ok_or(SubscriberError::InvalidConfig(
926 "hybrid historical source must expose one authoritative chain id",
927 ))?;
928 if live
929 .chain_id()
930 .is_some_and(|live_chain_id| live_chain_id != chain_id)
931 {
932 return Err(SubscriberError::InvalidConfig(
933 "hybrid historical and live sources must target the same chain",
934 ));
935 }
936 let historical_capabilities = historical.capabilities();
937 for required in [
938 SubscriberCapability::HistoricalBackfill,
939 SubscriberCapability::DurableReplay,
940 SubscriberCapability::Barriers,
941 ] {
942 if !historical_capabilities.supports(required) {
943 return Err(SubscriberError::InvalidConfig(
944 "hybrid historical source must provide durable backfill and explicit coverage barriers",
945 ));
946 }
947 }
948 if !live.capabilities().supports(SubscriberCapability::Live) {
949 return Err(SubscriberError::InvalidConfig(
950 "hybrid live source must advertise live canonical delivery",
951 ));
952 }
953 let lifecycle_intent = lifecycle_intent::<N>(&[], &HashMap::new())?;
954 Ok(Self {
955 historical,
956 live,
957 chain_id,
958 config,
959 phase: HybridPhase::Live,
960 fence: None,
961 drain_through: None,
962 pending_cutover: None,
963 acknowledged_historical_through: None,
964 live_buffer: VecDeque::new(),
965 buffered_live_records: 0,
966 buffered_live_bytes: 0,
967 recent_inputs: HashMap::new(),
968 recent_order: VecDeque::new(),
969 pending_inputs: HashMap::new(),
970 pending_output: None,
971 pending_live_rollback: None,
972 epoch: fresh_epoch(),
973 next_synthetic_token: 1,
974 lifecycle_generation: 1,
975 owner_generations: BTreeMap::new(),
976 restored_source_replays: HashSet::new(),
977 canonical_history: VecDeque::new(),
978 coverage_head: None,
979 safe_head: None,
980 finalized_head: None,
981 certified_historical: None,
982 recovery_anchor: None,
983 historical_position: SourcePosition::default(),
984 live_position: SourcePosition::default(),
985 last_committed_token: None,
986 pending_restore: None,
987 pending_restore_preparation: None,
988 prepared_restore_position: None,
989 poisoned: None,
990 base_interests: Vec::new(),
991 owners: HashMap::new(),
992 lifecycle_intent,
993 })
994 }
995
996 pub const fn phase(&self) -> HybridPhase {
998 self.phase
999 }
1000
1001 fn has_active_interests(&self) -> bool {
1002 !self.base_interests.is_empty()
1003 || self.owners.values().any(|interests| !interests.is_empty())
1004 }
1005
1006 fn validate_checkpoint_config_limits(
1007 &self,
1008 state: &HybridCheckpointV5,
1009 ) -> Result<(), SubscriberError> {
1010 self.validate_checkpoint_config_limits_for_restore(state, true)
1011 }
1012
1013 fn validate_checkpoint_config_limits_for_restore(
1014 &self,
1015 state: &HybridCheckpointV5,
1016 retain_live_cursor: bool,
1017 ) -> Result<(), SubscriberError> {
1018 if state.recent_inputs.len() > self.config.recent_input_capacity
1019 || recent_owner_entry_count(&state.recent_inputs)?
1020 > self.config.max_recent_owner_entries
1021 || state.canonical_history.len() > self.config.canonical_history_capacity
1022 || state.historical_position.canonical_history.len()
1023 > self.config.canonical_history_capacity
1024 || state.live_position.canonical_history.len() > self.config.canonical_history_capacity
1025 {
1026 return Err(SubscriberError::Provider(
1027 "hybrid checkpoint exceeds this coordinator's configured durable verification budgets"
1028 .into(),
1029 ));
1030 }
1031 for (label, position) in [
1032 ("historical", Some(&state.historical_position)),
1033 ("live", retain_live_cursor.then_some(&state.live_position)),
1034 ] {
1035 let Some(position) = position else {
1036 continue;
1037 };
1038 if position
1039 .delivery_token
1040 .as_ref()
1041 .is_some_and(|token| token.len() > self.config.max_source_delivery_token_bytes)
1042 {
1043 return Err(SubscriberError::Provider(format!(
1044 "hybrid {label} source delivery token exceeds this coordinator's configured opaque cursor bound"
1045 )));
1046 }
1047 if position.checkpoint.as_ref().is_some_and(|checkpoint| {
1048 checkpoint.len() > self.config.max_source_checkpoint_bytes
1049 }) {
1050 return Err(SubscriberError::Provider(format!(
1051 "hybrid {label} source checkpoint exceeds this coordinator's configured opaque cursor bound"
1052 )));
1053 }
1054 }
1055 if let Some(committed) = state.last_committed_token.as_ref()
1056 && committed.kind == HybridTokenKind::Forwarded
1057 && (committed.source != HybridSource::Live || retain_live_cursor)
1058 && committed.inner.len() > self.config.max_source_delivery_token_bytes
1059 {
1060 return Err(SubscriberError::Provider(
1061 "hybrid last committed forwarded token exceeds this coordinator's configured opaque cursor bound"
1062 .into(),
1063 ));
1064 }
1065 Ok(())
1066 }
1067
1068 fn validate_restore_position_preflight(
1075 &self,
1076 position: &SubscriberResumePosition,
1077 state: &HybridCheckpointV5,
1078 live_is_durable: bool,
1079 ) -> Result<Vec<BlockRef>, SubscriberError> {
1080 if position.chain_id != self.chain_id || state.chain_id != self.chain_id {
1081 return Err(SubscriberError::Provider(format!(
1082 "hybrid restore targets the wrong chain; expected {}",
1083 self.chain_id
1084 )));
1085 }
1086 self.validate_checkpoint_config_limits_for_restore(state, live_is_durable)?;
1087
1088 let token = position.delivery_token.as_ref().ok_or_else(|| {
1089 SubscriberError::Provider(
1090 "hybrid restore checkpoint is missing its matching delivery token".into(),
1091 )
1092 })?;
1093 let (token_epoch, source, kind, inner) = unwrap_token(token.clone())?;
1094 if token_epoch != state.epoch {
1095 return Err(SubscriberError::Provider(
1096 "hybrid checkpoint and delivery-token epochs differ".into(),
1097 ));
1098 }
1099 let committed = state.last_committed_token.as_ref().ok_or_else(|| {
1100 SubscriberError::Provider(
1101 "hybrid checkpoint is missing its last committed outer token".into(),
1102 )
1103 })?;
1104 if committed.source != source
1105 || committed.kind != kind
1106 || committed.inner.as_slice() != inner.as_bytes()
1107 {
1108 return Err(SubscriberError::Provider(
1109 "hybrid checkpoint does not match its last committed outer delivery token".into(),
1110 ));
1111 }
1112 match kind {
1113 HybridTokenKind::Synthetic => {
1114 let sequence = u64::from_be_bytes(inner.as_bytes().try_into().map_err(|_| {
1115 SubscriberError::Provider(
1116 "hybrid synthetic token has an invalid sequence width".into(),
1117 )
1118 })?);
1119 if state.next_synthetic_token
1120 != sequence.checked_add(1).ok_or_else(|| {
1121 SubscriberError::Provider(
1122 "restored hybrid synthetic token exhausted its sequence space".into(),
1123 )
1124 })?
1125 {
1126 return Err(SubscriberError::Provider(
1127 "hybrid checkpoint does not match its last synthetic delivery token".into(),
1128 ));
1129 }
1130 }
1131 HybridTokenKind::Forwarded => {
1132 let source_position = match source {
1133 HybridSource::Historical => &state.historical_position,
1134 HybridSource::Live => &state.live_position,
1135 };
1136 if source_position.delivery_token.as_deref() != Some(inner.as_bytes()) {
1137 return Err(SubscriberError::Provider(
1138 "hybrid checkpoint does not match its last forwarded delivery token".into(),
1139 ));
1140 }
1141 }
1142 }
1143
1144 if let Some(checkpoint_head) = state.coverage_head.as_ref()
1145 && compatible_block_ref(checkpoint_head, &position.coverage_head).is_err()
1146 {
1147 return Err(SubscriberError::Provider(format!(
1148 "hybrid checkpoint coverage {} does not match restored runtime coverage {}",
1149 checkpoint_head.number, position.coverage_head.number
1150 )));
1151 }
1152 let mut runtime_history = position.canonical_history.clone();
1153 if runtime_history
1154 .last()
1155 .is_none_or(|last| last.number < position.coverage_head.number)
1156 {
1157 runtime_history.push(position.coverage_head);
1158 }
1159 validate_checkpoint_history(
1160 "runtime resume",
1161 &runtime_history,
1162 Some(&position.coverage_head),
1163 )?;
1164 for runtime_block in &mut runtime_history {
1165 if let Some(checkpoint_block) = state
1166 .canonical_history
1167 .iter()
1168 .find(|block| block.number == runtime_block.number)
1169 {
1170 *runtime_block = compatible_block_ref(checkpoint_block, runtime_block).map_err(
1171 |_| {
1172 SubscriberError::Provider(format!(
1173 "runtime canonical history conflicts with the hybrid checkpoint at block {}",
1174 runtime_block.number
1175 ))
1176 },
1177 )?;
1178 }
1179 }
1180 for entry in &state.recent_inputs {
1181 if let Some(block) = entry.coverage.block.as_ref()
1182 && let Some(runtime_block) = runtime_history
1183 .iter()
1184 .find(|runtime| runtime.number == block.number)
1185 && compatible_block_ref(runtime_block, block).is_err()
1186 {
1187 return Err(SubscriberError::Provider(format!(
1188 "runtime canonical history conflicts with hybrid input coverage at block {}",
1189 block.number
1190 )));
1191 }
1192 }
1193 Ok(runtime_history)
1194 }
1195
1196 fn normalize_restore_candidate(
1205 &self,
1206 position: &SubscriberResumePosition,
1207 state: &mut HybridCheckpointV5,
1208 live_is_durable: bool,
1209 ) -> Result<bool, SubscriberError> {
1210 let mut runtime_history =
1211 self.validate_restore_position_preflight(position, state, live_is_durable)?;
1212 truncate_front_to(&mut runtime_history, self.config.canonical_history_capacity);
1213 state.coverage_head = Some(match state.coverage_head.as_ref() {
1214 Some(checkpoint_head) => {
1215 compatible_block_ref(checkpoint_head, &position.coverage_head)?
1216 }
1217 None => position.coverage_head,
1218 });
1219 state.canonical_history = runtime_history;
1220
1221 let mut rotate_epoch = false;
1222 if !live_is_durable {
1223 let retain_empty_barrier_ack = is_committed_empty_lifecycle_barrier(state);
1224 state.live_position.delivery_token = None;
1225 state.live_position.checkpoint = None;
1226 if !retain_empty_barrier_ack {
1227 state.live_position.delivery_digest = None;
1228 state.last_committed_token = None;
1229 rotate_epoch = true;
1230 }
1231 }
1232 self.validate_checkpoint_config_limits(state)?;
1233 validate_checkpoint_state(state)?;
1234 Ok(rotate_epoch)
1235 }
1236
1237 fn preflight_owner_topology(
1238 &self,
1239 next_owners: &HashMap<HandlerId, Vec<ReactiveInterest<N>>>,
1240 next_intent: &LifecycleIntent,
1241 generation: u64,
1242 next_owner_generations: &BTreeMap<HandlerId, u64>,
1243 transition: OwnerTopologyTransition,
1244 ) -> Result<(), SubscriberError> {
1245 validate_handler_ids(next_owners.keys())?;
1246 if !next_intent.has_active_interests() {
1247 return Ok(());
1250 }
1251 let mut state = self.checkpoint_state();
1252 state.lifecycle_generation = generation;
1253 state.owner_generations = next_owner_generations.clone();
1254 state.lifecycle_intent = next_intent.clone();
1255 state.certified_historical = None;
1256 match transition {
1257 OwnerTopologyTransition::DestructiveReset => {
1258 state.next_synthetic_token = 1;
1261 state.recent_inputs.clear();
1262 state.historical_position.delivery_token = None;
1263 state.historical_position.checkpoint = None;
1264 state.historical_position.delivery_digest = None;
1265 state.live_position.delivery_token = None;
1266 state.live_position.checkpoint = None;
1267 state.live_position.delivery_digest = None;
1268 state.last_committed_token = None;
1269 }
1270 OwnerTopologyTransition::Incremental { removed_owners } => {
1271 for entry in &mut state.recent_inputs {
1272 for owner in &removed_owners {
1273 entry.coverage.owners.remove(owner);
1274 }
1275 }
1276 }
1277 }
1278 self.preflight_maximum_canonical_log_delivery(&state, "hybrid owner topology")
1279 }
1280
1281 fn preflight_base_topology(
1282 &self,
1283 next_intent: &LifecycleIntent,
1284 generation: u64,
1285 ) -> Result<(), SubscriberError> {
1286 if !next_intent.has_active_interests() {
1287 return Ok(());
1290 }
1291 let mut state = self.checkpoint_state();
1292 state.next_synthetic_token = 1;
1293 state.lifecycle_generation = generation;
1294 state.owner_generations.clear();
1295 state.lifecycle_intent = next_intent.clone();
1296 state.recent_inputs.clear();
1297 state.certified_historical = None;
1298 state.historical_position.delivery_token = None;
1299 state.historical_position.checkpoint = None;
1300 state.historical_position.delivery_digest = None;
1301 state.live_position.delivery_token = None;
1302 state.live_position.checkpoint = None;
1303 state.live_position.delivery_digest = None;
1304 state.last_committed_token = None;
1305 self.preflight_maximum_canonical_log_delivery(&state, "hybrid base topology")
1306 }
1307
1308 fn preflight_restore_delivery_capacity(
1318 &self,
1319 candidate: &HybridCheckpointV5,
1320 ) -> Result<(), SubscriberError> {
1321 if !candidate.lifecycle_intent.has_active_interests() {
1322 return Ok(());
1323 }
1324 self.preflight_maximum_canonical_log_delivery(candidate, "hybrid restored checkpoint")
1325 }
1326
1327 fn preflight_maximum_canonical_log_delivery(
1349 &self,
1350 candidate: &HybridCheckpointV5,
1351 context: &str,
1352 ) -> Result<(), SubscriberError> {
1353 for source in [HybridSource::Historical, HybridSource::Live] {
1354 let source_label = match source {
1355 HybridSource::Historical => "historical",
1356 HybridSource::Live => "live",
1357 };
1358 for kind in [HybridTokenKind::Forwarded, HybridTokenKind::Synthetic] {
1359 let kind_label = match kind {
1360 HybridTokenKind::Forwarded => "forwarded-token",
1361 HybridTokenKind::Synthetic => "synthetic-token",
1362 };
1363 self.encode_maximum_canonical_log_delivery_variant(candidate, source, kind)
1364 .map_err(|error| {
1365 SubscriberError::Provider(format!(
1366 "{context} cannot retain one protected delivery witness for the maximum canonical-log delivery ({kind_label}) from the {source_label} source: {error}"
1367 ))
1368 })?;
1369 }
1370 }
1371 Ok(())
1372 }
1373
1374 #[cfg(test)]
1375 fn encode_maximum_canonical_log_delivery(
1376 &self,
1377 candidate: &HybridCheckpointV5,
1378 source: HybridSource,
1379 ) -> Result<Vec<u8>, SubscriberError> {
1380 self.encode_maximum_canonical_log_delivery_variant(
1381 candidate,
1382 source,
1383 HybridTokenKind::Forwarded,
1384 )
1385 }
1386
1387 fn encode_maximum_canonical_log_delivery_variant(
1388 &self,
1389 candidate: &HybridCheckpointV5,
1390 source: HybridSource,
1391 kind: HybridTokenKind,
1392 ) -> Result<Vec<u8>, SubscriberError> {
1393 let mut simulated = saturated_capacity_probe_state(candidate, &self.config)?;
1394 if kind == HybridTokenKind::Synthetic {
1395 simulated.next_synthetic_token = u64::MAX - 1;
1396 }
1397 self.validate_checkpoint_config_limits(&simulated)?;
1398 validate_checkpoint_state(&simulated)?;
1399 let mut commit = maximum_canonical_log_commit(
1400 &simulated,
1401 source,
1402 self.config.canonical_history_capacity,
1403 1,
1404 self.config.max_source_delivery_token_bytes,
1405 self.config.max_source_checkpoint_bytes,
1406 )?;
1407 if kind == HybridTokenKind::Synthetic {
1408 let sequence = simulated.next_synthetic_token;
1409 simulated.next_synthetic_token = simulated
1410 .next_synthetic_token
1411 .checked_add(1)
1412 .ok_or_else(|| {
1413 SubscriberError::Provider(
1414 "hybrid synthetic capacity-probe token space exhausted".into(),
1415 )
1416 })?;
1417 commit.source_token = None;
1418 commit.token_kind = HybridTokenKind::Synthetic;
1419 commit.token_bytes = sequence.to_be_bytes().to_vec();
1420 }
1421 let probe_block = commit.next_coverage_head.ok_or_else(|| {
1422 SubscriberError::Provider(
1423 "hybrid saturated canonical-log capacity probe did not advance coverage".into(),
1424 )
1425 })?;
1426 reserve_maximum_source_cursors(&mut simulated, &self.config, source, probe_block)?;
1427 apply_commit_to_checkpoint(
1428 &mut simulated,
1429 &commit,
1430 self.config.canonical_history_capacity,
1431 )?;
1432 fit_checkpoint_to_durable_limits(
1433 &mut simulated,
1434 1,
1435 self.config.recent_input_capacity,
1436 self.config.max_recent_owner_entries,
1437 )?;
1438 self.validate_checkpoint_config_limits(&simulated)?;
1439 validate_checkpoint_state(&simulated)?;
1440 encode_hybrid_checkpoint(&simulated)
1441 }
1442
1443 #[cfg(test)]
1444 fn encode_canonical_log_delivery_capacity_probe(
1445 &self,
1446 candidate: &HybridCheckpointV5,
1447 source: HybridSource,
1448 maximum_advances: usize,
1449 ) -> Result<Vec<u8>, SubscriberError> {
1450 let mut simulated = candidate.clone();
1451 let commit = maximum_canonical_log_commit(
1452 &simulated,
1453 source,
1454 self.config.canonical_history_capacity,
1455 maximum_advances,
1456 self.config.max_source_delivery_token_bytes,
1457 self.config.max_source_checkpoint_bytes,
1458 )?;
1459 let probe_block = commit.next_coverage_head.ok_or_else(|| {
1460 SubscriberError::Provider(
1461 "hybrid maximum canonical-log capacity probe did not advance coverage".into(),
1462 )
1463 })?;
1464 reserve_maximum_source_cursors(&mut simulated, &self.config, source, probe_block)?;
1465 apply_commit_to_checkpoint(
1466 &mut simulated,
1467 &commit,
1468 self.config.canonical_history_capacity,
1469 )?;
1470 fit_checkpoint_to_durable_limits(
1471 &mut simulated,
1472 1,
1473 self.config.recent_input_capacity,
1474 self.config.max_recent_owner_entries,
1475 )?;
1476 self.validate_checkpoint_config_limits(&simulated)?;
1477 validate_checkpoint_state(&simulated)?;
1478 encode_hybrid_checkpoint(&simulated)
1479 }
1480
1481 pub const fn fence(&self) -> Option<u64> {
1483 self.fence
1484 }
1485
1486 pub fn buffered_live_batches(&self) -> usize {
1488 self.live_buffer.len()
1489 }
1490
1491 pub const fn buffered_live_records(&self) -> usize {
1493 self.buffered_live_records
1494 }
1495
1496 pub const fn buffered_live_bytes(&self) -> usize {
1498 self.buffered_live_bytes
1499 }
1500
1501 pub fn poison_reason(&self) -> Option<&str> {
1503 self.poisoned.as_deref()
1504 }
1505
1506 pub const fn historical(&self) -> &H {
1508 &self.historical
1509 }
1510
1511 pub const fn live(&self) -> &L {
1513 &self.live
1514 }
1515
1516 pub fn into_sources(self) -> (H, L) {
1518 (self.historical, self.live)
1519 }
1520
1521 pub fn prepare_restore_lifecycle<'a>(
1565 &'a mut self,
1566 position: &SubscriberResumePosition,
1567 base_interests: &[ReactiveInterest<N>],
1568 owners: Vec<(HandlerId, Vec<ReactiveInterest<N>>)>,
1569 ) -> SubscriberOperation<'a, ()>
1570 where
1571 H: InterestOwnerSubscriber<N>,
1572 L: InterestOwnerSubscriber<N>,
1573 N: Send + 'static,
1574 {
1575 let position = position.clone();
1576 let base_interests = base_interests.to_vec();
1577 Box::pin(async move {
1578 if owners.is_empty() {
1579 return self
1580 .prepare_restore_base_lifecycle(&position, &base_interests)
1581 .await;
1582 }
1583 self.ensure_healthy()?;
1584 if self.pending_restore.is_some() {
1585 return Err(SubscriberError::Provider(
1586 "hybrid child restore reconciliation is already pending".into(),
1587 ));
1588 }
1589 if self.pending_cutover.is_some()
1590 || !self.pending_inputs.is_empty()
1591 || self.pending_output.is_some()
1592 || !self.live_buffer.is_empty()
1593 {
1594 return Err(SubscriberError::Provider(
1595 "hybrid live restore preparation requires a fresh coordinator with no in-flight delivery"
1596 .into(),
1597 ));
1598 }
1599 let intent = lifecycle_intent_from_entries(&base_interests, &owners)?;
1600 let checkpoint = position.subscriber_checkpoint.as_ref().ok_or_else(|| {
1601 SubscriberError::Provider(
1602 "hybrid live restore preparation requires its coordinator checkpoint".into(),
1603 )
1604 })?;
1605 let mut state = decode_hybrid_checkpoint(checkpoint.as_bytes())?;
1606 let live_is_durable = self
1607 .live
1608 .capabilities()
1609 .supports(SubscriberCapability::DurableReplay);
1610 self.normalize_restore_candidate(&position, &mut state, live_is_durable)?;
1611 if intent != state.lifecycle_intent {
1612 return Err(SubscriberError::Provider(
1613 "restored runtime interests do not match the hybrid checkpoint lifecycle intent"
1614 .into(),
1615 ));
1616 }
1617 let owner_topology = owners.iter().cloned().collect::<HashMap<_, _>>();
1618 validate_handler_ids(owner_topology.keys())?;
1619 self.preflight_restore_delivery_capacity(&state)?;
1620 if let Some(prepared) = self.prepared_restore_position.as_ref() {
1621 if prepared == &position && self.lifecycle_intent == intent {
1622 return Ok(());
1623 }
1624 return Err(SubscriberError::Provider(
1625 "hybrid live source is already prepared for a different restore position"
1626 .into(),
1627 ));
1628 }
1629 if self.pending_restore_preparation.is_none() {
1630 if self.lifecycle_generation != 1
1631 || !self.base_interests.is_empty()
1632 || !self.owners.is_empty()
1633 {
1634 return Err(SubscriberError::Provider(
1635 "hybrid live restore preparation requires a newly constructed coordinator"
1636 .into(),
1637 ));
1638 }
1639 self.pending_restore_preparation = Some(PendingRestorePreparation {
1640 position: position.clone(),
1641 intent: intent.clone(),
1642 base_interests: base_interests.clone(),
1643 owners: owners.clone(),
1644 live_topology_installed: false,
1645 });
1646 } else if self
1647 .pending_restore_preparation
1648 .as_ref()
1649 .is_none_or(|pending| pending.position != position || pending.intent != intent)
1650 {
1651 return Err(SubscriberError::Provider(
1652 "hybrid live restore preparation is pending for different interests or a different position"
1653 .into(),
1654 ));
1655 }
1656
1657 let pending_preparation = self
1658 .pending_restore_preparation
1659 .as_ref()
1660 .expect("restore preparation exists");
1661 let staged_base = pending_preparation.base_interests.clone();
1662 let staged_owners = pending_preparation.owners.clone();
1663 if !live_is_durable
1664 && !self
1665 .pending_restore_preparation
1666 .as_ref()
1667 .expect("restore preparation exists")
1668 .live_topology_installed
1669 {
1670 if staged_owners.is_empty() {
1671 self.live.register_interests(&staged_base).await?;
1674 } else {
1675 self.live
1680 .replace_interest_owners(staged_owners.clone())
1681 .await?;
1682 }
1683 self.pending_restore_preparation
1684 .as_mut()
1685 .expect("restore preparation survives registration")
1686 .live_topology_installed = true;
1687 }
1688 let staged_active = !base_interests.is_empty()
1689 || staged_owners
1690 .iter()
1691 .any(|(_, interests)| !interests.is_empty());
1692 if staged_active {
1693 self.ensure_source_chain(HybridSource::Live)?;
1694 }
1695
1696 self.base_interests = base_interests;
1697 self.owners = staged_owners.into_iter().collect();
1698 self.lifecycle_intent = intent;
1699 self.prepared_restore_position = Some(position);
1700 self.pending_restore_preparation = None;
1701 Ok(())
1702 })
1703 }
1704
1705 pub fn prepare_restore_base_lifecycle<'a>(
1733 &'a mut self,
1734 position: &SubscriberResumePosition,
1735 base_interests: &[ReactiveInterest<N>],
1736 ) -> SubscriberOperation<'a, ()>
1737 where
1738 N: Send + 'static,
1739 {
1740 let position = position.clone();
1741 let base_interests = base_interests.to_vec();
1742 Box::pin(async move {
1743 self.ensure_healthy()?;
1744 if self.pending_restore.is_some() {
1745 return Err(SubscriberError::Provider(
1746 "hybrid child restore reconciliation is already pending".into(),
1747 ));
1748 }
1749 if self.pending_cutover.is_some()
1750 || !self.pending_inputs.is_empty()
1751 || self.pending_output.is_some()
1752 || !self.live_buffer.is_empty()
1753 {
1754 return Err(SubscriberError::Provider(
1755 "hybrid live restore preparation requires a fresh coordinator with no in-flight delivery"
1756 .into(),
1757 ));
1758 }
1759 let intent = lifecycle_intent::<N>(&base_interests, &HashMap::new())?;
1760 let checkpoint = position.subscriber_checkpoint.as_ref().ok_or_else(|| {
1761 SubscriberError::Provider(
1762 "hybrid live restore preparation requires its coordinator checkpoint".into(),
1763 )
1764 })?;
1765 let mut state = decode_hybrid_checkpoint(checkpoint.as_bytes())?;
1766 let live_is_durable = self
1767 .live
1768 .capabilities()
1769 .supports(SubscriberCapability::DurableReplay);
1770 self.normalize_restore_candidate(&position, &mut state, live_is_durable)?;
1771 if intent != state.lifecycle_intent {
1772 return Err(SubscriberError::Provider(
1773 "restored runtime interests do not match the hybrid checkpoint lifecycle intent"
1774 .into(),
1775 ));
1776 }
1777 self.preflight_restore_delivery_capacity(&state)?;
1778 if let Some(prepared) = self.prepared_restore_position.as_ref() {
1779 if prepared == &position && self.lifecycle_intent == intent {
1780 return Ok(());
1781 }
1782 return Err(SubscriberError::Provider(
1783 "hybrid live source is already prepared for a different restore position"
1784 .into(),
1785 ));
1786 }
1787 if self.pending_restore_preparation.is_none() {
1788 if self.lifecycle_generation != 1
1789 || !self.base_interests.is_empty()
1790 || !self.owners.is_empty()
1791 {
1792 return Err(SubscriberError::Provider(
1793 "hybrid live restore preparation requires a newly constructed coordinator"
1794 .into(),
1795 ));
1796 }
1797 self.pending_restore_preparation = Some(PendingRestorePreparation {
1798 position: position.clone(),
1799 intent: intent.clone(),
1800 base_interests: base_interests.clone(),
1801 owners: Vec::new(),
1802 live_topology_installed: false,
1803 });
1804 } else if self
1805 .pending_restore_preparation
1806 .as_ref()
1807 .is_none_or(|pending| pending.position != position || pending.intent != intent)
1808 {
1809 return Err(SubscriberError::Provider(
1810 "hybrid live restore preparation is pending for different interests or a different position"
1811 .into(),
1812 ));
1813 }
1814
1815 if !live_is_durable
1816 && !self
1817 .pending_restore_preparation
1818 .as_ref()
1819 .expect("base restore preparation exists")
1820 .live_topology_installed
1821 {
1822 self.live.register_interests(&base_interests).await?;
1823 self.pending_restore_preparation
1824 .as_mut()
1825 .expect("base restore preparation survives registration")
1826 .live_topology_installed = true;
1827 }
1828 if !base_interests.is_empty() {
1829 self.ensure_source_chain(HybridSource::Live)?;
1830 }
1831
1832 self.base_interests = base_interests;
1833 self.owners.clear();
1834 self.lifecycle_intent = intent;
1835 self.prepared_restore_position = Some(position);
1836 self.pending_restore_preparation = None;
1837 Ok(())
1838 })
1839 }
1840
1841 fn prepare_empty_lifecycle_barrier(
1842 &self,
1843 lifecycle_generation: u64,
1844 owner_generations: BTreeMap<HandlerId, u64>,
1845 lifecycle_intent: LifecycleIntent,
1846 ) -> Result<PreparedEmptyLifecycleBarrier<N>, SubscriberError> {
1847 if lifecycle_intent.has_active_interests() {
1848 return Err(SubscriberError::Provider(
1849 "hybrid empty-lifecycle barrier was requested for an active topology".into(),
1850 ));
1851 }
1852
1853 let mut base_state = self.checkpoint_state();
1854 base_state.epoch = fresh_epoch();
1855 let sequence = 1_u64;
1856 base_state.next_synthetic_token = sequence.checked_add(1).ok_or_else(|| {
1857 SubscriberError::Provider("hybrid synthetic token space exhausted".into())
1858 })?;
1859 base_state.lifecycle_generation = lifecycle_generation;
1860 base_state.owner_generations = owner_generations;
1861 base_state.lifecycle_intent = lifecycle_intent;
1862 base_state.recent_inputs.clear();
1863 base_state.certified_historical = None;
1864 base_state.last_committed_token = None;
1865 for position in [
1866 &mut base_state.historical_position,
1867 &mut base_state.live_position,
1868 ] {
1869 position.delivery_token = None;
1874 }
1875 validate_checkpoint_state(&base_state)?;
1876
1877 let barrier_id = empty_lifecycle_barrier_id(&base_state)?;
1878 let control = ChainControl::Barrier {
1879 id: barrier_id,
1880 block: base_state.coverage_head,
1881 };
1882 let barrier_digest = source_delivery_digest::<N>(
1883 &[],
1884 std::slice::from_ref(&control),
1885 None,
1886 None,
1887 self.chain_id,
1888 )?;
1889 let raw_token = SubscriberDeliveryToken::new(sequence.to_be_bytes().to_vec());
1890 let pending = PendingCoordinatorCommit {
1891 audiences: Vec::new(),
1892 canonical: Vec::new(),
1893 source: HybridSource::Live,
1897 source_token: None,
1898 source_checkpoint: base_state.live_position.checkpoint.clone(),
1899 source_progress: None,
1900 source_observed_through: None,
1901 token_kind: HybridTokenKind::Synthetic,
1902 token_bytes: raw_token.as_bytes().to_vec(),
1903 source_delivery_digest: base_state
1908 .live_position
1909 .delivery_digest
1910 .unwrap_or(barrier_digest),
1911 next_safe_head: base_state.safe_head,
1912 next_finalized_head: base_state.finalized_head,
1913 next_canonical_history: base_state.canonical_history.clone(),
1914 next_coverage_head: base_state.coverage_head,
1915 };
1916 let mut committed_state = base_state.clone();
1917 apply_commit_to_checkpoint(
1918 &mut committed_state,
1919 &pending,
1920 self.config.canonical_history_capacity,
1921 )?;
1922 fit_checkpoint_to_durable_limits(
1923 &mut committed_state,
1924 0,
1925 self.config.recent_input_capacity,
1926 self.config.max_recent_owner_entries,
1927 )?;
1928 self.validate_checkpoint_config_limits(&committed_state)?;
1929 validate_checkpoint_state(&committed_state)?;
1930 let checkpoint = encode_hybrid_checkpoint(&committed_state)?;
1931 let output = ReactiveInputBatch::new(Vec::new())
1932 .with_chain_id(self.chain_id)
1933 .with_chain_controls([control])
1934 .with_subscriber_checkpoint(SubscriberCheckpoint::new(checkpoint))
1935 .with_delivery_token(wrap_token(
1936 base_state.epoch,
1937 HybridSource::Live,
1938 HybridTokenKind::Synthetic,
1939 raw_token,
1940 ));
1941
1942 Ok(PreparedEmptyLifecycleBarrier {
1943 base_state,
1944 pending,
1945 output,
1946 })
1947 }
1948
1949 fn install_empty_lifecycle_barrier(&mut self, prepared: PreparedEmptyLifecycleBarrier<N>) {
1950 let PreparedEmptyLifecycleBarrier {
1951 base_state,
1952 pending,
1953 output,
1954 } = prepared;
1955 self.install_checkpoint(base_state, false);
1956 self.phase = HybridPhase::Live;
1957 self.fence = None;
1958 self.drain_through = None;
1959 self.pending_cutover = None;
1960 self.acknowledged_historical_through = None;
1961 self.live_buffer.clear();
1962 self.buffered_live_records = 0;
1963 self.buffered_live_bytes = 0;
1964 self.pending_inputs.clear();
1965 self.pending_output = None;
1966 self.pending_live_rollback = None;
1967 self.restored_source_replays.clear();
1968 self.recovery_anchor = None;
1969 self.pending_restore_preparation = None;
1970 self.prepared_restore_position = None;
1971
1972 let key = (
1973 pending.source,
1974 pending.token_kind,
1975 pending.token_bytes.clone(),
1976 );
1977 self.pending_inputs.insert(key, pending);
1978 self.pending_output = Some(output);
1979 }
1980
1981 fn reset_delivery_state(&mut self) {
1985 debug_assert!(self.has_active_interests());
1986 self.epoch = fresh_epoch();
1987 self.next_synthetic_token = 1;
1988 self.phase = HybridPhase::CatchingUp;
1989 self.fence = None;
1990 self.drain_through = None;
1991 self.pending_cutover = None;
1992 self.acknowledged_historical_through = None;
1993 self.certified_historical = None;
1994 self.live_buffer.clear();
1995 self.buffered_live_records = 0;
1996 self.buffered_live_bytes = 0;
1997 self.recent_inputs.clear();
1998 self.recent_order.clear();
1999 self.pending_inputs.clear();
2000 self.pending_output = None;
2001 self.pending_live_rollback = None;
2002 self.restored_source_replays.clear();
2003 self.historical_position.delivery_token = None;
2004 self.historical_position.checkpoint = None;
2005 self.historical_position.delivery_digest = None;
2006 self.live_position.delivery_token = None;
2007 self.live_position.checkpoint = None;
2008 self.live_position.delivery_digest = None;
2009 self.last_committed_token = None;
2010 self.pending_restore_preparation = None;
2011 self.prepared_restore_position = None;
2012 }
2013
2014 fn ensure_healthy(&self) -> Result<(), SubscriberError> {
2015 match &self.poisoned {
2016 Some(reason) => Err(SubscriberError::Provider(format!(
2017 "hybrid coordinator is poisoned: {reason}"
2018 ))),
2019 None => Ok(()),
2020 }
2021 }
2022
2023 fn ensure_no_pending_restore(&self) -> Result<(), SubscriberError> {
2024 if self.pending_restore.is_some() {
2025 return Err(SubscriberError::Provider(
2026 "hybrid child restore reconciliation is pending; retry the exact same restore position before any other operation"
2027 .into(),
2028 ));
2029 }
2030 if self.pending_restore_preparation.is_some() {
2031 return Err(SubscriberError::Provider(
2032 "hybrid ephemeral-live restore preparation is pending; retry that exact preparation before any other operation"
2033 .into(),
2034 ));
2035 }
2036 if self.prepared_restore_position.is_some() {
2037 return Err(SubscriberError::Provider(
2038 "hybrid ephemeral live source is prepared; restore the exact durable position before any other operation"
2039 .into(),
2040 ));
2041 }
2042 Ok(())
2043 }
2044
2045 async fn reconcile_restored_source_acknowledgements(&mut self) -> Result<(), SubscriberError>
2046 where
2047 H: EventSubscriber<N>,
2048 L: EventSubscriber<N>,
2049 {
2050 for source in [HybridSource::Historical, HybridSource::Live] {
2051 if !self.restored_source_replays.contains(&source) {
2052 continue;
2053 }
2054 let token = match source {
2055 HybridSource::Historical => self.historical_position.delivery_token.clone(),
2056 HybridSource::Live => self.live_position.delivery_token.clone(),
2057 }
2058 .ok_or_else(|| {
2059 SubscriberError::Provider(format!(
2060 "hybrid restored {source:?} acknowledgement is missing its child token"
2061 ))
2062 })?;
2063 match source {
2064 HybridSource::Historical => {
2065 self.historical
2066 .acknowledge_delivery(SubscriberDeliveryToken::new(token))
2067 .await?;
2068 }
2069 HybridSource::Live => {
2070 self.live
2071 .acknowledge_delivery(SubscriberDeliveryToken::new(token))
2072 .await?;
2073 }
2074 }
2075 self.restored_source_replays.remove(&source);
2076 }
2077 Ok(())
2078 }
2079
2080 fn ensure_owner_managed_mode(&self) -> Result<(), SubscriberError> {
2081 if self.base_interests.is_empty() {
2082 Ok(())
2083 } else {
2084 Err(SubscriberError::InvalidConfig(EXCLUSIVE_TOPOLOGY_ERROR))
2085 }
2086 }
2087
2088 fn ensure_base_interest_mode(&self) -> Result<(), SubscriberError> {
2089 if self.owners.is_empty() {
2090 Ok(())
2091 } else {
2092 Err(SubscriberError::InvalidConfig(EXCLUSIVE_TOPOLOGY_ERROR))
2093 }
2094 }
2095
2096 fn ensure_source_chain(&self, source: HybridSource) -> Result<(), SubscriberError>
2097 where
2098 H: EventSubscriber<N>,
2099 L: EventSubscriber<N>,
2100 {
2101 let observed = match source {
2102 HybridSource::Historical => self.historical.chain_id(),
2103 HybridSource::Live => self.live.chain_id(),
2104 };
2105 match observed {
2106 Some(chain_id) if chain_id == self.chain_id => Ok(()),
2107 Some(chain_id) => Err(SubscriberError::Provider(format!(
2108 "hybrid {source:?} source changed to chain {chain_id}; expected {}",
2109 self.chain_id
2110 ))),
2111 None => Err(SubscriberError::Provider(format!(
2112 "hybrid {source:?} source has not resolved required chain {}",
2113 self.chain_id
2114 ))),
2115 }
2116 }
2117
2118 fn ensure_reconfigurable(&self) -> Result<(), SubscriberError> {
2119 self.ensure_no_pending_restore()?;
2120 if self.pending_cutover.is_some() || !self.pending_inputs.is_empty() {
2121 return Err(SubscriberError::Provider(
2122 "hybrid registration cannot change while a delivery is unacknowledged".into(),
2123 ));
2124 }
2125 if !self.live_buffer.is_empty() {
2126 return Err(SubscriberError::Provider(
2127 "hybrid registration cannot change while any buffered live delivery is pending"
2128 .into(),
2129 ));
2130 }
2131 if self.lifecycle_generation > 1 && self.phase != HybridPhase::Live {
2132 return Err(SubscriberError::Provider(
2133 "hybrid registration cannot change until the previous lifecycle revision has fully caught up and drained to Live"
2134 .into(),
2135 ));
2136 }
2137 Ok(())
2138 }
2139
2140 fn begin_reconfiguration_catchup(&mut self) {
2141 self.certified_historical = None;
2142 if !self.has_active_interests() {
2143 self.phase = HybridPhase::Live;
2144 self.fence = None;
2145 self.drain_through = None;
2146 self.pending_cutover = None;
2147 self.acknowledged_historical_through = None;
2148 self.recovery_anchor = None;
2149 return;
2150 }
2151 self.phase = HybridPhase::CatchingUp;
2152 self.drain_through = None;
2153 self.pending_cutover = None;
2154 self.acknowledged_historical_through = None;
2155 self.fence = self
2156 .live_buffer
2157 .front()
2158 .and_then(|batch| batch.max_canonical_block)
2159 .map(|head| head.saturating_sub(1));
2160 }
2161
2162 fn activate_lifecycle_recovery(&mut self) {
2163 let Some(LiveRollback::Recovery { anchor }) = self.pending_live_rollback.as_ref() else {
2164 return;
2165 };
2166 let anchor = *anchor;
2167 self.pending_live_rollback = None;
2168 self.phase = HybridPhase::Recovering;
2169 self.recovery_anchor = Some(anchor);
2170 self.fence = None;
2171 self.drain_through = None;
2172 self.pending_cutover = None;
2173 self.acknowledged_historical_through = None;
2174 self.certified_historical = None;
2175 self.live_buffer.clear();
2176 self.buffered_live_records = 0;
2177 self.buffered_live_bytes = 0;
2178 self.epoch = fresh_epoch();
2183 self.next_synthetic_token = 1;
2184 self.pending_inputs.clear();
2185 self.pending_output = None;
2186 self.restored_source_replays.clear();
2187 self.last_committed_token = None;
2188 for position in [&mut self.historical_position, &mut self.live_position] {
2189 position.delivery_token = None;
2190 position.checkpoint = None;
2191 position.delivery_digest = None;
2192 }
2193 }
2194
2195 fn poison(&mut self, reason: String) {
2196 self.phase = HybridPhase::Poisoned;
2197 self.poisoned = Some(reason);
2198 }
2199
2200 fn finish_acknowledged_cutover_if_ready(&mut self) {
2201 let Some((_fence, through)) = self
2202 .fence
2203 .zip(self.acknowledged_historical_through)
2204 .filter(|(fence, through)| through >= fence)
2205 else {
2206 return;
2207 };
2208 if self.pending_cutover.is_some()
2209 || !matches!(
2210 self.phase,
2211 HybridPhase::CatchingUp | HybridPhase::Recovering
2212 )
2213 {
2214 return;
2215 }
2216
2217 self.phase = HybridPhase::DrainingLive;
2222 self.drain_through = Some(through);
2223 self.recovery_anchor = None;
2224 self.acknowledged_historical_through = None;
2225 }
2226
2227 fn buffer_live(&mut self, batch: ReactiveInputBatch<N>) -> Result<(), SubscriberError> {
2228 if let Err(error) = self.ensure_source_chain(HybridSource::Live) {
2229 self.poison(error.to_string());
2230 return Err(error);
2231 }
2232 let batch = match BufferedBatch::from_batch(
2233 batch,
2234 self.chain_id,
2235 HybridIngressLimits::from(&self.config),
2236 &self.owner_generations,
2237 ) {
2238 Ok(batch) => batch,
2239 Err(error) => {
2240 self.poison(error.to_string());
2241 return Err(error);
2242 }
2243 };
2244 if let Err(error) = self.validate_owner_catchup_overlap(&batch) {
2245 self.poison(error.to_string());
2246 return Err(error);
2247 }
2248 if batch.token.is_some() && batch.max_canonical_block.is_none() {
2249 let message = "hybrid cannot buffer a tokened live delivery without canonical coverage; a durable live child must co-sequence finality and cursor-only controls with an acknowledgeable canonical batch";
2250 self.poison(message.into());
2251 return Err(SubscriberError::Unsupported(message));
2252 }
2253 if let Some(token) = batch.token.as_ref()
2254 && let Some(existing) = self.live_buffer.iter().find(|buffered| {
2255 buffered
2256 .token
2257 .as_ref()
2258 .is_some_and(|candidate| candidate.as_bytes() == token.as_bytes())
2259 })
2260 {
2261 if same_buffered_delivery(existing, &batch) {
2262 return Ok(());
2263 }
2264 let message = "hybrid live source reused one delivery token for different data";
2265 self.poison(message.into());
2266 return Err(SubscriberError::Provider(message.into()));
2267 }
2268 let record_count = batch.records.len();
2269 let next_records = self.buffered_live_records.saturating_add(record_count);
2270 let next_bytes = self
2271 .buffered_live_bytes
2272 .saturating_add(batch.accounted_bytes);
2273 if self.live_buffer.len() >= self.config.max_buffered_live_batches
2274 || next_records > self.config.max_buffered_live_records
2275 || next_bytes > self.config.max_buffered_live_bytes
2276 {
2277 let message = format!(
2278 "hybrid live buffer exceeded configured bounds before cutover \
2279 (batches {}/{}, records {}/{}, accounted bytes {}/{})",
2280 self.live_buffer.len().saturating_add(1),
2281 self.config.max_buffered_live_batches,
2282 next_records,
2283 self.config.max_buffered_live_records,
2284 next_bytes,
2285 self.config.max_buffered_live_bytes,
2286 );
2287 self.poison(message.clone());
2288 return Err(SubscriberError::Provider(message));
2289 }
2290 if self.fence.is_none()
2291 && let Some(head) = batch.max_canonical_block
2292 {
2293 self.fence = Some(head.saturating_sub(1));
2294 }
2295 self.buffered_live_records = next_records;
2296 self.buffered_live_bytes = next_bytes;
2297 self.live_buffer.push_back(batch);
2298 self.finish_acknowledged_cutover_if_ready();
2299 Ok(())
2300 }
2301
2302 fn emit_historical(
2303 &mut self,
2304 batch: ReactiveInputBatch<N>,
2305 ) -> Result<Option<ReactiveInputBatch<N>>, SubscriberError> {
2306 if let Err(error) = self.ensure_source_chain(HybridSource::Historical) {
2307 self.poison(error.to_string());
2308 return Err(error);
2309 }
2310 let mut batch = match BufferedBatch::from_batch(
2311 batch,
2312 self.chain_id,
2313 HybridIngressLimits::from(&self.config),
2314 &self.owner_generations,
2315 ) {
2316 Ok(batch) => batch,
2317 Err(error) => {
2318 self.poison(error.to_string());
2319 return Err(error);
2320 }
2321 };
2322 if let Err(error) = self.validate_owner_catchup_overlap(&batch) {
2323 self.poison(error.to_string());
2324 return Err(error);
2325 }
2326 if let Err(error) = self.reconcile_historical_recovery(&mut batch) {
2327 self.poison(error.to_string());
2328 return Err(error);
2329 }
2330 if let Err(error) = self.prepare_canonical_batch(&mut batch) {
2331 let error = error.into_subscriber_error();
2332 self.poison(error.to_string());
2333 return Err(error);
2334 }
2335 let original_max = batch.max_canonical_block;
2336 let token = batch.token.as_ref().ok_or_else(|| {
2337 SubscriberError::Provider(
2338 "hybrid durable historical delivery is missing its acknowledgement token".into(),
2339 )
2340 })?;
2341
2342 if self.pending_cutover.is_none()
2343 && self
2344 .fence
2345 .zip(original_max)
2346 .is_some_and(|(fence, observed)| observed >= fence)
2347 {
2348 self.pending_cutover = Some(PendingCutover {
2349 historical_token: token.as_bytes().to_vec(),
2350 through: original_max.expect("cutover condition has a block"),
2351 });
2352 }
2353
2354 let result = self.build_output(HybridSource::Historical, batch);
2355 if let Err(error) = &result {
2356 self.poison(error.to_string());
2357 }
2358 result
2359 }
2360
2361 fn reconcile_historical_recovery(
2362 &mut self,
2363 batch: &mut BufferedBatch<N>,
2364 ) -> Result<(), SubscriberError> {
2365 let Some(mut anchor) = self.recovery_anchor else {
2366 return Ok(());
2367 };
2368
2369 let mut controls = Vec::with_capacity(batch.controls.len());
2370 for mut control in std::mem::take(&mut batch.controls) {
2371 match &mut control {
2372 ChainControl::Reorg {
2373 common_ancestor,
2374 old_tip,
2375 ..
2376 } => {
2377 self.validate_reorg_ancestor(common_ancestor)?;
2378 if let Some(current_tip) = self.coverage_head.as_ref() {
2379 *old_tip = *current_tip;
2380 }
2381 anchor = *common_ancestor;
2382 self.recovery_anchor = Some(anchor);
2383 controls.push(control);
2384 }
2385 ChainControl::CanonicalProgress(block) => {
2386 if block.number <= anchor.number {
2387 self.validate_overlap_block(block)?;
2388 } else {
2389 controls.push(control);
2390 }
2391 }
2392 ChainControl::Barrier {
2393 block: Some(block), ..
2394 } if block.number <= anchor.number => {
2395 self.validate_overlap_block(block)?;
2396 }
2397 ChainControl::Safe(block) | ChainControl::Finalized(block)
2398 if block.number <= anchor.number =>
2399 {
2400 self.validate_overlap_block(block)?;
2401 controls.push(control);
2402 }
2403 _ => controls.push(control),
2404 }
2405 }
2406 batch.controls = controls;
2407
2408 let mut records = Vec::with_capacity(batch.records.len());
2409 for (record, audience, scope) in std::mem::take(&mut batch.records) {
2410 let suppress = scope_advances_canonical(scope)
2411 && canonical_block_ref(&record).is_some_and(|block| block.number <= anchor.number);
2412 if suppress {
2413 if let Some(block) = canonical_block_ref(&record) {
2414 self.validate_overlap_block(block)?;
2415 }
2416 self.validate_recent_witness(&record)?;
2417 } else {
2418 records.push((record, audience, scope));
2419 }
2420 }
2421 batch.records = records;
2422 Ok(())
2423 }
2424
2425 fn validate_overlap_block(&self, block: &BlockRef) -> Result<(), SubscriberError> {
2426 let known = self
2427 .canonical_history
2428 .iter()
2429 .find(|known| known.number == block.number)
2430 .or_else(|| {
2431 self.coverage_head
2432 .as_ref()
2433 .filter(|known| known.number == block.number)
2434 });
2435 let Some(known) = known else {
2436 return Err(SubscriberError::Provider(format!(
2437 "canonical overlap at block {} is outside retained hybrid history; increase canonical_history_capacity or perform a full resynchronization",
2438 block.number
2439 )));
2440 };
2441 compatible_block_ref(known, block).map(|_| ()).map_err(|_| {
2442 SubscriberError::Provider(format!(
2443 "historical/live overlap conflicts with committed canonical metadata at block {} without a usable reorg control",
2444 block.number
2445 ))
2446 })
2447 }
2448
2449 fn validate_recent_witness(
2450 &self,
2451 record: &ReactiveInputRecord<N>,
2452 ) -> Result<(), SubscriberError> {
2453 let (identity, witness) = validated_record(record, self.chain_id)?;
2454 let coverage = self.recent_inputs.get(&identity).ok_or_else(|| {
2455 SubscriberError::Provider(
2456 "historical overlap is outside the retained hybrid payload-witness window; a full resynchronization is required"
2457 .into(),
2458 )
2459 })?;
2460 let stored = coverage.witness.as_ref().ok_or_else(|| {
2461 SubscriberError::Provider(
2462 "hybrid recent-input journal is missing its payload witness".into(),
2463 )
2464 })?;
2465 ensure_witness_compatible(stored, &witness)?;
2466 Ok(())
2467 }
2468
2469 fn validate_owner_catchup_overlap(
2470 &self,
2471 batch: &BufferedBatch<N>,
2472 ) -> Result<(), SubscriberError> {
2473 for (record, _, scope) in &batch.records {
2474 if *scope != DeliveryScope::OwnerCatchup {
2475 continue;
2476 }
2477 let block = canonical_block_ref(record).ok_or_else(|| {
2478 SubscriberError::Provider(
2479 "hybrid owner-catchup input is missing an included canonical block".into(),
2480 )
2481 })?;
2482 self.validate_overlap_block(block).map_err(|error| {
2483 SubscriberError::Provider(format!(
2484 "hybrid owner-catchup block {} is not an exact retained canonical overlap; owner-only effects require a pre-existing core rollback-journal entry and cannot advance canonical state ({error})",
2485 block.number
2486 ))
2487 })?;
2488 }
2489 Ok(())
2490 }
2491
2492 fn validate_reorg_ancestor(&self, ancestor: &BlockRef) -> Result<(), SubscriberError> {
2493 if let Some(known) = self
2494 .canonical_history
2495 .iter()
2496 .find(|known| known.number == ancestor.number)
2497 {
2498 if known.hash == ancestor.hash {
2499 return Ok(());
2500 }
2501 return Err(SubscriberError::Provider(format!(
2502 "historical recovery reorg has a conflicting ancestor hash at block {}",
2503 ancestor.number
2504 )));
2505 }
2506 let Some(earliest) = self.canonical_history.front() else {
2507 return Err(SubscriberError::Provider(
2508 "historical recovery reorg has no retained rollback horizon".into(),
2509 ));
2510 };
2511 if ancestor.number < earliest.number {
2512 return Err(SubscriberError::Provider(format!(
2513 "historical recovery reorg ancestor {} is older than retained hybrid history; full resynchronization is required",
2514 ancestor.number
2515 )));
2516 }
2517 Ok(())
2518 }
2519
2520 fn prepare_canonical_batch(
2521 &self,
2522 batch: &mut BufferedBatch<N>,
2523 ) -> Result<(), CanonicalPlanError> {
2524 let validation_batch = ReactiveInputBatch::from_deliveries(batch.records.iter().map(
2525 |(record, audience, scope)| {
2526 ReactiveInputDelivery::new(record.clone(), audience.clone(), *scope)
2527 },
2528 ))
2529 .with_chain_id(self.chain_id)
2530 .with_chain_controls(batch.controls.clone());
2531 let sequence_state = CanonicalSequenceState::new(
2532 self.canonical_history.iter().copied().collect(),
2533 self.coverage_head,
2534 self.safe_head,
2535 self.finalized_head,
2536 );
2537 let validation = normalize_and_validate_canonical_sequence_diagnostic(
2538 &sequence_state,
2539 &validation_batch,
2540 )
2541 .map_err(classify_canonical_validation_error)?;
2542 batch.controls = validation.normalized_chain_controls().to_vec();
2543 batch.next_safe_head = validation.next_state().safe_head().copied();
2544 batch.next_finalized_head = validation.next_state().finalized_head().copied();
2545 batch.next_canonical_history = validation
2546 .next_state()
2547 .retained_canonical_history()
2548 .to_vec();
2549 batch.next_coverage_head = validation.next_state().coverage_head().copied();
2550 if let Some(coverage_head) = batch.next_coverage_head {
2551 if let Some(retained) = batch
2557 .next_canonical_history
2558 .iter_mut()
2559 .find(|retained| retained.number == coverage_head.number)
2560 {
2561 *retained = compatible_block_ref(retained, &coverage_head)
2562 .map_err(CanonicalPlanError::Invalid)?;
2563 } else {
2564 batch.next_canonical_history.push(coverage_head);
2565 batch
2566 .next_canonical_history
2567 .sort_by_key(|block| block.number);
2568 }
2569 }
2570 truncate_front_to(
2571 &mut batch.next_canonical_history,
2572 self.config.canonical_history_capacity,
2573 );
2574 batch.canonical.clear();
2575 for mutation in validation.mutations() {
2576 let projected = match mutation {
2577 CanonicalSequenceMutation::Rewind {
2578 common_ancestor: Some(ancestor),
2579 ..
2580 } => Some(CanonicalMutation::Rewind(*ancestor)),
2581 CanonicalSequenceMutation::Rewind {
2582 common_ancestor: None,
2583 ..
2584 } => Some(CanonicalMutation::Reset),
2585 CanonicalSequenceMutation::Canonical(block) => {
2586 Some(CanonicalMutation::Advance(*block))
2587 }
2588 CanonicalSequenceMutation::Safe(_) | CanonicalSequenceMutation::Finalized(_) => {
2589 None
2590 }
2591 _ => {
2592 return Err(CanonicalPlanError::Invalid(SubscriberError::Unsupported(
2593 "hybrid core validator returned an unknown canonical mutation",
2594 )));
2595 }
2596 };
2597 if let Some(projected) = projected {
2598 batch.canonical.push(projected);
2599 }
2600 }
2601 batch.source_progress = batch
2602 .canonical
2603 .iter()
2604 .rev()
2605 .find_map(|mutation| match mutation {
2606 CanonicalMutation::Rewind(block) | CanonicalMutation::Advance(block) => {
2607 Some(*block)
2608 }
2609 CanonicalMutation::Reset => None,
2610 });
2611 Ok(())
2612 }
2613
2614 fn emit_buffered_live(&mut self) -> Result<Option<ReactiveInputBatch<N>>, SubscriberError> {
2615 while let Some(mut batch) = self.live_buffer.pop_front() {
2616 self.buffered_live_records = self
2617 .buffered_live_records
2618 .saturating_sub(batch.records.len());
2619 self.buffered_live_bytes = self
2620 .buffered_live_bytes
2621 .saturating_sub(batch.accounted_bytes);
2622 let through = self.drain_through;
2623 if let Err(error) = self.reconcile_buffered_controls(&mut batch.controls, through) {
2624 self.poison(error.to_string());
2625 return Err(error);
2626 }
2627 let mut retained_records = Vec::with_capacity(batch.records.len());
2628 for record in std::mem::take(&mut batch.records) {
2629 let suppress_canonical_overlap = through.is_some_and(|through| {
2630 scope_advances_canonical(record.2)
2631 && canonical_block_number(&record.0).is_some_and(|number| number <= through)
2632 });
2633 let suppress_resolved_drop = through.is_some_and(|through| {
2634 scope_advances_canonical(record.2)
2635 && dropped_block_ref(&record.0).is_some_and(|dropped| {
2636 dropped.number <= through
2637 && !self.canonical_history.iter().any(|known| {
2638 known.number == dropped.number && known.hash == dropped.hash
2639 })
2640 })
2641 });
2642 if suppress_canonical_overlap {
2643 let block = canonical_block_ref(&record.0).expect("canonical number has block");
2644 if let Err(error) = self
2645 .validate_overlap_block(block)
2646 .and_then(|()| self.validate_recent_witness(&record.0))
2647 {
2648 self.poison(error.to_string());
2649 return Err(error);
2650 }
2651 } else if suppress_resolved_drop {
2652 } else {
2656 retained_records.push(record);
2657 }
2658 }
2659 batch.records = retained_records;
2660 if let Err(error) = self.prepare_canonical_batch(&mut batch) {
2661 match error {
2662 CanonicalPlanError::NeedsHistoricalRecovery(message) => {
2663 self.phase = HybridPhase::Recovering;
2664 self.recovery_anchor = self.coverage_head;
2665 return Err(SubscriberError::Provider(message));
2666 }
2667 CanonicalPlanError::Invalid(error) => {
2668 self.poison(error.to_string());
2669 return Err(error);
2670 }
2671 }
2672 }
2673 let output = self.build_output(HybridSource::Live, batch);
2674 let output = match output {
2675 Ok(output) => output,
2676 Err(error) => {
2677 self.poison(error.to_string());
2678 return Err(error);
2679 }
2680 };
2681 if let Some(output) = output {
2682 return Ok(Some(output));
2683 }
2684 }
2685 self.phase = HybridPhase::Live;
2686 self.fence = None;
2687 self.drain_through = None;
2688 self.acknowledged_historical_through = None;
2689 Ok(None)
2690 }
2691
2692 fn reconcile_buffered_controls(
2693 &self,
2694 controls: &mut Vec<ChainControl>,
2695 through: Option<u64>,
2696 ) -> Result<(), SubscriberError> {
2697 let Some(through) = through else {
2698 return Ok(());
2699 };
2700 let mut retained = Vec::with_capacity(controls.len());
2701 for control in std::mem::take(controls) {
2702 let keep = match &control {
2703 ChainControl::CanonicalProgress(block) if block.number <= through => {
2704 self.validate_overlap_block(block)?;
2705 false
2706 }
2707 ChainControl::CanonicalProgress(_) => true,
2708 ChainControl::Barrier {
2709 block: Some(block), ..
2710 } if block.number <= through => {
2711 self.validate_overlap_block(block)?;
2712 false
2713 }
2714 ChainControl::Reorg {
2715 common_ancestor,
2716 old_tip,
2717 ..
2718 } => {
2719 old_tip.number > through && common_ancestor.number >= through
2725 }
2726 ChainControl::Safe(block) | ChainControl::Finalized(block)
2727 if block.number <= through =>
2728 {
2729 self.validate_overlap_block(block)?;
2730 true
2731 }
2732 _ => true,
2733 };
2734 if keep {
2735 retained.push(control);
2736 }
2737 }
2738 *controls = retained;
2739 Ok(())
2740 }
2741
2742 fn emit_live(
2743 &mut self,
2744 batch: ReactiveInputBatch<N>,
2745 ) -> Result<Option<ReactiveInputBatch<N>>, SubscriberError> {
2746 if let Err(error) = self.ensure_source_chain(HybridSource::Live) {
2747 self.poison(error.to_string());
2748 return Err(error);
2749 }
2750 let batch = match BufferedBatch::from_batch(
2751 batch,
2752 self.chain_id,
2753 HybridIngressLimits::from(&self.config),
2754 &self.owner_generations,
2755 ) {
2756 Ok(batch) => batch,
2757 Err(error) => {
2758 self.poison(error.to_string());
2759 return Err(error);
2760 }
2761 };
2762 if let Err(error) = self.validate_owner_catchup_overlap(&batch) {
2763 self.poison(error.to_string());
2764 return Err(error);
2765 }
2766 let mut batch = batch;
2767 if let Err(error) = self.prepare_canonical_batch(&mut batch) {
2768 return match error {
2769 CanonicalPlanError::NeedsHistoricalRecovery(message) => {
2770 self.phase = HybridPhase::Recovering;
2771 self.recovery_anchor = self.coverage_head;
2772 Err(SubscriberError::Provider(message))
2773 }
2774 CanonicalPlanError::Invalid(error) => {
2775 self.poison(error.to_string());
2776 Err(error)
2777 }
2778 };
2779 }
2780 let result = self.build_output(HybridSource::Live, batch);
2781 if let Err(error) = &result {
2782 self.poison(error.to_string());
2783 }
2784 result
2785 }
2786
2787 fn filter_recent(
2788 &self,
2789 records: Vec<RoutedRecord<N>>,
2790 ) -> Result<(Vec<RoutedRecord<N>>, Vec<AudienceCommit>), SubscriberError> {
2791 let mut merged = Vec::<(RoutedRecord<N>, ReactiveInputIdentity, RecordWitness)>::new();
2792 let mut positions = HashMap::<ReactiveInputIdentity, usize>::new();
2793 for (record, audience, scope) in records {
2794 let (identity, witness) = validated_record(&record, self.chain_id)?;
2795 if let Some(index) = positions.get(&identity).copied() {
2796 let ((retained, retained_audience, retained_scope), _, retained_witness) =
2797 &mut merged[index];
2798 ensure_witness_compatible(retained_witness, &witness)?;
2799 let merged_duplicate = retained
2800 .merge_compatible_duplicate(&record)
2801 .map_err(|error| SubscriberError::Provider(error.to_string()))?;
2802 if !merged_duplicate {
2803 merged.push(((record, audience, scope), identity, witness));
2807 continue;
2808 }
2809 *retained_witness = record_witness(retained, self.chain_id)?;
2810 merge_audience(retained_audience, audience)?;
2811 merge_scope(retained_scope, scope)?;
2812 } else {
2813 positions.insert(identity, merged.len());
2814 merged.push(((record, audience, scope), identity, witness));
2815 }
2816 }
2817
2818 let mut staged = HashMap::<ReactiveInputIdentity, AudienceCoverage>::new();
2819 let mut commits = Vec::new();
2820 let mut output = Vec::new();
2821 for ((record, audience, scope), identity, witness) in merged {
2822 if !deduplicable(&record) {
2823 output.push((record, audience, scope));
2824 continue;
2825 }
2826 let coverage = match staged.entry(identity) {
2827 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
2828 std::collections::hash_map::Entry::Vacant(entry) => {
2829 let coverage = match self.recent_inputs.get(&identity) {
2830 Some(existing) => {
2831 let mut existing = existing.clone();
2832 let stored = existing.witness.as_mut().ok_or_else(|| {
2833 SubscriberError::Provider(
2834 "hybrid recent-input journal is missing its payload witness"
2835 .into(),
2836 )
2837 })?;
2838 merge_witness(stored, &witness)?;
2839 existing
2840 }
2841 None => AudienceCoverage {
2842 witness: Some(witness.clone()),
2843 ..AudienceCoverage::default()
2844 },
2845 };
2846 entry.insert(coverage)
2847 }
2848 };
2849 let Some(residual) = residual_audience(coverage, &audience, &self.owner_generations)
2850 else {
2851 commits.push(AudienceCommit {
2855 identity,
2856 audience: DeliveryAudience::Owners(Vec::new()),
2857 block: canonical_block_ref(&record).cloned(),
2858 witness: coverage
2859 .witness
2860 .clone()
2861 .expect("restored coverage has a validated witness"),
2862 });
2863 continue;
2864 };
2865 apply_audience_coverage(coverage, &residual, &self.owner_generations);
2866 commits.push(AudienceCommit {
2867 identity,
2868 audience: residual.clone(),
2869 block: canonical_block_ref(&record).cloned(),
2870 witness: coverage
2871 .witness
2872 .clone()
2873 .expect("new and restored coverage has a validated witness"),
2874 });
2875 output.push((record, residual, scope));
2876 }
2877 Ok((output, commits))
2878 }
2879
2880 fn build_output(
2881 &mut self,
2882 source: HybridSource,
2883 batch: BufferedBatch<N>,
2884 ) -> Result<Option<ReactiveInputBatch<N>>, SubscriberError> {
2885 let BufferedBatch {
2886 records,
2887 controls,
2888 token,
2889 checkpoint,
2890 payload_commitment,
2891 max_canonical_block,
2892 source_progress: source_progress_hint,
2893 delivery_digest: source_delivery_digest,
2894 mut canonical,
2895 mut next_safe_head,
2896 mut next_finalized_head,
2897 mut next_canonical_history,
2898 mut next_coverage_head,
2899 ..
2900 } = batch;
2901 let source_checkpoint = checkpoint.map(SubscriberCheckpoint::into_bytes);
2902 let source_position = match source {
2903 HybridSource::Historical => &self.historical_position,
2904 HybridSource::Live => &self.live_position,
2905 };
2906 let restored_replay = token.as_ref().is_some_and(|token| {
2907 source_position.delivery_token.as_deref() == Some(token.as_bytes())
2908 });
2909 if restored_replay {
2910 if source_position.delivery_digest != Some(source_delivery_digest) {
2911 let message = "hybrid source reused a committed delivery token for different data";
2912 self.poison(message.into());
2913 return Err(SubscriberError::Provider(message.into()));
2914 }
2915 if !self.restored_source_replays.contains(&source) {
2916 let message = "hybrid source reused one delivery token after it was committed";
2917 self.poison(message.into());
2918 return Err(SubscriberError::Provider(message.into()));
2919 }
2920 } else {
2921 self.restored_source_replays.remove(&source);
2922 }
2923 let (records, commits, controls, source_progress_hint) = if restored_replay {
2924 canonical.clear();
2928 next_safe_head = self.safe_head;
2929 next_finalized_head = self.finalized_head;
2930 next_canonical_history = self.canonical_history.iter().copied().collect();
2931 next_coverage_head = self.coverage_head;
2932 (Vec::new(), Vec::new(), Vec::new(), None)
2933 } else {
2934 let (records, commits) = match self.filter_recent(records) {
2935 Ok(filtered) => filtered,
2936 Err(error) => {
2937 self.poison(error.to_string());
2938 return Err(error);
2939 }
2940 };
2941 (records, commits, controls, source_progress_hint)
2942 };
2943 if records.is_empty() && controls.is_empty() && token.is_none() {
2944 return Ok(None);
2945 }
2946 let derived_source_progress = canonical.iter().rev().find_map(|mutation| match mutation {
2947 CanonicalMutation::Rewind(block) | CanonicalMutation::Advance(block) => Some(*block),
2948 CanonicalMutation::Reset => None,
2949 });
2950 let source_progress = source_progress_hint.or(derived_source_progress);
2951 let mut output =
2952 ReactiveInputBatch::from_deliveries(records.into_iter().map(
2953 |(record, audience, scope)| ReactiveInputDelivery::new(record, audience, scope),
2954 ))
2955 .with_chain_id(self.chain_id)
2956 .with_chain_controls(controls);
2957 if let Some(payload_commitment) = payload_commitment {
2958 output = output.with_payload_commitment(payload_commitment);
2959 }
2960 let (kind, token) = match token {
2961 Some(token) => (HybridTokenKind::Forwarded, token),
2962 None => {
2963 let sequence = self.next_synthetic_token;
2964 self.next_synthetic_token =
2965 self.next_synthetic_token.checked_add(1).ok_or_else(|| {
2966 SubscriberError::Provider("hybrid synthetic token space exhausted".into())
2967 })?;
2968 (
2969 HybridTokenKind::Synthetic,
2970 SubscriberDeliveryToken::new(sequence.to_be_bytes().to_vec()),
2971 )
2972 }
2973 };
2974 let source_token = (kind == HybridTokenKind::Forwarded).then(|| token.as_bytes().to_vec());
2975 let pending = PendingCoordinatorCommit {
2976 audiences: commits,
2977 canonical,
2978 source,
2979 source_token,
2980 source_checkpoint,
2981 source_progress,
2982 source_observed_through: max_canonical_block.and_then(|number| {
2983 next_canonical_history
2984 .iter()
2985 .find(|block| block.number == number)
2986 .copied()
2987 .or_else(|| next_coverage_head.filter(|block| block.number == number))
2988 }),
2989 token_kind: kind,
2990 token_bytes: token.as_bytes().to_vec(),
2991 source_delivery_digest,
2992 next_safe_head,
2993 next_finalized_head,
2994 next_canonical_history,
2995 next_coverage_head,
2996 };
2997 let durable_checkpoint = self.encode_checkpoint_after(&pending)?;
2998 self.pending_inputs
2999 .insert((source, kind, token.as_bytes().to_vec()), pending);
3000 output = output
3001 .with_subscriber_checkpoint(durable_checkpoint)
3002 .with_delivery_token(wrap_token(self.epoch, source, kind, token));
3003 self.pending_output = Some(output.clone());
3004 if restored_replay {
3005 self.restored_source_replays.remove(&source);
3006 }
3007 Ok(Some(output))
3008 }
3009
3010 fn commit_coordinator(
3011 &mut self,
3012 commit: PendingCoordinatorCommit,
3013 ) -> Result<(), SubscriberError> {
3014 let rewound = commit.canonical.iter().any(|mutation| {
3015 matches!(
3016 mutation,
3017 CanonicalMutation::Rewind(_) | CanonicalMutation::Reset
3018 )
3019 });
3020 let state = self.checkpoint_after(&commit)?;
3021 self.install_checkpoint(state, false);
3022 if rewound {
3023 self.restored_source_replays.clear();
3024 }
3025 Ok(())
3026 }
3027
3028 fn encode_checkpoint_after(
3029 &self,
3030 commit: &PendingCoordinatorCommit,
3031 ) -> Result<SubscriberCheckpoint, SubscriberError> {
3032 let state = self.checkpoint_after(commit)?;
3033 encode_hybrid_checkpoint(&state).map(SubscriberCheckpoint::new)
3034 }
3035
3036 fn checkpoint_after(
3037 &self,
3038 commit: &PendingCoordinatorCommit,
3039 ) -> Result<HybridCheckpointV5, SubscriberError> {
3040 let mut state = self.checkpoint_state();
3041 apply_commit_to_checkpoint(&mut state, commit, self.config.canonical_history_capacity)?;
3042 let protected_recent_inputs = commit
3043 .audiences
3044 .iter()
3045 .map(|audience| audience.identity)
3046 .collect::<HashSet<_>>()
3047 .len();
3048 fit_checkpoint_to_durable_limits(
3049 &mut state,
3050 protected_recent_inputs,
3051 self.config.recent_input_capacity,
3052 self.config.max_recent_owner_entries,
3053 )?;
3054 self.validate_checkpoint_config_limits(&state)?;
3055 validate_checkpoint_state(&state)?;
3056 Ok(state)
3057 }
3058
3059 fn checkpoint_state(&self) -> HybridCheckpointV5 {
3060 HybridCheckpointV5 {
3061 chain_id: self.chain_id,
3062 epoch: self.epoch,
3063 next_synthetic_token: self.next_synthetic_token,
3064 lifecycle_generation: self.lifecycle_generation,
3065 owner_generations: self.owner_generations.clone(),
3066 lifecycle_intent: self.lifecycle_intent.clone(),
3067 recent_inputs: self
3068 .recent_order
3069 .iter()
3070 .filter_map(|identity| {
3071 self.recent_inputs
3072 .get(identity)
3073 .cloned()
3074 .map(|coverage| StoredRecentInput {
3075 identity: *identity,
3076 coverage,
3077 })
3078 })
3079 .collect(),
3080 canonical_history: self.canonical_history.iter().cloned().collect(),
3081 coverage_head: self.coverage_head,
3082 safe_head: self.safe_head,
3083 finalized_head: self.finalized_head,
3084 certified_historical: self.certified_historical,
3085 historical_position: self.historical_position.clone(),
3086 live_position: self.live_position.clone(),
3087 last_committed_token: self.last_committed_token.clone(),
3088 }
3089 }
3090
3091 fn install_checkpoint(&mut self, state: HybridCheckpointV5, restored: bool) {
3092 self.epoch = state.epoch;
3093 self.next_synthetic_token = state.next_synthetic_token;
3094 self.lifecycle_generation = state.lifecycle_generation;
3095 self.owner_generations = state.owner_generations;
3096 self.lifecycle_intent = state.lifecycle_intent;
3097 self.recent_order = state
3098 .recent_inputs
3099 .iter()
3100 .map(|entry| entry.identity)
3101 .collect();
3102 self.recent_inputs = state
3103 .recent_inputs
3104 .into_iter()
3105 .map(|entry| (entry.identity, entry.coverage))
3106 .collect();
3107 self.canonical_history = state.canonical_history.into_iter().collect();
3108 self.coverage_head = state.coverage_head;
3109 self.safe_head = state.safe_head;
3110 self.finalized_head = state.finalized_head;
3111 self.certified_historical = state.certified_historical;
3112 self.historical_position = state.historical_position;
3113 self.live_position = state.live_position;
3114 self.last_committed_token = state.last_committed_token;
3115 if restored {
3116 self.restored_source_replays.clear();
3117 if self.historical_position.delivery_token.is_some() {
3118 self.restored_source_replays
3119 .insert(HybridSource::Historical);
3120 }
3121 if self.live_position.delivery_token.is_some()
3122 && self
3123 .live
3124 .capabilities()
3125 .supports(SubscriberCapability::DurableReplay)
3126 {
3127 self.restored_source_replays.insert(HybridSource::Live);
3128 }
3129 }
3130 }
3131
3132 fn reconcile_pending_restore(&mut self) -> Result<(), SubscriberError>
3133 where
3134 H: EventSubscriber<N>,
3135 L: EventSubscriber<N>,
3136 {
3137 let mut pending = self
3138 .pending_restore
3139 .take()
3140 .expect("restore reconciliation requires an intent");
3141 if !pending.historical_restored {
3142 let resume = pending
3143 .historical_resume
3144 .as_ref()
3145 .expect("unfinished historical restore has a position");
3146 if let Err(error) = self.historical.restore_position(resume) {
3147 self.pending_restore = Some(pending);
3148 return Err(error);
3149 }
3150 pending.historical_restored = true;
3151 }
3152 if !pending.live_restored {
3153 let resume = pending
3154 .live_resume
3155 .as_ref()
3156 .expect("unfinished live restore has a position");
3157 if let Err(error) = self.live.restore_position(resume) {
3158 self.pending_restore = Some(pending);
3159 return Err(error);
3160 }
3161 pending.live_restored = true;
3162 }
3163 if let Err(error) = self.ensure_source_chain(HybridSource::Historical) {
3164 self.pending_restore = Some(pending);
3165 return Err(error);
3166 }
3167 let lifecycle_nonempty = pending.state.lifecycle_intent.has_active_interests();
3168 if (pending.live_resume.is_some() || lifecycle_nonempty)
3169 && let Err(error) = self.ensure_source_chain(HybridSource::Live)
3170 {
3171 self.pending_restore = Some(pending);
3172 return Err(error);
3173 }
3174
3175 let coverage_head = pending.position.coverage_head;
3176 self.install_checkpoint(pending.state, true);
3177 self.phase = if lifecycle_nonempty {
3178 HybridPhase::Recovering
3179 } else {
3180 HybridPhase::Live
3181 };
3182 self.fence = None;
3183 self.drain_through = None;
3184 self.pending_cutover = None;
3185 self.acknowledged_historical_through = self
3190 .certified_historical
3191 .filter(|proof| proof.lifecycle_generation == self.lifecycle_generation)
3192 .map(|proof| proof.through.number);
3193 self.recovery_anchor = lifecycle_nonempty.then_some(coverage_head);
3194 self.buffered_live_records = 0;
3195 self.buffered_live_bytes = 0;
3196 self.prepared_restore_position = None;
3197 Ok(())
3198 }
3199}
3200
3201fn classify_canonical_validation_error(error: CanonicalSequenceError) -> CanonicalPlanError {
3202 let requires_history = error.requires_history();
3203 let error = error.into_reactive_error();
3204 if requires_history {
3205 return CanonicalPlanError::NeedsHistoricalRecovery(format!(
3206 "hybrid canonical sequence requires durable historical recovery: {error}"
3207 ));
3208 }
3209 CanonicalPlanError::Invalid(SubscriberError::Provider(format!(
3210 "hybrid canonical sequence validation failed: {error}"
3211 )))
3212}
3213
3214impl<H, L, N> EventSubscriber<N> for HybridSubscriber<H, L, N>
3215where
3216 H: EventSubscriber<N>,
3217 L: EventSubscriber<N>,
3218 N: Network + Send + 'static,
3219{
3220 fn chain_id(&self) -> Option<u64> {
3221 Some(self.chain_id)
3222 }
3223
3224 fn capabilities(&self) -> SubscriberCapabilities {
3225 let historical = self.historical.capabilities();
3226 let live = self.live.capabilities();
3227 let mut capabilities = Vec::new();
3228 if historical.supports(SubscriberCapability::HistoricalBackfill) {
3229 capabilities.push(SubscriberCapability::HistoricalBackfill);
3230 }
3231 if live.supports(SubscriberCapability::Live) {
3232 capabilities.push(SubscriberCapability::Live);
3233 }
3234 if historical.supports(SubscriberCapability::DurableReplay) {
3239 capabilities.push(SubscriberCapability::DurableReplay);
3240 }
3241 if historical.supports(SubscriberCapability::Barriers) {
3242 capabilities.push(SubscriberCapability::Barriers);
3243 }
3244 for capability in [
3245 SubscriberCapability::Logs,
3246 SubscriberCapability::PendingTransactionHashes,
3247 SubscriberCapability::OwnerScopedDelivery,
3248 SubscriberCapability::DynamicInterests,
3249 SubscriberCapability::ExplicitReorgs,
3250 SubscriberCapability::FinalityUpdates,
3251 ] {
3252 if historical.supports(capability) && live.supports(capability) {
3253 capabilities.push(capability);
3254 }
3255 }
3256 SubscriberCapabilities::new(capabilities)
3261 }
3262
3263 fn register_interests(
3264 &mut self,
3265 interests: &[ReactiveInterest<N>],
3266 ) -> SubscriberOperation<'_, ()> {
3267 let interests = interests.to_vec();
3268 Box::pin(async move {
3269 self.ensure_healthy()?;
3270 self.ensure_no_pending_restore()?;
3271 reconcile_pending_base_lifecycle(
3272 &mut self.historical,
3273 &mut self.live,
3274 &mut self.pending_live_rollback,
3275 &mut self.phase,
3276 &mut self.poisoned,
3277 )
3278 .await?;
3279 self.ensure_base_interest_mode()?;
3280 self.ensure_reconfigurable()?;
3281 let next_intent = lifecycle_intent(&interests, &HashMap::new())?;
3282 if next_intent == self.lifecycle_intent {
3283 return Ok(());
3284 }
3285 let next_active = !interests.is_empty();
3286 if next_active {
3287 self.ensure_source_chain(HybridSource::Historical)?;
3288 }
3289 if self.coverage_head.is_some() && next_active {
3290 return Err(SubscriberError::Unsupported(
3291 "hybrid cannot replace a base/unowned topology after canonical coverage exists because EventSubscriber has no atomic global-backfill rollback primitive; use handler-owned interests or restore a fresh coordinator",
3292 ));
3293 }
3294 let next_generation = self.lifecycle_generation.checked_add(1).ok_or_else(|| {
3295 SubscriberError::Provider("hybrid lifecycle generation exhausted".into())
3296 })?;
3297 self.preflight_base_topology(&next_intent, next_generation)?;
3298 let empty_barrier = if next_active {
3299 None
3300 } else {
3301 Some(self.prepare_empty_lifecycle_barrier(
3302 next_generation,
3303 BTreeMap::new(),
3304 next_intent.clone(),
3305 )?)
3306 };
3307 let previous = self.base_interests.clone();
3308 let gap_anchor = self.coverage_head.filter(|_| !previous.is_empty());
3309 self.pending_live_rollback = Some(LiveRollback::Base {
3310 previous: previous.clone(),
3311 gap_anchor,
3312 });
3313 self.live.register_interests(&interests).await?;
3314 if next_active && let Err(chain_error) = self.ensure_source_chain(HybridSource::Live) {
3315 return match reconcile_pending_base_lifecycle(
3316 &mut self.historical,
3317 &mut self.live,
3318 &mut self.pending_live_rollback,
3319 &mut self.phase,
3320 &mut self.poisoned,
3321 )
3322 .await
3323 {
3324 Ok(()) => Err(chain_error),
3325 Err(rollback) => Err(SubscriberError::Provider(format!(
3326 "live chain validation failed ({chain_error}); {rollback}"
3327 ))),
3328 };
3329 }
3330 if let Err(history_error) = self.historical.register_interests(&interests).await {
3331 return match reconcile_pending_base_lifecycle(
3332 &mut self.historical,
3333 &mut self.live,
3334 &mut self.pending_live_rollback,
3335 &mut self.phase,
3336 &mut self.poisoned,
3337 )
3338 .await
3339 {
3340 Ok(()) => Err(history_error),
3341 Err(rollback) => Err(SubscriberError::Provider(format!(
3342 "historical registration failed ({history_error}); {rollback}"
3343 ))),
3344 };
3345 }
3346 self.pending_live_rollback = None;
3347 self.base_interests = interests;
3348 self.owners.clear();
3349 self.lifecycle_generation = next_generation;
3350 self.owner_generations.clear();
3351 self.lifecycle_intent = next_intent;
3352 if let Some(prepared) = empty_barrier {
3353 self.install_empty_lifecycle_barrier(prepared);
3354 } else {
3355 self.reset_delivery_state();
3356 }
3357 Ok(())
3358 })
3359 }
3360
3361 fn next_batch(&mut self) -> SubscriberNextBatch<'_, N> {
3362 Box::pin(async move {
3363 self.ensure_healthy()?;
3364 self.ensure_no_pending_restore()?;
3365 self.reconcile_restored_source_acknowledgements().await?;
3370 self.activate_lifecycle_recovery();
3371 if self.pending_live_rollback.is_some() {
3372 return Err(SubscriberError::Provider(
3373 "hybrid lifecycle reconciliation is pending; retry the cancelled lifecycle operation before delivery".into(),
3374 ));
3375 }
3376 if let Some(output) = self.pending_output.as_ref() {
3377 return Ok(Some(output.clone()));
3378 }
3379 if !self.has_active_interests() {
3386 return Ok(None);
3387 }
3388 loop {
3389 match self.phase {
3390 HybridPhase::DrainingLive => {
3391 if let Some(batch) = self.emit_buffered_live()? {
3392 return Ok(Some(batch));
3393 }
3394 }
3395 HybridPhase::Live => match self.live.next_batch().await {
3396 Ok(Some(batch)) => {
3397 if let Some(batch) = self.emit_live(batch)? {
3398 return Ok(Some(batch));
3399 }
3400 }
3401 Ok(None) => {
3402 self.phase = HybridPhase::Recovering;
3403 self.recovery_anchor = self.coverage_head;
3404 return Err(SubscriberError::Provider(
3405 "hybrid live source ended; entering historical recovery".into(),
3406 ));
3407 }
3408 Err(error) => {
3409 self.phase = HybridPhase::Recovering;
3410 self.recovery_anchor = self.coverage_head;
3411 return Err(error);
3412 }
3413 },
3414 HybridPhase::CatchingUp | HybridPhase::Recovering => {
3415 if self.live_buffer.iter().any(|batch| batch.token.is_some()) {
3422 match self.historical.next_batch().await? {
3423 Some(batch) => {
3424 if let Some(batch) = self.emit_historical(batch)? {
3425 return Ok(Some(batch));
3426 }
3427 }
3428 None => {
3429 return Err(SubscriberError::Provider(
3430 "hybrid historical source ended without an acknowledged coverage proof for the live fence".into(),
3431 ));
3432 }
3433 }
3434 continue;
3435 }
3436 let (historical, live) = (&mut self.historical, &mut self.live);
3437 tokio::select! {
3438 historical_result = historical.next_batch() => {
3439 match historical_result? {
3440 Some(batch) => {
3441 if let Some(batch) = self.emit_historical(batch)? {
3442 return Ok(Some(batch));
3443 }
3444 }
3445 None => {
3446 return Err(SubscriberError::Provider(
3447 "hybrid historical source ended without an acknowledged coverage proof for the live fence".into(),
3448 ));
3449 }
3450 }
3451 }
3452 live_result = live.next_batch() => {
3453 match live_result {
3454 Ok(Some(batch)) => self.buffer_live(batch)?,
3455 Ok(None) => {
3456 self.phase = HybridPhase::Recovering;
3457 self.recovery_anchor = self.coverage_head;
3458 return Err(SubscriberError::Provider(
3459 "hybrid live source ended during catch-up/recovery".into(),
3460 ));
3461 }
3462 Err(error) => {
3463 self.phase = HybridPhase::Recovering;
3464 self.recovery_anchor = self.coverage_head;
3465 return Err(error);
3466 }
3467 }
3468 }
3469 }
3470 }
3471 HybridPhase::Poisoned => self.ensure_healthy()?,
3472 }
3473 }
3474 })
3475 }
3476
3477 fn restore_position(
3478 &mut self,
3479 position: &SubscriberResumePosition,
3480 ) -> Result<(), SubscriberError> {
3481 self.ensure_healthy()?;
3482 if self.pending_restore_preparation.is_some() {
3483 return Err(SubscriberError::Provider(
3484 "hybrid ephemeral-live restore preparation is incomplete".into(),
3485 ));
3486 }
3487 if let Some(pending) = self.pending_restore.as_ref() {
3488 if pending.position != *position {
3489 return Err(SubscriberError::Provider(
3490 "hybrid child restore reconciliation is pending for a different position"
3491 .into(),
3492 ));
3493 }
3494 return self.reconcile_pending_restore();
3495 }
3496 if self.pending_cutover.is_some()
3497 || !self.pending_inputs.is_empty()
3498 || self.pending_output.is_some()
3499 || !self.live_buffer.is_empty()
3500 {
3501 return Err(SubscriberError::Provider(
3502 "hybrid position restore requires a coordinator with no in-flight delivery".into(),
3503 ));
3504 }
3505 if position.chain_id != self.chain_id {
3506 return Err(SubscriberError::Provider(format!(
3507 "hybrid restore targets chain {}, but this coordinator targets {}",
3508 position.chain_id, self.chain_id
3509 )));
3510 }
3511 let checkpoint = position.subscriber_checkpoint.as_ref().ok_or_else(|| {
3512 SubscriberError::Provider(
3513 "hybrid restore requires its versioned coordinator checkpoint".into(),
3514 )
3515 })?;
3516 let mut state = decode_hybrid_checkpoint(checkpoint.as_bytes())?;
3517 let live_is_durable = self
3518 .live
3519 .capabilities()
3520 .supports(SubscriberCapability::DurableReplay);
3521 let rotate_epoch =
3522 self.normalize_restore_candidate(position, &mut state, live_is_durable)?;
3523 if let Some(prepared) = self.prepared_restore_position.as_ref()
3524 && prepared != position
3525 {
3526 return Err(SubscriberError::Provider(
3527 "hybrid live source was prepared for a different restore position".into(),
3528 ));
3529 }
3530 let lifecycle_requires_preparation = state.lifecycle_intent.requires_restore_preparation();
3531 if lifecycle_requires_preparation
3532 && self.prepared_restore_position.as_ref() != Some(position)
3533 {
3534 return Err(SubscriberError::Provider(
3535 "restoring a non-empty Hybrid lifecycle or owner topology requires prepare_restore_base_lifecycle (or owner-capable prepare_restore_lifecycle) first"
3536 .into(),
3537 ));
3538 }
3539 if let Some(prepared) = self.prepared_restore_position.as_ref()
3540 && (prepared != position || self.lifecycle_intent != state.lifecycle_intent)
3541 {
3542 return Err(SubscriberError::Provider(
3543 "prepared live subscriptions do not match the Hybrid checkpoint".into(),
3544 ));
3545 }
3546 self.preflight_restore_delivery_capacity(&state)?;
3550 if rotate_epoch {
3551 state.epoch = fresh_epoch();
3552 }
3553 validate_checkpoint_state(&state)?;
3554
3555 let historical_resume = source_resume_position(self.chain_id, &state.historical_position);
3556 let live_resume = live_is_durable
3557 .then(|| source_resume_position(self.chain_id, &state.live_position))
3558 .flatten();
3559 self.pending_restore = Some(PendingRestore {
3560 position: position.clone(),
3561 state,
3562 historical_restored: historical_resume.is_none(),
3563 live_restored: live_resume.is_none(),
3564 historical_resume,
3565 live_resume,
3566 });
3567 self.reconcile_pending_restore()
3568 }
3569
3570 fn acknowledge_delivery(
3571 &mut self,
3572 token: SubscriberDeliveryToken,
3573 ) -> SubscriberOperation<'_, ()> {
3574 Box::pin(async move {
3575 self.ensure_healthy()?;
3576 self.ensure_no_pending_restore()?;
3577 if self.pending_live_rollback.is_some() {
3578 return Err(SubscriberError::Provider(
3579 "hybrid lifecycle reconciliation is pending; retry the cancelled lifecycle operation before acknowledgement".into(),
3580 ));
3581 }
3582 let (epoch, source, kind, inner) = unwrap_token(token)?;
3583 if epoch != self.epoch {
3584 return Err(SubscriberError::Provider(
3585 "hybrid delivery token belongs to a different coordinator epoch".into(),
3586 ));
3587 }
3588 let token_key = (source, kind, inner.as_bytes().to_vec());
3589 if !self.pending_inputs.contains_key(&token_key) {
3590 if self.last_committed_token.as_ref()
3591 == Some(&StoredCommittedToken {
3592 source,
3593 kind,
3594 inner: inner.as_bytes().to_vec(),
3595 })
3596 {
3597 return Ok(());
3598 }
3599 return Err(SubscriberError::Provider(
3600 "hybrid acknowledgement does not match the in-flight delivery".into(),
3601 ));
3602 }
3603 let completes_cutover = kind == HybridTokenKind::Forwarded
3604 && source == HybridSource::Historical
3605 && self
3606 .pending_cutover
3607 .as_ref()
3608 .is_some_and(|pending| pending.historical_token == inner.as_bytes());
3609 if kind == HybridTokenKind::Forwarded {
3610 match source {
3611 HybridSource::Historical => {
3612 self.historical.acknowledge_delivery(inner.clone()).await?;
3613 }
3614 HybridSource::Live => self.live.acknowledge_delivery(inner).await?,
3615 }
3616 }
3617 if let Some(commit) = self.pending_inputs.get(&token_key).cloned() {
3618 let acknowledged_historical_through = (source == HybridSource::Historical
3619 && matches!(
3620 self.phase,
3621 HybridPhase::CatchingUp | HybridPhase::Recovering
3622 ))
3623 .then_some(commit.source_observed_through)
3624 .flatten()
3625 .map(|block| block.number);
3626 self.commit_coordinator(commit)?;
3627 self.pending_inputs.remove(&token_key);
3628 self.pending_output = None;
3629 if let Some(through) = acknowledged_historical_through {
3630 self.acknowledged_historical_through = Some(
3631 self.acknowledged_historical_through
3632 .map_or(through, |current| current.max(through)),
3633 );
3634 }
3635 }
3636 if completes_cutover {
3637 let pending = self
3638 .pending_cutover
3639 .take()
3640 .expect("matched pending cutover");
3641 self.drain_through = Some(pending.through);
3642 self.recovery_anchor = None;
3643 self.phase = HybridPhase::DrainingLive;
3644 self.acknowledged_historical_through = None;
3645 }
3646 if self.phase == HybridPhase::DrainingLive && self.live_buffer.is_empty() {
3647 self.phase = HybridPhase::Live;
3648 self.fence = None;
3649 self.drain_through = None;
3650 self.acknowledged_historical_through = None;
3651 }
3652 Ok(())
3653 })
3654 }
3655}
3656
3657impl<H, L, N> InterestOwnerSubscriber<N> for HybridSubscriber<H, L, N>
3658where
3659 H: InterestOwnerSubscriber<N>,
3660 L: InterestOwnerSubscriber<N>,
3661 N: Network + Send + 'static,
3662{
3663 fn replace_interest_owners(
3664 &mut self,
3665 owners: Vec<(HandlerId, Vec<ReactiveInterest<N>>)>,
3666 ) -> SubscriberOperation<'_, ()> {
3667 Box::pin(async move {
3668 self.ensure_healthy()?;
3669 self.ensure_no_pending_restore()?;
3670 reconcile_pending_owner_lifecycle(
3671 &mut self.historical,
3672 &mut self.live,
3673 &mut self.pending_live_rollback,
3674 &mut self.phase,
3675 &mut self.poisoned,
3676 )
3677 .await?;
3678 self.ensure_owner_managed_mode()?;
3679 self.ensure_reconfigurable()?;
3680 let next_owners = exact_owner_topology(
3681 &owners,
3682 "hybrid exact owner replacement contains a duplicate owner id",
3683 )?;
3684 let next_intent = lifecycle_intent(&[], &next_owners)?;
3685 if next_intent == self.lifecycle_intent {
3686 return Ok(());
3687 }
3688 let next_active = next_owners.values().any(|interests| !interests.is_empty());
3689 if next_active {
3690 self.ensure_source_chain(HybridSource::Historical)?;
3691 }
3692 if self.coverage_head.is_some() && next_active {
3693 return Err(SubscriberError::Unsupported(
3694 "hybrid exact owner replacement after canonical coverage requires replace_interest_owners_with_global_backfill so a destructive child reset cannot create an event gap",
3695 ));
3696 }
3697 let generation = self.lifecycle_generation.checked_add(1).ok_or_else(|| {
3698 SubscriberError::Provider("hybrid lifecycle generation exhausted".into())
3699 })?;
3700 let next_owner_generations = next_owners
3701 .keys()
3702 .cloned()
3703 .map(|owner| (owner, generation))
3704 .collect::<BTreeMap<_, _>>();
3705 self.preflight_owner_topology(
3706 &next_owners,
3707 &next_intent,
3708 generation,
3709 &next_owner_generations,
3710 OwnerTopologyTransition::DestructiveReset,
3711 )?;
3712 let empty_barrier = if next_active {
3713 None
3714 } else {
3715 Some(self.prepare_empty_lifecycle_barrier(
3716 generation,
3717 next_owner_generations.clone(),
3718 next_intent.clone(),
3719 )?)
3720 };
3721 let previous = LiveRollback::Topology {
3722 base: self.base_interests.clone(),
3723 owners: sorted_owner_entries(&self.owners),
3724 recovery_anchor: None,
3725 gap_anchor: self.coverage_head.filter(|_| {
3726 self.lifecycle_intent
3727 .owners
3728 .values()
3729 .any(|fingerprint| *fingerprint != empty_interest_fingerprint())
3730 }),
3731 };
3732 self.pending_live_rollback = Some(previous);
3733 self.live.replace_interest_owners(owners.clone()).await?;
3734 if next_active && let Err(chain_error) = self.ensure_source_chain(HybridSource::Live) {
3735 return match reconcile_pending_owner_lifecycle(
3736 &mut self.historical,
3737 &mut self.live,
3738 &mut self.pending_live_rollback,
3739 &mut self.phase,
3740 &mut self.poisoned,
3741 )
3742 .await
3743 {
3744 Ok(()) => Err(chain_error),
3745 Err(rollback) => Err(SubscriberError::Provider(format!(
3746 "live chain validation failed ({chain_error}); {rollback}"
3747 ))),
3748 };
3749 }
3750 if let Err(history_error) = self
3751 .historical
3752 .replace_interest_owners(owners.clone())
3753 .await
3754 {
3755 return match reconcile_pending_owner_lifecycle(
3756 &mut self.historical,
3757 &mut self.live,
3758 &mut self.pending_live_rollback,
3759 &mut self.phase,
3760 &mut self.poisoned,
3761 )
3762 .await
3763 {
3764 Ok(()) => Err(history_error),
3765 Err(rollback) => Err(SubscriberError::Provider(format!(
3766 "historical exact owner replacement failed ({history_error}); {rollback}"
3767 ))),
3768 };
3769 }
3770 self.pending_live_rollback = None;
3771 self.base_interests.clear();
3772 self.owners = next_owners;
3773 self.lifecycle_generation = generation;
3774 self.owner_generations = next_owner_generations;
3775 self.lifecycle_intent = next_intent;
3776 if let Some(prepared) = empty_barrier {
3777 self.install_empty_lifecycle_barrier(prepared);
3778 } else {
3779 self.reset_delivery_state();
3780 }
3781 Ok(())
3782 })
3783 }
3784
3785 fn replace_interest_owners_with_global_backfill(
3786 &mut self,
3787 owners: Vec<(HandlerId, Vec<ReactiveInterest<N>>)>,
3788 backfill: SubscriberBackfill,
3789 ) -> SubscriberOperation<'_, ()> {
3790 Box::pin(async move {
3791 self.ensure_healthy()?;
3792 self.ensure_no_pending_restore()?;
3793 reconcile_pending_owner_lifecycle(
3794 &mut self.historical,
3795 &mut self.live,
3796 &mut self.pending_live_rollback,
3797 &mut self.phase,
3798 &mut self.poisoned,
3799 )
3800 .await?;
3801 self.ensure_owner_managed_mode()?;
3802 self.ensure_reconfigurable()?;
3803 let next_owners = exact_owner_topology(
3804 &owners,
3805 "hybrid global owner replacement contains a duplicate owner id",
3806 )?;
3807 let next_intent = lifecycle_intent(&[], &next_owners)?;
3808 let next_active = next_owners.values().any(|interests| !interests.is_empty());
3809 if next_active {
3810 self.ensure_source_chain(HybridSource::Historical)?;
3811 }
3812 let generation = self.lifecycle_generation.checked_add(1).ok_or_else(|| {
3813 SubscriberError::Provider("hybrid lifecycle generation exhausted".into())
3814 })?;
3815 let next_owner_generations = next_owners
3816 .keys()
3817 .cloned()
3818 .map(|owner| (owner, generation))
3819 .collect::<BTreeMap<_, _>>();
3820 self.preflight_owner_topology(
3821 &next_owners,
3822 &next_intent,
3823 generation,
3824 &next_owner_generations,
3825 OwnerTopologyTransition::DestructiveReset,
3826 )?;
3827 let empty_barrier = if next_active {
3828 None
3829 } else {
3830 Some(self.prepare_empty_lifecycle_barrier(
3831 generation,
3832 next_owner_generations.clone(),
3833 next_intent.clone(),
3834 )?)
3835 };
3836 let previous = LiveRollback::Topology {
3837 base: self.base_interests.clone(),
3838 owners: sorted_owner_entries(&self.owners),
3839 recovery_anchor: self.coverage_head,
3840 gap_anchor: None,
3841 };
3842 self.pending_live_rollback = Some(previous);
3843 self.live.replace_interest_owners(owners.clone()).await?;
3844 if next_active && let Err(chain_error) = self.ensure_source_chain(HybridSource::Live) {
3845 return match reconcile_pending_owner_lifecycle(
3846 &mut self.historical,
3847 &mut self.live,
3848 &mut self.pending_live_rollback,
3849 &mut self.phase,
3850 &mut self.poisoned,
3851 )
3852 .await
3853 {
3854 Ok(()) => Err(chain_error),
3855 Err(rollback) => Err(SubscriberError::Provider(format!(
3856 "live chain validation failed ({chain_error}); {rollback}"
3857 ))),
3858 };
3859 }
3860 let history_result = if next_active {
3861 self.historical
3862 .replace_interest_owners_with_global_backfill(owners.clone(), backfill)
3863 .await
3864 } else {
3865 self.historical
3866 .replace_interest_owners(owners.clone())
3867 .await
3868 };
3869 if let Err(history_error) = history_result {
3870 return match reconcile_pending_owner_lifecycle(
3871 &mut self.historical,
3872 &mut self.live,
3873 &mut self.pending_live_rollback,
3874 &mut self.phase,
3875 &mut self.poisoned,
3876 )
3877 .await
3878 {
3879 Ok(()) => Err(history_error),
3880 Err(rollback) => Err(SubscriberError::Provider(format!(
3881 "historical global owner replacement failed ({history_error}); {rollback}"
3882 ))),
3883 };
3884 }
3885 self.pending_live_rollback = None;
3886 self.base_interests.clear();
3887 self.owners = next_owners;
3888 self.lifecycle_generation = generation;
3889 self.owner_generations = next_owner_generations;
3890 self.lifecycle_intent = next_intent;
3891 if let Some(prepared) = empty_barrier {
3892 self.install_empty_lifecycle_barrier(prepared);
3893 } else {
3894 self.reset_delivery_state();
3895 }
3896 Ok(())
3897 })
3898 }
3899
3900 fn upsert_interest_owners(
3901 &mut self,
3902 owners: Vec<(HandlerId, Vec<ReactiveInterest<N>>)>,
3903 ) -> SubscriberOperation<'_, ()> {
3904 Box::pin(async move {
3905 self.ensure_healthy()?;
3906 self.ensure_no_pending_restore()?;
3907 reconcile_pending_owner_lifecycle(
3908 &mut self.historical,
3909 &mut self.live,
3910 &mut self.pending_live_rollback,
3911 &mut self.phase,
3912 &mut self.poisoned,
3913 )
3914 .await?;
3915 self.ensure_owner_managed_mode()?;
3916 self.ensure_reconfigurable()?;
3917 if owners.is_empty() {
3918 return Ok(());
3919 }
3920 let mut unique_owners = HashSet::with_capacity(owners.len().min(1_024));
3921 if owners
3922 .iter()
3923 .any(|(owner, _)| !unique_owners.insert(owner.clone()))
3924 {
3925 return Err(SubscriberError::Provider(
3926 "hybrid bulk owner lifecycle contains a duplicate owner id".into(),
3927 ));
3928 }
3929 let mut next_owners = self.owners.clone();
3930 for (owner, interests) in &owners {
3931 next_owners.insert(owner.clone(), interests.clone());
3932 }
3933 let next_intent = lifecycle_intent(&self.base_interests, &next_owners)?;
3934 if next_intent == self.lifecycle_intent {
3935 return Ok(());
3936 }
3937 let next_active = next_owners.values().any(|interests| !interests.is_empty());
3938 if next_active {
3939 self.ensure_source_chain(HybridSource::Historical)?;
3940 }
3941 let generation = self.lifecycle_generation.checked_add(1).ok_or_else(|| {
3942 SubscriberError::Provider("hybrid lifecycle generation exhausted".into())
3943 })?;
3944 let mut next_owner_generations = self.owner_generations.clone();
3945 for (owner, _) in &owners {
3946 next_owner_generations.insert(owner.clone(), generation);
3947 }
3948 self.preflight_owner_topology(
3949 &next_owners,
3950 &next_intent,
3951 generation,
3952 &next_owner_generations,
3953 OwnerTopologyTransition::Incremental {
3954 removed_owners: Vec::new(),
3955 },
3956 )?;
3957 let empty_barrier = if next_active {
3958 None
3959 } else {
3960 Some(self.prepare_empty_lifecycle_barrier(
3961 generation,
3962 next_owner_generations.clone(),
3963 next_intent.clone(),
3964 )?)
3965 };
3966 let previous = owners
3967 .iter()
3968 .map(|(owner, _)| (owner.clone(), self.owners.get(owner).cloned()))
3969 .collect::<Vec<_>>();
3970 let gap_anchor = self.coverage_head.filter(|_| {
3971 owners.iter().any(|(owner, _)| {
3972 self.lifecycle_intent
3973 .owners
3974 .get(owner)
3975 .zip(next_intent.owners.get(owner))
3976 .is_some_and(|(current, next)| {
3977 current != next && *current != empty_interest_fingerprint()
3978 })
3979 })
3980 });
3981 self.pending_live_rollback = Some(LiveRollback::Bulk {
3982 previous: previous.clone(),
3983 gap_anchor,
3984 });
3985 self.live.upsert_interest_owners(owners.clone()).await?;
3986 if next_active && let Err(chain_error) = self.ensure_source_chain(HybridSource::Live) {
3987 return match reconcile_pending_owner_lifecycle(
3988 &mut self.historical,
3989 &mut self.live,
3990 &mut self.pending_live_rollback,
3991 &mut self.phase,
3992 &mut self.poisoned,
3993 )
3994 .await
3995 {
3996 Ok(()) => Err(chain_error),
3997 Err(rollback) => Err(SubscriberError::Provider(format!(
3998 "live chain validation failed ({chain_error}); {rollback}"
3999 ))),
4000 };
4001 }
4002 if let Err(history_error) = self.historical.upsert_interest_owners(owners.clone()).await
4003 {
4004 return match reconcile_pending_owner_lifecycle(
4005 &mut self.historical,
4006 &mut self.live,
4007 &mut self.pending_live_rollback,
4008 &mut self.phase,
4009 &mut self.poisoned,
4010 )
4011 .await
4012 {
4013 Ok(()) => Err(history_error),
4014 Err(rollback) => Err(SubscriberError::Provider(format!(
4015 "historical bulk owner registration failed ({history_error}); {rollback}"
4016 ))),
4017 };
4018 }
4019 self.pending_live_rollback = None;
4020 self.lifecycle_generation = generation;
4021 for (owner, interests) in owners {
4022 self.owner_generations.insert(owner.clone(), generation);
4023 self.owners.insert(owner, interests);
4024 }
4025 self.lifecycle_intent = next_intent;
4026 if let Some(prepared) = empty_barrier {
4027 self.install_empty_lifecycle_barrier(prepared);
4028 } else {
4029 self.begin_reconfiguration_catchup();
4030 }
4031 Ok(())
4032 })
4033 }
4034
4035 fn add_interest_owner(
4036 &mut self,
4037 owner: HandlerId,
4038 interests: &[ReactiveInterest<N>],
4039 ) -> SubscriberOperation<'_, ()> {
4040 let interests = interests.to_vec();
4041 Box::pin(async move {
4042 self.ensure_healthy()?;
4043 self.ensure_no_pending_restore()?;
4044 reconcile_pending_owner_lifecycle(
4045 &mut self.historical,
4046 &mut self.live,
4047 &mut self.pending_live_rollback,
4048 &mut self.phase,
4049 &mut self.poisoned,
4050 )
4051 .await?;
4052 self.ensure_owner_managed_mode()?;
4053 self.ensure_reconfigurable()?;
4054 let mut next_owners = self.owners.clone();
4055 next_owners.insert(owner.clone(), interests.clone());
4056 let next_intent = lifecycle_intent(&self.base_interests, &next_owners)?;
4057 if next_intent == self.lifecycle_intent {
4058 return Ok(());
4059 }
4060 let next_active = next_owners.values().any(|interests| !interests.is_empty());
4061 if next_active {
4062 self.ensure_source_chain(HybridSource::Historical)?;
4063 }
4064 let generation = self.lifecycle_generation.checked_add(1).ok_or_else(|| {
4065 SubscriberError::Provider("hybrid lifecycle generation exhausted".into())
4066 })?;
4067 let mut next_owner_generations = self.owner_generations.clone();
4068 next_owner_generations.insert(owner.clone(), generation);
4069 self.preflight_owner_topology(
4070 &next_owners,
4071 &next_intent,
4072 generation,
4073 &next_owner_generations,
4074 OwnerTopologyTransition::Incremental {
4075 removed_owners: Vec::new(),
4076 },
4077 )?;
4078 let empty_barrier = if next_active {
4079 None
4080 } else {
4081 Some(self.prepare_empty_lifecycle_barrier(
4082 generation,
4083 next_owner_generations,
4084 next_intent.clone(),
4085 )?)
4086 };
4087 let previous = self.owners.get(&owner).cloned();
4088 self.pending_live_rollback = Some(LiveRollback::Owner {
4089 owner: owner.clone(),
4090 previous: previous.clone(),
4091 gap_anchor: self.coverage_head.filter(|_| {
4092 previous
4093 .as_ref()
4094 .is_some_and(|interests| !interests.is_empty())
4095 }),
4096 });
4097 self.live
4098 .add_interest_owner(owner.clone(), &interests)
4099 .await?;
4100 if next_active && let Err(chain_error) = self.ensure_source_chain(HybridSource::Live) {
4101 return match reconcile_pending_owner_lifecycle(
4102 &mut self.historical,
4103 &mut self.live,
4104 &mut self.pending_live_rollback,
4105 &mut self.phase,
4106 &mut self.poisoned,
4107 )
4108 .await
4109 {
4110 Ok(()) => Err(chain_error),
4111 Err(rollback) => Err(SubscriberError::Provider(format!(
4112 "live chain validation failed ({chain_error}); {rollback}"
4113 ))),
4114 };
4115 }
4116 if let Err(history_error) = self
4117 .historical
4118 .add_interest_owner(owner.clone(), &interests)
4119 .await
4120 {
4121 return match reconcile_pending_owner_lifecycle(
4122 &mut self.historical,
4123 &mut self.live,
4124 &mut self.pending_live_rollback,
4125 &mut self.phase,
4126 &mut self.poisoned,
4127 )
4128 .await
4129 {
4130 Ok(()) => Err(history_error),
4131 Err(rollback) => Err(SubscriberError::Provider(format!(
4132 "historical owner registration failed ({history_error}); {rollback}"
4133 ))),
4134 };
4135 }
4136 self.pending_live_rollback = None;
4137 self.lifecycle_generation = generation;
4138 self.owner_generations
4139 .insert(owner.clone(), self.lifecycle_generation);
4140 self.owners.insert(owner, interests);
4141 self.lifecycle_intent = next_intent;
4142 if let Some(prepared) = empty_barrier {
4143 self.install_empty_lifecycle_barrier(prepared);
4144 } else {
4145 self.begin_reconfiguration_catchup();
4146 }
4147 Ok(())
4148 })
4149 }
4150
4151 fn add_interest_owner_with_backfill(
4152 &mut self,
4153 owner: HandlerId,
4154 interests: &[ReactiveInterest<N>],
4155 backfill: SubscriberBackfill,
4156 ) -> SubscriberOperation<'_, ()> {
4157 let interests = interests.to_vec();
4158 Box::pin(async move {
4159 self.ensure_healthy()?;
4160 self.ensure_no_pending_restore()?;
4161 reconcile_pending_owner_lifecycle(
4162 &mut self.historical,
4163 &mut self.live,
4164 &mut self.pending_live_rollback,
4165 &mut self.phase,
4166 &mut self.poisoned,
4167 )
4168 .await?;
4169 self.ensure_owner_managed_mode()?;
4170 self.ensure_reconfigurable()?;
4171 let mut next_owners = self.owners.clone();
4172 next_owners.insert(owner.clone(), interests.clone());
4173 let next_intent = lifecycle_intent(&self.base_interests, &next_owners)?;
4174 if next_intent == self.lifecycle_intent {
4175 return Ok(());
4176 }
4177 let next_active = next_owners.values().any(|interests| !interests.is_empty());
4178 if next_active {
4179 self.ensure_source_chain(HybridSource::Historical)?;
4180 }
4181 let generation = self.lifecycle_generation.checked_add(1).ok_or_else(|| {
4182 SubscriberError::Provider("hybrid lifecycle generation exhausted".into())
4183 })?;
4184 let mut next_owner_generations = self.owner_generations.clone();
4185 next_owner_generations.insert(owner.clone(), generation);
4186 self.preflight_owner_topology(
4187 &next_owners,
4188 &next_intent,
4189 generation,
4190 &next_owner_generations,
4191 OwnerTopologyTransition::Incremental {
4192 removed_owners: Vec::new(),
4193 },
4194 )?;
4195 let empty_barrier = if next_active {
4196 None
4197 } else {
4198 Some(self.prepare_empty_lifecycle_barrier(
4199 generation,
4200 next_owner_generations,
4201 next_intent.clone(),
4202 )?)
4203 };
4204 let previous = self.owners.get(&owner).cloned();
4205 self.pending_live_rollback = Some(LiveRollback::Owner {
4206 owner: owner.clone(),
4207 previous: previous.clone(),
4208 gap_anchor: self.coverage_head.filter(|_| {
4209 previous
4210 .as_ref()
4211 .is_some_and(|interests| !interests.is_empty())
4212 }),
4213 });
4214 self.live
4217 .add_interest_owner(owner.clone(), &interests)
4218 .await?;
4219 if next_active && let Err(chain_error) = self.ensure_source_chain(HybridSource::Live) {
4220 return match reconcile_pending_owner_lifecycle(
4221 &mut self.historical,
4222 &mut self.live,
4223 &mut self.pending_live_rollback,
4224 &mut self.phase,
4225 &mut self.poisoned,
4226 )
4227 .await
4228 {
4229 Ok(()) => Err(chain_error),
4230 Err(rollback) => Err(SubscriberError::Provider(format!(
4231 "live chain validation failed ({chain_error}); {rollback}"
4232 ))),
4233 };
4234 }
4235 let history_result = if next_active {
4236 self.historical
4237 .add_interest_owner_with_backfill(owner.clone(), &interests, backfill)
4238 .await
4239 } else {
4240 self.historical
4241 .add_interest_owner(owner.clone(), &interests)
4242 .await
4243 };
4244 if let Err(history_error) = history_result {
4245 return match reconcile_pending_owner_lifecycle(
4246 &mut self.historical,
4247 &mut self.live,
4248 &mut self.pending_live_rollback,
4249 &mut self.phase,
4250 &mut self.poisoned,
4251 )
4252 .await
4253 {
4254 Ok(()) => Err(history_error),
4255 Err(rollback) => Err(SubscriberError::Provider(format!(
4256 "historical owner registration failed ({history_error}); {rollback}"
4257 ))),
4258 };
4259 }
4260 self.pending_live_rollback = None;
4261 self.lifecycle_generation = generation;
4262 self.owner_generations
4263 .insert(owner.clone(), self.lifecycle_generation);
4264 self.owners.insert(owner, interests);
4265 self.lifecycle_intent = next_intent;
4266 if let Some(prepared) = empty_barrier {
4267 self.install_empty_lifecycle_barrier(prepared);
4268 } else {
4269 self.begin_reconfiguration_catchup();
4270 }
4271 Ok(())
4272 })
4273 }
4274
4275 fn add_interest_owner_with_canonical_catchup(
4276 &mut self,
4277 owner: HandlerId,
4278 interests: &[ReactiveInterest<N>],
4279 retained: BlockRef,
4280 ) -> SubscriberOperation<'_, ()> {
4281 let interests = interests.to_vec();
4282 Box::pin(async move {
4283 self.ensure_healthy()?;
4284 self.ensure_no_pending_restore()?;
4285 reconcile_pending_owner_lifecycle(
4286 &mut self.historical,
4287 &mut self.live,
4288 &mut self.pending_live_rollback,
4289 &mut self.phase,
4290 &mut self.poisoned,
4291 )
4292 .await?;
4293 self.ensure_owner_managed_mode()?;
4294 self.ensure_reconfigurable()?;
4295 let mut next_owners = self.owners.clone();
4296 next_owners.insert(owner.clone(), interests.clone());
4297 let next_intent = lifecycle_intent(&self.base_interests, &next_owners)?;
4298 if next_intent == self.lifecycle_intent {
4299 return Ok(());
4300 }
4301 let next_active = next_owners.values().any(|interests| !interests.is_empty());
4302 if next_active {
4303 self.ensure_source_chain(HybridSource::Historical)?;
4304 self.validate_overlap_block(&retained).map_err(|error| {
4305 SubscriberError::Provider(format!(
4306 "hybrid canonical owner activation baseline {} is not an exact retained canonical block ({error})",
4307 retained.number
4308 ))
4309 })?;
4310 }
4311 let generation = self.lifecycle_generation.checked_add(1).ok_or_else(|| {
4312 SubscriberError::Provider("hybrid lifecycle generation exhausted".into())
4313 })?;
4314 let mut next_owner_generations = self.owner_generations.clone();
4315 next_owner_generations.insert(owner.clone(), generation);
4316 self.preflight_owner_topology(
4317 &next_owners,
4318 &next_intent,
4319 generation,
4320 &next_owner_generations,
4321 OwnerTopologyTransition::Incremental {
4322 removed_owners: Vec::new(),
4323 },
4324 )?;
4325 let empty_barrier = if next_active {
4326 None
4327 } else {
4328 Some(self.prepare_empty_lifecycle_barrier(
4329 generation,
4330 next_owner_generations,
4331 next_intent.clone(),
4332 )?)
4333 };
4334 let previous = self.owners.get(&owner).cloned();
4335 self.pending_live_rollback = Some(LiveRollback::Owner {
4336 owner: owner.clone(),
4337 previous: previous.clone(),
4338 gap_anchor: self.coverage_head.filter(|_| {
4339 previous
4340 .as_ref()
4341 .is_some_and(|interests| !interests.is_empty())
4342 }),
4343 });
4344
4345 self.live
4349 .add_interest_owner(owner.clone(), &interests)
4350 .await?;
4351 if next_active && let Err(chain_error) = self.ensure_source_chain(HybridSource::Live) {
4352 return match reconcile_pending_owner_lifecycle(
4353 &mut self.historical,
4354 &mut self.live,
4355 &mut self.pending_live_rollback,
4356 &mut self.phase,
4357 &mut self.poisoned,
4358 )
4359 .await
4360 {
4361 Ok(()) => Err(chain_error),
4362 Err(rollback) => Err(SubscriberError::Provider(format!(
4363 "live chain validation failed ({chain_error}); {rollback}"
4364 ))),
4365 };
4366 }
4367 let history_result = if next_active {
4368 self.historical
4369 .add_interest_owner_with_canonical_catchup(owner.clone(), &interests, retained)
4370 .await
4371 } else {
4372 self.historical
4373 .add_interest_owner(owner.clone(), &interests)
4374 .await
4375 };
4376 if let Err(history_error) = history_result {
4377 return match reconcile_pending_owner_lifecycle(
4378 &mut self.historical,
4379 &mut self.live,
4380 &mut self.pending_live_rollback,
4381 &mut self.phase,
4382 &mut self.poisoned,
4383 )
4384 .await
4385 {
4386 Ok(()) => Err(history_error),
4387 Err(rollback) => Err(SubscriberError::Provider(format!(
4388 "historical canonical owner registration failed ({history_error}); {rollback}"
4389 ))),
4390 };
4391 }
4392
4393 self.pending_live_rollback = None;
4394 self.lifecycle_generation = generation;
4395 self.owner_generations
4396 .insert(owner.clone(), self.lifecycle_generation);
4397 self.owners.insert(owner, interests);
4398 self.lifecycle_intent = next_intent;
4399 if let Some(prepared) = empty_barrier {
4400 self.install_empty_lifecycle_barrier(prepared);
4401 } else {
4402 self.begin_reconfiguration_catchup();
4403 }
4404 Ok(())
4405 })
4406 }
4407
4408 fn remove_interest_owner(
4409 &mut self,
4410 owner: &HandlerId,
4411 ) -> SubscriberOperation<'_, Option<Vec<ReactiveInterest<N>>>> {
4412 let owner = owner.clone();
4413 Box::pin(async move {
4414 self.ensure_healthy()?;
4415 self.ensure_no_pending_restore()?;
4416 reconcile_pending_owner_lifecycle(
4417 &mut self.historical,
4418 &mut self.live,
4419 &mut self.pending_live_rollback,
4420 &mut self.phase,
4421 &mut self.poisoned,
4422 )
4423 .await?;
4424 self.ensure_owner_managed_mode()?;
4425 self.ensure_reconfigurable()?;
4426 if !self.owners.contains_key(&owner) {
4427 return Ok(None);
4428 }
4429 let mut next_owners = self.owners.clone();
4430 next_owners.remove(&owner);
4431 let next_intent = lifecycle_intent(&self.base_interests, &next_owners)?;
4432 let next_active = next_owners.values().any(|interests| !interests.is_empty());
4433 if next_active {
4434 self.ensure_source_chain(HybridSource::Historical)?;
4435 }
4436 let generation = self.lifecycle_generation.checked_add(1).ok_or_else(|| {
4437 SubscriberError::Provider("hybrid lifecycle generation exhausted".into())
4438 })?;
4439 let mut next_owner_generations = self.owner_generations.clone();
4440 next_owner_generations.remove(&owner);
4441 self.preflight_owner_topology(
4442 &next_owners,
4443 &next_intent,
4444 generation,
4445 &next_owner_generations,
4446 OwnerTopologyTransition::Incremental {
4447 removed_owners: vec![owner.clone()],
4448 },
4449 )?;
4450 let empty_barrier = if next_active {
4451 None
4452 } else {
4453 Some(self.prepare_empty_lifecycle_barrier(
4454 generation,
4455 next_owner_generations,
4456 next_intent.clone(),
4457 )?)
4458 };
4459 let previous = self.owners.get(&owner).cloned();
4460 self.pending_live_rollback = Some(LiveRollback::Owner {
4461 owner: owner.clone(),
4462 previous: previous.clone(),
4463 gap_anchor: self.coverage_head.filter(|_| {
4464 previous
4465 .as_ref()
4466 .is_some_and(|interests| !interests.is_empty())
4467 }),
4468 });
4469 self.live.remove_interest_owner(&owner).await?;
4470 if next_active && let Err(chain_error) = self.ensure_source_chain(HybridSource::Live) {
4471 return match reconcile_pending_owner_lifecycle(
4472 &mut self.historical,
4473 &mut self.live,
4474 &mut self.pending_live_rollback,
4475 &mut self.phase,
4476 &mut self.poisoned,
4477 )
4478 .await
4479 {
4480 Ok(()) => Err(chain_error),
4481 Err(rollback) => Err(SubscriberError::Provider(format!(
4482 "live chain validation failed ({chain_error}); {rollback}"
4483 ))),
4484 };
4485 }
4486 if let Err(history_error) = self.historical.remove_interest_owner(&owner).await {
4487 return match reconcile_pending_owner_lifecycle(
4488 &mut self.historical,
4489 &mut self.live,
4490 &mut self.pending_live_rollback,
4491 &mut self.phase,
4492 &mut self.poisoned,
4493 )
4494 .await
4495 {
4496 Ok(()) => Err(history_error),
4497 Err(rollback) => Err(SubscriberError::Provider(format!(
4498 "historical owner removal failed ({history_error}); {rollback}"
4499 ))),
4500 };
4501 }
4502 self.pending_live_rollback = None;
4503 let removed = self.owners.remove(&owner);
4504 self.lifecycle_generation = generation;
4505 self.owner_generations.remove(&owner);
4506 self.certified_historical = None;
4507 for coverage in self.recent_inputs.values_mut() {
4508 coverage.owners.remove(&owner);
4509 }
4510 self.lifecycle_intent = next_intent;
4511 if let Some(prepared) = empty_barrier {
4512 self.install_empty_lifecycle_barrier(prepared);
4513 }
4514 Ok(removed)
4515 })
4516 }
4517
4518 fn owner_interests(&self, owner: &HandlerId) -> Option<&[ReactiveInterest<N>]> {
4519 self.owners.get(owner).map(Vec::as_slice)
4520 }
4521}
4522
4523fn merge_audience(
4524 current: &mut DeliveryAudience,
4525 incoming: DeliveryAudience,
4526) -> Result<(), SubscriberError> {
4527 let merged = match (current.clone(), incoming) {
4528 (DeliveryAudience::All, _) | (_, DeliveryAudience::All) => DeliveryAudience::All,
4529 (DeliveryAudience::Owners(mut left), DeliveryAudience::Owners(right)) => {
4530 left.extend(right);
4531 normalize_owners(&mut left);
4532 DeliveryAudience::Owners(left)
4533 }
4534 (DeliveryAudience::AllExcept(mut excluded), DeliveryAudience::Owners(included))
4535 | (DeliveryAudience::Owners(included), DeliveryAudience::AllExcept(mut excluded)) => {
4536 let included = included.iter().collect::<HashSet<_>>();
4537 excluded.retain(|owner| !included.contains(owner));
4538 normalize_owners(&mut excluded);
4539 DeliveryAudience::AllExcept(excluded)
4540 }
4541 (DeliveryAudience::AllExcept(mut left), DeliveryAudience::AllExcept(right)) => {
4542 let right = right.iter().collect::<HashSet<_>>();
4543 left.retain(|owner| right.contains(owner));
4544 normalize_owners(&mut left);
4545 DeliveryAudience::AllExcept(left)
4546 }
4547 _ => {
4548 return Err(SubscriberError::Unsupported(
4549 "hybrid cannot merge an unknown delivery-audience variant",
4550 ));
4551 }
4552 };
4553 *current = merged;
4554 Ok(())
4555}
4556
4557fn merge_scope(
4558 current: &mut DeliveryScope,
4559 incoming: DeliveryScope,
4560) -> Result<(), SubscriberError> {
4561 *current = match (*current, incoming) {
4562 (DeliveryScope::Canonical, _) | (_, DeliveryScope::Canonical) => DeliveryScope::Canonical,
4563 (DeliveryScope::CanonicalProgress, _) | (_, DeliveryScope::CanonicalProgress) => {
4564 DeliveryScope::CanonicalProgress
4565 }
4566 (DeliveryScope::OwnerCatchup, DeliveryScope::OwnerCatchup) => DeliveryScope::OwnerCatchup,
4567 _ => {
4568 return Err(SubscriberError::Unsupported(
4569 "hybrid cannot merge an unknown delivery-scope variant",
4570 ));
4571 }
4572 };
4573 Ok(())
4574}
4575
4576fn preflight_ingress_routing<N: Network>(
4577 deliveries: &[ReactiveInputDelivery<N>],
4578 owner_generations: &BTreeMap<HandlerId, u64>,
4579 max_projected_owner_associations: usize,
4580) -> Result<(), SubscriberError> {
4581 let mut projected = 0usize;
4582 for delivery in deliveries {
4583 let audience = delivery.audience();
4584 let explicit_owners: &[HandlerId] = match audience {
4585 DeliveryAudience::Owners(owners) | DeliveryAudience::AllExcept(owners) => owners,
4586 DeliveryAudience::All => &[],
4587 _ => {
4588 return Err(SubscriberError::Unsupported(
4589 "hybrid source emitted an unknown delivery-audience variant",
4590 ));
4591 }
4592 };
4593 if explicit_owners.len() > max_projected_owner_associations {
4594 return Err(SubscriberError::Provider(format!(
4595 "hybrid child batch exceeds the explicit owner-audience ingress bound ({}/{max_projected_owner_associations})",
4596 explicit_owners.len()
4597 )));
4598 }
4599
4600 let associations = match audience {
4601 DeliveryAudience::All => owner_generations.len(),
4602 DeliveryAudience::AllExcept(excluded) => owner_generations
4603 .len()
4604 .checked_add(excluded.len())
4605 .ok_or_else(|| {
4606 SubscriberError::Provider(
4607 "hybrid child batch projected AllExcept owner work overflowed".into(),
4608 )
4609 })?,
4610 DeliveryAudience::Owners(owners) => owners.len(),
4611 _ => unreachable!("unknown audience rejected above"),
4612 };
4613 projected = projected.checked_add(associations).ok_or_else(|| {
4614 SubscriberError::Provider(
4615 "hybrid child batch projected owner associations overflowed".into(),
4616 )
4617 })?;
4618 if projected > max_projected_owner_associations {
4619 return Err(SubscriberError::Provider(format!(
4620 "hybrid child batch exceeds the projected owner associations ingress bound ({projected}/{max_projected_owner_associations})"
4621 )));
4622 }
4623
4624 validate_routing(audience, delivery.scope())?;
4625 if let Some(owner) = explicit_owners
4626 .iter()
4627 .find(|owner| !owner_generations.contains_key(*owner))
4628 {
4629 return Err(SubscriberError::Provider(format!(
4630 "hybrid source routed a delivery through unknown owner {:?}",
4631 owner.as_str()
4632 )));
4633 }
4634 }
4635 Ok(())
4636}
4637
4638fn validate_routing(
4639 audience: &DeliveryAudience,
4640 scope: DeliveryScope,
4641) -> Result<(), SubscriberError> {
4642 match audience {
4643 DeliveryAudience::All => {}
4644 DeliveryAudience::Owners(owners) => {
4645 validate_owner_list(owners, "owner audience")?;
4646 }
4647 DeliveryAudience::AllExcept(owners) => {
4648 validate_handler_ids(owners.iter())?;
4649 if has_duplicate_owners(owners) {
4650 return Err(SubscriberError::Provider(
4651 "hybrid source emitted a duplicate owner in an exclusion audience".into(),
4652 ));
4653 }
4654 }
4655 _ => {
4656 return Err(SubscriberError::Unsupported(
4657 "hybrid source emitted an unknown delivery-audience variant",
4658 ));
4659 }
4660 }
4661 match scope {
4662 DeliveryScope::Canonical | DeliveryScope::CanonicalProgress => Ok(()),
4663 DeliveryScope::OwnerCatchup if matches!(audience, DeliveryAudience::Owners(_)) => Ok(()),
4664 DeliveryScope::OwnerCatchup => Err(SubscriberError::Provider(
4665 "hybrid owner-catchup delivery requires a non-empty exact owner audience".into(),
4666 )),
4667 _ => Err(SubscriberError::Unsupported(
4668 "hybrid source emitted an unknown delivery-scope variant",
4669 )),
4670 }
4671}
4672
4673fn validate_owner_list(owners: &[HandlerId], label: &str) -> Result<(), SubscriberError> {
4674 if owners.is_empty() {
4675 return Err(SubscriberError::Provider(format!(
4676 "hybrid source emitted an empty {label}"
4677 )));
4678 }
4679 if has_duplicate_owners(owners) {
4680 return Err(SubscriberError::Provider(format!(
4681 "hybrid source emitted a duplicate owner in {label}"
4682 )));
4683 }
4684 validate_handler_ids(owners.iter())
4685}
4686
4687fn has_duplicate_owners(owners: &[HandlerId]) -> bool {
4688 let mut unique = HashSet::with_capacity(owners.len().min(1_024));
4689 owners.iter().any(|owner| !unique.insert(owner))
4690}
4691
4692fn validate_handler_ids<'a>(
4693 owners: impl IntoIterator<Item = &'a HandlerId>,
4694) -> Result<(), SubscriberError> {
4695 if let Some(owner) = owners
4696 .into_iter()
4697 .find(|owner| owner.as_str().len() > HYBRID_MAX_HANDLER_ID_BYTES)
4698 {
4699 return Err(SubscriberError::Provider(format!(
4700 "hybrid handler id {:?} exceeds the durable v5 limit of {} bytes",
4701 owner.as_str(),
4702 HYBRID_MAX_HANDLER_ID_BYTES
4703 )));
4704 }
4705 Ok(())
4706}
4707
4708fn residual_audience(
4709 coverage: &AudienceCoverage,
4710 requested: &DeliveryAudience,
4711 owner_generations: &BTreeMap<HandlerId, u64>,
4712) -> Option<DeliveryAudience> {
4713 let (base_requested, mut requested_owners, mut explicitly_excluded) = match requested {
4714 DeliveryAudience::All => (
4715 true,
4716 owner_generations.keys().cloned().collect::<Vec<_>>(),
4717 Vec::new(),
4718 ),
4719 DeliveryAudience::Owners(owners) => (false, owners.clone(), Vec::new()),
4720 DeliveryAudience::AllExcept(excluded) => {
4721 let excluded_set = excluded.iter().collect::<HashSet<_>>();
4722 (
4723 true,
4724 owner_generations
4725 .keys()
4726 .filter(|owner| !excluded_set.contains(owner))
4727 .cloned()
4728 .collect::<Vec<_>>(),
4729 excluded.clone(),
4730 )
4731 }
4732 _ => return Some(requested.clone()),
4733 };
4734 normalize_owners(&mut requested_owners);
4735 normalize_owners(&mut explicitly_excluded);
4736
4737 let mut covered = Vec::new();
4738 let mut uncovered = Vec::new();
4739 for owner in requested_owners {
4740 let generation = owner_generations.get(&owner).copied().unwrap_or(0);
4741 if coverage.owners.get(&owner) == Some(&generation) {
4742 covered.push(owner);
4743 } else {
4744 uncovered.push(owner);
4745 }
4746 }
4747
4748 if base_requested && !coverage.base {
4749 explicitly_excluded.extend(covered);
4750 normalize_owners(&mut explicitly_excluded);
4751 return Some(DeliveryAudience::AllExcept(explicitly_excluded));
4752 }
4753 if base_requested || matches!(requested, DeliveryAudience::Owners(_)) {
4754 normalize_owners(&mut uncovered);
4755 return (!uncovered.is_empty()).then_some(DeliveryAudience::Owners(uncovered));
4756 }
4757 None
4758}
4759
4760fn apply_audience_coverage(
4761 coverage: &mut AudienceCoverage,
4762 audience: &DeliveryAudience,
4763 owner_generations: &BTreeMap<HandlerId, u64>,
4764) {
4765 match audience {
4766 DeliveryAudience::All => {
4767 coverage.base = true;
4768 for (owner, generation) in owner_generations {
4769 coverage.owners.insert(owner.clone(), *generation);
4770 }
4771 }
4772 DeliveryAudience::Owners(owners) => {
4773 for owner in owners {
4774 coverage.owners.insert(
4775 owner.clone(),
4776 owner_generations.get(owner).copied().unwrap_or(0),
4777 );
4778 }
4779 }
4780 DeliveryAudience::AllExcept(excluded) => {
4781 coverage.base = true;
4782 let excluded = excluded.iter().collect::<HashSet<_>>();
4783 for (owner, generation) in owner_generations {
4784 if !excluded.contains(owner) {
4785 coverage.owners.insert(owner.clone(), *generation);
4786 }
4787 }
4788 }
4789 _ => {}
4790 }
4791}
4792
4793fn normalize_owners(owners: &mut Vec<HandlerId>) {
4794 owners.sort();
4795 owners.dedup();
4796}
4797
4798fn lifecycle_intent<N: Network>(
4799 base_interests: &[ReactiveInterest<N>],
4800 owners: &HashMap<HandlerId, Vec<ReactiveInterest<N>>>,
4801) -> Result<LifecycleIntent, SubscriberError> {
4802 validate_handler_ids(owners.keys())?;
4803 if !base_interests.is_empty() && !owners.is_empty() {
4804 return Err(SubscriberError::InvalidConfig(EXCLUSIVE_TOPOLOGY_ERROR));
4805 }
4806 let base = interest_fingerprint(base_interests)?;
4807 let owners = owners
4808 .iter()
4809 .map(|(owner, interests)| {
4810 interest_fingerprint(interests).map(|fingerprint| (owner.clone(), fingerprint))
4811 })
4812 .collect::<Result<BTreeMap<_, _>, _>>()?;
4813 Ok(LifecycleIntent { base, owners })
4814}
4815
4816fn lifecycle_intent_from_entries<N: Network>(
4817 base_interests: &[ReactiveInterest<N>],
4818 owners: &[(HandlerId, Vec<ReactiveInterest<N>>)],
4819) -> Result<LifecycleIntent, SubscriberError> {
4820 let mut unique = HashSet::with_capacity(owners.len().min(1_024));
4821 if owners
4822 .iter()
4823 .any(|(owner, _)| !unique.insert(owner.clone()))
4824 {
4825 return Err(SubscriberError::Provider(
4826 "hybrid lifecycle intent contains a duplicate owner id".into(),
4827 ));
4828 }
4829 let owners = owners.iter().cloned().collect::<HashMap<_, _>>();
4830 lifecycle_intent(base_interests, &owners)
4831}
4832
4833fn interest_fingerprint<N: Network>(
4834 interests: &[ReactiveInterest<N>],
4835) -> Result<[u8; 32], SubscriberError> {
4836 let portable = compile_portable_interests(interests).map_err(|error| {
4837 SubscriberError::Provider(format!(
4838 "hybrid durable lifecycle intent is not provider-portable: {error}"
4839 ))
4840 })?;
4841 let mut encoded = portable
4842 .into_iter()
4843 .map(|interest| {
4844 let mut hasher = CanonicalHasher::new(b"EFCHY-LIFECYCLE-INTEREST-V2");
4845 match interest.kind {
4846 Some(portable_interest::Kind::Log(log)) => {
4847 hasher.tag(1);
4848 hasher
4849 .sequence_len(log.addresses.len())
4850 .map_err(transcript_error)?;
4851 for address in log.addresses {
4852 hasher.bytes(&address).map_err(transcript_error)?;
4853 }
4854 hasher
4855 .sequence_len(log.topics.len())
4856 .map_err(transcript_error)?;
4857 for topic in log.topics {
4858 hasher
4859 .sequence_len(topic.values.len())
4860 .map_err(transcript_error)?;
4861 for value in topic.values {
4862 hasher.bytes(&value).map_err(transcript_error)?;
4863 }
4864 }
4865 }
4866 Some(portable_interest::Kind::Block(block)) => {
4867 hasher.tag(2);
4868 let mode = BlockMode::try_from(block.mode).map_err(|_| {
4869 SubscriberError::Provider(
4870 "hybrid lifecycle interest has an unknown block mode".into(),
4871 )
4872 })?;
4873 hasher.tag(match mode {
4874 BlockMode::Header => 1,
4875 BlockMode::FullBlock => 2,
4876 BlockMode::Unspecified => 0,
4877 });
4878 }
4879 None => {
4880 return Err(SubscriberError::Provider(
4881 "hybrid lifecycle interest is missing its portable kind".into(),
4882 ));
4883 }
4884 }
4885 Ok(hasher.finish())
4886 })
4887 .collect::<Result<Vec<[u8; 32]>, SubscriberError>>()?;
4888 encoded.sort();
4889 let mut hasher = CanonicalHasher::new(LIFECYCLE_FINGERPRINT_DOMAIN);
4890 hasher
4891 .sequence_len(encoded.len())
4892 .map_err(transcript_error)?;
4893 for interest in encoded {
4894 hasher.hash(&alloy_primitives::B256::from(interest));
4895 }
4896 Ok(hasher.finish())
4897}
4898
4899fn empty_interest_fingerprint() -> [u8; 32] {
4900 let mut hasher = CanonicalHasher::new(LIFECYCLE_FINGERPRINT_DOMAIN);
4901 hasher.u64(0);
4902 hasher.finish()
4903}
4904
4905fn empty_lifecycle_barrier_id(state: &HybridCheckpointV5) -> Result<Vec<u8>, SubscriberError> {
4906 let mut hasher = CanonicalHasher::new(EMPTY_LIFECYCLE_BARRIER_DOMAIN);
4907 hasher.u64(state.chain_id);
4908 hasher.bytes(&state.epoch).map_err(transcript_error)?;
4909 hasher.u64(state.lifecycle_generation);
4910 hasher.hash(&B256::from(state.lifecycle_intent.base));
4911 hasher
4912 .sequence_len(state.lifecycle_intent.owners.len())
4913 .map_err(transcript_error)?;
4914 for (owner, fingerprint) in &state.lifecycle_intent.owners {
4915 hasher.handler_id(owner).map_err(transcript_error)?;
4916 hasher.hash(&B256::from(*fingerprint));
4917 }
4918 Ok(hasher.finish().to_vec())
4919}
4920
4921fn is_committed_empty_lifecycle_barrier(state: &HybridCheckpointV5) -> bool {
4922 !state.lifecycle_intent.has_active_interests()
4923 && state.next_synthetic_token == 2
4924 && state.last_committed_token.as_ref().is_some_and(|token| {
4925 token.source == HybridSource::Live
4926 && token.kind == HybridTokenKind::Synthetic
4927 && token.inner.as_slice() == 1_u64.to_be_bytes()
4928 })
4929}
4930
4931fn validate_chain_controls(controls: &[ChainControl]) -> Result<(), SubscriberError> {
4932 let first_post_record = controls
4933 .iter()
4934 .position(|control| !matches!(control, ChainControl::Reorg { .. }))
4935 .unwrap_or(controls.len());
4936 if controls[first_post_record..]
4937 .iter()
4938 .any(|control| matches!(control, ChainControl::Reorg { .. }))
4939 {
4940 return Err(SubscriberError::Provider(
4941 "hybrid reorg controls must precede records and all post-record controls in a batch"
4942 .into(),
4943 ));
4944 }
4945 for control in controls {
4946 match control {
4947 ChainControl::Barrier { id, .. } if id.is_empty() => {
4948 return Err(SubscriberError::Provider(
4949 "hybrid source emitted an empty barrier id".into(),
4950 ));
4951 }
4952 ChainControl::Reorg {
4953 common_ancestor,
4954 old_tip,
4955 new_tip,
4956 } => {
4957 if common_ancestor.number > old_tip.number
4958 || common_ancestor.number > new_tip.number
4959 || (common_ancestor.number == old_tip.number
4960 && common_ancestor.hash != old_tip.hash)
4961 || (common_ancestor.number == new_tip.number
4962 && common_ancestor.hash != new_tip.hash)
4963 {
4964 return Err(SubscriberError::Provider(
4965 "hybrid source emitted an invalid reorg ancestry triple".into(),
4966 ));
4967 }
4968 compatible_same_height_metadata(common_ancestor, old_tip)?;
4969 compatible_same_height_metadata(common_ancestor, new_tip)?;
4970 }
4971 ChainControl::CanonicalProgress(_)
4972 | ChainControl::Safe(_)
4973 | ChainControl::Finalized(_)
4974 | ChainControl::Barrier { .. } => {}
4975 _ => {
4976 return Err(SubscriberError::Unsupported(
4977 "hybrid source emitted an unknown chain-control variant",
4978 ));
4979 }
4980 }
4981 }
4982 Ok(())
4983}
4984
4985fn compatible_same_height_metadata(
4986 left: &BlockRef,
4987 right: &BlockRef,
4988) -> Result<(), SubscriberError> {
4989 if left.number == right.number {
4990 compatible_block_ref(left, right)?;
4991 }
4992 Ok(())
4993}
4994
4995fn apply_commit_to_checkpoint(
4996 state: &mut HybridCheckpointV5,
4997 commit: &PendingCoordinatorCommit,
4998 history_capacity: usize,
4999) -> Result<(), SubscriberError> {
5000 for mutation in &commit.canonical {
5001 match mutation {
5002 CanonicalMutation::Rewind(ancestor) => {
5003 state.recent_inputs.retain(|entry| {
5004 entry
5005 .coverage
5006 .block
5007 .as_ref()
5008 .is_none_or(|block| block.number <= ancestor.number)
5009 });
5010 }
5011 CanonicalMutation::Reset => {
5012 state
5013 .recent_inputs
5014 .retain(|entry| entry.coverage.block.is_none());
5015 }
5016 CanonicalMutation::Advance(_) => {}
5017 }
5018 }
5019 let old_order = state
5020 .recent_inputs
5021 .iter()
5022 .map(|entry| entry.identity)
5023 .collect::<Vec<_>>();
5024 let mut entries = std::mem::take(&mut state.recent_inputs)
5025 .into_iter()
5026 .map(|entry| (entry.identity, entry))
5027 .collect::<HashMap<_, _>>();
5028 let mut touched = HashSet::with_capacity(commit.audiences.len());
5029 let mut refreshed = Vec::with_capacity(commit.audiences.len());
5030 for audience in &commit.audiences {
5031 let mut entry = entries
5032 .remove(&audience.identity)
5033 .or_else(|| {
5034 refreshed
5035 .iter()
5036 .position(|entry: &StoredRecentInput| entry.identity == audience.identity)
5037 .map(|index| refreshed.remove(index))
5038 })
5039 .unwrap_or_else(|| StoredRecentInput {
5040 identity: audience.identity,
5041 coverage: AudienceCoverage::default(),
5042 });
5043 let coverage = &mut entry.coverage;
5044 match coverage.witness.as_mut() {
5045 Some(stored) => merge_witness(stored, &audience.witness)?,
5046 None => coverage.witness = Some(audience.witness.clone()),
5047 }
5048 apply_audience_coverage(coverage, &audience.audience, &state.owner_generations);
5049 if audience.block.is_some() {
5050 coverage.block = match (coverage.block.as_ref(), audience.block.as_ref()) {
5051 (Some(stored), Some(incoming)) => Some(compatible_block_ref(stored, incoming)?),
5052 (None, Some(incoming)) => Some(*incoming),
5053 (Some(stored), None) => Some(*stored),
5054 (None, None) => None,
5055 };
5056 }
5057 touched.insert(audience.identity);
5058 refreshed.push(entry);
5059 }
5060 state.recent_inputs = old_order
5061 .into_iter()
5062 .filter(|identity| !touched.contains(identity))
5063 .filter_map(|identity| entries.remove(&identity))
5064 .chain(refreshed)
5065 .collect();
5066 let reorg_ancestors = commit
5067 .canonical
5068 .iter()
5069 .filter_map(|mutation| match mutation {
5070 CanonicalMutation::Rewind(ancestor) => Some(*ancestor),
5071 CanonicalMutation::Reset | CanonicalMutation::Advance(_) => None,
5072 })
5073 .collect::<Vec<_>>();
5074 if commit
5075 .canonical
5076 .iter()
5077 .any(|mutation| matches!(mutation, CanonicalMutation::Reset))
5078 || reorg_ancestors.iter().any(|ancestor| {
5079 state
5080 .certified_historical
5081 .is_some_and(|proof| proof.through.number > ancestor.number)
5082 })
5083 {
5084 state.certified_historical = None;
5085 }
5086 state.canonical_history = commit.next_canonical_history.clone();
5087 state.coverage_head = commit.next_coverage_head;
5088 state.safe_head = commit.next_safe_head;
5089 state.finalized_head = commit.next_finalized_head;
5090 let (source_position, other_position) = match commit.source {
5091 HybridSource::Historical => (&mut state.historical_position, &mut state.live_position),
5092 HybridSource::Live => (&mut state.live_position, &mut state.historical_position),
5093 };
5094 seed_reorg_ancestors(source_position, &reorg_ancestors, history_capacity)?;
5095 rewind_other_source(other_position, &reorg_ancestors, history_capacity)?;
5096 if commit
5097 .canonical
5098 .iter()
5099 .any(|mutation| matches!(mutation, CanonicalMutation::Reset))
5100 {
5101 other_position.canonical_history.clear();
5102 other_position.coverage_head = None;
5103 other_position.delivery_token = None;
5104 other_position.checkpoint = None;
5105 other_position.delivery_digest = None;
5106 }
5107 source_position.delivery_token = commit.source_token.clone();
5108 source_position.checkpoint = commit.source_checkpoint.clone();
5109 source_position.delivery_digest = Some(commit.source_delivery_digest);
5110 apply_canonical_mutations_to(
5111 &mut source_position.canonical_history,
5112 &mut source_position.coverage_head,
5113 &commit.canonical,
5114 history_capacity,
5115 )?;
5116 if let Some(progress) = commit.source_progress.as_ref()
5117 && source_position
5118 .coverage_head
5119 .as_ref()
5120 .is_none_or(|head| progress.number > head.number)
5121 {
5122 apply_canonical_mutations_to(
5123 &mut source_position.canonical_history,
5124 &mut source_position.coverage_head,
5125 &[CanonicalMutation::Advance(*progress)],
5126 history_capacity,
5127 )?;
5128 }
5129 if let Some(observed) = commit.source_observed_through.as_ref()
5130 && source_position
5131 .coverage_head
5132 .as_ref()
5133 .is_none_or(|head| observed.number >= head.number)
5134 {
5135 apply_canonical_mutations_to(
5136 &mut source_position.canonical_history,
5137 &mut source_position.coverage_head,
5138 &[CanonicalMutation::Advance(*observed)],
5139 history_capacity,
5140 )?;
5141 }
5142 if commit.source == HybridSource::Historical
5143 && let Some(through) = commit.source_observed_through
5144 {
5145 let next = CertifiedHistoricalCoverage {
5146 lifecycle_generation: state.lifecycle_generation,
5147 through,
5148 };
5149 state.certified_historical = Some(match state.certified_historical {
5156 Some(current)
5157 if current.lifecycle_generation == next.lifecycle_generation
5158 && current.through.number > next.through.number =>
5159 {
5160 current
5161 }
5162 Some(current)
5163 if current.lifecycle_generation == next.lifecycle_generation
5164 && current.through.number == next.through.number =>
5165 {
5166 CertifiedHistoricalCoverage {
5167 lifecycle_generation: current.lifecycle_generation,
5168 through: compatible_block_ref(¤t.through, &next.through)?,
5169 }
5170 }
5171 Some(_) | None => next,
5172 });
5173 }
5174 state.last_committed_token = Some(StoredCommittedToken {
5175 source: commit.source,
5176 kind: commit.token_kind,
5177 inner: commit.token_bytes.clone(),
5178 });
5179 Ok(())
5180}
5181
5182fn reserve_maximum_source_cursors(
5183 state: &mut HybridCheckpointV5,
5184 config: &HybridConfig,
5185 committing_source: HybridSource,
5186 block: BlockRef,
5187) -> Result<(), SubscriberError> {
5188 for (source, label, token_fill, checkpoint_fill, position) in [
5189 (
5190 HybridSource::Historical,
5191 "historical",
5192 0xe1,
5193 0xe2,
5194 &mut state.historical_position,
5195 ),
5196 (
5197 HybridSource::Live,
5198 "live",
5199 0x81,
5200 0x82,
5201 &mut state.live_position,
5202 ),
5203 ] {
5204 position.delivery_token = Some(capacity_probe_bytes(
5205 config.max_source_delivery_token_bytes,
5206 token_fill,
5207 label,
5208 "delivery token",
5209 )?);
5210 position.checkpoint = Some(capacity_probe_bytes(
5211 config.max_source_checkpoint_bytes,
5212 checkpoint_fill,
5213 label,
5214 "checkpoint",
5215 )?);
5216 if source != committing_source {
5217 apply_canonical_mutations_to(
5222 &mut position.canonical_history,
5223 &mut position.coverage_head,
5224 &[CanonicalMutation::Advance(block)],
5225 config.canonical_history_capacity,
5226 )?;
5227 }
5228 position.delivery_digest = Some([token_fill; 32]);
5229 }
5230 Ok(())
5231}
5232
5233fn saturated_capacity_probe_state(
5234 candidate: &HybridCheckpointV5,
5235 config: &HybridConfig,
5236) -> Result<HybridCheckpointV5, SubscriberError> {
5237 let history = saturated_capacity_probe_history(config.canonical_history_capacity)?;
5238 let head = *history.last().ok_or_else(|| {
5239 SubscriberError::Provider(
5240 "hybrid saturated capacity probe requires retained canonical history".into(),
5241 )
5242 })?;
5243 let mut state = candidate.clone();
5244
5245 state.recent_inputs.clear();
5250 state.next_synthetic_token = u64::MAX;
5251 state.canonical_history = history.clone();
5252 state.coverage_head = Some(head);
5253 state.safe_head = Some(head);
5254 state.finalized_head = Some(head);
5255 state.certified_historical = Some(CertifiedHistoricalCoverage {
5256 lifecycle_generation: state.lifecycle_generation,
5257 through: head,
5258 });
5259 state.historical_position =
5260 saturated_capacity_probe_source_position(&history, head, config, "historical", 0xe1, 0xe2)?;
5261 state.live_position =
5262 saturated_capacity_probe_source_position(&history, head, config, "live", 0x81, 0x82)?;
5263
5264 state.last_committed_token = None;
5268 Ok(state)
5269}
5270
5271fn saturated_capacity_probe_source_position(
5272 history: &[BlockRef],
5273 head: BlockRef,
5274 config: &HybridConfig,
5275 label: &str,
5276 token_fill: u8,
5277 checkpoint_fill: u8,
5278) -> Result<SourcePosition, SubscriberError> {
5279 Ok(SourcePosition {
5280 delivery_token: Some(capacity_probe_bytes(
5281 config.max_source_delivery_token_bytes,
5282 token_fill,
5283 label,
5284 "delivery token",
5285 )?),
5286 checkpoint: Some(capacity_probe_bytes(
5287 config.max_source_checkpoint_bytes,
5288 checkpoint_fill,
5289 label,
5290 "checkpoint",
5291 )?),
5292 coverage_head: Some(head),
5293 canonical_history: history.to_vec(),
5294 delivery_digest: Some([token_fill; 32]),
5295 })
5296}
5297
5298fn saturated_capacity_probe_history(capacity: usize) -> Result<Vec<BlockRef>, SubscriberError> {
5299 let capacity_u64 = u64::try_from(capacity).map_err(|_| {
5300 SubscriberError::Provider("hybrid saturated canonical-history capacity exceeds u64".into())
5301 })?;
5302 let first_number = u64::MAX.checked_sub(capacity_u64).ok_or_else(|| {
5303 SubscriberError::Provider("hybrid saturated canonical-history capacity underflowed".into())
5304 })?;
5305 let mut history = Vec::new();
5306 history.try_reserve_exact(capacity).map_err(|_| {
5307 SubscriberError::Provider(
5308 "hybrid could not allocate the saturated canonical-history capacity probe".into(),
5309 )
5310 })?;
5311 let mut parent_hash = capacity_probe_sequence_hash(0xc0, u64::MAX);
5312 for offset in 0..capacity {
5313 let offset = u64::try_from(offset).map_err(|_| {
5314 SubscriberError::Provider(
5315 "hybrid saturated canonical-history offset exceeds u64".into(),
5316 )
5317 })?;
5318 let number = first_number.checked_add(offset).ok_or_else(|| {
5319 SubscriberError::Provider("hybrid saturated canonical-history height overflowed".into())
5320 })?;
5321 let hash = capacity_probe_sequence_hash(0xc1, offset);
5322 history.push(BlockRef {
5323 number,
5324 hash,
5325 parent_hash: Some(parent_hash),
5326 timestamp: Some(u64::MAX),
5327 });
5328 parent_hash = hash;
5329 }
5330 Ok(history)
5331}
5332
5333fn capacity_probe_sequence_hash(fill: u8, sequence: u64) -> B256 {
5334 let mut bytes = [fill; 32];
5335 bytes[24..].copy_from_slice(&sequence.to_be_bytes());
5336 B256::from(bytes)
5337}
5338
5339fn capacity_probe_bytes(
5340 len: usize,
5341 fill: u8,
5342 source: &str,
5343 field: &str,
5344) -> Result<Vec<u8>, SubscriberError> {
5345 let mut bytes = Vec::new();
5346 bytes.try_reserve_exact(len).map_err(|_| {
5347 SubscriberError::Provider(format!(
5348 "hybrid could not allocate the configured {source} source {field} capacity probe"
5349 ))
5350 })?;
5351 bytes.resize(len, fill);
5352 Ok(bytes)
5353}
5354
5355fn maximum_canonical_log_commit(
5356 state: &HybridCheckpointV5,
5357 source: HybridSource,
5358 history_capacity: usize,
5359 maximum_advances: usize,
5360 max_delivery_token_bytes: usize,
5361 max_checkpoint_bytes: usize,
5362) -> Result<PendingCoordinatorCommit, SubscriberError> {
5363 let blocks = maximum_canonical_log_probe_blocks(state, maximum_advances)?;
5364 let record_block = *blocks.first().ok_or_else(|| {
5365 SubscriberError::Provider(
5366 "hybrid maximum canonical-log capacity probe produced no record block".into(),
5367 )
5368 })?;
5369 let coverage_tip = *blocks.last().ok_or_else(|| {
5370 SubscriberError::Provider(
5371 "hybrid maximum canonical-log capacity probe produced no coverage tip".into(),
5372 )
5373 })?;
5374 let canonical = blocks
5375 .iter()
5376 .copied()
5377 .map(CanonicalMutation::Advance)
5378 .collect::<Vec<_>>();
5379 maximum_canonical_log_commit_for_transition(
5380 state,
5381 source,
5382 canonical,
5383 record_block,
5384 coverage_tip,
5385 CapacityProbeLimits {
5386 history_capacity,
5387 max_delivery_token_bytes,
5388 max_checkpoint_bytes,
5389 },
5390 )
5391}
5392
5393fn maximum_canonical_log_commit_for_transition(
5394 state: &HybridCheckpointV5,
5395 source: HybridSource,
5396 canonical: Vec<CanonicalMutation>,
5397 record_block: BlockRef,
5398 coverage_tip: BlockRef,
5399 limits: CapacityProbeLimits,
5400) -> Result<PendingCoordinatorCommit, SubscriberError> {
5401 let CapacityProbeLimits {
5402 history_capacity,
5403 max_delivery_token_bytes,
5404 max_checkpoint_bytes,
5405 } = limits;
5406 let mut next_canonical_history = state.canonical_history.clone();
5407 let mut next_coverage_head = state.coverage_head;
5408 apply_canonical_mutations_to(
5409 &mut next_canonical_history,
5410 &mut next_coverage_head,
5411 &canonical,
5412 history_capacity,
5413 )?;
5414
5415 let existing = state
5416 .recent_inputs
5417 .iter()
5418 .map(|entry| entry.identity)
5419 .collect::<HashSet<_>>();
5420 let identity = (0..=existing.len())
5421 .find_map(|sequence| {
5422 let mut transaction_hash = [0xff; 32];
5423 transaction_hash[24..].copy_from_slice(&(sequence as u64).to_be_bytes());
5424 ReactiveInputIdentity::try_from_parts(
5425 InputRef::Log {
5426 chain_id: Some(state.chain_id),
5427 block_hash: record_block.hash,
5428 transaction_hash: B256::from(transaction_hash),
5429 log_index: u64::MAX,
5430 },
5431 ReactiveInputKind::CanonicalLog,
5432 )
5433 .ok()
5434 .filter(|identity| !existing.contains(identity))
5435 })
5436 .ok_or_else(|| {
5437 SubscriberError::Provider(
5438 "hybrid could not construct a unique durable capacity-probe identity".into(),
5439 )
5440 })?;
5441
5442 let audience = if state.lifecycle_intent.base != empty_interest_fingerprint() {
5443 DeliveryAudience::All
5444 } else {
5445 DeliveryAudience::Owners(state.owner_generations.keys().cloned().collect())
5446 };
5447 let source_label = match source {
5448 HybridSource::Historical => "historical",
5449 HybridSource::Live => "live",
5450 };
5451 let source_token = capacity_probe_bytes(
5452 max_delivery_token_bytes,
5453 match source {
5454 HybridSource::Historical => 0x91,
5455 HybridSource::Live => 0xa1,
5456 },
5457 source_label,
5458 "committing delivery token",
5459 )?;
5460 let source_checkpoint = capacity_probe_bytes(
5461 max_checkpoint_bytes,
5462 match source {
5463 HybridSource::Historical => 0x92,
5464 HybridSource::Live => 0xa2,
5465 },
5466 source_label,
5467 "committing checkpoint",
5468 )?;
5469 Ok(PendingCoordinatorCommit {
5470 audiences: vec![AudienceCommit {
5471 identity,
5472 audience,
5473 block: Some(record_block),
5474 witness: RecordWitness {
5475 payload_digest: [0xff; 32],
5476 chain_id: state.chain_id,
5477 lifecycle: WitnessLifecycle::Finalized,
5478 block: Some(record_block),
5479 transaction_index: Some(u64::MAX),
5480 log_index: Some(u64::MAX),
5481 log_block_timestamp: Some(u64::MAX),
5482 },
5483 }],
5484 canonical,
5485 source,
5486 source_token: Some(source_token.clone()),
5487 source_checkpoint: Some(source_checkpoint),
5488 source_progress: Some(coverage_tip),
5489 source_observed_through: Some(coverage_tip),
5490 token_kind: HybridTokenKind::Forwarded,
5491 token_bytes: source_token,
5492 source_delivery_digest: [0xfe; 32],
5493 next_safe_head: Some(coverage_tip),
5494 next_finalized_head: Some(coverage_tip),
5495 next_canonical_history,
5496 next_coverage_head,
5497 })
5498}
5499
5500#[cfg(test)]
5501fn maximum_canonical_log_probe_block(
5502 state: &HybridCheckpointV5,
5503) -> Result<BlockRef, SubscriberError> {
5504 maximum_canonical_log_probe_blocks(state, 1)?
5505 .pop()
5506 .ok_or_else(|| {
5507 SubscriberError::Provider(
5508 "hybrid maximum canonical-log capacity probe produced no block".into(),
5509 )
5510 })
5511}
5512
5513fn maximum_canonical_log_probe_blocks(
5514 state: &HybridCheckpointV5,
5515 maximum_advances: usize,
5516) -> Result<Vec<BlockRef>, SubscriberError> {
5517 if maximum_advances == 0 {
5518 return Err(SubscriberError::Provider(
5519 "hybrid maximum canonical-log capacity probe requires at least one advance".into(),
5520 ));
5521 }
5522 let mut known_hashes = capacity_probe_known_block_hashes(state)?;
5523 match state.coverage_head {
5524 Some(head) if head.number < u64::MAX => {
5525 let available = u64::MAX - head.number;
5526 let advance_count =
5527 maximum_advances.min(usize::try_from(available).unwrap_or(usize::MAX).max(1));
5528 maximum_canonical_log_probe_suffix(&mut known_hashes, Some(head), advance_count)
5529 }
5530 Some(head) => {
5531 let mut merged = head;
5532 for candidate in state
5533 .canonical_history
5534 .iter()
5535 .chain(state.historical_position.canonical_history.iter())
5536 .chain(state.live_position.canonical_history.iter())
5537 .chain(state.historical_position.coverage_head.iter())
5538 .chain(state.live_position.coverage_head.iter())
5539 .chain(state.safe_head.iter())
5540 .chain(state.finalized_head.iter())
5541 .chain(
5542 state
5543 .certified_historical
5544 .iter()
5545 .map(|proof| &proof.through),
5546 )
5547 .chain(
5548 state
5549 .recent_inputs
5550 .iter()
5551 .filter_map(|entry| entry.coverage.block.as_ref()),
5552 )
5553 .chain(
5554 state
5555 .recent_inputs
5556 .iter()
5557 .filter_map(|entry| entry.coverage.witness.as_ref()?.block.as_ref()),
5558 )
5559 .filter(|candidate| candidate.number == head.number)
5560 {
5561 merged = compatible_block_ref(&merged, candidate)?;
5562 }
5563 let predecessor = state
5564 .canonical_history
5565 .iter()
5566 .chain(state.historical_position.canonical_history.iter())
5567 .chain(state.live_position.canonical_history.iter())
5568 .rev()
5569 .find(|block| block.number.checked_add(1) == Some(head.number))
5570 .map(|block| block.hash);
5571 Ok(vec![BlockRef {
5572 parent_hash: Some(
5573 merged
5574 .parent_hash
5575 .or(predecessor)
5576 .map_or_else(|| unique_capacity_probe_hash(&known_hashes, 0xd2), Ok)?,
5577 ),
5578 timestamp: merged.timestamp.or(Some(u64::MAX)),
5579 ..merged
5580 }])
5581 }
5582 None => maximum_canonical_log_probe_suffix(&mut known_hashes, None, maximum_advances),
5583 }
5584}
5585
5586fn maximum_canonical_log_probe_suffix(
5587 known_hashes: &mut HashSet<B256>,
5588 prior_head: Option<BlockRef>,
5589 advance_count: usize,
5590) -> Result<Vec<BlockRef>, SubscriberError> {
5591 let count_minus_one = advance_count.checked_sub(1).ok_or_else(|| {
5592 SubscriberError::Provider(
5593 "hybrid maximum canonical-log capacity probe requires a non-empty suffix".into(),
5594 )
5595 })?;
5596 let count_minus_one = u64::try_from(count_minus_one).map_err(|_| {
5597 SubscriberError::Provider(
5598 "hybrid maximum canonical-log capacity probe suffix exceeds u64".into(),
5599 )
5600 })?;
5601 let first_number = u64::MAX.checked_sub(count_minus_one).ok_or_else(|| {
5602 SubscriberError::Provider(
5603 "hybrid maximum canonical-log capacity probe suffix underflowed".into(),
5604 )
5605 })?;
5606 let mut parent_hash = match prior_head {
5607 Some(head) if head.number.checked_add(1) == Some(first_number) => head.hash,
5608 Some(_) | None => {
5609 let parent = unique_capacity_probe_hash(known_hashes, 0xd2)?;
5610 known_hashes.insert(parent);
5611 parent
5612 }
5613 };
5614 let mut blocks = Vec::new();
5615 blocks.try_reserve_exact(advance_count).map_err(|_| {
5616 SubscriberError::Provider(
5617 "hybrid could not allocate the maximum canonical-log history probe".into(),
5618 )
5619 })?;
5620 for offset in 0..advance_count {
5621 let number = first_number
5622 .checked_add(u64::try_from(offset).map_err(|_| {
5623 SubscriberError::Provider(
5624 "hybrid maximum canonical-log capacity probe offset exceeds u64".into(),
5625 )
5626 })?)
5627 .ok_or_else(|| {
5628 SubscriberError::Provider(
5629 "hybrid maximum canonical-log capacity probe height overflowed".into(),
5630 )
5631 })?;
5632 let hash = unique_capacity_probe_hash(known_hashes, 0xd1)?;
5633 known_hashes.insert(hash);
5634 blocks.push(BlockRef {
5635 number,
5636 hash,
5637 parent_hash: Some(parent_hash),
5638 timestamp: Some(u64::MAX),
5639 });
5640 parent_hash = hash;
5641 }
5642 Ok(blocks)
5643}
5644
5645fn capacity_probe_state_blocks(state: &HybridCheckpointV5) -> impl Iterator<Item = &BlockRef> {
5646 state
5647 .canonical_history
5648 .iter()
5649 .chain(state.historical_position.canonical_history.iter())
5650 .chain(state.live_position.canonical_history.iter())
5651 .chain(state.coverage_head.iter())
5652 .chain(state.historical_position.coverage_head.iter())
5653 .chain(state.live_position.coverage_head.iter())
5654 .chain(state.safe_head.iter())
5655 .chain(state.finalized_head.iter())
5656 .chain(
5657 state
5658 .certified_historical
5659 .iter()
5660 .map(|proof| &proof.through),
5661 )
5662 .chain(
5663 state
5664 .recent_inputs
5665 .iter()
5666 .filter_map(|entry| entry.coverage.block.as_ref()),
5667 )
5668 .chain(
5669 state
5670 .recent_inputs
5671 .iter()
5672 .filter_map(|entry| entry.coverage.witness.as_ref()?.block.as_ref()),
5673 )
5674}
5675
5676fn capacity_probe_known_block_hashes(
5677 state: &HybridCheckpointV5,
5678) -> Result<HashSet<B256>, SubscriberError> {
5679 let expected = state
5680 .canonical_history
5681 .len()
5682 .saturating_add(state.historical_position.canonical_history.len())
5683 .saturating_add(state.live_position.canonical_history.len())
5684 .saturating_add(state.recent_inputs.len().saturating_mul(3))
5685 .saturating_add(8);
5686 let mut known = HashSet::new();
5687 known.try_reserve(expected).map_err(|_| {
5688 SubscriberError::Provider(
5689 "hybrid could not allocate the maximum canonical-log block-hash probe".into(),
5690 )
5691 })?;
5692 for block in capacity_probe_state_blocks(state) {
5693 known.insert(block.hash);
5694 }
5695 for entry in &state.recent_inputs {
5696 match entry.identity.input_ref() {
5697 InputRef::Log { block_hash, .. }
5698 | InputRef::Block {
5699 hash: block_hash, ..
5700 } => {
5701 known.insert(block_hash);
5702 }
5703 InputRef::PendingTx { .. } => {}
5704 }
5705 }
5706 Ok(known)
5707}
5708
5709fn unique_capacity_probe_hash(known: &HashSet<B256>, fill: u8) -> Result<B256, SubscriberError> {
5710 (0..=known.len())
5711 .map(|sequence| {
5712 let mut bytes = [fill; 32];
5713 bytes[24..].copy_from_slice(&(sequence as u64).to_be_bytes());
5714 B256::from(bytes)
5715 })
5716 .find(|candidate| !known.contains(candidate))
5717 .ok_or_else(|| {
5718 SubscriberError::Provider(
5719 "hybrid could not construct a unique maximum canonical-log block hash".into(),
5720 )
5721 })
5722}
5723
5724#[cfg(test)]
5725fn append_legacy_pending_capacity_probe(
5726 state: &mut HybridCheckpointV5,
5727 chain_id: u64,
5728 base: bool,
5729 owners: BTreeMap<HandlerId, u64>,
5730) -> Result<(), SubscriberError> {
5731 let existing = state
5732 .recent_inputs
5733 .iter()
5734 .map(|entry| entry.identity)
5735 .collect::<HashSet<_>>();
5736 let identity = (0..=existing.len())
5737 .find_map(|sequence| {
5738 let mut hash = [0x52; 32];
5739 hash[24..].copy_from_slice(&(sequence as u64).to_be_bytes());
5740 ReactiveInputIdentity::try_from_parts(
5741 InputRef::PendingTx {
5742 chain_id: Some(chain_id),
5743 hash: B256::from(hash),
5744 },
5745 ReactiveInputKind::PendingTxHash,
5746 )
5747 .ok()
5748 .filter(|identity| !existing.contains(identity))
5749 })
5750 .ok_or_else(|| {
5751 SubscriberError::Provider(
5752 "hybrid could not construct a unique legacy capacity-probe identity".into(),
5753 )
5754 })?;
5755 state.recent_inputs.push(StoredRecentInput {
5756 identity,
5757 coverage: AudienceCoverage {
5758 base,
5759 owners,
5760 block: None,
5761 witness: Some(RecordWitness {
5762 payload_digest: [0x52; 32],
5763 chain_id,
5764 lifecycle: WitnessLifecycle::Pending,
5765 block: None,
5766 transaction_index: None,
5767 log_index: None,
5768 log_block_timestamp: None,
5769 }),
5770 },
5771 });
5772 Ok(())
5773}
5774
5775fn fit_checkpoint_to_durable_limits(
5776 state: &mut HybridCheckpointV5,
5777 protected_recent_inputs: usize,
5778 recent_input_capacity: usize,
5779 max_recent_owner_entries: usize,
5780) -> Result<(), SubscriberError> {
5781 if protected_recent_inputs > state.recent_inputs.len() {
5782 return Err(SubscriberError::Provider(
5783 "hybrid current delivery has an invalid protected witness suffix".into(),
5784 ));
5785 }
5786 let max_drop = state.recent_inputs.len() - protected_recent_inputs;
5787 let mut required_drop = state
5788 .recent_inputs
5789 .len()
5790 .saturating_sub(recent_input_capacity);
5791 let mut retained_owner_entries = recent_owner_entry_count(&state.recent_inputs)?;
5792 for entry in state.recent_inputs.iter().take(required_drop) {
5793 retained_owner_entries = retained_owner_entries
5794 .checked_sub(entry.coverage.owners.len())
5795 .ok_or_else(|| {
5796 SubscriberError::Provider("hybrid recent owner-entry accounting underflowed".into())
5797 })?;
5798 }
5799 while retained_owner_entries > max_recent_owner_entries {
5800 let Some(entry) = state.recent_inputs.get(required_drop) else {
5801 break;
5802 };
5803 retained_owner_entries = retained_owner_entries
5804 .checked_sub(entry.coverage.owners.len())
5805 .ok_or_else(|| {
5806 SubscriberError::Provider("hybrid recent owner-entry accounting underflowed".into())
5807 })?;
5808 required_drop += 1;
5809 }
5810 if required_drop > max_drop {
5811 return Err(SubscriberError::Provider(format!(
5812 "hybrid current delivery needs {} protected recent witnesses and {retained_owner_entries} owner associations, exceeding the configured durable journal budgets",
5813 protected_recent_inputs
5814 )));
5815 }
5816
5817 let fits = |dropped: usize| -> Result<bool, SubscriberError> {
5818 if codec::validate_hybrid_checkpoint_v5_limits_after_dropping_recent(state, dropped)
5819 .is_err()
5820 {
5821 return Ok(false);
5822 }
5823 Ok(
5824 codec::checkpoint_payload_len_after_dropping_recent(state, dropped)?
5825 <= codec::MAX_CHECKPOINT_PAYLOAD_BYTES,
5826 )
5827 };
5828
5829 if !fits(max_drop)? {
5830 codec::validate_hybrid_checkpoint_v5_limits_after_dropping_recent(state, max_drop)?;
5833 let minimal_payload = codec::checkpoint_payload_len_after_dropping_recent(state, max_drop)?;
5834 return Err(SubscriberError::Provider(format!(
5835 "hybrid current delivery cannot fit its protected witness suffix in the {}-byte durable checkpoint payload (needs at least {minimal_payload} bytes)",
5836 codec::MAX_CHECKPOINT_PAYLOAD_BYTES
5837 )));
5838 }
5839
5840 let mut low = required_drop;
5841 let mut high = max_drop;
5842 while low < high {
5843 let middle = low + (high - low) / 2;
5844 if fits(middle)? {
5845 high = middle;
5846 } else {
5847 low = middle + 1;
5848 }
5849 }
5850 if low != 0 {
5851 state.recent_inputs.drain(..low);
5852 }
5853 codec::validate_hybrid_checkpoint_v5_state(state)?;
5854 Ok(())
5855}
5856
5857fn recent_owner_entry_count(entries: &[StoredRecentInput]) -> Result<usize, SubscriberError> {
5858 entries.iter().try_fold(0usize, |total, entry| {
5859 total
5860 .checked_add(entry.coverage.owners.len())
5861 .ok_or_else(|| {
5862 SubscriberError::Provider("hybrid recent owner-entry accounting overflowed".into())
5863 })
5864 })
5865}
5866
5867fn seed_reorg_ancestors(
5868 source: &mut SourcePosition,
5869 ancestors: &[BlockRef],
5870 capacity: usize,
5871) -> Result<(), SubscriberError> {
5872 for ancestor in ancestors {
5873 if source
5874 .coverage_head
5875 .as_ref()
5876 .is_some_and(|head| head.number > ancestor.number)
5877 && !source
5878 .canonical_history
5879 .iter()
5880 .any(|known| known.number == ancestor.number)
5881 {
5882 source.canonical_history.push(*ancestor);
5883 source.canonical_history.sort_by_key(|block| block.number);
5884 truncate_front_to(&mut source.canonical_history, capacity);
5885 }
5886 if let Some(known) = source
5887 .canonical_history
5888 .iter()
5889 .find(|known| known.number == ancestor.number)
5890 {
5891 compatible_block_ref(known, ancestor)?;
5892 }
5893 }
5894 Ok(())
5895}
5896
5897fn rewind_other_source(
5898 source: &mut SourcePosition,
5899 ancestors: &[BlockRef],
5900 capacity: usize,
5901) -> Result<(), SubscriberError> {
5902 for ancestor in ancestors {
5903 let Some(head_number) = source.coverage_head.as_ref().map(|head| head.number) else {
5904 continue;
5905 };
5906 if head_number < ancestor.number {
5907 continue;
5908 }
5909 seed_reorg_ancestors(source, std::slice::from_ref(ancestor), capacity)?;
5910 apply_canonical_mutations_to(
5911 &mut source.canonical_history,
5912 &mut source.coverage_head,
5913 &[CanonicalMutation::Rewind(*ancestor)],
5914 capacity,
5915 )?;
5916 if head_number > ancestor.number {
5917 source.delivery_token = None;
5918 source.checkpoint = None;
5919 source.delivery_digest = None;
5920 }
5921 }
5922 Ok(())
5923}
5924
5925fn apply_canonical_mutations_to(
5926 history: &mut Vec<BlockRef>,
5927 coverage_head: &mut Option<BlockRef>,
5928 mutations: &[CanonicalMutation],
5929 capacity: usize,
5930) -> Result<(), SubscriberError> {
5931 for mutation in mutations {
5932 match mutation {
5933 CanonicalMutation::Reset => {
5934 history.clear();
5935 *coverage_head = None;
5936 }
5937 CanonicalMutation::Rewind(ancestor) => {
5938 let known_ancestor = history.iter().find(|block| block.number == ancestor.number);
5939 if coverage_head.is_some()
5940 && known_ancestor.is_none()
5941 && history
5942 .first()
5943 .is_none_or(|oldest| oldest.number > ancestor.number)
5944 {
5945 return Err(SubscriberError::Provider(format!(
5946 "hybrid reorg ancestor {} is older than the retained canonical effect window",
5947 ancestor.number
5948 )));
5949 }
5950 if let Some(known) = known_ancestor
5951 && compatible_block_ref(known, ancestor).is_err()
5952 {
5953 return Err(SubscriberError::Provider(format!(
5954 "hybrid reorg ancestor hash or metadata mismatch at block {}",
5955 ancestor.number
5956 )));
5957 }
5958 if let Some(head) = coverage_head.as_ref()
5959 && ancestor.number > head.number
5960 {
5961 return Err(SubscriberError::Provider(
5962 "hybrid reorg ancestor is ahead of committed coverage".into(),
5963 ));
5964 }
5965 history.retain(|block| block.number <= ancestor.number);
5966 if let Some(known) = history
5967 .last_mut()
5968 .filter(|block| block.number == ancestor.number)
5969 {
5970 *known = compatible_block_ref(known, ancestor)?;
5971 } else {
5972 history.push(*ancestor);
5973 }
5974 *coverage_head = Some(match coverage_head.as_ref() {
5975 Some(head) if head.number == ancestor.number => {
5976 compatible_block_ref(head, ancestor)?
5977 }
5978 _ => *ancestor,
5979 });
5980 }
5981 CanonicalMutation::Advance(block) => {
5982 if let Some(head) = coverage_head.as_ref() {
5983 if block.number < head.number {
5984 let known = history
5985 .iter_mut()
5986 .find(|known| known.number == block.number)
5987 .ok_or_else(|| {
5988 SubscriberError::Provider(format!(
5989 "hybrid canonical block {} is outside retained history",
5990 block.number
5991 ))
5992 })?;
5993 *known = compatible_block_ref(known, block)?;
5994 continue;
5995 }
5996 if block.number == head.number {
5997 let merged = compatible_block_ref(head, block)?;
5998 *coverage_head = Some(merged);
5999 if let Some(known) = history
6000 .iter_mut()
6001 .find(|known| known.number == block.number)
6002 {
6003 *known = compatible_block_ref(known, &merged)?;
6004 }
6005 continue;
6006 }
6007 if block.number == head.number.saturating_add(1)
6008 && block.parent_hash.is_some_and(|parent| parent != head.hash)
6009 {
6010 return Err(SubscriberError::Provider(format!(
6011 "hybrid canonical parent mismatch at block {}",
6012 block.number
6013 )));
6014 }
6015 }
6016 if let Some(known) = history
6017 .iter_mut()
6018 .find(|known| known.number == block.number)
6019 {
6020 *known = compatible_block_ref(known, block)?;
6021 } else {
6022 history.push(*block);
6023 history.sort_by_key(|known| known.number);
6024 }
6025 *coverage_head = Some(match coverage_head.as_ref() {
6026 Some(head) if head.number == block.number => compatible_block_ref(head, block)?,
6027 _ => *block,
6028 });
6029 }
6030 }
6031 if history.len() > capacity {
6032 let excess = history.len() - capacity;
6033 history.drain(..excess);
6034 }
6035 }
6036 Ok(())
6037}
6038
6039fn encode_hybrid_checkpoint(state: &HybridCheckpointV5) -> Result<Vec<u8>, SubscriberError> {
6040 codec::encode_hybrid_checkpoint_v5(state)
6041}
6042
6043#[cfg(test)]
6044fn encode_hybrid_checkpoint_with_limit(
6045 state: &HybridCheckpointV5,
6046 payload_limit: usize,
6047) -> Result<Vec<u8>, SubscriberError> {
6048 codec::encode_hybrid_checkpoint_v5_with_limit(state, payload_limit)
6049}
6050
6051fn decode_hybrid_checkpoint(bytes: &[u8]) -> Result<HybridCheckpointV5, SubscriberError> {
6052 codec::decode_hybrid_checkpoint_v5(bytes)
6053}
6054
6055fn validate_checkpoint_state(state: &HybridCheckpointV5) -> Result<(), SubscriberError> {
6056 if state.next_synthetic_token == 0 {
6057 return Err(SubscriberError::Provider(
6058 "hybrid checkpoint synthetic token sequence must be non-zero".into(),
6059 ));
6060 }
6061 if state.lifecycle_generation == 0 {
6062 return Err(SubscriberError::Provider(
6063 "hybrid checkpoint lifecycle generation must be non-zero".into(),
6064 ));
6065 }
6066 if !state.lifecycle_intent.owners.is_empty()
6067 && state.lifecycle_intent.base != empty_interest_fingerprint()
6068 {
6069 return Err(SubscriberError::Provider(
6070 "hybrid checkpoint combines base/unowned and owner-managed interests; mixed topology cannot be restored atomically"
6071 .into(),
6072 ));
6073 }
6074 if state
6075 .owner_generations
6076 .values()
6077 .any(|generation| *generation == 0 || *generation > state.lifecycle_generation)
6078 {
6079 return Err(SubscriberError::Provider(
6080 "hybrid checkpoint contains an invalid owner generation".into(),
6081 ));
6082 }
6083 if state
6084 .owner_generations
6085 .keys()
6086 .ne(state.lifecycle_intent.owners.keys())
6087 {
6088 return Err(SubscriberError::Provider(
6089 "hybrid checkpoint owner generations do not match its lifecycle intent".into(),
6090 ));
6091 }
6092 let mut inputs = HashSet::with_capacity(state.recent_inputs.len().min(65_536));
6093 if state
6094 .recent_inputs
6095 .iter()
6096 .any(|entry| !inputs.insert(entry.identity))
6097 {
6098 return Err(SubscriberError::Provider(
6099 "hybrid checkpoint contains duplicate recent input identities".into(),
6100 ));
6101 }
6102 for entry in &state.recent_inputs {
6103 let chain_id = input_ref_chain_id(entry.identity.input_ref());
6104 if chain_id != Some(state.chain_id) {
6105 return Err(SubscriberError::Provider(
6106 "hybrid checkpoint contains a recent identity for the wrong or missing chain"
6107 .into(),
6108 ));
6109 }
6110 let witness = entry.coverage.witness.as_ref().ok_or_else(|| {
6111 SubscriberError::Provider(
6112 "hybrid checkpoint recent-input coverage is missing its payload witness".into(),
6113 )
6114 })?;
6115 if witness.chain_id != state.chain_id {
6116 return Err(SubscriberError::Provider(
6117 "hybrid checkpoint payload witness targets the wrong chain".into(),
6118 ));
6119 }
6120 if entry.coverage.owners.iter().any(|(owner, generation)| {
6121 *generation == 0
6122 || *generation > state.lifecycle_generation
6123 || state
6124 .owner_generations
6125 .get(owner)
6126 .is_none_or(|current| generation > current)
6127 }) {
6128 return Err(SubscriberError::Provider(
6129 "hybrid checkpoint recent-input coverage has an invalid owner generation".into(),
6130 ));
6131 }
6132 match entry.identity.kind() {
6133 ReactiveInputKind::CanonicalLog | ReactiveInputKind::BlockHeader => {
6134 if !matches!(
6135 witness.lifecycle,
6136 WitnessLifecycle::Included
6137 | WitnessLifecycle::Safe
6138 | WitnessLifecycle::Finalized
6139 ) || witness.block.is_none()
6140 {
6141 return Err(SubscriberError::Provider(
6142 "hybrid checkpoint canonical input witness has an invalid lifecycle".into(),
6143 ));
6144 }
6145 }
6146 ReactiveInputKind::ReorgSignalLog => {
6147 if witness.lifecycle != WitnessLifecycle::Reorg {
6148 return Err(SubscriberError::Provider(
6149 "hybrid checkpoint reorg input witness has an invalid lifecycle".into(),
6150 ));
6151 }
6152 }
6153 ReactiveInputKind::PendingTxHash => {
6154 if witness.lifecycle != WitnessLifecycle::Pending || witness.block.is_some() {
6155 return Err(SubscriberError::Provider(
6156 "hybrid checkpoint pending input witness has an invalid lifecycle".into(),
6157 ));
6158 }
6159 }
6160 ReactiveInputKind::FullBlock | ReactiveInputKind::PendingTx => {
6161 return Err(SubscriberError::Provider(
6162 "hybrid checkpoint contains a representation without a complete payload proof"
6163 .into(),
6164 ));
6165 }
6166 _ => {
6167 return Err(SubscriberError::Provider(
6168 "hybrid checkpoint contains an unknown input representation".into(),
6169 ));
6170 }
6171 }
6172 if let (Some(coverage_block), Some(witness_block)) =
6173 (entry.coverage.block.as_ref(), witness.block.as_ref())
6174 {
6175 compatible_block_ref(coverage_block, witness_block)?;
6176 }
6177 match entry.identity.input_ref() {
6178 InputRef::Log { block_hash, .. } => {
6179 if witness
6180 .block
6181 .as_ref()
6182 .is_some_and(|block| block.hash != block_hash)
6183 {
6184 return Err(SubscriberError::Provider(
6185 "hybrid checkpoint log identity conflicts with its witness block".into(),
6186 ));
6187 }
6188 }
6189 InputRef::Block { hash, number, .. } => {
6190 if witness
6191 .block
6192 .as_ref()
6193 .is_none_or(|block| block.hash != hash || block.number != number)
6194 {
6195 return Err(SubscriberError::Provider(
6196 "hybrid checkpoint block identity conflicts with its witness".into(),
6197 ));
6198 }
6199 }
6200 InputRef::PendingTx { .. } => {}
6201 }
6202 if let Some(block) = entry.coverage.block.as_ref() {
6203 if state
6204 .coverage_head
6205 .as_ref()
6206 .is_none_or(|head| block.number > head.number)
6207 {
6208 return Err(SubscriberError::Provider(
6209 "hybrid checkpoint recent-input coverage is ahead of canonical coverage".into(),
6210 ));
6211 }
6212 if let Some(known) = state
6213 .canonical_history
6214 .iter()
6215 .find(|known| known.number == block.number)
6216 {
6217 compatible_block_ref(known, block)?;
6218 }
6219 }
6220 }
6221 validate_checkpoint_history(
6222 "coordinator",
6223 &state.canonical_history,
6224 state.coverage_head.as_ref(),
6225 )?;
6226 validate_checkpoint_history(
6227 "historical source",
6228 &state.historical_position.canonical_history,
6229 state.historical_position.coverage_head.as_ref(),
6230 )?;
6231 validate_checkpoint_history(
6232 "live source",
6233 &state.live_position.canonical_history,
6234 state.live_position.coverage_head.as_ref(),
6235 )?;
6236 CanonicalSequenceState::new(
6237 state.canonical_history.clone(),
6238 state.coverage_head,
6239 state.safe_head,
6240 state.finalized_head,
6241 )
6242 .validate()
6243 .map_err(|error| {
6244 SubscriberError::Provider(format!(
6245 "hybrid checkpoint canonical sequence is invalid: {error}"
6246 ))
6247 })?;
6248 if let Some(proof) = state.certified_historical {
6249 if proof.lifecycle_generation != state.lifecycle_generation {
6250 return Err(SubscriberError::Provider(
6251 "hybrid checkpoint historical coverage proof belongs to a different lifecycle generation"
6252 .into(),
6253 ));
6254 }
6255 for (label, head) in [
6256 ("coordinator", state.coverage_head),
6257 ("historical source", state.historical_position.coverage_head),
6258 ] {
6259 let head = head.ok_or_else(|| {
6260 SubscriberError::Provider(format!(
6261 "hybrid checkpoint historical coverage proof has no {label} coverage head"
6262 ))
6263 })?;
6264 if proof.through.number > head.number {
6265 return Err(SubscriberError::Provider(format!(
6266 "hybrid checkpoint historical coverage proof is ahead of the {label}"
6267 )));
6268 }
6269 if proof.through.number == head.number {
6270 compatible_block_ref(&proof.through, &head)?;
6271 }
6272 }
6273 if let Some(known) = state
6274 .historical_position
6275 .canonical_history
6276 .iter()
6277 .find(|known| known.number == proof.through.number)
6278 {
6279 compatible_block_ref(known, &proof.through)?;
6280 }
6281 }
6282 validate_source_against_coordinator("historical source", &state.historical_position, state)?;
6283 validate_source_against_coordinator("live source", &state.live_position, state)?;
6284 if let Some(committed) = state.last_committed_token.as_ref() {
6285 if committed.inner.is_empty() {
6286 return Err(SubscriberError::Provider(
6287 "hybrid checkpoint contains an empty committed token".into(),
6288 ));
6289 }
6290 let source = match committed.source {
6291 HybridSource::Historical => &state.historical_position,
6292 HybridSource::Live => &state.live_position,
6293 };
6294 if source.delivery_digest.is_none() {
6295 return Err(SubscriberError::Provider(
6296 "hybrid committed token is missing its source delivery digest".into(),
6297 ));
6298 }
6299 match committed.kind {
6300 HybridTokenKind::Forwarded
6301 if source.delivery_token.as_deref() != Some(committed.inner.as_slice()) =>
6302 {
6303 return Err(SubscriberError::Provider(
6304 "hybrid checkpoint committed token does not match its source position".into(),
6305 ));
6306 }
6307 HybridTokenKind::Synthetic if source.delivery_token.is_some() => {
6308 return Err(SubscriberError::Provider(
6309 "hybrid synthetic commit retained a stale child delivery token".into(),
6310 ));
6311 }
6312 HybridTokenKind::Forwarded | HybridTokenKind::Synthetic => {}
6313 }
6314 }
6315 Ok(())
6316}
6317
6318fn validate_checkpoint_history(
6319 label: &str,
6320 history: &[BlockRef],
6321 coverage_head: Option<&BlockRef>,
6322) -> Result<(), SubscriberError> {
6323 if history.is_empty() {
6324 if coverage_head.is_some() {
6325 return Err(SubscriberError::Provider(format!(
6326 "hybrid {label} checkpoint has coverage without retained canonical history"
6327 )));
6328 }
6329 return Ok(());
6330 }
6331 if coverage_head.is_none() {
6332 return Err(SubscriberError::Provider(format!(
6333 "hybrid {label} checkpoint history has no coverage head"
6334 )));
6335 }
6336 for pair in history.windows(2) {
6337 let [previous, block] = pair else {
6338 unreachable!("window has exactly two entries")
6339 };
6340 if block.number <= previous.number {
6341 return Err(SubscriberError::Provider(format!(
6342 "hybrid {label} checkpoint history is not strictly ordered"
6343 )));
6344 }
6345 if block.number == previous.number.saturating_add(1)
6346 && block
6347 .parent_hash
6348 .is_some_and(|parent| parent != previous.hash)
6349 {
6350 return Err(SubscriberError::Provider(format!(
6351 "hybrid {label} checkpoint history has a parent-hash discontinuity at block {}",
6352 block.number
6353 )));
6354 }
6355 }
6356 let head = coverage_head.expect("checked above");
6357 let last = history.last().expect("non-empty history");
6358 if compatible_block_ref(last, head).is_err() {
6359 return Err(SubscriberError::Provider(format!(
6360 "hybrid {label} checkpoint history does not end at its coverage head"
6361 )));
6362 }
6363 Ok(())
6364}
6365
6366fn validate_source_against_coordinator(
6367 label: &str,
6368 source: &SourcePosition,
6369 state: &HybridCheckpointV5,
6370) -> Result<(), SubscriberError> {
6371 if source.delivery_token.is_some() && source.delivery_digest.is_none() {
6372 return Err(SubscriberError::Provider(format!(
6373 "hybrid {label} has a delivery token without its payload digest"
6374 )));
6375 }
6376 if source.delivery_token.is_some() && source.coverage_head.is_none() {
6377 return Err(SubscriberError::Provider(format!(
6378 "hybrid {label} has a forwarded delivery token without restorable canonical coverage"
6379 )));
6380 }
6381 if let Some(source_head) = source.coverage_head.as_ref() {
6382 let coordinator_head = state.coverage_head.as_ref().ok_or_else(|| {
6383 SubscriberError::Provider(format!(
6384 "hybrid {label} has coverage without coordinator coverage"
6385 ))
6386 })?;
6387 if source_head.number > coordinator_head.number {
6388 return Err(SubscriberError::Provider(format!(
6389 "hybrid {label} coverage is ahead of coordinator coverage"
6390 )));
6391 }
6392 if let Some(known) = state
6393 .canonical_history
6394 .iter()
6395 .find(|known| known.number == source_head.number)
6396 {
6397 compatible_block_ref(known, source_head)?;
6398 }
6399 }
6400 Ok(())
6401}
6402
6403fn compatible_block_ref(left: &BlockRef, right: &BlockRef) -> Result<BlockRef, SubscriberError> {
6404 if left.number != right.number || left.hash != right.hash {
6405 return Err(SubscriberError::Provider(format!(
6406 "hybrid canonical block identity conflicts at height {}",
6407 left.number
6408 )));
6409 }
6410 let parent_hash = merge_optional_metadata(
6411 left.parent_hash,
6412 right.parent_hash,
6413 "parent hash",
6414 left.number,
6415 )?;
6416 let timestamp =
6417 merge_optional_metadata(left.timestamp, right.timestamp, "timestamp", left.number)?;
6418 Ok(BlockRef {
6419 number: left.number,
6420 hash: left.hash,
6421 parent_hash,
6422 timestamp,
6423 })
6424}
6425
6426fn merge_optional_metadata<T: Copy + PartialEq>(
6427 left: Option<T>,
6428 right: Option<T>,
6429 label: &str,
6430 block_number: u64,
6431) -> Result<Option<T>, SubscriberError> {
6432 match (left, right) {
6433 (Some(left), Some(right)) if left != right => Err(SubscriberError::Provider(format!(
6434 "hybrid canonical {label} conflicts at block {block_number}"
6435 ))),
6436 (Some(value), _) | (_, Some(value)) => Ok(Some(value)),
6437 (None, None) => Ok(None),
6438 }
6439}
6440
6441fn input_ref_chain_id(input_ref: InputRef) -> Option<u64> {
6442 match input_ref {
6443 InputRef::Log { chain_id, .. }
6444 | InputRef::PendingTx { chain_id, .. }
6445 | InputRef::Block { chain_id, .. } => chain_id,
6446 }
6447}
6448
6449fn source_delivery_digest<N: Network>(
6450 records: &[RoutedRecord<N>],
6451 controls: &[ChainControl],
6452 checkpoint: Option<&SubscriberCheckpoint>,
6453 payload_commitment: Option<&SubscriberPayloadCommitment>,
6454 expected_chain_id: u64,
6455) -> Result<[u8; 32], SubscriberError> {
6456 let mut hasher = CanonicalHasher::new(b"EFCHY-SOURCE-DELIVERY-V1");
6457 hasher.u64(expected_chain_id);
6458 hasher
6459 .sequence_len(records.len())
6460 .map_err(transcript_error)?;
6461 for (record, audience, scope) in records {
6462 let (identity, witness) = validated_record(record, expected_chain_id)?;
6463 append_input_identity(&mut hasher, identity)?;
6464 append_record_witness(&mut hasher, &witness);
6465 append_exact_context(&mut hasher, &record.context)?;
6466 append_audience(&mut hasher, audience)?;
6467 append_scope(&mut hasher, *scope)?;
6468 }
6469 hasher
6470 .sequence_len(controls.len())
6471 .map_err(transcript_error)?;
6472 for control in controls {
6473 append_chain_control(&mut hasher, control)?;
6474 }
6475 match checkpoint {
6476 Some(checkpoint) => {
6477 hasher.bool(true);
6478 hasher
6479 .bytes(checkpoint.as_bytes())
6480 .map_err(transcript_error)?;
6481 }
6482 None => hasher.bool(false),
6483 }
6484 hasher.bool(payload_commitment.is_some());
6485 if let Some(commitment) = payload_commitment {
6486 hasher.hash(&commitment.digest());
6487 }
6488 Ok(hasher.finish())
6489}
6490
6491fn append_input_identity(
6492 hasher: &mut CanonicalHasher,
6493 identity: ReactiveInputIdentity,
6494) -> Result<(), SubscriberError> {
6495 match identity.input_ref() {
6496 InputRef::Log {
6497 chain_id,
6498 block_hash,
6499 transaction_hash,
6500 log_index,
6501 } => {
6502 hasher.tag(1);
6503 hasher.option_u64(chain_id);
6504 hasher.hash(&block_hash);
6505 hasher.hash(&transaction_hash);
6506 hasher.u64(log_index);
6507 }
6508 InputRef::PendingTx { chain_id, hash } => {
6509 hasher.tag(2);
6510 hasher.option_u64(chain_id);
6511 hasher.hash(&hash);
6512 }
6513 InputRef::Block {
6514 chain_id,
6515 hash,
6516 number,
6517 } => {
6518 hasher.tag(3);
6519 hasher.option_u64(chain_id);
6520 hasher.hash(&hash);
6521 hasher.u64(number);
6522 }
6523 }
6524 hasher.tag(match identity.kind() {
6525 ReactiveInputKind::CanonicalLog => 1,
6526 ReactiveInputKind::ReorgSignalLog => 2,
6527 ReactiveInputKind::BlockHeader => 3,
6528 ReactiveInputKind::FullBlock => 4,
6529 ReactiveInputKind::PendingTxHash => 5,
6530 ReactiveInputKind::PendingTx => 6,
6531 _ => {
6532 return Err(SubscriberError::Unsupported(
6533 "hybrid cannot transcript an unknown reactive-input kind",
6534 ));
6535 }
6536 });
6537 Ok(())
6538}
6539
6540fn append_record_witness(hasher: &mut CanonicalHasher, witness: &RecordWitness) {
6541 hasher
6542 .bytes(&witness.payload_digest)
6543 .expect("fixed digest length");
6544 hasher.u64(witness.chain_id);
6545 hasher.tag(match witness.lifecycle {
6546 WitnessLifecycle::Included => 1,
6547 WitnessLifecycle::Safe => 2,
6548 WitnessLifecycle::Finalized => 3,
6549 WitnessLifecycle::Reorg => 4,
6550 WitnessLifecycle::Pending => 5,
6551 });
6552 hasher.option_block_ref(witness.block.as_ref());
6553 hasher.option_u64(witness.transaction_index);
6554 hasher.option_u64(witness.log_index);
6555 hasher.option_u64(witness.log_block_timestamp);
6556}
6557
6558fn append_exact_context(
6559 hasher: &mut CanonicalHasher,
6560 context: &ReactiveContext,
6561) -> Result<(), SubscriberError> {
6562 hasher.option_u64(context.chain_id);
6563 hasher.tag(match context.source {
6564 InputSource::Batch => 1,
6565 InputSource::Subscription => 2,
6566 InputSource::Poll => 3,
6567 InputSource::Backfill => 4,
6568 InputSource::Synthetic => 5,
6569 _ => {
6570 return Err(SubscriberError::Unsupported(
6571 "hybrid cannot transcript an unknown input source",
6572 ));
6573 }
6574 });
6575 match &context.chain_status {
6576 ChainStatus::Pending => hasher.tag(1),
6577 ChainStatus::Included {
6578 block,
6579 confirmations,
6580 } => {
6581 hasher.tag(2);
6582 hasher.block_ref(block);
6583 hasher.u64(*confirmations);
6584 }
6585 ChainStatus::Safe { block } => {
6586 hasher.tag(3);
6587 hasher.block_ref(block);
6588 }
6589 ChainStatus::Finalized { block } => {
6590 hasher.tag(4);
6591 hasher.block_ref(block);
6592 }
6593 ChainStatus::Reorged { dropped_from } => {
6594 hasher.tag(5);
6595 hasher.block_ref(dropped_from);
6596 }
6597 _ => {
6598 return Err(SubscriberError::Unsupported(
6599 "hybrid cannot transcript an unknown chain status",
6600 ));
6601 }
6602 }
6603 hasher.option_block_ref(context.block.as_ref());
6604 hasher.option_u64(context.transaction_index);
6605 hasher.option_u64(context.log_index);
6606 Ok(())
6607}
6608
6609fn append_audience(
6610 hasher: &mut CanonicalHasher,
6611 audience: &DeliveryAudience,
6612) -> Result<(), SubscriberError> {
6613 let (tag, owners) = match audience {
6614 DeliveryAudience::All => (1, None),
6615 DeliveryAudience::Owners(owners) => (2, Some(owners)),
6616 DeliveryAudience::AllExcept(owners) => (3, Some(owners)),
6617 _ => {
6618 return Err(SubscriberError::Unsupported(
6619 "hybrid cannot transcript an unknown delivery audience",
6620 ));
6621 }
6622 };
6623 hasher.tag(tag);
6624 if let Some(owners) = owners {
6625 hasher
6626 .sequence_len(owners.len())
6627 .map_err(transcript_error)?;
6628 for owner in owners {
6629 hasher.handler_id(owner).map_err(transcript_error)?;
6630 }
6631 }
6632 Ok(())
6633}
6634
6635fn append_scope(hasher: &mut CanonicalHasher, scope: DeliveryScope) -> Result<(), SubscriberError> {
6636 hasher.tag(match scope {
6637 DeliveryScope::Canonical => 1,
6638 DeliveryScope::CanonicalProgress => 2,
6639 DeliveryScope::OwnerCatchup => 3,
6640 _ => {
6641 return Err(SubscriberError::Unsupported(
6642 "hybrid cannot transcript an unknown delivery scope",
6643 ));
6644 }
6645 });
6646 Ok(())
6647}
6648
6649fn append_chain_control(
6650 hasher: &mut CanonicalHasher,
6651 control: &ChainControl,
6652) -> Result<(), SubscriberError> {
6653 match control {
6654 ChainControl::Reorg {
6655 common_ancestor,
6656 old_tip,
6657 new_tip,
6658 } => {
6659 hasher.tag(1);
6660 hasher.block_ref(common_ancestor);
6661 hasher.block_ref(old_tip);
6662 hasher.block_ref(new_tip);
6663 }
6664 ChainControl::CanonicalProgress(block) => {
6665 hasher.tag(2);
6666 hasher.block_ref(block);
6667 }
6668 ChainControl::Safe(block) => {
6669 hasher.tag(3);
6670 hasher.block_ref(block);
6671 }
6672 ChainControl::Finalized(block) => {
6673 hasher.tag(4);
6674 hasher.block_ref(block);
6675 }
6676 ChainControl::Barrier { id, block } => {
6677 hasher.tag(5);
6678 hasher.bytes(id).map_err(transcript_error)?;
6679 hasher.option_block_ref(block.as_ref());
6680 }
6681 _ => {
6682 return Err(SubscriberError::Unsupported(
6683 "hybrid cannot transcript an unknown chain control",
6684 ));
6685 }
6686 }
6687 Ok(())
6688}
6689
6690fn transcript_error(error: transcript::TranscriptLengthError) -> SubscriberError {
6691 SubscriberError::Provider(error.to_string())
6692}
6693
6694fn validated_record<N: Network>(
6695 record: &ReactiveInputRecord<N>,
6696 expected_chain_id: u64,
6697) -> Result<(ReactiveInputIdentity, RecordWitness), SubscriberError> {
6698 if matches!(
6699 record.input,
6700 ReactiveInput::FullBlock(_) | ReactiveInput::PendingTx(_)
6701 ) {
6702 return Err(SubscriberError::Unsupported(
6703 "hybrid delivery does not accept full blocks or hydrated pending transactions until their complete bodies can be verified generically",
6704 ));
6705 }
6706 if record.context.chain_id != Some(expected_chain_id) {
6707 return Err(SubscriberError::Provider(format!(
6708 "hybrid source record has chain {:?}; expected {expected_chain_id}",
6709 record.context.chain_id
6710 )));
6711 }
6712 let identity = record
6713 .validated_identity()
6714 .map_err(|error| SubscriberError::Provider(error.to_string()))?;
6715 if input_ref_chain_id(identity.input_ref()) != Some(expected_chain_id) {
6716 return Err(SubscriberError::Provider(
6717 "hybrid validated input identity has a wrong or missing chain id".into(),
6718 ));
6719 }
6720 let witness = record_witness(record, expected_chain_id)?;
6721 Ok((identity, witness))
6722}
6723
6724fn record_witness<N: Network>(
6725 record: &ReactiveInputRecord<N>,
6726 expected_chain_id: u64,
6727) -> Result<RecordWitness, SubscriberError> {
6728 let identity = record
6729 .validated_identity()
6730 .map_err(|error| SubscriberError::Provider(error.to_string()))?;
6731 let lifecycle = match &record.context.chain_status {
6732 ChainStatus::Pending => WitnessLifecycle::Pending,
6733 ChainStatus::Included { .. } => WitnessLifecycle::Included,
6734 ChainStatus::Safe { .. } => WitnessLifecycle::Safe,
6735 ChainStatus::Finalized { .. } => WitnessLifecycle::Finalized,
6736 ChainStatus::Reorged { .. } => WitnessLifecycle::Reorg,
6737 _ => {
6738 return Err(SubscriberError::Provider(
6739 "hybrid does not support an unknown chain lifecycle variant".into(),
6740 ));
6741 }
6742 };
6743 let mut payload = CanonicalHasher::new(b"EFCHY-RECORD-PAYLOAD-V1");
6744 append_input_identity(&mut payload, identity)?;
6745 let log_block_timestamp = match &record.input {
6746 ReactiveInput::Log(log) => {
6747 payload.tag(1);
6748 payload
6749 .bytes(log.inner.address.as_slice())
6750 .map_err(transcript_error)?;
6751 payload
6752 .sequence_len(log.topics().len())
6753 .map_err(transcript_error)?;
6754 for topic in log.topics() {
6755 payload.hash(topic);
6756 }
6757 payload
6758 .bytes(&log.inner.data.data)
6759 .map_err(transcript_error)?;
6760 payload.hash(log.block_hash.as_ref().expect("validated log block hash"));
6761 payload.u64(log.block_number.expect("validated log block"));
6762 payload.hash(
6763 log.transaction_hash
6764 .as_ref()
6765 .expect("validated log transaction hash"),
6766 );
6767 payload.u64(log.transaction_index.expect("validated transaction index"));
6768 payload.u64(log.log_index.expect("validated log index"));
6769 payload.bool(log.removed);
6770 log.block_timestamp
6771 }
6772 ReactiveInput::BlockHeader(header) => {
6773 payload.tag(2);
6774 let (header_digest, _) = header_body_digest(header, usize::MAX)?;
6775 payload.hash(&header_digest);
6776 None
6777 }
6778 ReactiveInput::PendingTxHash(hash) => {
6779 payload.tag(3);
6780 payload.hash(hash);
6781 None
6782 }
6783 ReactiveInput::FullBlock(_) | ReactiveInput::PendingTx(_) => None,
6784 };
6785 Ok(RecordWitness {
6786 payload_digest: payload.finish(),
6787 chain_id: expected_chain_id,
6788 lifecycle,
6789 block: record.context.block,
6790 transaction_index: record.context.transaction_index,
6791 log_index: record.context.log_index,
6792 log_block_timestamp,
6793 })
6794}
6795
6796fn ensure_witness_compatible(
6797 left: &RecordWitness,
6798 right: &RecordWitness,
6799) -> Result<(), SubscriberError> {
6800 let mut merged = left.clone();
6801 merge_witness(&mut merged, right)
6802}
6803
6804fn merge_witness(left: &mut RecordWitness, right: &RecordWitness) -> Result<(), SubscriberError> {
6805 if left.payload_digest != right.payload_digest
6806 || left.chain_id != right.chain_id
6807 || left.transaction_index != right.transaction_index
6808 || left.log_index != right.log_index
6809 {
6810 return Err(SubscriberError::Provider(
6811 "hybrid duplicate identity carries conflicting payload or required positions".into(),
6812 ));
6813 }
6814 left.lifecycle = merge_witness_lifecycle(left.lifecycle, right.lifecycle)?;
6815 left.block = match (left.block.as_ref(), right.block.as_ref()) {
6816 (Some(left), Some(right)) => Some(compatible_block_ref(left, right)?),
6817 (Some(left), None) => Some(*left),
6818 (None, Some(right)) => Some(*right),
6819 (None, None) => None,
6820 };
6821 left.log_block_timestamp = merge_optional_metadata(
6822 left.log_block_timestamp,
6823 right.log_block_timestamp,
6824 "log timestamp",
6825 left.block.as_ref().map_or(0, |block| block.number),
6826 )?;
6827 Ok(())
6828}
6829
6830fn merge_witness_lifecycle(
6831 left: WitnessLifecycle,
6832 right: WitnessLifecycle,
6833) -> Result<WitnessLifecycle, SubscriberError> {
6834 use WitnessLifecycle::{Finalized, Included, Pending, Reorg, Safe};
6835 match (left, right) {
6836 (Pending, Pending) => Ok(Pending),
6837 (Reorg, Reorg) => Ok(Reorg),
6838 (Included | Safe | Finalized, Included | Safe | Finalized) => Ok(match (left, right) {
6839 (Finalized, _) | (_, Finalized) => Finalized,
6840 (Safe, _) | (_, Safe) => Safe,
6841 _ => Included,
6842 }),
6843 _ => Err(SubscriberError::Provider(
6844 "hybrid duplicate identity carries conflicting lifecycle context".into(),
6845 )),
6846 }
6847}
6848
6849fn truncate_front_to<T>(values: &mut Vec<T>, capacity: usize) {
6850 if values.len() > capacity {
6851 values.drain(..values.len() - capacity);
6852 }
6853}
6854
6855fn fresh_epoch() -> [u8; 16] {
6856 let now = SystemTime::now()
6857 .duration_since(UNIX_EPOCH)
6858 .unwrap_or_default()
6859 .as_nanos();
6860 let nonce = NEXT_EPOCH_NONCE.fetch_add(1, Ordering::Relaxed);
6861 let mut seed = Vec::with_capacity(16 + 4 + 8);
6862 seed.extend_from_slice(&now.to_be_bytes());
6863 seed.extend_from_slice(&std::process::id().to_be_bytes());
6864 seed.extend_from_slice(&nonce.to_be_bytes());
6865 let hash = keccak256(seed);
6866 hash[..16].try_into().expect("fixed hash prefix")
6867}
6868
6869fn source_resume_position(
6870 chain_id: u64,
6871 source: &SourcePosition,
6872) -> Option<SubscriberResumePosition> {
6873 let coverage_head = source.coverage_head?;
6874 let canonical_history = if source.canonical_history.is_empty() {
6875 vec![coverage_head]
6876 } else {
6877 source.canonical_history.clone()
6878 };
6879 Some(SubscriberResumePosition::new(
6880 chain_id,
6881 coverage_head,
6882 canonical_history,
6883 source
6884 .delivery_token
6885 .clone()
6886 .map(SubscriberDeliveryToken::new),
6887 source.checkpoint.clone().map(SubscriberCheckpoint::new),
6888 ))
6889}
6890
6891async fn reconcile_pending_owner_lifecycle<H, L, N>(
6892 historical: &mut H,
6893 live: &mut L,
6894 pending: &mut Option<LiveRollback<N>>,
6895 phase: &mut HybridPhase,
6896 poisoned: &mut Option<String>,
6897) -> Result<(), SubscriberError>
6898where
6899 H: InterestOwnerSubscriber<N>,
6900 L: InterestOwnerSubscriber<N>,
6901 N: Network,
6902{
6903 let Some(rollback) = pending.clone() else {
6904 return Ok(());
6905 };
6906 if matches!(rollback, LiveRollback::Recovery { .. }) {
6907 return Err(SubscriberError::Provider(
6908 "hybrid destructive lifecycle compensation is awaiting historical gap certification; poll the coordinator until it returns to Live before retrying registration"
6909 .into(),
6910 ));
6911 }
6912 let historical_result = apply_historical_owner_lifecycle_rollback(historical, &rollback).await;
6913 let live_result = apply_owner_lifecycle_rollback(live, &rollback).await;
6914 if historical_result.is_ok() && live_result.is_ok() {
6915 if let LiveRollback::Topology {
6916 base,
6917 owners,
6918 recovery_anchor: Some(anchor),
6919 ..
6920 } = &rollback
6921 && base.is_empty()
6922 && !owners.is_empty()
6923 {
6924 *pending = Some(LiveRollback::Recovery { anchor: *anchor });
6925 return Err(SubscriberError::Provider(
6926 "hybrid previous topology was restored after a destructive live reset; historical gap certification is required before Live"
6927 .into(),
6928 ));
6929 }
6930 if let Some(anchor) = rollback.gap_anchor() {
6931 let scope = rollback.gap_scope();
6932 *pending = None;
6933 return Err(poison_uncertifiable_lifecycle_gap(
6934 phase, poisoned, anchor, scope,
6935 ));
6936 }
6937 *pending = None;
6938 return Ok(());
6939 }
6940 Err(SubscriberError::Provider(format!(
6941 "hybrid lifecycle rollback remains pending (historical: {}; live: {})",
6942 result_description(&historical_result),
6943 result_description(&live_result),
6944 )))
6945}
6946
6947async fn apply_owner_lifecycle_rollback<S, N>(
6948 subscriber: &mut S,
6949 rollback: &LiveRollback<N>,
6950) -> Result<(), SubscriberError>
6951where
6952 S: InterestOwnerSubscriber<N>,
6953 N: Network,
6954{
6955 match rollback {
6956 LiveRollback::Base { previous, .. } => subscriber.register_interests(previous).await,
6957 LiveRollback::Topology { base, owners, .. } => {
6958 if !base.is_empty() && !owners.is_empty() {
6959 return Err(SubscriberError::InvalidConfig(EXCLUSIVE_TOPOLOGY_ERROR));
6960 }
6961 if base.is_empty() {
6962 subscriber.replace_interest_owners(owners.clone()).await
6963 } else {
6964 subscriber.register_interests(base).await
6965 }
6966 }
6967 LiveRollback::Owner {
6968 owner, previous, ..
6969 } => rollback_owner(subscriber, owner, previous.as_deref()).await,
6970 LiveRollback::Bulk { previous, .. } => rollback_bulk(subscriber, previous).await,
6971 LiveRollback::Recovery { .. } => Err(SubscriberError::Provider(
6972 "hybrid lifecycle recovery cannot be applied as a child topology mutation".into(),
6973 )),
6974 }
6975}
6976
6977async fn apply_historical_owner_lifecycle_rollback<S, N>(
6978 subscriber: &mut S,
6979 rollback: &LiveRollback<N>,
6980) -> Result<(), SubscriberError>
6981where
6982 S: InterestOwnerSubscriber<N>,
6983 N: Network,
6984{
6985 match rollback {
6986 LiveRollback::Topology {
6987 base,
6988 owners,
6989 recovery_anchor: Some(anchor),
6990 ..
6991 } if base.is_empty() && !owners.is_empty() => {
6992 let backfill = SubscriberBackfill::after_canonical_block(*anchor)?;
6993 subscriber
6994 .replace_interest_owners_with_global_backfill(owners.clone(), backfill)
6995 .await
6996 }
6997 LiveRollback::Recovery { .. } => Err(SubscriberError::Provider(
6998 "hybrid historical lifecycle recovery is already staged".into(),
6999 )),
7000 _ => apply_owner_lifecycle_rollback(subscriber, rollback).await,
7001 }
7002}
7003
7004fn exact_owner_topology<N: Network>(
7005 owners: &[(HandlerId, Vec<ReactiveInterest<N>>)],
7006 operation: &'static str,
7007) -> Result<HashMap<HandlerId, Vec<ReactiveInterest<N>>>, SubscriberError> {
7008 let mut exact = HashMap::with_capacity(owners.len());
7009 for (owner, interests) in owners {
7010 if exact.insert(owner.clone(), interests.clone()).is_some() {
7011 return Err(SubscriberError::InvalidConfig(operation));
7012 }
7013 }
7014 Ok(exact)
7015}
7016
7017fn sorted_owner_entries<N: Network>(
7018 owners: &HashMap<HandlerId, Vec<ReactiveInterest<N>>>,
7019) -> Vec<(HandlerId, Vec<ReactiveInterest<N>>)> {
7020 let mut entries = owners
7021 .iter()
7022 .map(|(owner, interests)| (owner.clone(), interests.clone()))
7023 .collect::<Vec<_>>();
7024 entries.sort_by(|(left, _), (right, _)| left.cmp(right));
7025 entries
7026}
7027
7028async fn reconcile_pending_base_lifecycle<H, L, N>(
7029 historical: &mut H,
7030 live: &mut L,
7031 pending: &mut Option<LiveRollback<N>>,
7032 phase: &mut HybridPhase,
7033 poisoned: &mut Option<String>,
7034) -> Result<(), SubscriberError>
7035where
7036 H: EventSubscriber<N>,
7037 L: EventSubscriber<N>,
7038 N: Network,
7039{
7040 let Some(LiveRollback::Base {
7041 previous,
7042 gap_anchor,
7043 }) = pending.as_ref()
7044 else {
7045 return if pending.is_none() {
7046 Ok(())
7047 } else {
7048 Err(SubscriberError::Provider(
7049 "hybrid owner lifecycle is pending; retry an owner lifecycle operation".into(),
7050 ))
7051 };
7052 };
7053 let previous = previous.clone();
7054 let gap_anchor = *gap_anchor;
7055 let historical_result = historical.register_interests(&previous).await;
7056 let live_result = live.register_interests(&previous).await;
7057 if historical_result.is_ok() && live_result.is_ok() {
7058 if let Some(anchor) = gap_anchor {
7059 *pending = None;
7060 return Err(poison_uncertifiable_lifecycle_gap(
7061 phase,
7062 poisoned,
7063 anchor,
7064 "base subscription",
7065 ));
7066 }
7067 *pending = None;
7068 return Ok(());
7069 }
7070 Err(SubscriberError::Provider(format!(
7071 "hybrid base lifecycle rollback remains pending (historical: {}; live: {})",
7072 result_description(&historical_result),
7073 result_description(&live_result),
7074 )))
7075}
7076
7077fn poison_uncertifiable_lifecycle_gap(
7078 phase: &mut HybridPhase,
7079 poisoned: &mut Option<String>,
7080 anchor: BlockRef,
7081 scope: &str,
7082) -> SubscriberError {
7083 let reason = format!(
7084 "{scope} changed on the live source after acknowledged coverage at block {}, but the durable historical commit did not complete; ordinary compensation restored the prior filters without proving the intervening live mutation window is free of an event gap. Reconstruct the coordinator from an authoritative checkpoint",
7085 anchor.number
7086 );
7087 *phase = HybridPhase::Poisoned;
7088 *poisoned = Some(reason.clone());
7089 SubscriberError::Provider(format!("hybrid coordinator is poisoned: {reason}"))
7090}
7091
7092fn result_description(result: &Result<(), SubscriberError>) -> String {
7093 match result {
7094 Ok(()) => "restored".into(),
7095 Err(error) => error.to_string(),
7096 }
7097}
7098
7099async fn rollback_bulk<S, N>(
7100 subscriber: &mut S,
7101 previous: &[(HandlerId, Option<Vec<ReactiveInterest<N>>>)],
7102) -> Result<(), SubscriberError>
7103where
7104 S: InterestOwnerSubscriber<N>,
7105 N: Network,
7106{
7107 let restore = previous
7108 .iter()
7109 .filter_map(|(owner, interests)| {
7110 interests
7111 .as_ref()
7112 .map(|interests| (owner.clone(), interests.clone()))
7113 })
7114 .collect::<Vec<_>>();
7115 if !restore.is_empty() {
7116 subscriber.upsert_interest_owners(restore).await?;
7117 }
7118 for (owner, interests) in previous {
7119 if interests.is_none() {
7120 subscriber.remove_interest_owner(owner).await?;
7121 }
7122 }
7123 Ok(())
7124}
7125
7126async fn rollback_owner<S, N>(
7127 subscriber: &mut S,
7128 owner: &HandlerId,
7129 previous: Option<&[ReactiveInterest<N>]>,
7130) -> Result<(), SubscriberError>
7131where
7132 S: InterestOwnerSubscriber<N>,
7133 N: Network,
7134{
7135 if let Some(previous) = previous {
7136 subscriber.add_interest_owner(owner.clone(), previous).await
7137 } else {
7138 subscriber.remove_interest_owner(owner).await.map(|_| ())
7139 }
7140}
7141
7142fn canonical_block_number<N: Network>(record: &ReactiveInputRecord<N>) -> Option<u64> {
7143 canonical_block_ref(record).map(|block| block.number)
7144}
7145
7146fn canonical_block_ref<N: Network>(record: &ReactiveInputRecord<N>) -> Option<&BlockRef> {
7147 match &record.context.chain_status {
7148 ChainStatus::Included { block, .. }
7149 | ChainStatus::Safe { block }
7150 | ChainStatus::Finalized { block } => Some(block),
7151 ChainStatus::Pending | ChainStatus::Reorged { .. } => None,
7152 _ => None,
7153 }
7154}
7155
7156fn dropped_block_ref<N: Network>(record: &ReactiveInputRecord<N>) -> Option<&BlockRef> {
7157 match &record.context.chain_status {
7158 ChainStatus::Reorged { dropped_from } => Some(dropped_from),
7159 _ if matches!(&record.input, ReactiveInput::Log(log) if log.removed) => {
7160 record.context.block.as_ref()
7161 }
7162 _ => None,
7163 }
7164}
7165
7166fn coverage_record_block_number<N: Network>(record: &ReactiveInputRecord<N>) -> Option<u64> {
7167 canonical_block_ref(record).map(|block| block.number)
7168}
7169
7170fn coverage_control_block_number(control: &ChainControl) -> Option<u64> {
7171 match control {
7172 ChainControl::CanonicalProgress(block) => Some(block.number),
7173 ChainControl::Barrier { block, .. } => block.as_ref().map(|block| block.number),
7174 _ => None,
7175 }
7176}
7177
7178fn scope_advances_canonical(scope: DeliveryScope) -> bool {
7179 matches!(
7180 scope,
7181 DeliveryScope::Canonical | DeliveryScope::CanonicalProgress
7182 )
7183}
7184
7185struct BoundedHeaderBuffer {
7186 bytes: Vec<u8>,
7187 encoded_limit: usize,
7188 limit_exceeded: bool,
7189}
7190
7191impl BoundedHeaderBuffer {
7192 fn new(encoded_limit: usize) -> Self {
7193 Self {
7194 bytes: Vec::new(),
7195 encoded_limit,
7196 limit_exceeded: false,
7197 }
7198 }
7199}
7200
7201impl std::io::Write for BoundedHeaderBuffer {
7202 fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
7203 let Some(next_len) = self.bytes.len().checked_add(bytes.len()) else {
7204 self.limit_exceeded = true;
7205 return Err(std::io::Error::other(
7206 "serialized block-header length overflow",
7207 ));
7208 };
7209 if next_len > self.encoded_limit {
7210 self.limit_exceeded = true;
7211 return Err(std::io::Error::other(
7212 "serialized block header exceeds ingress byte bound",
7213 ));
7214 }
7215 self.bytes
7216 .try_reserve(bytes.len())
7217 .map_err(|_| std::io::Error::other("serialized block-header allocation failed"))?;
7218 self.bytes.extend_from_slice(bytes);
7219 Ok(bytes.len())
7220 }
7221
7222 fn flush(&mut self) -> std::io::Result<()> {
7223 Ok(())
7224 }
7225}
7226
7227struct HeaderDigestWriter {
7228 hasher: Keccak256,
7229 encoded_len: usize,
7230 encoded_limit: usize,
7231 limit_exceeded: bool,
7232}
7233
7234fn write_canonical_json(
7235 value: &serde_json::Value,
7236 writer: &mut HeaderDigestWriter,
7237) -> std::io::Result<()> {
7238 use std::io::Write as _;
7239
7240 match value {
7241 serde_json::Value::Array(values) => {
7242 writer.write_all(b"[")?;
7243 for (index, value) in values.iter().enumerate() {
7244 if index != 0 {
7245 writer.write_all(b",")?;
7246 }
7247 write_canonical_json(value, writer)?;
7248 }
7249 writer.write_all(b"]")
7250 }
7251 serde_json::Value::Object(values) => {
7252 let mut entries = Vec::new();
7253 entries
7254 .try_reserve(values.len())
7255 .map_err(|_| std::io::Error::other("block-header key allocation failed"))?;
7256 entries.extend(values.iter());
7257 entries.sort_unstable_by_key(|(key, _)| *key);
7258
7259 writer.write_all(b"{")?;
7260 for (index, (key, value)) in entries.into_iter().enumerate() {
7261 if index != 0 {
7262 writer.write_all(b",")?;
7263 }
7264 serde_json::to_writer(&mut *writer, key).map_err(std::io::Error::other)?;
7265 writer.write_all(b":")?;
7266 write_canonical_json(value, writer)?;
7267 }
7268 writer.write_all(b"}")
7269 }
7270 _ => serde_json::to_writer(writer, value).map_err(std::io::Error::other),
7271 }
7272}
7273
7274impl HeaderDigestWriter {
7275 fn new(encoded_limit: usize) -> Self {
7276 let mut hasher = Keccak256::new();
7277 hasher.update(HEADER_WITNESS_DOMAIN);
7278 Self {
7279 hasher,
7280 encoded_len: 0,
7281 encoded_limit,
7282 limit_exceeded: false,
7283 }
7284 }
7285}
7286
7287impl std::io::Write for HeaderDigestWriter {
7288 fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
7289 let Some(next_len) = self.encoded_len.checked_add(bytes.len()) else {
7290 self.limit_exceeded = true;
7291 return Err(std::io::Error::other(
7292 "serialized block-header length overflow",
7293 ));
7294 };
7295 if next_len > self.encoded_limit {
7296 self.limit_exceeded = true;
7297 return Err(std::io::Error::other(
7298 "serialized block header exceeds ingress byte bound",
7299 ));
7300 }
7301 self.hasher.update(bytes);
7302 self.encoded_len = next_len;
7303 Ok(bytes.len())
7304 }
7305
7306 fn flush(&mut self) -> std::io::Result<()> {
7307 Ok(())
7308 }
7309}
7310
7311fn header_body_digest<T: Serialize + ?Sized>(
7312 header: &T,
7313 encoded_limit: usize,
7314) -> Result<(alloy_primitives::B256, usize), SubscriberError> {
7315 let encoded_limit = encoded_limit.min(MAX_HEADER_CANONICALIZATION_BYTES);
7319 let mut bounded = BoundedHeaderBuffer::new(encoded_limit);
7320 if let Err(error) = serde_json::to_writer(&mut bounded, header) {
7321 if bounded.limit_exceeded {
7322 return Err(SubscriberError::Provider(format!(
7323 "hybrid serialized block header exceeds the remaining ingress byte bound ({encoded_limit} bytes)"
7324 )));
7325 }
7326 return Err(SubscriberError::Provider(format!(
7327 "failed to encode exact hybrid block-header witness: {error}"
7328 )));
7329 }
7330 let value = serde_json::from_slice(&bounded.bytes).map_err(|error| {
7331 SubscriberError::Provider(format!(
7332 "failed to canonicalize exact hybrid block-header witness: {error}"
7333 ))
7334 })?;
7335 let mut writer = HeaderDigestWriter::new(encoded_limit);
7336 if let Err(error) = write_canonical_json(&value, &mut writer) {
7337 if writer.limit_exceeded {
7338 return Err(SubscriberError::Provider(format!(
7339 "hybrid serialized block header exceeds the remaining ingress byte bound ({encoded_limit} bytes)"
7340 )));
7341 }
7342 return Err(SubscriberError::Provider(format!(
7343 "failed to canonicalize exact hybrid block-header witness: {error}"
7344 )));
7345 }
7346 Ok((writer.hasher.finalize(), writer.encoded_len))
7347}
7348
7349fn accounted_record_bytes<N: Network>(
7350 record: &ReactiveInputRecord<N>,
7351 audience: &DeliveryAudience,
7352 remaining: usize,
7353) -> Result<usize, SubscriberError> {
7354 const RECORD_OVERHEAD: usize = 512;
7355 let audience_bytes = accounted_audience_bytes(audience);
7356 let input_bytes = match &record.input {
7357 ReactiveInput::Log(log) => RECORD_OVERHEAD
7358 .saturating_add(log.topics().len().saturating_mul(32))
7359 .saturating_add(log.inner.data.data.len()),
7360 ReactiveInput::BlockHeader(header) => {
7361 let fixed = RECORD_OVERHEAD.saturating_add(audience_bytes);
7362 let header_limit = remaining.saturating_sub(fixed);
7363 let (_, encoded_len) = header_body_digest(header, header_limit)?;
7364 RECORD_OVERHEAD.saturating_add(encoded_len)
7365 }
7366 ReactiveInput::PendingTxHash(_) => RECORD_OVERHEAD,
7367 ReactiveInput::FullBlock(_) | ReactiveInput::PendingTx(_) => usize::MAX / 2,
7372 };
7373 Ok(input_bytes.saturating_add(audience_bytes))
7374}
7375
7376fn accounted_audience_bytes(audience: &DeliveryAudience) -> usize {
7377 const AUDIENCE_OVERHEAD: usize = 64;
7378 match audience {
7379 DeliveryAudience::All => AUDIENCE_OVERHEAD,
7380 DeliveryAudience::Owners(owners) | DeliveryAudience::AllExcept(owners) => owners
7381 .iter()
7382 .map(|owner| owner.as_str().len().saturating_add(32))
7383 .fold(AUDIENCE_OVERHEAD, usize::saturating_add),
7384 _ => usize::MAX / 2,
7385 }
7386}
7387
7388fn accounted_control_bytes(control: &ChainControl) -> usize {
7389 const CONTROL_OVERHEAD: usize = 256;
7390 match control {
7391 ChainControl::Barrier { id, .. } => CONTROL_OVERHEAD.saturating_add(id.len()),
7392 ChainControl::Reorg { .. }
7393 | ChainControl::Safe(_)
7394 | ChainControl::Finalized(_)
7395 | ChainControl::CanonicalProgress(_) => CONTROL_OVERHEAD,
7396 _ => usize::MAX / 2,
7397 }
7398}
7399
7400fn same_buffered_delivery<N: Network>(left: &BufferedBatch<N>, right: &BufferedBatch<N>) -> bool {
7401 left.records.len() == right.records.len()
7402 && left.records.iter().zip(&right.records).all(
7403 |(
7404 (left_record, left_audience, left_scope),
7405 (right_record, right_audience, right_scope),
7406 )| {
7407 same_supported_record(left_record, right_record)
7408 && left_audience == right_audience
7409 && left_scope == right_scope
7410 },
7411 )
7412 && left.controls == right.controls
7413 && left.checkpoint == right.checkpoint
7414 && left.payload_commitment == right.payload_commitment
7415}
7416
7417fn same_supported_record<N: Network>(
7418 left: &ReactiveInputRecord<N>,
7419 right: &ReactiveInputRecord<N>,
7420) -> bool {
7421 if left.context != right.context || left.input_ref() != right.input_ref() {
7422 return false;
7423 }
7424 match (&left.input, &right.input) {
7425 (ReactiveInput::Log(left), ReactiveInput::Log(right)) => left == right,
7426 (ReactiveInput::PendingTxHash(left), ReactiveInput::PendingTxHash(right)) => left == right,
7427 (ReactiveInput::BlockHeader(left), ReactiveInput::BlockHeader(right)) => {
7431 match (
7432 header_body_digest(left, usize::MAX),
7433 header_body_digest(right, usize::MAX),
7434 ) {
7435 (Ok((left, left_len)), Ok((right, right_len))) => {
7436 left_len == right_len && left == right
7437 }
7438 _ => false,
7439 }
7440 }
7441 (ReactiveInput::FullBlock(_), ReactiveInput::FullBlock(_))
7445 | (ReactiveInput::PendingTx(_), ReactiveInput::PendingTx(_)) => false,
7446 _ => false,
7447 }
7448}
7449
7450fn deduplicable<N: Network>(record: &ReactiveInputRecord<N>) -> bool {
7451 record.is_payload_deduplicable()
7456 && !matches!(record.context.chain_status, ChainStatus::Reorged { .. })
7457 && !matches!(&record.input, ReactiveInput::Log(log) if log.removed)
7458}
7459
7460fn wrap_token(
7461 epoch: [u8; 16],
7462 source: HybridSource,
7463 kind: HybridTokenKind,
7464 token: SubscriberDeliveryToken,
7465) -> SubscriberDeliveryToken {
7466 let inner = token.into_bytes();
7467 let mut bytes = Vec::with_capacity(TOKEN_MAGIC.len() + 16 + 2 + inner.len());
7468 bytes.extend_from_slice(TOKEN_MAGIC);
7469 bytes.extend_from_slice(&epoch);
7470 bytes.push(source.tag());
7471 bytes.push(kind.tag());
7472 bytes.extend_from_slice(&inner);
7473 SubscriberDeliveryToken::new(bytes)
7474}
7475
7476fn unwrap_token(
7477 token: SubscriberDeliveryToken,
7478) -> Result<
7479 (
7480 [u8; 16],
7481 HybridSource,
7482 HybridTokenKind,
7483 SubscriberDeliveryToken,
7484 ),
7485 SubscriberError,
7486> {
7487 let bytes = token.into_bytes();
7488 const PREFIX_LEN: usize = 8 + 16;
7489 if bytes.len() < PREFIX_LEN + 2 || &bytes[..TOKEN_MAGIC.len()] != TOKEN_MAGIC {
7490 return Err(SubscriberError::Provider(
7491 "invalid hybrid delivery-token envelope".into(),
7492 ));
7493 }
7494 let epoch = bytes[8..PREFIX_LEN]
7495 .try_into()
7496 .expect("validated epoch width");
7497 let source = HybridSource::from_tag(bytes[PREFIX_LEN])?;
7498 let kind = HybridTokenKind::from_tag(bytes[PREFIX_LEN + 1])?;
7499 Ok((
7500 epoch,
7501 source,
7502 kind,
7503 SubscriberDeliveryToken::new(bytes[(PREFIX_LEN + 2)..].to_vec()),
7504 ))
7505}
7506
7507#[cfg(test)]
7508mod token_tests {
7509 use super::*;
7510 use std::sync::{
7511 Arc,
7512 atomic::{AtomicUsize, Ordering as AtomicOrdering},
7513 };
7514
7515 use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog};
7516 use alloy_rpc_types_eth::{Filter, Log};
7517 use evm_fork_cache::reactive::{CanonicalRollbackKind, LogInterest};
7518
7519 struct RestoreCountingSource {
7520 historical: bool,
7521 durable: bool,
7522 polls: Arc<AtomicUsize>,
7523 mutations: Arc<AtomicUsize>,
7524 }
7525
7526 impl EventSubscriber<Ethereum> for RestoreCountingSource {
7527 fn chain_id(&self) -> Option<u64> {
7528 Some(1)
7529 }
7530
7531 fn capabilities(&self) -> SubscriberCapabilities {
7532 let mut capabilities = vec![
7533 SubscriberCapability::Logs,
7534 SubscriberCapability::OwnerScopedDelivery,
7535 SubscriberCapability::DynamicInterests,
7536 SubscriberCapability::Barriers,
7537 ];
7538 if self.durable {
7539 capabilities.push(SubscriberCapability::DurableReplay);
7540 }
7541 capabilities.push(if self.historical {
7542 SubscriberCapability::HistoricalBackfill
7543 } else {
7544 SubscriberCapability::Live
7545 });
7546 SubscriberCapabilities::new(capabilities)
7547 }
7548
7549 fn register_interests(
7550 &mut self,
7551 _interests: &[ReactiveInterest<Ethereum>],
7552 ) -> SubscriberOperation<'_, ()> {
7553 Box::pin(async move {
7554 self.mutations.fetch_add(1, AtomicOrdering::SeqCst);
7555 Ok(())
7556 })
7557 }
7558
7559 fn next_batch(&mut self) -> SubscriberNextBatch<'_, Ethereum> {
7560 Box::pin(async move {
7561 self.polls.fetch_add(1, AtomicOrdering::SeqCst);
7562 Ok(None)
7563 })
7564 }
7565
7566 fn restore_position(
7567 &mut self,
7568 _position: &SubscriberResumePosition,
7569 ) -> Result<(), SubscriberError> {
7570 Ok(())
7571 }
7572 }
7573
7574 impl InterestOwnerSubscriber<Ethereum> for RestoreCountingSource {
7575 fn upsert_interest_owners(
7576 &mut self,
7577 _owners: Vec<(HandlerId, Vec<ReactiveInterest<Ethereum>>)>,
7578 ) -> SubscriberOperation<'_, ()> {
7579 Box::pin(async move {
7580 self.mutations.fetch_add(1, AtomicOrdering::SeqCst);
7581 Ok(())
7582 })
7583 }
7584
7585 fn replace_interest_owners(
7586 &mut self,
7587 _owners: Vec<(HandlerId, Vec<ReactiveInterest<Ethereum>>)>,
7588 ) -> SubscriberOperation<'_, ()> {
7589 Box::pin(async move {
7590 self.mutations.fetch_add(1, AtomicOrdering::SeqCst);
7591 Ok(())
7592 })
7593 }
7594
7595 fn add_interest_owner(
7596 &mut self,
7597 _owner: HandlerId,
7598 _interests: &[ReactiveInterest<Ethereum>],
7599 ) -> SubscriberOperation<'_, ()> {
7600 Box::pin(async move {
7601 self.mutations.fetch_add(1, AtomicOrdering::SeqCst);
7602 Ok(())
7603 })
7604 }
7605
7606 fn add_interest_owner_with_backfill(
7607 &mut self,
7608 owner: HandlerId,
7609 interests: &[ReactiveInterest<Ethereum>],
7610 _backfill: SubscriberBackfill,
7611 ) -> SubscriberOperation<'_, ()> {
7612 self.add_interest_owner(owner, interests)
7613 }
7614
7615 fn remove_interest_owner(
7616 &mut self,
7617 _owner: &HandlerId,
7618 ) -> SubscriberOperation<'_, Option<Vec<ReactiveInterest<Ethereum>>>> {
7619 Box::pin(async move {
7620 self.mutations.fetch_add(1, AtomicOrdering::SeqCst);
7621 Ok(None)
7622 })
7623 }
7624
7625 fn owner_interests(&self, _owner: &HandlerId) -> Option<&[ReactiveInterest<Ethereum>]> {
7626 None
7627 }
7628 }
7629
7630 struct OrderedHeader<'a>(&'a [(&'a str, u64)]);
7631
7632 impl Serialize for OrderedHeader<'_> {
7633 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
7634 where
7635 S: serde::Serializer,
7636 {
7637 let mut map = serializer.serialize_map(Some(self.0.len()))?;
7638 for (key, value) in self.0 {
7639 serde::ser::SerializeMap::serialize_entry(&mut map, key, value)?;
7640 }
7641 serde::ser::SerializeMap::end(map)
7642 }
7643 }
7644
7645 fn checkpoint() -> HybridCheckpointV5 {
7646 HybridCheckpointV5 {
7647 chain_id: 1,
7648 epoch: [9; 16],
7649 next_synthetic_token: 7,
7650 lifecycle_generation: 3,
7651 owner_generations: BTreeMap::new(),
7652 lifecycle_intent: lifecycle_intent::<Ethereum>(&[], &HashMap::new())
7653 .expect("empty lifecycle"),
7654 recent_inputs: Vec::new(),
7655 canonical_history: Vec::new(),
7656 coverage_head: None,
7657 safe_head: None,
7658 finalized_head: None,
7659 certified_historical: None,
7660 historical_position: SourcePosition::default(),
7661 live_position: SourcePosition::default(),
7662 last_committed_token: None,
7663 }
7664 }
7665
7666 fn golden_block(number: u64, hash: u8, parent_hash: u8) -> BlockRef {
7667 BlockRef {
7668 number,
7669 hash: B256::repeat_byte(hash),
7670 parent_hash: Some(B256::repeat_byte(parent_hash)),
7671 timestamp: Some(number.saturating_mul(12)),
7672 }
7673 }
7674
7675 fn golden_identity(input_ref: InputRef, kind: ReactiveInputKind) -> ReactiveInputIdentity {
7676 ReactiveInputIdentity::try_from_parts(input_ref, kind)
7677 .expect("golden identity must pair a compatible reference and representation")
7678 }
7679
7680 fn golden_witness(
7681 payload: u8,
7682 lifecycle: WitnessLifecycle,
7683 block: Option<BlockRef>,
7684 transaction_index: Option<u64>,
7685 log_index: Option<u64>,
7686 log_block_timestamp: Option<u64>,
7687 ) -> RecordWitness {
7688 RecordWitness {
7689 payload_digest: [payload; 32],
7690 chain_id: 1,
7691 lifecycle,
7692 block,
7693 transaction_index,
7694 log_index,
7695 log_block_timestamp,
7696 }
7697 }
7698
7699 fn fully_populated_checkpoint(
7700 committed_source: HybridSource,
7701 committed_kind: HybridTokenKind,
7702 ) -> HybridCheckpointV5 {
7703 let block_100 = golden_block(100, 0x64, 0x63);
7704 let block_101 = golden_block(101, 0x65, 0x64);
7705 let block_102 = golden_block(102, 0x66, 0x65);
7706 let owner_alpha = HandlerId::new("owner-alpha");
7707 let owner_beta = HandlerId::new("owner-beta");
7708 let owner_generations = BTreeMap::from([(owner_alpha.clone(), 2), (owner_beta.clone(), 3)]);
7709 let owner_digests = BTreeMap::from([
7710 (owner_alpha.clone(), [0xa1; 32]),
7711 (owner_beta.clone(), [0xb2; 32]),
7712 ]);
7713 let recent_inputs = vec![
7714 StoredRecentInput {
7715 identity: golden_identity(
7716 InputRef::Log {
7717 chain_id: Some(1),
7718 block_hash: block_102.hash,
7719 transaction_hash: B256::repeat_byte(0xc1),
7720 log_index: 7,
7721 },
7722 ReactiveInputKind::CanonicalLog,
7723 ),
7724 coverage: AudienceCoverage {
7725 base: true,
7726 owners: BTreeMap::new(),
7727 block: Some(block_102),
7728 witness: Some(golden_witness(
7729 0x11,
7730 WitnessLifecycle::Included,
7731 Some(block_102),
7732 Some(2),
7733 Some(7),
7734 Some(1_224),
7735 )),
7736 },
7737 },
7738 StoredRecentInput {
7739 identity: golden_identity(
7740 InputRef::Log {
7741 chain_id: Some(1),
7742 block_hash: block_100.hash,
7743 transaction_hash: B256::repeat_byte(0xc2),
7744 log_index: 8,
7745 },
7746 ReactiveInputKind::CanonicalLog,
7747 ),
7748 coverage: AudienceCoverage {
7749 base: false,
7750 owners: BTreeMap::from([(owner_alpha.clone(), 2)]),
7751 block: Some(block_100),
7752 witness: Some(golden_witness(
7753 0x22,
7754 WitnessLifecycle::Finalized,
7755 Some(block_100),
7756 Some(3),
7757 Some(8),
7758 None,
7759 )),
7760 },
7761 },
7762 StoredRecentInput {
7763 identity: golden_identity(
7764 InputRef::Block {
7765 chain_id: Some(1),
7766 hash: block_101.hash,
7767 number: block_101.number,
7768 },
7769 ReactiveInputKind::BlockHeader,
7770 ),
7771 coverage: AudienceCoverage {
7772 base: true,
7773 owners: BTreeMap::from([(owner_alpha.clone(), 2), (owner_beta.clone(), 3)]),
7774 block: Some(block_101),
7775 witness: Some(golden_witness(
7776 0x33,
7777 WitnessLifecycle::Safe,
7778 Some(block_101),
7779 None,
7780 None,
7781 None,
7782 )),
7783 },
7784 },
7785 StoredRecentInput {
7786 identity: golden_identity(
7787 InputRef::Log {
7788 chain_id: Some(1),
7789 block_hash: block_101.hash,
7790 transaction_hash: B256::repeat_byte(0xc4),
7791 log_index: 9,
7792 },
7793 ReactiveInputKind::ReorgSignalLog,
7794 ),
7795 coverage: AudienceCoverage {
7796 base: false,
7797 owners: BTreeMap::from([(owner_beta.clone(), 3)]),
7798 block: Some(block_101),
7799 witness: Some(golden_witness(
7800 0x44,
7801 WitnessLifecycle::Reorg,
7802 Some(block_101),
7803 Some(4),
7804 Some(9),
7805 Some(1_212),
7806 )),
7807 },
7808 },
7809 StoredRecentInput {
7810 identity: golden_identity(
7811 InputRef::PendingTx {
7812 chain_id: Some(1),
7813 hash: B256::repeat_byte(0xd5),
7814 },
7815 ReactiveInputKind::PendingTxHash,
7816 ),
7817 coverage: AudienceCoverage {
7818 base: false,
7819 owners: BTreeMap::new(),
7820 block: None,
7821 witness: Some(golden_witness(
7822 0x55,
7823 WitnessLifecycle::Pending,
7824 None,
7825 None,
7826 None,
7827 None,
7828 )),
7829 },
7830 },
7831 ];
7832 let historical_token = vec![0xf1, 0xf2, 0xf3];
7833 let live_token = vec![0xe1, 0xe2, 0xe3, 0xe4];
7834 let mut historical_position = SourcePosition {
7835 delivery_token: Some(historical_token.clone()),
7836 checkpoint: Some(vec![0xa0, 0xa1, 0xa2]),
7837 coverage_head: Some(block_102),
7838 canonical_history: vec![block_100, block_101, block_102],
7839 delivery_digest: Some([0xa5; 32]),
7840 };
7841 let mut live_position = SourcePosition {
7842 delivery_token: Some(live_token.clone()),
7843 checkpoint: Some(vec![0xb0, 0xb1]),
7844 coverage_head: Some(block_101),
7845 canonical_history: vec![block_100, block_101],
7846 delivery_digest: Some([0xb5; 32]),
7847 };
7848 let committed_inner = match (committed_source, committed_kind) {
7849 (HybridSource::Historical, HybridTokenKind::Forwarded) => historical_token,
7850 (HybridSource::Live, HybridTokenKind::Forwarded) => live_token,
7851 (HybridSource::Historical, HybridTokenKind::Synthetic) => {
7852 historical_position.delivery_token = None;
7853 5_u64.to_be_bytes().to_vec()
7854 }
7855 (HybridSource::Live, HybridTokenKind::Synthetic) => {
7856 live_position.delivery_token = None;
7857 6_u64.to_be_bytes().to_vec()
7858 }
7859 };
7860 HybridCheckpointV5 {
7861 chain_id: 1,
7862 epoch: [0x9a; 16],
7863 next_synthetic_token: 7,
7864 lifecycle_generation: 3,
7865 owner_generations,
7866 lifecycle_intent: LifecycleIntent {
7867 base: empty_interest_fingerprint(),
7868 owners: owner_digests,
7869 },
7870 recent_inputs,
7871 canonical_history: vec![block_100, block_101, block_102],
7872 coverage_head: Some(block_102),
7873 safe_head: Some(block_101),
7874 finalized_head: Some(block_100),
7875 certified_historical: Some(CertifiedHistoricalCoverage {
7876 lifecycle_generation: 3,
7877 through: block_102,
7878 }),
7879 historical_position,
7880 live_position,
7881 last_committed_token: Some(StoredCommittedToken {
7882 source: committed_source,
7883 kind: committed_kind,
7884 inner: committed_inner,
7885 }),
7886 }
7887 }
7888
7889 fn restore_test_interest() -> ReactiveInterest<Ethereum> {
7890 ReactiveInterest::Logs(LogInterest {
7891 provider_filter: Filter::new().address(Address::repeat_byte(0xaa)),
7892 local_matcher: None,
7893 route_key: None,
7894 })
7895 }
7896
7897 fn restore_position_for(state: &HybridCheckpointV5) -> SubscriberResumePosition {
7898 let committed = state
7899 .last_committed_token
7900 .as_ref()
7901 .expect("restore fixture carries a committed token");
7902 SubscriberResumePosition::new(
7903 state.chain_id,
7904 state
7905 .coverage_head
7906 .expect("restore fixture carries coverage"),
7907 state.canonical_history.clone(),
7908 Some(wrap_token(
7909 state.epoch,
7910 committed.source,
7911 committed.kind,
7912 SubscriberDeliveryToken::new(committed.inner.clone()),
7913 )),
7914 Some(SubscriberCheckpoint::new(
7915 encode_hybrid_checkpoint(state).expect("encode restore fixture"),
7916 )),
7917 )
7918 }
7919
7920 fn fill_opaque_checkpoint_to_v5_limit(state: &mut HybridCheckpointV5) {
7921 let mut low = 0usize;
7922 let mut high = codec::MAX_CHECKPOINT_PAYLOAD_BYTES + 1;
7923 while low + 1 < high {
7924 let middle = low + (high - low) / 2;
7925 state.historical_position.checkpoint = Some(vec![0x7a; middle]);
7926 if codec::checkpoint_payload_len_after_dropping_recent(state, 0)
7927 .expect("measure checkpoint fixture")
7928 <= codec::MAX_CHECKPOINT_PAYLOAD_BYTES
7929 {
7930 low = middle;
7931 } else {
7932 high = middle;
7933 }
7934 }
7935 state.historical_position.checkpoint = Some(vec![0x7a; low]);
7936 encode_hybrid_checkpoint(state).expect("near-limit checkpoint remains valid");
7937 }
7938
7939 fn fill_ephemeral_live_checkpoint_to_v5_limit(state: &mut HybridCheckpointV5) {
7940 let mut low = 0usize;
7941 let mut high = codec::MAX_CHECKPOINT_PAYLOAD_BYTES + 1;
7942 while low + 1 < high {
7943 let middle = low + (high - low) / 2;
7944 state.live_position.checkpoint = Some(vec![0x6c; middle]);
7945 if codec::checkpoint_payload_len_after_dropping_recent(state, 0)
7946 .expect("measure live checkpoint fixture")
7947 <= codec::MAX_CHECKPOINT_PAYLOAD_BYTES
7948 {
7949 low = middle;
7950 } else {
7951 high = middle;
7952 }
7953 }
7954 state.live_position.checkpoint = Some(vec![0x6c; low]);
7955 encode_hybrid_checkpoint(state).expect("near-limit live checkpoint remains valid");
7956 }
7957
7958 fn maximum_opaque_checkpoint_config() -> HybridConfig {
7959 HybridConfig {
7960 max_source_checkpoint_bytes: HYBRID_MAX_SOURCE_CHECKPOINT_BYTES,
7961 ..HybridConfig::default()
7962 }
7963 }
7964
7965 fn restore_counting_sources(
7966 live_mutations: Arc<AtomicUsize>,
7967 ) -> (
7968 RestoreCountingSource,
7969 RestoreCountingSource,
7970 Arc<AtomicUsize>,
7971 ) {
7972 let historical_mutations = Arc::new(AtomicUsize::new(0));
7973 (
7974 RestoreCountingSource {
7975 historical: true,
7976 durable: true,
7977 polls: Arc::new(AtomicUsize::new(0)),
7978 mutations: Arc::clone(&historical_mutations),
7979 },
7980 RestoreCountingSource {
7981 historical: false,
7982 durable: false,
7983 polls: Arc::new(AtomicUsize::new(0)),
7984 mutations: live_mutations,
7985 },
7986 historical_mutations,
7987 )
7988 }
7989
7990 fn checkpoint_hex(bytes: &[u8]) -> String {
7991 use std::fmt::Write as _;
7992
7993 let mut encoded = String::with_capacity(bytes.len().saturating_mul(2));
7994 for byte in bytes {
7995 write!(encoded, "{byte:02x}").expect("writing to String cannot fail");
7996 }
7997 encoded
7998 }
7999
8000 #[test]
8001 fn every_structured_incomplete_rollback_requests_historical_recovery() {
8002 for kind in [
8003 CanonicalRollbackKind::Explicit,
8004 CanonicalRollbackKind::Removed,
8005 CanonicalRollbackKind::ImplicitParent,
8006 CanonicalRollbackKind::MissingReplacement,
8007 ] {
8008 let classified =
8009 classify_canonical_validation_error(CanonicalSequenceError::IncompleteRollback {
8010 common_ancestor: 99,
8011 oldest_retained: Some(100),
8012 kind,
8013 });
8014 assert!(matches!(
8015 classified,
8016 CanonicalPlanError::NeedsHistoricalRecovery(_)
8017 ));
8018 }
8019 }
8020
8021 #[test]
8022 fn intrinsically_invalid_canonical_input_remains_fatal() {
8023 let classified = classify_canonical_validation_error(CanonicalSequenceError::Invalid(
8024 evm_fork_cache::reactive::ReactiveError::InvalidChainControl {
8025 message: "contradictory test input".into(),
8026 },
8027 ));
8028 assert!(matches!(classified, CanonicalPlanError::Invalid(_)));
8029 }
8030
8031 fn stored_recent_input(
8032 block_number: u64,
8033 owners: BTreeMap<HandlerId, u64>,
8034 ) -> StoredRecentInput {
8035 let block = BlockRef {
8036 number: block_number,
8037 hash: B256::repeat_byte(block_number as u8),
8038 parent_hash: None,
8039 timestamp: None,
8040 };
8041 let record = ReactiveInputRecord::<Ethereum>::new(
8042 ReactiveInput::Log(Log {
8043 inner: PrimitiveLog::new_unchecked(
8044 Address::repeat_byte(0xaa),
8045 vec![B256::repeat_byte(0xbb)],
8046 Bytes::new(),
8047 ),
8048 block_hash: Some(block.hash),
8049 block_number: Some(block.number),
8050 block_timestamp: None,
8051 transaction_hash: Some(B256::repeat_byte(block_number.wrapping_add(1) as u8)),
8052 transaction_index: Some(0),
8053 log_index: Some(block_number),
8054 removed: false,
8055 }),
8056 ReactiveContext {
8057 chain_id: Some(1),
8058 source: InputSource::Backfill,
8059 chain_status: ChainStatus::Included {
8060 block,
8061 confirmations: 0,
8062 },
8063 block: Some(block),
8064 transaction_index: Some(0),
8065 log_index: Some(block_number),
8066 },
8067 );
8068 StoredRecentInput {
8069 identity: record.validated_identity().expect("valid test identity"),
8070 coverage: AudienceCoverage {
8071 base: false,
8072 owners,
8073 block: Some(block),
8074 witness: Some(record_witness(&record, 1).expect("test witness")),
8075 },
8076 }
8077 }
8078
8079 #[test]
8080 fn token_envelope_round_trips_source_and_opaque_bytes() {
8081 for source in [HybridSource::Historical, HybridSource::Live] {
8082 for kind in [HybridTokenKind::Forwarded, HybridTokenKind::Synthetic] {
8083 let wrapped = wrap_token(
8084 [7; 16],
8085 source,
8086 kind,
8087 SubscriberDeliveryToken::new(vec![0, 1, 2, 255]),
8088 );
8089 let (epoch, decoded_source, decoded_kind, inner) =
8090 unwrap_token(wrapped).expect("decode token");
8091 assert_eq!(epoch, [7; 16]);
8092 assert_eq!(decoded_source, source);
8093 assert_eq!(decoded_kind, kind);
8094 assert_eq!(inner.as_bytes(), &[0, 1, 2, 255]);
8095 }
8096 }
8097 }
8098
8099 #[test]
8100 fn header_witness_is_stable_across_object_field_order() {
8101 let (left, left_len) =
8102 header_body_digest(&OrderedHeader(&[("beta", 2), ("alpha", 1)]), 1_024)
8103 .expect("left witness");
8104 let (right, right_len) =
8105 header_body_digest(&OrderedHeader(&[("alpha", 1), ("beta", 2)]), 1_024)
8106 .expect("right witness");
8107 let (changed, _) = header_body_digest(&OrderedHeader(&[("alpha", 1), ("beta", 3)]), 1_024)
8108 .expect("changed witness");
8109
8110 assert_eq!(left, right);
8111 assert_eq!(left_len, right_len);
8112 assert_ne!(left, changed);
8113 }
8114
8115 #[test]
8116 fn header_canonicalization_scratch_has_an_independent_hard_cap() {
8117 let oversized = "x".repeat(MAX_HEADER_CANONICALIZATION_BYTES + 1);
8118 let error = header_body_digest(&oversized, usize::MAX)
8119 .expect_err("canonicalization scratch must remain bounded");
8120 assert!(
8121 error
8122 .to_string()
8123 .contains("serialized block header exceeds")
8124 );
8125 assert!(
8126 error
8127 .to_string()
8128 .contains(&MAX_HEADER_CANONICALIZATION_BYTES.to_string())
8129 );
8130 }
8131
8132 #[test]
8133 fn checkpoint_rejects_corruption_truncation_and_unknown_versions() {
8134 let encoded = encode_hybrid_checkpoint(&checkpoint()).expect("encode checkpoint");
8135 assert_eq!(
8136 decode_hybrid_checkpoint(&encoded)
8137 .expect("decode checkpoint")
8138 .next_synthetic_token,
8139 7
8140 );
8141
8142 let mut corrupt = encoded.clone();
8143 *corrupt.last_mut().expect("payload") ^= 1;
8144 assert!(
8145 decode_hybrid_checkpoint(&corrupt)
8146 .expect_err("checksum mismatch")
8147 .to_string()
8148 .contains("checksum")
8149 );
8150
8151 assert!(
8152 decode_hybrid_checkpoint(&encoded[..encoded.len() - 1])
8153 .expect_err("truncated checkpoint")
8154 .to_string()
8155 .contains("payload length")
8156 );
8157
8158 let mut future = encoded;
8159 let mut legacy = future.clone();
8160 legacy[8..10].copy_from_slice(&4_u16.to_be_bytes());
8161 let legacy_error = decode_hybrid_checkpoint(&legacy).expect_err("legacy checkpoint");
8162 assert!(
8163 legacy_error.to_string().contains("version 4")
8164 && legacy_error.to_string().contains("resynchronization")
8165 );
8166
8167 future[8..10].copy_from_slice(&(CHECKPOINT_VERSION + 1).to_be_bytes());
8168 assert!(
8169 decode_hybrid_checkpoint(&future)
8170 .expect_err("future checkpoint")
8171 .to_string()
8172 .contains("unsupported hybrid checkpoint version")
8173 );
8174
8175 let mut semantically_invalid = checkpoint();
8176 semantically_invalid.next_synthetic_token = 0;
8177 assert!(
8178 encode_hybrid_checkpoint(&semantically_invalid)
8179 .expect_err("zero synthetic sequence")
8180 .to_string()
8181 .contains("synthetic token sequence")
8182 );
8183
8184 let mut trailing = encode_hybrid_checkpoint(&checkpoint()).expect("encode checkpoint");
8185 let payload_start = 18;
8186 trailing.push(0xff);
8187 let payload_len = trailing.len() - payload_start;
8188 trailing[10..14].copy_from_slice(&(payload_len as u32).to_be_bytes());
8189 let checksum = crc32fast::hash(&trailing[payload_start..]);
8190 trailing[14..18].copy_from_slice(&checksum.to_be_bytes());
8191 assert!(
8192 decode_hybrid_checkpoint(&trailing)
8193 .expect_err("trailing payload")
8194 .to_string()
8195 .contains("trailing bytes")
8196 );
8197
8198 let mut oversized = vec![0_u8; 18 + MAX_CHECKPOINT_BYTES + 1];
8199 oversized[..8].copy_from_slice(CHECKPOINT_MAGIC);
8200 oversized[8..10].copy_from_slice(&CHECKPOINT_VERSION.to_be_bytes());
8201 oversized[10..14].copy_from_slice(&((MAX_CHECKPOINT_BYTES + 1) as u32).to_be_bytes());
8202 assert!(
8203 decode_hybrid_checkpoint(&oversized)
8204 .expect_err("oversized checkpoint")
8205 .to_string()
8206 .contains("oversized")
8207 );
8208 }
8209
8210 #[test]
8211 fn checkpoint_encoding_stops_at_the_bound_before_materializing_oversized_payload() {
8212 const TEST_LIMIT: usize = 64;
8213 let state = checkpoint();
8214 let error = encode_hybrid_checkpoint_with_limit(&state, TEST_LIMIT)
8215 .expect_err("bounded production encoder must fail closed");
8216 assert!(error.to_string().contains("exceeds 64 bytes"));
8217 }
8218
8219 #[test]
8220 fn dense_owner_fanout_evicts_oldest_witnesses_but_protects_the_current_delivery() {
8221 let owner_a = HandlerId::new("owner-a");
8222 let owner_b = HandlerId::new("owner-b");
8223 let owners = BTreeMap::from([(owner_a.clone(), 3), (owner_b.clone(), 3)]);
8224 let mut state = checkpoint();
8225 state.owner_generations = owners.clone();
8226 state.lifecycle_intent.owners =
8227 BTreeMap::from([(owner_a, [0x11; 32]), (owner_b, [0x22; 32])]);
8228 state.recent_inputs = (0..3)
8229 .map(|block| stored_recent_input(block, owners.clone()))
8230 .collect();
8231 let head = BlockRef {
8232 number: 2,
8233 hash: B256::repeat_byte(2),
8234 parent_hash: None,
8235 timestamp: None,
8236 };
8237 state.canonical_history.push(head);
8238 state.coverage_head = Some(head);
8239
8240 fit_checkpoint_to_durable_limits(&mut state, 1, 10, 2)
8241 .expect("newest two-owner witness fits");
8242 assert_eq!(state.recent_inputs.len(), 1);
8243 assert_eq!(
8244 state.recent_inputs[0]
8245 .coverage
8246 .block
8247 .as_ref()
8248 .unwrap()
8249 .number,
8250 2
8251 );
8252
8253 let error = fit_checkpoint_to_durable_limits(&mut state, 1, 10, 1)
8254 .expect_err("protected fanout cannot be silently discarded");
8255 assert!(error.to_string().contains("protected recent witnesses"));
8256 }
8257
8258 #[test]
8259 fn saturated_capacity_preflight_encodes_the_real_maximum_commit_for_each_source() {
8260 let config = HybridConfig {
8261 max_source_delivery_token_bytes: 19,
8262 max_source_checkpoint_bytes: 23,
8263 canonical_history_capacity: 4,
8264 ..HybridConfig::default()
8265 };
8266 let live_mutations = Arc::new(AtomicUsize::new(0));
8267 let (history, live, _) = restore_counting_sources(live_mutations);
8268 let hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
8269 let mut state = checkpoint();
8270 state.lifecycle_intent.base = [0x41; 32];
8271 let head = golden_block(100, 0x64, 0x63);
8272 state.canonical_history = vec![head];
8273 state.coverage_head = Some(head);
8274 state.historical_position.canonical_history = vec![head];
8275 state.historical_position.coverage_head = Some(head);
8276 state.live_position.canonical_history = vec![head];
8277 state.live_position.coverage_head = Some(head);
8278
8279 hybrid
8280 .preflight_maximum_canonical_log_delivery(&state, "test checkpoint")
8281 .expect("both eligible source commits fit");
8282
8283 for source in [HybridSource::Historical, HybridSource::Live] {
8284 let encoded = hybrid
8285 .encode_maximum_canonical_log_delivery(&state, source)
8286 .expect("the exact simulated commit encodes");
8287 let committed =
8288 decode_hybrid_checkpoint(&encoded).expect("the saturated simulated commit decodes");
8289 assert_eq!(committed.chain_id, state.chain_id);
8290 assert_eq!(committed.epoch, state.epoch);
8291 assert_eq!(committed.next_synthetic_token, u64::MAX);
8292 assert_eq!(committed.lifecycle_generation, state.lifecycle_generation);
8293 assert_eq!(committed.owner_generations, state.owner_generations);
8294 assert_eq!(committed.lifecycle_intent, state.lifecycle_intent);
8295 assert_eq!(
8296 committed.canonical_history.len(),
8297 config.canonical_history_capacity
8298 );
8299 let block = committed
8300 .coverage_head
8301 .expect("maximum commit advances coverage");
8302 assert_eq!(block.number, u64::MAX);
8303 assert!(block.parent_hash.is_some());
8304 assert_eq!(block.timestamp, Some(u64::MAX));
8305 let record_block = block;
8306 assert_eq!(
8307 committed.canonical_history.first().unwrap().number,
8308 u64::MAX - 3
8309 );
8310 for pair in committed.canonical_history.windows(2) {
8311 assert_eq!(pair[1].number, pair[0].number + 1);
8312 assert_eq!(pair[1].parent_hash, Some(pair[0].hash));
8313 }
8314 assert!(
8315 committed
8316 .canonical_history
8317 .iter()
8318 .all(|retained| retained.timestamp == Some(u64::MAX))
8319 );
8320 assert_eq!(committed.safe_head, Some(block));
8321 assert_eq!(committed.finalized_head, Some(block));
8322
8323 for position in [&committed.historical_position, &committed.live_position] {
8324 assert_eq!(
8325 position
8326 .delivery_token
8327 .as_ref()
8328 .expect("reserved delivery token")
8329 .len(),
8330 config.max_source_delivery_token_bytes
8331 );
8332 assert_eq!(
8333 position
8334 .checkpoint
8335 .as_ref()
8336 .expect("reserved child checkpoint")
8337 .len(),
8338 config.max_source_checkpoint_bytes
8339 );
8340 assert!(position.delivery_digest.is_some());
8341 }
8342 let source_position = match source {
8343 HybridSource::Historical => &committed.historical_position,
8344 HybridSource::Live => &committed.live_position,
8345 };
8346 let other_position = match source {
8347 HybridSource::Historical => &committed.live_position,
8348 HybridSource::Live => &committed.historical_position,
8349 };
8350 assert_eq!(source_position.coverage_head, Some(block));
8351 assert_eq!(
8352 source_position.canonical_history,
8353 committed.canonical_history
8354 );
8355 assert_eq!(other_position.coverage_head, Some(block));
8356 assert_eq!(
8357 other_position.canonical_history,
8358 committed.canonical_history
8359 );
8360 let last = committed
8361 .last_committed_token
8362 .as_ref()
8363 .expect("forwarded last-commit proof");
8364 assert_eq!(last.source, source);
8365 assert_eq!(last.kind, HybridTokenKind::Forwarded);
8366 assert_eq!(
8367 last.inner,
8368 source_position
8369 .delivery_token
8370 .clone()
8371 .expect("matching source token")
8372 );
8373 assert_eq!(
8374 committed.certified_historical,
8375 if source == HybridSource::Historical {
8376 Some(CertifiedHistoricalCoverage {
8377 lifecycle_generation: state.lifecycle_generation,
8378 through: block,
8379 })
8380 } else {
8381 Some(CertifiedHistoricalCoverage {
8382 lifecycle_generation: state.lifecycle_generation,
8383 through: committed.canonical_history[config.canonical_history_capacity - 2],
8384 })
8385 }
8386 );
8387
8388 assert_eq!(committed.recent_inputs.len(), 1);
8389 let recent = committed
8390 .recent_inputs
8391 .last()
8392 .expect("protected canonical-log witness");
8393 assert!(recent.coverage.base);
8394 assert!(recent.coverage.owners.is_empty());
8395 assert_eq!(recent.coverage.block, Some(record_block));
8396 match recent.identity.input_ref() {
8397 InputRef::Log {
8398 chain_id,
8399 block_hash,
8400 log_index,
8401 ..
8402 } => {
8403 assert_eq!(chain_id, Some(state.chain_id));
8404 assert_eq!(block_hash, record_block.hash);
8405 assert_eq!(log_index, u64::MAX);
8406 }
8407 _ => panic!("maximum probe must use a canonical-log identity"),
8408 }
8409 assert_eq!(recent.identity.kind(), ReactiveInputKind::CanonicalLog);
8410 let witness = recent
8411 .coverage
8412 .witness
8413 .as_ref()
8414 .expect("complete maximum witness");
8415 assert_eq!(witness.lifecycle, WitnessLifecycle::Finalized);
8416 assert_eq!(witness.block, Some(record_block));
8417 assert_eq!(witness.transaction_index, Some(u64::MAX));
8418 assert_eq!(witness.log_index, Some(u64::MAX));
8419 assert_eq!(witness.log_block_timestamp, Some(u64::MAX));
8420 }
8421 }
8422
8423 #[test]
8424 fn saturated_capacity_preflight_covers_synthetic_and_single_byte_rlp_boundaries() {
8425 let config = HybridConfig {
8426 max_source_delivery_token_bytes: 1,
8427 max_source_checkpoint_bytes: 1,
8428 canonical_history_capacity: 3,
8429 ..HybridConfig::default()
8430 };
8431 let mut state = checkpoint();
8432 state.lifecycle_intent.base = [0x41; 32];
8433
8434 let high_byte_state =
8435 saturated_capacity_probe_state(&state, &config).expect("saturated cursor state");
8436 let mut low_byte_state = high_byte_state.clone();
8437 low_byte_state.historical_position.delivery_token = Some(vec![0x71]);
8438 low_byte_state.historical_position.checkpoint = Some(vec![0x72]);
8439 let high_byte_len =
8440 codec::checkpoint_payload_len_after_dropping_recent(&high_byte_state, 0)
8441 .expect("measure high-byte RLP payload");
8442 let low_byte_len = codec::checkpoint_payload_len_after_dropping_recent(&low_byte_state, 0)
8443 .expect("measure low-byte RLP payload");
8444 assert!(
8445 high_byte_len >= low_byte_len + 2,
8446 "two one-byte opaque vectors at or above 0x80 need RLP prefixes"
8447 );
8448
8449 let (history, live, _) = restore_counting_sources(Arc::new(AtomicUsize::new(0)));
8450 let hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
8451 hybrid
8452 .preflight_maximum_canonical_log_delivery(&state, "single-byte test checkpoint")
8453 .expect("all four source/token-kind probes fit");
8454
8455 for source in [HybridSource::Historical, HybridSource::Live] {
8456 for kind in [HybridTokenKind::Forwarded, HybridTokenKind::Synthetic] {
8457 let encoded = hybrid
8458 .encode_maximum_canonical_log_delivery_variant(&state, source, kind)
8459 .expect("saturated source/token-kind simulation encodes");
8460 let committed = decode_hybrid_checkpoint(&encoded)
8461 .expect("decode saturated source/token-kind simulation");
8462 assert_eq!(committed.next_synthetic_token, u64::MAX);
8463 assert_eq!(
8464 committed.canonical_history.len(),
8465 config.canonical_history_capacity
8466 );
8467 for position in [&committed.historical_position, &committed.live_position] {
8468 assert_eq!(position.canonical_history, committed.canonical_history);
8469 assert!(
8470 position
8471 .checkpoint
8472 .as_ref()
8473 .is_some_and(|bytes| bytes.len() == 1 && bytes[0] >= 0x80)
8474 );
8475 }
8476
8477 let committing = match source {
8478 HybridSource::Historical => &committed.historical_position,
8479 HybridSource::Live => &committed.live_position,
8480 };
8481 let other = match source {
8482 HybridSource::Historical => &committed.live_position,
8483 HybridSource::Live => &committed.historical_position,
8484 };
8485 assert!(
8486 other
8487 .delivery_token
8488 .as_ref()
8489 .is_some_and(|bytes| bytes.len() == 1 && bytes[0] >= 0x80)
8490 );
8491 let last = committed
8492 .last_committed_token
8493 .as_ref()
8494 .expect("last-commit proof");
8495 assert_eq!(last.source, source);
8496 assert_eq!(last.kind, kind);
8497 match kind {
8498 HybridTokenKind::Forwarded => {
8499 assert!(
8500 committing
8501 .delivery_token
8502 .as_ref()
8503 .is_some_and(|bytes| bytes.len() == 1 && bytes[0] >= 0x80)
8504 );
8505 assert_eq!(last.inner.len(), 1);
8506 }
8507 HybridTokenKind::Synthetic => {
8508 assert!(committing.delivery_token.is_none());
8509 assert_eq!(last.inner, (u64::MAX - 1).to_be_bytes());
8510 assert_eq!(last.inner.len(), 8);
8511 }
8512 }
8513 }
8514 }
8515 }
8516
8517 #[test]
8518 fn maximum_capacity_probe_reserves_every_admitted_canonical_history_entry() {
8519 let config = HybridConfig {
8520 max_buffered_live_bytes: 1_024,
8521 max_source_delivery_token_bytes: 19,
8522 max_source_checkpoint_bytes: 23,
8523 canonical_history_capacity: 3,
8524 ..HybridConfig::default()
8525 };
8526 let live_mutations = Arc::new(AtomicUsize::new(0));
8527 let (history, live, _) = restore_counting_sources(live_mutations);
8528 let hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
8529 let mut state = checkpoint();
8530 state.lifecycle_intent.base = [0x41; 32];
8531 let head = golden_block(100, 0x64, 0x63);
8532 state.canonical_history = vec![head];
8533 state.coverage_head = Some(head);
8534
8535 for source in [HybridSource::Historical, HybridSource::Live] {
8536 let encoded = hybrid
8537 .encode_maximum_canonical_log_delivery(&state, source)
8538 .expect("maximum admitted canonical sequence encodes");
8539 let committed =
8540 decode_hybrid_checkpoint(&encoded).expect("maximum canonical sequence decodes");
8541 assert_eq!(
8542 committed.canonical_history.len(),
8543 config.canonical_history_capacity
8544 );
8545 let source_position = match source {
8546 HybridSource::Historical => &committed.historical_position,
8547 HybridSource::Live => &committed.live_position,
8548 };
8549 assert_eq!(
8550 source_position.canonical_history.len(),
8551 config.canonical_history_capacity
8552 );
8553 for pair in committed.canonical_history.windows(2) {
8554 assert_eq!(pair[1].number, pair[0].number + 1);
8555 assert_eq!(pair[1].parent_hash, Some(pair[0].hash));
8556 }
8557 assert!(
8558 committed
8559 .canonical_history
8560 .iter()
8561 .all(|block| block.timestamp == Some(u64::MAX))
8562 );
8563 }
8564 }
8565
8566 #[test]
8567 fn capacity_probe_uses_the_same_derived_control_ceiling_as_ingress() {
8568 assert_eq!(maximum_ingress_control_count(255), 0);
8569 assert_eq!(maximum_ingress_control_count(256), 0);
8570 assert_eq!(maximum_ingress_control_count(511), 0);
8571 assert_eq!(maximum_ingress_control_count(512), 1);
8572 assert_eq!(maximum_ingress_control_count(1_024), 3);
8573
8574 let mut config = HybridConfig {
8575 max_buffered_live_bytes: 1_024,
8576 canonical_history_capacity: 8,
8577 ..HybridConfig::default()
8578 };
8579 assert_eq!(maximum_admitted_canonical_advances(&config), 4);
8580 config.canonical_history_capacity = 3;
8581 assert_eq!(maximum_admitted_canonical_advances(&config), 3);
8582 }
8583
8584 #[test]
8585 fn maximum_capacity_probe_handles_sparse_adjacent_and_terminal_heights() {
8586 let sparse_head = golden_block(100, 0x64, 0x63);
8587 let mut sparse = checkpoint();
8588 sparse.canonical_history = vec![sparse_head];
8589 sparse.coverage_head = Some(sparse_head);
8590 let sparse_probe =
8591 maximum_canonical_log_probe_block(&sparse).expect("construct sparse probe");
8592 assert_eq!(sparse_probe.number, u64::MAX);
8593 assert_ne!(sparse_probe.parent_hash, Some(sparse_head.hash));
8594
8595 let adjacent_head = golden_block(u64::MAX - 1, 0x71, 0x70);
8596 let mut adjacent = checkpoint();
8597 adjacent.canonical_history = vec![adjacent_head];
8598 adjacent.coverage_head = Some(adjacent_head);
8599 let adjacent_probe =
8600 maximum_canonical_log_probe_block(&adjacent).expect("construct adjacent probe");
8601 assert_eq!(adjacent_probe.number, u64::MAX);
8602 assert_eq!(adjacent_probe.parent_hash, Some(adjacent_head.hash));
8603
8604 let terminal_head = BlockRef {
8605 number: u64::MAX,
8606 hash: B256::repeat_byte(0x81),
8607 parent_hash: None,
8608 timestamp: None,
8609 };
8610 let mut terminal = checkpoint();
8611 terminal.canonical_history = vec![terminal_head];
8612 terminal.coverage_head = Some(terminal_head);
8613 let terminal_probe =
8614 maximum_canonical_log_probe_block(&terminal).expect("enrich terminal probe");
8615 assert_eq!(terminal_probe.number, u64::MAX);
8616 assert_eq!(terminal_probe.hash, terminal_head.hash);
8617 assert!(terminal_probe.parent_hash.is_some());
8618 assert_eq!(terminal_probe.timestamp, Some(u64::MAX));
8619 }
8620
8621 #[test]
8622 fn saturated_capacity_probe_subsumes_divergent_source_histories_for_both_sources() {
8623 let config = HybridConfig {
8624 max_source_delivery_token_bytes: 7,
8625 max_source_checkpoint_bytes: 11,
8626 canonical_history_capacity: 3,
8627 ..HybridConfig::default()
8628 };
8629 let coordinator_oldest = golden_block(0, 0x01, 0x00);
8630 let coordinator_tip = golden_block(u64::MAX, 0xf1, 0xf0);
8631 let historical_tip = golden_block(u64::MAX - 1, 0xe1, 0xe0);
8632 let live_history = vec![
8633 golden_block(0, 0x11, 0x10),
8634 golden_block(1, 0x12, 0x11),
8635 golden_block(2, 0x13, 0x12),
8636 ];
8637 let mut state = checkpoint();
8638 state.lifecycle_intent.base = [0x41; 32];
8639 state.canonical_history = vec![coordinator_oldest, coordinator_tip];
8640 state.coverage_head = Some(coordinator_tip);
8641 state.historical_position.canonical_history = vec![historical_tip];
8642 state.historical_position.coverage_head = Some(historical_tip);
8643 state.live_position.canonical_history = live_history.clone();
8644 state.live_position.coverage_head = live_history.last().copied();
8645 validate_checkpoint_state(&state).expect("divergent source candidate is valid");
8646
8647 let (history, live, _) = restore_counting_sources(Arc::new(AtomicUsize::new(0)));
8648 let hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
8649 for source in [HybridSource::Historical, HybridSource::Live] {
8650 let encoded = hybrid
8651 .encode_maximum_canonical_log_delivery(&state, source)
8652 .expect("saturated simulation succeeds");
8653 let committed =
8654 decode_hybrid_checkpoint(&encoded).expect("decode saturated simulation");
8655 let head = committed.coverage_head.expect("saturated head");
8656 assert_eq!(head.number, u64::MAX);
8657 for position in [&committed.historical_position, &committed.live_position] {
8658 assert_eq!(position.canonical_history, committed.canonical_history);
8659 assert_eq!(position.coverage_head, Some(head));
8660 }
8661 assert_eq!(
8662 committed.canonical_history.len(),
8663 config.canonical_history_capacity
8664 );
8665 }
8666 }
8667
8668 #[test]
8669 fn saturated_history_builder_populates_the_wire_maximum_in_one_pass() {
8670 let history = saturated_capacity_probe_history(HYBRID_MAX_CANONICAL_HISTORY)
8671 .expect("maximum retained history builds");
8672 assert_eq!(history.len(), HYBRID_MAX_CANONICAL_HISTORY);
8673 assert_eq!(
8674 history.first().expect("first saturated block").number,
8675 u64::MAX - HYBRID_MAX_CANONICAL_HISTORY as u64
8676 );
8677 assert_eq!(
8678 history.last().expect("last saturated block").number,
8679 u64::MAX - 1
8680 );
8681 assert_eq!(
8682 history
8683 .iter()
8684 .map(|block| block.hash)
8685 .collect::<HashSet<_>>()
8686 .len(),
8687 HYBRID_MAX_CANONICAL_HISTORY
8688 );
8689 assert!(
8690 history
8691 .iter()
8692 .all(|block| { block.parent_hash.is_some() && block.timestamp == Some(u64::MAX) })
8693 );
8694 assert!(history.windows(2).all(|pair| {
8695 pair[1].number == pair[0].number + 1 && pair[1].parent_hash == Some(pair[0].hash)
8696 }));
8697 }
8698
8699 #[test]
8700 fn maximum_owner_capacity_commit_persists_exact_installed_fanout() {
8701 let config = HybridConfig {
8702 max_source_delivery_token_bytes: 7,
8703 max_source_checkpoint_bytes: 11,
8704 ..HybridConfig::default()
8705 };
8706 let live_mutations = Arc::new(AtomicUsize::new(0));
8707 let (history, live, _) = restore_counting_sources(live_mutations);
8708 let hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
8709 let owner_a = HandlerId::new("capacity-owner-a");
8710 let owner_b = HandlerId::new("capacity-owner-b");
8711 let mut state = checkpoint();
8712 state.owner_generations = BTreeMap::from([(owner_a.clone(), 3), (owner_b.clone(), 2)]);
8713 state.lifecycle_intent.owners = BTreeMap::from([
8714 (owner_a.clone(), [0xa1; 32]),
8715 (owner_b.clone(), empty_interest_fingerprint()),
8716 ]);
8717
8718 let encoded = hybrid
8719 .encode_maximum_canonical_log_delivery(&state, HybridSource::Historical)
8720 .expect("maximum owner-fanout commit encodes");
8721 let committed = decode_hybrid_checkpoint(&encoded).expect("decode maximum owner fanout");
8722 let coverage = &committed
8723 .recent_inputs
8724 .last()
8725 .expect("protected owner witness")
8726 .coverage;
8727
8728 assert!(!coverage.base);
8729 assert_eq!(coverage.owners, state.owner_generations);
8730 }
8731
8732 #[test]
8733 fn checkpoint_rejects_mixed_base_and_owner_topology() {
8734 assert_eq!(
8735 empty_interest_fingerprint(),
8736 interest_fingerprint::<Ethereum>(&[]).expect("empty fingerprint")
8737 );
8738 let mut mixed = checkpoint();
8739 let owner = HandlerId::new("mixed-owner");
8740 mixed.lifecycle_intent.base = [0x41; 32];
8741 mixed
8742 .lifecycle_intent
8743 .owners
8744 .insert(owner.clone(), [0x42; 32]);
8745 mixed.owner_generations.insert(owner, 3);
8746 let error = encode_hybrid_checkpoint(&mixed)
8747 .expect_err("mixed topology cannot be encoded atomically");
8748 assert!(error.to_string().contains("base/unowned"));
8749 assert!(error.to_string().contains("owner-managed"));
8750 }
8751
8752 #[test]
8753 fn checkpoint_v5_encoding_has_a_stable_golden_digest() {
8754 let encoded = encode_hybrid_checkpoint(&checkpoint()).expect("encode checkpoint");
8755 assert_eq!(&encoded[..8], CHECKPOINT_MAGIC);
8756 assert_eq!(&encoded[8..10], &CHECKPOINT_VERSION.to_be_bytes());
8757 assert_eq!(
8758 keccak256(&encoded),
8759 B256::from([
8760 0x48, 0xc8, 0x96, 0x5b, 0x9d, 0x19, 0x8b, 0xf8, 0xe4, 0x8a, 0x32, 0xe0, 0x22, 0xa9,
8761 0x0f, 0x17, 0x39, 0x7a, 0xae, 0xcd, 0xb9, 0x97, 0x71, 0x80, 0x9c, 0x60, 0xb3, 0x7b,
8762 0x22, 0x18, 0x53, 0x86,
8763 ])
8764 );
8765 }
8766
8767 #[test]
8768 fn fully_populated_v5_checkpoints_match_permanent_wire_fixtures() {
8769 let cases = [
8770 (
8771 "forwarded historical",
8772 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded),
8773 include_str!("../testdata/hybrid_checkpoint_v5_forwarded_historical.hex"),
8774 ),
8775 (
8776 "synthetic live",
8777 fully_populated_checkpoint(HybridSource::Live, HybridTokenKind::Synthetic),
8778 include_str!("../testdata/hybrid_checkpoint_v5_synthetic_live.hex"),
8779 ),
8780 ];
8781
8782 for (label, expected_state, fixture) in cases {
8783 let encoded = encode_hybrid_checkpoint(&expected_state)
8784 .unwrap_or_else(|error| panic!("encode {label} checkpoint: {error}"));
8785 assert_eq!(checkpoint_hex(&encoded), fixture.trim(), "{label} bytes");
8786 let decoded = decode_hybrid_checkpoint(&encoded)
8787 .unwrap_or_else(|error| panic!("decode {label} checkpoint: {error}"));
8788 assert_eq!(decoded, expected_state, "{label} decoded state");
8789 assert_eq!(
8790 encode_hybrid_checkpoint(&decoded)
8791 .unwrap_or_else(|error| panic!("re-encode {label} checkpoint: {error}")),
8792 encoded,
8793 "{label} byte-for-byte re-encoding"
8794 );
8795 }
8796 }
8797
8798 #[tokio::test]
8799 async fn active_base_restore_rejects_a_checkpoint_without_one_record_headroom_before_mutation()
8800 {
8801 let interest = restore_test_interest();
8802 let mut state =
8803 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
8804 state.owner_generations.clear();
8805 state.lifecycle_intent = LifecycleIntent {
8806 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
8807 .expect("base fingerprint"),
8808 owners: BTreeMap::new(),
8809 };
8810 state.recent_inputs.clear();
8811 fill_opaque_checkpoint_to_v5_limit(&mut state);
8812 let original_recent = state.recent_inputs.clone();
8813 let position = restore_position_for(&state);
8814 let live_mutations = Arc::new(AtomicUsize::new(0));
8815 let (history, live, historical_mutations) =
8816 restore_counting_sources(Arc::clone(&live_mutations));
8817 let mut hybrid = HybridSubscriber::new(history, live, maximum_opaque_checkpoint_config())
8818 .expect("coordinator");
8819
8820 let error = hybrid
8821 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
8822 .await
8823 .expect_err("one protected base witness cannot fit");
8824
8825 assert!(
8826 error
8827 .to_string()
8828 .contains("cannot retain one protected delivery witness"),
8829 "unexpected capacity error: {error}"
8830 );
8831 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
8832 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
8833 assert_eq!(
8834 decode_hybrid_checkpoint(
8835 position
8836 .subscriber_checkpoint
8837 .as_ref()
8838 .expect("checkpoint")
8839 .as_bytes()
8840 )
8841 .expect("decode unchanged checkpoint")
8842 .recent_inputs,
8843 original_recent
8844 );
8845 }
8846
8847 #[tokio::test]
8848 async fn active_restore_rejects_when_only_the_legacy_pending_probe_would_fit() {
8849 let interest = restore_test_interest();
8850 let mut state =
8851 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
8852 state.owner_generations.clear();
8853 state.lifecycle_intent = LifecycleIntent {
8854 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
8855 .expect("base fingerprint"),
8856 owners: BTreeMap::new(),
8857 };
8858 state.recent_inputs.clear();
8859 fill_opaque_checkpoint_to_v5_limit(&mut state);
8860 let opaque = state
8861 .historical_position
8862 .checkpoint
8863 .as_mut()
8864 .expect("opaque historical checkpoint");
8865 opaque.truncate(opaque.len().saturating_sub(512));
8866 encode_hybrid_checkpoint(&state).expect("legacy-probe-only fixture remains valid");
8867 let mut legacy_simulation = state.clone();
8868 append_legacy_pending_capacity_probe(
8869 &mut legacy_simulation,
8870 state.chain_id,
8871 true,
8872 BTreeMap::new(),
8873 )
8874 .expect("construct legacy pending probe");
8875 fit_checkpoint_to_durable_limits(
8876 &mut legacy_simulation,
8877 1,
8878 HybridConfig::default().recent_input_capacity,
8879 HybridConfig::default().max_recent_owner_entries,
8880 )
8881 .expect("the legacy tiny pending probe fits this fixture");
8882 encode_hybrid_checkpoint(&legacy_simulation)
8883 .expect("the legacy tiny pending probe encodes");
8884 let position = restore_position_for(&state);
8885 let live_mutations = Arc::new(AtomicUsize::new(0));
8886 let (history, live, historical_mutations) =
8887 restore_counting_sources(Arc::clone(&live_mutations));
8888 let mut hybrid = HybridSubscriber::new(history, live, maximum_opaque_checkpoint_config())
8889 .expect("coordinator");
8890
8891 let error = hybrid
8892 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
8893 .await
8894 .expect_err(
8895 "the real maximum canonical-log commit must not inherit the legacy tiny-probe result",
8896 );
8897
8898 assert!(
8899 error.to_string().contains("maximum canonical-log delivery"),
8900 "unexpected capacity error: {error}"
8901 );
8902 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
8903 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
8904 }
8905
8906 #[tokio::test]
8907 async fn active_restore_rejects_when_one_advance_fits_but_admitted_history_does_not() {
8908 let interest = restore_test_interest();
8909 let head = golden_block(100, 0x64, 0x63);
8910 let mut state = checkpoint();
8911 state.lifecycle_intent = LifecycleIntent {
8912 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
8913 .expect("base fingerprint"),
8914 owners: BTreeMap::new(),
8915 };
8916 state.canonical_history = vec![head];
8917 state.coverage_head = Some(head);
8918 state.safe_head = Some(head);
8919 state.finalized_head = Some(head);
8920 state.certified_historical = Some(CertifiedHistoricalCoverage {
8921 lifecycle_generation: state.lifecycle_generation,
8922 through: head,
8923 });
8924 state.historical_position = SourcePosition {
8925 delivery_token: Some(vec![0x11]),
8926 checkpoint: Some(vec![0x12]),
8927 delivery_digest: Some([0x13; 32]),
8928 coverage_head: Some(head),
8929 canonical_history: vec![head],
8930 };
8931 state.live_position = SourcePosition {
8932 delivery_token: Some(vec![0x21]),
8933 checkpoint: Some(vec![0x22]),
8934 delivery_digest: Some([0x23; 32]),
8935 coverage_head: Some(head),
8936 canonical_history: vec![head],
8937 };
8938 state.last_committed_token = Some(StoredCommittedToken {
8939 source: HybridSource::Historical,
8940 kind: HybridTokenKind::Forwarded,
8941 inner: vec![0x11],
8942 });
8943
8944 let one_advance_config = HybridConfig {
8945 max_buffered_live_bytes: 1_024,
8946 max_source_delivery_token_bytes: 1,
8947 max_source_checkpoint_bytes: 1,
8948 canonical_history_capacity: 3,
8949 ..HybridConfig::default()
8950 };
8951 let (history, live, _) = restore_counting_sources(Arc::new(AtomicUsize::new(0)));
8952 let one_advance_hybrid =
8953 HybridSubscriber::new(history, live, one_advance_config).expect("coordinator");
8954 let encoded = one_advance_hybrid
8955 .encode_canonical_log_delivery_capacity_probe(&state, HybridSource::Historical, 1)
8956 .expect("legacy one-advance simulation fits");
8957 let mut one_advance =
8958 decode_hybrid_checkpoint(&encoded).expect("decode one-advance simulation");
8959 assert_eq!(one_advance.canonical_history.len(), 2);
8960
8961 let mut low = 1usize;
8962 let mut high = codec::MAX_CHECKPOINT_PAYLOAD_BYTES / 2;
8963 while low + 1 < high {
8964 let middle = low + (high - low) / 2;
8965 one_advance.historical_position.checkpoint = Some(vec![0x92; middle]);
8966 one_advance.live_position.checkpoint = Some(vec![0x82; middle]);
8967 if codec::checkpoint_payload_len_after_dropping_recent(&one_advance, 0)
8968 .expect("measure one-advance checkpoint")
8969 <= codec::MAX_CHECKPOINT_PAYLOAD_BYTES
8970 {
8971 low = middle;
8972 } else {
8973 high = middle;
8974 }
8975 }
8976 one_advance.historical_position.checkpoint = Some(vec![0x92; low]);
8977 one_advance.live_position.checkpoint = Some(vec![0x82; low]);
8978 encode_hybrid_checkpoint(&one_advance)
8979 .expect("legacy one-advance checkpoint fits exactly below the bound");
8980
8981 state.historical_position.checkpoint = Some(vec![0x12; low]);
8982 state.live_position.checkpoint = Some(vec![0x22; low]);
8983 let position = restore_position_for(&state);
8984 let historical_mutations = Arc::new(AtomicUsize::new(0));
8985 let live_mutations = Arc::new(AtomicUsize::new(0));
8986 let history = RestoreCountingSource {
8987 historical: true,
8988 durable: true,
8989 polls: Arc::new(AtomicUsize::new(0)),
8990 mutations: Arc::clone(&historical_mutations),
8991 };
8992 let live = RestoreCountingSource {
8993 historical: false,
8994 durable: true,
8995 polls: Arc::new(AtomicUsize::new(0)),
8996 mutations: Arc::clone(&live_mutations),
8997 };
8998 let config = HybridConfig {
8999 max_buffered_live_bytes: 1_024,
9000 max_source_delivery_token_bytes: 1,
9001 max_source_checkpoint_bytes: low,
9002 canonical_history_capacity: 3,
9003 ..HybridConfig::default()
9004 };
9005 let mut hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
9006
9007 let error = hybrid
9008 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
9009 .await
9010 .expect_err("the full admitted history must be reserved before child mutation");
9011
9012 assert!(
9013 error.to_string().contains("maximum canonical-log delivery"),
9014 "unexpected capacity error: {error}"
9015 );
9016 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9017 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9018 }
9019
9020 #[tokio::test]
9021 async fn terminal_restore_reserves_reorg_replacement_history_before_child_mutation() {
9022 let interest = restore_test_interest();
9023 let ancestor = golden_block(u64::MAX - 3, 0x91, 0x90);
9024 let old_tip = golden_block(u64::MAX, 0x94, 0x93);
9025 let mut state = checkpoint();
9026 state.lifecycle_intent = LifecycleIntent {
9027 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
9028 .expect("base fingerprint"),
9029 owners: BTreeMap::new(),
9030 };
9031 state.canonical_history = vec![ancestor, old_tip];
9032 state.coverage_head = Some(old_tip);
9033 state.safe_head = Some(ancestor);
9034 state.finalized_head = Some(ancestor);
9035 state.certified_historical = Some(CertifiedHistoricalCoverage {
9036 lifecycle_generation: state.lifecycle_generation,
9037 through: ancestor,
9038 });
9039 state.historical_position = SourcePosition {
9040 delivery_token: Some(vec![0x11]),
9041 checkpoint: Some(vec![0x12]),
9042 delivery_digest: Some([0x13; 32]),
9043 coverage_head: Some(old_tip),
9044 canonical_history: vec![ancestor, old_tip],
9045 };
9046 state.live_position = SourcePosition {
9047 delivery_token: Some(vec![0x21]),
9048 checkpoint: Some(vec![0x22]),
9049 delivery_digest: Some([0x23; 32]),
9050 coverage_head: Some(old_tip),
9051 canonical_history: vec![ancestor, old_tip],
9052 };
9053 state.last_committed_token = Some(StoredCommittedToken {
9054 source: HybridSource::Historical,
9055 kind: HybridTokenKind::Forwarded,
9056 inner: vec![0x11],
9057 });
9058
9059 let terminal_config = HybridConfig {
9060 max_buffered_live_bytes: 1_024,
9062 max_source_delivery_token_bytes: 1,
9063 max_source_checkpoint_bytes: 1,
9064 canonical_history_capacity: 4,
9065 ..HybridConfig::default()
9066 };
9067 let (history, live, _) = restore_counting_sources(Arc::new(AtomicUsize::new(0)));
9068 let terminal_hybrid =
9069 HybridSubscriber::new(history, live, terminal_config).expect("coordinator");
9070 let encoded = terminal_hybrid
9071 .encode_maximum_canonical_log_delivery(&state, HybridSource::Historical)
9072 .expect("terminal same-height enrichment fits");
9073 let mut terminal_enrichment =
9074 decode_hybrid_checkpoint(&encoded).expect("decode terminal enrichment");
9075 assert_eq!(
9076 terminal_enrichment.canonical_history.len(),
9077 terminal_config.canonical_history_capacity
9078 );
9079
9080 let mut low = 1usize;
9081 let mut high = codec::MAX_CHECKPOINT_PAYLOAD_BYTES / 2;
9082 while low + 1 < high {
9083 let middle = low + (high - low) / 2;
9084 terminal_enrichment.historical_position.checkpoint = Some(vec![0x92; middle]);
9085 terminal_enrichment.live_position.checkpoint = Some(vec![0x82; middle]);
9086 if codec::checkpoint_payload_len_after_dropping_recent(&terminal_enrichment, 0)
9087 .expect("measure terminal-enrichment checkpoint")
9088 <= codec::MAX_CHECKPOINT_PAYLOAD_BYTES
9089 {
9090 low = middle;
9091 } else {
9092 high = middle;
9093 }
9094 }
9095 terminal_enrichment.historical_position.checkpoint = Some(vec![0x92; low]);
9096 terminal_enrichment.live_position.checkpoint = Some(vec![0x82; low]);
9097 encode_hybrid_checkpoint(&terminal_enrichment)
9098 .expect("terminal enrichment fits exactly below the bound");
9099
9100 state.historical_position.checkpoint = Some(vec![0x12; low]);
9101 state.live_position.checkpoint = Some(vec![0x22; low]);
9102 let position = restore_position_for(&state);
9103 let historical_mutations = Arc::new(AtomicUsize::new(0));
9104 let live_mutations = Arc::new(AtomicUsize::new(0));
9105 let history = RestoreCountingSource {
9106 historical: true,
9107 durable: true,
9108 polls: Arc::new(AtomicUsize::new(0)),
9109 mutations: Arc::clone(&historical_mutations),
9110 };
9111 let live = RestoreCountingSource {
9112 historical: false,
9113 durable: true,
9114 polls: Arc::new(AtomicUsize::new(0)),
9115 mutations: Arc::clone(&live_mutations),
9116 };
9117 let config = HybridConfig {
9118 max_buffered_live_bytes: 1_024,
9119 max_source_delivery_token_bytes: 1,
9120 max_source_checkpoint_bytes: low,
9121 canonical_history_capacity: 4,
9122 ..HybridConfig::default()
9123 };
9124 let mut hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
9125
9126 let error = hybrid
9127 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
9128 .await
9129 .expect_err("terminal reorg replacement must be reserved before child mutation");
9130
9131 assert!(
9132 error.to_string().contains("maximum canonical-log delivery"),
9133 "unexpected terminal reorg capacity error: {error}"
9134 );
9135 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9136 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9137 }
9138
9139 #[tokio::test]
9140 async fn synthetic_capacity_boundary_rejects_restore_before_child_mutation() {
9141 let interest = restore_test_interest();
9142 let mut state =
9143 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9144 state.owner_generations.clear();
9145 state.lifecycle_intent = LifecycleIntent {
9146 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
9147 .expect("base fingerprint"),
9148 owners: BTreeMap::new(),
9149 };
9150 state.recent_inputs.clear();
9151 state.historical_position.delivery_token = Some(vec![0xe1]);
9152 state.historical_position.checkpoint = Some(vec![0xe2]);
9153 state.live_position.delivery_token = Some(vec![0x81]);
9154 state.live_position.checkpoint = Some(vec![0x82]);
9155 state.last_committed_token = Some(StoredCommittedToken {
9156 source: HybridSource::Historical,
9157 kind: HybridTokenKind::Forwarded,
9158 inner: vec![0xe1],
9159 });
9160 validate_checkpoint_state(&state).expect("single-byte restore candidate is valid");
9161
9162 let sizing_config = HybridConfig {
9163 max_source_delivery_token_bytes: 1,
9164 max_source_checkpoint_bytes: 1,
9165 canonical_history_capacity: 3,
9166 ..HybridConfig::default()
9167 };
9168 let (history, live, _) = restore_counting_sources(Arc::new(AtomicUsize::new(0)));
9169 let sizing_hybrid =
9170 HybridSubscriber::new(history, live, sizing_config).expect("sizing coordinator");
9171 let encoded = sizing_hybrid
9172 .encode_maximum_canonical_log_delivery_variant(
9173 &state,
9174 HybridSource::Historical,
9175 HybridTokenKind::Forwarded,
9176 )
9177 .expect("one-byte forwarded probe initially fits");
9178 let mut forwarded =
9179 decode_hybrid_checkpoint(&encoded).expect("decode forwarded capacity probe");
9180
9181 let mut low = 1usize;
9182 let mut high = codec::MAX_CHECKPOINT_PAYLOAD_BYTES / 2;
9183 while low + 1 < high {
9184 let middle = low + (high - low) / 2;
9185 forwarded
9186 .historical_position
9187 .checkpoint
9188 .as_mut()
9189 .expect("historical capacity-probe checkpoint")
9190 .resize(middle, 0xe2);
9191 forwarded
9192 .live_position
9193 .checkpoint
9194 .as_mut()
9195 .expect("live capacity-probe checkpoint")
9196 .resize(middle, 0x82);
9197 if codec::checkpoint_payload_len_after_dropping_recent(&forwarded, 0)
9198 .expect("measure forwarded boundary")
9199 <= codec::MAX_CHECKPOINT_PAYLOAD_BYTES
9200 {
9201 low = middle;
9202 } else {
9203 high = middle;
9204 }
9205 }
9206 drop(forwarded);
9207
9208 let boundary_config = HybridConfig {
9209 max_source_delivery_token_bytes: 1,
9210 max_source_checkpoint_bytes: low,
9211 canonical_history_capacity: 3,
9212 ..HybridConfig::default()
9213 };
9214 let (history, live, _) = restore_counting_sources(Arc::new(AtomicUsize::new(0)));
9215 let boundary_hybrid =
9216 HybridSubscriber::new(history, live, boundary_config).expect("boundary coordinator");
9217 boundary_hybrid
9218 .encode_maximum_canonical_log_delivery_variant(
9219 &state,
9220 HybridSource::Historical,
9221 HybridTokenKind::Forwarded,
9222 )
9223 .expect("maximum forwarded-token state still fits");
9224 boundary_hybrid
9225 .encode_maximum_canonical_log_delivery_variant(
9226 &state,
9227 HybridSource::Historical,
9228 HybridTokenKind::Synthetic,
9229 )
9230 .expect_err("larger synthetic-token state alone must exceed the envelope");
9231
9232 let position = restore_position_for(&state);
9233 let historical_mutations = Arc::new(AtomicUsize::new(0));
9234 let live_mutations = Arc::new(AtomicUsize::new(0));
9235 let history = RestoreCountingSource {
9236 historical: true,
9237 durable: true,
9238 polls: Arc::new(AtomicUsize::new(0)),
9239 mutations: Arc::clone(&historical_mutations),
9240 };
9241 let live = RestoreCountingSource {
9242 historical: false,
9243 durable: true,
9244 polls: Arc::new(AtomicUsize::new(0)),
9245 mutations: Arc::clone(&live_mutations),
9246 };
9247 let mut hybrid =
9248 HybridSubscriber::new(history, live, boundary_config).expect("restore coordinator");
9249 let error = hybrid
9250 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
9251 .await
9252 .expect_err("synthetic-token headroom must be proven before child mutation");
9253
9254 assert!(
9255 error.to_string().contains("synthetic-token"),
9256 "unexpected synthetic capacity error: {error}"
9257 );
9258 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9259 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9260 }
9261
9262 #[tokio::test]
9263 async fn owner_restore_rejects_a_checkpoint_without_one_record_headroom_before_mutation() {
9264 let interest = restore_test_interest();
9265 let owner = HandlerId::new("near-limit-owner");
9266 let mut state =
9267 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9268 state.owner_generations = BTreeMap::from([(owner.clone(), state.lifecycle_generation)]);
9269 state.lifecycle_intent = LifecycleIntent {
9270 base: empty_interest_fingerprint(),
9271 owners: BTreeMap::from([(
9272 owner.clone(),
9273 interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
9274 .expect("owner fingerprint"),
9275 )]),
9276 };
9277 state.recent_inputs.clear();
9278 fill_opaque_checkpoint_to_v5_limit(&mut state);
9279 let original_recent = state.recent_inputs.clone();
9280 let position = restore_position_for(&state);
9281 let live_mutations = Arc::new(AtomicUsize::new(0));
9282 let (history, live, historical_mutations) =
9283 restore_counting_sources(Arc::clone(&live_mutations));
9284 let mut hybrid = HybridSubscriber::new(history, live, maximum_opaque_checkpoint_config())
9285 .expect("coordinator");
9286
9287 let error = hybrid
9288 .prepare_restore_lifecycle(&position, &[], vec![(owner, vec![interest])])
9289 .await
9290 .expect_err("one protected owner witness cannot fit");
9291
9292 assert!(
9293 error
9294 .to_string()
9295 .contains("cannot retain one protected delivery witness"),
9296 "unexpected capacity error: {error}"
9297 );
9298 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9299 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9300 assert_eq!(
9301 decode_hybrid_checkpoint(
9302 position
9303 .subscriber_checkpoint
9304 .as_ref()
9305 .expect("checkpoint")
9306 .as_bytes()
9307 )
9308 .expect("decode unchanged checkpoint")
9309 .recent_inputs,
9310 original_recent
9311 );
9312 }
9313
9314 #[tokio::test]
9315 async fn restore_preflight_accounts_for_runtime_history_growth_before_live_mutation() {
9316 let interest = restore_test_interest();
9317 let mut state =
9318 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9319 state.owner_generations.clear();
9320 state.lifecycle_intent = LifecycleIntent {
9321 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
9322 .expect("base fingerprint"),
9323 owners: BTreeMap::new(),
9324 };
9325 state.recent_inputs.clear();
9326 state.canonical_history = vec![state.coverage_head.expect("fixture coverage")];
9327 fill_opaque_checkpoint_to_v5_limit(&mut state);
9328 let opaque = state
9329 .historical_position
9330 .checkpoint
9331 .as_mut()
9332 .expect("opaque historical checkpoint");
9333 opaque.truncate(opaque.len().saturating_sub(4_096));
9334 encode_hybrid_checkpoint(&state).expect("short-history fixture remains valid");
9335
9336 let mut position = restore_position_for(&state);
9337 position.canonical_history = (0..=102)
9338 .map(|number| {
9339 let mut block = golden_block(number, number as u8, number.saturating_sub(1) as u8);
9340 if number == 0 {
9341 block.parent_hash = None;
9342 }
9343 block
9344 })
9345 .collect();
9346 let live_mutations = Arc::new(AtomicUsize::new(0));
9347 let (history, live, historical_mutations) =
9348 restore_counting_sources(Arc::clone(&live_mutations));
9349 let mut hybrid = HybridSubscriber::new(history, live, maximum_opaque_checkpoint_config())
9350 .expect("coordinator");
9351
9352 let error = hybrid
9353 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
9354 .await
9355 .expect_err("installed runtime history must leave one-record durable headroom");
9356
9357 assert!(
9358 error.to_string().contains("protected delivery witness"),
9359 "unexpected capacity error: {error}"
9360 );
9361 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9362 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9363 }
9364
9365 #[tokio::test]
9366 async fn restore_preflight_discards_only_an_ephemeral_live_cursor_before_capacity_proof() {
9367 let interest = restore_test_interest();
9368 let mut state =
9369 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9370 state.owner_generations.clear();
9371 state.lifecycle_intent = LifecycleIntent {
9372 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
9373 .expect("base fingerprint"),
9374 owners: BTreeMap::new(),
9375 };
9376 state.recent_inputs.clear();
9377 fill_ephemeral_live_checkpoint_to_v5_limit(&mut state);
9378 let position = restore_position_for(&state);
9379 let live_mutations = Arc::new(AtomicUsize::new(0));
9380 let (history, live, historical_mutations) =
9381 restore_counting_sources(Arc::clone(&live_mutations));
9382 let mut hybrid =
9383 HybridSubscriber::new(history, live, HybridConfig::default()).expect("coordinator");
9384
9385 hybrid
9386 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
9387 .await
9388 .expect("discarded ephemeral cursor leaves delivery headroom");
9389 hybrid
9390 .restore_position(&position)
9391 .expect("restore normalized candidate");
9392
9393 let restored = hybrid.checkpoint_state();
9394 assert!(restored.live_position.delivery_token.is_none());
9395 assert!(restored.live_position.checkpoint.is_none());
9396 assert!(restored.live_position.delivery_digest.is_none());
9397 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9398 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 1);
9399 }
9400
9401 #[tokio::test]
9402 async fn restore_rejects_an_oversized_historical_cursor_before_child_mutation() {
9403 let interest = restore_test_interest();
9404 let mut state =
9405 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9406 state.owner_generations.clear();
9407 state.lifecycle_intent = LifecycleIntent {
9408 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
9409 .expect("base fingerprint"),
9410 owners: BTreeMap::new(),
9411 };
9412 state.recent_inputs.clear();
9413 state.historical_position.delivery_token = Some(vec![0x51; 33]);
9414 state.last_committed_token = Some(StoredCommittedToken {
9415 source: HybridSource::Historical,
9416 kind: HybridTokenKind::Forwarded,
9417 inner: vec![0x51; 33],
9418 });
9419 let position = restore_position_for(&state);
9420 let live_mutations = Arc::new(AtomicUsize::new(0));
9421 let (history, live, historical_mutations) =
9422 restore_counting_sources(Arc::clone(&live_mutations));
9423 let config = HybridConfig {
9424 max_source_delivery_token_bytes: 32,
9425 ..HybridConfig::default()
9426 };
9427 let mut hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
9428
9429 let error = hybrid
9430 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
9431 .await
9432 .expect_err("oversized historical token must fail pure restore validation");
9433
9434 assert!(error.to_string().contains("opaque cursor bound"));
9435 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9436 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9437 }
9438
9439 #[tokio::test]
9440 async fn restore_rejects_an_oversized_durable_live_cursor_before_child_mutation() {
9441 let interest = restore_test_interest();
9442 let mut state =
9443 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9444 state.owner_generations.clear();
9445 state.lifecycle_intent = LifecycleIntent {
9446 base: interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
9447 .expect("base fingerprint"),
9448 owners: BTreeMap::new(),
9449 };
9450 state.recent_inputs.clear();
9451 state.live_position.checkpoint = Some(vec![0x61; 33]);
9452 let position = restore_position_for(&state);
9453 let historical_mutations = Arc::new(AtomicUsize::new(0));
9454 let live_mutations = Arc::new(AtomicUsize::new(0));
9455 let history = RestoreCountingSource {
9456 historical: true,
9457 durable: true,
9458 polls: Arc::new(AtomicUsize::new(0)),
9459 mutations: Arc::clone(&historical_mutations),
9460 };
9461 let live = RestoreCountingSource {
9462 historical: false,
9463 durable: true,
9464 polls: Arc::new(AtomicUsize::new(0)),
9465 mutations: Arc::clone(&live_mutations),
9466 };
9467 let config = HybridConfig {
9468 max_source_checkpoint_bytes: 32,
9469 ..HybridConfig::default()
9470 };
9471 let mut hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
9472
9473 let error = hybrid
9474 .prepare_restore_base_lifecycle(&position, std::slice::from_ref(&interest))
9475 .await
9476 .expect_err("oversized durable-live checkpoint must fail pure restore validation");
9477
9478 assert!(error.to_string().contains("opaque cursor bound"));
9479 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9480 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9481 }
9482
9483 #[tokio::test]
9484 async fn restore_capacity_simulation_never_evicts_installed_recent_witnesses() {
9485 let interest = restore_test_interest();
9486 let owner = HandlerId::new("preserved-owner");
9487 let mut state =
9488 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9489 state.owner_generations = BTreeMap::from([(owner.clone(), state.lifecycle_generation)]);
9490 state.lifecycle_intent = LifecycleIntent {
9491 base: empty_interest_fingerprint(),
9492 owners: BTreeMap::from([(
9493 owner.clone(),
9494 interest_fingerprint::<Ethereum>(std::slice::from_ref(&interest))
9495 .expect("owner fingerprint"),
9496 )]),
9497 };
9498 for entry in &mut state.recent_inputs {
9499 entry.coverage.base = false;
9500 entry.coverage.owners = BTreeMap::from([(owner.clone(), state.lifecycle_generation)]);
9501 }
9502 let original_recent = state.recent_inputs.clone();
9503 let position = restore_position_for(&state);
9504 let live_mutations = Arc::new(AtomicUsize::new(0));
9505 let (history, live, _) = restore_counting_sources(Arc::clone(&live_mutations));
9506 let mut hybrid =
9507 HybridSubscriber::new(history, live, HybridConfig::default()).expect("coordinator");
9508
9509 hybrid
9510 .prepare_restore_lifecycle(&position, &[], vec![(owner, vec![interest])])
9511 .await
9512 .expect("prepare owner restore");
9513 hybrid
9514 .restore_position(&position)
9515 .expect("install prepared restore");
9516
9517 assert_eq!(hybrid.checkpoint_state().recent_inputs, original_recent);
9518 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 1);
9519 }
9520
9521 #[tokio::test]
9522 async fn near_limit_restored_effective_empty_owner_topology_does_not_poll_sources() {
9523 let owner = HandlerId::new("restored-empty-owner");
9524 let mut state =
9525 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9526 state.owner_generations = BTreeMap::from([(owner.clone(), state.lifecycle_generation)]);
9527 state.lifecycle_intent = LifecycleIntent {
9528 base: empty_interest_fingerprint(),
9529 owners: BTreeMap::from([(owner.clone(), empty_interest_fingerprint())]),
9530 };
9531 state.recent_inputs.clear();
9532 fill_opaque_checkpoint_to_v5_limit(&mut state);
9533 let checkpoint = SubscriberCheckpoint::new(
9534 encode_hybrid_checkpoint(&state).expect("encode empty-owner checkpoint"),
9535 );
9536 let token = wrap_token(
9537 state.epoch,
9538 HybridSource::Historical,
9539 HybridTokenKind::Forwarded,
9540 SubscriberDeliveryToken::new(vec![0xf1, 0xf2, 0xf3]),
9541 );
9542 let position = SubscriberResumePosition::new(
9543 state.chain_id,
9544 state.coverage_head.expect("golden coverage"),
9545 state.canonical_history.clone(),
9546 Some(token),
9547 Some(checkpoint),
9548 );
9549 let historical_polls = Arc::new(AtomicUsize::new(0));
9550 let live_polls = Arc::new(AtomicUsize::new(0));
9551 let historical_mutations = Arc::new(AtomicUsize::new(0));
9552 let live_mutations = Arc::new(AtomicUsize::new(0));
9553 let history = RestoreCountingSource {
9554 historical: true,
9555 durable: true,
9556 polls: Arc::clone(&historical_polls),
9557 mutations: Arc::clone(&historical_mutations),
9558 };
9559 let live = RestoreCountingSource {
9560 historical: false,
9561 durable: false,
9562 polls: Arc::clone(&live_polls),
9563 mutations: Arc::clone(&live_mutations),
9564 };
9565 let mut hybrid = HybridSubscriber::new(history, live, maximum_opaque_checkpoint_config())
9566 .expect("coordinator");
9567
9568 hybrid
9569 .prepare_restore_lifecycle(&position, &[], vec![(owner, Vec::new())])
9570 .await
9571 .expect("prepare empty owner topology");
9572 hybrid
9573 .restore_position(&position)
9574 .expect("restore empty owner topology");
9575 let delivery = hybrid.next_batch().await.expect("poll restored topology");
9576
9577 assert!(delivery.is_none());
9578 assert_eq!(historical_polls.load(AtomicOrdering::SeqCst), 0);
9579 assert_eq!(live_polls.load(AtomicOrdering::SeqCst), 0);
9580 assert_eq!(hybrid.phase(), HybridPhase::Live);
9581
9582 let historical_before = historical_mutations.load(AtomicOrdering::SeqCst);
9583 let live_before = live_mutations.load(AtomicOrdering::SeqCst);
9584 let error = hybrid
9585 .add_interest_owner(
9586 HandlerId::new("activated-after-empty-restore"),
9587 &[restore_test_interest()],
9588 )
9589 .await
9590 .expect_err("activation must re-prove exact preserved checkpoint headroom");
9591 assert!(
9592 error.to_string().contains("one protected delivery witness"),
9593 "unexpected activation capacity error: {error}"
9594 );
9595 assert_eq!(
9596 historical_mutations.load(AtomicOrdering::SeqCst),
9597 historical_before
9598 );
9599 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), live_before);
9600 }
9601
9602 #[tokio::test]
9603 async fn base_destructive_and_incremental_capacity_failures_precede_child_mutation() {
9604 let historical_mutations = Arc::new(AtomicUsize::new(0));
9605 let live_mutations = Arc::new(AtomicUsize::new(0));
9606 let history = RestoreCountingSource {
9607 historical: true,
9608 durable: true,
9609 polls: Arc::new(AtomicUsize::new(0)),
9610 mutations: Arc::clone(&historical_mutations),
9611 };
9612 let live = RestoreCountingSource {
9613 historical: false,
9614 durable: false,
9615 polls: Arc::new(AtomicUsize::new(0)),
9616 mutations: Arc::clone(&live_mutations),
9617 };
9618 let config = HybridConfig {
9619 max_source_checkpoint_bytes: HYBRID_MAX_SOURCE_CHECKPOINT_BYTES,
9620 ..HybridConfig::default()
9621 };
9622 let mut hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
9623
9624 let error = hybrid
9625 .register_interests(&[restore_test_interest()])
9626 .await
9627 .expect_err("effective-empty base activation must preflight maximum cursor reserve");
9628
9629 assert!(
9630 error.to_string().contains("maximum canonical-log delivery"),
9631 "unexpected base preflight error: {error}"
9632 );
9633 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9634 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9635 assert!(hybrid.base_interests.is_empty());
9636
9637 for destructive in [true, false] {
9638 let historical_mutations = Arc::new(AtomicUsize::new(0));
9639 let live_mutations = Arc::new(AtomicUsize::new(0));
9640 let history = RestoreCountingSource {
9641 historical: true,
9642 durable: true,
9643 polls: Arc::new(AtomicUsize::new(0)),
9644 mutations: Arc::clone(&historical_mutations),
9645 };
9646 let live = RestoreCountingSource {
9647 historical: false,
9648 durable: false,
9649 polls: Arc::new(AtomicUsize::new(0)),
9650 mutations: Arc::clone(&live_mutations),
9651 };
9652 let config = HybridConfig {
9653 max_source_checkpoint_bytes: HYBRID_MAX_SOURCE_CHECKPOINT_BYTES,
9654 ..HybridConfig::default()
9655 };
9656 let mut hybrid = HybridSubscriber::new(history, live, config).expect("coordinator");
9657 let owner = HandlerId::new(if destructive {
9658 "destructive-capacity-owner"
9659 } else {
9660 "incremental-capacity-owner"
9661 });
9662 let interest = restore_test_interest();
9663
9664 let error = if destructive {
9665 hybrid
9666 .replace_interest_owners(vec![(owner, vec![interest])])
9667 .await
9668 .expect_err("destructive replacement must preflight maximum cursor reserve")
9669 } else {
9670 hybrid
9671 .add_interest_owner(owner, &[interest])
9672 .await
9673 .expect_err("incremental change must preflight maximum cursor reserve")
9674 };
9675
9676 assert!(
9677 error.to_string().contains("maximum canonical-log delivery"),
9678 "unexpected topology preflight error: {error}"
9679 );
9680 assert_eq!(historical_mutations.load(AtomicOrdering::SeqCst), 0);
9681 assert_eq!(live_mutations.load(AtomicOrdering::SeqCst), 0);
9682 assert!(hybrid.owners.is_empty());
9683 }
9684 }
9685
9686 #[test]
9687 fn empty_owner_topology_still_requires_restore_preparation() {
9688 let intent = LifecycleIntent {
9689 base: empty_interest_fingerprint(),
9690 owners: BTreeMap::from([(HandlerId::new("empty-owner"), empty_interest_fingerprint())]),
9691 };
9692
9693 assert!(!intent.has_active_interests());
9694 assert!(intent.requires_restore_preparation());
9695 }
9696
9697 #[test]
9698 fn older_historical_page_cannot_regress_certified_coverage() {
9699 let mut state =
9700 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9701 let higher = state
9702 .certified_historical
9703 .expect("fixture carries higher historical proof");
9704 let lower = state.canonical_history[1];
9705 let commit = PendingCoordinatorCommit {
9706 audiences: Vec::new(),
9707 canonical: Vec::new(),
9708 source: HybridSource::Historical,
9709 source_token: Some(vec![0x11]),
9710 source_checkpoint: Some(vec![0x22]),
9711 source_progress: Some(lower),
9712 source_observed_through: Some(lower),
9713 token_kind: HybridTokenKind::Forwarded,
9714 token_bytes: vec![0x11],
9715 source_delivery_digest: [0x33; 32],
9716 next_safe_head: state.safe_head,
9717 next_finalized_head: state.finalized_head,
9718 next_canonical_history: state.canonical_history.clone(),
9719 next_coverage_head: state.coverage_head,
9720 };
9721
9722 apply_commit_to_checkpoint(&mut state, &commit, 32).expect("apply older page");
9723 let encoded = encode_hybrid_checkpoint(&state).expect("persist monotonic proof");
9724 let restored = decode_hybrid_checkpoint(&encoded).expect("restore monotonic proof");
9725
9726 assert_eq!(restored.certified_historical, Some(higher));
9727 }
9728
9729 #[test]
9730 fn same_height_historical_proof_merges_compatible_metadata() {
9731 let mut state =
9732 fully_populated_checkpoint(HybridSource::Historical, HybridTokenKind::Forwarded);
9733 let through = state.coverage_head.expect("fixture coverage");
9734 state.certified_historical = Some(CertifiedHistoricalCoverage {
9735 lifecycle_generation: state.lifecycle_generation,
9736 through: BlockRef {
9737 parent_hash: None,
9738 timestamp: None,
9739 ..through
9740 },
9741 });
9742 let commit = PendingCoordinatorCommit {
9743 audiences: Vec::new(),
9744 canonical: Vec::new(),
9745 source: HybridSource::Historical,
9746 source_token: Some(vec![0x12]),
9747 source_checkpoint: Some(vec![0x23]),
9748 source_progress: Some(through),
9749 source_observed_through: Some(through),
9750 token_kind: HybridTokenKind::Forwarded,
9751 token_bytes: vec![0x12],
9752 source_delivery_digest: [0x34; 32],
9753 next_safe_head: state.safe_head,
9754 next_finalized_head: state.finalized_head,
9755 next_canonical_history: state.canonical_history.clone(),
9756 next_coverage_head: state.coverage_head,
9757 };
9758
9759 apply_commit_to_checkpoint(&mut state, &commit, 32).expect("merge proof metadata");
9760
9761 assert_eq!(
9762 state.certified_historical,
9763 Some(CertifiedHistoricalCoverage {
9764 lifecycle_generation: state.lifecycle_generation,
9765 through,
9766 })
9767 );
9768 encode_hybrid_checkpoint(&state).expect("merged proof remains durably valid");
9769 }
9770
9771 #[test]
9772 fn same_height_historical_proof_rejects_conflicting_metadata() {
9773 let partial = BlockRef {
9774 number: 10,
9775 hash: B256::repeat_byte(0xaa),
9776 parent_hash: None,
9777 timestamp: None,
9778 };
9779 let certified = BlockRef {
9780 timestamp: Some(100),
9781 ..partial
9782 };
9783 let conflicting = BlockRef {
9784 timestamp: Some(101),
9785 ..partial
9786 };
9787 let mut state = checkpoint();
9788 state.canonical_history = vec![partial];
9789 state.coverage_head = Some(partial);
9790 state.historical_position = SourcePosition {
9791 coverage_head: Some(partial),
9792 canonical_history: vec![partial],
9793 ..SourcePosition::default()
9794 };
9795 state.certified_historical = Some(CertifiedHistoricalCoverage {
9796 lifecycle_generation: state.lifecycle_generation,
9797 through: certified,
9798 });
9799 validate_checkpoint_state(&state).expect("partial checkpoint is valid");
9800 let commit = PendingCoordinatorCommit {
9801 audiences: Vec::new(),
9802 canonical: Vec::new(),
9803 source: HybridSource::Historical,
9804 source_token: None,
9805 source_checkpoint: None,
9806 source_progress: Some(conflicting),
9807 source_observed_through: Some(conflicting),
9808 token_kind: HybridTokenKind::Synthetic,
9809 token_bytes: 1_u64.to_be_bytes().to_vec(),
9810 source_delivery_digest: [0x35; 32],
9811 next_safe_head: None,
9812 next_finalized_head: None,
9813 next_canonical_history: vec![conflicting],
9814 next_coverage_head: Some(conflicting),
9815 };
9816
9817 let error = apply_commit_to_checkpoint(&mut state, &commit, 32)
9818 .expect_err("conflicting proof metadata must fail closed");
9819
9820 assert!(error.to_string().contains("timestamp conflicts"));
9821 }
9822
9823 #[test]
9824 fn default_recent_window_fits_the_durable_checkpoint_envelope_for_one_owner() {
9825 let config = HybridConfig::default();
9826 let owner = HandlerId::new("representative-owner");
9827 let mut state = checkpoint();
9828 state.owner_generations.insert(owner.clone(), 3);
9829 state.lifecycle_intent.owners.insert(
9830 owner.clone(),
9831 interest_fingerprint::<Ethereum>(&[]).expect("empty owner interests"),
9832 );
9833 state.recent_inputs = (0..config.recent_input_capacity)
9834 .map(|index| {
9835 let block = BlockRef {
9836 number: index as u64,
9837 hash: B256::repeat_byte(index as u8),
9838 parent_hash: None,
9839 timestamp: None,
9840 };
9841 let record = ReactiveInputRecord::<Ethereum>::new(
9842 ReactiveInput::Log(Log {
9843 inner: PrimitiveLog::new_unchecked(
9844 Address::repeat_byte(0xaa),
9845 vec![B256::repeat_byte(0xbb)],
9846 Bytes::new(),
9847 ),
9848 block_hash: Some(block.hash),
9849 block_number: Some(block.number),
9850 block_timestamp: None,
9851 transaction_hash: Some(B256::repeat_byte(index.wrapping_add(1) as u8)),
9852 transaction_index: Some(0),
9853 log_index: Some(index as u64),
9854 removed: false,
9855 }),
9856 evm_fork_cache::reactive::ReactiveContext {
9857 chain_id: Some(1),
9858 source: evm_fork_cache::reactive::InputSource::Backfill,
9859 chain_status: ChainStatus::Included {
9860 block,
9861 confirmations: 0,
9862 },
9863 block: Some(block),
9864 transaction_index: Some(0),
9865 log_index: Some(index as u64),
9866 },
9867 );
9868 StoredRecentInput {
9869 identity: record.validated_identity().expect("valid test identity"),
9870 coverage: AudienceCoverage {
9871 base: false,
9872 owners: BTreeMap::from([(owner.clone(), 3)]),
9873 block: Some(block),
9874 witness: Some(record_witness(&record, 1).expect("test witness")),
9875 },
9876 }
9877 })
9878 .collect();
9879 let head = BlockRef {
9880 number: config.recent_input_capacity.saturating_sub(1) as u64,
9881 hash: B256::repeat_byte(config.recent_input_capacity.saturating_sub(1) as u8),
9882 parent_hash: None,
9883 timestamp: None,
9884 };
9885 state.canonical_history.push(head);
9886 state.coverage_head = Some(head);
9887
9888 let encoded =
9889 encode_hybrid_checkpoint(&state).expect("default journal must remain checkpointable");
9890 decode_hybrid_checkpoint(&encoded)
9891 .expect("the bounded decoder must accept every default-sized encoded journal");
9892 }
9893}