1use std::{
2 collections::HashSet,
3 pin::Pin,
4 sync::{
5 Arc, Mutex as StdMutex,
6 atomic::{AtomicU64, Ordering},
7 },
8 time::Duration,
9};
10
11use async_trait::async_trait;
12use evm_fork_cache_event_protocol::{
13 MAX_MESSAGE_SIZE_BYTES, PROTOCOL_VERSION,
14 v1::{
15 Acknowledge, AcknowledgementCommitted, ApplyDesiredState, BlockRef, ClientMessage, Cursor,
16 Delivery, ErrorCode, Hello, HelloAccepted, PendingDeliveryResume, ProtocolError,
17 RuntimeCheckpointPosition, ServerMessage, ServiceLimits, SourceCapabilities,
18 client_message,
19 event_stream_server::{EventStream, EventStreamServer},
20 portable_interest, server_message,
21 },
22};
23use futures::{Stream, StreamExt};
24use tokio::sync::{Mutex, mpsc, watch};
25use tokio_stream::wrappers::ReceiverStream;
26use tonic::{Request, Response, Status, metadata::MetadataMap};
27
28use prost::Message;
29
30use crate::{
31 DesiredStateError, PersistedSession, SessionStore, SessionStoreError, validate_desired_state,
32};
33
34#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
36pub struct PreparationId(u64);
37
38impl PreparationId {
39 const fn new(value: u64) -> Self {
40 Self(value)
41 }
42}
43
44#[derive(Clone, Copy, Debug)]
50#[non_exhaustive]
51pub struct DeliveryRequest<'a> {
52 desired_state: &'a ApplyDesiredState,
53 acknowledged_cursor: Option<&'a Cursor>,
54 required_reorg_anchor: Option<&'a BlockRef>,
55}
56
57impl<'a> DeliveryRequest<'a> {
58 pub const fn new(
60 desired_state: &'a ApplyDesiredState,
61 acknowledged_cursor: Option<&'a Cursor>,
62 ) -> Self {
63 Self {
64 desired_state,
65 acknowledged_cursor,
66 required_reorg_anchor: None,
67 }
68 }
69
70 #[must_use]
72 pub const fn with_required_reorg_anchor(mut self, anchor: Option<&'a BlockRef>) -> Self {
73 self.required_reorg_anchor = anchor;
74 self
75 }
76
77 pub const fn desired_state(self) -> &'a ApplyDesiredState {
79 self.desired_state
80 }
81
82 pub const fn acknowledged_cursor(self) -> Option<&'a Cursor> {
84 self.acknowledged_cursor
85 }
86
87 pub const fn required_reorg_anchor(self) -> Option<&'a BlockRef> {
95 self.required_reorg_anchor
96 }
97}
98
99#[async_trait]
111pub trait EventSource: Send + Sync + 'static {
112 fn capabilities(&self) -> SourceCapabilities;
114
115 fn capabilities_for_chain(&self, _chain_id: u64) -> SourceCapabilities {
118 self.capabilities()
119 }
120
121 async fn prepare_desired_state(
139 &self,
140 _preparation_id: PreparationId,
141 _desired_state: &ApplyDesiredState,
142 _acknowledged_cursor: Option<&Cursor>,
143 ) -> Result<Option<Cursor>, EventSourceError> {
144 Ok(None)
145 }
146
147 async fn activate_desired_state(
159 &self,
160 _preparation_id: PreparationId,
161 _desired_state: &ApplyDesiredState,
162 _acknowledged_cursor: Option<&Cursor>,
163 ) -> Result<(), EventSourceError> {
164 Ok(())
165 }
166
167 async fn abort_desired_state(
177 &self,
178 _preparation_id: PreparationId,
179 _desired_state: &ApplyDesiredState,
180 ) -> Result<(), EventSourceError> {
181 Ok(())
182 }
183
184 async fn release_session(
199 &self,
200 _session_id: &str,
201 _chain_id: u64,
202 ) -> Result<(), EventSourceError> {
203 Ok(())
204 }
205
206 async fn next_delivery(
223 &self,
224 request: DeliveryRequest<'_>,
225 ) -> Result<Option<Delivery>, EventSourceError>;
226
227 async fn wait_for_update(&self, _request: DeliveryRequest<'_>) -> Result<(), EventSourceError> {
242 std::future::pending().await
243 }
244
245 async fn acknowledge(
261 &self,
262 chain_id: u64,
263 acknowledgement: &Acknowledge,
264 committed_cursor: &Cursor,
265 ) -> Result<(), EventSourceError>;
266}
267
268#[derive(Clone, Copy, Debug, PartialEq, Eq)]
271#[non_exhaustive]
272pub enum EventSourceErrorKind {
273 InvalidRequest,
275 Unsupported,
277 ResourceExhausted,
279 Unavailable,
281 Internal,
283}
284
285#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
287#[error("event source failed ({kind:?}): {message}")]
288#[non_exhaustive]
289pub struct EventSourceError {
290 pub kind: EventSourceErrorKind,
292 pub message: String,
294}
295
296impl EventSourceError {
297 pub fn invalid(message: impl Into<String>) -> Self {
299 Self {
300 kind: EventSourceErrorKind::InvalidRequest,
301 message: message.into(),
302 }
303 }
304
305 pub fn unsupported(message: impl Into<String>) -> Self {
307 Self {
308 kind: EventSourceErrorKind::Unsupported,
309 message: message.into(),
310 }
311 }
312
313 pub fn resource_exhausted(message: impl Into<String>) -> Self {
315 Self {
316 kind: EventSourceErrorKind::ResourceExhausted,
317 message: message.into(),
318 }
319 }
320
321 pub fn unavailable(message: impl Into<String>) -> Self {
323 Self {
324 kind: EventSourceErrorKind::Unavailable,
325 message: message.into(),
326 }
327 }
328
329 pub fn internal(message: impl Into<String>) -> Self {
331 Self {
332 kind: EventSourceErrorKind::Internal,
333 message: message.into(),
334 }
335 }
336}
337
338pub struct EventService<P> {
340 store: Arc<Mutex<SessionStore>>,
341 provider: Arc<P>,
342 poll_interval: Duration,
343 client_hello_timeout: Duration,
344 source_operation_timeout: Duration,
345 client_send_timeout: Duration,
346 next_preparation_id: Arc<AtomicU64>,
347 active_sessions: Arc<StdMutex<HashSet<(String, u64)>>>,
348 limits: EventServiceLimits,
349 authorizer: Arc<dyn SessionAuthorizer>,
350 metrics: EventServiceMetrics,
351 shutdown: watch::Sender<bool>,
352}
353
354#[derive(Clone)]
356pub struct EventServiceShutdown {
357 sender: watch::Sender<bool>,
358}
359
360impl EventServiceShutdown {
361 pub fn shutdown(&self) {
363 self.sender.send_replace(true);
364 }
365}
366
367#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
369#[non_exhaustive]
370pub enum EventServiceConfigError {
371 #[error("event-service poll interval must be greater than zero")]
373 ZeroPollInterval,
374 #[error("event-service source operation timeout must be greater than zero")]
375 ZeroSourceOperationTimeout,
377 #[error("event-service client Hello timeout must be greater than zero")]
378 ZeroClientHelloTimeout,
380 #[error("event-service client send timeout must be greater than zero")]
381 ZeroClientSendTimeout,
383}
384
385#[derive(Clone, Default)]
387pub struct EventServiceMetrics {
388 inner: Arc<EventServiceMetricCounters>,
389}
390
391#[derive(Default)]
392struct EventServiceMetricCounters {
393 active_sessions: AtomicU64,
394 sessions_accepted: AtomicU64,
395 authentication_rejections: AtomicU64,
396 lease_rejections: AtomicU64,
397 desired_states_committed: AtomicU64,
398 deliveries_persisted: AtomicU64,
399 deliveries_replayed: AtomicU64,
400 acknowledgements_committed: AtomicU64,
401 source_errors: AtomicU64,
402}
403
404#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
406#[non_exhaustive]
407pub struct EventServiceMetricsSnapshot {
408 pub active_sessions: u64,
410 pub sessions_accepted: u64,
412 pub authentication_rejections: u64,
414 pub lease_rejections: u64,
416 pub desired_states_committed: u64,
418 pub deliveries_persisted: u64,
420 pub deliveries_replayed: u64,
422 pub acknowledgements_committed: u64,
424 pub source_errors: u64,
426}
427
428impl EventServiceMetrics {
429 pub fn snapshot(&self) -> EventServiceMetricsSnapshot {
431 EventServiceMetricsSnapshot {
432 active_sessions: self.inner.active_sessions.load(Ordering::Relaxed),
433 sessions_accepted: self.inner.sessions_accepted.load(Ordering::Relaxed),
434 authentication_rejections: self.inner.authentication_rejections.load(Ordering::Relaxed),
435 lease_rejections: self.inner.lease_rejections.load(Ordering::Relaxed),
436 desired_states_committed: self.inner.desired_states_committed.load(Ordering::Relaxed),
437 deliveries_persisted: self.inner.deliveries_persisted.load(Ordering::Relaxed),
438 deliveries_replayed: self.inner.deliveries_replayed.load(Ordering::Relaxed),
439 acknowledgements_committed: self
440 .inner
441 .acknowledgements_committed
442 .load(Ordering::Relaxed),
443 source_errors: self.inner.source_errors.load(Ordering::Relaxed),
444 }
445 }
446}
447
448pub trait SessionAuthorizer: Send + Sync + 'static {
450 fn authorize(&self, metadata: &MetadataMap) -> Result<(), String>;
458
459 fn authorize_session(
468 &self,
469 metadata: &MetadataMap,
470 _session_id: &str,
471 _chain_id: u64,
472 ) -> Result<(), String> {
473 self.authorize(metadata)
474 }
475}
476
477struct AllowAll;
478
479impl SessionAuthorizer for AllowAll {
480 fn authorize(&self, _metadata: &MetadataMap) -> Result<(), String> {
481 Ok(())
482 }
483}
484
485#[derive(Clone, Copy, Debug, PartialEq, Eq)]
487#[non_exhaustive]
488pub struct EventServiceLimits {
489 pub max_owners: usize,
491 pub max_interests_per_owner: usize,
493 pub max_addresses_per_log_interest: usize,
495 pub max_values_per_topic: usize,
497 pub max_bounded_backfill_blocks: u64,
499 pub max_identifier_bytes: usize,
501 pub max_total_interests: usize,
503 pub max_total_filter_values: usize,
505 pub max_desired_state_bytes: usize,
507 pub max_delivery_bytes: usize,
509 pub max_active_sessions: usize,
511 pub max_persisted_sessions: usize,
513}
514
515impl Default for EventServiceLimits {
516 fn default() -> Self {
517 Self {
518 max_owners: 4_096,
519 max_interests_per_owner: 256,
520 max_addresses_per_log_interest: 4_096,
521 max_values_per_topic: 1_024,
522 max_bounded_backfill_blocks: 10_000_000,
523 max_identifier_bytes: 256,
524 max_total_interests: 16_384,
525 max_total_filter_values: 1_000_000,
526 max_desired_state_bytes: 16 * 1024 * 1024,
527 max_delivery_bytes: MAX_MESSAGE_SIZE_BYTES,
528 max_active_sessions: 4_096,
529 max_persisted_sessions: 65_536,
530 }
531 }
532}
533
534impl EventServiceLimits {
535 fn wire(self) -> ServiceLimits {
536 ServiceLimits {
537 max_owners: self.max_owners.min(u32::MAX as usize) as u32,
538 max_interests_per_owner: self.max_interests_per_owner.min(u32::MAX as usize) as u32,
539 max_addresses_per_log_interest: self
540 .max_addresses_per_log_interest
541 .min(u32::MAX as usize) as u32,
542 max_values_per_topic: self.max_values_per_topic.min(u32::MAX as usize) as u32,
543 max_bounded_backfill_blocks: self.max_bounded_backfill_blocks,
544 max_identifier_bytes: self.max_identifier_bytes.min(u32::MAX as usize) as u32,
545 max_total_interests: self.max_total_interests.min(u32::MAX as usize) as u32,
546 max_total_filter_values: self.max_total_filter_values.min(u64::MAX as usize) as u64,
547 max_desired_state_bytes: self.max_desired_state_bytes.min(u64::MAX as usize) as u64,
548 max_delivery_bytes: delivery_limit(self).min(u64::MAX as usize) as u64,
549 max_active_sessions: self.max_active_sessions.min(u32::MAX as usize) as u32,
550 max_persisted_sessions: self.max_persisted_sessions.min(u64::MAX as usize) as u64,
551 }
552 }
553}
554
555impl<P> EventService<P>
556where
557 P: EventSource,
558{
559 pub fn new(
566 store: Arc<Mutex<SessionStore>>,
567 provider: Arc<P>,
568 poll_interval: Duration,
569 ) -> Result<Self, EventServiceConfigError> {
570 if poll_interval.is_zero() {
571 return Err(EventServiceConfigError::ZeroPollInterval);
572 }
573 let (shutdown, _) = watch::channel(false);
574 Ok(Self {
575 store,
576 provider,
577 poll_interval,
578 client_hello_timeout: Duration::from_secs(10),
579 source_operation_timeout: Duration::from_secs(30),
580 client_send_timeout: Duration::from_secs(30),
581 next_preparation_id: Arc::new(AtomicU64::new(1)),
582 active_sessions: Arc::new(StdMutex::new(HashSet::new())),
583 limits: EventServiceLimits::default(),
584 authorizer: Arc::new(AllowAll),
585 metrics: EventServiceMetrics::default(),
586 shutdown,
587 })
588 }
589
590 pub fn with_limits(mut self, limits: EventServiceLimits) -> Self {
592 self.limits = limits;
593 self
594 }
595
596 pub fn with_source_operation_timeout(
604 mut self,
605 timeout: Duration,
606 ) -> Result<Self, EventServiceConfigError> {
607 if timeout.is_zero() {
608 return Err(EventServiceConfigError::ZeroSourceOperationTimeout);
609 }
610 self.source_operation_timeout = timeout;
611 Ok(self)
612 }
613
614 pub fn with_client_hello_timeout(
623 mut self,
624 timeout: Duration,
625 ) -> Result<Self, EventServiceConfigError> {
626 if timeout.is_zero() {
627 return Err(EventServiceConfigError::ZeroClientHelloTimeout);
628 }
629 self.client_hello_timeout = timeout;
630 Ok(self)
631 }
632
633 pub fn with_client_send_timeout(
640 mut self,
641 timeout: Duration,
642 ) -> Result<Self, EventServiceConfigError> {
643 if timeout.is_zero() {
644 return Err(EventServiceConfigError::ZeroClientSendTimeout);
645 }
646 self.client_send_timeout = timeout;
647 Ok(self)
648 }
649
650 pub fn with_authorizer(mut self, authorizer: Arc<dyn SessionAuthorizer>) -> Self {
652 self.authorizer = authorizer;
653 self
654 }
655
656 pub fn metrics(&self) -> EventServiceMetrics {
658 self.metrics.clone()
659 }
660
661 pub fn active_session_count(&self) -> u64 {
663 self.metrics.snapshot().active_sessions
664 }
665
666 pub fn shutdown_handle(&self) -> EventServiceShutdown {
669 EventServiceShutdown {
670 sender: self.shutdown.clone(),
671 }
672 }
673
674 pub fn into_server(self) -> EventStreamServer<Self> {
676 EventStreamServer::new(self)
677 .max_decoding_message_size(MAX_MESSAGE_SIZE_BYTES)
678 .max_encoding_message_size(MAX_MESSAGE_SIZE_BYTES)
679 }
680}
681
682impl<P> Clone for EventService<P> {
683 fn clone(&self) -> Self {
684 Self {
685 store: Arc::clone(&self.store),
686 provider: Arc::clone(&self.provider),
687 poll_interval: self.poll_interval,
688 client_hello_timeout: self.client_hello_timeout,
689 source_operation_timeout: self.source_operation_timeout,
690 client_send_timeout: self.client_send_timeout,
691 next_preparation_id: Arc::clone(&self.next_preparation_id),
692 active_sessions: Arc::clone(&self.active_sessions),
693 limits: self.limits,
694 authorizer: Arc::clone(&self.authorizer),
695 metrics: self.metrics.clone(),
696 shutdown: self.shutdown.clone(),
697 }
698 }
699}
700
701type ResponseStream = Pin<Box<dyn Stream<Item = Result<ServerMessage, Status>> + Send + 'static>>;
702
703#[tonic::async_trait]
704impl<P> EventStream for EventService<P>
705where
706 P: EventSource,
707{
708 type SessionStream = ResponseStream;
709
710 async fn session(
711 &self,
712 request: Request<tonic::Streaming<ClientMessage>>,
713 ) -> Result<Response<Self::SessionStream>, Status> {
714 if self.authorizer.authorize(request.metadata()).is_err() {
715 self.metrics
716 .inner
717 .authentication_rejections
718 .fetch_add(1, Ordering::Relaxed);
719 return Err(Status::unauthenticated(
720 "event session authentication failed",
721 ));
722 }
723 if *self.shutdown.borrow() {
724 return Err(Status::unavailable("event service is shutting down"));
725 }
726 let request_metadata = request.metadata().clone();
727 let mut inbound = request.into_inner();
728 let (outbound, receiver) = mpsc::channel(32);
729 let store = Arc::clone(&self.store);
730 let provider = Arc::clone(&self.provider);
731 let next_preparation_id = Arc::clone(&self.next_preparation_id);
732 let active_sessions = Arc::clone(&self.active_sessions);
733 let poll_interval = self.poll_interval;
734 let client_hello_timeout = self.client_hello_timeout;
735 let source_operation_timeout = self.source_operation_timeout;
736 let client_send_timeout = self.client_send_timeout;
737 let limits = self.limits;
738 let authorizer = Arc::clone(&self.authorizer);
739 let metrics = self.metrics.clone();
740 let shutdown = wait_for_service_shutdown(self.shutdown.subscribe());
741 tokio::spawn(async move {
742 let mut interval = tokio::time::interval(poll_interval);
743 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
744 let hello_deadline = tokio::time::sleep(client_hello_timeout);
745 tokio::pin!(hello_deadline);
746 tokio::pin!(shutdown);
747 let mut identity: Option<(String, u64)> = None;
748 let mut desired_state: Option<ApplyDesiredState> = None;
749 let mut cursor: Option<Cursor> = None;
750 let mut required_reorg_anchor: Option<BlockRef> = None;
751 let mut has_pending = false;
752 let mut pending_was_sent = false;
756 let mut delivery_requested = false;
757 let mut session_lease: Option<SessionLease> = None;
758 let mut position_confirmed = false;
759
760 loop {
761 let update_desired_state = desired_state.clone();
762 let update_cursor = cursor.clone();
763 let update_reorg_anchor = required_reorg_anchor.clone();
764 tokio::select! {
765 () = &mut shutdown => {
766 break;
767 }
768 _ = &mut hello_deadline, if identity.is_none() => {
769 let _ = outbound.try_send(Err(Status::deadline_exceeded(
770 "event session Hello timed out",
771 )));
772 break;
773 }
774 inbound_message = inbound.next() => {
775 let Some(inbound_message) = inbound_message else { break };
776 let message = match inbound_message {
777 Ok(message) => message,
778 Err(_) => break,
779 };
780 match message.message {
781 Some(client_message::Message::Hello(hello)) => {
782 if identity.is_some() {
783 if !send_error(&outbound, ErrorCode::InvalidInterest, "session already negotiated", 0, false).await { break; }
784 continue;
785 }
786 if hello.protocol_version != PROTOCOL_VERSION {
787 if !send_error(&outbound, ErrorCode::ProtocolVersion, "unsupported protocol version", 0, false).await { break; }
788 continue;
789 }
790 if hello.session_id.len() > limits.max_identifier_bytes {
791 if !send_error(&outbound, ErrorCode::ResourceExhausted, "session identifier exceeds the configured byte limit", 0, false).await { break; }
792 continue;
793 }
794 if hello.session_id.is_empty() {
795 if !send_error(&outbound, ErrorCode::InvalidInterest, "session identifier is empty", 0, false).await { break; }
796 continue;
797 }
798 if authorizer.authorize_session(
799 &request_metadata,
800 &hello.session_id,
801 hello.chain_id,
802 ).is_err() {
803 metrics.inner.authentication_rejections.fetch_add(1, Ordering::Relaxed);
804 if !send_error(&outbound, ErrorCode::Authentication, "event session is not authorized for this identity", 0, false).await { break; }
805 continue;
806 }
807 let lease = match SessionLease::acquire(
808 Arc::clone(&active_sessions),
809 (hello.session_id.clone(), hello.chain_id),
810 limits.max_active_sessions,
811 metrics.clone(),
812 ) {
813 Ok(lease) => lease,
814 Err(LeaseError::InUse) => {
815 metrics.inner.lease_rejections.fetch_add(1, Ordering::Relaxed);
816 if !send_error(&outbound, ErrorCode::SessionInUse, "another connection owns this session", 0, true).await { break; }
817 continue;
818 }
819 Err(LeaseError::Capacity) => {
820 metrics.inner.lease_rejections.fetch_add(1, Ordering::Relaxed);
821 if !send_error(&outbound, ErrorCode::ResourceExhausted, "active session capacity is exhausted", 0, true).await { break; }
822 continue;
823 }
824 Err(LeaseError::Poisoned) => {
825 let _ = send_internal(&outbound).await;
826 break;
827 }
828 };
829 let persisted = match store.lock().await.load(&hello.session_id, hello.chain_id) {
830 Ok(persisted) => persisted,
831 Err(_) => {
832 let _ = send_internal(&outbound).await;
833 break;
834 }
835 };
836 let committed_revision = persisted.desired_state.as_ref().map_or(0, |state| state.new_revision);
837 let hello_position = match validate_hello_position(&hello, &persisted) {
838 Ok(position) => position,
839 Err(_) => {
840 if !send_error(&outbound, ErrorCode::RevisionConflict, "client resume position does not match durable service state", committed_revision, false).await { break; }
841 continue;
842 }
843 };
844 if persisted.pending_delivery.as_ref().is_some_and(|delivery| {
845 encoded_delivery_len(delivery) > MAX_MESSAGE_SIZE_BYTES
846 }) {
847 if !send_error(&outbound, ErrorCode::ResourceExhausted, "durable pending delivery exceeds the hard transport limit", committed_revision, false).await { break; }
848 continue;
849 }
850 session_lease = Some(lease);
851 let prepared = if let Some(committed) = persisted.desired_state.as_ref() {
852 let Some(preparation_id) = allocate_preparation_id(
853 next_preparation_id.as_ref(),
854 ) else {
855 let _ = send_error(
856 &outbound,
857 ErrorCode::ResourceExhausted,
858 "source preparation identity space is exhausted",
859 committed_revision,
860 false,
861 )
862 .await;
863 break;
864 };
865 if source_call(source_operation_timeout, provider.prepare_desired_state(
866 preparation_id, committed, persisted.acknowledged_cursor.as_ref(),
867 )).await.is_err() {
868 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
869 if source_call(
870 source_operation_timeout,
871 provider.abort_desired_state(preparation_id, committed),
872 )
873 .await
874 .is_err()
875 {
876 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
877 }
878 if !send_source_unavailable(&outbound).await { break; }
879 break;
880 }
881 Some((preparation_id, committed))
882 } else {
883 None
884 };
885 if let Some((preparation_id, committed)) = prepared
886 && source_call(source_operation_timeout, provider.activate_desired_state(
887 preparation_id,
888 committed,
889 persisted.acknowledged_cursor.as_ref(),
890 )).await.is_err()
891 {
892 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
893 let _ = source_call(source_operation_timeout, provider.abort_desired_state(preparation_id, committed)).await;
894 if !send_source_unavailable(&outbound).await { break; }
895 break;
896 }
897 identity = Some((hello.session_id.clone(), hello.chain_id));
898 position_confirmed = hello_position == HelloPosition::Confirmed;
899 metrics.inner.sessions_accepted.fetch_add(1, Ordering::Relaxed);
900 desired_state = persisted.desired_state.clone();
901 cursor = persisted.acknowledged_cursor.clone();
902 required_reorg_anchor = persisted.expected_reorg_tip.clone();
903 has_pending = persisted.pending_delivery.is_some();
904 pending_was_sent = has_pending;
905 delivery_requested = false;
906 if !send_outbound(&outbound, ServerMessage {
907 message: Some(server_message::Message::HelloAccepted(HelloAccepted {
908 protocol_version: PROTOCOL_VERSION,
909 session_id: hello.session_id,
910 chain_id: hello.chain_id,
911 committed_revision,
912 acknowledged_cursor: cursor.clone(),
913 desired_state: desired_state.clone(),
914 capabilities: Some(provider.capabilities_for_chain(hello.chain_id)),
915 service_limits: Some(limits.wire()),
916 runtime_checkpoint_position: Some(RuntimeCheckpointPosition {
917 cursor: persisted.runtime_checkpoint_cursor.clone(),
918 }),
919 })),
920 }, client_send_timeout).await { break; }
921 }
925 Some(client_message::Message::ApplyDesiredState(request)) => {
926 let Some((session_id, chain_id)) = &identity else {
927 if !send_error(&outbound, ErrorCode::ProtocolVersion, "Hello must be sent first", 0, false).await { break; }
928 continue;
929 };
930 if &request.session_id != session_id || request.chain_id != *chain_id {
931 if !send_error(&outbound, ErrorCode::InvalidInterest, "desired state does not match negotiated session", 0, false).await { break; }
932 continue;
933 }
934 if !position_confirmed {
935 if !send_error(&outbound, ErrorCode::RevisionConflict, "runtime checkpoint proof is required on a new session", desired_state.as_ref().map_or(0, |state| state.new_revision), false).await { break; }
936 continue;
937 }
938 if let Err(error) = validate_limits(&request, limits) {
939 if !send_error(&outbound, ErrorCode::ResourceExhausted, error, request.expected_revision, false).await { break; }
940 continue;
941 }
942 if let Err(error) = validate_desired_state(&request) {
943 let (code, message) = desired_state_error(&error);
944 if !send_error(&outbound, code, message, request.expected_revision, false).await { break; }
945 continue;
946 }
947 if request.expected_revision > i64::MAX as u64
948 || request.new_revision > i64::MAX as u64
949 {
950 if !send_error(
951 &outbound,
952 ErrorCode::InvalidInterest,
953 "desired-state revision exceeds the durable storage range",
954 request.expected_revision,
955 false,
956 ).await { break; }
957 continue;
958 }
959 let Some(preparation_id) = allocate_preparation_id(
960 next_preparation_id.as_ref(),
961 ) else {
962 if !send_error(
963 &outbound,
964 ErrorCode::ResourceExhausted,
965 "source preparation identity space is exhausted",
966 request.expected_revision,
967 false,
968 ).await { break; }
969 continue;
970 };
971 let prepared_cursor = match source_call(source_operation_timeout, provider.prepare_desired_state(
972 preparation_id,
973 &request,
974 cursor.as_ref(),
975 )).await {
976 Ok(cursor) => cursor,
977 Err(error) => {
978 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
979 let abort_failed = source_call(
980 source_operation_timeout,
981 provider.abort_desired_state(preparation_id, &request),
982 )
983 .await
984 .is_err();
985 if abort_failed {
986 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
987 let _ = send_source_unavailable(&outbound).await;
991 break;
992 }
993 let (code, message, retryable) = source_error_response(&error);
994 if !send_error(&outbound, code, message, request.expected_revision, retryable).await { break; }
995 continue;
996 }
997 };
998 let result = store.lock().await.apply_desired_state_with_cursor_and_limit(
999 request.clone(),
1000 prepared_cursor.as_ref(),
1001 delivery_limit(limits),
1002 limits.max_persisted_sessions,
1003 );
1004 match result {
1005 Ok((applied, newly_committed)) => {
1006 if newly_committed {
1007 metrics.inner.desired_states_committed.fetch_add(1, Ordering::Relaxed);
1008 metrics.inner.deliveries_persisted.fetch_add(1, Ordering::Relaxed);
1009 }
1010 if source_call(source_operation_timeout, provider.activate_desired_state(
1011 preparation_id,
1012 &request,
1013 cursor.as_ref(),
1014 )).await.is_err() {
1015 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1016 let _ = source_call(source_operation_timeout, provider.abort_desired_state(preparation_id, &request)).await;
1017 if !send_source_unavailable(&outbound).await { break; }
1021 break;
1022 }
1023 desired_state = Some(request.clone());
1024 let pending = match store.lock().await.load(session_id, *chain_id) {
1025 Ok(persisted) => persisted.pending_delivery,
1026 Err(_) => {
1027 let _ = send_internal(&outbound).await;
1028 break;
1029 }
1030 };
1031 has_pending = pending.is_some();
1032 pending_was_sent = !newly_committed && has_pending;
1033 delivery_requested = false;
1034 if !send_outbound(&outbound, ServerMessage {
1035 message: Some(server_message::Message::DesiredStateApplied(applied)),
1036 }, client_send_timeout).await { break; }
1037 }
1040 Err(error) => {
1041 if source_call(source_operation_timeout, provider.abort_desired_state(preparation_id, &request)).await.is_err() {
1042 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1043 let _ = send_source_unavailable(&outbound).await;
1047 break;
1048 }
1049 let (code, committed, retryable) = store_error_code(&error);
1050 if code == ErrorCode::Internal {
1051 let _ = send_internal(&outbound).await;
1052 break;
1053 }
1054 if !send_error(&outbound, code, store_error_message(&error), committed, retryable).await { break; }
1055 }
1056 }
1057 }
1058 Some(client_message::Message::Acknowledge(acknowledgement)) => {
1059 let Some((session_id, chain_id)) = &identity else {
1060 if !send_error(&outbound, ErrorCode::ProtocolVersion, "Hello must be sent first", 0, false).await { break; }
1061 continue;
1062 };
1063 if &acknowledgement.session_id != session_id {
1064 if !send_error(&outbound, ErrorCode::InvalidInterest, "acknowledgement does not match negotiated session", 0, false).await { break; }
1065 continue;
1066 }
1067 if !position_confirmed {
1068 if !send_error(&outbound, ErrorCode::RevisionConflict, "runtime checkpoint proof is required on a new session", desired_state.as_ref().map_or(0, |state| state.new_revision), false).await { break; }
1069 continue;
1070 }
1071 let acknowledgement_result = {
1072 let mut authority = store.lock().await;
1073 match authority.acknowledge_with_status(*chain_id, &acknowledgement) {
1074 Ok((committed_cursor, newly_committed)) => authority
1075 .load(session_id, *chain_id)
1076 .map(|persisted| {
1077 (
1078 committed_cursor,
1079 newly_committed,
1080 persisted.expected_reorg_tip,
1081 )
1082 }),
1083 Err(error) => Err(error),
1084 }
1085 };
1086 match acknowledgement_result {
1087 Ok((committed_cursor, newly_committed, committed_reorg_anchor)) => {
1088 required_reorg_anchor = committed_reorg_anchor;
1089 if newly_committed {
1090 metrics
1091 .inner
1092 .acknowledgements_committed
1093 .fetch_add(1, Ordering::Relaxed);
1094 }
1095 if source_call(source_operation_timeout, provider.acknowledge(
1096 *chain_id,
1097 &acknowledgement,
1098 &committed_cursor,
1099 )).await.is_err()
1100 {
1101 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1102 if !send_source_unavailable(&outbound).await { break; }
1103 break;
1104 }
1105 if !send_outbound(&outbound, ServerMessage {
1106 message: Some(server_message::Message::AcknowledgementCommitted(
1107 AcknowledgementCommitted {
1108 session_id: session_id.clone(),
1109 sequence: acknowledgement.sequence,
1110 cursor: Some(committed_cursor.clone()),
1111 },
1112 )),
1113 }, client_send_timeout).await {
1114 break;
1115 }
1116 cursor = Some(committed_cursor);
1117 has_pending = false;
1118 pending_was_sent = false;
1119 delivery_requested = false;
1120 interval.reset_immediately();
1121 }
1122 Err(_) => {
1123 let _ = send_internal(&outbound).await;
1124 break;
1125 }
1126 }
1127 }
1128 Some(client_message::Message::Heartbeat(heartbeat)) => {
1129 if identity.is_none() {
1130 if !send_error(&outbound, ErrorCode::ProtocolVersion, "Hello must be sent first", 0, false).await { break; }
1131 continue;
1132 }
1133 if !send_outbound(&outbound, ServerMessage {
1134 message: Some(server_message::Message::Heartbeat(heartbeat)),
1135 }, client_send_timeout).await { break; }
1136 }
1137 Some(client_message::Message::DeliveryDemand(demand)) => {
1138 let Some((session_id, chain_id)) = &identity else {
1139 if !send_error(&outbound, ErrorCode::ProtocolVersion, "Hello must be sent first", 0, false).await { break; }
1140 continue;
1141 };
1142 if &demand.session_id != session_id {
1143 if !send_error(&outbound, ErrorCode::InvalidInterest, "delivery demand does not match negotiated session", 0, false).await { break; }
1144 continue;
1145 }
1146 if !position_confirmed {
1147 if !send_error(&outbound, ErrorCode::RevisionConflict, "runtime checkpoint proof is required on a new session", desired_state.as_ref().map_or(0, |state| state.new_revision), false).await { break; }
1148 continue;
1149 }
1150 if has_pending {
1151 let pending = store.lock().await.load(session_id, *chain_id)
1152 .ok()
1153 .and_then(|persisted| persisted.pending_delivery);
1154 match pending {
1155 Some(delivery) if encoded_delivery_len(&delivery) <= MAX_MESSAGE_SIZE_BYTES => {
1156 if pending_was_sent {
1157 metrics.inner.deliveries_replayed.fetch_add(1, Ordering::Relaxed);
1158 }
1159 if !send_outbound(&outbound, ServerMessage {
1160 message: Some(server_message::Message::Delivery(delivery)),
1161 }, client_send_timeout).await { break; }
1162 pending_was_sent = true;
1163 }
1164 Some(_) => {
1165 if !send_error(&outbound, ErrorCode::ResourceExhausted, "durable pending delivery exceeds the hard transport limit", desired_state.as_ref().map_or(0, |state| state.new_revision), false).await { break; }
1166 }
1167 None => {
1168 has_pending = false;
1169 pending_was_sent = false;
1170 delivery_requested = true;
1171 interval.reset_immediately();
1172 }
1173 }
1174 } else {
1175 delivery_requested = true;
1176 interval.reset_immediately();
1177 }
1178 }
1179 None => {
1180 if !send_error(&outbound, ErrorCode::InvalidInterest, "empty client message", 0, false).await { break; }
1181 }
1182 }
1183 }
1184 _ = interval.tick(), if identity.is_some() && position_confirmed && desired_state.is_some() && delivery_requested && !has_pending => {
1185 let Some(desired) = desired_state.as_ref() else {
1186 continue;
1187 };
1188 let request = DeliveryRequest::new(desired, cursor.as_ref())
1189 .with_required_reorg_anchor(required_reorg_anchor.as_ref());
1190 match source_call(source_operation_timeout, provider.next_delivery(request)).await {
1191 Ok(Some(delivery)) => {
1192 if encoded_delivery_len(&delivery) > delivery_limit(limits) {
1193 delivery_requested = false;
1194 if !send_error(&outbound, ErrorCode::ResourceExhausted, "source delivery exceeds the configured encoded-byte limit", desired.new_revision, false).await { break; }
1195 continue;
1196 }
1197 let save_result = {
1198 store.lock().await.save_pending_with_status(desired, &delivery)
1199 };
1200 match save_result {
1201 Ok(newly_persisted) => {
1202 if newly_persisted {
1203 metrics.inner.deliveries_persisted.fetch_add(1, Ordering::Relaxed);
1204 }
1205 has_pending = true;
1206 pending_was_sent = true;
1207 delivery_requested = false;
1208 if !send_outbound(&outbound, ServerMessage {
1209 message: Some(server_message::Message::Delivery(delivery)),
1210 }, client_send_timeout).await { break; }
1211 }
1212 Err(_) => {
1213 let _ = send_internal(&outbound).await;
1214 break;
1215 }
1216 }
1217 }
1218 Ok(None) => {}
1219 Err(error) => {
1220 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1221 delivery_requested = false;
1222 let (code, message, retryable) = source_error_response(&error);
1223 if !send_error(&outbound, code, message, desired.new_revision, retryable).await { break; }
1224 }
1225 }
1226 }
1227 update = wait_for_provider_update(
1228 provider.as_ref(),
1229 update_desired_state.as_ref(),
1230 update_cursor.as_ref(),
1231 update_reorg_anchor.as_ref(),
1232 ), if identity.is_some() && position_confirmed && delivery_requested && !has_pending => {
1233 match update {
1234 Ok(()) => {
1235 let Some(desired) = desired_state.as_ref() else {
1236 continue;
1237 };
1238 let request = DeliveryRequest::new(desired, cursor.as_ref())
1239 .with_required_reorg_anchor(required_reorg_anchor.as_ref());
1240 match source_call(source_operation_timeout, provider.next_delivery(request)).await {
1241 Ok(Some(delivery)) => {
1242 if encoded_delivery_len(&delivery) > delivery_limit(limits) {
1243 delivery_requested = false;
1244 if !send_error(&outbound, ErrorCode::ResourceExhausted, "source delivery exceeds the configured encoded-byte limit", desired.new_revision, false).await { break; }
1245 continue;
1246 }
1247 let save_result = {
1248 store.lock().await.save_pending_with_status(desired, &delivery)
1249 };
1250 match save_result {
1251 Ok(newly_persisted) => {
1252 if newly_persisted {
1253 metrics.inner.deliveries_persisted.fetch_add(1, Ordering::Relaxed);
1254 }
1255 has_pending = true;
1256 pending_was_sent = true;
1257 delivery_requested = false;
1258 if !send_outbound(&outbound, ServerMessage {
1259 message: Some(server_message::Message::Delivery(delivery)),
1260 }, client_send_timeout).await { break; }
1261 }
1262 Err(_) => {
1263 let _ = send_internal(&outbound).await;
1264 break;
1265 }
1266 }
1267 }
1268 Ok(None) => {}
1269 Err(error) => {
1270 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1271 delivery_requested = false;
1272 let (code, message, retryable) = source_error_response(&error);
1273 if !send_error(&outbound, code, message, desired.new_revision, retryable).await { break; }
1274 }
1275 }
1276 }
1277 Err(error) => {
1278 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1279 delivery_requested = false;
1280 let (code, message, retryable) = source_error_response(&error);
1281 if !send_error(&outbound, code, message, desired_state.as_ref().map_or(0, |state| state.new_revision), retryable).await { break; }
1282 }
1283 }
1284 }
1285 }
1286 }
1287 if let Some(lease) = session_lease.as_mut() {
1288 let release = tokio::time::timeout(
1289 source_operation_timeout,
1290 provider.release_session(&lease.key.0, lease.key.1),
1291 )
1292 .await;
1293 if !matches!(release, Ok(Ok(()))) {
1294 if matches!(release, Ok(Err(_))) {
1295 metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1296 }
1297 lease.retain_on_drop();
1301 }
1302 }
1303 drop(session_lease);
1304 });
1305 Ok(Response::new(Box::pin(ReceiverStream::new(receiver))))
1306 }
1307}
1308
1309struct SessionLease {
1310 registry: Arc<StdMutex<HashSet<(String, u64)>>>,
1311 key: (String, u64),
1312 metrics: EventServiceMetrics,
1313 retain_registry_entry: bool,
1314}
1315
1316impl SessionLease {
1317 fn acquire(
1318 registry: Arc<StdMutex<HashSet<(String, u64)>>>,
1319 key: (String, u64),
1320 max_active_sessions: usize,
1321 metrics: EventServiceMetrics,
1322 ) -> Result<Self, LeaseError> {
1323 let mut sessions = registry.lock().map_err(|_| LeaseError::Poisoned)?;
1324 if sessions.contains(&key) {
1325 return Err(LeaseError::InUse);
1326 }
1327 if sessions.len() >= max_active_sessions {
1328 return Err(LeaseError::Capacity);
1329 }
1330 sessions.insert(key.clone());
1331 drop(sessions);
1332 metrics
1333 .inner
1334 .active_sessions
1335 .fetch_add(1, Ordering::Relaxed);
1336 Ok(Self {
1337 registry,
1338 key,
1339 metrics,
1340 retain_registry_entry: false,
1341 })
1342 }
1343
1344 fn retain_on_drop(&mut self) {
1345 self.retain_registry_entry = true;
1346 }
1347}
1348
1349impl Drop for SessionLease {
1350 fn drop(&mut self) {
1351 if self.retain_registry_entry {
1352 return;
1353 }
1354 if let Ok(mut sessions) = self.registry.lock()
1355 && sessions.remove(&self.key)
1356 {
1357 self.metrics
1358 .inner
1359 .active_sessions
1360 .fetch_sub(1, Ordering::Relaxed);
1361 }
1362 }
1363}
1364
1365enum LeaseError {
1366 InUse,
1367 Capacity,
1368 Poisoned,
1369}
1370
1371fn allocate_preparation_id(next: &AtomicU64) -> Option<PreparationId> {
1372 next.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| {
1373 value.checked_add(1)
1374 })
1375 .ok()
1376 .map(PreparationId::new)
1377}
1378
1379async fn source_call<T>(
1380 timeout: Duration,
1381 operation: impl std::future::Future<Output = Result<T, EventSourceError>>,
1382) -> Result<T, EventSourceError> {
1383 tokio::time::timeout(timeout, operation)
1384 .await
1385 .map_err(|_| EventSourceError::unavailable("source operation timed out"))?
1386}
1387
1388async fn wait_for_service_shutdown(mut shutdown: watch::Receiver<bool>) {
1389 if *shutdown.borrow_and_update() {
1390 return;
1391 }
1392 loop {
1393 if shutdown.changed().await.is_err() {
1394 return;
1395 }
1396 if *shutdown.borrow_and_update() {
1397 return;
1398 }
1399 }
1400}
1401
1402async fn wait_for_provider_update<P>(
1403 provider: &P,
1404 desired_state: Option<&ApplyDesiredState>,
1405 acknowledged_cursor: Option<&Cursor>,
1406 required_reorg_anchor: Option<&BlockRef>,
1407) -> Result<(), EventSourceError>
1408where
1409 P: EventSource,
1410{
1411 let Some(desired_state) = desired_state else {
1412 return std::future::pending().await;
1413 };
1414 provider
1415 .wait_for_update(
1416 DeliveryRequest::new(desired_state, acknowledged_cursor)
1417 .with_required_reorg_anchor(required_reorg_anchor),
1418 )
1419 .await
1420}
1421
1422async fn send_internal(sender: &mpsc::Sender<Result<ServerMessage, Status>>) -> bool {
1423 sender
1424 .try_send(Err(Status::unavailable("internal event service failure")))
1425 .is_ok()
1426}
1427
1428async fn send_source_unavailable(sender: &mpsc::Sender<Result<ServerMessage, Status>>) -> bool {
1429 sender
1430 .try_send(Err(Status::unavailable("event source unavailable")))
1431 .is_ok()
1432}
1433
1434async fn send_outbound(
1435 sender: &mpsc::Sender<Result<ServerMessage, Status>>,
1436 message: ServerMessage,
1437 timeout: Duration,
1438) -> bool {
1439 matches!(
1440 tokio::time::timeout(timeout, sender.send(Ok(message))).await,
1441 Ok(Ok(()))
1442 )
1443}
1444
1445async fn send_error(
1446 sender: &mpsc::Sender<Result<ServerMessage, Status>>,
1447 code: ErrorCode,
1448 message: &str,
1449 committed_revision: u64,
1450 retryable: bool,
1451) -> bool {
1452 sender
1453 .try_send(Ok(ServerMessage {
1454 message: Some(server_message::Message::Error(ProtocolError {
1455 code: code.into(),
1456 message: message.to_owned(),
1457 committed_revision,
1458 retryable,
1459 })),
1460 }))
1461 .is_ok()
1462}
1463
1464fn store_error_code(error: &SessionStoreError) -> (ErrorCode, u64, bool) {
1465 match error {
1466 SessionStoreError::RevisionConflict { committed, .. } => {
1467 (ErrorCode::RevisionConflict, *committed, false)
1468 }
1469 SessionStoreError::DesiredState(DesiredStateError::ProtocolVersion { .. }) => {
1470 (ErrorCode::ProtocolVersion, 0, false)
1471 }
1472 SessionStoreError::DesiredState(DesiredStateError::UnsupportedInterest { .. }) => {
1473 (ErrorCode::UnsupportedInterest, 0, false)
1474 }
1475 SessionStoreError::DesiredState(DesiredStateError::InvalidState(_)) => {
1476 (ErrorCode::InvalidInterest, 0, false)
1477 }
1478 SessionStoreError::PendingDelivery => (ErrorCode::PendingDelivery, 0, true),
1479 SessionStoreError::PersistedSessionLimit { .. } => (ErrorCode::ResourceExhausted, 0, false),
1480 SessionStoreError::SequenceOverflow => (ErrorCode::ResourceExhausted, 0, false),
1481 SessionStoreError::IntegerRange(_) => (ErrorCode::InvalidInterest, 0, false),
1482 SessionStoreError::DeliveryTooLarge { .. } => (ErrorCode::ResourceExhausted, 0, false),
1483 _ => (ErrorCode::Internal, 0, true),
1484 }
1485}
1486
1487fn store_error_message(error: &SessionStoreError) -> &'static str {
1488 match error {
1489 SessionStoreError::RevisionConflict { .. } => "desired-state revision conflict",
1490 SessionStoreError::DesiredState(DesiredStateError::ProtocolVersion { .. }) => {
1491 "unsupported protocol version"
1492 }
1493 SessionStoreError::DesiredState(DesiredStateError::UnsupportedInterest { .. }) => {
1494 "desired state contains an unsupported interest"
1495 }
1496 SessionStoreError::DesiredState(DesiredStateError::InvalidState(_)) => {
1497 "desired state is invalid"
1498 }
1499 SessionStoreError::PendingDelivery => {
1500 "pending delivery must be acknowledged before desired state changes"
1501 }
1502 SessionStoreError::PersistedSessionLimit { .. } => {
1503 "persisted session capacity is exhausted"
1504 }
1505 SessionStoreError::SequenceOverflow => "delivery sequence space is exhausted",
1506 SessionStoreError::IntegerRange(_) => "value exceeds the durable storage range",
1507 SessionStoreError::DeliveryTooLarge { .. } => {
1508 "activation delivery exceeds the configured encoded-byte limit"
1509 }
1510 _ => "internal event service failure",
1511 }
1512}
1513
1514fn desired_state_error(error: &DesiredStateError) -> (ErrorCode, &'static str) {
1515 match error {
1516 DesiredStateError::ProtocolVersion { .. } => {
1517 (ErrorCode::ProtocolVersion, "unsupported protocol version")
1518 }
1519 DesiredStateError::UnsupportedInterest { .. } => (
1520 ErrorCode::UnsupportedInterest,
1521 "desired state contains an unsupported interest",
1522 ),
1523 DesiredStateError::InvalidState(_) => {
1524 (ErrorCode::InvalidInterest, "desired state is invalid")
1525 }
1526 DesiredStateError::RevisionConflict { .. } => (
1527 ErrorCode::RevisionConflict,
1528 "desired-state revision conflict",
1529 ),
1530 }
1531}
1532
1533fn source_error_response(error: &EventSourceError) -> (ErrorCode, &'static str, bool) {
1534 match error.kind {
1535 EventSourceErrorKind::InvalidRequest => (
1536 ErrorCode::InvalidInterest,
1537 "event source rejected the request",
1538 false,
1539 ),
1540 EventSourceErrorKind::Unsupported => (
1541 ErrorCode::UnsupportedInterest,
1542 "event source does not support the request",
1543 false,
1544 ),
1545 EventSourceErrorKind::ResourceExhausted => (
1546 ErrorCode::ResourceExhausted,
1547 "event source resource limit exceeded",
1548 true,
1549 ),
1550 EventSourceErrorKind::Unavailable => (
1551 ErrorCode::SourceUnavailable,
1552 "event source is temporarily unavailable",
1553 true,
1554 ),
1555 EventSourceErrorKind::Internal => {
1556 (ErrorCode::Internal, "internal event source failure", true)
1557 }
1558 }
1559}
1560
1561fn encoded_delivery_len(delivery: &Delivery) -> usize {
1562 let delivery_bytes = delivery.encoded_len();
1563 delivery_bytes
1564 .saturating_add(prost::length_delimiter_len(delivery_bytes))
1565 .saturating_add(1)
1566}
1567
1568#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1569enum HelloPosition {
1570 Confirmed,
1571 DiscoveryOnly,
1572}
1573
1574fn validate_hello_position(
1575 hello: &Hello,
1576 persisted: &PersistedSession,
1577) -> Result<HelloPosition, &'static str> {
1578 let transport_sequence = persisted
1579 .acknowledged_cursor
1580 .as_ref()
1581 .map_or(0, |cursor| cursor.batch_sequence);
1582 let runtime_sequence = persisted
1583 .runtime_checkpoint_cursor
1584 .as_ref()
1585 .map_or(0, |cursor| cursor.batch_sequence);
1586 let Some(resume) = hello.pending_delivery_resume.as_ref() else {
1587 return if hello.acknowledged_sequence == runtime_sequence {
1588 Ok(HelloPosition::Confirmed)
1589 } else if hello.acknowledged_sequence == 0 {
1590 Ok(HelloPosition::DiscoveryOnly)
1594 } else {
1595 Err("client position does not equal runtime checkpoint authority")
1596 };
1597 };
1598
1599 if hello.acknowledged_sequence == runtime_sequence {
1600 let cursor = persisted
1601 .runtime_checkpoint_cursor
1602 .as_ref()
1603 .ok_or("zero sequence cannot carry a pending-delivery proof")?;
1604 resume_matches_cursor(
1605 resume,
1606 hello.acknowledged_sequence.to_be_bytes().as_slice(),
1607 cursor,
1608 )?;
1609 return Ok(HelloPosition::Confirmed);
1610 }
1611
1612 let expected_pending = transport_sequence
1613 .checked_add(1)
1614 .ok_or("durable delivery sequence is exhausted")?;
1615 let pending = persisted
1616 .pending_delivery
1617 .as_ref()
1618 .ok_or("service has no pending delivery matching the restored position")?;
1619 let cursor = pending
1620 .cursor
1621 .as_ref()
1622 .ok_or("pending delivery cursor is missing")?;
1623 if hello.acknowledged_sequence != expected_pending
1624 || pending.sequence != expected_pending
1625 || pending.delivery_token != resume.delivery_token
1626 {
1627 return Err("pending delivery sequence or token does not match");
1628 }
1629 resume_matches_cursor(resume, &pending.delivery_token, cursor)?;
1630 Ok(HelloPosition::Confirmed)
1631}
1632
1633fn resume_matches_cursor(
1634 resume: &PendingDeliveryResume,
1635 expected_token: &[u8],
1636 cursor: &Cursor,
1637) -> Result<(), &'static str> {
1638 let checkpoint_matches = if cursor.provider_checkpoint.is_empty() {
1639 resume.provider_checkpoint.is_none()
1640 } else {
1641 resume.provider_checkpoint.as_deref() == Some(cursor.provider_checkpoint.as_slice())
1642 };
1643 if resume.delivery_token != expected_token
1644 || !checkpoint_matches
1645 || resume.coverage_head.as_ref() != cursor.canonical_head.as_ref()
1646 {
1647 return Err("pending-delivery proof does not match its cursor");
1648 }
1649 Ok(())
1650}
1651
1652const fn delivery_limit(limits: EventServiceLimits) -> usize {
1653 if limits.max_delivery_bytes < MAX_MESSAGE_SIZE_BYTES {
1654 limits.max_delivery_bytes
1655 } else {
1656 MAX_MESSAGE_SIZE_BYTES
1657 }
1658}
1659
1660fn validate_limits(
1661 request: &ApplyDesiredState,
1662 limits: EventServiceLimits,
1663) -> Result<(), &'static str> {
1664 if request.session_id.len() > limits.max_identifier_bytes {
1665 return Err("session identifier exceeds the configured byte limit");
1666 }
1667 if request.owners.len() > limits.max_owners {
1668 return Err("owner count exceeds the configured limit");
1669 }
1670 if request.encoded_len() > limits.max_desired_state_bytes {
1671 return Err("desired state exceeds the configured encoded-byte limit");
1672 }
1673 let mut total_interests = 0usize;
1674 let mut total_filter_values = 0usize;
1675 for owner in &request.owners {
1676 if owner.owner_id.len() > limits.max_identifier_bytes {
1677 return Err("owner identifier exceeds the configured byte limit");
1678 }
1679 if owner.interests.len() > limits.max_interests_per_owner {
1680 return Err("owner interest count exceeds the configured limit");
1681 }
1682 total_interests = total_interests.saturating_add(owner.interests.len());
1683 if total_interests > limits.max_total_interests {
1684 return Err("total interest count exceeds the configured limit");
1685 }
1686 if owner.backfill.as_ref().is_some_and(|backfill| {
1687 backfill.to_block_excl.is_some_and(|end| {
1688 end.saturating_sub(backfill.from_block) > limits.max_bounded_backfill_blocks
1689 })
1690 }) {
1691 return Err("bounded backfill exceeds the configured block limit");
1692 }
1693 for interest in &owner.interests {
1694 if let Some(portable_interest::Kind::Log(log)) = &interest.kind {
1695 total_filter_values = total_filter_values.saturating_add(log.addresses.len());
1696 total_filter_values = total_filter_values.saturating_add(
1697 log.topics
1698 .iter()
1699 .map(|topic| topic.values.len())
1700 .fold(0usize, usize::saturating_add),
1701 );
1702 if total_filter_values > limits.max_total_filter_values {
1703 return Err("total log filter value count exceeds the configured limit");
1704 }
1705 if log.addresses.len() > limits.max_addresses_per_log_interest {
1706 return Err("log address count exceeds the configured limit");
1707 }
1708 if log
1709 .topics
1710 .iter()
1711 .any(|topic| topic.values.len() > limits.max_values_per_topic)
1712 {
1713 return Err("log topic value count exceeds the configured limit");
1714 }
1715 }
1716 }
1717 }
1718 Ok(())
1719}
1720
1721#[cfg(test)]
1722mod tests {
1723 use std::sync::atomic::AtomicU64;
1724
1725 use super::allocate_preparation_id;
1726
1727 #[test]
1728 fn preparation_identifiers_never_wrap_or_repeat() {
1729 let next = AtomicU64::new(u64::MAX - 1);
1730 assert!(allocate_preparation_id(&next).is_some());
1731 assert!(allocate_preparation_id(&next).is_none());
1732 assert!(allocate_preparation_id(&next).is_none());
1733 }
1734}