Skip to main content

evm_fork_cache_event_service/
service.rs

1use std::{
2    collections::HashSet,
3    pin::Pin,
4    sync::{
5        Arc, Mutex as StdMutex,
6        atomic::{AtomicU64, Ordering},
7    },
8    time::Duration,
9};
10
11use async_trait::async_trait;
12use evm_fork_cache_event_protocol::{
13    MAX_MESSAGE_SIZE_BYTES, PROTOCOL_VERSION,
14    v1::{
15        Acknowledge, AcknowledgementCommitted, ApplyDesiredState, BlockRef, ClientMessage, Cursor,
16        Delivery, ErrorCode, Hello, HelloAccepted, PendingDeliveryResume, ProtocolError,
17        RuntimeCheckpointPosition, ServerMessage, ServiceLimits, SourceCapabilities,
18        client_message,
19        event_stream_server::{EventStream, EventStreamServer},
20        portable_interest, server_message,
21    },
22};
23use futures::{Stream, StreamExt};
24use tokio::sync::{Mutex, mpsc, watch};
25use tokio_stream::wrappers::ReceiverStream;
26use tonic::{Request, Response, Status, metadata::MetadataMap};
27
28use prost::Message;
29
30use crate::{
31    DesiredStateError, PersistedSession, SessionStore, SessionStoreError, validate_desired_state,
32};
33
34/// Process-local identity for one prepared desired-state candidate.
35#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
36pub struct PreparationId(u64);
37
38impl PreparationId {
39    const fn new(value: u64) -> Self {
40        Self(value)
41    }
42}
43
44/// Durable, provider-neutral inputs for producing one source delivery.
45///
46/// The service constructs this request only from committed session state. Its
47/// private fields and accessors leave room for later source constraints without
48/// forcing providers to destructure a version-sensitive public shape.
49#[derive(Clone, Copy, Debug)]
50#[non_exhaustive]
51pub struct DeliveryRequest<'a> {
52    desired_state: &'a ApplyDesiredState,
53    acknowledged_cursor: Option<&'a Cursor>,
54    required_reorg_anchor: Option<&'a BlockRef>,
55}
56
57impl<'a> DeliveryRequest<'a> {
58    /// Construct an unconstrained poll after the committed cursor.
59    pub const fn new(
60        desired_state: &'a ApplyDesiredState,
61        acknowledged_cursor: Option<&'a Cursor>,
62    ) -> Self {
63        Self {
64            desired_state,
65            acknowledged_cursor,
66            required_reorg_anchor: None,
67        }
68    }
69
70    /// Require the first canonical replacement delivery to certify this anchor.
71    #[must_use]
72    pub const fn with_required_reorg_anchor(mut self, anchor: Option<&'a BlockRef>) -> Self {
73        self.required_reorg_anchor = anchor;
74        self
75    }
76
77    /// Complete authoritative desired state for the active revision.
78    pub const fn desired_state(self) -> &'a ApplyDesiredState {
79        self.desired_state
80    }
81
82    /// Last transport cursor committed by the durable service.
83    pub const fn acknowledged_cursor(self) -> Option<&'a Cursor> {
84        self.acknowledged_cursor
85    }
86
87    /// Exact replacement anchor promised by an acknowledged reorg, if any.
88    ///
89    /// A source must return `None` until it can honor this constraint, or return
90    /// a delivery that explicitly certifies the anchor. A blockful barrier must
91    /// stop at the anchor. Data may end later only when it includes a complete
92    /// continuous descendant suffix through its terminal cursor. Ignoring this
93    /// value is a source contract violation and the service rejects the output.
94    pub const fn required_reorg_anchor(self) -> Option<&'a BlockRef> {
95        self.required_reorg_anchor
96    }
97}
98
99/// Provider-neutral source boundary used by the durable event service.
100///
101/// # Cancellation and failure atomicity
102///
103/// The service applies deadlines by dropping these async futures. Every method
104/// must therefore be cancellation-safe as well as failure-atomic: if its future
105/// is dropped or returns `Err`, a retry with the same durable inputs must remain
106/// valid and must not observe a partially published transition. Implementations
107/// may stage idempotent work behind a preparation or delivery token, but must
108/// bound and eventually release such work. Durable service state, not an
109/// in-memory source task, is the authority after restart.
110#[async_trait]
111pub trait EventSource: Send + Sync + 'static {
112    /// Advertise the guarantees and topology implemented by this source.
113    fn capabilities(&self) -> SourceCapabilities;
114
115    /// Advertise guarantees for a particular chain. Multi-chain sources should
116    /// override this; single-capability sources retain the legacy behavior.
117    fn capabilities_for_chain(&self, _chain_id: u64) -> SourceCapabilities {
118        self.capabilities()
119    }
120
121    /// Validate and fully prepare a desired state before it becomes durable.
122    ///
123    /// A source that establishes a provider-native activation boundary may
124    /// return the corresponding portable cursor. The service persists that
125    /// cursor in the activation barrier, so acknowledging the barrier cannot
126    /// rewind a newly prepared source (for example, from archive head to
127    /// genesis). Sources without a prepared boundary return `None`.
128    ///
129    /// Cancellation or `Err` must leave the active revision unchanged. Any
130    /// staged candidate must remain safely retryable by `preparation_id` or
131    /// removable by [`EventSource::abort_desired_state`].
132    ///
133    /// # Errors
134    ///
135    /// Returns [`EventSourceError`] when the desired state is invalid or
136    /// unsupported, resources are exhausted, the provider is unavailable, or
137    /// preparation cannot complete atomically.
138    async fn prepare_desired_state(
139        &self,
140        _preparation_id: PreparationId,
141        _desired_state: &ApplyDesiredState,
142        _acknowledged_cursor: Option<&Cursor>,
143    ) -> Result<Option<Cursor>, EventSourceError> {
144        Ok(None)
145    }
146
147    /// Activate a previously prepared desired state after its durable commit.
148    ///
149    /// Cancellation or `Err` must not expose a partially activated revision;
150    /// the previous active source must remain usable and the candidate must
151    /// remain retryable or abortable.
152    ///
153    /// # Errors
154    ///
155    /// Returns [`EventSourceError`] when the prepared candidate is missing or
156    /// inconsistent, resources are unavailable, or activation cannot complete
157    /// atomically.
158    async fn activate_desired_state(
159        &self,
160        _preparation_id: PreparationId,
161        _desired_state: &ApplyDesiredState,
162        _acknowledged_cursor: Option<&Cursor>,
163    ) -> Result<(), EventSourceError> {
164        Ok(())
165    }
166
167    /// Discard a prepared desired state when its durable commit fails.
168    ///
169    /// This operation must be idempotent. Cancellation or `Err` must leave a
170    /// later abort retry able to finish cleanup without losing the active source.
171    ///
172    /// # Errors
173    ///
174    /// Returns [`EventSourceError`] when cleanup cannot currently complete.
175    /// Implementations must leave the same abort safely retryable.
176    async fn abort_desired_state(
177        &self,
178        _preparation_id: PreparationId,
179        _desired_state: &ApplyDesiredState,
180    ) -> Result<(), EventSourceError> {
181        Ok(())
182    }
183
184    /// Release all process-local state for a stream that owned this durable
185    /// session. The service invokes this at most once for each successfully
186    /// negotiated lease, after it stops using every other source method.
187    ///
188    /// Implementations must be idempotent and cancellation-safe. An error or
189    /// cancellation at [`EventService::with_source_operation_timeout`] may mean
190    /// cleanup is partial. In either case the service retains the lease (fail
191    /// closed), so another generation cannot overlap it; a service restart is
192    /// needed to recover that session identity.
193    ///
194    /// # Errors
195    ///
196    /// Returns [`EventSourceError`] when process-local session cleanup cannot
197    /// complete. The service then retains the lease to prevent overlap.
198    async fn release_session(
199        &self,
200        _session_id: &str,
201        _chain_id: u64,
202    ) -> Result<(), EventSourceError> {
203        Ok(())
204    }
205
206    /// Produce the next delivery after the durable cursor, or `None` when caught up.
207    ///
208    /// Implementations must inspect [`DeliveryRequest::required_reorg_anchor`].
209    /// Returning `None` or a typed unsupported error is valid when the source
210    /// cannot yet certify it; returning output that skips it is a contract
211    /// violation and will be rejected before outbox persistence.
212    ///
213    /// Cancellation or `Err` must not advance the acknowledged source cursor.
214    /// Repeating the call with the same inputs must return the same pending
215    /// token and payload, or reconstruct an equivalent delivery without skipping
216    /// data. Provider reads performed before cancellation are observations only.
217    ///
218    /// # Errors
219    ///
220    /// Returns [`EventSourceError`] for invalid/unsupported requests, resource
221    /// exhaustion, source unavailability, or an internal continuity failure.
222    async fn next_delivery(
223        &self,
224        request: DeliveryRequest<'_>,
225    ) -> Result<Option<Delivery>, EventSourceError>;
226
227    /// Wait until the source reports that a newer archive height may be
228    /// available. Providers without a push signal can retain the default;
229    /// [`EventService`] always keeps its polling interval as a fallback.
230    /// The request is identical to the one used for delivery production, so a
231    /// push source can wait for a required reorg anchor instead of waking on a
232    /// newer but insufficient source hint.
233    /// This future is observational: cancellation or `Err` must not consume a
234    /// unique wakeup needed for correctness, because polling remains authoritative.
235    ///
236    /// # Errors
237    ///
238    /// Returns [`EventSourceError`] when the wakeup request is invalid or the
239    /// source cannot observe updates. Failure must not consume correctness
240    /// state.
241    async fn wait_for_update(&self, _request: DeliveryRequest<'_>) -> Result<(), EventSourceError> {
242        std::future::pending().await
243    }
244
245    /// Reconcile provider-local state after the service has already committed
246    /// runtime ingestion and its durable cursor.
247    ///
248    /// This is a post-commit hook, not a transaction participant. It must be
249    /// idempotent by delivery token/sequence and treat `committed_cursor` as the
250    /// authority. Cancellation or `Err` cannot roll back the service commit; a
251    /// retry (including after restart with no pending in-memory delivery) must
252    /// converge provider-local state to that committed cursor without replaying
253    /// side effects or rejecting an already-applied acknowledgement.
254    ///
255    /// # Errors
256    ///
257    /// Returns [`EventSourceError`] when provider-local reconciliation cannot
258    /// currently converge to the already committed cursor. The error cannot
259    /// roll back service authority.
260    async fn acknowledge(
261        &self,
262        chain_id: u64,
263        acknowledgement: &Acknowledge,
264        committed_cursor: &Cursor,
265    ) -> Result<(), EventSourceError>;
266}
267
268/// Stable classification for source failures. The public protocol exposes the
269/// class and retry disposition, never provider credentials or raw internals.
270#[derive(Clone, Copy, Debug, PartialEq, Eq)]
271#[non_exhaustive]
272pub enum EventSourceErrorKind {
273    /// The requested source operation or query is malformed.
274    InvalidRequest,
275    /// The source cannot provide the requested capability.
276    Unsupported,
277    /// A configured or provider resource limit was exceeded.
278    ResourceExhausted,
279    /// The source is temporarily unavailable and the operation may be retried.
280    Unavailable,
281    /// The source encountered an unexpected non-client failure.
282    Internal,
283}
284
285/// Typed source worker failure surfaced through the versioned protocol.
286#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
287#[error("event source failed ({kind:?}): {message}")]
288#[non_exhaustive]
289pub struct EventSourceError {
290    /// Stable protocol-facing failure classification.
291    pub kind: EventSourceErrorKind,
292    /// Sanitized diagnostic message safe to return to the client.
293    pub message: String,
294}
295
296impl EventSourceError {
297    /// Construct an invalid-request source failure.
298    pub fn invalid(message: impl Into<String>) -> Self {
299        Self {
300            kind: EventSourceErrorKind::InvalidRequest,
301            message: message.into(),
302        }
303    }
304
305    /// Construct an unsupported-operation source failure.
306    pub fn unsupported(message: impl Into<String>) -> Self {
307        Self {
308            kind: EventSourceErrorKind::Unsupported,
309            message: message.into(),
310        }
311    }
312
313    /// Construct a resource-exhaustion source failure.
314    pub fn resource_exhausted(message: impl Into<String>) -> Self {
315        Self {
316            kind: EventSourceErrorKind::ResourceExhausted,
317            message: message.into(),
318        }
319    }
320
321    /// Construct a retryable source-unavailable failure.
322    pub fn unavailable(message: impl Into<String>) -> Self {
323        Self {
324            kind: EventSourceErrorKind::Unavailable,
325            message: message.into(),
326        }
327    }
328
329    /// Construct an unexpected internal source failure.
330    pub fn internal(message: impl Into<String>) -> Self {
331        Self {
332            kind: EventSourceErrorKind::Internal,
333            message: message.into(),
334        }
335    }
336}
337
338/// Durable tonic event service.
339pub struct EventService<P> {
340    store: Arc<Mutex<SessionStore>>,
341    provider: Arc<P>,
342    poll_interval: Duration,
343    client_hello_timeout: Duration,
344    source_operation_timeout: Duration,
345    client_send_timeout: Duration,
346    next_preparation_id: Arc<AtomicU64>,
347    active_sessions: Arc<StdMutex<HashSet<(String, u64)>>>,
348    limits: EventServiceLimits,
349    authorizer: Arc<dyn SessionAuthorizer>,
350    metrics: EventServiceMetrics,
351    shutdown: watch::Sender<bool>,
352}
353
354/// Cloneable signal used to close active event sessions during server shutdown.
355#[derive(Clone)]
356pub struct EventServiceShutdown {
357    sender: watch::Sender<bool>,
358}
359
360impl EventServiceShutdown {
361    /// Signal every active session loop to release its source state and lease.
362    pub fn shutdown(&self) {
363        self.sender.send_replace(true);
364    }
365}
366
367/// Invalid event-service construction parameters.
368#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
369#[non_exhaustive]
370pub enum EventServiceConfigError {
371    /// Tokio intervals cannot represent a zero-duration polling fallback.
372    #[error("event-service poll interval must be greater than zero")]
373    ZeroPollInterval,
374    #[error("event-service source operation timeout must be greater than zero")]
375    /// A zero source-operation timeout was configured.
376    ZeroSourceOperationTimeout,
377    #[error("event-service client Hello timeout must be greater than zero")]
378    /// A zero client-Hello timeout was configured.
379    ZeroClientHelloTimeout,
380    #[error("event-service client send timeout must be greater than zero")]
381    /// A zero outbound client-send timeout was configured.
382    ZeroClientSendTimeout,
383}
384
385/// Lock-free operational counters shared by all service clones.
386#[derive(Clone, Default)]
387pub struct EventServiceMetrics {
388    inner: Arc<EventServiceMetricCounters>,
389}
390
391#[derive(Default)]
392struct EventServiceMetricCounters {
393    active_sessions: AtomicU64,
394    sessions_accepted: AtomicU64,
395    authentication_rejections: AtomicU64,
396    lease_rejections: AtomicU64,
397    desired_states_committed: AtomicU64,
398    deliveries_persisted: AtomicU64,
399    deliveries_replayed: AtomicU64,
400    acknowledgements_committed: AtomicU64,
401    source_errors: AtomicU64,
402}
403
404/// Point-in-time service health counters.
405#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
406#[non_exhaustive]
407pub struct EventServiceMetricsSnapshot {
408    /// Streams currently holding a session lease.
409    pub active_sessions: u64,
410    /// Session streams accepted since service start.
411    pub sessions_accepted: u64,
412    /// Connections rejected by the configured authorizer.
413    pub authentication_rejections: u64,
414    /// Streams rejected because the session lease or capacity was unavailable.
415    pub lease_rejections: u64,
416    /// Desired-state revisions durably committed.
417    pub desired_states_committed: u64,
418    /// Deliveries durably placed in a session outbox.
419    pub deliveries_persisted: u64,
420    /// Pending deliveries replayed from a durable outbox.
421    pub deliveries_replayed: u64,
422    /// New delivery acknowledgements durably committed.
423    pub acknowledgements_committed: u64,
424    /// Source operations that returned an error.
425    pub source_errors: u64,
426}
427
428impl EventServiceMetrics {
429    /// Snapshot monotonic counters and the current active-session gauge.
430    pub fn snapshot(&self) -> EventServiceMetricsSnapshot {
431        EventServiceMetricsSnapshot {
432            active_sessions: self.inner.active_sessions.load(Ordering::Relaxed),
433            sessions_accepted: self.inner.sessions_accepted.load(Ordering::Relaxed),
434            authentication_rejections: self.inner.authentication_rejections.load(Ordering::Relaxed),
435            lease_rejections: self.inner.lease_rejections.load(Ordering::Relaxed),
436            desired_states_committed: self.inner.desired_states_committed.load(Ordering::Relaxed),
437            deliveries_persisted: self.inner.deliveries_persisted.load(Ordering::Relaxed),
438            deliveries_replayed: self.inner.deliveries_replayed.load(Ordering::Relaxed),
439            acknowledgements_committed: self
440                .inner
441                .acknowledgements_committed
442                .load(Ordering::Relaxed),
443            source_errors: self.inner.source_errors.load(Ordering::Relaxed),
444        }
445    }
446}
447
448/// Connection-level authorization hook for bearer, mTLS, or gateway identity.
449pub trait SessionAuthorizer: Send + Sync + 'static {
450    /// Authorize a new transport connection from its request metadata.
451    ///
452    /// # Errors
453    ///
454    /// Returns a client-safe rejection reason when the connection is not
455    /// authorized. The message must not contain credentials or private
456    /// authentication material.
457    fn authorize(&self, metadata: &MetadataMap) -> Result<(), String>;
458
459    /// Authorize the principal for the durable identity requested by `Hello`.
460    /// The default preserves connection-only policies while allowing mTLS or
461    /// bearer implementations to prevent cross-session takeover.
462    ///
463    /// # Errors
464    ///
465    /// Returns a client-safe rejection reason when the principal may not claim
466    /// the requested session and chain. The message must not expose secrets.
467    fn authorize_session(
468        &self,
469        metadata: &MetadataMap,
470        _session_id: &str,
471        _chain_id: u64,
472    ) -> Result<(), String> {
473        self.authorize(metadata)
474    }
475}
476
477struct AllowAll;
478
479impl SessionAuthorizer for AllowAll {
480    fn authorize(&self, _metadata: &MetadataMap) -> Result<(), String> {
481        Ok(())
482    }
483}
484
485/// Resource policy enforced before desired state reaches a provider.
486#[derive(Clone, Copy, Debug, PartialEq, Eq)]
487#[non_exhaustive]
488pub struct EventServiceLimits {
489    /// Maximum owners in one desired state.
490    pub max_owners: usize,
491    /// Maximum portable interests attached to one owner.
492    pub max_interests_per_owner: usize,
493    /// Maximum address alternatives in one log interest.
494    pub max_addresses_per_log_interest: usize,
495    /// Maximum alternatives in one indexed log topic.
496    pub max_values_per_topic: usize,
497    /// Maximum number of blocks in one explicitly bounded backfill.
498    pub max_bounded_backfill_blocks: u64,
499    /// Maximum UTF-8 bytes in a session or owner identifier.
500    pub max_identifier_bytes: usize,
501    /// Maximum interests across the entire desired state.
502    pub max_total_interests: usize,
503    /// Maximum address and topic alternatives across the desired state.
504    pub max_total_filter_values: usize,
505    /// Maximum encoded desired-state size accepted from a client.
506    pub max_desired_state_bytes: usize,
507    /// Maximum encoded delivery size persisted or sent to a client.
508    pub max_delivery_bytes: usize,
509    /// Maximum simultaneously leased session streams.
510    pub max_active_sessions: usize,
511    /// Maximum durable session identities retained in the SQLite authority.
512    pub max_persisted_sessions: usize,
513}
514
515impl Default for EventServiceLimits {
516    fn default() -> Self {
517        Self {
518            max_owners: 4_096,
519            max_interests_per_owner: 256,
520            max_addresses_per_log_interest: 4_096,
521            max_values_per_topic: 1_024,
522            max_bounded_backfill_blocks: 10_000_000,
523            max_identifier_bytes: 256,
524            max_total_interests: 16_384,
525            max_total_filter_values: 1_000_000,
526            max_desired_state_bytes: 16 * 1024 * 1024,
527            max_delivery_bytes: MAX_MESSAGE_SIZE_BYTES,
528            max_active_sessions: 4_096,
529            max_persisted_sessions: 65_536,
530        }
531    }
532}
533
534impl EventServiceLimits {
535    fn wire(self) -> ServiceLimits {
536        ServiceLimits {
537            max_owners: self.max_owners.min(u32::MAX as usize) as u32,
538            max_interests_per_owner: self.max_interests_per_owner.min(u32::MAX as usize) as u32,
539            max_addresses_per_log_interest: self
540                .max_addresses_per_log_interest
541                .min(u32::MAX as usize) as u32,
542            max_values_per_topic: self.max_values_per_topic.min(u32::MAX as usize) as u32,
543            max_bounded_backfill_blocks: self.max_bounded_backfill_blocks,
544            max_identifier_bytes: self.max_identifier_bytes.min(u32::MAX as usize) as u32,
545            max_total_interests: self.max_total_interests.min(u32::MAX as usize) as u32,
546            max_total_filter_values: self.max_total_filter_values.min(u64::MAX as usize) as u64,
547            max_desired_state_bytes: self.max_desired_state_bytes.min(u64::MAX as usize) as u64,
548            max_delivery_bytes: delivery_limit(self).min(u64::MAX as usize) as u64,
549            max_active_sessions: self.max_active_sessions.min(u32::MAX as usize) as u32,
550            max_persisted_sessions: self.max_persisted_sessions.min(u64::MAX as usize) as u64,
551        }
552    }
553}
554
555impl<P> EventService<P>
556where
557    P: EventSource,
558{
559    /// Construct a service sharing one durable session database and provider.
560    ///
561    /// # Errors
562    ///
563    /// Returns [`EventServiceConfigError::ZeroPollInterval`] when the polling
564    /// fallback is zero.
565    pub fn new(
566        store: Arc<Mutex<SessionStore>>,
567        provider: Arc<P>,
568        poll_interval: Duration,
569    ) -> Result<Self, EventServiceConfigError> {
570        if poll_interval.is_zero() {
571            return Err(EventServiceConfigError::ZeroPollInterval);
572        }
573        let (shutdown, _) = watch::channel(false);
574        Ok(Self {
575            store,
576            provider,
577            poll_interval,
578            client_hello_timeout: Duration::from_secs(10),
579            source_operation_timeout: Duration::from_secs(30),
580            client_send_timeout: Duration::from_secs(30),
581            next_preparation_id: Arc::new(AtomicU64::new(1)),
582            active_sessions: Arc::new(StdMutex::new(HashSet::new())),
583            limits: EventServiceLimits::default(),
584            authorizer: Arc::new(AllowAll),
585            metrics: EventServiceMetrics::default(),
586            shutdown,
587        })
588    }
589
590    /// Replace the default resource policy advertised and enforced by the service.
591    pub fn with_limits(mut self, limits: EventServiceLimits) -> Self {
592        self.limits = limits;
593        self
594    }
595
596    /// Bound source operations that would otherwise stall session control
597    /// traffic indefinitely. Provider wakeups remain independently cancelable.
598    ///
599    /// # Errors
600    ///
601    /// Returns [`EventServiceConfigError::ZeroSourceOperationTimeout`] when
602    /// `timeout` is zero.
603    pub fn with_source_operation_timeout(
604        mut self,
605        timeout: Duration,
606    ) -> Result<Self, EventServiceConfigError> {
607        if timeout.is_zero() {
608            return Err(EventServiceConfigError::ZeroSourceOperationTimeout);
609        }
610        self.source_operation_timeout = timeout;
611        Ok(self)
612    }
613
614    /// Bound how long a newly opened stream may remain unnegotiated. This
615    /// protects the pre-lease path, which is intentionally outside the active
616    /// durable-session quota.
617    ///
618    /// # Errors
619    ///
620    /// Returns [`EventServiceConfigError::ZeroClientHelloTimeout`] when
621    /// `timeout` is zero.
622    pub fn with_client_hello_timeout(
623        mut self,
624        timeout: Duration,
625    ) -> Result<Self, EventServiceConfigError> {
626        if timeout.is_zero() {
627            return Err(EventServiceConfigError::ZeroClientHelloTimeout);
628        }
629        self.client_hello_timeout = timeout;
630        Ok(self)
631    }
632
633    /// Bound response backpressure from a client that stops reading its stream.
634    ///
635    /// # Errors
636    ///
637    /// Returns [`EventServiceConfigError::ZeroClientSendTimeout`] when
638    /// `timeout` is zero.
639    pub fn with_client_send_timeout(
640        mut self,
641        timeout: Duration,
642    ) -> Result<Self, EventServiceConfigError> {
643        if timeout.is_zero() {
644            return Err(EventServiceConfigError::ZeroClientSendTimeout);
645        }
646        self.client_send_timeout = timeout;
647        Ok(self)
648    }
649
650    /// Install a connection-level authorization policy.
651    pub fn with_authorizer(mut self, authorizer: Arc<dyn SessionAuthorizer>) -> Self {
652        self.authorizer = authorizer;
653        self
654    }
655
656    /// Shared operational counters for health endpoints or metrics exporters.
657    pub fn metrics(&self) -> EventServiceMetrics {
658        self.metrics.clone()
659    }
660
661    /// Current number of leased sessions.
662    pub fn active_session_count(&self) -> u64 {
663        self.metrics.snapshot().active_sessions
664    }
665
666    /// Obtain a signal that should be triggered together with the tonic server's
667    /// graceful-shutdown future so long-lived bidirectional streams can close.
668    pub fn shutdown_handle(&self) -> EventServiceShutdown {
669        EventServiceShutdown {
670            sender: self.shutdown.clone(),
671        }
672    }
673
674    /// Wrap this implementation in the generated tonic server.
675    pub fn into_server(self) -> EventStreamServer<Self> {
676        EventStreamServer::new(self)
677            .max_decoding_message_size(MAX_MESSAGE_SIZE_BYTES)
678            .max_encoding_message_size(MAX_MESSAGE_SIZE_BYTES)
679    }
680}
681
682impl<P> Clone for EventService<P> {
683    fn clone(&self) -> Self {
684        Self {
685            store: Arc::clone(&self.store),
686            provider: Arc::clone(&self.provider),
687            poll_interval: self.poll_interval,
688            client_hello_timeout: self.client_hello_timeout,
689            source_operation_timeout: self.source_operation_timeout,
690            client_send_timeout: self.client_send_timeout,
691            next_preparation_id: Arc::clone(&self.next_preparation_id),
692            active_sessions: Arc::clone(&self.active_sessions),
693            limits: self.limits,
694            authorizer: Arc::clone(&self.authorizer),
695            metrics: self.metrics.clone(),
696            shutdown: self.shutdown.clone(),
697        }
698    }
699}
700
701type ResponseStream = Pin<Box<dyn Stream<Item = Result<ServerMessage, Status>> + Send + 'static>>;
702
703#[tonic::async_trait]
704impl<P> EventStream for EventService<P>
705where
706    P: EventSource,
707{
708    type SessionStream = ResponseStream;
709
710    async fn session(
711        &self,
712        request: Request<tonic::Streaming<ClientMessage>>,
713    ) -> Result<Response<Self::SessionStream>, Status> {
714        if self.authorizer.authorize(request.metadata()).is_err() {
715            self.metrics
716                .inner
717                .authentication_rejections
718                .fetch_add(1, Ordering::Relaxed);
719            return Err(Status::unauthenticated(
720                "event session authentication failed",
721            ));
722        }
723        if *self.shutdown.borrow() {
724            return Err(Status::unavailable("event service is shutting down"));
725        }
726        let request_metadata = request.metadata().clone();
727        let mut inbound = request.into_inner();
728        let (outbound, receiver) = mpsc::channel(32);
729        let store = Arc::clone(&self.store);
730        let provider = Arc::clone(&self.provider);
731        let next_preparation_id = Arc::clone(&self.next_preparation_id);
732        let active_sessions = Arc::clone(&self.active_sessions);
733        let poll_interval = self.poll_interval;
734        let client_hello_timeout = self.client_hello_timeout;
735        let source_operation_timeout = self.source_operation_timeout;
736        let client_send_timeout = self.client_send_timeout;
737        let limits = self.limits;
738        let authorizer = Arc::clone(&self.authorizer);
739        let metrics = self.metrics.clone();
740        let shutdown = wait_for_service_shutdown(self.shutdown.subscribe());
741        tokio::spawn(async move {
742            let mut interval = tokio::time::interval(poll_interval);
743            interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
744            let hello_deadline = tokio::time::sleep(client_hello_timeout);
745            tokio::pin!(hello_deadline);
746            tokio::pin!(shutdown);
747            let mut identity: Option<(String, u64)> = None;
748            let mut desired_state: Option<ApplyDesiredState> = None;
749            let mut cursor: Option<Cursor> = None;
750            let mut required_reorg_anchor: Option<BlockRef> = None;
751            let mut has_pending = false;
752            // A pending item loaded at Hello or already emitted on this stream
753            // is a replay on its next send. A newly committed activation has
754            // not been sent yet, so its first demand is the original delivery.
755            let mut pending_was_sent = false;
756            let mut delivery_requested = false;
757            let mut session_lease: Option<SessionLease> = None;
758            let mut position_confirmed = false;
759
760            loop {
761                let update_desired_state = desired_state.clone();
762                let update_cursor = cursor.clone();
763                let update_reorg_anchor = required_reorg_anchor.clone();
764                tokio::select! {
765                    () = &mut shutdown => {
766                        break;
767                    }
768                    _ = &mut hello_deadline, if identity.is_none() => {
769                        let _ = outbound.try_send(Err(Status::deadline_exceeded(
770                            "event session Hello timed out",
771                        )));
772                        break;
773                    }
774                    inbound_message = inbound.next() => {
775                        let Some(inbound_message) = inbound_message else { break };
776                        let message = match inbound_message {
777                            Ok(message) => message,
778                            Err(_) => break,
779                        };
780                        match message.message {
781                            Some(client_message::Message::Hello(hello)) => {
782                                if identity.is_some() {
783                                    if !send_error(&outbound, ErrorCode::InvalidInterest, "session already negotiated", 0, false).await { break; }
784                                    continue;
785                                }
786                                if hello.protocol_version != PROTOCOL_VERSION {
787                                    if !send_error(&outbound, ErrorCode::ProtocolVersion, "unsupported protocol version", 0, false).await { break; }
788                                    continue;
789                                }
790                                if hello.session_id.len() > limits.max_identifier_bytes {
791                                    if !send_error(&outbound, ErrorCode::ResourceExhausted, "session identifier exceeds the configured byte limit", 0, false).await { break; }
792                                    continue;
793                                }
794                                if hello.session_id.is_empty() {
795                                    if !send_error(&outbound, ErrorCode::InvalidInterest, "session identifier is empty", 0, false).await { break; }
796                                    continue;
797                                }
798                                if authorizer.authorize_session(
799                                    &request_metadata,
800                                    &hello.session_id,
801                                    hello.chain_id,
802                                ).is_err() {
803                                    metrics.inner.authentication_rejections.fetch_add(1, Ordering::Relaxed);
804                                    if !send_error(&outbound, ErrorCode::Authentication, "event session is not authorized for this identity", 0, false).await { break; }
805                                    continue;
806                                }
807                                let lease = match SessionLease::acquire(
808                                    Arc::clone(&active_sessions),
809                                    (hello.session_id.clone(), hello.chain_id),
810                                    limits.max_active_sessions,
811                                    metrics.clone(),
812                                ) {
813                                    Ok(lease) => lease,
814                                    Err(LeaseError::InUse) => {
815                                        metrics.inner.lease_rejections.fetch_add(1, Ordering::Relaxed);
816                                        if !send_error(&outbound, ErrorCode::SessionInUse, "another connection owns this session", 0, true).await { break; }
817                                        continue;
818                                    }
819                                    Err(LeaseError::Capacity) => {
820                                        metrics.inner.lease_rejections.fetch_add(1, Ordering::Relaxed);
821                                        if !send_error(&outbound, ErrorCode::ResourceExhausted, "active session capacity is exhausted", 0, true).await { break; }
822                                        continue;
823                                    }
824                                    Err(LeaseError::Poisoned) => {
825                                        let _ = send_internal(&outbound).await;
826                                        break;
827                                    }
828                                };
829                                let persisted = match store.lock().await.load(&hello.session_id, hello.chain_id) {
830                                    Ok(persisted) => persisted,
831                                    Err(_) => {
832                                        let _ = send_internal(&outbound).await;
833                                        break;
834                                    }
835                                };
836                                let committed_revision = persisted.desired_state.as_ref().map_or(0, |state| state.new_revision);
837                                let hello_position = match validate_hello_position(&hello, &persisted) {
838                                    Ok(position) => position,
839                                    Err(_) => {
840                                        if !send_error(&outbound, ErrorCode::RevisionConflict, "client resume position does not match durable service state", committed_revision, false).await { break; }
841                                        continue;
842                                    }
843                                };
844                                if persisted.pending_delivery.as_ref().is_some_and(|delivery| {
845                                    encoded_delivery_len(delivery) > MAX_MESSAGE_SIZE_BYTES
846                                }) {
847                                    if !send_error(&outbound, ErrorCode::ResourceExhausted, "durable pending delivery exceeds the hard transport limit", committed_revision, false).await { break; }
848                                    continue;
849                                }
850                                session_lease = Some(lease);
851                                let prepared = if let Some(committed) = persisted.desired_state.as_ref() {
852                                    let Some(preparation_id) = allocate_preparation_id(
853                                        next_preparation_id.as_ref(),
854                                    ) else {
855                                        let _ = send_error(
856                                            &outbound,
857                                            ErrorCode::ResourceExhausted,
858                                            "source preparation identity space is exhausted",
859                                            committed_revision,
860                                            false,
861                                        )
862                                        .await;
863                                        break;
864                                    };
865                                    if source_call(source_operation_timeout, provider.prepare_desired_state(
866                                        preparation_id, committed, persisted.acknowledged_cursor.as_ref(),
867                                    )).await.is_err() {
868                                        metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
869                                        if source_call(
870                                            source_operation_timeout,
871                                            provider.abort_desired_state(preparation_id, committed),
872                                        )
873                                        .await
874                                        .is_err()
875                                        {
876                                            metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
877                                        }
878                                        if !send_source_unavailable(&outbound).await { break; }
879                                        break;
880                                    }
881                                    Some((preparation_id, committed))
882                                } else {
883                                    None
884                                };
885                                if let Some((preparation_id, committed)) = prepared
886                                    && source_call(source_operation_timeout, provider.activate_desired_state(
887                                        preparation_id,
888                                        committed,
889                                        persisted.acknowledged_cursor.as_ref(),
890                                    )).await.is_err()
891                                {
892                                    metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
893                                    let _ = source_call(source_operation_timeout, provider.abort_desired_state(preparation_id, committed)).await;
894                                    if !send_source_unavailable(&outbound).await { break; }
895                                    break;
896                                }
897                                identity = Some((hello.session_id.clone(), hello.chain_id));
898                                position_confirmed = hello_position == HelloPosition::Confirmed;
899                                metrics.inner.sessions_accepted.fetch_add(1, Ordering::Relaxed);
900                                desired_state = persisted.desired_state.clone();
901                                cursor = persisted.acknowledged_cursor.clone();
902                                required_reorg_anchor = persisted.expected_reorg_tip.clone();
903                                has_pending = persisted.pending_delivery.is_some();
904                                pending_was_sent = has_pending;
905                                delivery_requested = false;
906                                if !send_outbound(&outbound, ServerMessage {
907                                    message: Some(server_message::Message::HelloAccepted(HelloAccepted {
908                                        protocol_version: PROTOCOL_VERSION,
909                                        session_id: hello.session_id,
910                                        chain_id: hello.chain_id,
911                                        committed_revision,
912                                        acknowledged_cursor: cursor.clone(),
913                                        desired_state: desired_state.clone(),
914                                        capabilities: Some(provider.capabilities_for_chain(hello.chain_id)),
915                                        service_limits: Some(limits.wire()),
916                                        runtime_checkpoint_position: Some(RuntimeCheckpointPosition {
917                                            cursor: persisted.runtime_checkpoint_cursor.clone(),
918                                        }),
919                                    })),
920                                }, client_send_timeout).await { break; }
921                                // Pending deliveries are replayed only in response to
922                                // DeliveryDemand. This keeps one replay per connection
923                                // demand and avoids duplicate durable items after reconnect.
924                            }
925                            Some(client_message::Message::ApplyDesiredState(request)) => {
926                                let Some((session_id, chain_id)) = &identity else {
927                                    if !send_error(&outbound, ErrorCode::ProtocolVersion, "Hello must be sent first", 0, false).await { break; }
928                                    continue;
929                                };
930                                if &request.session_id != session_id || request.chain_id != *chain_id {
931                                    if !send_error(&outbound, ErrorCode::InvalidInterest, "desired state does not match negotiated session", 0, false).await { break; }
932                                    continue;
933                                }
934                                if !position_confirmed {
935                                    if !send_error(&outbound, ErrorCode::RevisionConflict, "runtime checkpoint proof is required on a new session", desired_state.as_ref().map_or(0, |state| state.new_revision), false).await { break; }
936                                    continue;
937                                }
938                                if let Err(error) = validate_limits(&request, limits) {
939                                    if !send_error(&outbound, ErrorCode::ResourceExhausted, error, request.expected_revision, false).await { break; }
940                                    continue;
941                                }
942                                if let Err(error) = validate_desired_state(&request) {
943                                    let (code, message) = desired_state_error(&error);
944                                    if !send_error(&outbound, code, message, request.expected_revision, false).await { break; }
945                                    continue;
946                                }
947                                if request.expected_revision > i64::MAX as u64
948                                    || request.new_revision > i64::MAX as u64
949                                {
950                                    if !send_error(
951                                        &outbound,
952                                        ErrorCode::InvalidInterest,
953                                        "desired-state revision exceeds the durable storage range",
954                                        request.expected_revision,
955                                        false,
956                                    ).await { break; }
957                                    continue;
958                                }
959                                let Some(preparation_id) = allocate_preparation_id(
960                                    next_preparation_id.as_ref(),
961                                ) else {
962                                    if !send_error(
963                                        &outbound,
964                                        ErrorCode::ResourceExhausted,
965                                        "source preparation identity space is exhausted",
966                                        request.expected_revision,
967                                        false,
968                                    ).await { break; }
969                                    continue;
970                                };
971                                let prepared_cursor = match source_call(source_operation_timeout, provider.prepare_desired_state(
972                                    preparation_id,
973                                    &request,
974                                    cursor.as_ref(),
975                                )).await {
976                                    Ok(cursor) => cursor,
977                                    Err(error) => {
978                                        metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
979                                        let abort_failed = source_call(
980                                            source_operation_timeout,
981                                            provider.abort_desired_state(preparation_id, &request),
982                                        )
983                                        .await
984                                        .is_err();
985                                        if abort_failed {
986                                            metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
987                                            // Preparation may have partially mutated provider
988                                            // state. Fence this generation and let bounded
989                                            // session cleanup decide whether the lease can move.
990                                            let _ = send_source_unavailable(&outbound).await;
991                                            break;
992                                        }
993                                        let (code, message, retryable) = source_error_response(&error);
994                                        if !send_error(&outbound, code, message, request.expected_revision, retryable).await { break; }
995                                        continue;
996                                    }
997                                };
998                                let result = store.lock().await.apply_desired_state_with_cursor_and_limit(
999                                    request.clone(),
1000                                    prepared_cursor.as_ref(),
1001                                    delivery_limit(limits),
1002                                    limits.max_persisted_sessions,
1003                                );
1004                                match result {
1005                                    Ok((applied, newly_committed)) => {
1006                                        if newly_committed {
1007                                            metrics.inner.desired_states_committed.fetch_add(1, Ordering::Relaxed);
1008                                            metrics.inner.deliveries_persisted.fetch_add(1, Ordering::Relaxed);
1009                                        }
1010                                        if source_call(source_operation_timeout, provider.activate_desired_state(
1011                                            preparation_id,
1012                                            &request,
1013                                            cursor.as_ref(),
1014                                        )).await.is_err() {
1015                                            metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1016                                            let _ = source_call(source_operation_timeout, provider.abort_desired_state(preparation_id, &request)).await;
1017                                            // The desired state is already durable. Closing the
1018                                            // stream makes this an explicitly uncertain result;
1019                                            // reconnect will restore and reactivate that revision.
1020                                            if !send_source_unavailable(&outbound).await { break; }
1021                                            break;
1022                                        }
1023                                        desired_state = Some(request.clone());
1024                                        let pending = match store.lock().await.load(session_id, *chain_id) {
1025                                            Ok(persisted) => persisted.pending_delivery,
1026                                            Err(_) => {
1027                                                let _ = send_internal(&outbound).await;
1028                                                break;
1029                                            }
1030                                        };
1031                                        has_pending = pending.is_some();
1032                                        pending_was_sent = !newly_committed && has_pending;
1033                                        delivery_requested = false;
1034                                        if !send_outbound(&outbound, ServerMessage {
1035                                            message: Some(server_message::Message::DesiredStateApplied(applied)),
1036                                        }, client_send_timeout).await { break; }
1037                                        // The activation remains in the durable outbox and
1038                                        // is delivered on the client's next DeliveryDemand.
1039                                    }
1040                                    Err(error) => {
1041                                        if source_call(source_operation_timeout, provider.abort_desired_state(preparation_id, &request)).await.is_err() {
1042                                            metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1043                                            // The durable CAS rejected the candidate, but failed
1044                                            // compensation leaves provider state indeterminate.
1045                                            // Never accept another operation on this generation.
1046                                            let _ = send_source_unavailable(&outbound).await;
1047                                            break;
1048                                        }
1049                                        let (code, committed, retryable) = store_error_code(&error);
1050                                        if code == ErrorCode::Internal {
1051                                            let _ = send_internal(&outbound).await;
1052                                            break;
1053                                        }
1054                                        if !send_error(&outbound, code, store_error_message(&error), committed, retryable).await { break; }
1055                                    }
1056                                }
1057                            }
1058                            Some(client_message::Message::Acknowledge(acknowledgement)) => {
1059                                let Some((session_id, chain_id)) = &identity else {
1060                                    if !send_error(&outbound, ErrorCode::ProtocolVersion, "Hello must be sent first", 0, false).await { break; }
1061                                    continue;
1062                                };
1063                                if &acknowledgement.session_id != session_id {
1064                                    if !send_error(&outbound, ErrorCode::InvalidInterest, "acknowledgement does not match negotiated session", 0, false).await { break; }
1065                                    continue;
1066                                }
1067                                if !position_confirmed {
1068                                    if !send_error(&outbound, ErrorCode::RevisionConflict, "runtime checkpoint proof is required on a new session", desired_state.as_ref().map_or(0, |state| state.new_revision), false).await { break; }
1069                                    continue;
1070                                }
1071                                let acknowledgement_result = {
1072                                    let mut authority = store.lock().await;
1073                                    match authority.acknowledge_with_status(*chain_id, &acknowledgement) {
1074                                        Ok((committed_cursor, newly_committed)) => authority
1075                                            .load(session_id, *chain_id)
1076                                            .map(|persisted| {
1077                                                (
1078                                                    committed_cursor,
1079                                                    newly_committed,
1080                                                    persisted.expected_reorg_tip,
1081                                                )
1082                                            }),
1083                                        Err(error) => Err(error),
1084                                    }
1085                                };
1086                                match acknowledgement_result {
1087                                    Ok((committed_cursor, newly_committed, committed_reorg_anchor)) => {
1088                                        required_reorg_anchor = committed_reorg_anchor;
1089                                        if newly_committed {
1090                                            metrics
1091                                                .inner
1092                                                .acknowledgements_committed
1093                                                .fetch_add(1, Ordering::Relaxed);
1094                                        }
1095                                        if source_call(source_operation_timeout, provider.acknowledge(
1096                                            *chain_id,
1097                                            &acknowledgement,
1098                                            &committed_cursor,
1099                                        )).await.is_err()
1100                                        {
1101                                            metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1102                                            if !send_source_unavailable(&outbound).await { break; }
1103                                            break;
1104                                        }
1105                                        if !send_outbound(&outbound, ServerMessage {
1106                                            message: Some(server_message::Message::AcknowledgementCommitted(
1107                                                AcknowledgementCommitted {
1108                                                    session_id: session_id.clone(),
1109                                                    sequence: acknowledgement.sequence,
1110                                                    cursor: Some(committed_cursor.clone()),
1111                                                },
1112                                            )),
1113                                        }, client_send_timeout).await {
1114                                            break;
1115                                        }
1116                                        cursor = Some(committed_cursor);
1117                                        has_pending = false;
1118                                        pending_was_sent = false;
1119                                        delivery_requested = false;
1120                                        interval.reset_immediately();
1121                                    }
1122                                    Err(_) => {
1123                                        let _ = send_internal(&outbound).await;
1124                                        break;
1125                                    }
1126                                }
1127                            }
1128                            Some(client_message::Message::Heartbeat(heartbeat)) => {
1129                                if identity.is_none() {
1130                                    if !send_error(&outbound, ErrorCode::ProtocolVersion, "Hello must be sent first", 0, false).await { break; }
1131                                    continue;
1132                                }
1133                                if !send_outbound(&outbound, ServerMessage {
1134                                    message: Some(server_message::Message::Heartbeat(heartbeat)),
1135                                }, client_send_timeout).await { break; }
1136                            }
1137                            Some(client_message::Message::DeliveryDemand(demand)) => {
1138                                let Some((session_id, chain_id)) = &identity else {
1139                                    if !send_error(&outbound, ErrorCode::ProtocolVersion, "Hello must be sent first", 0, false).await { break; }
1140                                    continue;
1141                                };
1142                                if &demand.session_id != session_id {
1143                                    if !send_error(&outbound, ErrorCode::InvalidInterest, "delivery demand does not match negotiated session", 0, false).await { break; }
1144                                    continue;
1145                                }
1146                                if !position_confirmed {
1147                                    if !send_error(&outbound, ErrorCode::RevisionConflict, "runtime checkpoint proof is required on a new session", desired_state.as_ref().map_or(0, |state| state.new_revision), false).await { break; }
1148                                    continue;
1149                                }
1150                                if has_pending {
1151                                    let pending = store.lock().await.load(session_id, *chain_id)
1152                                        .ok()
1153                                        .and_then(|persisted| persisted.pending_delivery);
1154                                    match pending {
1155                                        Some(delivery) if encoded_delivery_len(&delivery) <= MAX_MESSAGE_SIZE_BYTES => {
1156                                            if pending_was_sent {
1157                                                metrics.inner.deliveries_replayed.fetch_add(1, Ordering::Relaxed);
1158                                            }
1159                                            if !send_outbound(&outbound, ServerMessage {
1160                                                message: Some(server_message::Message::Delivery(delivery)),
1161                                            }, client_send_timeout).await { break; }
1162                                            pending_was_sent = true;
1163                                        }
1164                                        Some(_) => {
1165                                            if !send_error(&outbound, ErrorCode::ResourceExhausted, "durable pending delivery exceeds the hard transport limit", desired_state.as_ref().map_or(0, |state| state.new_revision), false).await { break; }
1166                                        }
1167                                        None => {
1168                                            has_pending = false;
1169                                            pending_was_sent = false;
1170                                            delivery_requested = true;
1171                                            interval.reset_immediately();
1172                                        }
1173                                    }
1174                                } else {
1175                                    delivery_requested = true;
1176                                    interval.reset_immediately();
1177                                }
1178                            }
1179                            None => {
1180                                if !send_error(&outbound, ErrorCode::InvalidInterest, "empty client message", 0, false).await { break; }
1181                            }
1182                        }
1183                    }
1184                    _ = interval.tick(), if identity.is_some() && position_confirmed && desired_state.is_some() && delivery_requested && !has_pending => {
1185                        let Some(desired) = desired_state.as_ref() else {
1186                            continue;
1187                        };
1188                        let request = DeliveryRequest::new(desired, cursor.as_ref())
1189                            .with_required_reorg_anchor(required_reorg_anchor.as_ref());
1190                        match source_call(source_operation_timeout, provider.next_delivery(request)).await {
1191                            Ok(Some(delivery)) => {
1192                                if encoded_delivery_len(&delivery) > delivery_limit(limits) {
1193                                    delivery_requested = false;
1194                                    if !send_error(&outbound, ErrorCode::ResourceExhausted, "source delivery exceeds the configured encoded-byte limit", desired.new_revision, false).await { break; }
1195                                    continue;
1196                                }
1197                                let save_result = {
1198                                    store.lock().await.save_pending_with_status(desired, &delivery)
1199                                };
1200                                match save_result {
1201                                    Ok(newly_persisted) => {
1202                                        if newly_persisted {
1203                                            metrics.inner.deliveries_persisted.fetch_add(1, Ordering::Relaxed);
1204                                        }
1205                                        has_pending = true;
1206                                        pending_was_sent = true;
1207                                        delivery_requested = false;
1208                                        if !send_outbound(&outbound, ServerMessage {
1209                                            message: Some(server_message::Message::Delivery(delivery)),
1210                                        }, client_send_timeout).await { break; }
1211                                    }
1212                                    Err(_) => {
1213                                        let _ = send_internal(&outbound).await;
1214                                        break;
1215                                    }
1216                                }
1217                            }
1218                            Ok(None) => {}
1219                            Err(error) => {
1220                                metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1221                                delivery_requested = false;
1222                                let (code, message, retryable) = source_error_response(&error);
1223                                if !send_error(&outbound, code, message, desired.new_revision, retryable).await { break; }
1224                            }
1225                        }
1226                    }
1227                    update = wait_for_provider_update(
1228                        provider.as_ref(),
1229                        update_desired_state.as_ref(),
1230                        update_cursor.as_ref(),
1231                        update_reorg_anchor.as_ref(),
1232                    ), if identity.is_some() && position_confirmed && delivery_requested && !has_pending => {
1233                        match update {
1234                            Ok(()) => {
1235                                let Some(desired) = desired_state.as_ref() else {
1236                                    continue;
1237                                };
1238                                let request = DeliveryRequest::new(desired, cursor.as_ref())
1239                                    .with_required_reorg_anchor(required_reorg_anchor.as_ref());
1240                                match source_call(source_operation_timeout, provider.next_delivery(request)).await {
1241                                    Ok(Some(delivery)) => {
1242                                        if encoded_delivery_len(&delivery) > delivery_limit(limits) {
1243                                            delivery_requested = false;
1244                                            if !send_error(&outbound, ErrorCode::ResourceExhausted, "source delivery exceeds the configured encoded-byte limit", desired.new_revision, false).await { break; }
1245                                            continue;
1246                                        }
1247                                        let save_result = {
1248                                            store.lock().await.save_pending_with_status(desired, &delivery)
1249                                        };
1250                                        match save_result {
1251                                            Ok(newly_persisted) => {
1252                                                if newly_persisted {
1253                                                    metrics.inner.deliveries_persisted.fetch_add(1, Ordering::Relaxed);
1254                                                }
1255                                                has_pending = true;
1256                                                pending_was_sent = true;
1257                                                delivery_requested = false;
1258                                                if !send_outbound(&outbound, ServerMessage {
1259                                                    message: Some(server_message::Message::Delivery(delivery)),
1260                                                }, client_send_timeout).await { break; }
1261                                            }
1262                                            Err(_) => {
1263                                                let _ = send_internal(&outbound).await;
1264                                                break;
1265                                            }
1266                                        }
1267                                    }
1268                                    Ok(None) => {}
1269                                    Err(error) => {
1270                                        metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1271                                        delivery_requested = false;
1272                                        let (code, message, retryable) = source_error_response(&error);
1273                                        if !send_error(&outbound, code, message, desired.new_revision, retryable).await { break; }
1274                                    }
1275                                }
1276                            }
1277                            Err(error) => {
1278                                metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1279                                delivery_requested = false;
1280                                let (code, message, retryable) = source_error_response(&error);
1281                                if !send_error(&outbound, code, message, desired_state.as_ref().map_or(0, |state| state.new_revision), retryable).await { break; }
1282                            }
1283                        }
1284                    }
1285                }
1286            }
1287            if let Some(lease) = session_lease.as_mut() {
1288                let release = tokio::time::timeout(
1289                    source_operation_timeout,
1290                    provider.release_session(&lease.key.0, lease.key.1),
1291                )
1292                .await;
1293                if !matches!(release, Ok(Ok(()))) {
1294                    if matches!(release, Ok(Err(_))) {
1295                        metrics.inner.source_errors.fetch_add(1, Ordering::Relaxed);
1296                    }
1297                    // Failed or canceled cleanup may have partially mutated
1298                    // provider state. Preserve the lease so a replacement
1299                    // generation cannot race it.
1300                    lease.retain_on_drop();
1301                }
1302            }
1303            drop(session_lease);
1304        });
1305        Ok(Response::new(Box::pin(ReceiverStream::new(receiver))))
1306    }
1307}
1308
1309struct SessionLease {
1310    registry: Arc<StdMutex<HashSet<(String, u64)>>>,
1311    key: (String, u64),
1312    metrics: EventServiceMetrics,
1313    retain_registry_entry: bool,
1314}
1315
1316impl SessionLease {
1317    fn acquire(
1318        registry: Arc<StdMutex<HashSet<(String, u64)>>>,
1319        key: (String, u64),
1320        max_active_sessions: usize,
1321        metrics: EventServiceMetrics,
1322    ) -> Result<Self, LeaseError> {
1323        let mut sessions = registry.lock().map_err(|_| LeaseError::Poisoned)?;
1324        if sessions.contains(&key) {
1325            return Err(LeaseError::InUse);
1326        }
1327        if sessions.len() >= max_active_sessions {
1328            return Err(LeaseError::Capacity);
1329        }
1330        sessions.insert(key.clone());
1331        drop(sessions);
1332        metrics
1333            .inner
1334            .active_sessions
1335            .fetch_add(1, Ordering::Relaxed);
1336        Ok(Self {
1337            registry,
1338            key,
1339            metrics,
1340            retain_registry_entry: false,
1341        })
1342    }
1343
1344    fn retain_on_drop(&mut self) {
1345        self.retain_registry_entry = true;
1346    }
1347}
1348
1349impl Drop for SessionLease {
1350    fn drop(&mut self) {
1351        if self.retain_registry_entry {
1352            return;
1353        }
1354        if let Ok(mut sessions) = self.registry.lock()
1355            && sessions.remove(&self.key)
1356        {
1357            self.metrics
1358                .inner
1359                .active_sessions
1360                .fetch_sub(1, Ordering::Relaxed);
1361        }
1362    }
1363}
1364
1365enum LeaseError {
1366    InUse,
1367    Capacity,
1368    Poisoned,
1369}
1370
1371fn allocate_preparation_id(next: &AtomicU64) -> Option<PreparationId> {
1372    next.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| {
1373        value.checked_add(1)
1374    })
1375    .ok()
1376    .map(PreparationId::new)
1377}
1378
1379async fn source_call<T>(
1380    timeout: Duration,
1381    operation: impl std::future::Future<Output = Result<T, EventSourceError>>,
1382) -> Result<T, EventSourceError> {
1383    tokio::time::timeout(timeout, operation)
1384        .await
1385        .map_err(|_| EventSourceError::unavailable("source operation timed out"))?
1386}
1387
1388async fn wait_for_service_shutdown(mut shutdown: watch::Receiver<bool>) {
1389    if *shutdown.borrow_and_update() {
1390        return;
1391    }
1392    loop {
1393        if shutdown.changed().await.is_err() {
1394            return;
1395        }
1396        if *shutdown.borrow_and_update() {
1397            return;
1398        }
1399    }
1400}
1401
1402async fn wait_for_provider_update<P>(
1403    provider: &P,
1404    desired_state: Option<&ApplyDesiredState>,
1405    acknowledged_cursor: Option<&Cursor>,
1406    required_reorg_anchor: Option<&BlockRef>,
1407) -> Result<(), EventSourceError>
1408where
1409    P: EventSource,
1410{
1411    let Some(desired_state) = desired_state else {
1412        return std::future::pending().await;
1413    };
1414    provider
1415        .wait_for_update(
1416            DeliveryRequest::new(desired_state, acknowledged_cursor)
1417                .with_required_reorg_anchor(required_reorg_anchor),
1418        )
1419        .await
1420}
1421
1422async fn send_internal(sender: &mpsc::Sender<Result<ServerMessage, Status>>) -> bool {
1423    sender
1424        .try_send(Err(Status::unavailable("internal event service failure")))
1425        .is_ok()
1426}
1427
1428async fn send_source_unavailable(sender: &mpsc::Sender<Result<ServerMessage, Status>>) -> bool {
1429    sender
1430        .try_send(Err(Status::unavailable("event source unavailable")))
1431        .is_ok()
1432}
1433
1434async fn send_outbound(
1435    sender: &mpsc::Sender<Result<ServerMessage, Status>>,
1436    message: ServerMessage,
1437    timeout: Duration,
1438) -> bool {
1439    matches!(
1440        tokio::time::timeout(timeout, sender.send(Ok(message))).await,
1441        Ok(Ok(()))
1442    )
1443}
1444
1445async fn send_error(
1446    sender: &mpsc::Sender<Result<ServerMessage, Status>>,
1447    code: ErrorCode,
1448    message: &str,
1449    committed_revision: u64,
1450    retryable: bool,
1451) -> bool {
1452    sender
1453        .try_send(Ok(ServerMessage {
1454            message: Some(server_message::Message::Error(ProtocolError {
1455                code: code.into(),
1456                message: message.to_owned(),
1457                committed_revision,
1458                retryable,
1459            })),
1460        }))
1461        .is_ok()
1462}
1463
1464fn store_error_code(error: &SessionStoreError) -> (ErrorCode, u64, bool) {
1465    match error {
1466        SessionStoreError::RevisionConflict { committed, .. } => {
1467            (ErrorCode::RevisionConflict, *committed, false)
1468        }
1469        SessionStoreError::DesiredState(DesiredStateError::ProtocolVersion { .. }) => {
1470            (ErrorCode::ProtocolVersion, 0, false)
1471        }
1472        SessionStoreError::DesiredState(DesiredStateError::UnsupportedInterest { .. }) => {
1473            (ErrorCode::UnsupportedInterest, 0, false)
1474        }
1475        SessionStoreError::DesiredState(DesiredStateError::InvalidState(_)) => {
1476            (ErrorCode::InvalidInterest, 0, false)
1477        }
1478        SessionStoreError::PendingDelivery => (ErrorCode::PendingDelivery, 0, true),
1479        SessionStoreError::PersistedSessionLimit { .. } => (ErrorCode::ResourceExhausted, 0, false),
1480        SessionStoreError::SequenceOverflow => (ErrorCode::ResourceExhausted, 0, false),
1481        SessionStoreError::IntegerRange(_) => (ErrorCode::InvalidInterest, 0, false),
1482        SessionStoreError::DeliveryTooLarge { .. } => (ErrorCode::ResourceExhausted, 0, false),
1483        _ => (ErrorCode::Internal, 0, true),
1484    }
1485}
1486
1487fn store_error_message(error: &SessionStoreError) -> &'static str {
1488    match error {
1489        SessionStoreError::RevisionConflict { .. } => "desired-state revision conflict",
1490        SessionStoreError::DesiredState(DesiredStateError::ProtocolVersion { .. }) => {
1491            "unsupported protocol version"
1492        }
1493        SessionStoreError::DesiredState(DesiredStateError::UnsupportedInterest { .. }) => {
1494            "desired state contains an unsupported interest"
1495        }
1496        SessionStoreError::DesiredState(DesiredStateError::InvalidState(_)) => {
1497            "desired state is invalid"
1498        }
1499        SessionStoreError::PendingDelivery => {
1500            "pending delivery must be acknowledged before desired state changes"
1501        }
1502        SessionStoreError::PersistedSessionLimit { .. } => {
1503            "persisted session capacity is exhausted"
1504        }
1505        SessionStoreError::SequenceOverflow => "delivery sequence space is exhausted",
1506        SessionStoreError::IntegerRange(_) => "value exceeds the durable storage range",
1507        SessionStoreError::DeliveryTooLarge { .. } => {
1508            "activation delivery exceeds the configured encoded-byte limit"
1509        }
1510        _ => "internal event service failure",
1511    }
1512}
1513
1514fn desired_state_error(error: &DesiredStateError) -> (ErrorCode, &'static str) {
1515    match error {
1516        DesiredStateError::ProtocolVersion { .. } => {
1517            (ErrorCode::ProtocolVersion, "unsupported protocol version")
1518        }
1519        DesiredStateError::UnsupportedInterest { .. } => (
1520            ErrorCode::UnsupportedInterest,
1521            "desired state contains an unsupported interest",
1522        ),
1523        DesiredStateError::InvalidState(_) => {
1524            (ErrorCode::InvalidInterest, "desired state is invalid")
1525        }
1526        DesiredStateError::RevisionConflict { .. } => (
1527            ErrorCode::RevisionConflict,
1528            "desired-state revision conflict",
1529        ),
1530    }
1531}
1532
1533fn source_error_response(error: &EventSourceError) -> (ErrorCode, &'static str, bool) {
1534    match error.kind {
1535        EventSourceErrorKind::InvalidRequest => (
1536            ErrorCode::InvalidInterest,
1537            "event source rejected the request",
1538            false,
1539        ),
1540        EventSourceErrorKind::Unsupported => (
1541            ErrorCode::UnsupportedInterest,
1542            "event source does not support the request",
1543            false,
1544        ),
1545        EventSourceErrorKind::ResourceExhausted => (
1546            ErrorCode::ResourceExhausted,
1547            "event source resource limit exceeded",
1548            true,
1549        ),
1550        EventSourceErrorKind::Unavailable => (
1551            ErrorCode::SourceUnavailable,
1552            "event source is temporarily unavailable",
1553            true,
1554        ),
1555        EventSourceErrorKind::Internal => {
1556            (ErrorCode::Internal, "internal event source failure", true)
1557        }
1558    }
1559}
1560
1561fn encoded_delivery_len(delivery: &Delivery) -> usize {
1562    let delivery_bytes = delivery.encoded_len();
1563    delivery_bytes
1564        .saturating_add(prost::length_delimiter_len(delivery_bytes))
1565        .saturating_add(1)
1566}
1567
1568#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1569enum HelloPosition {
1570    Confirmed,
1571    DiscoveryOnly,
1572}
1573
1574fn validate_hello_position(
1575    hello: &Hello,
1576    persisted: &PersistedSession,
1577) -> Result<HelloPosition, &'static str> {
1578    let transport_sequence = persisted
1579        .acknowledged_cursor
1580        .as_ref()
1581        .map_or(0, |cursor| cursor.batch_sequence);
1582    let runtime_sequence = persisted
1583        .runtime_checkpoint_cursor
1584        .as_ref()
1585        .map_or(0, |cursor| cursor.batch_sequence);
1586    let Some(resume) = hello.pending_delivery_resume.as_ref() else {
1587        return if hello.acknowledged_sequence == runtime_sequence {
1588            Ok(HelloPosition::Confirmed)
1589        } else if hello.acknowledged_sequence == 0 {
1590            // A proof-free sequence-zero connection may inspect authority so a
1591            // synchronous runtime restore hook can discover the exact cursor.
1592            // The stream remains server-gated until a new proof-bearing Hello.
1593            Ok(HelloPosition::DiscoveryOnly)
1594        } else {
1595            Err("client position does not equal runtime checkpoint authority")
1596        };
1597    };
1598
1599    if hello.acknowledged_sequence == runtime_sequence {
1600        let cursor = persisted
1601            .runtime_checkpoint_cursor
1602            .as_ref()
1603            .ok_or("zero sequence cannot carry a pending-delivery proof")?;
1604        resume_matches_cursor(
1605            resume,
1606            hello.acknowledged_sequence.to_be_bytes().as_slice(),
1607            cursor,
1608        )?;
1609        return Ok(HelloPosition::Confirmed);
1610    }
1611
1612    let expected_pending = transport_sequence
1613        .checked_add(1)
1614        .ok_or("durable delivery sequence is exhausted")?;
1615    let pending = persisted
1616        .pending_delivery
1617        .as_ref()
1618        .ok_or("service has no pending delivery matching the restored position")?;
1619    let cursor = pending
1620        .cursor
1621        .as_ref()
1622        .ok_or("pending delivery cursor is missing")?;
1623    if hello.acknowledged_sequence != expected_pending
1624        || pending.sequence != expected_pending
1625        || pending.delivery_token != resume.delivery_token
1626    {
1627        return Err("pending delivery sequence or token does not match");
1628    }
1629    resume_matches_cursor(resume, &pending.delivery_token, cursor)?;
1630    Ok(HelloPosition::Confirmed)
1631}
1632
1633fn resume_matches_cursor(
1634    resume: &PendingDeliveryResume,
1635    expected_token: &[u8],
1636    cursor: &Cursor,
1637) -> Result<(), &'static str> {
1638    let checkpoint_matches = if cursor.provider_checkpoint.is_empty() {
1639        resume.provider_checkpoint.is_none()
1640    } else {
1641        resume.provider_checkpoint.as_deref() == Some(cursor.provider_checkpoint.as_slice())
1642    };
1643    if resume.delivery_token != expected_token
1644        || !checkpoint_matches
1645        || resume.coverage_head.as_ref() != cursor.canonical_head.as_ref()
1646    {
1647        return Err("pending-delivery proof does not match its cursor");
1648    }
1649    Ok(())
1650}
1651
1652const fn delivery_limit(limits: EventServiceLimits) -> usize {
1653    if limits.max_delivery_bytes < MAX_MESSAGE_SIZE_BYTES {
1654        limits.max_delivery_bytes
1655    } else {
1656        MAX_MESSAGE_SIZE_BYTES
1657    }
1658}
1659
1660fn validate_limits(
1661    request: &ApplyDesiredState,
1662    limits: EventServiceLimits,
1663) -> Result<(), &'static str> {
1664    if request.session_id.len() > limits.max_identifier_bytes {
1665        return Err("session identifier exceeds the configured byte limit");
1666    }
1667    if request.owners.len() > limits.max_owners {
1668        return Err("owner count exceeds the configured limit");
1669    }
1670    if request.encoded_len() > limits.max_desired_state_bytes {
1671        return Err("desired state exceeds the configured encoded-byte limit");
1672    }
1673    let mut total_interests = 0usize;
1674    let mut total_filter_values = 0usize;
1675    for owner in &request.owners {
1676        if owner.owner_id.len() > limits.max_identifier_bytes {
1677            return Err("owner identifier exceeds the configured byte limit");
1678        }
1679        if owner.interests.len() > limits.max_interests_per_owner {
1680            return Err("owner interest count exceeds the configured limit");
1681        }
1682        total_interests = total_interests.saturating_add(owner.interests.len());
1683        if total_interests > limits.max_total_interests {
1684            return Err("total interest count exceeds the configured limit");
1685        }
1686        if owner.backfill.as_ref().is_some_and(|backfill| {
1687            backfill.to_block_excl.is_some_and(|end| {
1688                end.saturating_sub(backfill.from_block) > limits.max_bounded_backfill_blocks
1689            })
1690        }) {
1691            return Err("bounded backfill exceeds the configured block limit");
1692        }
1693        for interest in &owner.interests {
1694            if let Some(portable_interest::Kind::Log(log)) = &interest.kind {
1695                total_filter_values = total_filter_values.saturating_add(log.addresses.len());
1696                total_filter_values = total_filter_values.saturating_add(
1697                    log.topics
1698                        .iter()
1699                        .map(|topic| topic.values.len())
1700                        .fold(0usize, usize::saturating_add),
1701                );
1702                if total_filter_values > limits.max_total_filter_values {
1703                    return Err("total log filter value count exceeds the configured limit");
1704                }
1705                if log.addresses.len() > limits.max_addresses_per_log_interest {
1706                    return Err("log address count exceeds the configured limit");
1707                }
1708                if log
1709                    .topics
1710                    .iter()
1711                    .any(|topic| topic.values.len() > limits.max_values_per_topic)
1712                {
1713                    return Err("log topic value count exceeds the configured limit");
1714                }
1715            }
1716        }
1717    }
1718    Ok(())
1719}
1720
1721#[cfg(test)]
1722mod tests {
1723    use std::sync::atomic::AtomicU64;
1724
1725    use super::allocate_preparation_id;
1726
1727    #[test]
1728    fn preparation_identifiers_never_wrap_or_repeat() {
1729        let next = AtomicU64::new(u64::MAX - 1);
1730        assert!(allocate_preparation_id(&next).is_some());
1731        assert!(allocate_preparation_id(&next).is_none());
1732        assert!(allocate_preparation_id(&next).is_none());
1733    }
1734}