Skip to main content

evm_fork_cache_remote/
lib.rs

1#![doc = include_str!("../README.md")]
2#![warn(missing_docs)]
3
4mod hybrid;
5
6pub use hybrid::{
7    HYBRID_MAX_CANONICAL_HISTORY, HYBRID_MAX_HANDLER_ID_BYTES, HYBRID_MAX_RECENT_INPUTS,
8    HYBRID_MAX_RECENT_OWNER_ENTRIES, HYBRID_MAX_SOURCE_CHECKPOINT_BYTES,
9    HYBRID_MAX_SOURCE_DELIVERY_TOKEN_BYTES, HybridConfig, HybridPhase, HybridSource,
10    HybridSubscriber,
11};
12
13use std::{
14    collections::{HashMap, HashSet, VecDeque},
15    marker::PhantomData,
16    time::Duration,
17};
18
19use alloy_consensus::Header as ConsensusHeader;
20use alloy_network::{Ethereum, Network};
21use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog, U256, keccak256};
22use alloy_rlp::Decodable;
23use alloy_rpc_types_eth::{Filter, FilterBlockOption, Header as RpcHeader, Log as RpcLog};
24use async_trait::async_trait;
25use evm_fork_cache::reactive::{
26    BlockInterest as RuntimeBlockInterest, BlockInterestMode, BlockRef as RuntimeBlockRef,
27    ChainControl, ChainStatus, DeliveryAudience, DeliveryScope as RuntimeDeliveryScope,
28    EventSubscriber, HandlerId, InputSource, InterestOwnerSubscriber,
29    LogInterest as RuntimeLogInterest, ReactiveContext, ReactiveInput, ReactiveInputBatch,
30    ReactiveInputDelivery, ReactiveInputRecord, ReactiveInterest, SubscriberBackfill,
31    SubscriberCapabilities, SubscriberCapability, SubscriberCheckpoint, SubscriberDeliveryToken,
32    SubscriberError, SubscriberNextBatch, SubscriberOperation, SubscriberPayloadCommitment,
33    SubscriberResumePosition,
34};
35use evm_fork_cache_event_protocol::v1::{
36    Acknowledge, ApplyDesiredState, Backfill, BlockInterest, BlockMode, BlockRef as WireBlockRef,
37    Capability as WireCapability, ClientMessage, Cursor, Delivery, DeliveryDemand,
38    DeliveryScope as WireDeliveryScope, DesiredStateApplied, ErrorCode, FinalityKind, Hello,
39    HelloAccepted, LogEvent as WireLog, LogInterest as WireLogInterest, OwnerInterests,
40    PendingDeliveryResume, PortableInterest, ProtocolError, ServerMessage, SourceCapabilities,
41    TopicValues, chain_event, client_message, delivery, event_stream_client::EventStreamClient,
42    portable_interest, server_message,
43};
44use evm_fork_cache_event_protocol::{MAX_MESSAGE_SIZE_BYTES, PROTOCOL_VERSION};
45use prost::Message;
46use tokio::sync::mpsc;
47use tokio_stream::wrappers::ReceiverStream;
48use tonic::transport::Channel;
49
50/// Compile provider-portable interests while leaving local matchers and route
51/// extraction on the runtime machine.
52///
53/// # Errors
54///
55/// Returns [`RemoteError::UnsupportedFilterBlockOption`] for log filters with
56/// a block range or block hash, and [`RemoteError::UnsupportedInterest`] for
57/// full-block or pending-transaction interests that protocol v1 cannot carry.
58pub fn compile_portable_interests<N: Network>(
59    interests: &[ReactiveInterest<N>],
60) -> Result<Vec<PortableInterest>, RemoteError> {
61    interests
62        .iter()
63        .map(|interest| match interest {
64            ReactiveInterest::Logs(log) => {
65                if log.provider_filter.block_option != FilterBlockOption::default() {
66                    return Err(RemoteError::UnsupportedFilterBlockOption);
67                }
68                let mut addresses: Vec<_> = log
69                    .provider_filter
70                    .address
71                    .iter()
72                    .map(|address| address.as_slice().to_vec())
73                    .collect();
74                addresses.sort();
75
76                let last_topic = log
77                    .provider_filter
78                    .topics
79                    .iter()
80                    .rposition(|topic| !topic.is_empty());
81                let topics = last_topic.map_or_else(Vec::new, |last| {
82                    log.provider_filter.topics[..=last]
83                        .iter()
84                        .map(|topic| {
85                            let mut values: Vec<_> = topic
86                                .iter()
87                                .map(|value| value.as_slice().to_vec())
88                                .collect();
89                            values.sort();
90                            TopicValues { values }
91                        })
92                        .collect()
93                });
94                Ok(PortableInterest {
95                    kind: Some(portable_interest::Kind::Log(WireLogInterest {
96                        addresses,
97                        topics,
98                    })),
99                })
100            }
101            ReactiveInterest::Blocks(block) => {
102                let mode = match block.mode {
103                    BlockInterestMode::Header => BlockMode::Header,
104                    BlockInterestMode::FullBlock => {
105                        return Err(RemoteError::UnsupportedInterest("full block"));
106                    }
107                };
108                Ok(PortableInterest {
109                    kind: Some(portable_interest::Kind::Block(BlockInterest {
110                        mode: mode.into(),
111                    })),
112                })
113            }
114            ReactiveInterest::PendingTransactions(_) => {
115                Err(RemoteError::UnsupportedInterest("pending transaction"))
116            }
117        })
118        .collect()
119}
120
121/// Remote interest compilation or transport failure.
122#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
123#[non_exhaustive]
124pub enum RemoteError {
125    /// The current service does not support this reactive interest.
126    #[error("remote subscriber does not support {0} interests")]
127    UnsupportedInterest(&'static str),
128    /// Block ranges belong to owner backfill/cursor state, not provider filters.
129    #[error("remote log interests cannot carry an Alloy filter block range or block hash")]
130    UnsupportedFilterBlockOption,
131    /// The service returned desired state that cannot reconstruct a subscriber.
132    #[error("invalid authoritative desired state: {0}")]
133    InvalidAuthoritativeState(&'static str),
134}
135
136/// Narrow transport boundary implemented by the tonic client and test doubles.
137#[async_trait]
138pub trait RemoteEventTransport: Send {
139    /// Runtime capabilities negotiated or otherwise guaranteed by this
140    /// transport's remote authority.
141    ///
142    /// The subscriber snapshots this value when it is constructed. Custom
143    /// transports should return only capabilities they enforce end to end;
144    /// the default is deliberately empty and therefore fails closed in
145    /// coordinators that require historical or durable behavior.
146    fn capabilities(&self) -> SubscriberCapabilities {
147        SubscriberCapabilities::default()
148    }
149
150    /// Validate and seed a runtime-restored durable position before polling.
151    ///
152    /// # Errors
153    ///
154    /// Returns an error when the position is malformed, belongs to another
155    /// chain/session authority, or conflicts with the transport's durable
156    /// cursor or pending replay proof. Failure must leave the prior transport
157    /// authority usable.
158    fn restore_position(
159        &mut self,
160        _position: &SubscriberResumePosition,
161    ) -> Result<(), RemoteTransportError> {
162        Ok(())
163    }
164
165    /// Sequence durably acknowledged by the service after a restore hand-off.
166    /// A transport returns `None` when it has no separate remote authority.
167    fn durable_acknowledged_sequence(&self) -> Option<u64> {
168        None
169    }
170
171    /// Exact durable cursor currently accepted by the remote authority.
172    fn durable_acknowledged_cursor(&self) -> Option<Cursor> {
173        None
174    }
175
176    /// Atomically replace the authoritative desired state.
177    ///
178    /// # Errors
179    ///
180    /// Returns an error for transport failure, protocol-invalid confirmation,
181    /// or a structured service rejection. An uncertain failure must remain
182    /// safely retryable with the same request.
183    async fn apply_desired_state(
184        &mut self,
185        request: ApplyDesiredState,
186    ) -> Result<DesiredStateApplied, RemoteTransportError>;
187
188    /// Receive the next ordered data or chain-control delivery.
189    ///
190    /// # Errors
191    ///
192    /// Returns an error when the transport is unavailable or the service
193    /// violates the negotiated delivery protocol. Failure must not silently
194    /// consume an unacknowledged delivery.
195    async fn next_delivery(&mut self) -> Result<Option<Delivery>, RemoteTransportError>;
196
197    /// Commit one successfully ingested data batch.
198    ///
199    /// # Errors
200    ///
201    /// Returns an error for transport failure, a mismatched acknowledgement,
202    /// or a service rejection. An uncertain result must be reconciled before a
203    /// later delivery is exposed.
204    async fn acknowledge(
205        &mut self,
206        acknowledgement: Acknowledge,
207    ) -> Result<(), RemoteTransportError>;
208}
209
210/// Remote service or network failure.
211#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
212#[non_exhaustive]
213pub enum RemoteTransportError {
214    /// The remote endpoint is unavailable or rejected the request.
215    #[error("remote event service unavailable: {0}")]
216    Unavailable(String),
217    /// The service returned an invalid lifecycle acknowledgement.
218    #[error("remote event service protocol error: {0}")]
219    Protocol(String),
220    /// A structured application-level rejection from the remote service.
221    #[error("remote event service error {code:?} at revision {committed_revision}: {message}")]
222    Remote {
223        /// Stable protocol error classification.
224        code: ErrorCode,
225        /// Sanitized server-provided explanation.
226        message: String,
227        /// Service-authoritative desired-state revision at rejection time.
228        committed_revision: u64,
229        /// Whether reconnecting and retrying the same operation may succeed.
230        retryable: bool,
231    },
232}
233
234impl RemoteTransportError {
235    fn operation_uncertain(&self) -> bool {
236        matches!(
237            self,
238            Self::Unavailable(_)
239                | Self::Remote {
240                    retryable: true,
241                    ..
242                }
243        )
244    }
245}
246
247/// Network and control-operation deadlines for the tonic transport. Delivery
248/// waits are intentionally unbounded because an idle chain/source is healthy;
249/// cancellation remains immediate through the borrowed future.
250#[derive(Clone, Debug, PartialEq, Eq)]
251#[non_exhaustive]
252pub struct GrpcTransportConfig {
253    /// Maximum time to establish one HTTP/2 channel.
254    pub connect_timeout: Duration,
255    /// Maximum time to establish the stream and receive `HelloAccepted`.
256    pub handshake_timeout: Duration,
257    /// Maximum time to wait for apply/ack confirmation. Delivery waits are not
258    /// governed by this value. This is one absolute deadline for the whole
259    /// control operation; heartbeats and reconnects do not extend it.
260    pub control_response_timeout: Duration,
261    /// Interval between HTTP/2 PING frames used to detect a half-open channel.
262    pub http2_keep_alive_interval: Duration,
263    /// Maximum time to wait for an HTTP/2 keepalive PING acknowledgement.
264    pub http2_keep_alive_timeout: Duration,
265    /// Send HTTP/2 keepalive PING frames even while no RPC messages are moving.
266    /// This should normally remain enabled for a long-lived event stream.
267    pub http2_keep_alive_while_idle: bool,
268    /// Number of handshake attempts made after a stream disconnect.
269    pub reconnect_attempts: usize,
270    /// Delay before the second reconnect attempt.
271    pub reconnect_initial_delay: Duration,
272    /// Upper bound for exponential reconnect delay.
273    pub reconnect_max_delay: Duration,
274}
275
276impl Default for GrpcTransportConfig {
277    fn default() -> Self {
278        Self {
279            connect_timeout: Duration::from_secs(10),
280            // The shipped service may spend up to 45 seconds preparing or
281            // proving source state before Hello/apply confirmation.
282            handshake_timeout: Duration::from_secs(60),
283            control_response_timeout: Duration::from_secs(60),
284            http2_keep_alive_interval: Duration::from_secs(30),
285            http2_keep_alive_timeout: Duration::from_secs(10),
286            http2_keep_alive_while_idle: true,
287            reconnect_attempts: 5,
288            reconnect_initial_delay: Duration::from_millis(200),
289            reconnect_max_delay: Duration::from_secs(3),
290        }
291    }
292}
293
294/// Tonic transport for one long-lived, bidirectional event session.
295pub struct GrpcEventTransport {
296    endpoint: String,
297    session_id: String,
298    chain_id: u64,
299    acknowledged_sequence: u64,
300    hello_acknowledged_sequence: u64,
301    authorization: Option<String>,
302    reconnects: u64,
303    sender: mpsc::Sender<ClientMessage>,
304    inbound: tonic::Streaming<ServerMessage>,
305    buffered: VecDeque<ServerMessage>,
306    accepted: HelloAccepted,
307    pending_apply: Option<PendingOperation<ApplyDesiredState>>,
308    pending_acknowledgement: Option<PendingOperation<Acknowledge>>,
309    delivery_demand_sent: bool,
310    in_flight_delivery: Option<Delivery>,
311    config: GrpcTransportConfig,
312    position_confirmation_required: bool,
313    resume_handshake_required: bool,
314    pending_delivery_resume: Option<PendingDeliveryResume>,
315}
316
317#[derive(Clone)]
318struct PendingOperation<T> {
319    value: T,
320    sent: bool,
321}
322
323struct OpenedSession {
324    sender: mpsc::Sender<ClientMessage>,
325    inbound: tonic::Streaming<ServerMessage>,
326    accepted: HelloAccepted,
327}
328
329enum StreamReceive {
330    Message(Box<ServerMessage>),
331    Reconnected,
332}
333
334impl GrpcEventTransport {
335    /// Connect to an event service and negotiate a versioned session.
336    ///
337    /// # Errors
338    ///
339    /// Returns [`RemoteTransportError`] when the endpoint or transport
340    /// configuration is invalid, the HTTP/2 connection/handshake fails, or the
341    /// service rejects or violates protocol negotiation.
342    pub async fn connect(
343        endpoint: impl Into<String>,
344        session_id: impl Into<String>,
345        chain_id: u64,
346        acknowledged_sequence: u64,
347    ) -> Result<Self, RemoteTransportError> {
348        Self::connect_with_authorization_and_config(
349            endpoint,
350            session_id,
351            chain_id,
352            acknowledged_sequence,
353            None,
354            GrpcTransportConfig::default(),
355        )
356        .await
357    }
358
359    /// Connect with an HTTP authorization metadata value, typically
360    /// `Bearer <token>`. The value is retained only for reconnects.
361    ///
362    /// # Errors
363    ///
364    /// Returns [`RemoteTransportError`] for invalid authorization metadata,
365    /// endpoint/configuration errors, connection or handshake failure, and
366    /// service rejection or protocol-invalid negotiation.
367    pub async fn connect_with_authorization(
368        endpoint: impl Into<String>,
369        session_id: impl Into<String>,
370        chain_id: u64,
371        acknowledged_sequence: u64,
372        authorization: Option<String>,
373    ) -> Result<Self, RemoteTransportError> {
374        Self::connect_with_authorization_and_config(
375            endpoint,
376            session_id,
377            chain_id,
378            acknowledged_sequence,
379            authorization,
380            GrpcTransportConfig::default(),
381        )
382        .await
383    }
384
385    /// Connect using explicit network/control deadlines.
386    ///
387    /// # Errors
388    ///
389    /// Returns [`RemoteTransportError`] when a configured deadline/retry bound
390    /// is invalid, the connection or handshake fails, or negotiation is
391    /// rejected or protocol-invalid.
392    pub async fn connect_with_config(
393        endpoint: impl Into<String>,
394        session_id: impl Into<String>,
395        chain_id: u64,
396        acknowledged_sequence: u64,
397        config: GrpcTransportConfig,
398    ) -> Result<Self, RemoteTransportError> {
399        Self::connect_with_authorization_and_config(
400            endpoint,
401            session_id,
402            chain_id,
403            acknowledged_sequence,
404            None,
405            config,
406        )
407        .await
408    }
409
410    /// Connect with authorization and explicit network/control deadlines.
411    ///
412    /// # Errors
413    ///
414    /// Returns [`RemoteTransportError`] for invalid metadata or configuration,
415    /// connection/handshake failure, and service rejection or
416    /// protocol-invalid negotiation.
417    pub async fn connect_with_authorization_and_config(
418        endpoint: impl Into<String>,
419        session_id: impl Into<String>,
420        chain_id: u64,
421        acknowledged_sequence: u64,
422        authorization: Option<String>,
423        config: GrpcTransportConfig,
424    ) -> Result<Self, RemoteTransportError> {
425        Self::connect_with_resume(
426            endpoint.into(),
427            session_id.into(),
428            chain_id,
429            acknowledged_sequence,
430            authorization,
431            config,
432            None,
433        )
434        .await
435    }
436
437    async fn connect_with_resume(
438        endpoint: String,
439        session_id: String,
440        chain_id: u64,
441        acknowledged_sequence: u64,
442        authorization: Option<String>,
443        config: GrpcTransportConfig,
444        pending_delivery_resume: Option<PendingDeliveryResume>,
445    ) -> Result<Self, RemoteTransportError> {
446        validate_transport_config(&config)?;
447        validate_resume_proof_sequence(acknowledged_sequence, pending_delivery_resume.as_ref())?;
448        let opened = open_session(
449            &endpoint,
450            &session_id,
451            chain_id,
452            acknowledged_sequence,
453            pending_delivery_resume.as_ref(),
454            authorization.as_deref(),
455            &config,
456        )
457        .await?;
458        let durable_sequence = accepted_sequence(&opened.accepted)?;
459        let runtime_sequence = accepted_runtime_sequence(&opened.accepted);
460        let accepted_runtime_position = acknowledged_sequence == runtime_sequence;
461        let accepted_pending_replay = pending_delivery_resume.is_some()
462            && durable_sequence.checked_add(1) == Some(acknowledged_sequence);
463        Ok(Self {
464            endpoint,
465            session_id,
466            chain_id,
467            acknowledged_sequence: durable_sequence,
468            hello_acknowledged_sequence: acknowledged_sequence,
469            authorization,
470            reconnects: 0,
471            sender: opened.sender,
472            inbound: opened.inbound,
473            buffered: VecDeque::new(),
474            accepted: opened.accepted,
475            pending_apply: None,
476            pending_acknowledgement: None,
477            delivery_demand_sent: false,
478            in_flight_delivery: None,
479            config,
480            position_confirmation_required: !accepted_runtime_position && !accepted_pending_replay,
481            resume_handshake_required: false,
482            pending_delivery_resume,
483        })
484    }
485
486    /// Session metadata returned by the authoritative service.
487    pub fn accepted(&self) -> &HelloAccepted {
488        &self.accepted
489    }
490
491    /// Number of transport sessions successfully replaced after disconnect.
492    pub const fn reconnect_count(&self) -> u64 {
493        self.reconnects
494    }
495
496    async fn reconnect(&mut self) -> Result<(), RemoteTransportError> {
497        let mut delay = self
498            .config
499            .reconnect_initial_delay
500            .min(self.config.reconnect_max_delay);
501        let mut last_error = None;
502        let hello_sequence = self
503            .pending_delivery_resume
504            .as_ref()
505            .map_or(Ok(self.acknowledged_sequence), resume_proof_sequence)?;
506        for attempt in 0..self.config.reconnect_attempts {
507            if attempt > 0 {
508                tokio::time::sleep(delay).await;
509                delay = delay.saturating_mul(2).min(self.config.reconnect_max_delay);
510            }
511            match open_session(
512                &self.endpoint,
513                &self.session_id,
514                self.chain_id,
515                hello_sequence,
516                self.pending_delivery_resume.as_ref(),
517                self.authorization.as_deref(),
518                &self.config,
519            )
520            .await
521            {
522                Ok(opened) => {
523                    self.validate_reconnected_authority(&opened.accepted)?;
524                    let durable_sequence = accepted_sequence(&opened.accepted)?;
525                    self.sender = opened.sender;
526                    self.inbound = opened.inbound;
527                    self.accepted = opened.accepted;
528                    self.acknowledged_sequence = durable_sequence;
529                    self.hello_acknowledged_sequence = hello_sequence;
530                    self.buffered.clear();
531                    if let Some(pending) = self.pending_apply.as_mut() {
532                        pending.sent = false;
533                    }
534                    if let Some(pending) = self.pending_acknowledgement.as_mut() {
535                        pending.sent = false;
536                    }
537                    self.delivery_demand_sent = false;
538                    self.reconnects = self.reconnects.saturating_add(1);
539                    return Ok(());
540                }
541                Err(error @ RemoteTransportError::Protocol(_)) => return Err(error),
542                Err(
543                    error @ RemoteTransportError::Remote {
544                        retryable: false, ..
545                    },
546                ) => {
547                    return Err(error);
548                }
549                Err(error) => last_error = Some(error),
550            }
551        }
552        Err(last_error.unwrap_or_else(|| {
553            RemoteTransportError::Unavailable("session reconnect attempts exhausted".into())
554        }))
555    }
556
557    async fn send(&mut self, message: client_message::Message) -> Result<(), RemoteTransportError> {
558        let message = ClientMessage {
559            message: Some(message),
560        };
561        match tokio::time::timeout(
562            self.config.control_response_timeout,
563            self.sender.send(message.clone()),
564        )
565        .await
566        {
567            Ok(Ok(())) => return Ok(()),
568            Err(_) => {
569                return Err(RemoteTransportError::Unavailable(
570                    "timed out queueing event-session request".into(),
571                ));
572            }
573            Ok(Err(_)) => {}
574        }
575        self.reconnect().await?;
576        tokio::time::timeout(
577            self.config.control_response_timeout,
578            self.sender.send(message),
579        )
580        .await
581        .map_err(|_| {
582            RemoteTransportError::Unavailable(
583                "timed out queueing event-session request after reconnect".into(),
584            )
585        })?
586        .map_err(|_| RemoteTransportError::Unavailable("event session closed".into()))
587    }
588
589    async fn send_until(
590        &mut self,
591        message: client_message::Message,
592        deadline: tokio::time::Instant,
593    ) -> Result<(), RemoteTransportError> {
594        let message = ClientMessage {
595            message: Some(message),
596        };
597        match tokio::time::timeout_at(deadline, self.sender.send(message.clone())).await {
598            Ok(Ok(())) => return Ok(()),
599            Err(_) => return Err(control_deadline_elapsed()),
600            Ok(Err(_)) => {}
601        }
602        tokio::time::timeout_at(deadline, self.reconnect())
603            .await
604            .map_err(|_| control_deadline_elapsed())??;
605        tokio::time::timeout_at(deadline, self.sender.send(message))
606            .await
607            .map_err(|_| control_deadline_elapsed())?
608            .map_err(|_| RemoteTransportError::Unavailable("event session closed".into()))
609    }
610
611    async fn ensure_resume_handshake(&mut self) -> Result<(), RemoteTransportError> {
612        if self.position_confirmation_required {
613            return Err(RemoteTransportError::Protocol(
614                "restore_position must confirm the runtime checkpoint before an existing session can advance"
615                    .into(),
616            ));
617        }
618        if !self.resume_handshake_required {
619            return Ok(());
620        }
621        // `restore_position` is synchronous, so close the original request
622        // stream here and negotiate a fresh Hello carrying the restored
623        // sequence before any control or data operation can proceed.
624        let (placeholder, receiver) = mpsc::channel(1);
625        drop(receiver);
626        drop(std::mem::replace(&mut self.sender, placeholder));
627        self.reconnect().await?;
628        self.resume_handshake_required = false;
629        Ok(())
630    }
631
632    fn validate_reconnected_authority(
633        &self,
634        accepted: &HelloAccepted,
635    ) -> Result<(), RemoteTransportError> {
636        let desired_matches_current = accepted.committed_revision
637            == self.accepted.committed_revision
638            && accepted.desired_state == self.accepted.desired_state;
639        let desired_matches_pending = self.pending_apply.as_ref().is_some_and(|pending| {
640            accepted.committed_revision == pending.value.new_revision
641                && accepted.desired_state.as_ref() == Some(&pending.value)
642        });
643        if !desired_matches_current && !desired_matches_pending {
644            return Err(RemoteTransportError::Protocol(
645                "service authority changed unexpectedly during reconnect".into(),
646            ));
647        }
648        if accepted.capabilities != self.accepted.capabilities {
649            return Err(RemoteTransportError::Protocol(
650                "service capabilities changed during reconnect".into(),
651            ));
652        }
653        let durable_sequence = accepted_sequence(accepted)?;
654        let acknowledgement_matches_pending = self
655            .pending_acknowledgement
656            .as_ref()
657            .is_some_and(|pending| durable_sequence == pending.value.sequence);
658        if durable_sequence != self.acknowledged_sequence && !acknowledgement_matches_pending {
659            return Err(RemoteTransportError::Protocol(
660                "service acknowledgement cursor changed unexpectedly during reconnect".into(),
661            ));
662        }
663        Ok(())
664    }
665
666    async fn receive_stream(&mut self) -> Result<StreamReceive, RemoteTransportError> {
667        match self.inbound.message().await {
668            Ok(Some(message)) => Ok(StreamReceive::Message(Box::new(message))),
669            Ok(None) | Err(_) => {
670                self.reconnect().await?;
671                Ok(StreamReceive::Reconnected)
672            }
673        }
674    }
675
676    async fn receive_control(
677        &mut self,
678        deadline: tokio::time::Instant,
679    ) -> Result<StreamReceive, RemoteTransportError> {
680        tokio::time::timeout_at(deadline, self.receive_stream())
681            .await
682            .map_err(|_| {
683                RemoteTransportError::Unavailable(
684                    "timed out waiting for event-service control confirmation".into(),
685                )
686            })?
687    }
688}
689
690async fn open_session(
691    endpoint: &str,
692    session_id: &str,
693    chain_id: u64,
694    acknowledged_sequence: u64,
695    pending_delivery_resume: Option<&PendingDeliveryResume>,
696    authorization: Option<&str>,
697    config: &GrpcTransportConfig,
698) -> Result<OpenedSession, RemoteTransportError> {
699    let endpoint = tonic::transport::Endpoint::from_shared(endpoint.to_owned())
700        .map_err(|error| RemoteTransportError::Protocol(error.to_string()))?
701        .connect_timeout(config.connect_timeout)
702        .http2_keep_alive_interval(config.http2_keep_alive_interval)
703        .keep_alive_timeout(config.http2_keep_alive_timeout)
704        .keep_alive_while_idle(config.http2_keep_alive_while_idle);
705    let channel = tokio::time::timeout(config.connect_timeout, endpoint.connect())
706        .await
707        .map_err(|_| {
708            RemoteTransportError::Unavailable("event-service connection timed out".into())
709        })?
710        .map_err(unavailable)?;
711    let mut client = EventStreamClient::<Channel>::new(channel)
712        .max_decoding_message_size(MAX_MESSAGE_SIZE_BYTES)
713        .max_encoding_message_size(MAX_MESSAGE_SIZE_BYTES);
714    let (sender, receiver) = mpsc::channel(32);
715    let mut request = tonic::Request::new(ReceiverStream::new(receiver));
716    if let Some(authorization) = authorization {
717        let value = authorization.parse().map_err(|_| {
718            RemoteTransportError::Protocol("invalid authorization metadata value".into())
719        })?;
720        request.metadata_mut().insert("authorization", value);
721    }
722    let response = tokio::time::timeout(config.handshake_timeout, client.session(request))
723        .await
724        .map_err(|_| RemoteTransportError::Unavailable("event-session handshake timed out".into()))?
725        .map_err(status_error)?;
726    let mut inbound = response.into_inner();
727    sender
728        .send(ClientMessage {
729            message: Some(client_message::Message::Hello(Hello {
730                protocol_version: PROTOCOL_VERSION,
731                session_id: session_id.to_owned(),
732                chain_id,
733                acknowledged_sequence,
734                pending_delivery_resume: pending_delivery_resume.cloned(),
735            })),
736        })
737        .await
738        .map_err(|_| RemoteTransportError::Unavailable("session request closed".into()))?;
739    let accepted = match tokio::time::timeout(config.handshake_timeout, inbound.message())
740        .await
741        .map_err(|_| RemoteTransportError::Unavailable("HelloAccepted timed out".into()))?
742        .map_err(status_error)?
743    {
744        Some(ServerMessage {
745            message: Some(server_message::Message::HelloAccepted(accepted)),
746        }) => accepted,
747        Some(ServerMessage {
748            message: Some(server_message::Message::Error(error)),
749        }) => return Err(protocol_error(error)),
750        Some(_) => {
751            return Err(RemoteTransportError::Protocol(
752                "service did not begin with HelloAccepted".into(),
753            ));
754        }
755        None => {
756            return Err(RemoteTransportError::Unavailable(
757                "service closed during session negotiation".into(),
758            ));
759        }
760    };
761    if accepted.protocol_version != PROTOCOL_VERSION
762        || accepted.session_id != session_id
763        || accepted.chain_id != chain_id
764    {
765        return Err(RemoteTransportError::Protocol(format!(
766            "invalid HelloAccepted: version={}, session={}, chain={}",
767            accepted.protocol_version, accepted.session_id, accepted.chain_id
768        )));
769    }
770    validate_hello_authority(
771        &accepted,
772        session_id,
773        chain_id,
774        acknowledged_sequence,
775        pending_delivery_resume,
776    )?;
777    Ok(OpenedSession {
778        sender,
779        inbound,
780        accepted,
781    })
782}
783
784fn accepted_sequence(accepted: &HelloAccepted) -> Result<u64, RemoteTransportError> {
785    Ok(accepted
786        .acknowledged_cursor
787        .as_ref()
788        .map_or(0, |cursor| cursor.batch_sequence))
789}
790
791fn accepted_runtime_cursor(accepted: &HelloAccepted) -> Option<&Cursor> {
792    accepted
793        .runtime_checkpoint_position
794        .as_ref()
795        .map_or(accepted.acknowledged_cursor.as_ref(), |position| {
796            position.cursor.as_ref()
797        })
798}
799
800fn accepted_runtime_sequence(accepted: &HelloAccepted) -> u64 {
801    accepted_runtime_cursor(accepted).map_or(0, |cursor| cursor.batch_sequence)
802}
803
804fn validate_hello_authority(
805    accepted: &HelloAccepted,
806    session_id: &str,
807    chain_id: u64,
808    requested_acknowledged_sequence: u64,
809    pending_delivery_resume: Option<&PendingDeliveryResume>,
810) -> Result<(), RemoteTransportError> {
811    let durable_sequence = accepted_sequence(accepted)?;
812    let runtime_sequence = accepted_runtime_sequence(accepted);
813    let exact_pending_resume = pending_delivery_resume.is_some()
814        && durable_sequence.checked_add(1) == Some(requested_acknowledged_sequence);
815    let sequence_zero_discovery =
816        requested_acknowledged_sequence == 0 && pending_delivery_resume.is_none();
817    let legacy_authority = accepted.runtime_checkpoint_position.is_none();
818    let requested_matches_authority = if legacy_authority {
819        requested_acknowledged_sequence <= durable_sequence
820    } else {
821        requested_acknowledged_sequence == runtime_sequence
822    };
823    if !requested_matches_authority && !exact_pending_resume && !sequence_zero_discovery {
824        return Err(RemoteTransportError::Protocol(
825            "service runtime checkpoint authority does not match the restored client position"
826                .into(),
827        ));
828    }
829    if let Some(cursor) = accepted.acknowledged_cursor.as_ref()
830        && (cursor.chain_id != chain_id
831            || cursor.query_revision > accepted.committed_revision
832            || cursor.batch_sequence == 0)
833    {
834        return Err(RemoteTransportError::Protocol(
835            "HelloAccepted contains an invalid acknowledged cursor".into(),
836        ));
837    }
838    if let Some(head) = accepted
839        .acknowledged_cursor
840        .as_ref()
841        .and_then(|cursor| cursor.canonical_head.as_ref())
842    {
843        runtime_block_ref(head).map_err(|error| {
844            RemoteTransportError::Protocol(format!(
845                "HelloAccepted canonical head is invalid: {error}"
846            ))
847        })?;
848    }
849    if let Some(runtime_cursor) = accepted_runtime_cursor(accepted) {
850        if runtime_cursor.chain_id != chain_id
851            || runtime_cursor.query_revision > accepted.committed_revision
852            || runtime_cursor.batch_sequence == 0
853            || runtime_cursor.batch_sequence > durable_sequence
854            || (runtime_cursor.batch_sequence == durable_sequence
855                && accepted.acknowledged_cursor.as_ref() != Some(runtime_cursor))
856        {
857            return Err(RemoteTransportError::Protocol(
858                "HelloAccepted contains an invalid runtime checkpoint cursor".into(),
859            ));
860        }
861        if let Some(head) = runtime_cursor.canonical_head.as_ref() {
862            runtime_block_ref(head).map_err(|error| {
863                RemoteTransportError::Protocol(format!(
864                    "HelloAccepted runtime checkpoint head is invalid: {error}"
865                ))
866            })?;
867        }
868    }
869    match accepted.desired_state.as_ref() {
870        Some(desired)
871            if desired.protocol_version == PROTOCOL_VERSION
872                && desired.session_id == session_id
873                && desired.chain_id == chain_id
874                && desired.new_revision == accepted.committed_revision
875                && desired.expected_revision.checked_add(1)
876                    == Some(accepted.committed_revision) => {}
877        None if accepted.committed_revision == 0 => {}
878        _ => {
879            return Err(RemoteTransportError::Protocol(
880                "HelloAccepted desired state does not match its authority".into(),
881            ));
882        }
883    }
884    Ok(())
885}
886
887fn resume_sequence(position: &SubscriberResumePosition) -> Result<u64, RemoteTransportError> {
888    let Some(token) = position.delivery_token.as_ref() else {
889        if position.subscriber_checkpoint.is_some() {
890            return Err(RemoteTransportError::Protocol(
891                "restored subscriber checkpoint is missing its delivery token".into(),
892            ));
893        }
894        return Ok(0);
895    };
896    let bytes = token.as_bytes().try_into().map_err(|_| {
897        RemoteTransportError::Protocol(
898            "remote delivery tokens must be exactly eight-byte big-endian sequences".into(),
899        )
900    })?;
901    Ok(u64::from_be_bytes(bytes))
902}
903
904fn resume_proof_sequence(resume: &PendingDeliveryResume) -> Result<u64, RemoteTransportError> {
905    let bytes = resume.delivery_token.as_slice().try_into().map_err(|_| {
906        RemoteTransportError::Protocol(
907            "pending-delivery resume tokens must be exactly eight-byte big-endian sequences".into(),
908        )
909    })?;
910    Ok(u64::from_be_bytes(bytes))
911}
912
913fn validate_resume_proof_sequence(
914    acknowledged_sequence: u64,
915    resume: Option<&PendingDeliveryResume>,
916) -> Result<(), RemoteTransportError> {
917    if let Some(resume) = resume
918        && resume_proof_sequence(resume)? != acknowledged_sequence
919    {
920        return Err(RemoteTransportError::Protocol(
921            "pending-delivery resume proof does not match its Hello sequence".into(),
922        ));
923    }
924    Ok(())
925}
926
927fn pending_delivery_resume(
928    position: &SubscriberResumePosition,
929) -> Result<Option<PendingDeliveryResume>, RemoteTransportError> {
930    let sequence = resume_sequence(position)?;
931    let Some(token) = position.delivery_token.as_ref() else {
932        return Ok(None);
933    };
934    let resume = PendingDeliveryResume {
935        delivery_token: token.as_bytes().to_vec(),
936        provider_checkpoint: position
937            .subscriber_checkpoint
938            .as_ref()
939            .and_then(|checkpoint| {
940                (!checkpoint.as_bytes().is_empty()).then(|| checkpoint.as_bytes().to_vec())
941            }),
942        coverage_head: Some(wire_block_ref(&position.coverage_head)?),
943    };
944    validate_resume_proof_sequence(sequence, Some(&resume))?;
945    Ok(Some(resume))
946}
947
948fn resume_proof_matches_cursor(resume: &PendingDeliveryResume, cursor: &Cursor) -> bool {
949    cursor.provider_checkpoint == resume.provider_checkpoint.as_deref().unwrap_or_default()
950        && cursor.canonical_head.as_ref() == resume.coverage_head.as_ref()
951}
952
953fn wire_block_ref(block: &RuntimeBlockRef) -> Result<WireBlockRef, RemoteTransportError> {
954    let parent_hash = block.parent_hash.ok_or_else(|| {
955        RemoteTransportError::Protocol(
956            "pending-delivery resume coverage is missing its parent hash".into(),
957        )
958    })?;
959    let timestamp = block.timestamp.ok_or_else(|| {
960        RemoteTransportError::Protocol(
961            "pending-delivery resume coverage is missing its timestamp".into(),
962        )
963    })?;
964    Ok(WireBlockRef {
965        number: block.number,
966        hash: block.hash.as_slice().to_vec(),
967        parent_hash: parent_hash.as_slice().to_vec(),
968        timestamp,
969    })
970}
971
972fn validate_resume_position(
973    accepted: &HelloAccepted,
974    position: &SubscriberResumePosition,
975) -> Result<(), RemoteTransportError> {
976    let sequence = resume_sequence(position)?;
977    let durable_sequence = accepted_sequence(accepted)?;
978    let runtime_sequence = accepted_runtime_sequence(accepted);
979    let resumes_pending = durable_sequence.checked_add(1) == Some(sequence);
980    if runtime_sequence != sequence && !resumes_pending {
981        return Err(RemoteTransportError::Protocol(format!(
982            "service runtime checkpoint sequence {runtime_sequence} does not match restored runtime sequence {sequence}"
983        )));
984    }
985    if resumes_pending {
986        return Ok(());
987    }
988    let cursor = accepted_runtime_cursor(accepted);
989    let restored_checkpoint = position.subscriber_checkpoint.as_ref();
990    let checkpoint_matches = match cursor {
991        None => restored_checkpoint.is_none(),
992        Some(cursor) if cursor.provider_checkpoint.is_empty() => restored_checkpoint.is_none(),
993        Some(cursor) => restored_checkpoint
994            .is_some_and(|checkpoint| checkpoint.as_bytes() == cursor.provider_checkpoint),
995    };
996    if !checkpoint_matches {
997        return Err(RemoteTransportError::Protocol(
998            "service provider checkpoint does not match the restored runtime checkpoint".into(),
999        ));
1000    }
1001    let service_head = cursor.and_then(|cursor| cursor.canonical_head.as_ref());
1002    let coverage_matches = service_head
1003        .map(runtime_block_ref)
1004        .transpose()
1005        .map_err(|error| {
1006            RemoteTransportError::Protocol(format!(
1007                "service runtime checkpoint head is invalid: {error}"
1008            ))
1009        })?
1010        .as_ref()
1011        == Some(&position.coverage_head);
1012    if !coverage_matches {
1013        return Err(RemoteTransportError::Protocol(
1014            "service canonical head does not exactly match restored runtime coverage".into(),
1015        ));
1016    }
1017    Ok(())
1018}
1019
1020#[async_trait]
1021impl RemoteEventTransport for GrpcEventTransport {
1022    fn capabilities(&self) -> SubscriberCapabilities {
1023        runtime_capabilities(self.accepted.capabilities.as_ref())
1024    }
1025
1026    fn restore_position(
1027        &mut self,
1028        position: &SubscriberResumePosition,
1029    ) -> Result<(), RemoteTransportError> {
1030        validate_resume_position(&self.accepted, position)?;
1031        let sequence = resume_sequence(position)?;
1032        let resume = pending_delivery_resume(position)?;
1033        let proof_changed = self.pending_delivery_resume != resume;
1034        self.acknowledged_sequence = accepted_sequence(&self.accepted)?;
1035        self.in_flight_delivery = None;
1036        self.resume_handshake_required =
1037            self.hello_acknowledged_sequence != sequence || proof_changed;
1038        self.pending_delivery_resume = resume;
1039        self.position_confirmation_required = false;
1040        Ok(())
1041    }
1042
1043    fn durable_acknowledged_sequence(&self) -> Option<u64> {
1044        Some(self.acknowledged_sequence)
1045    }
1046
1047    fn durable_acknowledged_cursor(&self) -> Option<Cursor> {
1048        self.accepted.acknowledged_cursor.clone()
1049    }
1050
1051    async fn apply_desired_state(
1052        &mut self,
1053        request: ApplyDesiredState,
1054    ) -> Result<DesiredStateApplied, RemoteTransportError> {
1055        let deadline = tokio::time::Instant::now()
1056            .checked_add(self.config.control_response_timeout)
1057            .ok_or_else(|| {
1058                RemoteTransportError::Protocol("control response deadline is out of range".into())
1059            })?;
1060        tokio::time::timeout_at(deadline, self.ensure_resume_handshake())
1061            .await
1062            .map_err(|_| control_deadline_elapsed())??;
1063        if self.pending_acknowledgement.is_some() {
1064            return Err(RemoteTransportError::Protocol(
1065                "an acknowledgement must be reconciled before desired state can change".into(),
1066            ));
1067        }
1068        match self.pending_apply.as_ref() {
1069            Some(pending) if pending.value != request => {
1070                return Err(RemoteTransportError::Protocol(
1071                    "a different desired-state operation is still pending".into(),
1072                ));
1073            }
1074            None => {
1075                self.pending_apply = Some(PendingOperation {
1076                    value: request.clone(),
1077                    sent: false,
1078                });
1079            }
1080            Some(_) => {}
1081        }
1082        loop {
1083            if !self
1084                .pending_apply
1085                .as_ref()
1086                .is_some_and(|pending| pending.sent)
1087            {
1088                self.send_until(
1089                    client_message::Message::ApplyDesiredState(request.clone()),
1090                    deadline,
1091                )
1092                .await?;
1093                self.pending_apply
1094                    .as_mut()
1095                    .expect("pending apply exists")
1096                    .sent = true;
1097            }
1098            let message = match self.receive_control(deadline).await? {
1099                StreamReceive::Message(message) => *message,
1100                StreamReceive::Reconnected => continue,
1101            };
1102            match message.message {
1103                Some(server_message::Message::DesiredStateApplied(applied)) => {
1104                    if applied.session_id != request.session_id
1105                        || applied.revision != request.new_revision
1106                        || applied.activation_sequence
1107                            != self.acknowledged_sequence.checked_add(1).ok_or_else(|| {
1108                                RemoteTransportError::Protocol(
1109                                    "delivery sequence overflow during desired-state apply".into(),
1110                                )
1111                            })?
1112                    {
1113                        return Err(RemoteTransportError::Protocol(
1114                            "desired-state confirmation does not match the request".into(),
1115                        ));
1116                    }
1117                    self.pending_apply = None;
1118                    self.accepted.committed_revision = request.new_revision;
1119                    self.accepted.desired_state = Some(request.clone());
1120                    return Ok(applied);
1121                }
1122                Some(server_message::Message::Delivery(_)) => self.buffered.push_back(message),
1123                Some(server_message::Message::AcknowledgementCommitted(_)) => {
1124                    return Err(RemoteTransportError::Protocol(
1125                        "unexpected acknowledgement while applying desired state".into(),
1126                    ));
1127                }
1128                Some(server_message::Message::Error(error)) => {
1129                    let error = protocol_error(error);
1130                    if !error.operation_uncertain() {
1131                        self.pending_apply = None;
1132                    }
1133                    return Err(error);
1134                }
1135                Some(server_message::Message::Heartbeat(_)) => {}
1136                Some(server_message::Message::HelloAccepted(_)) | None => {
1137                    return Err(RemoteTransportError::Protocol(
1138                        "unexpected message while applying desired state".into(),
1139                    ));
1140                }
1141            }
1142        }
1143    }
1144
1145    async fn next_delivery(&mut self) -> Result<Option<Delivery>, RemoteTransportError> {
1146        self.ensure_resume_handshake().await?;
1147        if self.pending_apply.is_some() || self.pending_acknowledgement.is_some() {
1148            return Err(RemoteTransportError::Protocol(
1149                "a cancelled control operation must be retried before receiving delivery".into(),
1150            ));
1151        }
1152        if let Some(delivery) = self.in_flight_delivery.as_ref() {
1153            return Ok(Some(delivery.clone()));
1154        }
1155        loop {
1156            let message = if let Some(message) = self.buffered.pop_front() {
1157                message
1158            } else {
1159                if !self.delivery_demand_sent {
1160                    self.send(client_message::Message::DeliveryDemand(DeliveryDemand {
1161                        session_id: self.session_id.clone(),
1162                    }))
1163                    .await?;
1164                    self.delivery_demand_sent = true;
1165                }
1166                match self.receive_stream().await? {
1167                    StreamReceive::Message(message) => *message,
1168                    StreamReceive::Reconnected => continue,
1169                }
1170            };
1171            match message.message {
1172                Some(server_message::Message::Delivery(delivery)) => {
1173                    self.delivery_demand_sent = false;
1174                    let expected_sequence =
1175                        self.acknowledged_sequence.checked_add(1).ok_or_else(|| {
1176                            RemoteTransportError::Protocol("delivery sequence overflow".into())
1177                        })?;
1178                    validate_transport_delivery(
1179                        DeliveryAuthority {
1180                            session_id: &self.session_id,
1181                            chain_id: self.chain_id,
1182                            revision: self.accepted.committed_revision,
1183                            expected_sequence,
1184                            acknowledged_cursor: self.accepted.acknowledged_cursor.as_ref(),
1185                            activation_baseline: desired_state_global_baseline(
1186                                self.accepted.desired_state.as_ref(),
1187                            ),
1188                            requires_open_backfill_boundary: desired_state_has_open_backfill(
1189                                self.accepted.desired_state.as_ref(),
1190                            ),
1191                        },
1192                        &delivery,
1193                    )?;
1194                    self.in_flight_delivery = Some(delivery.clone());
1195                    return Ok(Some(delivery));
1196                }
1197                Some(server_message::Message::Heartbeat(_)) => {}
1198                Some(server_message::Message::Error(error)) => {
1199                    self.delivery_demand_sent = false;
1200                    return Err(protocol_error(error));
1201                }
1202                Some(server_message::Message::DesiredStateApplied(_))
1203                | Some(server_message::Message::AcknowledgementCommitted(_))
1204                | Some(server_message::Message::HelloAccepted(_))
1205                | None => {
1206                    return Err(RemoteTransportError::Protocol(
1207                        "unexpected control message on data stream".into(),
1208                    ));
1209                }
1210            }
1211        }
1212    }
1213
1214    async fn acknowledge(
1215        &mut self,
1216        acknowledgement: Acknowledge,
1217    ) -> Result<(), RemoteTransportError> {
1218        let deadline = tokio::time::Instant::now()
1219            .checked_add(self.config.control_response_timeout)
1220            .ok_or_else(|| {
1221                RemoteTransportError::Protocol("control response deadline is out of range".into())
1222            })?;
1223        tokio::time::timeout_at(deadline, self.ensure_resume_handshake())
1224            .await
1225            .map_err(|_| control_deadline_elapsed())??;
1226        if self.pending_apply.is_some() {
1227            return Err(RemoteTransportError::Protocol(
1228                "a desired-state operation must be reconciled before acknowledgement".into(),
1229            ));
1230        }
1231        let (expected_cursor, restored_resume) =
1232            if let Some(delivery) = self.in_flight_delivery.as_ref() {
1233                if acknowledgement.session_id != delivery.session_id
1234                    || acknowledgement.sequence != delivery.sequence
1235                    || acknowledgement.delivery_token != delivery.delivery_token
1236                {
1237                    return Err(RemoteTransportError::Protocol(
1238                        "acknowledgement does not match the in-flight delivery".into(),
1239                    ));
1240                }
1241                let cursor = delivery.cursor.clone().ok_or_else(|| {
1242                    RemoteTransportError::Protocol("in-flight delivery cursor is missing".into())
1243                })?;
1244                (Some(cursor), None)
1245            } else {
1246                let resume = self.pending_delivery_resume.clone().ok_or_else(|| {
1247                    RemoteTransportError::Protocol(
1248                        "acknowledgement requires an in-flight or exactly restored delivery".into(),
1249                    )
1250                })?;
1251                if acknowledgement.session_id != self.session_id
1252                    || acknowledgement.sequence != resume_proof_sequence(&resume)?
1253                    || acknowledgement.delivery_token != resume.delivery_token
1254                {
1255                    return Err(RemoteTransportError::Protocol(
1256                        "acknowledgement does not match the restored pending delivery".into(),
1257                    ));
1258                }
1259                (None, Some(resume))
1260            };
1261        match self.pending_acknowledgement.as_ref() {
1262            Some(pending) if pending.value != acknowledgement => {
1263                return Err(RemoteTransportError::Protocol(
1264                    "a different acknowledgement is still pending".into(),
1265                ));
1266            }
1267            None => {
1268                self.pending_acknowledgement = Some(PendingOperation {
1269                    value: acknowledgement.clone(),
1270                    sent: false,
1271                });
1272            }
1273            Some(_) => {}
1274        }
1275        loop {
1276            if !self
1277                .pending_acknowledgement
1278                .as_ref()
1279                .is_some_and(|pending| pending.sent)
1280            {
1281                self.send_until(
1282                    client_message::Message::Acknowledge(acknowledgement.clone()),
1283                    deadline,
1284                )
1285                .await?;
1286                self.pending_acknowledgement
1287                    .as_mut()
1288                    .expect("pending acknowledgement exists")
1289                    .sent = true;
1290            }
1291            let message = match self.receive_control(deadline).await? {
1292                StreamReceive::Message(message) => *message,
1293                StreamReceive::Reconnected => continue,
1294            };
1295            match message.message {
1296                Some(server_message::Message::AcknowledgementCommitted(committed)) => {
1297                    let cursor = committed.cursor.as_ref().ok_or_else(|| {
1298                        RemoteTransportError::Protocol(
1299                            "acknowledgement confirmation is missing its cursor".into(),
1300                        )
1301                    })?;
1302                    if committed.session_id != acknowledgement.session_id
1303                        || committed.sequence != acknowledgement.sequence
1304                        || cursor.chain_id != self.chain_id
1305                        || cursor.query_revision != self.accepted.committed_revision
1306                        || cursor.batch_sequence != acknowledgement.sequence
1307                        || expected_cursor
1308                            .as_ref()
1309                            .is_some_and(|expected| expected != cursor)
1310                        || restored_resume
1311                            .as_ref()
1312                            .is_some_and(|resume| !resume_proof_matches_cursor(resume, cursor))
1313                    {
1314                        return Err(RemoteTransportError::Protocol(
1315                            "acknowledgement confirmation does not exactly match the request and delivered cursor"
1316                                .into(),
1317                        ));
1318                    }
1319                    self.pending_acknowledgement = None;
1320                    self.acknowledged_sequence = committed.sequence;
1321                    self.accepted.acknowledged_cursor = committed.cursor;
1322                    self.in_flight_delivery = None;
1323                    if self
1324                        .pending_delivery_resume
1325                        .as_ref()
1326                        .and_then(|resume| resume_proof_sequence(resume).ok())
1327                        .is_some_and(|sequence| sequence <= committed.sequence)
1328                    {
1329                        self.pending_delivery_resume = None;
1330                    }
1331                    return Ok(());
1332                }
1333                Some(server_message::Message::Delivery(delivery))
1334                    if delivery.sequence == acknowledgement.sequence
1335                        && delivery.delivery_token == acknowledgement.delivery_token =>
1336                {
1337                    // A reconnect replays the still-pending item before it
1338                    // can receive this idempotent acknowledgement retry.
1339                }
1340                Some(server_message::Message::Error(error)) => {
1341                    let error = protocol_error(error);
1342                    if !error.operation_uncertain() {
1343                        self.pending_acknowledgement = None;
1344                    }
1345                    return Err(error);
1346                }
1347                Some(server_message::Message::Heartbeat(_)) => {}
1348                Some(server_message::Message::Delivery(_))
1349                | Some(server_message::Message::DesiredStateApplied(_))
1350                | Some(server_message::Message::HelloAccepted(_))
1351                | None => {
1352                    return Err(RemoteTransportError::Protocol(
1353                        "unexpected message while acknowledging delivery".into(),
1354                    ));
1355                }
1356            }
1357        }
1358    }
1359}
1360
1361fn unavailable(error: impl std::fmt::Display) -> RemoteTransportError {
1362    RemoteTransportError::Unavailable(error.to_string())
1363}
1364
1365fn control_deadline_elapsed() -> RemoteTransportError {
1366    RemoteTransportError::Unavailable(
1367        "timed out waiting for event-service control operation".into(),
1368    )
1369}
1370
1371fn status_error(status: tonic::Status) -> RemoteTransportError {
1372    match status.code() {
1373        tonic::Code::Unauthenticated | tonic::Code::PermissionDenied => {
1374            RemoteTransportError::Remote {
1375                code: ErrorCode::Authentication,
1376                message: "event session authentication failed".into(),
1377                committed_revision: 0,
1378                retryable: false,
1379            }
1380        }
1381        tonic::Code::ResourceExhausted => RemoteTransportError::Remote {
1382            code: ErrorCode::ResourceExhausted,
1383            message: "event service resource limit exceeded".into(),
1384            committed_revision: 0,
1385            retryable: true,
1386        },
1387        _ => RemoteTransportError::Unavailable(status.to_string()),
1388    }
1389}
1390
1391fn validate_transport_config(config: &GrpcTransportConfig) -> Result<(), RemoteTransportError> {
1392    if config.connect_timeout.is_zero()
1393        || config.handshake_timeout.is_zero()
1394        || config.control_response_timeout.is_zero()
1395        || config.http2_keep_alive_interval.is_zero()
1396        || config.http2_keep_alive_timeout.is_zero()
1397        || config.reconnect_initial_delay.is_zero()
1398        || config.reconnect_max_delay.is_zero()
1399        || config.reconnect_attempts == 0
1400    {
1401        return Err(RemoteTransportError::Protocol(
1402            "transport timeouts, keepalive/reconnect durations, and reconnect attempts must be greater than zero"
1403                .into(),
1404        ));
1405    }
1406    Ok(())
1407}
1408
1409fn protocol_error(error: ProtocolError) -> RemoteTransportError {
1410    match ErrorCode::try_from(error.code) {
1411        Ok(ErrorCode::Unspecified) | Err(_) => RemoteTransportError::Protocol(format!(
1412            "service returned unknown error code {}",
1413            error.code
1414        )),
1415        Ok(code) => RemoteTransportError::Remote {
1416            code,
1417            message: error.message,
1418            committed_revision: error.committed_revision,
1419            retryable: error.retryable,
1420        },
1421    }
1422}
1423
1424#[derive(Clone, Copy)]
1425struct DeliveryAuthority<'a> {
1426    session_id: &'a str,
1427    chain_id: u64,
1428    revision: u64,
1429    expected_sequence: u64,
1430    acknowledged_cursor: Option<&'a Cursor>,
1431    activation_baseline: Option<&'a WireBlockRef>,
1432    requires_open_backfill_boundary: bool,
1433}
1434
1435fn validate_transport_delivery(
1436    authority: DeliveryAuthority<'_>,
1437    delivery: &Delivery,
1438) -> Result<(), RemoteTransportError> {
1439    let cursor = delivery.cursor.as_ref().ok_or_else(|| {
1440        RemoteTransportError::Protocol("delivery is missing its authoritative cursor".into())
1441    })?;
1442    if delivery.session_id != authority.session_id
1443        || delivery.sequence != authority.expected_sequence
1444        || delivery.query_revision != authority.revision
1445        || delivery.delivery_token != delivery.sequence.to_be_bytes()
1446        || delivery.payload.is_none()
1447        || cursor.chain_id != authority.chain_id
1448        || cursor.query_revision != authority.revision
1449        || cursor.batch_sequence != delivery.sequence
1450    {
1451        return Err(RemoteTransportError::Protocol(
1452            "delivery identity, revision, sequence, token, payload, or cursor is invalid".into(),
1453        ));
1454    }
1455    validate_wire_delivery_progress(
1456        authority.acknowledged_cursor,
1457        authority.activation_baseline,
1458        delivery,
1459        authority.requires_open_backfill_boundary,
1460    )
1461    .map_err(|message| RemoteTransportError::Protocol(message.into()))?;
1462    Ok(())
1463}
1464
1465fn desired_state_has_open_backfill(desired_state: Option<&ApplyDesiredState>) -> bool {
1466    desired_state.is_some_and(|desired_state| {
1467        desired_state.owners.iter().any(|owner| {
1468            owner
1469                .backfill
1470                .as_ref()
1471                .is_some_and(|backfill| backfill.to_block_excl.is_none())
1472        })
1473    })
1474}
1475
1476fn desired_state_global_baseline(
1477    desired_state: Option<&ApplyDesiredState>,
1478) -> Option<&WireBlockRef> {
1479    desired_state?
1480        .owners
1481        .iter()
1482        .find(|owner| owner.canonical)
1483        .and_then(|owner| owner.backfill.as_ref())
1484        .and_then(|backfill| backfill.retained_baseline.as_ref())
1485}
1486
1487fn validate_wire_delivery_progress(
1488    acknowledged_cursor: Option<&Cursor>,
1489    activation_baseline: Option<&WireBlockRef>,
1490    delivery: &Delivery,
1491    requires_open_backfill_boundary: bool,
1492) -> Result<(), &'static str> {
1493    let cursor = delivery
1494        .cursor
1495        .as_ref()
1496        .ok_or("delivery cursor is missing")?;
1497    let payload = delivery
1498        .payload
1499        .as_ref()
1500        .ok_or("delivery payload is missing")?;
1501    if matches!(payload, delivery::Payload::Barrier(barrier) if barrier.id.is_empty()) {
1502        return Err("barrier identifier is empty");
1503    }
1504    let is_reorg = matches!(payload, delivery::Payload::Reorg(_));
1505    let is_activation = is_wire_activation(acknowledged_cursor, activation_baseline, delivery);
1506    let is_scan_progress = is_wire_cursor_progress(delivery);
1507    if delivery.checkpoint_neutral != (is_activation || is_scan_progress) {
1508        return Err(
1509            "checkpoint-neutral marker must identify exactly an activation or scan-only progress barrier",
1510        );
1511    }
1512    if is_activation {
1513        if requires_open_backfill_boundary && cursor.owner_backfill_activation_block.is_none() {
1514            return Err("open owner backfill activation is missing its portable boundary");
1515        }
1516        if cursor
1517            .owner_backfill_activation_block
1518            .is_some_and(|activation| activation < cursor.next_block)
1519        {
1520            return Err("owner backfill activation boundary precedes its scan cursor");
1521        }
1522    } else if acknowledged_cursor.is_some_and(|acknowledged| {
1523        acknowledged.query_revision == cursor.query_revision
1524            && acknowledged.owner_backfill_activation_block
1525                != cursor.owner_backfill_activation_block
1526    }) {
1527        return Err("owner backfill activation boundary changed within one revision");
1528    }
1529    if !is_reorg
1530        && acknowledged_cursor.is_some_and(|acknowledged| {
1531            (cursor.next_block < acknowledged.next_block && !is_activation)
1532                || wire_head_regresses_or_changes(acknowledged, cursor)
1533        })
1534    {
1535        return Err("non-reorg delivery cursor regresses canonical progress");
1536    }
1537    if let delivery::Payload::Reorg(reorg) = payload {
1538        let ancestor = reorg
1539            .common_ancestor
1540            .as_ref()
1541            .ok_or("reorg is missing its common ancestor")?;
1542        let old_tip = reorg
1543            .old_tip
1544            .as_ref()
1545            .ok_or("reorg is missing its old tip")?;
1546        let new_tip = reorg
1547            .new_tip
1548            .as_ref()
1549            .ok_or("reorg is missing its new tip")?;
1550        validate_wire_reorg_shape(ancestor, old_tip, new_tip)?;
1551        if acknowledged_cursor.and_then(|cursor| cursor.canonical_head.as_ref()) != Some(old_tip)
1552            || old_tip.number <= ancestor.number
1553            || new_tip.number <= ancestor.number
1554            || ancestor.number.checked_add(1) != Some(cursor.next_block)
1555        {
1556            return Err("reorg ancestry, prior tip, or cursor successor is invalid");
1557        }
1558    }
1559
1560    let expected_head = match payload {
1561        delivery::Payload::Data(data) => {
1562            if data.records.is_empty() {
1563                return Err("data delivery contains no records");
1564            }
1565            let mut head = None;
1566            let mut last_number = None;
1567            let mut only_owner_catchup = true;
1568            let mut highest_canonical_record = None;
1569            let mut highest_canonical_coverage = None;
1570            for record in &data.records {
1571                let scope = WireDeliveryScope::try_from(record.scope)
1572                    .map_err(|_| "data record has an unknown delivery scope")?;
1573                if scope == WireDeliveryScope::Unspecified {
1574                    return Err("data record has an unspecified delivery scope");
1575                }
1576                if scope == WireDeliveryScope::OwnerCatchup {
1577                    continue;
1578                }
1579                only_owner_catchup = false;
1580                let Some(event) = record.event.as_ref().and_then(|event| event.event.as_ref())
1581                else {
1582                    return Err("data record is missing its chain event");
1583                };
1584                let (record_number, block) = match event {
1585                    chain_event::Event::BlockHeader(header) => (
1586                        header.block.as_ref().map(|block| block.number),
1587                        header.block.as_ref(),
1588                    ),
1589                    chain_event::Event::BlockProgress(progress) => (
1590                        progress.block.as_ref().map(|block| block.number),
1591                        progress.block.as_ref(),
1592                    ),
1593                    chain_event::Event::Log(log) => (Some(log.block_number), None),
1594                };
1595                let record_number =
1596                    record_number.ok_or("canonical block record is missing its block reference")?;
1597                highest_canonical_record = Some(
1598                    highest_canonical_record
1599                        .map_or(record_number, |known: u64| known.max(record_number)),
1600                );
1601                if let Some(block) = block {
1602                    highest_canonical_coverage = Some(
1603                        highest_canonical_coverage
1604                            .map_or(block.number, |known: u64| known.max(block.number)),
1605                    );
1606                    if last_number.is_some_and(|previous| block.number < previous) {
1607                        return Err("canonical block records are not ordered by height");
1608                    }
1609                    last_number = Some(block.number);
1610                    head = Some(block);
1611                }
1612            }
1613            if !only_owner_catchup
1614                && highest_canonical_record
1615                    .zip(highest_canonical_coverage)
1616                    .is_none_or(|(record, coverage)| coverage < record)
1617            {
1618                return Err(
1619                    "canonical data is not certified by a final block identity at or above every canonical record",
1620                );
1621            }
1622            if only_owner_catchup {
1623                acknowledged_cursor.and_then(|acknowledged| acknowledged.canonical_head.as_ref())
1624            } else {
1625                head
1626            }
1627        }
1628        delivery::Payload::Reorg(reorg) => reorg.common_ancestor.as_ref(),
1629        delivery::Payload::Finality(_) => {
1630            acknowledged_cursor.and_then(|acknowledged| acknowledged.canonical_head.as_ref())
1631        }
1632        delivery::Payload::Barrier(barrier) => barrier
1633            .block
1634            .as_ref()
1635            .or_else(|| {
1636                acknowledged_cursor.and_then(|acknowledged| acknowledged.canonical_head.as_ref())
1637            })
1638            .or(if is_activation {
1639                activation_baseline
1640            } else {
1641                None
1642            }),
1643    };
1644    if cursor.canonical_head.as_ref() != expected_head {
1645        return Err("delivery cursor canonical head disagrees with its payload");
1646    }
1647    let cursor_is_behind_coverage = cursor
1648        .canonical_head
1649        .as_ref()
1650        .is_some_and(|head| cursor.next_block <= head.number);
1651    let is_owner_catchup_data = matches!(payload, delivery::Payload::Data(data)
1652    if data.records.iter().all(|record| {
1653        record.scope == i32::from(WireDeliveryScope::OwnerCatchup)
1654    }));
1655    let preserves_acknowledged_head = acknowledged_cursor
1656        .is_none_or(|acknowledged| cursor.canonical_head == acknowledged.canonical_head);
1657    if cursor_is_behind_coverage
1658        && !(preserves_acknowledged_head
1659            && (is_activation || is_owner_catchup_data || is_scan_progress))
1660    {
1661        return Err("delivery cursor next block does not follow its canonical head");
1662    }
1663    Ok(())
1664}
1665
1666fn validate_wire_reorg_shape(
1667    ancestor: &WireBlockRef,
1668    old_tip: &WireBlockRef,
1669    new_tip: &WireBlockRef,
1670) -> Result<(), &'static str> {
1671    let direct_old_parent_mismatch =
1672        old_tip.number == ancestor.number.saturating_add(1) && old_tip.parent_hash != ancestor.hash;
1673    let direct_new_parent_mismatch =
1674        new_tip.number == ancestor.number.saturating_add(1) && new_tip.parent_hash != ancestor.hash;
1675    if old_tip.number <= ancestor.number
1676        || new_tip.number <= ancestor.number
1677        || old_tip.timestamp <= ancestor.timestamp
1678        || new_tip.timestamp <= ancestor.timestamp
1679        || old_tip.hash == new_tip.hash
1680        || old_tip.hash == ancestor.hash
1681        || new_tip.hash == ancestor.hash
1682        || direct_old_parent_mismatch
1683        || direct_new_parent_mismatch
1684    {
1685        return Err("reorg tips do not describe two distinct descendant branches");
1686    }
1687    Ok(())
1688}
1689
1690fn is_wire_activation(
1691    acknowledged_cursor: Option<&Cursor>,
1692    activation_baseline: Option<&WireBlockRef>,
1693    delivery: &Delivery,
1694) -> bool {
1695    let Some(cursor) = delivery.cursor.as_ref() else {
1696        return false;
1697    };
1698    let expected_revision =
1699        acknowledged_cursor.map_or(Some(1), |cursor| cursor.query_revision.checked_add(1));
1700    let expected_head = acknowledged_cursor
1701        .and_then(|cursor| cursor.canonical_head.as_ref())
1702        .or(activation_baseline);
1703    expected_revision == Some(delivery.query_revision)
1704        && cursor.canonical_head.as_ref() == expected_head
1705        && matches!(
1706            delivery.payload.as_ref(),
1707            Some(delivery::Payload::Barrier(barrier))
1708                if barrier.block.is_none()
1709                    && barrier.id
1710                        == format!("desired-state:{}", delivery.query_revision).as_bytes()
1711        )
1712}
1713
1714fn is_wire_cursor_progress(delivery: &Delivery) -> bool {
1715    let Some(cursor) = delivery.cursor.as_ref() else {
1716        return false;
1717    };
1718    matches!(
1719        delivery.payload.as_ref(),
1720        Some(delivery::Payload::Barrier(barrier))
1721            if barrier.block.is_none()
1722                && barrier.id
1723                    == format!(
1724                        "source-progress:{}:{}",
1725                        delivery.query_revision, cursor.next_block
1726                    )
1727                    .as_bytes()
1728    )
1729}
1730
1731fn wire_head_regresses_or_changes(previous: &Cursor, current: &Cursor) -> bool {
1732    match (
1733        previous.canonical_head.as_ref(),
1734        current.canonical_head.as_ref(),
1735    ) {
1736        (Some(previous), Some(current)) => {
1737            current.number < previous.number
1738                || (current.number == previous.number && current != previous)
1739        }
1740        (Some(_), None) => true,
1741        _ => false,
1742    }
1743}
1744
1745#[derive(Clone)]
1746struct OwnedInterests<N: Network> {
1747    interests: Vec<ReactiveInterest<N>>,
1748    backfill: Option<SubscriberBackfill>,
1749}
1750
1751#[derive(Clone)]
1752struct PendingCandidate<N: Network> {
1753    request: ApplyDesiredState,
1754    base_interests: Vec<ReactiveInterest<N>>,
1755    global_backfill: Option<SubscriberBackfill>,
1756    owners: HashMap<HandlerId, OwnedInterests<N>>,
1757}
1758
1759#[derive(Clone)]
1760struct PendingNeutralAcknowledgement {
1761    sequence: u64,
1762    token: Vec<u8>,
1763    cursor: Cursor,
1764}
1765
1766type RestoredDesiredState = (
1767    Vec<ReactiveInterest<Ethereum>>,
1768    Option<SubscriberBackfill>,
1769    HashMap<HandlerId, OwnedInterests<Ethereum>>,
1770);
1771
1772/// Remote, owner-aware event subscriber backed by an authoritative service.
1773pub struct RemoteSubscriber<T, N: Network = Ethereum> {
1774    session_id: String,
1775    chain_id: u64,
1776    committed_revision: u64,
1777    base_interests: Vec<ReactiveInterest<N>>,
1778    global_backfill: Option<SubscriberBackfill>,
1779    owners: HashMap<HandlerId, OwnedInterests<N>>,
1780    pending_candidate: Option<PendingCandidate<N>>,
1781    pending_neutral_acknowledgement: Option<PendingNeutralAcknowledgement>,
1782    acknowledged_sequence: u64,
1783    acknowledged_cursor: Option<Cursor>,
1784    acknowledged_token: Option<Vec<u8>>,
1785    delivered_sequence: Option<u64>,
1786    delivered_cursor: Option<Cursor>,
1787    capabilities: SubscriberCapabilities,
1788    transport: T,
1789    network: PhantomData<fn() -> N>,
1790}
1791
1792impl<T, N: Network> RemoteSubscriber<T, N> {
1793    /// Create a remote subscriber for one durable session and chain.
1794    pub fn new(session_id: impl Into<String>, chain_id: u64, transport: T) -> Self
1795    where
1796        T: RemoteEventTransport,
1797    {
1798        Self::new_at_revision(session_id, chain_id, 0, transport)
1799    }
1800
1801    /// Restore a remote subscriber at a service-authoritative revision.
1802    pub fn new_at_revision(
1803        session_id: impl Into<String>,
1804        chain_id: u64,
1805        committed_revision: u64,
1806        transport: T,
1807    ) -> Self
1808    where
1809        T: RemoteEventTransport,
1810    {
1811        let capabilities = transport.capabilities();
1812        Self {
1813            session_id: session_id.into(),
1814            chain_id,
1815            committed_revision,
1816            base_interests: Vec::new(),
1817            global_backfill: None,
1818            owners: HashMap::new(),
1819            pending_candidate: None,
1820            pending_neutral_acknowledgement: None,
1821            acknowledged_sequence: 0,
1822            acknowledged_cursor: None,
1823            acknowledged_token: None,
1824            delivered_sequence: None,
1825            delivered_cursor: None,
1826            capabilities,
1827            transport,
1828            network: PhantomData,
1829        }
1830    }
1831
1832    /// Last authoritative desired-state revision acknowledged by the service.
1833    pub fn committed_revision(&self) -> u64 {
1834        self.committed_revision
1835    }
1836
1837    /// Borrow the transport.
1838    pub fn transport(&self) -> &T {
1839        &self.transport
1840    }
1841
1842    /// Mutably borrow the transport.
1843    pub fn transport_mut(&mut self) -> &mut T {
1844        &mut self.transport
1845    }
1846}
1847
1848impl<T> RemoteSubscriber<T, Ethereum>
1849where
1850    T: RemoteEventTransport,
1851{
1852    /// Restore local registration mirrors from the service-authoritative wire
1853    /// state. Provider-portable filters are exact; local-only matchers and
1854    /// route-key extractors cannot cross the protocol boundary and are reset.
1855    ///
1856    /// # Errors
1857    ///
1858    /// Returns [`RemoteError::InvalidAuthoritativeState`] when the supplied
1859    /// session, chain, revision, owner topology, filter, or backfill state does
1860    /// not form one self-consistent authoritative snapshot.
1861    pub fn new_from_authoritative(
1862        session_id: impl Into<String>,
1863        chain_id: u64,
1864        transport: T,
1865        desired_state: Option<ApplyDesiredState>,
1866        committed_revision: u64,
1867    ) -> Result<Self, RemoteError> {
1868        let session_id = session_id.into();
1869        let capabilities = transport.capabilities();
1870        let (base_interests, global_backfill, owners) = match desired_state {
1871            Some(desired_state) => decode_authoritative_desired_state(
1872                &session_id,
1873                chain_id,
1874                committed_revision,
1875                desired_state,
1876            )?,
1877            None if committed_revision == 0 => (Vec::new(), None, HashMap::new()),
1878            None => {
1879                return Err(RemoteError::InvalidAuthoritativeState(
1880                    "non-zero revision is missing desired state",
1881                ));
1882            }
1883        };
1884        let mut subscriber = Self {
1885            session_id,
1886            chain_id,
1887            committed_revision,
1888            base_interests,
1889            global_backfill,
1890            owners,
1891            pending_candidate: None,
1892            pending_neutral_acknowledgement: None,
1893            acknowledged_sequence: 0,
1894            acknowledged_cursor: None,
1895            acknowledged_token: None,
1896            delivered_sequence: None,
1897            delivered_cursor: None,
1898            capabilities,
1899            transport,
1900            network: PhantomData,
1901        };
1902        subscriber.restore_transport_authority();
1903        Ok(subscriber)
1904    }
1905}
1906
1907impl RemoteSubscriber<GrpcEventTransport, Ethereum> {
1908    /// Connect and restore the authoritative desired-state revision.
1909    ///
1910    /// If this sequence-zero handshake finds an existing nonzero durable
1911    /// cursor, operations remain gated until [`EventSubscriber::restore_position`]
1912    /// confirms the corresponding runtime checkpoint. Persisted callers that
1913    /// already have that position should prefer [`Self::connect_from_position`].
1914    ///
1915    /// # Errors
1916    ///
1917    /// Returns [`RemoteTransportError`] when transport negotiation fails or the
1918    /// service's authoritative desired state/cursor is internally inconsistent.
1919    pub async fn connect(
1920        endpoint: impl Into<String>,
1921        session_id: impl Into<String>,
1922        chain_id: u64,
1923    ) -> Result<Self, RemoteTransportError> {
1924        let session_id = session_id.into();
1925        let transport = GrpcEventTransport::connect(endpoint, &session_id, chain_id, 0).await?;
1926        let acknowledged_sequence = accepted_sequence(transport.accepted())?;
1927        let revision = transport.accepted().committed_revision;
1928        let desired_state = transport.accepted().desired_state.clone();
1929        let mut subscriber =
1930            Self::new_from_authoritative(session_id, chain_id, transport, desired_state, revision)
1931                .map_err(|error| RemoteTransportError::Protocol(error.to_string()))?;
1932        subscriber.acknowledged_sequence = acknowledged_sequence;
1933        Ok(subscriber)
1934    }
1935
1936    /// Connect using the delivery token/checkpoint restored with runtime state.
1937    /// Unlike the convenience `connect` constructor, this sends the restored
1938    /// sequence and cursor proof in `Hello`. Exact service/runtime agreement is
1939    /// required except for the proven crash window where that exact delivery
1940    /// remains one sequence ahead in the service's durable outbox.
1941    ///
1942    /// # Errors
1943    ///
1944    /// Returns [`RemoteTransportError`] when the restored position is malformed
1945    /// or disagrees with remote authority, transport negotiation fails, or the
1946    /// accepted authoritative state is protocol-invalid.
1947    pub async fn connect_from_position(
1948        endpoint: impl Into<String>,
1949        session_id: impl Into<String>,
1950        chain_id: u64,
1951        position: &SubscriberResumePosition,
1952    ) -> Result<Self, RemoteTransportError> {
1953        let session_id = session_id.into();
1954        let sequence = resume_sequence(position)?;
1955        let resume = pending_delivery_resume(position)?;
1956        let mut transport = GrpcEventTransport::connect_with_resume(
1957            endpoint.into(),
1958            session_id.clone(),
1959            chain_id,
1960            sequence,
1961            None,
1962            GrpcTransportConfig::default(),
1963            resume,
1964        )
1965        .await?;
1966        transport.restore_position(position)?;
1967        let acknowledged_sequence = accepted_sequence(transport.accepted())?;
1968        let revision = transport.accepted().committed_revision;
1969        let desired_state = transport.accepted().desired_state.clone();
1970        let mut subscriber =
1971            Self::new_from_authoritative(session_id, chain_id, transport, desired_state, revision)
1972                .map_err(|error| RemoteTransportError::Protocol(error.to_string()))?;
1973        subscriber.acknowledged_sequence = acknowledged_sequence;
1974        Ok(subscriber)
1975    }
1976
1977    /// Connect with bearer authorization retained across transport reconnects.
1978    /// Existing nonzero sessions have the same restore gate as [`Self::connect`].
1979    ///
1980    /// # Security
1981    ///
1982    /// Use an HTTPS endpoint, or plaintext only inside an authenticated and
1983    /// trusted local service mesh. The API retains the bearer token in memory
1984    /// so it can authenticate reconnect attempts.
1985    ///
1986    /// # Errors
1987    ///
1988    /// Returns [`RemoteTransportError`] for invalid authorization metadata,
1989    /// failed/rejected negotiation, or an inconsistent authoritative state.
1990    pub async fn connect_with_bearer_token(
1991        endpoint: impl Into<String>,
1992        session_id: impl Into<String>,
1993        chain_id: u64,
1994        bearer_token: impl AsRef<str>,
1995    ) -> Result<Self, RemoteTransportError> {
1996        let session_id = session_id.into();
1997        let transport = GrpcEventTransport::connect_with_authorization(
1998            endpoint,
1999            &session_id,
2000            chain_id,
2001            0,
2002            Some(format!("Bearer {}", bearer_token.as_ref())),
2003        )
2004        .await?;
2005        let acknowledged_sequence = accepted_sequence(transport.accepted())?;
2006        let revision = transport.accepted().committed_revision;
2007        let desired_state = transport.accepted().desired_state.clone();
2008        let mut subscriber =
2009            Self::new_from_authoritative(session_id, chain_id, transport, desired_state, revision)
2010                .map_err(|error| RemoteTransportError::Protocol(error.to_string()))?;
2011        subscriber.acknowledged_sequence = acknowledged_sequence;
2012        Ok(subscriber)
2013    }
2014
2015    /// Bearer-authenticated variant of `connect_from_position`.
2016    ///
2017    /// # Security
2018    ///
2019    /// Use an HTTPS endpoint, or plaintext only inside an authenticated and
2020    /// trusted local service mesh. The API retains the bearer token in memory
2021    /// so it can authenticate reconnect attempts.
2022    ///
2023    /// # Errors
2024    ///
2025    /// Returns [`RemoteTransportError`] when authorization metadata or the
2026    /// restored position is invalid, negotiation fails, or remote authority
2027    /// disagrees with the supplied durable proof.
2028    pub async fn connect_with_bearer_token_from_position(
2029        endpoint: impl Into<String>,
2030        session_id: impl Into<String>,
2031        chain_id: u64,
2032        bearer_token: impl AsRef<str>,
2033        position: &SubscriberResumePosition,
2034    ) -> Result<Self, RemoteTransportError> {
2035        let session_id = session_id.into();
2036        let sequence = resume_sequence(position)?;
2037        let resume = pending_delivery_resume(position)?;
2038        let mut transport = GrpcEventTransport::connect_with_resume(
2039            endpoint.into(),
2040            session_id.clone(),
2041            chain_id,
2042            sequence,
2043            Some(format!("Bearer {}", bearer_token.as_ref())),
2044            GrpcTransportConfig::default(),
2045            resume,
2046        )
2047        .await?;
2048        transport.restore_position(position)?;
2049        let acknowledged_sequence = accepted_sequence(transport.accepted())?;
2050        let revision = transport.accepted().committed_revision;
2051        let desired_state = transport.accepted().desired_state.clone();
2052        let mut subscriber =
2053            Self::new_from_authoritative(session_id, chain_id, transport, desired_state, revision)
2054                .map_err(|error| RemoteTransportError::Protocol(error.to_string()))?;
2055        subscriber.acknowledged_sequence = acknowledged_sequence;
2056        Ok(subscriber)
2057    }
2058}
2059
2060impl<T, N> RemoteSubscriber<T, N>
2061where
2062    T: RemoteEventTransport,
2063    N: Network,
2064{
2065    fn commit_candidate(
2066        &mut self,
2067        candidate: PendingCandidate<N>,
2068        applied: &DesiredStateApplied,
2069    ) -> Result<(), SubscriberError> {
2070        if applied.session_id != self.session_id
2071            || applied.revision != candidate.request.new_revision
2072            || applied.activation_sequence
2073                != self.acknowledged_sequence.checked_add(1).ok_or_else(|| {
2074                    SubscriberError::Provider("remote delivery sequence overflow".into())
2075                })?
2076        {
2077            return Err(SubscriberError::Provider(format!(
2078                "invalid desired-state acknowledgement: session={}, revision={}",
2079                applied.session_id, applied.revision
2080            )));
2081        }
2082        self.base_interests = candidate.base_interests;
2083        self.global_backfill = candidate.global_backfill;
2084        self.owners = candidate.owners;
2085        self.committed_revision = candidate.request.new_revision;
2086        self.pending_candidate = None;
2087        self.maybe_complete_backfill();
2088        Ok(())
2089    }
2090
2091    fn restore_transport_authority(&mut self) {
2092        if let Some(sequence) = self.transport.durable_acknowledged_sequence() {
2093            self.acknowledged_sequence = sequence;
2094            self.acknowledged_token = (sequence != 0).then(|| sequence.to_be_bytes().to_vec());
2095        }
2096        self.acknowledged_cursor = self.transport.durable_acknowledged_cursor();
2097        self.maybe_complete_backfill();
2098    }
2099
2100    fn has_incomplete_backfill(&self) -> bool {
2101        self.global_backfill.is_some() || self.owners.values().any(|owner| owner.backfill.is_some())
2102    }
2103
2104    fn has_open_backfill(&self) -> bool {
2105        self.global_backfill
2106            .as_ref()
2107            .is_some_and(|backfill| backfill.end_block().is_none())
2108            || self.owners.values().any(|owner| {
2109                owner
2110                    .backfill
2111                    .as_ref()
2112                    .is_some_and(|backfill| backfill.end_block().is_none())
2113            })
2114    }
2115
2116    fn maybe_complete_backfill(&mut self) {
2117        if !self.has_incomplete_backfill() {
2118            return;
2119        }
2120        let Some(cursor) = self.acknowledged_cursor.as_ref() else {
2121            return;
2122        };
2123        if cursor.query_revision != self.committed_revision {
2124            return;
2125        }
2126        if let Some(backfill) = self.global_backfill.as_ref() {
2127            let target = backfill
2128                .end_block()
2129                .and_then(|end| end.checked_add(1))
2130                .or(cursor.owner_backfill_activation_block);
2131            if target.is_some_and(|target| cursor.next_block >= target) {
2132                self.global_backfill = None;
2133            }
2134        }
2135        for owner in self.owners.values_mut() {
2136            let Some(backfill) = owner.backfill.as_ref() else {
2137                continue;
2138            };
2139            let target = backfill
2140                .end_block()
2141                .and_then(|end| end.checked_add(1))
2142                .or(cursor.owner_backfill_activation_block);
2143            if target.is_some_and(|target| cursor.next_block >= target) {
2144                owner.backfill = None;
2145            }
2146        }
2147    }
2148
2149    async fn reconcile_neutral_acknowledgement(&mut self) -> Result<(), SubscriberError> {
2150        let Some(pending) = self.pending_neutral_acknowledgement.clone() else {
2151            return Ok(());
2152        };
2153        self.transport
2154            .acknowledge(Acknowledge {
2155                session_id: self.session_id.clone(),
2156                sequence: pending.sequence,
2157                delivery_token: pending.token.clone(),
2158            })
2159            .await
2160            .map_err(transport_error)?;
2161        self.acknowledged_sequence = pending.sequence;
2162        self.acknowledged_cursor = Some(pending.cursor);
2163        self.acknowledged_token = Some(pending.token);
2164        self.pending_neutral_acknowledgement = None;
2165        self.maybe_complete_backfill();
2166        Ok(())
2167    }
2168
2169    async fn reconcile_pending_candidate(&mut self) -> Result<(), SubscriberError> {
2170        let Some(candidate) = self.pending_candidate.clone() else {
2171            return Ok(());
2172        };
2173        let applied = match self
2174            .transport
2175            .apply_desired_state(candidate.request.clone())
2176            .await
2177        {
2178            Ok(applied) => applied,
2179            Err(error) => {
2180                if !error.operation_uncertain() {
2181                    self.pending_candidate = None;
2182                }
2183                return Err(transport_error(error));
2184            }
2185        };
2186        self.commit_candidate(candidate, &applied)
2187    }
2188
2189    async fn apply_candidate(
2190        &mut self,
2191        base_interests: Vec<ReactiveInterest<N>>,
2192        global_backfill: Option<SubscriberBackfill>,
2193        owners: HashMap<HandlerId, OwnedInterests<N>>,
2194        supersedes_incomplete_backfill: bool,
2195    ) -> Result<(), SubscriberError> {
2196        self.reconcile_neutral_acknowledgement().await?;
2197        if self.pending_candidate.is_some() {
2198            return Err(SubscriberError::Provider(
2199                "remote desired-state reconciliation is still pending".into(),
2200            ));
2201        }
2202        if self.has_incomplete_backfill() && !supersedes_incomplete_backfill {
2203            return Err(SubscriberError::Provider(
2204                "remote owner backfill is still in progress; lifecycle mutation is fenced until an acknowledged cursor reaches global coverage"
2205                    .into(),
2206            ));
2207        }
2208        let new_revision =
2209            self.committed_revision
2210                .checked_add(1)
2211                .ok_or(SubscriberError::InvalidConfig(
2212                    "remote desired-state revision overflow",
2213                ))?;
2214        let request = build_desired_state(
2215            &self.session_id,
2216            self.chain_id,
2217            self.committed_revision,
2218            new_revision,
2219            &base_interests,
2220            global_backfill.as_ref(),
2221            &owners,
2222        )
2223        .map_err(remote_error)?;
2224        let candidate = PendingCandidate {
2225            request: request.clone(),
2226            base_interests,
2227            global_backfill,
2228            owners,
2229        };
2230        self.pending_candidate = Some(candidate.clone());
2231        let applied = match self.transport.apply_desired_state(request).await {
2232            Ok(applied) => applied,
2233            Err(error) => {
2234                if !error.operation_uncertain() {
2235                    self.pending_candidate = None;
2236                }
2237                return Err(transport_error(error));
2238            }
2239        };
2240        self.commit_candidate(candidate, &applied)
2241    }
2242}
2243
2244impl<T> EventSubscriber<Ethereum> for RemoteSubscriber<T, Ethereum>
2245where
2246    T: RemoteEventTransport,
2247{
2248    fn chain_id(&self) -> Option<u64> {
2249        Some(self.chain_id)
2250    }
2251
2252    fn capabilities(&self) -> SubscriberCapabilities {
2253        self.capabilities.clone()
2254    }
2255
2256    fn restore_position(
2257        &mut self,
2258        position: &SubscriberResumePosition,
2259    ) -> Result<(), SubscriberError> {
2260        if self.pending_candidate.is_some()
2261            || self.pending_neutral_acknowledgement.is_some()
2262            || self.delivered_sequence.is_some()
2263        {
2264            return Err(SubscriberError::Provider(
2265                "cannot restore a remote position while an operation or delivery is pending".into(),
2266            ));
2267        }
2268        self.transport
2269            .restore_position(position)
2270            .map_err(transport_error)?;
2271        let restored_sequence = resume_sequence(position).map_err(transport_error)?;
2272        let durable_sequence = self
2273            .transport
2274            .durable_acknowledged_sequence()
2275            .unwrap_or(restored_sequence);
2276        self.acknowledged_sequence = durable_sequence;
2277        self.acknowledged_cursor = self.transport.durable_acknowledged_cursor();
2278        self.acknowledged_token = (self.acknowledged_sequence != 0)
2279            .then(|| self.acknowledged_sequence.to_be_bytes().to_vec());
2280        self.delivered_sequence = (position.delivery_token.is_some()
2281            && durable_sequence.checked_add(1) == Some(restored_sequence))
2282        .then_some(restored_sequence);
2283        self.delivered_cursor = None;
2284        self.maybe_complete_backfill();
2285        Ok(())
2286    }
2287
2288    fn register_interests(
2289        &mut self,
2290        interests: &[ReactiveInterest<Ethereum>],
2291    ) -> SubscriberOperation<'_, ()> {
2292        let base_interests = interests.to_vec();
2293        Box::pin(async move {
2294            self.reconcile_pending_candidate().await?;
2295            self.apply_candidate(base_interests, None, HashMap::new(), false)
2296                .await
2297        })
2298    }
2299
2300    fn next_batch(&mut self) -> SubscriberNextBatch<'_, Ethereum> {
2301        Box::pin(async move {
2302            self.reconcile_pending_candidate().await?;
2303            self.reconcile_neutral_acknowledgement().await?;
2304            loop {
2305                match self
2306                    .transport
2307                    .next_delivery()
2308                    .await
2309                    .map_err(transport_error)?
2310                {
2311                    None => return Ok(None),
2312                    Some(delivery) => {
2313                        let expected_sequence = self
2314                            .delivered_sequence
2315                            .map_or_else(|| self.acknowledged_sequence.checked_add(1), Some)
2316                            .ok_or_else(|| {
2317                                SubscriberError::Provider(
2318                                    "remote delivery sequence overflow".into(),
2319                                )
2320                            })?;
2321                        let sequence = delivery.sequence;
2322                        let delivered_cursor = delivery.cursor.clone();
2323                        let checkpoint_neutral = delivery.checkpoint_neutral;
2324                        let delivery_token = delivery.delivery_token.clone();
2325                        let requires_open_backfill_boundary = self.has_open_backfill();
2326                        let activation_baseline = self
2327                            .global_backfill
2328                            .as_ref()
2329                            .and_then(SubscriberBackfill::retained_anchor)
2330                            .map(wire_retained_baseline)
2331                            .transpose()
2332                            .map_err(remote_error)?;
2333                        let authoritative_owners: HashSet<_> =
2334                            self.owners.keys().map(HandlerId::as_str).collect();
2335                        let batch = decode_delivery(
2336                            DeliveryAuthority {
2337                                session_id: &self.session_id,
2338                                chain_id: self.chain_id,
2339                                revision: self.committed_revision,
2340                                expected_sequence,
2341                                acknowledged_cursor: self.acknowledged_cursor.as_ref(),
2342                                activation_baseline: activation_baseline.as_ref(),
2343                                requires_open_backfill_boundary,
2344                            },
2345                            delivery,
2346                            &authoritative_owners,
2347                        )
2348                        .map_err(|error| SubscriberError::Provider(error.to_string()))?;
2349                        if checkpoint_neutral {
2350                            self.pending_neutral_acknowledgement =
2351                                Some(PendingNeutralAcknowledgement {
2352                                    sequence,
2353                                    token: delivery_token,
2354                                    cursor: delivered_cursor.ok_or_else(|| {
2355                                        SubscriberError::Provider(
2356                                            "checkpoint-neutral delivery is missing its cursor"
2357                                                .into(),
2358                                        )
2359                                    })?,
2360                                });
2361                            self.reconcile_neutral_acknowledgement().await?;
2362                            continue;
2363                        }
2364                        self.delivered_sequence = Some(sequence);
2365                        self.delivered_cursor = delivered_cursor;
2366                        return Ok(Some(batch));
2367                    }
2368                }
2369            }
2370        })
2371    }
2372
2373    fn acknowledge_delivery(
2374        &mut self,
2375        token: SubscriberDeliveryToken,
2376    ) -> SubscriberOperation<'_, ()> {
2377        Box::pin(async move {
2378            self.reconcile_pending_candidate().await?;
2379            self.reconcile_neutral_acknowledgement().await?;
2380            let bytes = token.as_bytes();
2381            let sequence = u64::from_be_bytes(bytes.try_into().map_err(|_| {
2382                SubscriberError::Provider("invalid remote delivery token width".into())
2383            })?);
2384            if self.delivered_sequence.is_none()
2385                && self.acknowledged_sequence == sequence
2386                && self.acknowledged_token.as_deref() == Some(bytes)
2387            {
2388                return Ok(());
2389            }
2390            if self.delivered_sequence != Some(sequence) {
2391                return Err(SubscriberError::Provider(
2392                    "remote acknowledgement does not match the delivered sequence".into(),
2393                ));
2394            }
2395            let acknowledged_token = token.as_bytes().to_vec();
2396            self.transport
2397                .acknowledge(Acknowledge {
2398                    session_id: self.session_id.clone(),
2399                    sequence,
2400                    delivery_token: acknowledged_token.clone(),
2401                })
2402                .await
2403                .map_err(transport_error)?;
2404            self.acknowledged_sequence = sequence;
2405            self.acknowledged_cursor = self
2406                .delivered_cursor
2407                .take()
2408                .or_else(|| self.transport.durable_acknowledged_cursor());
2409            self.acknowledged_token = Some(acknowledged_token);
2410            self.delivered_sequence = None;
2411            self.maybe_complete_backfill();
2412            Ok(())
2413        })
2414    }
2415}
2416
2417fn runtime_capabilities(wire: Option<&SourceCapabilities>) -> SubscriberCapabilities {
2418    let Some(wire) = wire else {
2419        return SubscriberCapabilities::default();
2420    };
2421    let mut capabilities = Vec::new();
2422    let mut has_safe_head = false;
2423    let mut has_finalized_head = false;
2424    for capability in &wire.capabilities {
2425        let Ok(capability) = WireCapability::try_from(*capability) else {
2426            continue;
2427        };
2428        let mapped = match capability {
2429            WireCapability::Historical => Some(SubscriberCapability::HistoricalBackfill),
2430            WireCapability::Live => Some(SubscriberCapability::Live),
2431            WireCapability::Logs => Some(SubscriberCapability::Logs),
2432            WireCapability::Headers => Some(SubscriberCapability::BlockHeaders),
2433            WireCapability::DynamicFilters => Some(SubscriberCapability::DynamicInterests),
2434            WireCapability::ExplicitReorgs => Some(SubscriberCapability::ExplicitReorgs),
2435            WireCapability::SafeHead => {
2436                has_safe_head = true;
2437                None
2438            }
2439            WireCapability::FinalizedHead => {
2440                has_finalized_head = true;
2441                None
2442            }
2443            WireCapability::OwnerScopedDelivery => Some(SubscriberCapability::OwnerScopedDelivery),
2444            WireCapability::DurableReplay => Some(SubscriberCapability::DurableReplay),
2445            WireCapability::Unspecified
2446            | WireCapability::Transactions
2447            | WireCapability::FullBlocks
2448            | WireCapability::Pending
2449            | WireCapability::ServerFiltering
2450            | WireCapability::NativeCheckpoint => None,
2451        };
2452        if let Some(mapped) = mapped {
2453            capabilities.push(mapped);
2454        }
2455    }
2456    if has_safe_head && has_finalized_head {
2457        capabilities.push(SubscriberCapability::FinalityUpdates);
2458    }
2459    capabilities.push(SubscriberCapability::Barriers);
2460    SubscriberCapabilities::new(capabilities)
2461}
2462
2463#[cfg(test)]
2464mod capability_tests {
2465    use super::*;
2466
2467    #[test]
2468    fn wire_capabilities_never_overclaim_unimplemented_runtime_shapes() {
2469        let wire = SourceCapabilities {
2470            capabilities: vec![
2471                WireCapability::FullBlocks.into(),
2472                WireCapability::Pending.into(),
2473                WireCapability::SafeHead.into(),
2474            ],
2475            sources: Vec::new(),
2476        };
2477        let capabilities = runtime_capabilities(Some(&wire));
2478        assert!(!capabilities.supports(SubscriberCapability::FullBlocks));
2479        assert!(!capabilities.supports(SubscriberCapability::PendingTransactions));
2480        assert!(!capabilities.supports(SubscriberCapability::FinalityUpdates));
2481    }
2482
2483    #[test]
2484    fn finality_requires_both_safe_and_finalized_wire_heads() {
2485        let wire = SourceCapabilities {
2486            capabilities: vec![
2487                WireCapability::SafeHead.into(),
2488                WireCapability::FinalizedHead.into(),
2489            ],
2490            sources: Vec::new(),
2491        };
2492        assert!(runtime_capabilities(Some(&wire)).supports(SubscriberCapability::FinalityUpdates));
2493    }
2494}
2495
2496#[cfg(test)]
2497mod resume_tests {
2498    use super::*;
2499
2500    fn pending_position(coverage_head: RuntimeBlockRef) -> SubscriberResumePosition {
2501        SubscriberResumePosition::new(
2502            1,
2503            coverage_head,
2504            Vec::new(),
2505            Some(SubscriberDeliveryToken::new(1_u64.to_be_bytes().to_vec())),
2506            None,
2507        )
2508    }
2509
2510    #[test]
2511    fn pending_resume_rejects_incomplete_coverage_metadata() {
2512        let missing_parent = pending_position(RuntimeBlockRef {
2513            number: 12,
2514            hash: B256::repeat_byte(0x12),
2515            parent_hash: None,
2516            timestamp: Some(1_700_000_012),
2517        });
2518        assert!(
2519            pending_delivery_resume(&missing_parent)
2520                .expect_err("missing parent hash cannot become a durable wire proof")
2521                .to_string()
2522                .contains("missing its parent hash")
2523        );
2524
2525        let missing_timestamp = pending_position(RuntimeBlockRef {
2526            number: 12,
2527            hash: B256::repeat_byte(0x12),
2528            parent_hash: Some(B256::repeat_byte(0x11)),
2529            timestamp: None,
2530        });
2531        assert!(
2532            pending_delivery_resume(&missing_timestamp)
2533                .expect_err("missing timestamp cannot become a durable wire proof")
2534                .to_string()
2535                .contains("missing its timestamp")
2536        );
2537    }
2538
2539    #[test]
2540    fn pending_resume_preserves_complete_coverage_metadata_exactly() {
2541        let position = pending_position(RuntimeBlockRef {
2542            number: 12,
2543            hash: B256::repeat_byte(0x12),
2544            parent_hash: Some(B256::repeat_byte(0x11)),
2545            timestamp: Some(1_700_000_012),
2546        });
2547        let resume = pending_delivery_resume(&position)
2548            .expect("complete proof")
2549            .expect("pending resume");
2550        let coverage = resume.coverage_head.expect("coverage head");
2551        assert_eq!(coverage.number, 12);
2552        assert_eq!(coverage.hash, vec![0x12; 32]);
2553        assert_eq!(coverage.parent_hash, vec![0x11; 32]);
2554        assert_eq!(coverage.timestamp, 1_700_000_012);
2555    }
2556
2557    #[test]
2558    fn runtime_resume_requires_exact_checkpoint_presence_and_coverage() {
2559        let head = WireBlockRef {
2560            number: 12,
2561            hash: vec![0x12; 32],
2562            parent_hash: vec![0x11; 32],
2563            timestamp: 1_700_000_012,
2564        };
2565        let cursor = Cursor {
2566            chain_id: 1,
2567            query_revision: 1,
2568            next_block: 13,
2569            canonical_head: Some(head),
2570            batch_sequence: 1,
2571            provider_checkpoint: b"checkpoint".to_vec(),
2572            owner_backfill_activation_block: None,
2573        };
2574        let accepted = HelloAccepted {
2575            acknowledged_cursor: Some(cursor.clone()),
2576            runtime_checkpoint_position: Some(
2577                evm_fork_cache_event_protocol::v1::RuntimeCheckpointPosition {
2578                    cursor: Some(cursor),
2579                },
2580            ),
2581            ..Default::default()
2582        };
2583        let coverage = RuntimeBlockRef {
2584            number: 12,
2585            hash: B256::repeat_byte(0x12),
2586            parent_hash: Some(B256::repeat_byte(0x11)),
2587            timestamp: Some(1_700_000_012),
2588        };
2589        let exact = SubscriberResumePosition::new(
2590            1,
2591            coverage,
2592            Vec::new(),
2593            Some(SubscriberDeliveryToken::new(1_u64.to_be_bytes().to_vec())),
2594            Some(SubscriberCheckpoint::new(b"checkpoint".to_vec())),
2595        );
2596        validate_resume_position(&accepted, &exact).expect("exact runtime position");
2597
2598        let mut missing_checkpoint = exact.clone();
2599        missing_checkpoint.subscriber_checkpoint = None;
2600        assert!(
2601            validate_resume_position(&accepted, &missing_checkpoint)
2602                .expect_err("missing checkpoint presence must not equal nonempty authority")
2603                .to_string()
2604                .contains("provider checkpoint")
2605        );
2606
2607        let mut wrong_coverage = exact;
2608        wrong_coverage.coverage_head.hash = B256::repeat_byte(0xee);
2609        assert!(
2610            validate_resume_position(&accepted, &wrong_coverage)
2611                .expect_err("coverage identity must match exactly")
2612                .to_string()
2613                .contains("canonical head")
2614        );
2615    }
2616}
2617
2618impl<T> InterestOwnerSubscriber<Ethereum> for RemoteSubscriber<T, Ethereum>
2619where
2620    T: RemoteEventTransport,
2621{
2622    fn upsert_interest_owners(
2623        &mut self,
2624        owner_updates: Vec<(HandlerId, Vec<ReactiveInterest<Ethereum>>)>,
2625    ) -> SubscriberOperation<'_, ()> {
2626        Box::pin(async move {
2627            self.reconcile_pending_candidate().await?;
2628            let mut seen = HashSet::with_capacity(owner_updates.len());
2629            let mut owners = self.owners.clone();
2630            for (owner, interests) in owner_updates {
2631                if !seen.insert(owner.clone()) {
2632                    return Err(SubscriberError::InvalidConfig(
2633                        "bulk remote owner upsert contains a duplicate owner",
2634                    ));
2635                }
2636                owners.insert(
2637                    owner,
2638                    OwnedInterests {
2639                        interests,
2640                        backfill: None,
2641                    },
2642                );
2643            }
2644            self.apply_candidate(self.base_interests.clone(), None, owners, false)
2645                .await
2646        })
2647    }
2648
2649    fn replace_interest_owners(
2650        &mut self,
2651        replacement: Vec<(HandlerId, Vec<ReactiveInterest<Ethereum>>)>,
2652    ) -> SubscriberOperation<'_, ()> {
2653        Box::pin(async move {
2654            self.reconcile_pending_candidate().await?;
2655            let mut owners = HashMap::with_capacity(replacement.len());
2656            for (owner, interests) in replacement {
2657                if owners
2658                    .insert(
2659                        owner,
2660                        OwnedInterests {
2661                            interests,
2662                            backfill: None,
2663                        },
2664                    )
2665                    .is_some()
2666                {
2667                    return Err(SubscriberError::InvalidConfig(
2668                        "exact remote owner replacement contains a duplicate owner",
2669                    ));
2670                }
2671            }
2672            self.apply_candidate(Vec::new(), None, owners, true).await
2673        })
2674    }
2675
2676    fn replace_interest_owners_with_global_backfill(
2677        &mut self,
2678        replacement: Vec<(HandlerId, Vec<ReactiveInterest<Ethereum>>)>,
2679        backfill: SubscriberBackfill,
2680    ) -> SubscriberOperation<'_, ()> {
2681        Box::pin(async move {
2682            self.reconcile_pending_candidate().await?;
2683            let mut owners = HashMap::with_capacity(replacement.len());
2684            for (owner, interests) in replacement {
2685                if owners
2686                    .insert(
2687                        owner,
2688                        OwnedInterests {
2689                            interests,
2690                            backfill: None,
2691                        },
2692                    )
2693                    .is_some()
2694                {
2695                    return Err(SubscriberError::InvalidConfig(
2696                        "global remote owner replacement contains a duplicate owner",
2697                    ));
2698                }
2699            }
2700            self.apply_candidate(Vec::new(), Some(backfill), owners, true)
2701                .await
2702        })
2703    }
2704
2705    fn add_interest_owner(
2706        &mut self,
2707        owner: HandlerId,
2708        interests: &[ReactiveInterest<Ethereum>],
2709    ) -> SubscriberOperation<'_, ()> {
2710        let interests = interests.to_vec();
2711        Box::pin(async move {
2712            self.reconcile_pending_candidate().await?;
2713            let mut owners = self.owners.clone();
2714            owners.insert(
2715                owner,
2716                OwnedInterests {
2717                    interests,
2718                    backfill: None,
2719                },
2720            );
2721            self.apply_candidate(self.base_interests.clone(), None, owners, false)
2722                .await
2723        })
2724    }
2725
2726    fn add_interest_owner_with_backfill(
2727        &mut self,
2728        owner: HandlerId,
2729        interests: &[ReactiveInterest<Ethereum>],
2730        backfill: SubscriberBackfill,
2731    ) -> SubscriberOperation<'_, ()> {
2732        let interests = interests.to_vec();
2733        Box::pin(async move {
2734            self.reconcile_pending_candidate().await?;
2735            let (owner_backfill, global_backfill) = split_mid_lifecycle_backfill(backfill)?;
2736            let mut owners = self.owners.clone();
2737            owners.insert(
2738                owner,
2739                OwnedInterests {
2740                    interests,
2741                    backfill: Some(owner_backfill),
2742                },
2743            );
2744            self.apply_candidate(self.base_interests.clone(), global_backfill, owners, false)
2745                .await
2746        })
2747    }
2748
2749    fn add_interest_owner_with_canonical_catchup(
2750        &mut self,
2751        owner: HandlerId,
2752        interests: &[ReactiveInterest<Ethereum>],
2753        retained: RuntimeBlockRef,
2754    ) -> SubscriberOperation<'_, ()> {
2755        let interests = interests.to_vec();
2756        Box::pin(async move {
2757            self.reconcile_pending_candidate().await?;
2758            let mut owners = self.owners.clone();
2759            owners.insert(
2760                owner,
2761                OwnedInterests {
2762                    interests,
2763                    backfill: Some(SubscriberBackfill::from_block(retained.number)),
2764                },
2765            );
2766            let global_backfill = SubscriberBackfill::after_canonical_block(retained)?;
2767            self.apply_candidate(
2768                self.base_interests.clone(),
2769                Some(global_backfill),
2770                owners,
2771                false,
2772            )
2773            .await
2774        })
2775    }
2776
2777    fn remove_interest_owner(
2778        &mut self,
2779        owner: &HandlerId,
2780    ) -> SubscriberOperation<'_, Option<Vec<ReactiveInterest<Ethereum>>>> {
2781        let owner = owner.clone();
2782        Box::pin(async move {
2783            self.reconcile_pending_candidate().await?;
2784            let mut owners = self.owners.clone();
2785            let removed = owners.remove(&owner).map(|owned| owned.interests);
2786            self.apply_candidate(self.base_interests.clone(), None, owners, false)
2787                .await?;
2788            Ok(removed)
2789        })
2790    }
2791
2792    fn owner_interests(&self, owner: &HandlerId) -> Option<&[ReactiveInterest<Ethereum>]> {
2793        self.owners
2794            .get(owner)
2795            .map(|owned| owned.interests.as_slice())
2796    }
2797}
2798
2799fn split_mid_lifecycle_backfill(
2800    backfill: SubscriberBackfill,
2801) -> Result<(SubscriberBackfill, Option<SubscriberBackfill>), SubscriberError> {
2802    let Some(baseline) = backfill.retained_anchor().copied() else {
2803        return Ok((backfill, None));
2804    };
2805    if baseline.number.checked_add(1) == Some(backfill.start_block()) {
2806        return Ok((backfill, None));
2807    }
2808    if baseline.number != backfill.start_block() {
2809        return Err(SubscriberError::InvalidConfig(
2810            "retained owner backfill anchor is not its start or exact predecessor",
2811        ));
2812    }
2813    let owner_backfill = backfill.end_block().map_or_else(
2814        || SubscriberBackfill::from_block(backfill.start_block()),
2815        |end| SubscriberBackfill::range(backfill.start_block(), end),
2816    );
2817    Ok((owner_backfill, None))
2818}
2819
2820fn build_desired_state<N: Network>(
2821    session_id: &str,
2822    chain_id: u64,
2823    expected_revision: u64,
2824    new_revision: u64,
2825    base_interests: &[ReactiveInterest<N>],
2826    global_backfill: Option<&SubscriberBackfill>,
2827    owners: &HashMap<HandlerId, OwnedInterests<N>>,
2828) -> Result<ApplyDesiredState, RemoteError> {
2829    let mut wire_owners = Vec::with_capacity(
2830        owners.len() + usize::from(!base_interests.is_empty() || global_backfill.is_some()),
2831    );
2832    if !base_interests.is_empty() || global_backfill.is_some() {
2833        wire_owners.push(OwnerInterests {
2834            owner_id: String::new(),
2835            interests: compile_portable_interests(base_interests)?,
2836            backfill: global_backfill.map(encode_backfill).transpose()?,
2837            canonical: true,
2838        });
2839    }
2840    for (owner, owned) in owners {
2841        wire_owners.push(OwnerInterests {
2842            owner_id: owner.as_str().to_owned(),
2843            interests: compile_portable_interests(&owned.interests)?,
2844            backfill: owned.backfill.as_ref().map(encode_backfill).transpose()?,
2845            canonical: false,
2846        });
2847    }
2848    wire_owners.sort_by(|left, right| left.owner_id.cmp(&right.owner_id));
2849    Ok(ApplyDesiredState {
2850        protocol_version: PROTOCOL_VERSION,
2851        session_id: session_id.to_owned(),
2852        chain_id,
2853        expected_revision,
2854        new_revision,
2855        owners: wire_owners,
2856    })
2857}
2858
2859fn encode_backfill(backfill: &SubscriberBackfill) -> Result<Backfill, RemoteError> {
2860    let to_block_excl = backfill
2861        .end_block()
2862        .map(|end| {
2863            end.checked_add(1)
2864                .ok_or(RemoteError::InvalidAuthoritativeState(
2865                    "backfill exclusive upper bound overflows u64",
2866                ))
2867        })
2868        .transpose()?;
2869    if let Some(baseline) = backfill.retained_anchor()
2870        && baseline.number.checked_add(1) != Some(backfill.start_block())
2871    {
2872        return Err(RemoteError::InvalidAuthoritativeState(
2873            "retained backfill baseline must immediately precede its start",
2874        ));
2875    }
2876    Ok(Backfill {
2877        from_block: backfill.start_block(),
2878        to_block_excl,
2879        retained_baseline: backfill
2880            .retained_anchor()
2881            .map(wire_retained_baseline)
2882            .transpose()?,
2883    })
2884}
2885
2886fn wire_retained_baseline(block: &RuntimeBlockRef) -> Result<WireBlockRef, RemoteError> {
2887    let parent_hash = block
2888        .parent_hash
2889        .ok_or(RemoteError::InvalidAuthoritativeState(
2890            "retained backfill baseline is missing its parent hash",
2891        ))?;
2892    let timestamp = block
2893        .timestamp
2894        .ok_or(RemoteError::InvalidAuthoritativeState(
2895            "retained backfill baseline is missing its timestamp",
2896        ))?;
2897    Ok(WireBlockRef {
2898        number: block.number,
2899        hash: block.hash.as_slice().to_vec(),
2900        parent_hash: parent_hash.as_slice().to_vec(),
2901        timestamp,
2902    })
2903}
2904
2905fn decode_authoritative_backfill(backfill: Backfill) -> Result<SubscriberBackfill, RemoteError> {
2906    if backfill
2907        .to_block_excl
2908        .is_some_and(|end| end < backfill.from_block)
2909        || (backfill.retained_baseline.is_none()
2910            && backfill
2911                .to_block_excl
2912                .is_some_and(|end| end == backfill.from_block))
2913    {
2914        return Err(RemoteError::InvalidAuthoritativeState(
2915            "owner backfill range is invalid",
2916        ));
2917    }
2918    if let Some(baseline) = backfill.retained_baseline {
2919        if baseline.number.checked_add(1) != Some(backfill.from_block) {
2920            return Err(RemoteError::InvalidAuthoritativeState(
2921                "retained backfill baseline must immediately precede its start",
2922            ));
2923        }
2924        let baseline = runtime_block_ref(&baseline).map_err(|_| {
2925            RemoteError::InvalidAuthoritativeState(
2926                "retained backfill baseline has an invalid block identity",
2927            )
2928        })?;
2929        return backfill
2930            .to_block_excl
2931            .map_or_else(
2932                || SubscriberBackfill::after_canonical_block(baseline),
2933                |end| SubscriberBackfill::after_canonical_block_through(baseline, end - 1),
2934            )
2935            .map_err(|_| {
2936                RemoteError::InvalidAuthoritativeState(
2937                    "retained backfill baseline or upper bound is invalid",
2938                )
2939            });
2940    }
2941    Ok(backfill.to_block_excl.map_or_else(
2942        || SubscriberBackfill::from_block(backfill.from_block),
2943        |end| SubscriberBackfill::range(backfill.from_block, end - 1),
2944    ))
2945}
2946
2947fn decode_authoritative_desired_state(
2948    session_id: &str,
2949    chain_id: u64,
2950    committed_revision: u64,
2951    desired_state: ApplyDesiredState,
2952) -> Result<RestoredDesiredState, RemoteError> {
2953    if desired_state.protocol_version != PROTOCOL_VERSION
2954        || desired_state.session_id != session_id
2955        || desired_state.chain_id != chain_id
2956        || desired_state.new_revision != committed_revision
2957        || desired_state.expected_revision.checked_add(1) != Some(committed_revision)
2958    {
2959        return Err(RemoteError::InvalidAuthoritativeState(
2960            "identity, version, or revision does not match HelloAccepted",
2961        ));
2962    }
2963    let mut base_interests = Vec::new();
2964    let mut global_backfill = None;
2965    let mut has_base_interests = false;
2966    let mut owners = HashMap::new();
2967    for owner in desired_state.owners {
2968        let interests = decode_portable_interests(owner.interests)?;
2969        if owner.canonical {
2970            if !owner.owner_id.is_empty() {
2971                return Err(RemoteError::InvalidAuthoritativeState(
2972                    "canonical interests must have an empty owner id",
2973                ));
2974            }
2975            if has_base_interests {
2976                return Err(RemoteError::InvalidAuthoritativeState(
2977                    "base owner appears more than once",
2978                ));
2979            }
2980            has_base_interests = true;
2981            base_interests = interests;
2982            global_backfill = owner
2983                .backfill
2984                .map(decode_authoritative_backfill)
2985                .transpose()?;
2986        } else if owner.owner_id.is_empty() {
2987            return Err(RemoteError::InvalidAuthoritativeState(
2988                "non-canonical owner id is empty",
2989            ));
2990        } else {
2991            let backfill = owner
2992                .backfill
2993                .map(decode_authoritative_backfill)
2994                .transpose()?;
2995            let owner_id = HandlerId::try_new(owner.owner_id).map_err(|_| {
2996                RemoteError::InvalidAuthoritativeState("non-canonical owner id is empty")
2997            })?;
2998            if owners
2999                .insert(
3000                    owner_id,
3001                    OwnedInterests {
3002                        interests,
3003                        backfill,
3004                    },
3005                )
3006                .is_some()
3007            {
3008                return Err(RemoteError::InvalidAuthoritativeState(
3009                    "owner appears more than once",
3010                ));
3011            }
3012        }
3013    }
3014    Ok((base_interests, global_backfill, owners))
3015}
3016
3017fn decode_portable_interests(
3018    interests: Vec<PortableInterest>,
3019) -> Result<Vec<ReactiveInterest<Ethereum>>, RemoteError> {
3020    interests
3021        .into_iter()
3022        .map(|interest| match interest.kind {
3023            Some(portable_interest::Kind::Log(log)) => {
3024                let addresses = log
3025                    .addresses
3026                    .into_iter()
3027                    .map(|address| {
3028                        Address::try_from(address.as_slice()).map_err(|_| {
3029                            RemoteError::InvalidAuthoritativeState("log address is not 20 bytes")
3030                        })
3031                    })
3032                    .collect::<Result<Vec<_>, _>>()?;
3033                if log.topics.len() > 4 {
3034                    return Err(RemoteError::InvalidAuthoritativeState(
3035                        "log filter has more than four topic positions",
3036                    ));
3037                }
3038                let mut filter = Filter::new().address(addresses);
3039                for (index, accepted) in log.topics.into_iter().enumerate() {
3040                    let topics = accepted
3041                        .values
3042                        .into_iter()
3043                        .map(|topic| {
3044                            B256::try_from(topic.as_slice()).map_err(|_| {
3045                                RemoteError::InvalidAuthoritativeState("log topic is not 32 bytes")
3046                            })
3047                        })
3048                        .collect::<Result<Vec<_>, _>>()?;
3049                    filter = match index {
3050                        0 => filter.event_signature(topics),
3051                        1 => filter.topic1(topics),
3052                        2 => filter.topic2(topics),
3053                        3 => filter.topic3(topics),
3054                        _ => unreachable!("topic count was bounded above"),
3055                    };
3056                }
3057                Ok(ReactiveInterest::Logs(RuntimeLogInterest {
3058                    provider_filter: filter,
3059                    local_matcher: None,
3060                    route_key: None,
3061                }))
3062            }
3063            Some(portable_interest::Kind::Block(block)) => match BlockMode::try_from(block.mode) {
3064                Ok(BlockMode::Header) => Ok(ReactiveInterest::Blocks(RuntimeBlockInterest {
3065                    mode: BlockInterestMode::Header,
3066                })),
3067                Ok(BlockMode::FullBlock) => Err(RemoteError::UnsupportedInterest("full block")),
3068                _ => Err(RemoteError::InvalidAuthoritativeState(
3069                    "block mode is unspecified or unknown",
3070                )),
3071            },
3072            None => Err(RemoteError::InvalidAuthoritativeState(
3073                "portable interest kind is missing",
3074            )),
3075        })
3076        .collect()
3077}
3078
3079fn remote_error(error: RemoteError) -> SubscriberError {
3080    match error {
3081        RemoteError::UnsupportedInterest(kind) => SubscriberError::Unsupported(kind),
3082        RemoteError::UnsupportedFilterBlockOption => SubscriberError::Unsupported(
3083            "remote log filters with block constraints are unsupported",
3084        ),
3085        RemoteError::InvalidAuthoritativeState(message) => {
3086            SubscriberError::Provider(format!("invalid authoritative desired state: {message}"))
3087        }
3088    }
3089}
3090
3091fn transport_error(error: RemoteTransportError) -> SubscriberError {
3092    SubscriberError::Provider(error.to_string())
3093}
3094
3095fn decode_delivery(
3096    authority: DeliveryAuthority<'_>,
3097    delivery: Delivery,
3098    authoritative_owners: &HashSet<&str>,
3099) -> Result<ReactiveInputBatch<Ethereum>, RemoteDecodeError> {
3100    let payload_commitment = SubscriberPayloadCommitment::new(keccak256(delivery.encode_to_vec()));
3101    if delivery.session_id != authority.session_id {
3102        return Err(RemoteDecodeError::Session {
3103            expected: authority.session_id.to_owned(),
3104            received: delivery.session_id,
3105        });
3106    }
3107    let cursor = delivery
3108        .cursor
3109        .as_ref()
3110        .ok_or(RemoteDecodeError::MissingField("delivery.cursor"))?;
3111    if cursor.chain_id != authority.chain_id {
3112        return Err(RemoteDecodeError::Chain {
3113            expected: authority.chain_id,
3114            received: cursor.chain_id,
3115        });
3116    }
3117    if let Some(head) = cursor.canonical_head.as_ref() {
3118        runtime_block_ref(head)?;
3119    }
3120    if delivery.query_revision != authority.revision
3121        || delivery.sequence != authority.expected_sequence
3122        || cursor.query_revision != delivery.query_revision
3123        || cursor.batch_sequence != delivery.sequence
3124    {
3125        return Err(RemoteDecodeError::Cursor);
3126    }
3127    if delivery.delivery_token.as_slice() != delivery.sequence.to_be_bytes() {
3128        return Err(RemoteDecodeError::DeliveryToken);
3129    }
3130    validate_wire_delivery_progress(
3131        authority.acknowledged_cursor,
3132        authority.activation_baseline,
3133        &delivery,
3134        authority.requires_open_backfill_boundary,
3135    )
3136    .map_err(RemoteDecodeError::CursorAuthority)?;
3137    let token = delivery.delivery_token;
3138    let checkpoint = cursor.provider_checkpoint.clone();
3139    let payload = delivery
3140        .payload
3141        .ok_or(RemoteDecodeError::MissingField("delivery.payload"))?;
3142    let mut batch = match payload {
3143        delivery::Payload::Data(data) => {
3144            decode_data_payload(authority.chain_id, data.records, authoritative_owners)?
3145        }
3146        delivery::Payload::Reorg(reorg) => {
3147            ReactiveInputBatch::new(Vec::new()).with_chain_controls([ChainControl::Reorg {
3148                common_ancestor: runtime_block_ref(
3149                    &reorg
3150                        .common_ancestor
3151                        .ok_or(RemoteDecodeError::MissingField("reorg.common_ancestor"))?,
3152                )?,
3153                old_tip: runtime_block_ref(
3154                    &reorg
3155                        .old_tip
3156                        .ok_or(RemoteDecodeError::MissingField("reorg.old_tip"))?,
3157                )?,
3158                new_tip: runtime_block_ref(
3159                    &reorg
3160                        .new_tip
3161                        .ok_or(RemoteDecodeError::MissingField("reorg.new_tip"))?,
3162                )?,
3163            }])
3164        }
3165        delivery::Payload::Finality(finality) => {
3166            let block = runtime_block_ref(
3167                &finality
3168                    .block
3169                    .ok_or(RemoteDecodeError::MissingField("finality.block"))?,
3170            )?;
3171            let control = match FinalityKind::try_from(finality.kind)
3172                .map_err(|_| RemoteDecodeError::Finality(finality.kind))?
3173            {
3174                FinalityKind::Safe => ChainControl::Safe(block),
3175                FinalityKind::Finalized => ChainControl::Finalized(block),
3176                FinalityKind::Unspecified => {
3177                    return Err(RemoteDecodeError::Finality(finality.kind));
3178                }
3179            };
3180            ReactiveInputBatch::new(Vec::new()).with_chain_controls([control])
3181        }
3182        delivery::Payload::Barrier(barrier) => {
3183            if barrier.id.is_empty() {
3184                return Err(RemoteDecodeError::EmptyBarrierId);
3185            }
3186            ReactiveInputBatch::new(Vec::new()).with_chain_controls([ChainControl::Barrier {
3187                id: barrier.id,
3188                block: barrier.block.as_ref().map(runtime_block_ref).transpose()?,
3189            }])
3190        }
3191    };
3192    batch = batch
3193        .with_chain_id(authority.chain_id)
3194        .with_payload_commitment(payload_commitment)
3195        .with_delivery_token(SubscriberDeliveryToken::new(token));
3196    if !checkpoint.is_empty() {
3197        batch = batch.with_subscriber_checkpoint(SubscriberCheckpoint::new(checkpoint));
3198    }
3199    Ok(batch)
3200}
3201
3202fn decode_data_payload(
3203    expected_chain_id: u64,
3204    wire_records: Vec<evm_fork_cache_event_protocol::v1::EventRecord>,
3205    authoritative_owners: &HashSet<&str>,
3206) -> Result<ReactiveInputBatch<Ethereum>, RemoteDecodeError> {
3207    if wire_records.is_empty() {
3208        return Err(RemoteDecodeError::EmptyData);
3209    }
3210    let mut blocks = HashMap::new();
3211    // Canonical ordering places logs before the final compact progress record
3212    // at the same height. Pre-index every explicit block identity so those logs
3213    // receive the complete parent/timestamp proof rather than a synthetic
3214    // partial context that only the later control could enrich.
3215    for wire_record in &wire_records {
3216        let Some(event) = wire_record
3217            .event
3218            .as_ref()
3219            .and_then(|event| event.event.as_ref())
3220        else {
3221            continue;
3222        };
3223        let wire = match event {
3224            chain_event::Event::BlockHeader(header) => header.block.as_ref(),
3225            chain_event::Event::BlockProgress(progress) => progress.block.as_ref(),
3226            chain_event::Event::Log(_) => None,
3227        };
3228        if let Some(wire) = wire {
3229            let block = runtime_block_ref(wire)?;
3230            if let Some(known) = blocks.insert(block.number, block)
3231                && known != block
3232            {
3233                return Err(RemoteDecodeError::BlockHashMismatch(block.number));
3234            }
3235        }
3236    }
3237    let mut explicit_headers = HashSet::new();
3238    let mut explicit_progress = HashSet::new();
3239    let mut log_identities = HashSet::new();
3240    let mut transaction_indexes = HashMap::<(B256, B256), u64>::new();
3241    let mut transaction_hashes = HashMap::<(B256, u64), B256>::new();
3242    let mut last_event_order = None;
3243    let mut records = Vec::with_capacity(wire_records.len());
3244    let mut canonical_progress = None;
3245    for wire_record in wire_records {
3246        let audience = decode_audience(
3247            wire_record.canonical_audience,
3248            wire_record.owner_ids,
3249            authoritative_owners,
3250        )?;
3251        let scope = decode_delivery_scope(wire_record.scope)?;
3252        if scope == RuntimeDeliveryScope::OwnerCatchup && matches!(audience, DeliveryAudience::All)
3253        {
3254            return Err(RemoteDecodeError::OwnerCatchupAudience);
3255        }
3256        let event = wire_record
3257            .event
3258            .ok_or(RemoteDecodeError::MissingField("event_record.event"))?;
3259        let event = event
3260            .event
3261            .ok_or(RemoteDecodeError::MissingField("chain_event.event"))?;
3262        let event_order = match &event {
3263            chain_event::Event::BlockHeader(header) => (
3264                header.block.as_ref().map_or(0, |block| block.number),
3265                0_u8,
3266                0,
3267                0,
3268            ),
3269            chain_event::Event::Log(log) => {
3270                (log.block_number, 1_u8, log.transaction_index, log.log_index)
3271            }
3272            chain_event::Event::BlockProgress(progress) => (
3273                progress.block.as_ref().map_or(0, |block| block.number),
3274                2_u8,
3275                0,
3276                0,
3277            ),
3278        };
3279        if last_event_order.is_some_and(|previous| event_order < previous) {
3280            return Err(RemoteDecodeError::EventOrder);
3281        }
3282        last_event_order = Some(event_order);
3283        match event {
3284            chain_event::Event::BlockHeader(header) => {
3285                let wire = header
3286                    .block
3287                    .ok_or(RemoteDecodeError::MissingField("block_header.block"))?;
3288                let block = runtime_block_ref(&wire)?;
3289                if !explicit_headers.insert(block.number) {
3290                    return Err(RemoteDecodeError::DuplicateBlock(block.number));
3291                }
3292                let hash = block.hash;
3293                if header.consensus_header_rlp.is_empty() {
3294                    return Err(RemoteDecodeError::MissingHeaderRlp);
3295                }
3296                let mut encoded = header.consensus_header_rlp.as_slice();
3297                let inner = ConsensusHeader::decode(&mut encoded)
3298                    .map_err(|error| RemoteDecodeError::HeaderRlp(error.to_string()))?;
3299                if !encoded.is_empty() {
3300                    return Err(RemoteDecodeError::HeaderRlp(
3301                        "trailing bytes after consensus header".into(),
3302                    ));
3303                }
3304                if inner.hash_slow() != hash {
3305                    return Err(RemoteDecodeError::HeaderHash);
3306                }
3307                if inner.number != block.number
3308                    || Some(inner.parent_hash) != block.parent_hash
3309                    || Some(inner.timestamp) != block.timestamp
3310                {
3311                    return Err(RemoteDecodeError::HeaderMetadata);
3312                }
3313                let rpc_header = RpcHeader {
3314                    hash,
3315                    inner,
3316                    total_difficulty: decode_u256(
3317                        &header.total_difficulty,
3318                        "block_header.total_difficulty",
3319                    )?,
3320                    size: decode_u256(&header.size, "block_header.size")?,
3321                };
3322                if let Some(known) = blocks.insert(block.number, block)
3323                    && known != block
3324                {
3325                    return Err(RemoteDecodeError::BlockHashMismatch(block.number));
3326                }
3327                records.push(ReactiveInputDelivery::new(
3328                    ReactiveInputRecord::new(
3329                        ReactiveInput::BlockHeader(rpc_header),
3330                        included_context(expected_chain_id, block, None, None, input_source(scope)),
3331                    ),
3332                    audience,
3333                    scope,
3334                ));
3335            }
3336            chain_event::Event::Log(log) => {
3337                let block_hash = decode_b256(&log.block_hash, "log.block_hash")?;
3338                let transaction_hash = decode_b256(&log.transaction_hash, "log.transaction_hash")?;
3339                if !log_identities.insert((block_hash, log.log_index)) {
3340                    return Err(RemoteDecodeError::DuplicateLog(log.log_index));
3341                }
3342                if transaction_indexes
3343                    .insert((block_hash, transaction_hash), log.transaction_index)
3344                    .is_some_and(|known| known != log.transaction_index)
3345                    || transaction_hashes
3346                        .insert((block_hash, log.transaction_index), transaction_hash)
3347                        .is_some_and(|known| known != transaction_hash)
3348                {
3349                    return Err(RemoteDecodeError::TransactionIdentity);
3350                }
3351                let block = blocks
3352                    .get(&log.block_number)
3353                    .copied()
3354                    .unwrap_or(RuntimeBlockRef {
3355                        number: log.block_number,
3356                        hash: block_hash,
3357                        parent_hash: None,
3358                        timestamp: Some(log.block_timestamp),
3359                    });
3360                if block.hash != block_hash || block.timestamp != Some(log.block_timestamp) {
3361                    return Err(RemoteDecodeError::BlockHashMismatch(log.block_number));
3362                }
3363                blocks.entry(log.block_number).or_insert(block);
3364                let rpc_log = decode_log(&log)?;
3365                let context = if log.removed {
3366                    ReactiveContext {
3367                        chain_id: Some(expected_chain_id),
3368                        source: input_source(scope),
3369                        chain_status: ChainStatus::Reorged {
3370                            dropped_from: block,
3371                        },
3372                        block: Some(block),
3373                        transaction_index: Some(log.transaction_index),
3374                        log_index: Some(log.log_index),
3375                    }
3376                } else {
3377                    included_context(
3378                        expected_chain_id,
3379                        block,
3380                        Some(log.transaction_index),
3381                        Some(log.log_index),
3382                        input_source(scope),
3383                    )
3384                };
3385                records.push(ReactiveInputDelivery::new(
3386                    ReactiveInputRecord::new(ReactiveInput::Log(rpc_log), context),
3387                    audience,
3388                    scope,
3389                ));
3390            }
3391            chain_event::Event::BlockProgress(progress) => {
3392                if scope == RuntimeDeliveryScope::OwnerCatchup {
3393                    return Err(RemoteDecodeError::OwnerProgress);
3394                }
3395                let wire = progress
3396                    .block
3397                    .ok_or(RemoteDecodeError::MissingField("block_progress.block"))?;
3398                let block = runtime_block_ref(&wire)?;
3399                if !explicit_progress.insert(block.number) {
3400                    return Err(RemoteDecodeError::DuplicateBlock(block.number));
3401                }
3402                if let Some(known) = blocks.insert(block.number, block)
3403                    && known != block
3404                {
3405                    return Err(RemoteDecodeError::BlockHashMismatch(block.number));
3406                }
3407                // Runtime chain controls are applied after every record in the
3408                // envelope. Retaining an earlier page-level progress proof
3409                // would therefore regress behind a later canonical record.
3410                // Every proof is still decoded and identity-checked above;
3411                // only the highest authenticated coverage boundary is needed
3412                // to certify the complete ordered page.
3413                canonical_progress = Some(block);
3414            }
3415        }
3416    }
3417    Ok(
3418        ReactiveInputBatch::from_deliveries(records).with_chain_controls(
3419            canonical_progress
3420                .into_iter()
3421                .map(ChainControl::CanonicalProgress),
3422        ),
3423    )
3424}
3425
3426fn decode_audience(
3427    canonical: bool,
3428    owner_ids: Vec<String>,
3429    authoritative_owners: &HashSet<&str>,
3430) -> Result<DeliveryAudience, RemoteDecodeError> {
3431    if canonical {
3432        if !owner_ids.is_empty() {
3433            return Err(RemoteDecodeError::Audience);
3434        }
3435        return Ok(DeliveryAudience::All);
3436    }
3437    if owner_ids.is_empty() {
3438        return Err(RemoteDecodeError::Audience);
3439    }
3440    let mut seen = HashSet::with_capacity(owner_ids.len());
3441    let mut owners = Vec::with_capacity(owner_ids.len());
3442    for owner in owner_ids {
3443        if owner.is_empty()
3444            || !authoritative_owners.contains(owner.as_str())
3445            || !seen.insert(owner.clone())
3446        {
3447            return Err(RemoteDecodeError::Audience);
3448        }
3449        owners.push(HandlerId::try_new(owner).map_err(|_| RemoteDecodeError::Audience)?);
3450    }
3451    Ok(DeliveryAudience::Owners(owners))
3452}
3453
3454fn decode_delivery_scope(scope: i32) -> Result<RuntimeDeliveryScope, RemoteDecodeError> {
3455    match WireDeliveryScope::try_from(scope) {
3456        Ok(WireDeliveryScope::Canonical) => Ok(RuntimeDeliveryScope::Canonical),
3457        Ok(WireDeliveryScope::CanonicalProgress) => Ok(RuntimeDeliveryScope::CanonicalProgress),
3458        Ok(WireDeliveryScope::OwnerCatchup) => Ok(RuntimeDeliveryScope::OwnerCatchup),
3459        Ok(WireDeliveryScope::Unspecified) | Err(_) => Err(RemoteDecodeError::DeliveryScope(scope)),
3460    }
3461}
3462
3463const fn input_source(scope: RuntimeDeliveryScope) -> InputSource {
3464    match scope {
3465        RuntimeDeliveryScope::Canonical => InputSource::Subscription,
3466        RuntimeDeliveryScope::CanonicalProgress | RuntimeDeliveryScope::OwnerCatchup => {
3467            InputSource::Backfill
3468        }
3469        _ => InputSource::Subscription,
3470    }
3471}
3472
3473fn decode_log(log: &WireLog) -> Result<RpcLog, RemoteDecodeError> {
3474    if log.topics.len() > 4 {
3475        return Err(RemoteDecodeError::TopicCount(log.topics.len()));
3476    }
3477    let address = decode_address(&log.address, "log.address")?;
3478    let topics = log
3479        .topics
3480        .iter()
3481        .map(|topic| decode_b256(topic, "log.topic"))
3482        .collect::<Result<Vec<_>, _>>()?;
3483    Ok(RpcLog {
3484        inner: PrimitiveLog::new_unchecked(address, topics, Bytes::from(log.data.clone())),
3485        block_hash: Some(decode_b256(&log.block_hash, "log.block_hash")?),
3486        block_number: Some(log.block_number),
3487        block_timestamp: Some(log.block_timestamp),
3488        transaction_hash: Some(decode_b256(&log.transaction_hash, "log.transaction_hash")?),
3489        transaction_index: Some(log.transaction_index),
3490        log_index: Some(log.log_index),
3491        removed: log.removed,
3492    })
3493}
3494
3495fn runtime_block_ref(
3496    block: &evm_fork_cache_event_protocol::v1::BlockRef,
3497) -> Result<RuntimeBlockRef, RemoteDecodeError> {
3498    Ok(RuntimeBlockRef {
3499        number: block.number,
3500        hash: decode_b256(&block.hash, "block.hash")?,
3501        parent_hash: Some(decode_b256(&block.parent_hash, "block.parent_hash")?),
3502        timestamp: Some(block.timestamp),
3503    })
3504}
3505
3506fn included_context(
3507    chain_id: u64,
3508    block: RuntimeBlockRef,
3509    transaction_index: Option<u64>,
3510    log_index: Option<u64>,
3511    source: InputSource,
3512) -> ReactiveContext {
3513    ReactiveContext {
3514        chain_id: Some(chain_id),
3515        source,
3516        chain_status: ChainStatus::Included {
3517            block,
3518            confirmations: 0,
3519        },
3520        block: Some(block),
3521        transaction_index,
3522        log_index,
3523    }
3524}
3525
3526fn decode_address(bytes: &[u8], field: &'static str) -> Result<Address, RemoteDecodeError> {
3527    if bytes.len() != 20 {
3528        return Err(RemoteDecodeError::Width {
3529            field,
3530            expected: 20,
3531            received: bytes.len(),
3532        });
3533    }
3534    Ok(Address::from_slice(bytes))
3535}
3536
3537fn decode_b256(bytes: &[u8], field: &'static str) -> Result<B256, RemoteDecodeError> {
3538    if bytes.len() != 32 {
3539        return Err(RemoteDecodeError::Width {
3540            field,
3541            expected: 32,
3542            received: bytes.len(),
3543        });
3544    }
3545    Ok(B256::from_slice(bytes))
3546}
3547
3548fn decode_u256(bytes: &[u8], field: &'static str) -> Result<Option<U256>, RemoteDecodeError> {
3549    if bytes.len() > 32 {
3550        return Err(RemoteDecodeError::QuantityWidth {
3551            field,
3552            received: bytes.len(),
3553        });
3554    }
3555    Ok((!bytes.is_empty()).then(|| U256::from_be_slice(bytes)))
3556}
3557
3558#[derive(Debug, thiserror::Error)]
3559enum RemoteDecodeError {
3560    #[error("remote data delivery contains no records")]
3561    EmptyData,
3562    #[error("remote barrier identifier is empty")]
3563    EmptyBarrierId,
3564    #[error("remote delivery is missing `{0}`")]
3565    MissingField(&'static str),
3566    #[error("remote delivery session mismatch: expected `{expected}`, received `{received}`")]
3567    Session { expected: String, received: String },
3568    #[error("remote delivery chain mismatch: expected {expected}, received {received}")]
3569    Chain { expected: u64, received: u64 },
3570    #[error("remote delivery token does not encode its sequence")]
3571    DeliveryToken,
3572    #[error("remote delivery cursor does not match its query revision and sequence")]
3573    Cursor,
3574    #[error("remote delivery cursor authority is invalid: {0}")]
3575    CursorAuthority(&'static str),
3576    #[error("remote finality kind {0} is invalid")]
3577    Finality(i32),
3578    #[error("`{field}` must be {expected} bytes, got {received}")]
3579    Width {
3580        field: &'static str,
3581        expected: usize,
3582        received: usize,
3583    },
3584    #[error("`{field}` must be at most 32 bytes, got {received}")]
3585    QuantityWidth {
3586        field: &'static str,
3587        received: usize,
3588    },
3589    #[error("invalid consensus header RLP: {0}")]
3590    HeaderRlp(String),
3591    #[error("full block-header delivery is missing consensus header RLP")]
3592    MissingHeaderRlp,
3593    #[error("consensus header RLP hash does not match its block reference")]
3594    HeaderHash,
3595    #[error("consensus header number, parent, or timestamp does not match its block reference")]
3596    HeaderMetadata,
3597    #[error("log block hash does not match block event at height {0}")]
3598    BlockHashMismatch(u64),
3599    #[error("block {0} appears more than once as a header/progress record")]
3600    DuplicateBlock(u64),
3601    #[error("wire delivery audience is empty, duplicated, or contradictory")]
3602    Audience,
3603    #[error("wire delivery scope {0} is unspecified or unknown")]
3604    DeliveryScope(i32),
3605    #[error("owner catch-up cannot emit canonical block progress")]
3606    OwnerProgress,
3607    #[error("owner catch-up must target named owners rather than canonical broadcast")]
3608    OwnerCatchupAudience,
3609    #[error("EVM logs may carry at most four topics, got {0}")]
3610    TopicCount(usize),
3611    #[error("log identity with index {0} appears more than once in one delivery")]
3612    DuplicateLog(u64),
3613    #[error("remote data records are not in canonical event order")]
3614    EventOrder,
3615    #[error("remote logs disagree on transaction hash/index identity within a block")]
3616    TransactionIdentity,
3617}