Skip to main content

evm_fork_cache_hypersync/
managed_provider.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    future::Future,
4    sync::{
5        Arc,
6        atomic::{AtomicU64, Ordering},
7    },
8    time::Duration,
9};
10
11use async_trait::async_trait;
12use evm_fork_cache_event_protocol::v1::{
13    Acknowledge, ApplyDesiredState, Capability, Cursor, Delivery, SourceCapabilities,
14    SourceDescriptor, SourceRole, portable_interest,
15};
16use evm_fork_cache_event_service::{DeliveryRequest, EventSource, EventSourceError, PreparationId};
17use futures::StreamExt;
18use hypersync_client::net_types::{BlockField, Query};
19use tokio::{
20    sync::{Mutex, Notify},
21    task::AbortHandle,
22};
23
24use crate::{
25    ChainDataSource, ChainHeightStream, HyperSyncDataSource, MAX_DELIVERY_SIZE_BYTES,
26    QueryPlanError, SourceEngine, SourceEngineError, SourceError, SourceResponseLimits,
27    SourceResume,
28    normalize::{DecodedSourceCheckpoint, block_ref, decode_source_checkpoint},
29    page_validation::validate_source_page,
30    source_engine::validate_response_counts,
31};
32
33const SOURCE_REQUEST_ATTEMPTS: u32 = 3;
34const MAX_SOURCE_REQUEST_ATTEMPT: Duration = Duration::from_secs(8);
35
36/// Factory for independently configured per-chain data sources.
37pub trait ChainDataSourceFactory: Send + Sync + 'static {
38    /// Source implementation produced for each managed session.
39    type Source: ChainDataSource + 'static;
40
41    /// Create a source for one EVM chain.
42    ///
43    /// # Errors
44    ///
45    /// Returns [`SourceError`] when the chain is unsupported or its
46    /// provider/client configuration cannot be constructed.
47    fn create(&self, chain_id: u64) -> Result<Self::Source, SourceError>;
48
49    /// Whether this factory can reconstruct and hash-verify complete headers on a chain.
50    fn supports_full_headers(&self, _chain_id: u64) -> bool {
51        false
52    }
53
54    /// Whether any configured chain supports complete hash-verified headers.
55    fn supports_full_headers_on_any_chain(&self) -> bool {
56        false
57    }
58}
59
60/// HyperSync source factory using one Envio API token across configured chains.
61#[derive(Clone)]
62pub struct HyperSyncSourceFactory {
63    api_token: String,
64    full_header_chains: HashSet<u64>,
65}
66
67impl HyperSyncSourceFactory {
68    /// Store the API token used when constructing per-chain clients.
69    pub fn new(api_token: impl Into<String>) -> Self {
70        Self {
71            api_token: api_token.into(),
72            full_header_chains: HashSet::new(),
73        }
74    }
75
76    /// Opt a chain into full headers after independently proving the provider schema is complete.
77    ///
78    /// The current HyperSync schema omits `requests_hash`, so this must not be
79    /// enabled for post-Prague Ethereum blocks.
80    pub fn with_hash_verified_headers_for_chain(mut self, chain_id: u64) -> Self {
81        self.full_header_chains.insert(chain_id);
82        self
83    }
84}
85
86impl ChainDataSourceFactory for HyperSyncSourceFactory {
87    type Source = HyperSyncDataSource;
88
89    fn create(&self, chain_id: u64) -> Result<Self::Source, SourceError> {
90        HyperSyncDataSource::new(chain_id, &self.api_token)
91    }
92
93    fn supports_full_headers(&self, chain_id: u64) -> bool {
94        self.full_header_chains.contains(&chain_id)
95    }
96
97    fn supports_full_headers_on_any_chain(&self) -> bool {
98        !self.full_header_chains.is_empty()
99    }
100}
101
102struct ManagedEngine<S> {
103    revision: u64,
104    engine: Arc<Mutex<SourceEngine<S>>>,
105    height_driver: Option<Arc<HeightDriver>>,
106}
107
108type SessionKey = (String, u64);
109type EngineMap<S> = HashMap<SessionKey, ManagedEngine<S>>;
110type PreparedEngineMap<S> = HashMap<(SessionKey, PreparationId), ManagedEngine<S>>;
111
112struct ProviderState<S> {
113    engines: EngineMap<S>,
114    prepared: PreparedEngineMap<S>,
115}
116
117impl<S> Default for ProviderState<S> {
118    fn default() -> Self {
119        Self {
120            engines: HashMap::new(),
121            prepared: HashMap::new(),
122        }
123    }
124}
125
126struct HeightDriver {
127    latest_hint: Arc<AtomicU64>,
128    update: Arc<Notify>,
129    abort: AbortHandle,
130}
131
132impl HeightDriver {
133    fn spawn(mut updates: ChainHeightStream) -> Arc<Self> {
134        let latest_hint = Arc::new(AtomicU64::new(0));
135        let update = Arc::new(Notify::new());
136        let task_latest = Arc::clone(&latest_hint);
137        let task_update = Arc::clone(&update);
138        let task = tokio::spawn(async move {
139            while let Some(height) = updates.next().await {
140                task_latest.store(height, Ordering::Release);
141                task_update.notify_one();
142            }
143        });
144        let abort = task.abort_handle();
145        drop(task);
146        Arc::new(Self {
147            latest_hint,
148            update,
149            abort,
150        })
151    }
152}
153
154impl Drop for HeightDriver {
155    fn drop(&mut self) {
156        self.abort.abort();
157    }
158}
159
160impl<S> Clone for ManagedEngine<S> {
161    fn clone(&self) -> Self {
162        Self {
163            revision: self.revision,
164            engine: Arc::clone(&self.engine),
165            height_driver: self.height_driver.as_ref().map(Arc::clone),
166        }
167    }
168}
169
170/// Maintains one in-memory source engine per durable session while SQLite
171/// remains authoritative across process restarts.
172pub struct ManagedEventProvider<F>
173where
174    F: ChainDataSourceFactory,
175{
176    factory: F,
177    reorg_depth: usize,
178    request_timeout: Duration,
179    max_delivery_bytes: usize,
180    response_limits: SourceResponseLimits,
181    max_resident_sessions: usize,
182    state: Mutex<ProviderState<F::Source>>,
183}
184
185impl<F> ManagedEventProvider<F>
186where
187    F: ChainDataSourceFactory,
188{
189    /// Create a provider retaining `reorg_depth` canonical blocks per session.
190    ///
191    /// # Panics
192    ///
193    /// Panics when `reorg_depth` is zero. Configuration-facing callers should
194    /// prefer [`Self::try_new`] to report that invalid boundary without panic.
195    pub fn new(factory: F, reorg_depth: usize) -> Self {
196        Self::try_new(factory, reorg_depth)
197            .expect("managed source reorg depth must be greater than zero")
198    }
199
200    /// Fallibly create a provider with a nonzero retained canonical history.
201    ///
202    /// # Errors
203    ///
204    /// Returns [`SourceError::InvalidReorgDepth`] when `reorg_depth` is zero.
205    pub fn try_new(factory: F, reorg_depth: usize) -> Result<Self, SourceError> {
206        if reorg_depth == 0 {
207            return Err(SourceError::InvalidReorgDepth);
208        }
209        Ok(Self {
210            factory,
211            reorg_depth,
212            request_timeout: Duration::from_secs(45),
213            max_delivery_bytes: MAX_DELIVERY_SIZE_BYTES,
214            response_limits: SourceResponseLimits::default(),
215            max_resident_sessions: 4_096,
216            state: Mutex::new(ProviderState::default()),
217        })
218    }
219
220    /// Bound each provider height/query operation, including its internal retries.
221    ///
222    /// # Errors
223    ///
224    /// Returns [`SourceError::InvalidRequestTimeout`] when the timeout is zero.
225    pub fn with_request_timeout(mut self, request_timeout: Duration) -> Result<Self, SourceError> {
226        if request_timeout.is_zero() {
227            return Err(SourceError::InvalidRequestTimeout);
228        }
229        self.request_timeout = request_timeout;
230        Ok(self)
231    }
232
233    /// Bound the encoded source delivery before the service adds its outer envelope.
234    ///
235    /// # Errors
236    ///
237    /// Returns [`SourceError::InvalidDeliverySizeLimit`] when the limit is zero
238    /// or exceeds [`MAX_DELIVERY_SIZE_BYTES`].
239    pub fn with_max_delivery_bytes(
240        mut self,
241        max_delivery_bytes: usize,
242    ) -> Result<Self, SourceError> {
243        if max_delivery_bytes == 0 || max_delivery_bytes > MAX_DELIVERY_SIZE_BYTES {
244            return Err(SourceError::InvalidDeliverySizeLimit {
245                requested: max_delivery_bytes,
246                maximum: MAX_DELIVERY_SIZE_BYTES,
247            });
248        }
249        self.max_delivery_bytes = max_delivery_bytes;
250        Ok(self)
251    }
252
253    /// Override hard local decoded-response limits independently of provider soft targets.
254    pub fn with_response_limits(mut self, response_limits: SourceResponseLimits) -> Self {
255        self.response_limits = response_limits;
256        self
257    }
258
259    /// Bound the number of distinct session/chain engines retained in memory.
260    ///
261    /// # Errors
262    ///
263    /// Returns [`SourceError::InvalidSessionLimit`] when the limit is zero.
264    pub fn with_max_resident_sessions(
265        mut self,
266        max_resident_sessions: usize,
267    ) -> Result<Self, SourceError> {
268        if max_resident_sessions == 0 {
269            return Err(SourceError::InvalidSessionLimit);
270        }
271        self.max_resident_sessions = max_resident_sessions;
272        Ok(self)
273    }
274
275    async fn build_engine(
276        &self,
277        desired_state: &ApplyDesiredState,
278        acknowledged_cursor: Option<&Cursor>,
279    ) -> Result<ManagedEngine<F::Source>, EventSourceError> {
280        if requests_full_headers(desired_state)
281            && !self.factory.supports_full_headers(desired_state.chain_id)
282        {
283            return Err(EventSourceError::unsupported(format!(
284                "complete hash-verified block headers are unavailable for chain {}",
285                desired_state.chain_id
286            )));
287        }
288        let source = self
289            .factory
290            .create(desired_state.chain_id)
291            .map_err(source_error)?;
292        let backfill_start = desired_state
293            .owners
294            .iter()
295            .filter_map(|owner| owner.backfill.as_ref().map(|backfill| backfill.from_block))
296            .min();
297        let global_backfill = desired_state
298            .owners
299            .iter()
300            .find(|owner| owner.canonical)
301            .and_then(|owner| owner.backfill.as_ref());
302        let (start_block, computed_activation_block) = match acknowledged_cursor {
303            Some(cursor) if cursor.query_revision < desired_state.new_revision => (
304                backfill_start.map_or(cursor.next_block, |from| from.min(cursor.next_block)),
305                cursor.next_block,
306            ),
307            Some(cursor) => (cursor.next_block, cursor.next_block),
308            None => match backfill_start {
309                Some(from_block) => {
310                    let head = available_height(&source, self.request_timeout)
311                        .await
312                        .map_err(source_error)?;
313                    (from_block, head)
314                }
315                None => {
316                    let head = available_height(&source, self.request_timeout)
317                        .await
318                        .map_err(source_error)?;
319                    (head, head)
320                }
321            },
322        };
323        let committed_sequence = acknowledged_cursor.map_or(0, |cursor| cursor.batch_sequence);
324        let decoded_checkpoint = acknowledged_cursor
325            .map(checkpoints_from_cursor)
326            .transpose()?
327            .unwrap_or_default();
328        let checkpoint =
329            if acknowledged_cursor.is_some_and(|cursor| cursor.next_block == start_block) {
330                decoded_checkpoint.clone()
331            } else {
332                DecodedSourceCheckpoint::default()
333            };
334        verify_retained_backfill_baselines(
335            &source,
336            desired_state,
337            &checkpoint.canonical_blocks,
338            self.request_timeout,
339            self.response_limits,
340        )
341        .await?;
342        let global_baseline =
343            global_backfill.and_then(|backfill| backfill.retained_baseline.clone());
344        if let (Some(acknowledged), Some(baseline)) = (
345            acknowledged_cursor.and_then(|cursor| cursor.canonical_head.as_ref()),
346            global_baseline.as_ref(),
347        ) && acknowledged != baseline
348        {
349            return Err(EventSourceError::invalid(
350                "global backfill baseline does not match acknowledged canonical coverage",
351            ));
352        }
353        let mut canonical_blocks = checkpoint.canonical_blocks;
354        if canonical_blocks.is_empty()
355            && let Some(baseline) = global_baseline.clone()
356        {
357            canonical_blocks.push(baseline);
358        }
359        let activation_block = acknowledged_cursor
360            .filter(|cursor| cursor.query_revision == desired_state.new_revision)
361            .and(decoded_checkpoint.activation_block)
362            .unwrap_or(computed_activation_block);
363        let owner_backfill_activation_block = acknowledged_cursor
364            .filter(|cursor| cursor.query_revision == desired_state.new_revision)
365            .map_or(Some(computed_activation_block), |cursor| {
366                cursor.owner_backfill_activation_block
367            });
368        let height_driver = source.height_stream().map(HeightDriver::spawn);
369        let engine = SourceEngine::restore(
370            source,
371            desired_state.clone(),
372            SourceResume {
373                next_block: start_block,
374                sequence: committed_sequence,
375                activation_block,
376                owner_backfill_activation_block,
377                canonical_blocks,
378                provider_checkpoint: checkpoint.rollback_guard,
379                coverage_head: acknowledged_cursor
380                    .and_then(|cursor| cursor.canonical_head.clone())
381                    .or(global_baseline),
382            },
383            self.reorg_depth,
384        )
385        .map_err(|error| EventSourceError::unavailable(error.to_string()))?
386        .with_max_delivery_bytes(self.max_delivery_bytes)
387        .map_err(source_engine_error)?
388        .with_response_limits(self.response_limits);
389        let managed = ManagedEngine {
390            revision: desired_state.new_revision,
391            engine: Arc::new(Mutex::new(engine)),
392            height_driver,
393        };
394        Ok(managed)
395    }
396
397    async fn engine(
398        &self,
399        desired_state: &ApplyDesiredState,
400        acknowledged_cursor: Option<&Cursor>,
401    ) -> Result<ManagedEngine<F::Source>, EventSourceError> {
402        let key = (desired_state.session_id.clone(), desired_state.chain_id);
403        if let Some(existing) = self.state.lock().await.engines.get(&key).cloned()
404            && existing.revision == desired_state.new_revision
405        {
406            return Ok(existing);
407        }
408        let managed = self
409            .build_engine(desired_state, acknowledged_cursor)
410            .await?;
411        let mut state = self.state.lock().await;
412        if let Some(existing) = state.engines.get(&key).cloned()
413            && existing.revision == desired_state.new_revision
414        {
415            return Ok(existing);
416        }
417        ensure_session_capacity(&state, &key, self.max_resident_sessions)?;
418        state.engines.insert(key, managed.clone());
419        Ok(managed)
420    }
421}
422
423#[async_trait]
424impl<F> EventSource for ManagedEventProvider<F>
425where
426    F: ChainDataSourceFactory,
427{
428    fn capabilities(&self) -> SourceCapabilities {
429        self.source_capabilities(self.factory.supports_full_headers_on_any_chain())
430    }
431
432    fn capabilities_for_chain(&self, chain_id: u64) -> SourceCapabilities {
433        self.source_capabilities(self.factory.supports_full_headers(chain_id))
434    }
435
436    async fn prepare_desired_state(
437        &self,
438        preparation_id: PreparationId,
439        desired_state: &ApplyDesiredState,
440        acknowledged_cursor: Option<&Cursor>,
441    ) -> Result<Option<Cursor>, EventSourceError> {
442        let key = (desired_state.session_id.clone(), desired_state.chain_id);
443        {
444            let state = self.state.lock().await;
445            ensure_session_capacity(&state, &key, self.max_resident_sessions)?;
446        }
447        let managed = self
448            .build_engine(desired_state, acknowledged_cursor)
449            .await?;
450        let activation_cursor = managed
451            .engine
452            .lock()
453            .await
454            .activation_cursor()
455            .map_err(source_engine_error)?;
456        let mut state = self.state.lock().await;
457        ensure_session_capacity(&state, &key, self.max_resident_sessions)?;
458        state
459            .prepared
460            .retain(|(prepared_key, _), _| prepared_key != &key);
461        state.prepared.insert((key, preparation_id), managed);
462        Ok(Some(activation_cursor))
463    }
464
465    async fn activate_desired_state(
466        &self,
467        preparation_id: PreparationId,
468        desired_state: &ApplyDesiredState,
469        _acknowledged_cursor: Option<&Cursor>,
470    ) -> Result<(), EventSourceError> {
471        let key = (desired_state.session_id.clone(), desired_state.chain_id);
472        let mut state = self.state.lock().await;
473        let managed = state
474            .prepared
475            .remove(&(key.clone(), preparation_id))
476            .ok_or_else(|| EventSourceError::internal("prepared source candidate is missing"))?;
477        if managed.revision != desired_state.new_revision {
478            return Err(EventSourceError::internal(
479                "prepared source candidate revision does not match activation",
480            ));
481        }
482        state.engines.insert(key, managed);
483        Ok(())
484    }
485
486    async fn abort_desired_state(
487        &self,
488        preparation_id: PreparationId,
489        desired_state: &ApplyDesiredState,
490    ) -> Result<(), EventSourceError> {
491        let key = (desired_state.session_id.clone(), desired_state.chain_id);
492        self.state
493            .lock()
494            .await
495            .prepared
496            .remove(&(key, preparation_id));
497        Ok(())
498    }
499
500    async fn release_session(
501        &self,
502        session_id: &str,
503        chain_id: u64,
504    ) -> Result<(), EventSourceError> {
505        let key = (session_id.to_owned(), chain_id);
506        let mut state = self.state.lock().await;
507        state.engines.remove(&key);
508        state
509            .prepared
510            .retain(|(prepared_key, _), _| prepared_key != &key);
511        Ok(())
512    }
513
514    async fn next_delivery(
515        &self,
516        request: DeliveryRequest<'_>,
517    ) -> Result<Option<Delivery>, EventSourceError> {
518        let desired_state = request.desired_state();
519        let acknowledged_cursor = request.acknowledged_cursor();
520        let required_target = required_reorg_target(request)?;
521        let managed = self.engine(desired_state, acknowledged_cursor).await?;
522        let mut engine = managed.engine.lock().await;
523        // SSE archive heights are wakeup hints only: reconnects, provider
524        // corrections, and cross-replica lag can move them in either direction.
525        // Reconcile every query target against the authoritative REST height.
526        let archive_height = available_height(engine.source(), self.request_timeout)
527            .await
528            .map_err(source_error)?;
529        if let Some(driver) = managed.height_driver.as_ref() {
530            driver.latest_hint.store(archive_height, Ordering::Release);
531        }
532        if archive_height <= engine.committed_next_block() && required_target.is_none() {
533            return Ok(None);
534        }
535        let target = match required_target {
536            Some(target) if archive_height < target => return Ok(None),
537            Some(target) => target,
538            None => archive_height,
539        };
540        tokio::time::timeout(self.request_timeout, engine.next_batch(target))
541            .await
542            .map_err(|_| {
543                source_error(SourceError::RequestTimeout {
544                    millis: self.request_timeout.as_millis(),
545                })
546            })?
547            .map_err(source_engine_error)
548    }
549
550    async fn wait_for_update(&self, request: DeliveryRequest<'_>) -> Result<(), EventSourceError> {
551        let desired_state = request.desired_state();
552        let acknowledged_cursor = request.acknowledged_cursor();
553        let required_target = required_reorg_target(request)?;
554        let managed = self.engine(desired_state, acknowledged_cursor).await?;
555        let Some(driver) = managed.height_driver.as_ref() else {
556            return std::future::pending().await;
557        };
558        let committed_next_block = managed.engine.lock().await.committed_next_block();
559        loop {
560            let latest_hint = driver.latest_hint.load(Ordering::Acquire);
561            let ready = required_target.map_or(latest_hint > committed_next_block, |target| {
562                latest_hint >= target
563            });
564            if ready {
565                return Ok(());
566            }
567            driver.update.notified().await;
568        }
569    }
570
571    async fn acknowledge(
572        &self,
573        chain_id: u64,
574        acknowledgement: &Acknowledge,
575        committed_cursor: &Cursor,
576    ) -> Result<(), EventSourceError> {
577        let key = (acknowledgement.session_id.clone(), chain_id);
578        let managed = self.state.lock().await.engines.get(&key).cloned();
579        let Some(managed) = managed else {
580            // SQLite remains authoritative. A replayed pending batch can be
581            // acknowledged before its in-memory engine is restored.
582            return Ok(());
583        };
584        let mut engine = managed.engine.lock().await;
585        if acknowledgement.sequence <= engine.committed_sequence() {
586            return Ok(());
587        }
588        match engine.acknowledge(&acknowledgement.delivery_token) {
589            Ok(()) => Ok(()),
590            Err(SourceEngineError::NoPendingDelivery)
591                if committed_cursor.query_revision == managed.revision =>
592            {
593                let checkpoint = checkpoints_from_cursor(committed_cursor)?;
594                let activation_block = checkpoint
595                    .activation_block
596                    .unwrap_or_else(|| engine.activation_block());
597                engine
598                    .synchronize_committed_cursor(SourceResume {
599                        next_block: committed_cursor.next_block,
600                        sequence: acknowledgement.sequence,
601                        activation_block,
602                        owner_backfill_activation_block: committed_cursor
603                            .owner_backfill_activation_block,
604                        canonical_blocks: checkpoint.canonical_blocks,
605                        provider_checkpoint: checkpoint.rollback_guard,
606                        coverage_head: committed_cursor.canonical_head.clone(),
607                    })
608                    .map_err(source_engine_error)
609            }
610            Err(error) => Err(source_engine_error(error)),
611        }
612    }
613}
614
615impl<F> ManagedEventProvider<F>
616where
617    F: ChainDataSourceFactory,
618{
619    fn source_capabilities(&self, supports_full_headers: bool) -> SourceCapabilities {
620        let mut capabilities = vec![
621            Capability::Historical.into(),
622            Capability::Logs.into(),
623            Capability::ServerFiltering.into(),
624            Capability::DynamicFilters.into(),
625            Capability::ExplicitReorgs.into(),
626            Capability::NativeCheckpoint.into(),
627            Capability::OwnerScopedDelivery.into(),
628            Capability::DurableReplay.into(),
629        ];
630        if supports_full_headers {
631            capabilities.push(Capability::Headers.into());
632        }
633        SourceCapabilities {
634            capabilities: capabilities.clone(),
635            sources: vec![SourceDescriptor {
636                source_id: "hypersync".into(),
637                role: SourceRole::Historical.into(),
638                capabilities,
639            }],
640        }
641    }
642}
643
644fn checkpoints_from_cursor(cursor: &Cursor) -> Result<DecodedSourceCheckpoint, EventSourceError> {
645    let mut checkpoint = decode_source_checkpoint(&cursor.provider_checkpoint)
646        .map_err(|error| EventSourceError::invalid(error.to_string()))?;
647    if checkpoint.activation_block.is_some()
648        && cursor.owner_backfill_activation_block.is_some()
649        && checkpoint.activation_block != cursor.owner_backfill_activation_block
650    {
651        return Err(EventSourceError::invalid(
652            "portable owner backfill activation boundary conflicts with provider checkpoint",
653        ));
654    }
655    if checkpoint.activation_block.is_none() {
656        checkpoint.activation_block = cursor.owner_backfill_activation_block;
657    }
658    if checkpoint.canonical_blocks.is_empty()
659        && let Some(head) = cursor.canonical_head.clone()
660        && head.number.checked_add(1) == Some(cursor.next_block)
661    {
662        checkpoint.canonical_blocks.push(head);
663    }
664    Ok(checkpoint)
665}
666
667fn requests_full_headers(desired_state: &ApplyDesiredState) -> bool {
668    desired_state.owners.iter().any(|owner| {
669        owner.interests.iter().any(|interest| {
670            matches!(
671                interest.kind.as_ref(),
672                Some(portable_interest::Kind::Block(_))
673            )
674        })
675    })
676}
677
678async fn verify_retained_backfill_baselines<S: ChainDataSource>(
679    source: &S,
680    desired_state: &ApplyDesiredState,
681    retained_history: &[evm_fork_cache_event_protocol::v1::BlockRef],
682    request_timeout: Duration,
683    response_limits: SourceResponseLimits,
684) -> Result<(), EventSourceError> {
685    let mut baselines = BTreeMap::new();
686    for baseline in desired_state
687        .owners
688        .iter()
689        .filter_map(|owner| owner.backfill.as_ref())
690        .filter_map(|backfill| backfill.retained_baseline.as_ref())
691    {
692        if let Some(existing) = baselines.insert(baseline.number, baseline)
693            && existing != baseline
694        {
695            return Err(EventSourceError::invalid(
696                "retained backfill baselines conflict at the same block height",
697            ));
698        }
699    }
700    for baseline in baselines.into_values() {
701        if baseline.hash.len() != 32 || baseline.parent_hash.len() != 32 {
702            return Err(EventSourceError::invalid(
703                "retained backfill baseline hashes must be 32 bytes",
704            ));
705        }
706        if let Some(retained) = retained_history
707            .iter()
708            .find(|retained| retained.number == baseline.number)
709        {
710            if retained != baseline {
711                return Err(EventSourceError::invalid(
712                    "retained backfill baseline conflicts with durable canonical history",
713                ));
714            }
715            continue;
716        }
717        let end = baseline.number.checked_add(1).ok_or_else(|| {
718            EventSourceError::invalid("retained backfill baseline has no successor")
719        })?;
720        let mut query = Query::new()
721            .from_block(baseline.number)
722            .to_block_excl(end)
723            .include_all_blocks()
724            .select_block_fields([
725                BlockField::Number,
726                BlockField::Hash,
727                BlockField::ParentHash,
728                BlockField::Timestamp,
729            ]);
730        query.max_num_blocks = Some(1);
731        query.max_num_logs = Some(1);
732        let page = retry_source_request(request_timeout, || source.query(query.clone()))
733            .await
734            .map_err(source_error)?;
735        validate_response_counts(&page, response_limits).map_err(source_engine_error)?;
736        validate_source_page(&page, baseline.number, end)
737            .map_err(|error| EventSourceError::unavailable(error.to_string()))?;
738        let observed =
739            block_ref(page.blocks.first().ok_or_else(|| {
740                EventSourceError::unavailable("retained baseline is unavailable")
741            })?)
742            .map_err(|error| EventSourceError::unavailable(error.to_string()))?;
743        if &observed != baseline {
744            return Err(EventSourceError::invalid(
745                "retained backfill baseline does not match provider canonical identity",
746            ));
747        }
748    }
749    Ok(())
750}
751
752fn ensure_session_capacity<S>(
753    state: &ProviderState<S>,
754    key: &SessionKey,
755    limit: usize,
756) -> Result<(), EventSourceError> {
757    let already_resident = state.engines.contains_key(key)
758        || state
759            .prepared
760            .keys()
761            .any(|(prepared_key, _)| prepared_key == key);
762    if already_resident {
763        return Ok(());
764    }
765    let mut resident = HashSet::with_capacity(state.engines.len() + state.prepared.len());
766    resident.extend(state.engines.keys().cloned());
767    resident.extend(
768        state
769            .prepared
770            .keys()
771            .map(|(prepared_key, _)| prepared_key.clone()),
772    );
773    if resident.len() >= limit {
774        return Err(EventSourceError::resource_exhausted(format!(
775            "managed source reached its {limit}-session resident limit"
776        )));
777    }
778    Ok(())
779}
780
781fn required_reorg_target(request: DeliveryRequest<'_>) -> Result<Option<u64>, EventSourceError> {
782    let Some(anchor) = request.required_reorg_anchor() else {
783        return Ok(None);
784    };
785    if anchor.hash.len() != 32 || anchor.parent_hash.len() != 32 {
786        return Err(EventSourceError::invalid(
787            "required reorg anchor hashes must be 32 bytes",
788        ));
789    }
790    let target = anchor
791        .number
792        .checked_add(1)
793        .ok_or_else(|| EventSourceError::invalid("required reorg anchor has no successor"))?;
794    let cursor = request.acknowledged_cursor().ok_or_else(|| {
795        EventSourceError::invalid(
796            "required reorg anchor is missing its acknowledged predecessor cursor",
797        )
798    })?;
799    if cursor.next_block > anchor.number
800        || cursor
801            .canonical_head
802            .as_ref()
803            .is_some_and(|head| head.number >= anchor.number)
804    {
805        return Err(EventSourceError::invalid(
806            "required reorg anchor does not follow the acknowledged source position",
807        ));
808    }
809    if let Some(head) = cursor.canonical_head.as_ref()
810        && head
811            .number
812            .checked_add(1)
813            .is_some_and(|successor| successor == anchor.number)
814        && anchor.parent_hash != head.hash
815    {
816        return Err(EventSourceError::invalid(
817            "required reorg anchor does not descend from the acknowledged canonical head",
818        ));
819    }
820    Ok(Some(target))
821}
822
823fn source_error(error: SourceError) -> EventSourceError {
824    match error {
825        SourceError::InvalidRequestTimeout
826        | SourceError::InvalidSessionLimit
827        | SourceError::InvalidReorgDepth
828        | SourceError::InvalidDeliverySizeLimit { .. }
829        | SourceError::InvalidResponseLimit { .. } => EventSourceError::invalid(error.to_string()),
830        SourceError::Request(_) => {
831            EventSourceError::unavailable("upstream chain data source request failed")
832        }
833        SourceError::UnavailableHeight | SourceError::RequestTimeout { .. } => {
834            EventSourceError::unavailable(error.to_string())
835        }
836    }
837}
838
839fn source_engine_error(error: SourceEngineError) -> EventSourceError {
840    match error {
841        SourceEngineError::QueryPlan(QueryPlanError::TooManyCompiledLogFilters { .. })
842        | SourceEngineError::DeliveryTooLarge { .. }
843        | SourceEngineError::ResponseLimitExceeded { .. }
844        | SourceEngineError::SequenceExhausted => {
845            EventSourceError::resource_exhausted(error.to_string())
846        }
847        SourceEngineError::QueryPlan(_) => EventSourceError::invalid(error.to_string()),
848        SourceEngineError::Source(error) => source_error(error),
849        SourceEngineError::Normalize(_)
850        | SourceEngineError::InvalidPage(_)
851        | SourceEngineError::Canonical(_)
852        | SourceEngineError::NoProgress { .. }
853        | SourceEngineError::MissingRollbackGuard
854        | SourceEngineError::OwnerCatchupReorg { .. }
855        | SourceEngineError::ResumeCursorMismatch { .. }
856        | SourceEngineError::ResumeGuardMismatch(_)
857        | SourceEngineError::CoverageCursorMismatch { .. }
858        | SourceEngineError::CoverageBoundaryConflict { .. } => {
859            EventSourceError::unavailable(error.to_string())
860        }
861        SourceEngineError::InvalidDeliverySizeLimit { .. }
862        | SourceEngineError::NoPendingDelivery
863        | SourceEngineError::DeliveryTokenMismatch
864        | SourceEngineError::MissingCursor
865        | SourceEngineError::PendingDelivery
866        | SourceEngineError::SequenceRegression { .. }
867        | SourceEngineError::BlockNumberOverflow
868        | SourceEngineError::BlockTimestampOverflow(_) => {
869            EventSourceError::internal(error.to_string())
870        }
871    }
872}
873
874async fn available_height<S: ChainDataSource>(
875    source: &S,
876    request_timeout: Duration,
877) -> Result<u64, SourceError> {
878    let height = retry_source_request(request_timeout, || source.height()).await?;
879    if height == 0 {
880        return Err(SourceError::UnavailableHeight);
881    }
882    Ok(height)
883}
884
885async fn retry_source_request<T, F, Fut>(
886    request_timeout: Duration,
887    mut request: F,
888) -> Result<T, SourceError>
889where
890    F: FnMut() -> Fut,
891    Fut: Future<Output = Result<T, SourceError>>,
892{
893    let deadline = tokio::time::Instant::now() + request_timeout;
894    let mut last_error = None;
895
896    for attempt in 0..SOURCE_REQUEST_ATTEMPTS {
897        let now = tokio::time::Instant::now();
898        if now >= deadline {
899            break;
900        }
901        let attempts_left = SOURCE_REQUEST_ATTEMPTS - attempt;
902        let fair_share = (deadline - now) / attempts_left;
903        let attempt_timeout = fair_share.min(MAX_SOURCE_REQUEST_ATTEMPT);
904        match tokio::time::timeout(attempt_timeout, request()).await {
905            Ok(Ok(value)) => return Ok(value),
906            Ok(Err(error)) => last_error = Some(error),
907            Err(_) => last_error = None,
908        }
909    }
910
911    Err(last_error.unwrap_or(SourceError::RequestTimeout {
912        millis: request_timeout.as_millis(),
913    }))
914}
915
916#[cfg(test)]
917mod source_error_tests {
918    use super::*;
919
920    #[test]
921    fn upstream_request_details_are_not_exposed_to_protocol_clients() {
922        let secret = "https://user:secret-token@indexer.invalid/private";
923        let public = source_error(SourceError::Request(secret.into()));
924        assert_eq!(
925            public.kind,
926            evm_fork_cache_event_service::EventSourceErrorKind::Unavailable
927        );
928        assert!(!public.message.contains(secret));
929        assert!(!public.to_string().contains(secret));
930        assert_eq!(public.message, "upstream chain data source request failed");
931    }
932}