1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 future::Future,
4 sync::{
5 Arc,
6 atomic::{AtomicU64, Ordering},
7 },
8 time::Duration,
9};
10
11use async_trait::async_trait;
12use evm_fork_cache_event_protocol::v1::{
13 Acknowledge, ApplyDesiredState, Capability, Cursor, Delivery, SourceCapabilities,
14 SourceDescriptor, SourceRole, portable_interest,
15};
16use evm_fork_cache_event_service::{DeliveryRequest, EventSource, EventSourceError, PreparationId};
17use futures::StreamExt;
18use hypersync_client::net_types::{BlockField, Query};
19use tokio::{
20 sync::{Mutex, Notify},
21 task::AbortHandle,
22};
23
24use crate::{
25 ChainDataSource, ChainHeightStream, HyperSyncDataSource, MAX_DELIVERY_SIZE_BYTES,
26 QueryPlanError, SourceEngine, SourceEngineError, SourceError, SourceResponseLimits,
27 SourceResume,
28 normalize::{DecodedSourceCheckpoint, block_ref, decode_source_checkpoint},
29 page_validation::validate_source_page,
30 source_engine::validate_response_counts,
31};
32
33const SOURCE_REQUEST_ATTEMPTS: u32 = 3;
34const MAX_SOURCE_REQUEST_ATTEMPT: Duration = Duration::from_secs(8);
35
36pub trait ChainDataSourceFactory: Send + Sync + 'static {
38 type Source: ChainDataSource + 'static;
40
41 fn create(&self, chain_id: u64) -> Result<Self::Source, SourceError>;
48
49 fn supports_full_headers(&self, _chain_id: u64) -> bool {
51 false
52 }
53
54 fn supports_full_headers_on_any_chain(&self) -> bool {
56 false
57 }
58}
59
60#[derive(Clone)]
62pub struct HyperSyncSourceFactory {
63 api_token: String,
64 full_header_chains: HashSet<u64>,
65}
66
67impl HyperSyncSourceFactory {
68 pub fn new(api_token: impl Into<String>) -> Self {
70 Self {
71 api_token: api_token.into(),
72 full_header_chains: HashSet::new(),
73 }
74 }
75
76 pub fn with_hash_verified_headers_for_chain(mut self, chain_id: u64) -> Self {
81 self.full_header_chains.insert(chain_id);
82 self
83 }
84}
85
86impl ChainDataSourceFactory for HyperSyncSourceFactory {
87 type Source = HyperSyncDataSource;
88
89 fn create(&self, chain_id: u64) -> Result<Self::Source, SourceError> {
90 HyperSyncDataSource::new(chain_id, &self.api_token)
91 }
92
93 fn supports_full_headers(&self, chain_id: u64) -> bool {
94 self.full_header_chains.contains(&chain_id)
95 }
96
97 fn supports_full_headers_on_any_chain(&self) -> bool {
98 !self.full_header_chains.is_empty()
99 }
100}
101
102struct ManagedEngine<S> {
103 revision: u64,
104 engine: Arc<Mutex<SourceEngine<S>>>,
105 height_driver: Option<Arc<HeightDriver>>,
106}
107
108type SessionKey = (String, u64);
109type EngineMap<S> = HashMap<SessionKey, ManagedEngine<S>>;
110type PreparedEngineMap<S> = HashMap<(SessionKey, PreparationId), ManagedEngine<S>>;
111
112struct ProviderState<S> {
113 engines: EngineMap<S>,
114 prepared: PreparedEngineMap<S>,
115}
116
117impl<S> Default for ProviderState<S> {
118 fn default() -> Self {
119 Self {
120 engines: HashMap::new(),
121 prepared: HashMap::new(),
122 }
123 }
124}
125
126struct HeightDriver {
127 latest_hint: Arc<AtomicU64>,
128 update: Arc<Notify>,
129 abort: AbortHandle,
130}
131
132impl HeightDriver {
133 fn spawn(mut updates: ChainHeightStream) -> Arc<Self> {
134 let latest_hint = Arc::new(AtomicU64::new(0));
135 let update = Arc::new(Notify::new());
136 let task_latest = Arc::clone(&latest_hint);
137 let task_update = Arc::clone(&update);
138 let task = tokio::spawn(async move {
139 while let Some(height) = updates.next().await {
140 task_latest.store(height, Ordering::Release);
141 task_update.notify_one();
142 }
143 });
144 let abort = task.abort_handle();
145 drop(task);
146 Arc::new(Self {
147 latest_hint,
148 update,
149 abort,
150 })
151 }
152}
153
154impl Drop for HeightDriver {
155 fn drop(&mut self) {
156 self.abort.abort();
157 }
158}
159
160impl<S> Clone for ManagedEngine<S> {
161 fn clone(&self) -> Self {
162 Self {
163 revision: self.revision,
164 engine: Arc::clone(&self.engine),
165 height_driver: self.height_driver.as_ref().map(Arc::clone),
166 }
167 }
168}
169
170pub struct ManagedEventProvider<F>
173where
174 F: ChainDataSourceFactory,
175{
176 factory: F,
177 reorg_depth: usize,
178 request_timeout: Duration,
179 max_delivery_bytes: usize,
180 response_limits: SourceResponseLimits,
181 max_resident_sessions: usize,
182 state: Mutex<ProviderState<F::Source>>,
183}
184
185impl<F> ManagedEventProvider<F>
186where
187 F: ChainDataSourceFactory,
188{
189 pub fn new(factory: F, reorg_depth: usize) -> Self {
196 Self::try_new(factory, reorg_depth)
197 .expect("managed source reorg depth must be greater than zero")
198 }
199
200 pub fn try_new(factory: F, reorg_depth: usize) -> Result<Self, SourceError> {
206 if reorg_depth == 0 {
207 return Err(SourceError::InvalidReorgDepth);
208 }
209 Ok(Self {
210 factory,
211 reorg_depth,
212 request_timeout: Duration::from_secs(45),
213 max_delivery_bytes: MAX_DELIVERY_SIZE_BYTES,
214 response_limits: SourceResponseLimits::default(),
215 max_resident_sessions: 4_096,
216 state: Mutex::new(ProviderState::default()),
217 })
218 }
219
220 pub fn with_request_timeout(mut self, request_timeout: Duration) -> Result<Self, SourceError> {
226 if request_timeout.is_zero() {
227 return Err(SourceError::InvalidRequestTimeout);
228 }
229 self.request_timeout = request_timeout;
230 Ok(self)
231 }
232
233 pub fn with_max_delivery_bytes(
240 mut self,
241 max_delivery_bytes: usize,
242 ) -> Result<Self, SourceError> {
243 if max_delivery_bytes == 0 || max_delivery_bytes > MAX_DELIVERY_SIZE_BYTES {
244 return Err(SourceError::InvalidDeliverySizeLimit {
245 requested: max_delivery_bytes,
246 maximum: MAX_DELIVERY_SIZE_BYTES,
247 });
248 }
249 self.max_delivery_bytes = max_delivery_bytes;
250 Ok(self)
251 }
252
253 pub fn with_response_limits(mut self, response_limits: SourceResponseLimits) -> Self {
255 self.response_limits = response_limits;
256 self
257 }
258
259 pub fn with_max_resident_sessions(
265 mut self,
266 max_resident_sessions: usize,
267 ) -> Result<Self, SourceError> {
268 if max_resident_sessions == 0 {
269 return Err(SourceError::InvalidSessionLimit);
270 }
271 self.max_resident_sessions = max_resident_sessions;
272 Ok(self)
273 }
274
275 async fn build_engine(
276 &self,
277 desired_state: &ApplyDesiredState,
278 acknowledged_cursor: Option<&Cursor>,
279 ) -> Result<ManagedEngine<F::Source>, EventSourceError> {
280 if requests_full_headers(desired_state)
281 && !self.factory.supports_full_headers(desired_state.chain_id)
282 {
283 return Err(EventSourceError::unsupported(format!(
284 "complete hash-verified block headers are unavailable for chain {}",
285 desired_state.chain_id
286 )));
287 }
288 let source = self
289 .factory
290 .create(desired_state.chain_id)
291 .map_err(source_error)?;
292 let backfill_start = desired_state
293 .owners
294 .iter()
295 .filter_map(|owner| owner.backfill.as_ref().map(|backfill| backfill.from_block))
296 .min();
297 let global_backfill = desired_state
298 .owners
299 .iter()
300 .find(|owner| owner.canonical)
301 .and_then(|owner| owner.backfill.as_ref());
302 let (start_block, computed_activation_block) = match acknowledged_cursor {
303 Some(cursor) if cursor.query_revision < desired_state.new_revision => (
304 backfill_start.map_or(cursor.next_block, |from| from.min(cursor.next_block)),
305 cursor.next_block,
306 ),
307 Some(cursor) => (cursor.next_block, cursor.next_block),
308 None => match backfill_start {
309 Some(from_block) => {
310 let head = available_height(&source, self.request_timeout)
311 .await
312 .map_err(source_error)?;
313 (from_block, head)
314 }
315 None => {
316 let head = available_height(&source, self.request_timeout)
317 .await
318 .map_err(source_error)?;
319 (head, head)
320 }
321 },
322 };
323 let committed_sequence = acknowledged_cursor.map_or(0, |cursor| cursor.batch_sequence);
324 let decoded_checkpoint = acknowledged_cursor
325 .map(checkpoints_from_cursor)
326 .transpose()?
327 .unwrap_or_default();
328 let checkpoint =
329 if acknowledged_cursor.is_some_and(|cursor| cursor.next_block == start_block) {
330 decoded_checkpoint.clone()
331 } else {
332 DecodedSourceCheckpoint::default()
333 };
334 verify_retained_backfill_baselines(
335 &source,
336 desired_state,
337 &checkpoint.canonical_blocks,
338 self.request_timeout,
339 self.response_limits,
340 )
341 .await?;
342 let global_baseline =
343 global_backfill.and_then(|backfill| backfill.retained_baseline.clone());
344 if let (Some(acknowledged), Some(baseline)) = (
345 acknowledged_cursor.and_then(|cursor| cursor.canonical_head.as_ref()),
346 global_baseline.as_ref(),
347 ) && acknowledged != baseline
348 {
349 return Err(EventSourceError::invalid(
350 "global backfill baseline does not match acknowledged canonical coverage",
351 ));
352 }
353 let mut canonical_blocks = checkpoint.canonical_blocks;
354 if canonical_blocks.is_empty()
355 && let Some(baseline) = global_baseline.clone()
356 {
357 canonical_blocks.push(baseline);
358 }
359 let activation_block = acknowledged_cursor
360 .filter(|cursor| cursor.query_revision == desired_state.new_revision)
361 .and(decoded_checkpoint.activation_block)
362 .unwrap_or(computed_activation_block);
363 let owner_backfill_activation_block = acknowledged_cursor
364 .filter(|cursor| cursor.query_revision == desired_state.new_revision)
365 .map_or(Some(computed_activation_block), |cursor| {
366 cursor.owner_backfill_activation_block
367 });
368 let height_driver = source.height_stream().map(HeightDriver::spawn);
369 let engine = SourceEngine::restore(
370 source,
371 desired_state.clone(),
372 SourceResume {
373 next_block: start_block,
374 sequence: committed_sequence,
375 activation_block,
376 owner_backfill_activation_block,
377 canonical_blocks,
378 provider_checkpoint: checkpoint.rollback_guard,
379 coverage_head: acknowledged_cursor
380 .and_then(|cursor| cursor.canonical_head.clone())
381 .or(global_baseline),
382 },
383 self.reorg_depth,
384 )
385 .map_err(|error| EventSourceError::unavailable(error.to_string()))?
386 .with_max_delivery_bytes(self.max_delivery_bytes)
387 .map_err(source_engine_error)?
388 .with_response_limits(self.response_limits);
389 let managed = ManagedEngine {
390 revision: desired_state.new_revision,
391 engine: Arc::new(Mutex::new(engine)),
392 height_driver,
393 };
394 Ok(managed)
395 }
396
397 async fn engine(
398 &self,
399 desired_state: &ApplyDesiredState,
400 acknowledged_cursor: Option<&Cursor>,
401 ) -> Result<ManagedEngine<F::Source>, EventSourceError> {
402 let key = (desired_state.session_id.clone(), desired_state.chain_id);
403 if let Some(existing) = self.state.lock().await.engines.get(&key).cloned()
404 && existing.revision == desired_state.new_revision
405 {
406 return Ok(existing);
407 }
408 let managed = self
409 .build_engine(desired_state, acknowledged_cursor)
410 .await?;
411 let mut state = self.state.lock().await;
412 if let Some(existing) = state.engines.get(&key).cloned()
413 && existing.revision == desired_state.new_revision
414 {
415 return Ok(existing);
416 }
417 ensure_session_capacity(&state, &key, self.max_resident_sessions)?;
418 state.engines.insert(key, managed.clone());
419 Ok(managed)
420 }
421}
422
423#[async_trait]
424impl<F> EventSource for ManagedEventProvider<F>
425where
426 F: ChainDataSourceFactory,
427{
428 fn capabilities(&self) -> SourceCapabilities {
429 self.source_capabilities(self.factory.supports_full_headers_on_any_chain())
430 }
431
432 fn capabilities_for_chain(&self, chain_id: u64) -> SourceCapabilities {
433 self.source_capabilities(self.factory.supports_full_headers(chain_id))
434 }
435
436 async fn prepare_desired_state(
437 &self,
438 preparation_id: PreparationId,
439 desired_state: &ApplyDesiredState,
440 acknowledged_cursor: Option<&Cursor>,
441 ) -> Result<Option<Cursor>, EventSourceError> {
442 let key = (desired_state.session_id.clone(), desired_state.chain_id);
443 {
444 let state = self.state.lock().await;
445 ensure_session_capacity(&state, &key, self.max_resident_sessions)?;
446 }
447 let managed = self
448 .build_engine(desired_state, acknowledged_cursor)
449 .await?;
450 let activation_cursor = managed
451 .engine
452 .lock()
453 .await
454 .activation_cursor()
455 .map_err(source_engine_error)?;
456 let mut state = self.state.lock().await;
457 ensure_session_capacity(&state, &key, self.max_resident_sessions)?;
458 state
459 .prepared
460 .retain(|(prepared_key, _), _| prepared_key != &key);
461 state.prepared.insert((key, preparation_id), managed);
462 Ok(Some(activation_cursor))
463 }
464
465 async fn activate_desired_state(
466 &self,
467 preparation_id: PreparationId,
468 desired_state: &ApplyDesiredState,
469 _acknowledged_cursor: Option<&Cursor>,
470 ) -> Result<(), EventSourceError> {
471 let key = (desired_state.session_id.clone(), desired_state.chain_id);
472 let mut state = self.state.lock().await;
473 let managed = state
474 .prepared
475 .remove(&(key.clone(), preparation_id))
476 .ok_or_else(|| EventSourceError::internal("prepared source candidate is missing"))?;
477 if managed.revision != desired_state.new_revision {
478 return Err(EventSourceError::internal(
479 "prepared source candidate revision does not match activation",
480 ));
481 }
482 state.engines.insert(key, managed);
483 Ok(())
484 }
485
486 async fn abort_desired_state(
487 &self,
488 preparation_id: PreparationId,
489 desired_state: &ApplyDesiredState,
490 ) -> Result<(), EventSourceError> {
491 let key = (desired_state.session_id.clone(), desired_state.chain_id);
492 self.state
493 .lock()
494 .await
495 .prepared
496 .remove(&(key, preparation_id));
497 Ok(())
498 }
499
500 async fn release_session(
501 &self,
502 session_id: &str,
503 chain_id: u64,
504 ) -> Result<(), EventSourceError> {
505 let key = (session_id.to_owned(), chain_id);
506 let mut state = self.state.lock().await;
507 state.engines.remove(&key);
508 state
509 .prepared
510 .retain(|(prepared_key, _), _| prepared_key != &key);
511 Ok(())
512 }
513
514 async fn next_delivery(
515 &self,
516 request: DeliveryRequest<'_>,
517 ) -> Result<Option<Delivery>, EventSourceError> {
518 let desired_state = request.desired_state();
519 let acknowledged_cursor = request.acknowledged_cursor();
520 let required_target = required_reorg_target(request)?;
521 let managed = self.engine(desired_state, acknowledged_cursor).await?;
522 let mut engine = managed.engine.lock().await;
523 let archive_height = available_height(engine.source(), self.request_timeout)
527 .await
528 .map_err(source_error)?;
529 if let Some(driver) = managed.height_driver.as_ref() {
530 driver.latest_hint.store(archive_height, Ordering::Release);
531 }
532 if archive_height <= engine.committed_next_block() && required_target.is_none() {
533 return Ok(None);
534 }
535 let target = match required_target {
536 Some(target) if archive_height < target => return Ok(None),
537 Some(target) => target,
538 None => archive_height,
539 };
540 tokio::time::timeout(self.request_timeout, engine.next_batch(target))
541 .await
542 .map_err(|_| {
543 source_error(SourceError::RequestTimeout {
544 millis: self.request_timeout.as_millis(),
545 })
546 })?
547 .map_err(source_engine_error)
548 }
549
550 async fn wait_for_update(&self, request: DeliveryRequest<'_>) -> Result<(), EventSourceError> {
551 let desired_state = request.desired_state();
552 let acknowledged_cursor = request.acknowledged_cursor();
553 let required_target = required_reorg_target(request)?;
554 let managed = self.engine(desired_state, acknowledged_cursor).await?;
555 let Some(driver) = managed.height_driver.as_ref() else {
556 return std::future::pending().await;
557 };
558 let committed_next_block = managed.engine.lock().await.committed_next_block();
559 loop {
560 let latest_hint = driver.latest_hint.load(Ordering::Acquire);
561 let ready = required_target.map_or(latest_hint > committed_next_block, |target| {
562 latest_hint >= target
563 });
564 if ready {
565 return Ok(());
566 }
567 driver.update.notified().await;
568 }
569 }
570
571 async fn acknowledge(
572 &self,
573 chain_id: u64,
574 acknowledgement: &Acknowledge,
575 committed_cursor: &Cursor,
576 ) -> Result<(), EventSourceError> {
577 let key = (acknowledgement.session_id.clone(), chain_id);
578 let managed = self.state.lock().await.engines.get(&key).cloned();
579 let Some(managed) = managed else {
580 return Ok(());
583 };
584 let mut engine = managed.engine.lock().await;
585 if acknowledgement.sequence <= engine.committed_sequence() {
586 return Ok(());
587 }
588 match engine.acknowledge(&acknowledgement.delivery_token) {
589 Ok(()) => Ok(()),
590 Err(SourceEngineError::NoPendingDelivery)
591 if committed_cursor.query_revision == managed.revision =>
592 {
593 let checkpoint = checkpoints_from_cursor(committed_cursor)?;
594 let activation_block = checkpoint
595 .activation_block
596 .unwrap_or_else(|| engine.activation_block());
597 engine
598 .synchronize_committed_cursor(SourceResume {
599 next_block: committed_cursor.next_block,
600 sequence: acknowledgement.sequence,
601 activation_block,
602 owner_backfill_activation_block: committed_cursor
603 .owner_backfill_activation_block,
604 canonical_blocks: checkpoint.canonical_blocks,
605 provider_checkpoint: checkpoint.rollback_guard,
606 coverage_head: committed_cursor.canonical_head.clone(),
607 })
608 .map_err(source_engine_error)
609 }
610 Err(error) => Err(source_engine_error(error)),
611 }
612 }
613}
614
615impl<F> ManagedEventProvider<F>
616where
617 F: ChainDataSourceFactory,
618{
619 fn source_capabilities(&self, supports_full_headers: bool) -> SourceCapabilities {
620 let mut capabilities = vec![
621 Capability::Historical.into(),
622 Capability::Logs.into(),
623 Capability::ServerFiltering.into(),
624 Capability::DynamicFilters.into(),
625 Capability::ExplicitReorgs.into(),
626 Capability::NativeCheckpoint.into(),
627 Capability::OwnerScopedDelivery.into(),
628 Capability::DurableReplay.into(),
629 ];
630 if supports_full_headers {
631 capabilities.push(Capability::Headers.into());
632 }
633 SourceCapabilities {
634 capabilities: capabilities.clone(),
635 sources: vec![SourceDescriptor {
636 source_id: "hypersync".into(),
637 role: SourceRole::Historical.into(),
638 capabilities,
639 }],
640 }
641 }
642}
643
644fn checkpoints_from_cursor(cursor: &Cursor) -> Result<DecodedSourceCheckpoint, EventSourceError> {
645 let mut checkpoint = decode_source_checkpoint(&cursor.provider_checkpoint)
646 .map_err(|error| EventSourceError::invalid(error.to_string()))?;
647 if checkpoint.activation_block.is_some()
648 && cursor.owner_backfill_activation_block.is_some()
649 && checkpoint.activation_block != cursor.owner_backfill_activation_block
650 {
651 return Err(EventSourceError::invalid(
652 "portable owner backfill activation boundary conflicts with provider checkpoint",
653 ));
654 }
655 if checkpoint.activation_block.is_none() {
656 checkpoint.activation_block = cursor.owner_backfill_activation_block;
657 }
658 if checkpoint.canonical_blocks.is_empty()
659 && let Some(head) = cursor.canonical_head.clone()
660 && head.number.checked_add(1) == Some(cursor.next_block)
661 {
662 checkpoint.canonical_blocks.push(head);
663 }
664 Ok(checkpoint)
665}
666
667fn requests_full_headers(desired_state: &ApplyDesiredState) -> bool {
668 desired_state.owners.iter().any(|owner| {
669 owner.interests.iter().any(|interest| {
670 matches!(
671 interest.kind.as_ref(),
672 Some(portable_interest::Kind::Block(_))
673 )
674 })
675 })
676}
677
678async fn verify_retained_backfill_baselines<S: ChainDataSource>(
679 source: &S,
680 desired_state: &ApplyDesiredState,
681 retained_history: &[evm_fork_cache_event_protocol::v1::BlockRef],
682 request_timeout: Duration,
683 response_limits: SourceResponseLimits,
684) -> Result<(), EventSourceError> {
685 let mut baselines = BTreeMap::new();
686 for baseline in desired_state
687 .owners
688 .iter()
689 .filter_map(|owner| owner.backfill.as_ref())
690 .filter_map(|backfill| backfill.retained_baseline.as_ref())
691 {
692 if let Some(existing) = baselines.insert(baseline.number, baseline)
693 && existing != baseline
694 {
695 return Err(EventSourceError::invalid(
696 "retained backfill baselines conflict at the same block height",
697 ));
698 }
699 }
700 for baseline in baselines.into_values() {
701 if baseline.hash.len() != 32 || baseline.parent_hash.len() != 32 {
702 return Err(EventSourceError::invalid(
703 "retained backfill baseline hashes must be 32 bytes",
704 ));
705 }
706 if let Some(retained) = retained_history
707 .iter()
708 .find(|retained| retained.number == baseline.number)
709 {
710 if retained != baseline {
711 return Err(EventSourceError::invalid(
712 "retained backfill baseline conflicts with durable canonical history",
713 ));
714 }
715 continue;
716 }
717 let end = baseline.number.checked_add(1).ok_or_else(|| {
718 EventSourceError::invalid("retained backfill baseline has no successor")
719 })?;
720 let mut query = Query::new()
721 .from_block(baseline.number)
722 .to_block_excl(end)
723 .include_all_blocks()
724 .select_block_fields([
725 BlockField::Number,
726 BlockField::Hash,
727 BlockField::ParentHash,
728 BlockField::Timestamp,
729 ]);
730 query.max_num_blocks = Some(1);
731 query.max_num_logs = Some(1);
732 let page = retry_source_request(request_timeout, || source.query(query.clone()))
733 .await
734 .map_err(source_error)?;
735 validate_response_counts(&page, response_limits).map_err(source_engine_error)?;
736 validate_source_page(&page, baseline.number, end)
737 .map_err(|error| EventSourceError::unavailable(error.to_string()))?;
738 let observed =
739 block_ref(page.blocks.first().ok_or_else(|| {
740 EventSourceError::unavailable("retained baseline is unavailable")
741 })?)
742 .map_err(|error| EventSourceError::unavailable(error.to_string()))?;
743 if &observed != baseline {
744 return Err(EventSourceError::invalid(
745 "retained backfill baseline does not match provider canonical identity",
746 ));
747 }
748 }
749 Ok(())
750}
751
752fn ensure_session_capacity<S>(
753 state: &ProviderState<S>,
754 key: &SessionKey,
755 limit: usize,
756) -> Result<(), EventSourceError> {
757 let already_resident = state.engines.contains_key(key)
758 || state
759 .prepared
760 .keys()
761 .any(|(prepared_key, _)| prepared_key == key);
762 if already_resident {
763 return Ok(());
764 }
765 let mut resident = HashSet::with_capacity(state.engines.len() + state.prepared.len());
766 resident.extend(state.engines.keys().cloned());
767 resident.extend(
768 state
769 .prepared
770 .keys()
771 .map(|(prepared_key, _)| prepared_key.clone()),
772 );
773 if resident.len() >= limit {
774 return Err(EventSourceError::resource_exhausted(format!(
775 "managed source reached its {limit}-session resident limit"
776 )));
777 }
778 Ok(())
779}
780
781fn required_reorg_target(request: DeliveryRequest<'_>) -> Result<Option<u64>, EventSourceError> {
782 let Some(anchor) = request.required_reorg_anchor() else {
783 return Ok(None);
784 };
785 if anchor.hash.len() != 32 || anchor.parent_hash.len() != 32 {
786 return Err(EventSourceError::invalid(
787 "required reorg anchor hashes must be 32 bytes",
788 ));
789 }
790 let target = anchor
791 .number
792 .checked_add(1)
793 .ok_or_else(|| EventSourceError::invalid("required reorg anchor has no successor"))?;
794 let cursor = request.acknowledged_cursor().ok_or_else(|| {
795 EventSourceError::invalid(
796 "required reorg anchor is missing its acknowledged predecessor cursor",
797 )
798 })?;
799 if cursor.next_block > anchor.number
800 || cursor
801 .canonical_head
802 .as_ref()
803 .is_some_and(|head| head.number >= anchor.number)
804 {
805 return Err(EventSourceError::invalid(
806 "required reorg anchor does not follow the acknowledged source position",
807 ));
808 }
809 if let Some(head) = cursor.canonical_head.as_ref()
810 && head
811 .number
812 .checked_add(1)
813 .is_some_and(|successor| successor == anchor.number)
814 && anchor.parent_hash != head.hash
815 {
816 return Err(EventSourceError::invalid(
817 "required reorg anchor does not descend from the acknowledged canonical head",
818 ));
819 }
820 Ok(Some(target))
821}
822
823fn source_error(error: SourceError) -> EventSourceError {
824 match error {
825 SourceError::InvalidRequestTimeout
826 | SourceError::InvalidSessionLimit
827 | SourceError::InvalidReorgDepth
828 | SourceError::InvalidDeliverySizeLimit { .. }
829 | SourceError::InvalidResponseLimit { .. } => EventSourceError::invalid(error.to_string()),
830 SourceError::Request(_) => {
831 EventSourceError::unavailable("upstream chain data source request failed")
832 }
833 SourceError::UnavailableHeight | SourceError::RequestTimeout { .. } => {
834 EventSourceError::unavailable(error.to_string())
835 }
836 }
837}
838
839fn source_engine_error(error: SourceEngineError) -> EventSourceError {
840 match error {
841 SourceEngineError::QueryPlan(QueryPlanError::TooManyCompiledLogFilters { .. })
842 | SourceEngineError::DeliveryTooLarge { .. }
843 | SourceEngineError::ResponseLimitExceeded { .. }
844 | SourceEngineError::SequenceExhausted => {
845 EventSourceError::resource_exhausted(error.to_string())
846 }
847 SourceEngineError::QueryPlan(_) => EventSourceError::invalid(error.to_string()),
848 SourceEngineError::Source(error) => source_error(error),
849 SourceEngineError::Normalize(_)
850 | SourceEngineError::InvalidPage(_)
851 | SourceEngineError::Canonical(_)
852 | SourceEngineError::NoProgress { .. }
853 | SourceEngineError::MissingRollbackGuard
854 | SourceEngineError::OwnerCatchupReorg { .. }
855 | SourceEngineError::ResumeCursorMismatch { .. }
856 | SourceEngineError::ResumeGuardMismatch(_)
857 | SourceEngineError::CoverageCursorMismatch { .. }
858 | SourceEngineError::CoverageBoundaryConflict { .. } => {
859 EventSourceError::unavailable(error.to_string())
860 }
861 SourceEngineError::InvalidDeliverySizeLimit { .. }
862 | SourceEngineError::NoPendingDelivery
863 | SourceEngineError::DeliveryTokenMismatch
864 | SourceEngineError::MissingCursor
865 | SourceEngineError::PendingDelivery
866 | SourceEngineError::SequenceRegression { .. }
867 | SourceEngineError::BlockNumberOverflow
868 | SourceEngineError::BlockTimestampOverflow(_) => {
869 EventSourceError::internal(error.to_string())
870 }
871 }
872}
873
874async fn available_height<S: ChainDataSource>(
875 source: &S,
876 request_timeout: Duration,
877) -> Result<u64, SourceError> {
878 let height = retry_source_request(request_timeout, || source.height()).await?;
879 if height == 0 {
880 return Err(SourceError::UnavailableHeight);
881 }
882 Ok(height)
883}
884
885async fn retry_source_request<T, F, Fut>(
886 request_timeout: Duration,
887 mut request: F,
888) -> Result<T, SourceError>
889where
890 F: FnMut() -> Fut,
891 Fut: Future<Output = Result<T, SourceError>>,
892{
893 let deadline = tokio::time::Instant::now() + request_timeout;
894 let mut last_error = None;
895
896 for attempt in 0..SOURCE_REQUEST_ATTEMPTS {
897 let now = tokio::time::Instant::now();
898 if now >= deadline {
899 break;
900 }
901 let attempts_left = SOURCE_REQUEST_ATTEMPTS - attempt;
902 let fair_share = (deadline - now) / attempts_left;
903 let attempt_timeout = fair_share.min(MAX_SOURCE_REQUEST_ATTEMPT);
904 match tokio::time::timeout(attempt_timeout, request()).await {
905 Ok(Ok(value)) => return Ok(value),
906 Ok(Err(error)) => last_error = Some(error),
907 Err(_) => last_error = None,
908 }
909 }
910
911 Err(last_error.unwrap_or(SourceError::RequestTimeout {
912 millis: request_timeout.as_millis(),
913 }))
914}
915
916#[cfg(test)]
917mod source_error_tests {
918 use super::*;
919
920 #[test]
921 fn upstream_request_details_are_not_exposed_to_protocol_clients() {
922 let secret = "https://user:secret-token@indexer.invalid/private";
923 let public = source_error(SourceError::Request(secret.into()));
924 assert_eq!(
925 public.kind,
926 evm_fork_cache_event_service::EventSourceErrorKind::Unavailable
927 );
928 assert!(!public.message.contains(secret));
929 assert!(!public.to_string().contains(secret));
930 assert_eq!(public.message, "upstream chain data source request failed");
931 }
932}