use std::pin::Pin;
use async_trait::async_trait;
use evm_fork_cache_event_protocol::{
MAX_MESSAGE_SIZE_BYTES,
v1::{ApplyDesiredState, Barrier, BlockRef, Cursor, Delivery, Reorg, delivery},
};
use futures::Stream;
use hypersync_client::{
format::Hash,
net_types::{BlockField, Query, RollbackGuard},
};
use prost::Message;
use crate::{
CanonicalError, CanonicalTracker, NormalizeError, QueryPlanError, ReorgDelta, SourcePage,
SourcePageError,
normalize::{block_ref, encode_source_checkpoint, normalize_page_at},
page_validation::validate_source_page,
query_plan::{MAX_BLOCKS_PER_QUERY, MAX_LOGS_PER_QUERY, compile_query_with_limits},
};
pub const MAX_DELIVERY_SIZE_BYTES: usize = MAX_MESSAGE_SIZE_BYTES - 64 * 1024;
pub const MAX_BLOCKS_PER_RESPONSE: usize = MAX_BLOCKS_PER_QUERY * 2;
pub const MAX_LOGS_PER_RESPONSE: usize = MAX_LOGS_PER_QUERY * 2;
pub const MAX_DYNAMIC_BYTES_PER_RESPONSE: usize = 16 * 1024 * 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub struct SourceResponseLimits {
max_blocks: usize,
max_logs: usize,
max_dynamic_bytes: usize,
}
impl SourceResponseLimits {
pub fn new(
max_blocks: usize,
max_logs: usize,
max_dynamic_bytes: usize,
) -> Result<Self, SourceError> {
if max_blocks == 0 {
return Err(SourceError::InvalidResponseLimit { resource: "blocks" });
}
if max_logs == 0 {
return Err(SourceError::InvalidResponseLimit { resource: "logs" });
}
if max_dynamic_bytes == 0 {
return Err(SourceError::InvalidResponseLimit {
resource: "dynamic bytes",
});
}
Ok(Self {
max_blocks,
max_logs,
max_dynamic_bytes,
})
}
pub const fn max_blocks(&self) -> usize {
self.max_blocks
}
pub const fn max_logs(&self) -> usize {
self.max_logs
}
pub const fn max_dynamic_bytes(&self) -> usize {
self.max_dynamic_bytes
}
}
impl Default for SourceResponseLimits {
fn default() -> Self {
Self {
max_blocks: MAX_BLOCKS_PER_RESPONSE,
max_logs: MAX_LOGS_PER_RESPONSE,
max_dynamic_bytes: MAX_DYNAMIC_BYTES_PER_RESPONSE,
}
}
}
pub type ChainHeightStream = Pin<Box<dyn Stream<Item = u64> + Send + 'static>>;
#[async_trait]
pub trait ChainDataSource: Send + Sync {
async fn height(&self) -> Result<u64, SourceError>;
async fn query(&self, query: Query) -> Result<SourcePage, SourceError>;
fn height_stream(&self) -> Option<ChainHeightStream> {
None
}
}
pub struct SourceEngine<S> {
source: S,
desired_state: ApplyDesiredState,
committed_next_block: u64,
committed_sequence: u64,
activation_block: u64,
owner_backfill_activation_block: Option<u64>,
coverage_head: Option<BlockRef>,
canonical: CanonicalTracker,
previous_guard: Option<RollbackGuard>,
max_delivery_bytes: usize,
response_limits: SourceResponseLimits,
pending: Option<PendingDelivery>,
queued: Option<PendingDelivery>,
}
struct PendingDelivery {
delivery: Delivery,
rollback_guard: Option<RollbackGuard>,
advances_source: bool,
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct SourceResume {
pub next_block: u64,
pub sequence: u64,
pub activation_block: u64,
pub owner_backfill_activation_block: Option<u64>,
pub canonical_blocks: Vec<BlockRef>,
pub provider_checkpoint: Option<RollbackGuard>,
pub coverage_head: Option<BlockRef>,
}
impl SourceResume {
pub fn new(next_block: u64, sequence: u64, activation_block: u64) -> Self {
Self {
next_block,
sequence,
activation_block,
owner_backfill_activation_block: Some(activation_block),
canonical_blocks: Vec::new(),
provider_checkpoint: None,
coverage_head: None,
}
}
pub fn with_canonical_blocks(mut self, canonical_blocks: Vec<BlockRef>) -> Self {
self.canonical_blocks = canonical_blocks;
self
}
pub fn with_provider_checkpoint(mut self, provider_checkpoint: Option<RollbackGuard>) -> Self {
self.provider_checkpoint = provider_checkpoint;
self
}
pub fn with_coverage_head(mut self, coverage_head: Option<BlockRef>) -> Self {
self.coverage_head = coverage_head;
self
}
}
impl<S> SourceEngine<S>
where
S: ChainDataSource,
{
pub fn new(
source: S,
desired_state: ApplyDesiredState,
committed_next_block: u64,
activation_block: u64,
reorg_depth: usize,
) -> Self {
Self {
source,
desired_state,
committed_next_block,
committed_sequence: 0,
activation_block,
owner_backfill_activation_block: Some(activation_block),
coverage_head: None,
canonical: CanonicalTracker::new(reorg_depth),
previous_guard: None,
max_delivery_bytes: MAX_DELIVERY_SIZE_BYTES,
response_limits: SourceResponseLimits::default(),
pending: None,
queued: None,
}
}
pub fn restore(
source: S,
desired_state: ApplyDesiredState,
resume: SourceResume,
reorg_depth: usize,
) -> Result<Self, SourceEngineError> {
let mut canonical = CanonicalTracker::new(reorg_depth);
if !resume.canonical_blocks.is_empty() {
canonical.apply_blocks(resume.canonical_blocks)?;
}
let previous_guard =
validate_resume_state(&canonical, resume.next_block, resume.provider_checkpoint)?;
validate_coverage_state(&canonical, resume.next_block, resume.coverage_head.as_ref())?;
Ok(Self {
source,
desired_state,
committed_next_block: resume.next_block,
committed_sequence: resume.sequence,
activation_block: resume.activation_block,
owner_backfill_activation_block: resume.owner_backfill_activation_block,
coverage_head: resume.coverage_head,
canonical,
previous_guard,
max_delivery_bytes: MAX_DELIVERY_SIZE_BYTES,
response_limits: SourceResponseLimits::default(),
pending: None,
queued: None,
})
}
pub fn with_max_delivery_bytes(
mut self,
max_delivery_bytes: usize,
) -> Result<Self, SourceEngineError> {
if max_delivery_bytes == 0 || max_delivery_bytes > MAX_DELIVERY_SIZE_BYTES {
return Err(SourceEngineError::InvalidDeliverySizeLimit {
requested: max_delivery_bytes,
maximum: MAX_DELIVERY_SIZE_BYTES,
});
}
self.max_delivery_bytes = max_delivery_bytes;
Ok(self)
}
pub fn with_response_limits(mut self, response_limits: SourceResponseLimits) -> Self {
self.response_limits = response_limits;
self
}
pub async fn next_batch(
&mut self,
to_block_excl: u64,
) -> Result<Option<Delivery>, SourceEngineError> {
if let Some(pending) = &self.pending {
return Ok(Some(pending.delivery.clone()));
}
if self.committed_next_block >= to_block_excl {
return Ok(None);
}
let sequence = self
.committed_sequence
.checked_add(1)
.ok_or(SourceEngineError::SequenceExhausted)?;
let mut query_limits = QueryLimits::for_range(self.committed_next_block, to_block_excl);
let mut initial_anchor = None;
let (mut delivery, rollback_guard, canonical, reorg) = loop {
let page = self
.query_page(self.committed_next_block, to_block_excl, query_limits)
.await?;
if initial_anchor.is_none() {
initial_anchor = self.query_canonical_anchor(&page).await?;
}
let prepared = if self.rollback_detected(&page) {
self.recover_rollback(to_block_excl, sequence, query_limits)
.await?
} else {
self.prepare_batch(page, sequence, initial_anchor.as_ref())?
};
let encoded_len = prepared.0.encoded_len();
if encoded_len <= self.max_delivery_bytes {
break prepared;
}
if !query_limits.shrink() {
return Err(SourceEngineError::DeliveryTooLarge {
encoded_len,
limit: self.max_delivery_bytes,
});
}
};
if let Some(reorg) = reorg {
if reorg.old_tip.number < self.activation_block
&& !global_backfill_contains(&self.desired_state, reorg.old_tip.number)
{
return Err(SourceEngineError::OwnerCatchupReorg {
activation_block: self.activation_block,
common_ancestor: reorg.common_ancestor.number,
old_tip: reorg.old_tip.number,
new_tip: reorg.new_tip.number,
});
}
let replacement_sequence = sequence
.checked_add(1)
.ok_or(SourceEngineError::SequenceExhausted)?;
set_delivery_sequence(&mut delivery, replacement_sequence)?;
let control = self.reorg_delivery(sequence, reorg)?;
let encoded_len = control.encoded_len();
if encoded_len > self.max_delivery_bytes {
return Err(SourceEngineError::DeliveryTooLarge {
encoded_len,
limit: self.max_delivery_bytes,
});
}
self.canonical = canonical;
self.queued = Some(PendingDelivery {
delivery,
rollback_guard,
advances_source: true,
});
self.pending = Some(PendingDelivery {
delivery: control.clone(),
rollback_guard: self.previous_guard.clone(),
advances_source: false,
});
Ok(Some(control))
} else {
self.canonical = canonical;
self.pending = Some(PendingDelivery {
delivery: delivery.clone(),
rollback_guard,
advances_source: true,
});
Ok(Some(delivery))
}
}
pub fn acknowledge(&mut self, delivery_token: &[u8]) -> Result<(), SourceEngineError> {
let pending = self
.pending
.as_ref()
.ok_or(SourceEngineError::NoPendingDelivery)?;
if pending.delivery.delivery_token != delivery_token {
return Err(SourceEngineError::DeliveryTokenMismatch);
}
let cursor = pending
.delivery
.cursor
.as_ref()
.ok_or(SourceEngineError::MissingCursor)?;
let committed_next_block = cursor.next_block;
let committed_sequence = pending.delivery.sequence;
let advances_source = pending.advances_source;
let rollback_guard = pending.rollback_guard.clone();
let coverage_head = cursor.canonical_head.clone();
if advances_source {
self.committed_next_block = committed_next_block;
self.previous_guard = rollback_guard;
}
self.committed_sequence = committed_sequence;
self.coverage_head = coverage_head;
self.pending = self.queued.take();
Ok(())
}
pub fn committed_next_block(&self) -> u64 {
self.committed_next_block
}
pub fn activation_cursor(&self) -> Result<Cursor, SourceEngineError> {
Ok(Cursor {
chain_id: self.desired_state.chain_id,
query_revision: self.desired_state.new_revision,
next_block: self.committed_next_block,
canonical_head: self.coverage_head.clone(),
batch_sequence: self.committed_sequence,
provider_checkpoint: encode_source_checkpoint(
self.previous_guard.as_ref(),
self.activation_block,
self.canonical.blocks(),
)?,
owner_backfill_activation_block: self.owner_backfill_activation_block,
})
}
pub fn committed_sequence(&self) -> u64 {
self.committed_sequence
}
pub fn acknowledge_external_control(&mut self, sequence: u64) -> Result<(), SourceEngineError> {
if self.pending.is_some() {
return Err(SourceEngineError::PendingDelivery);
}
if sequence < self.committed_sequence {
return Err(SourceEngineError::SequenceRegression {
committed: self.committed_sequence,
received: sequence,
});
}
self.committed_sequence = sequence;
Ok(())
}
pub fn synchronize_committed_cursor(
&mut self,
resume: SourceResume,
) -> Result<(), SourceEngineError> {
if self.pending.is_some() {
return Err(SourceEngineError::PendingDelivery);
}
if resume.sequence < self.committed_sequence {
return Err(SourceEngineError::SequenceRegression {
committed: self.committed_sequence,
received: resume.sequence,
});
}
let mut canonical = self.canonical.clone();
canonical.restore_blocks(resume.canonical_blocks)?;
let previous_guard =
validate_resume_state(&canonical, resume.next_block, resume.provider_checkpoint)?;
validate_coverage_state(&canonical, resume.next_block, resume.coverage_head.as_ref())?;
self.canonical = canonical;
self.committed_next_block = resume.next_block;
self.committed_sequence = resume.sequence;
self.activation_block = resume.activation_block;
self.previous_guard = previous_guard;
self.coverage_head = resume.coverage_head;
self.owner_backfill_activation_block = resume.owner_backfill_activation_block;
Ok(())
}
pub fn activation_block(&self) -> u64 {
self.activation_block
}
pub fn source(&self) -> &S {
&self.source
}
async fn query_page(
&self,
from_block: u64,
to_block_excl: u64,
limits: QueryLimits,
) -> Result<SourcePage, SourceEngineError> {
let query = compile_query_with_limits(
&self.desired_state,
from_block,
Some(to_block_excl),
limits.max_blocks,
limits.max_logs,
)?;
let page = self.source.query(query).await?;
validate_response_counts(&page, self.response_limits)?;
if page.next_block <= from_block {
return Err(SourceEngineError::NoProgress {
from_block,
next_block: page.next_block,
});
}
validate_source_page(&page, from_block, to_block_excl)?;
Ok(page)
}
async fn query_canonical_anchor(
&self,
page: &SourcePage,
) -> Result<Option<BlockRef>, SourceEngineError> {
if !self.canonical.is_empty() {
return Ok(None);
}
let Some(guard) = page.rollback_guard.as_ref() else {
return Ok(None);
};
let Some(anchor_number) = guard.first_block_number.checked_sub(1) else {
return Ok(None);
};
let anchor_end = anchor_number
.checked_add(1)
.ok_or(SourceEngineError::BlockNumberOverflow)?;
let mut query = Query::new()
.from_block(anchor_number)
.to_block_excl(anchor_end)
.include_all_blocks()
.select_block_fields([
BlockField::Number,
BlockField::Hash,
BlockField::ParentHash,
BlockField::Timestamp,
]);
query.max_num_blocks = Some(1);
query.max_num_logs = Some(1);
let anchor_page = self.source.query(query).await?;
validate_response_counts(&anchor_page, self.response_limits)?;
if anchor_page.next_block <= anchor_number {
return Err(SourceEngineError::NoProgress {
from_block: anchor_number,
next_block: anchor_page.next_block,
});
}
validate_source_page(&anchor_page, anchor_number, anchor_end)?;
let anchor = block_ref(
anchor_page
.blocks
.first()
.expect("validated one-block anchor page"),
)?;
if anchor.hash.as_slice() != guard.first_parent_hash.as_ref() {
return Err(CanonicalError::UnknownParent {
number: guard.first_block_number,
parent_hash: guard.first_parent_hash.as_ref().to_vec(),
}
.into());
}
Ok(Some(anchor))
}
fn rollback_detected(&self, page: &SourcePage) -> bool {
match (&self.previous_guard, &page.rollback_guard) {
(Some(previous), Some(current)) => {
current.first_block_number <= previous.block_number.saturating_add(1)
&& current.first_parent_hash != previous.hash
}
_ => false,
}
}
async fn recover_rollback(
&self,
to_block_excl: u64,
sequence: u64,
limits: QueryLimits,
) -> Result<PreparedPage, SourceEngineError> {
let previous = self
.previous_guard
.as_ref()
.ok_or(SourceEngineError::MissingRollbackGuard)?;
let oldest = self
.canonical
.blocks()
.next()
.map(|block| block.number)
.ok_or(CanonicalError::HistoryExhausted {
required_block: previous.block_number,
})?;
let newest = self
.canonical
.tip()
.map(|block| block.number.min(previous.block_number))
.ok_or(CanonicalError::HistoryExhausted {
required_block: previous.block_number,
})?;
let mut low = oldest;
let mut high = newest;
let mut common_ancestor = None;
while low <= high {
let candidate = low + (high - low) / 2;
let observed = self.query_canonical_block(candidate).await?;
let expected = self
.canonical
.block(candidate)
.expect("binary probe stays inside retained canonical history");
if observed.hash == expected.hash {
if observed != *expected {
return Err(
CanonicalError::ConflictingBlockMetadata { number: candidate }.into(),
);
}
common_ancestor = Some(candidate);
let Some(next) = candidate.checked_add(1) else {
break;
};
low = next;
} else if candidate == 0 {
break;
} else {
high = candidate - 1;
}
}
let common_ancestor = common_ancestor.ok_or(CanonicalError::HistoryExhausted {
required_block: oldest.saturating_sub(1),
})?;
let replacement_start = common_ancestor
.checked_add(1)
.ok_or(SourceEngineError::BlockNumberOverflow)?;
let page = self
.query_page(replacement_start, to_block_excl, limits)
.await?;
self.prepare_batch(page, sequence, None)
}
async fn query_canonical_block(&self, number: u64) -> Result<BlockRef, SourceEngineError> {
let end = number
.checked_add(1)
.ok_or(SourceEngineError::BlockNumberOverflow)?;
let mut query = Query::new()
.from_block(number)
.to_block_excl(end)
.include_all_blocks()
.select_block_fields([
BlockField::Number,
BlockField::Hash,
BlockField::ParentHash,
BlockField::Timestamp,
]);
query.max_num_blocks = Some(1);
query.max_num_logs = Some(1);
let page = self.source.query(query).await?;
validate_response_counts(&page, self.response_limits)?;
if page.next_block <= number {
return Err(SourceEngineError::NoProgress {
from_block: number,
next_block: page.next_block,
});
}
validate_source_page(&page, number, end)?;
block_ref(page.blocks.first().expect("validated single-block page")).map_err(Into::into)
}
fn prepare_batch(
&self,
page: SourcePage,
sequence: u64,
initial_anchor: Option<&BlockRef>,
) -> Result<PreparedPage, SourceEngineError> {
let rollback_guard = page.rollback_guard.clone();
let page_blocks = page
.blocks
.iter()
.map(block_ref)
.collect::<Result<Vec<_>, _>>()?;
let crosses_untracked_coverage_boundary =
self.coverage_head.as_ref().is_some_and(|coverage| {
self.canonical.is_empty() || self.committed_next_block <= coverage.number
});
if initial_anchor.is_some() || crosses_untracked_coverage_boundary {
validate_coverage_boundary(self.coverage_head.as_ref(), &page_blocks, initial_anchor)?;
}
let mut delivery =
normalize_page_at(&self.desired_state, sequence, page, self.activation_block)?;
let mut canonical = self.canonical.clone();
if canonical.is_empty()
&& let Some(anchor) = initial_anchor
{
canonical.apply_blocks([anchor.clone()])?;
}
let reorg = canonical.apply_blocks(page_blocks)?;
delivery
.cursor
.as_mut()
.ok_or(SourceEngineError::MissingCursor)?
.provider_checkpoint = encode_source_checkpoint(
rollback_guard.as_ref(),
self.activation_block,
canonical.blocks(),
)?;
let cursor = delivery
.cursor
.as_mut()
.expect("delivery cursor was validated above");
cursor.owner_backfill_activation_block = self.owner_backfill_activation_block;
if cursor.canonical_head.is_none() {
cursor.canonical_head = self.coverage_head.clone();
}
if matches!(
delivery.payload.as_ref(),
Some(delivery::Payload::Data(data)) if data.records.is_empty()
) {
let progress_block = delivery
.cursor
.as_ref()
.and_then(|cursor| cursor.canonical_head.clone())
.filter(|head| self.coverage_head.as_ref() != Some(head));
delivery.payload = Some(delivery::Payload::Barrier(Barrier {
id: format!(
"source-progress:{}:{}",
self.desired_state.new_revision,
delivery
.cursor
.as_ref()
.expect("validated cursor")
.next_block
)
.into_bytes(),
block: progress_block.clone(),
}));
delivery.checkpoint_neutral = progress_block.is_none();
}
Ok((delivery, rollback_guard, canonical, reorg))
}
fn reorg_delivery(
&self,
sequence: u64,
reorg: ReorgDelta,
) -> Result<Delivery, SourceEngineError> {
let next_block = reorg
.common_ancestor
.number
.checked_add(1)
.ok_or(SourceEngineError::BlockNumberOverflow)?;
let rollback_guard = rollback_guard_at(&reorg.common_ancestor)?;
let canonical_prefix: Vec<_> = self
.canonical
.blocks()
.take_while(|block| block.number <= reorg.common_ancestor.number)
.collect();
Ok(Delivery {
session_id: self.desired_state.session_id.clone(),
sequence,
query_revision: self.desired_state.new_revision,
delivery_token: sequence.to_be_bytes().to_vec(),
cursor: Some(Cursor {
chain_id: self.desired_state.chain_id,
query_revision: self.desired_state.new_revision,
next_block,
canonical_head: Some(reorg.common_ancestor.clone()),
batch_sequence: sequence,
provider_checkpoint: encode_source_checkpoint(
Some(&rollback_guard),
self.activation_block,
canonical_prefix.into_iter(),
)?,
owner_backfill_activation_block: self.owner_backfill_activation_block,
}),
payload: Some(delivery::Payload::Reorg(Reorg {
common_ancestor: Some(reorg.common_ancestor),
old_tip: Some(reorg.old_tip),
new_tip: Some(reorg.new_tip),
})),
checkpoint_neutral: false,
})
}
}
fn global_backfill_contains(desired_state: &ApplyDesiredState, block_number: u64) -> bool {
desired_state.owners.iter().any(|owner| {
owner.canonical
&& owner.backfill.as_ref().is_some_and(|backfill| {
block_number >= backfill.from_block
&& backfill.to_block_excl.is_none_or(|end| block_number < end)
})
})
}
fn validate_coverage_state(
canonical: &CanonicalTracker,
next_block: u64,
coverage_head: Option<&BlockRef>,
) -> Result<(), SourceEngineError> {
let Some(coverage_head) = coverage_head else {
return Ok(());
};
let mut validator = CanonicalTracker::new(1);
validator.apply_blocks([coverage_head.clone()])?;
let coverage_successor = coverage_head
.number
.checked_add(1)
.ok_or(SourceEngineError::BlockNumberOverflow)?;
if next_block > coverage_successor {
return Err(SourceEngineError::CoverageCursorMismatch {
coverage_successor,
received: next_block,
});
}
if let Some(scanned) = canonical.block(coverage_head.number)
&& scanned != coverage_head
{
return Err(SourceEngineError::CoverageBoundaryConflict {
number: coverage_head.number,
});
}
Ok(())
}
fn validate_coverage_boundary(
coverage_head: Option<&BlockRef>,
page_blocks: &[BlockRef],
initial_anchor: Option<&BlockRef>,
) -> Result<(), SourceEngineError> {
let Some(coverage_head) = coverage_head else {
return Ok(());
};
if let Some(observed) = page_blocks
.iter()
.find(|block| block.number == coverage_head.number)
.or_else(|| initial_anchor.filter(|block| block.number == coverage_head.number))
&& observed != coverage_head
{
return Err(SourceEngineError::CoverageBoundaryConflict {
number: coverage_head.number,
});
}
if let Some(first) = page_blocks.first()
&& first.number == coverage_head.number.saturating_add(1)
&& first.parent_hash != coverage_head.hash
{
return Err(SourceEngineError::CoverageBoundaryConflict {
number: coverage_head.number,
});
}
Ok(())
}
pub(crate) fn validate_response_counts(
page: &SourcePage,
limits: SourceResponseLimits,
) -> Result<(), SourceEngineError> {
if page.blocks.len() > limits.max_blocks {
return Err(SourceEngineError::ResponseLimitExceeded {
resource: "blocks",
observed: page.blocks.len(),
limit: limits.max_blocks,
});
}
if page.logs.len() > limits.max_logs {
return Err(SourceEngineError::ResponseLimitExceeded {
resource: "logs",
observed: page.logs.len(),
limit: limits.max_logs,
});
}
validate_response_dynamic_bytes(page, limits.max_dynamic_bytes)?;
Ok(())
}
fn validate_response_dynamic_bytes(
page: &SourcePage,
limit: usize,
) -> Result<(), SourceEngineError> {
let mut observed = std::mem::size_of::<hypersync_client::simple_types::Block>()
.saturating_mul(page.blocks.capacity())
.saturating_add(
std::mem::size_of::<hypersync_client::simple_types::Log>()
.saturating_mul(page.logs.capacity()),
);
check_dynamic_bytes(observed, limit)?;
macro_rules! add_bytes {
($value:expr) => {
if let Some(value) = $value.as_ref() {
observed = observed.saturating_add(value.as_ref().len());
check_dynamic_bytes(observed, limit)?;
}
};
}
for block in &page.blocks {
add_bytes!(block.hash);
add_bytes!(block.parent_hash);
add_bytes!(block.nonce);
add_bytes!(block.sha3_uncles);
add_bytes!(block.logs_bloom);
add_bytes!(block.transactions_root);
add_bytes!(block.state_root);
add_bytes!(block.receipts_root);
add_bytes!(block.miner);
add_bytes!(block.difficulty);
add_bytes!(block.total_difficulty);
add_bytes!(block.extra_data);
add_bytes!(block.size);
add_bytes!(block.gas_limit);
add_bytes!(block.gas_used);
add_bytes!(block.timestamp);
add_bytes!(block.base_fee_per_gas);
add_bytes!(block.blob_gas_used);
add_bytes!(block.excess_blob_gas);
add_bytes!(block.parent_beacon_block_root);
add_bytes!(block.withdrawals_root);
add_bytes!(block.send_count);
add_bytes!(block.send_root);
add_bytes!(block.mix_hash);
if let Some(uncles) = &block.uncles {
observed = observed
.saturating_add(std::mem::size_of::<Hash>().saturating_mul(uncles.capacity()));
check_dynamic_bytes(observed, limit)?;
for uncle in uncles {
observed = observed.saturating_add(uncle.as_ref().len());
check_dynamic_bytes(observed, limit)?;
}
}
if let Some(withdrawals) = &block.withdrawals {
observed = observed.saturating_add(
std::mem::size_of::<hypersync_client::format::Withdrawal>()
.saturating_mul(withdrawals.capacity()),
);
check_dynamic_bytes(observed, limit)?;
for withdrawal in withdrawals {
add_bytes!(withdrawal.index);
add_bytes!(withdrawal.validator_index);
add_bytes!(withdrawal.address);
add_bytes!(withdrawal.amount);
}
}
}
for log in &page.logs {
add_bytes!(log.transaction_hash);
add_bytes!(log.block_hash);
add_bytes!(log.address);
add_bytes!(log.data);
for topic in &log.topics {
add_bytes!(topic);
}
}
if let Some(guard) = &page.rollback_guard {
observed = observed
.saturating_add(guard.hash.as_ref().len())
.saturating_add(guard.first_parent_hash.as_ref().len());
check_dynamic_bytes(observed, limit)?;
}
Ok(())
}
fn check_dynamic_bytes(observed: usize, limit: usize) -> Result<(), SourceEngineError> {
if observed > limit {
return Err(SourceEngineError::ResponseLimitExceeded {
resource: "dynamic bytes",
observed,
limit,
});
}
Ok(())
}
fn rollback_guard_at(block: &BlockRef) -> Result<RollbackGuard, SourceEngineError> {
let hash: [u8; 32] =
block
.hash
.as_slice()
.try_into()
.map_err(|_| CanonicalError::InvalidHashWidth {
field: "hash",
width: block.hash.len(),
})?;
let parent_hash: [u8; 32] =
block
.parent_hash
.as_slice()
.try_into()
.map_err(|_| CanonicalError::InvalidHashWidth {
field: "parent_hash",
width: block.parent_hash.len(),
})?;
Ok(RollbackGuard {
block_number: block.number,
timestamp: i64::try_from(block.timestamp)
.map_err(|_| SourceEngineError::BlockTimestampOverflow(block.timestamp))?,
hash: Hash::from(hash),
first_block_number: block.number,
first_parent_hash: Hash::from(parent_hash),
})
}
fn validate_resume_state(
canonical: &CanonicalTracker,
next_block: u64,
provider_checkpoint: Option<RollbackGuard>,
) -> Result<Option<RollbackGuard>, SourceEngineError> {
if provider_checkpoint
.as_ref()
.is_some_and(|guard| guard.first_block_number > guard.block_number)
{
return Err(SourceEngineError::ResumeGuardMismatch("first_block_number"));
}
let Some(tip) = canonical.tip() else {
if let Some(guard) = provider_checkpoint {
if guard.timestamp < 0 {
return Err(SourceEngineError::ResumeGuardMismatch("timestamp"));
}
let expected = guard
.block_number
.checked_add(1)
.ok_or(SourceEngineError::BlockNumberOverflow)?;
if next_block != expected {
return Err(SourceEngineError::ResumeCursorMismatch {
expected,
received: next_block,
});
}
return Ok(Some(guard));
}
return Ok(None);
};
let expected = tip
.number
.checked_add(1)
.ok_or(SourceEngineError::BlockNumberOverflow)?;
if next_block != expected {
return Err(SourceEngineError::ResumeCursorMismatch {
expected,
received: next_block,
});
}
let Some(guard) = provider_checkpoint else {
return rollback_guard_at(tip).map(Some);
};
if guard.block_number != tip.number {
return Err(SourceEngineError::ResumeGuardMismatch("block_number"));
}
if guard.hash.as_ref() != tip.hash {
return Err(SourceEngineError::ResumeGuardMismatch("hash"));
}
let timestamp = i64::try_from(tip.timestamp)
.map_err(|_| SourceEngineError::BlockTimestampOverflow(tip.timestamp))?;
if guard.timestamp != timestamp {
return Err(SourceEngineError::ResumeGuardMismatch("timestamp"));
}
Ok(Some(guard))
}
type PreparedPage = (
Delivery,
Option<RollbackGuard>,
CanonicalTracker,
Option<ReorgDelta>,
);
#[derive(Clone, Copy)]
struct QueryLimits {
max_blocks: usize,
max_logs: usize,
}
impl QueryLimits {
fn for_range(from_block: u64, to_block_excl: u64) -> Self {
let span = to_block_excl.saturating_sub(from_block);
let max_blocks = usize::try_from(span)
.unwrap_or(MAX_BLOCKS_PER_QUERY)
.clamp(1, MAX_BLOCKS_PER_QUERY);
Self {
max_blocks,
max_logs: MAX_LOGS_PER_QUERY,
}
}
fn shrink(&mut self) -> bool {
if self.max_blocks == 1 && self.max_logs == 1 {
return false;
}
self.max_blocks = self.max_blocks.div_ceil(2).max(1);
self.max_logs = self.max_logs.div_ceil(2).max(1);
true
}
}
fn set_delivery_sequence(delivery: &mut Delivery, sequence: u64) -> Result<(), SourceEngineError> {
delivery.sequence = sequence;
delivery.delivery_token = sequence.to_be_bytes().to_vec();
delivery
.cursor
.as_mut()
.ok_or(SourceEngineError::MissingCursor)?
.batch_sequence = sequence;
Ok(())
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
#[non_exhaustive]
pub enum SourceError {
#[error("chain data source request failed: {0}")]
Request(String),
#[error("chain data source archive height is unavailable")]
UnavailableHeight,
#[error("chain data source request timed out after {millis}ms")]
RequestTimeout {
millis: u128,
},
#[error("chain data source request timeout must be greater than zero")]
InvalidRequestTimeout,
#[error("managed source resident-session limit must be greater than zero")]
InvalidSessionLimit,
#[error("managed source reorg depth must be greater than zero")]
InvalidReorgDepth,
#[error("source delivery size limit {requested} must be within 1..={maximum} bytes")]
InvalidDeliverySizeLimit {
requested: usize,
maximum: usize,
},
#[error("source response limit for {resource} must be greater than zero")]
InvalidResponseLimit {
resource: &'static str,
},
}
impl SourceError {
pub fn request(message: impl Into<String>) -> Self {
Self::Request(message.into())
}
pub const fn unavailable_height() -> Self {
Self::UnavailableHeight
}
pub fn request_timeout(timeout: std::time::Duration) -> Self {
Self::RequestTimeout {
millis: timeout.as_millis(),
}
}
}
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum SourceEngineError {
#[error(transparent)]
QueryPlan(#[from] QueryPlanError),
#[error(transparent)]
Source(#[from] SourceError),
#[error(transparent)]
Normalize(#[from] NormalizeError),
#[error(transparent)]
InvalidPage(#[from] SourcePageError),
#[error(transparent)]
Canonical(#[from] CanonicalError),
#[error("source page made no progress from {from_block}: next_block={next_block}")]
NoProgress {
from_block: u64,
next_block: u64,
},
#[error("no delivery is awaiting acknowledgement")]
NoPendingDelivery,
#[error("delivery token does not match the in-flight batch")]
DeliveryTokenMismatch,
#[error("delivered batch is missing its cursor")]
MissingCursor,
#[error("rollback recovery is missing its previous continuity guard")]
MissingRollbackGuard,
#[error(
"owner catch-up fork before activation block {activation_block} cannot be emitted as a global reorg (common ancestor {common_ancestor}, old tip {old_tip}, new tip {new_tip})"
)]
OwnerCatchupReorg {
activation_block: u64,
common_ancestor: u64,
old_tip: u64,
new_tip: u64,
},
#[error("cannot acknowledge an external control while a source delivery is pending")]
PendingDelivery,
#[error("delivery sequence regressed from {committed} to {received}")]
SequenceRegression {
committed: u64,
received: u64,
},
#[error("resume cursor is {received}, expected canonical successor {expected}")]
ResumeCursorMismatch {
expected: u64,
received: u64,
},
#[error("resume provider guard conflicts with canonical `{0}`")]
ResumeGuardMismatch(&'static str),
#[error(
"resume scan cursor is {received}, beyond preserved coverage successor {coverage_successor}"
)]
CoverageCursorMismatch {
coverage_successor: u64,
received: u64,
},
#[error("rescanned canonical identity conflicts with preserved coverage at block {number}")]
CoverageBoundaryConflict {
number: u64,
},
#[error("delivery sequence is exhausted")]
SequenceExhausted,
#[error("canonical block number cannot advance beyond u64::MAX")]
BlockNumberOverflow,
#[error("canonical block timestamp {0} does not fit in i64")]
BlockTimestampOverflow(u64),
#[error("delivery size limit {requested} must be within 1..={maximum} bytes")]
InvalidDeliverySizeLimit {
requested: usize,
maximum: usize,
},
#[error("encoded delivery is {encoded_len} bytes, exceeding the {limit}-byte limit")]
DeliveryTooLarge {
encoded_len: usize,
limit: usize,
},
#[error(
"provider response contains {observed} {resource}, exceeding the hard limit of {limit}"
)]
ResponseLimitExceeded {
resource: &'static str,
observed: usize,
limit: usize,
},
}