use std::collections::VecDeque;
use std::future::Future;
use std::sync::Once;
use std::time::Duration;
use alloy::{
network::Ethereum,
primitives::B256,
providers::{DynProvider, Provider},
rpc::types::{BlockNumberOrTag, Filter, Header, Log},
};
use async_stream::stream;
use futures_util::{StreamExt, TryStreamExt, pin_mut, stream::Stream};
use tokio::time::{Instant, sleep, sleep_until};
use tracing::{debug, info, warn};
use crate::rpc::cursor::{TipGate, TipWaitOutcome, next_cursor_after_batch, wait_for_tip};
use crate::rpc::error_tracking::{
ErrorCategory, ProviderErrorTracker, find_active_provider, truncate_error, truncate_error_short,
};
use crate::rpc::{
LogRange, RpcError,
config::{CaptureTarget, ProviderSettings},
progress::ScanProgress,
};
const MAX_BLOCKS_PER_WS_REQUEST: u64 = 300;
const IDLE_HEAD_AWAIT_SECS: u64 = 120;
const REORG_RING_TOP_UP_STEPS: u64 = 8;
const SELF_HEAL_WALK_MAX_BLOCKS: u64 = 1_024;
#[derive(Debug, Clone)]
struct HeadCoalescer {
block_cursor: Option<u64>,
latest_seen_head: Option<u64>,
flush_deadline: Option<Instant>,
window: Duration,
progress: Option<ScanProgress>,
}
impl HeadCoalescer {
fn new(block_cursor: Option<u64>, window: Duration) -> Self {
Self {
block_cursor,
latest_seen_head: None,
flush_deadline: None,
window,
progress: None,
}
}
fn with_progress(mut self, progress: Option<ScanProgress>) -> Self {
self.progress = progress;
self
}
fn record_tip(&self, tip: u64) {
if let Some(p) = &self.progress {
p.record_observed_tip(tip);
}
}
fn block_cursor(&self) -> Option<u64> {
self.block_cursor
}
fn observe_head(&mut self, head_number: u64, now: Instant) -> bool {
let from_block = self.block_cursor.unwrap_or(head_number);
if head_number < from_block {
return false;
}
self.latest_seen_head = Some(
self.latest_seen_head
.map_or(head_number, |latest| latest.max(head_number)),
);
if self.flush_deadline.is_none() {
self.flush_deadline = Some(now + self.window);
}
true
}
fn pending_range(&self) -> Option<(u64, u64)> {
let to_block = self.latest_seen_head?;
let from_block = self.block_cursor.unwrap_or(to_block);
(to_block >= from_block).then_some((from_block, to_block))
}
fn flush_deadline(&self) -> Option<Instant> {
self.pending_range()?;
self.flush_deadline
}
fn should_flush(&self, now: Instant) -> bool {
let Some((from_block, to_block)) = self.pending_range() else {
return false;
};
if to_block - from_block + 1 >= MAX_BLOCKS_PER_WS_REQUEST {
return true;
}
self.flush_deadline.is_some_and(|deadline| now >= deadline)
}
fn advance_cursor(&mut self, next_block: u64) {
self.block_cursor = Some(next_block);
if let Some(p) = &self.progress {
p.record_scanned_to(next_block.saturating_sub(1));
}
if self
.latest_seen_head
.is_some_and(|latest| next_block > latest)
{
self.latest_seen_head = None;
self.flush_deadline = None;
}
}
fn defer_flush(&mut self) {
self.flush_deadline = None;
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum LinkOutcome {
Linked,
Reorg { common_ancestor: u64 },
Detached {
parent_hash: B256,
child_number: u64,
},
Gap { ring_tip: u64 },
NoRecordedParent,
}
#[derive(Debug, Clone)]
struct HeadLinkage {
ring: VecDeque<(u64, B256)>,
depth: u64,
}
impl HeadLinkage {
fn new(depth: u64) -> Self {
Self {
ring: VecDeque::new(),
depth,
}
}
fn enabled(&self) -> bool {
self.depth > 0
}
fn floor(&self) -> Option<u64> {
self.ring.front().map(|&(number, _)| number)
}
fn lowest_worth_recording(&self) -> Option<u64> {
let (latest, _) = self.latest()?;
Some(latest.saturating_sub(self.depth.saturating_sub(1)))
}
fn recovery_window_depth(&self) -> u64 {
self.depth
}
fn hash_at(&self, number: u64) -> Option<B256> {
let &(front, _) = self.ring.front()?;
let offset = number.checked_sub(front)? as usize;
self.ring.get(offset).map(|&(_, hash)| hash)
}
fn matches(&self, number: u64, hash: B256) -> bool {
self.hash_at(number) == Some(hash)
}
fn push(&mut self, number: u64, hash: B256) {
self.ring.push_back((number, hash));
while self.ring.len() as u64 > self.depth {
self.ring.pop_front();
}
}
fn extend_downward(&mut self, pairs: &[(u64, B256)]) {
if !self.enabled() {
return;
}
let Some(front) = self.floor() else {
return; };
let deepest = self.lowest_worth_recording().unwrap_or(front);
let below: Vec<(u64, B256)> = pairs
.iter()
.copied()
.filter(|&(n, _)| (deepest..front).contains(&n))
.collect();
let Some(&(top, _)) = below.last() else {
return; };
let contiguous =
top + 1 == front && below.windows(2).all(|pair| pair[1].0 == pair[0].0 + 1);
if !contiguous {
return;
}
for &(number, hash) in below.iter().rev() {
self.ring.push_front((number, hash));
}
}
fn latest(&self) -> Option<(u64, B256)> {
self.ring.back().copied()
}
fn reanchor(&mut self, ancestor: u64, number: u64, hash: B256) {
while self.ring.back().is_some_and(|&(n, _)| n > ancestor) {
self.ring.pop_back();
}
self.push(number, hash);
}
fn observe(&mut self, number: u64, hash: B256, parent_hash: B256) -> LinkOutcome {
if !self.enabled() {
return LinkOutcome::Linked;
}
let Some(&(tip_number, tip_hash)) = self.ring.back() else {
self.push(number, hash);
return LinkOutcome::Linked;
};
if number == tip_number + 1 && parent_hash == tip_hash {
self.push(number, hash);
return LinkOutcome::Linked;
}
if self.hash_at(number) == Some(hash) {
return LinkOutcome::Linked;
}
if number > tip_number + 1 {
return LinkOutcome::Gap {
ring_tip: tip_number,
};
}
let Some(parent_number) = number.checked_sub(1) else {
return LinkOutcome::NoRecordedParent;
};
match self.hash_at(parent_number) {
Some(canonical_parent) if canonical_parent == parent_hash => {
self.reanchor(parent_number, number, hash);
LinkOutcome::Reorg {
common_ancestor: parent_number,
}
}
Some(_) => LinkOutcome::Detached {
parent_hash,
child_number: number,
},
None => LinkOutcome::NoRecordedParent,
}
}
}
async fn get_block_by_hash_with_http_fallback(
caller: &str,
providers: &ProviderSettings,
reused: Option<&[DynProvider<Ethereum>]>,
start_index: usize,
skip_first: bool,
hash: B256,
is_match: impl Fn(&Header) -> bool,
) -> Result<(usize, Box<Header>), RpcError> {
let n = providers.http_providers.len();
let mut attempted_hosts: Vec<String> = Vec::new();
let mut last_err = String::new();
for step in usize::from(skip_first)..n {
let idx = fallback_http_index(start_index, step, n);
let host = providers.http_settings(idx).host();
let http_provider = resolve_http_provider(providers, reused, idx);
let outcome = match http_provider.get_block_by_hash(hash).await {
Ok(Some(block)) if is_match(&block.header) => return Ok((idx, Box::new(block.header))),
Ok(Some(_)) => "answered a non-matching block".to_string(),
Ok(None) => "has no block".to_string(),
Err(e) => truncate_error_short(e, 100),
};
warn!("{caller}: provider {host} did not answer for block {hash}: {outcome}");
last_err = outcome;
attempted_hosts.push(host);
}
Err(RpcError::ConnectionError(if attempted_hosts.is_empty() {
format!("{caller}: no provider left to try for block {hash}")
} else {
format!(
"{caller}: no provider answered for block {hash} after trying [{}]: {}",
attempted_hosts.join(", "),
last_err
)
}))
}
fn provider_set_preconditions(
n: usize,
reused: Option<&[DynProvider<Ethereum>]>,
) -> Option<RpcError> {
if n == 0 {
return Some(RpcError::ConnectionError(
"no HTTP providers configured to resolve reorg-walk steps".to_string(),
));
}
if let Some(set) = reused
&& set.len() != n
{
return Some(RpcError::ConnectionError(format!(
"reused HTTP client count {} does not match configured provider count {n}",
set.len(),
)));
}
None
}
pub(crate) async fn collect_window_hashes(
provider: &DynProvider<Ethereum>,
providers: &ProviderSettings,
reused: Option<&[DynProvider<Ethereum>]>,
primary_index: usize,
tip_hash: B256,
tip_number: u64,
from_block: u64,
) -> Result<Vec<(u64, B256)>, RpcError> {
let n = providers.http_providers.len();
if let Some(err) = provider_set_preconditions(n, reused) {
return Err(err);
}
if tip_number < from_block {
return Ok(Vec::new());
}
let primary_index = primary_index % n;
let mut collected: Vec<(u64, B256)> = Vec::new();
let mut current_hash = tip_hash;
let mut current_number = tip_number;
loop {
collected.push((current_number, current_hash));
if current_number == from_block {
break;
}
let hash = current_hash;
let header = match provider.get_block_by_hash(hash).await {
Ok(Some(block))
if block.header.hash == hash && block.header.number == current_number =>
{
Box::new(block.header)
}
_ => {
let (_, header) = get_block_by_hash_with_http_fallback(
"collect_window_hashes",
providers,
reused,
primary_index,
true,
hash,
|h| h.hash == hash && h.number == current_number,
)
.await?;
header
}
};
let parent_hash = header.parent_hash;
let Some(parent_number) = current_number.checked_sub(1) else {
return Err(RpcError::Gap);
};
if parent_number < from_block {
break;
}
current_hash = parent_hash;
current_number = parent_number;
}
Ok(collected)
}
pub(crate) async fn fetch_logs_by_hash_range(
providers: &ProviderSettings,
pairs: &[(u64, B256)],
base_filter: Option<&Filter>,
) -> Result<Vec<LogRange>, RpcError> {
let mut ranges: Vec<LogRange> = Vec::with_capacity(pairs.len());
for &(number, hash) in pairs {
let query = base_filter
.cloned()
.unwrap_or_else(Filter::new)
.at_block_hash(hash);
let (_, logs) =
get_logs_with_http_fallback_retry(providers, &query, 0, number, number, None).await?;
ranges.push(LogRange {
from_block: number,
to_block: number,
logs,
tip_hash: Some(hash),
});
}
Ok(ranges)
}
async fn bound_flush_time<F, T>(max_secs: u64, label: &str, fut: F) -> Result<T, RpcError>
where
F: Future<Output = Result<T, RpcError>>,
{
if max_secs == 0 {
return fut.await;
}
match tokio::time::timeout(Duration::from_secs(max_secs), fut).await {
Ok(result) => result,
Err(_) => Err(RpcError::ConnectionError(format!(
"{label}: flush exceeded its {max_secs}s aggregate timeout — provider \
likely stalled mid-flush; switching provider"
))),
}
}
fn live_head_gap_event() -> &'static str {
"live_head_gap"
}
fn live_head_reorg_beyond_ring_event() -> &'static str {
"live_head_reorg_beyond_ring"
}
fn walk_failure_event(err: &RpcError) -> Option<&'static str> {
match err {
RpcError::ReorgBeyondRing => Some(live_head_reorg_beyond_ring_event()),
_ => None,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FlushFailure {
EndStream,
SwitchProvider,
}
fn classify_flush_failure(err: &RpcError) -> FlushFailure {
match err {
RpcError::Gap | RpcError::ReorgBeyondRing | RpcError::Reorg { .. } => {
FlushFailure::EndStream
}
RpcError::ConnectionError(_)
| RpcError::SubscriptionError(_)
| RpcError::LogFetchError(_)
| RpcError::TransportError(_)
| RpcError::CursorPastTip { .. }
| RpcError::ProviderStalled { .. }
| RpcError::AllProvidersSuspended(_)
| RpcError::AllProvidersIdle { .. }
| RpcError::Other(_) => FlushFailure::SwitchProvider,
}
}
fn flush_failure_log_line(err: &RpcError, failure: FlushFailure) -> String {
let disposition = match failure {
FlushFailure::EndStream => "ending stream for consumer recovery",
FlushFailure::SwitchProvider => "retrying the same window",
};
format!(
"by-hash window flush failed: {} — {disposition}",
truncate_error(err)
)
}
fn flush_timeout_label(providers: &ProviderSettings, ws_host: &str) -> String {
let fallback = providers
.http_providers
.first()
.map(|settings| settings.host())
.unwrap_or_else(|| "<none>".to_string());
format!("ws={ws_host} fallback={fallback}")
}
async fn flush_window_range(
linkage: &mut HeadLinkage,
coalescer: &mut HeadCoalescer,
providers: &ProviderSettings,
filter: Option<&Filter>,
provider_index: usize,
reused: Option<&[DynProvider<Ethereum>]>,
) -> Result<Vec<LogRange>, RpcError> {
let Some((from_block, pending_to_block)) = coalescer.pending_range() else {
return Ok(vec![]);
};
if providers.http_providers.is_empty() {
return Err(RpcError::ConnectionError(
"No HTTP providers configured for window flush".to_string(),
));
}
let to_block = std::cmp::min(from_block + MAX_BLOCKS_PER_WS_REQUEST - 1, pending_to_block);
let pairs: Vec<(u64, B256)> = if (from_block..=to_block).all(|n| linkage.hash_at(n).is_some()) {
(from_block..=to_block)
.map(|n| (n, linkage.hash_at(n).expect("just verified")))
.collect()
} else {
let Some((top_n, top_hash)) = linkage.latest() else {
return Err(RpcError::Gap);
};
let walk_blocks = top_n.saturating_sub(from_block) + 1;
let walk_bound = SELF_HEAL_WALK_MAX_BLOCKS.saturating_add(linkage.recovery_window_depth());
if walk_blocks > walk_bound {
warn!(
"flush window [{}, {}] would need a {}-block self-heal walk from ring anchor \
{} — beyond the {}-block bound; ending the flush rather than walking it",
from_block, to_block, walk_blocks, top_n, walk_bound
);
return Err(RpcError::Gap);
}
let walk_provider = resolve_http_provider(providers, reused, provider_index);
warn!(
"flush window [{}, {}] missing ring entries — self-heal walking back up \
to {} blocks from ring anchor {} to {} ({})",
from_block,
to_block,
top_n.saturating_sub(from_block) + 1,
top_n,
from_block,
if top_n < from_block {
"ring anchor already trails the window's start — the walk \
immediately no-ops, nothing is fetched"
} else {
"uses the same client-reuse mechanism as the main fetch path when \
available, but the walk itself is sequential; not covered by the \
concurrent fetch path"
}
);
let mut pairs_desc = collect_window_hashes(
&walk_provider,
providers,
reused,
provider_index,
top_hash,
top_n,
from_block,
)
.await?;
pairs_desc.reverse();
linkage.extend_downward(&pairs_desc);
pairs_desc
.into_iter()
.filter(|&(n, _)| n <= to_block)
.collect()
};
let last_pending = pairs.last().copied();
let bound = providers.http_settings(provider_index).max_concurrency as usize;
let fetch = move |(n, hash): (u64, B256)| async move {
let query = filter
.cloned()
.unwrap_or_else(Filter::new)
.at_block_hash(hash);
let (_, logs) =
get_logs_with_http_fallback_retry(providers, &query, provider_index, n, n, reused)
.await?;
Ok::<LogRange, RpcError>(LogRange {
from_block: n,
to_block: n,
logs,
tip_hash: Some(hash),
})
};
let ranges = sort_ranges_ascending(fetch_concurrently(pairs, bound, fetch).await?);
top_up_reorg_ring(linkage, providers, provider_index, reused).await;
if let Some((last, _)) = last_pending {
coalescer.advance_cursor(last + 1);
}
Ok(ranges)
}
async fn top_up_reorg_ring(
linkage: &mut HeadLinkage,
providers: &ProviderSettings,
provider_index: usize,
reused: Option<&[DynProvider<Ethereum>]>,
) {
let (Some(floor), Some(target)) = (linkage.floor(), linkage.lowest_worth_recording()) else {
return;
};
if floor <= target {
return;
}
let floor_hash = linkage
.hash_at(floor)
.expect("the ring's own floor is always recorded at its own number");
let walk_from = target.max(floor.saturating_sub(REORG_RING_TOP_UP_STEPS));
let walk_provider = resolve_http_provider(providers, reused, provider_index);
match collect_window_hashes(
&walk_provider,
providers,
reused,
provider_index,
floor_hash,
floor,
walk_from,
)
.await
{
Ok(mut pairs_desc) => {
pairs_desc.reverse();
linkage.extend_downward(&pairs_desc);
info!(
"reorg ring top-up: floor {floor} -> {} (target {target})",
linkage.floor().unwrap_or(floor)
);
}
Err(err) => {
debug!("reorg ring top-up failed, will retry next flush: {err}");
}
}
}
#[allow(clippy::manual_async_fn)]
fn fetch_concurrently<In, Out, F, Fut>(
items: Vec<In>,
bound: usize,
fetch: F,
) -> impl Future<Output = Result<Vec<Out>, RpcError>> + Send
where
In: Send,
Out: Send,
F: Fn(In) -> Fut + Send,
Fut: Future<Output = Result<Out, RpcError>> + Send,
{
async move {
futures_util::stream::iter(items.into_iter().map(fetch).map(Ok::<Fut, RpcError>))
.try_buffer_unordered(bound.max(1))
.try_collect()
.await
}
}
fn sort_ranges_ascending(mut ranges: Vec<LogRange>) -> Vec<LogRange> {
ranges.sort_by_key(|range| range.from_block);
ranges
}
async fn walk_to_common_ancestor(
provider: &DynProvider<Ethereum>,
providers: &ProviderSettings,
reused: Option<&[DynProvider<Ethereum>]>,
linkage: &HeadLinkage,
detached_parent: B256,
child_number: u64,
depth: u64,
) -> Result<u64, RpcError> {
let n = providers.http_providers.len();
if let Some(err) = provider_set_preconditions(n, reused) {
return Err(err);
}
let mut parent_hash = detached_parent;
let Some(mut parent_number) = child_number.checked_sub(1) else {
return Err(RpcError::Gap);
};
let floor = linkage.floor().unwrap_or(u64::MAX);
let mut confirmed_a_step = false;
let mut http_cursor = 0;
for _ in 0..depth {
if linkage.matches(parent_number, parent_hash) {
return Ok(parent_number);
}
if parent_number <= floor {
return if parent_number == floor && confirmed_a_step {
Err(RpcError::ReorgBeyondRing)
} else {
Err(RpcError::Gap)
};
}
let hash = parent_hash;
let header = match provider.get_block_by_hash(hash).await {
Ok(Some(block)) if block.header.hash == hash => Box::new(block.header),
_ => {
let (answered_index, header) = get_block_by_hash_with_http_fallback(
"walk_to_common_ancestor",
providers,
reused,
http_cursor,
false,
hash,
|h| h.hash == hash,
)
.await?;
http_cursor = answered_index;
header
}
};
parent_hash = header.parent_hash;
confirmed_a_step = true;
let Some(next) = parent_number.checked_sub(1) else {
return Err(RpcError::Gap);
};
parent_number = next;
}
Err(RpcError::Gap)
}
fn fallback_http_index(start_index: usize, attempt: usize, num_http_providers: usize) -> usize {
(start_index + attempt) % num_http_providers
}
fn fallback_http_exhausted_error(
from_block: u64,
to_block: u64,
attempted_hosts: &[String],
last_err: &str,
) -> RpcError {
RpcError::LogFetchError(format!(
"HTTP fallback exhausted for blocks {}-{} after trying providers [{}]: {}",
from_block,
to_block,
attempted_hosts.join(", "),
last_err
))
}
pub(crate) fn resolve_http_provider(
providers: &ProviderSettings,
reused: Option<&[DynProvider<Ethereum>]>,
idx: usize,
) -> DynProvider<Ethereum> {
assert!(
!providers.http_providers.is_empty(),
"resolve_http_provider: no HTTP providers configured"
);
match reused {
Some(set) => set[idx % set.len()].clone(),
None => providers.connect_http(idx),
}
}
async fn get_logs_with_http_fallback_retry(
providers: &ProviderSettings,
query_filter: &Filter,
start_index: usize,
from_block: u64,
to_block: u64,
reused: Option<&[DynProvider<Ethereum>]>,
) -> Result<(usize, Vec<alloy::rpc::types::Log>), RpcError> {
let num_http_providers = providers.http_providers.len();
let mut attempted_hosts: Vec<String> = Vec::with_capacity(num_http_providers);
let mut last_err = String::new();
for attempt in 0..num_http_providers {
let provider_idx = fallback_http_index(start_index, attempt, num_http_providers);
let provider_host = providers.http_settings(provider_idx).host();
attempted_hosts.push(provider_host.clone());
let http_provider = resolve_http_provider(providers, reused, provider_idx);
match http_provider.get_logs(query_filter).await {
Ok(logs) => return Ok((provider_idx, logs)),
Err(err) => {
last_err = err.to_string();
warn!(
"HTTP fallback {} failed for blocks {}-{}: {}",
provider_host,
from_block,
to_block,
truncate_error_short(&last_err, 100)
);
}
}
}
Err(fallback_http_exhausted_error(
from_block,
to_block,
&attempted_hosts,
&last_err,
))
}
enum YieldBounds {
Full,
Partial(u64),
Unattributable,
}
fn yield_bounds(logs: &[Log], to_block: u64) -> YieldBounds {
let Some(watermark) = logs.iter().filter_map(|log| log.block_number).max() else {
return YieldBounds::Unattributable;
};
if watermark >= to_block {
YieldBounds::Full
} else {
YieldBounds::Partial(watermark)
}
}
enum EmptyFlushOutcome {
Deferred,
VerifiedEmpty,
CaughtUp,
Stalled {
to_block: u64,
reported_tip: u64,
stalled_cycles: u32,
},
RepollFailed(String),
TipFetchFailed(String),
}
async fn handle_empty_flush<F, Fut, E>(
coalescer: &mut HeadCoalescer,
gate: &mut TipGate,
providers: &ProviderSettings,
host: &str,
to_block: u64,
span_capped: bool,
get_tip: F,
) -> EmptyFlushOutcome
where
F: Fn() -> Fut,
Fut: Future<Output = Result<u64, E>>,
E: std::fmt::Display,
{
if !span_capped {
coalescer.defer_flush();
return EmptyFlushOutcome::Deferred;
}
let reported_tip = match get_tip().await {
Ok(tip) => tip,
Err(e) => return EmptyFlushOutcome::TipFetchFailed(e.to_string()),
};
coalescer.record_tip(reported_tip);
if next_cursor_after_batch(to_block, true, reported_tip).is_err() {
match wait_for_tip(to_block, reported_tip, gate, providers, host, get_tip).await {
TipWaitOutcome::CaughtUp => EmptyFlushOutcome::CaughtUp,
TipWaitOutcome::Terminal {
to_block,
reported_tip,
stalled_cycles,
} => EmptyFlushOutcome::Stalled {
to_block,
reported_tip,
stalled_cycles,
},
TipWaitOutcome::RepollFailed(err) => EmptyFlushOutcome::RepollFailed(err),
}
} else {
EmptyFlushOutcome::VerifiedEmpty
}
}
enum IdleHeadAwait {
Header(Box<Header>),
StreamEnded,
Expired,
}
async fn await_head_within_idle_bound<S>(s: &mut S) -> IdleHeadAwait
where
S: Stream<Item = Header> + Unpin,
{
match tokio::time::timeout(Duration::from_secs(IDLE_HEAD_AWAIT_SECS), s.next()).await {
Ok(Some(header)) => IdleHeadAwait::Header(Box::new(header)),
Ok(None) => IdleHeadAwait::StreamEnded,
Err(_) => IdleHeadAwait::Expired,
}
}
struct IdleRotationCounter {
consecutive_expiries: usize,
}
enum IdleRotation {
Continue,
AllIdle,
}
impl IdleRotationCounter {
fn new() -> Self {
Self {
consecutive_expiries: 0,
}
}
fn reset(&mut self) {
self.consecutive_expiries = 0;
}
fn expire(&mut self, threshold: usize) -> IdleRotation {
self.consecutive_expiries += 1;
if self.consecutive_expiries >= threshold.max(1) {
IdleRotation::AllIdle
} else {
IdleRotation::Continue
}
}
}
async fn stream_heads_provider(
provider: &DynProvider<Ethereum>,
provider_host: &str,
) -> impl Stream<Item = Header> + Send {
stream! {
let head_stream = provider.subscribe_blocks().await;
match head_stream {
Ok(heads) => {
debug!("WebSocket connected, streaming headers...");
let mut s = heads.into_stream();
while let Some(header) = s.next().await {
yield header;
}
}
Err(e) => {
warn!(
"Failed to subscribe to blocks on {}: {} - retrying",
provider_host,
truncate_error_short(e, 100)
);
}
}
}
}
pub async fn stream_heads_with_logs_with_progress(
providers: &ProviderSettings,
filter: Option<&Filter>,
progress: Option<&ScanProgress>,
) -> impl Stream<Item = Result<LogRange, RpcError>> + Send {
let num_providers = providers.wss_endpoints.len();
let progress = progress.cloned();
stream! {
let mut provider_trackers: Vec<ProviderErrorTracker> = (0..num_providers)
.map(|i| {
let host = providers.wss_endpoints[i]
.host_str()
.unwrap_or("unknown")
.to_string();
ProviderErrorTracker::new(host)
})
.collect();
let reused_http_clients = providers.build_reused_http_clients();
let mut provider_index = 0;
let mut coalescer = HeadCoalescer::new(
filter.and_then(|f| f.get_from_block()),
Duration::from_millis(providers.live_head_coalesce_window_ms),
)
.with_progress(progress);
let mut linkage = HeadLinkage::new(providers.recovery_window_depth);
let mut gate = TipGate::new();
let mut idle_rotation = IdleRotationCounter::new();
loop {
let active_provider = find_active_provider(&mut provider_trackers, provider_index);
let Some(idx) = active_provider else {
yield Err(RpcError::AllProvidersSuspended(format!(
"All {} WebSocket providers suspended. Unable to continue streaming.",
num_providers
)));
return;
};
provider_index = idx;
let provider = match providers.connect_ws(provider_index).await {
Ok(p) => {
provider_trackers[provider_index].record_success();
p
}
Err(e) => {
let suspended = provider_trackers[provider_index].record_error();
let backoff = provider_trackers[provider_index].backoff_duration();
if suspended {
warn!(
"WebSocket provider {} suspended after connection failure: {}",
provider_trackers[provider_index].identifier(),
truncate_error_short(&e, 100)
);
} else {
warn!(
"Failed to connect to WebSocket {}: {} - retrying in {:?}",
provider_trackers[provider_index].identifier(),
truncate_error_short(&e, 100),
backoff
);
sleep(backoff).await;
}
provider_index = (provider_index + 1) % num_providers;
continue;
}
};
gate.reset();
let provider_host = provider_trackers[provider_index].identifier().to_string();
let s = stream_heads_provider(&provider, &provider_host).await;
pin_mut!(s);
let mut should_switch_provider = false;
loop {
if coalescer.should_flush(Instant::now()) {
'flush: while let Some((from_block, pending_to_block)) = coalescer.pending_range() {
let to_block =
std::cmp::min(from_block + MAX_BLOCKS_PER_WS_REQUEST - 1, pending_to_block);
if linkage.enabled() {
let flush_timeout_label =
flush_timeout_label(providers, &provider_host);
match bound_flush_time(
providers.flush_max_total_secs,
&flush_timeout_label,
flush_window_range(
&mut linkage,
&mut coalescer,
providers,
filter,
0,
Some(&reused_http_clients),
),
)
.await
{
Ok(ranges) => {
for range in ranges {
provider_trackers[provider_index].record_success();
yield Ok(range);
}
}
Err(e) => {
let failure = classify_flush_failure(&e);
warn!("{}", flush_failure_log_line(&e, failure));
match failure {
FlushFailure::EndStream => {
yield Err(e);
return;
}
FlushFailure::SwitchProvider => {
should_switch_provider = true;
break 'flush;
}
}
}
}
continue;
}
let query_filter = filter
.cloned()
.unwrap_or_else(Filter::new)
.from_block(from_block)
.to_block(to_block);
let logs = provider.get_logs(&query_filter).await;
match logs {
Ok(logs) => {
provider_trackers[provider_index].record_success();
let watermark_bounds =
(!logs.is_empty()).then(|| yield_bounds(&logs, to_block));
if let Some(YieldBounds::Partial(watermark)) = watermark_bounds {
yield Ok(LogRange {
from_block,
to_block: watermark,
logs,
tip_hash: None,
});
coalescer.advance_cursor(watermark + 1);
gate.reset();
coalescer.defer_flush();
break 'flush;
}
let treat_as_empty =
!matches!(watermark_bounds, Some(YieldBounds::Full));
if treat_as_empty {
let span_capped =
to_block - from_block + 1 == MAX_BLOCKS_PER_WS_REQUEST;
match handle_empty_flush(
&mut coalescer,
&mut gate,
providers,
&provider_host,
to_block,
span_capped,
|| provider.get_block_number(),
)
.await
{
EmptyFlushOutcome::Deferred => break 'flush,
EmptyFlushOutcome::VerifiedEmpty => {}
EmptyFlushOutcome::CaughtUp => continue 'flush,
EmptyFlushOutcome::Stalled {
to_block,
reported_tip,
stalled_cycles,
} => {
yield Err(RpcError::ProviderStalled {
to_block,
reported_tip,
stalled_cycles,
});
return;
}
EmptyFlushOutcome::RepollFailed(err) => {
let suspended = provider_trackers
[provider_index]
.record_error();
if suspended {
warn!(
"WebSocket provider {} suspended after tip re-poll failed during tip-wait for blocks {}-{}: {}",
provider_trackers[provider_index].identifier(),
from_block,
to_block,
truncate_error_short(&err, 100),
);
} else {
let backoff = provider_trackers
[provider_index]
.backoff_duration();
warn!(
"Tip re-poll failed during wait for blocks {}-{} from {}: {} - backing off {:?}",
from_block,
to_block,
provider_trackers[provider_index].identifier(),
truncate_error_short(&err, 100),
backoff
);
sleep(backoff).await;
}
should_switch_provider = true;
break 'flush;
}
EmptyFlushOutcome::TipFetchFailed(err_str) => {
let suspended =
provider_trackers[provider_index].record_error();
if suspended {
warn!(
"WebSocket provider {} suspended after tip fetch failed during empty-batch guard for blocks {}-{}: {}",
provider_trackers[provider_index].identifier(),
from_block, to_block,
truncate_error_short(&err_str, 100)
);
} else {
let backoff = provider_trackers[provider_index].backoff_duration();
warn!(
"Tip fetch failed during empty-batch guard for blocks {}-{} from {}: {} - backing off {:?}",
from_block, to_block,
provider_trackers[provider_index].identifier(),
truncate_error_short(&err_str, 100),
backoff
);
sleep(backoff).await;
}
should_switch_provider = true;
break 'flush;
}
}
}
let logs = if treat_as_empty { Vec::new() } else { logs };
yield Ok(LogRange {
from_block,
to_block,
logs,
tip_hash: None,
});
coalescer.advance_cursor(to_block + 1);
gate.reset();
}
Err(e) => {
let error_msg = e.to_string();
let error_category = ErrorCategory::from_error_msg(&error_msg);
match error_category {
ErrorCategory::RangeTooWide => {
warn!(
"Range too wide for blocks {}-{}, falling back to HTTP RPC: {}",
from_block, to_block, truncate_error_short(&error_msg, 100)
);
match get_logs_with_http_fallback_retry(
providers,
&query_filter,
0,
from_block,
to_block,
Some(&reused_http_clients),
)
.await
{
Ok((http_provider_idx, logs)) => {
provider_trackers[provider_index].record_success();
let watermark_bounds = (!logs.is_empty())
.then(|| yield_bounds(&logs, to_block));
if let Some(YieldBounds::Partial(watermark)) =
watermark_bounds
{
yield Ok(LogRange {
from_block,
to_block: watermark,
logs,
tip_hash: None,
});
coalescer.advance_cursor(watermark + 1);
gate.reset();
coalescer.defer_flush();
break 'flush;
}
let treat_as_empty = !matches!(
watermark_bounds,
Some(YieldBounds::Full)
);
if treat_as_empty {
let http_provider = providers.connect_http(http_provider_idx);
let http_host = providers.http_settings(http_provider_idx).host();
match handle_empty_flush(
&mut coalescer,
&mut gate,
providers,
&http_host,
to_block,
true,
|| http_provider.get_block_number(),
)
.await
{
EmptyFlushOutcome::Deferred => {
unreachable!(
"span_capped: true never defers"
)
}
EmptyFlushOutcome::VerifiedEmpty => {}
EmptyFlushOutcome::CaughtUp => {
continue 'flush
}
EmptyFlushOutcome::Stalled {
to_block,
reported_tip,
stalled_cycles,
} => {
yield Err(
RpcError::ProviderStalled {
to_block,
reported_tip,
stalled_cycles,
},
);
return;
}
EmptyFlushOutcome::RepollFailed(err) => {
warn!(
"HTTP fallback {} tip re-poll failed during tip-wait for blocks {}-{}: {}",
http_host,
from_block,
to_block,
truncate_error_short(&err, 100),
);
should_switch_provider = true;
break 'flush;
}
EmptyFlushOutcome::TipFetchFailed(err) => {
warn!(
"HTTP fallback {} tip fetch failed during empty-batch guard for blocks {}-{}: {}",
http_host, from_block, to_block,
truncate_error_short(&err, 100)
);
should_switch_provider = true;
break 'flush;
}
}
}
let logs = if treat_as_empty {
Vec::new()
} else {
logs
};
yield Ok(LogRange {
from_block,
to_block,
logs,
tip_hash: None,
});
coalescer.advance_cursor(to_block + 1);
gate.reset();
}
Err(http_err) => {
warn!(
"HTTP fallback retry exhausted for blocks {}-{}: {}",
from_block,
to_block,
truncate_error_short(&http_err, 100)
);
should_switch_provider = true;
break;
}
}
}
ErrorCategory::RateLimit => {
provider_trackers[provider_index].record_error();
let backoff = provider_trackers[provider_index].backoff_duration();
warn!(
"Rate limit hit on WS provider {} for blocks {}-{}: {} - backing off {:?}",
provider_trackers[provider_index].identifier(),
from_block, to_block,
truncate_error_short(&error_msg, 100),
backoff
);
sleep(backoff).await;
continue;
}
ErrorCategory::Connection | ErrorCategory::Other => {
let suspended = provider_trackers[provider_index].record_error();
if suspended {
warn!(
"WebSocket provider {} suspended after error on blocks {}-{}: {}",
provider_trackers[provider_index].identifier(),
from_block, to_block,
truncate_error_short(&error_msg, 100)
);
} else {
let backoff = provider_trackers[provider_index].backoff_duration();
warn!(
"Error fetching logs for blocks {}-{} from {}: {} - backing off {:?}",
from_block, to_block,
provider_trackers[provider_index].identifier(),
truncate_error_short(&error_msg, 100),
backoff
);
sleep(backoff).await;
}
should_switch_provider = true;
break;
}
}
}
}
}
if should_switch_provider {
break;
}
continue;
}
let maybe_header = if let Some(deadline) = coalescer.flush_deadline() {
tokio::select! {
maybe_header = s.next() => maybe_header,
_ = sleep_until(deadline) => {
continue;
}
}
} else {
match await_head_within_idle_bound(&mut s).await {
IdleHeadAwait::Header(header) => Some(*header),
IdleHeadAwait::StreamEnded => None,
IdleHeadAwait::Expired => match idle_rotation.expire(num_providers) {
IdleRotation::Continue => {
warn!(
"No head from {} in {}s - idle bound expired, rotating provider",
provider_trackers[provider_index].identifier(),
IDLE_HEAD_AWAIT_SECS
);
should_switch_provider = true;
break;
}
IdleRotation::AllIdle => {
warn!(
"No head from any of {} configured provider(s) within {}s - a \
full quiet rotation",
num_providers, IDLE_HEAD_AWAIT_SECS
);
yield Err(RpcError::AllProvidersIdle {
providers_tried: num_providers,
idle_secs: IDLE_HEAD_AWAIT_SECS,
});
return;
}
},
}
};
match maybe_header {
Some(header) => {
idle_rotation.reset();
let now = Instant::now();
let head_number = header.number;
if !coalescer.observe_head(head_number, now) {
let from_block = coalescer.block_cursor().unwrap_or(head_number);
warn!(
"Skipping header {} as it's before the cursor {}",
head_number, from_block
);
}
match linkage.observe(head_number, header.hash, header.parent_hash) {
LinkOutcome::Linked => {}
LinkOutcome::Reorg { common_ancestor } => {
warn!(
"reorg at head {}: common ancestor {} — ending stream for consumer recovery",
head_number, common_ancestor
);
yield Err(RpcError::Reorg { common_ancestor });
return;
}
LinkOutcome::Gap { ring_tip } => {
warn!(
event = live_head_gap_event(),
ring_tip,
head = head_number,
gap = head_number.saturating_sub(ring_tip),
"live heads skipped ahead of the recovery ring — ending stream for consumer recovery"
);
yield Err(RpcError::Gap);
return;
}
LinkOutcome::Detached {
parent_hash,
child_number,
} => {
match walk_to_common_ancestor(
&provider,
providers,
Some(&reused_http_clients),
&linkage,
parent_hash,
child_number,
providers.recovery_window_depth,
)
.await
{
Ok(ancestor) => {
warn!(
"reorg at head {} (walked new chain): common ancestor {} — ending stream",
head_number, ancestor
);
yield Err(RpcError::Reorg { common_ancestor: ancestor });
return;
}
Err(err) => {
match walk_failure_event(&err) {
Some(event_name) => warn!(
event = event_name,
head = head_number,
"reorg walk at head {head_number} found a canonical hash disagreement beyond the recovery ring — ending stream for consumer recovery"
),
None => warn!(
"reorg walk at head {} ended the stream: {}",
head_number,
truncate_error(&err)
),
}
yield Err(err);
return;
}
}
}
LinkOutcome::NoRecordedParent => {
warn!(
"head {} has no canonical record at its parent's height — ending stream for consumer recovery",
head_number
);
yield Err(RpcError::Gap);
return;
}
}
}
None => break,
}
}
if !should_switch_provider {
provider_trackers[provider_index].record_error();
let backoff = provider_trackers[provider_index].backoff_duration();
warn!(
"WS connection to {} lost or stream ended - reconnecting in {:?}",
provider_trackers[provider_index].identifier(),
backoff
);
sleep(backoff).await;
}
provider_index = (provider_index + 1) % num_providers;
}
}
}
pub async fn stream_heads_with_logs(
providers: &ProviderSettings,
filter: Option<&Filter>,
) -> impl Stream<Item = Result<LogRange, RpcError>> + Send {
stream_heads_with_logs_with_progress(providers, filter, None).await
}
async fn poll_providers_for_block_number<F, Fut>(
providers: &ProviderSettings,
empty_set_phrase: &str,
attempt_phrase: &str,
fetch: F,
) -> Result<u64, RpcError>
where
F: Fn(DynProvider<Ethereum>) -> Fut,
Fut: Future<Output = Result<u64, String>>,
{
let num_providers = providers.http_providers.len();
if num_providers == 0 {
return Err(RpcError::ConnectionError(format!(
"No HTTP providers configured for {empty_set_phrase}"
)));
}
let mut index: usize = 0;
loop {
let provider_index = index % num_providers;
let provider = providers.connect_http(provider_index);
match fetch(provider).await {
Ok(block_number) => return Ok(block_number),
Err(e) => {
let provider_host = providers.http_settings(provider_index).host();
if index >= 10 {
return Err(RpcError::ConnectionError(format!(
"Failed to fetch {attempt_phrase} after {} attempts from HTTP provider {}: {}",
index, provider_host, e
)));
}
index += 1;
warn!(
"Failed to fetch {attempt_phrase} from HTTP provider {} (index {}) - retrying",
provider_host, provider_index
);
let progressive_sleep = (5 * index).min(50) as u64;
sleep(Duration::from_secs(progressive_sleep)).await;
}
}
}
}
pub async fn last_block(providers: &ProviderSettings) -> Result<u64, RpcError> {
poll_providers_for_block_number(
providers,
"latest block fetch",
"latest block number",
|provider| async move { provider.get_block_number().await.map_err(|e| e.to_string()) },
)
.await
}
struct CapturePolicyNotice {
latest: Once,
lag: Once,
finalized: Once,
}
impl CapturePolicyNotice {
const fn new() -> Self {
Self {
latest: Once::new(),
lag: Once::new(),
finalized: Once::new(),
}
}
fn first_message(&self, target: CaptureTarget, block: u64) -> Option<String> {
let gate = match target {
CaptureTarget::Latest => &self.latest,
CaptureTarget::Lag(_) => &self.lag,
CaptureTarget::Finalized => &self.finalized,
};
let mut won = false;
gate.call_once(|| won = true);
if !won {
return None;
}
Some(match target {
CaptureTarget::Latest => format!(
"ethl capture target is `Latest` (block {}): capturing to the raw chain \
tip with NO reorg protection — a reorg can rewrite recently captured \
blocks. Reorg-prone chains should opt into `Lag`/`Finalized`.",
block,
),
CaptureTarget::Lag(n) => format!(
"ethl DEEP LAG ACTIVE (reorg-safety): capturing to tip - {} = block {}.\n\
This build AVOIDS reorgs by lagging; events within {} blocks of tip are DELAYED.",
n, block, n,
),
CaptureTarget::Finalized => format!(
"ethl DEEP LAG ACTIVE (reorg-safety): capturing to `finalized` block {}.\n\
(~2 epochs ≈ 64 blocks ≈ 13 min behind chain tip).\n\
This build AVOIDS reorgs by lagging; events within ~64 blocks of tip are DELAYED.",
block
),
})
}
}
static CAPTURE_POLICY_NOTICE: CapturePolicyNotice = CapturePolicyNotice::new();
pub async fn resolve_capture_target(settings: &ProviderSettings) -> Result<u64, RpcError> {
match settings.capture_target {
CaptureTarget::Latest => {
let tip = last_block(settings).await?;
if let Some(message) = CAPTURE_POLICY_NOTICE.first_message(settings.capture_target, tip)
{
warn!("{message}");
}
Ok(tip)
}
CaptureTarget::Lag(n) => {
let tip = last_block(settings).await?;
let target_block = tip.saturating_sub(n);
if let Some(message) =
CAPTURE_POLICY_NOTICE.first_message(settings.capture_target, target_block)
{
warn!("{message}");
}
Ok(target_block)
}
CaptureTarget::Finalized => {
let finalized = poll_providers_for_block_number(
settings,
"finalized block fetch",
"finalized block number",
|provider| async move {
let result = provider
.get_block_by_number(BlockNumberOrTag::Finalized)
.await
.map_err(|e| e.to_string())?;
result.map(|block| block.header.number).ok_or_else(|| {
"Provider returned no finalized block; cannot determine safe capture target. \
Ensure the provider supports the `finalized` tag (mainnet only)."
.to_string()
})
},
)
.await?;
if let Some(message) =
CAPTURE_POLICY_NOTICE.first_message(settings.capture_target, finalized)
{
warn!("{message}");
}
Ok(finalized)
}
}
}
#[cfg(test)]
mod tests {
use super::{
CapturePolicyNotice, EmptyFlushOutcome, FlushFailure, HeadCoalescer, HeadLinkage,
IDLE_HEAD_AWAIT_SECS, IdleHeadAwait, IdleRotation, IdleRotationCounter, LinkOutcome,
MAX_BLOCKS_PER_WS_REQUEST, REORG_RING_TOP_UP_STEPS, RpcError, SELF_HEAL_WALK_MAX_BLOCKS,
TipGate, YieldBounds, await_head_within_idle_bound, bound_flush_time,
classify_flush_failure, collect_window_hashes, fallback_http_exhausted_error,
fallback_http_index, fetch_concurrently, fetch_logs_by_hash_range, flush_failure_log_line,
flush_timeout_label, flush_window_range, handle_empty_flush, live_head_gap_event,
live_head_reorg_beyond_ring_event, resolve_http_provider, sort_ranges_ascending,
walk_failure_event, walk_to_common_ancestor, yield_bounds,
};
use crate::rpc::LogRange;
use crate::rpc::config::{CaptureTarget, HttpRpcSettings, ProviderSettings};
use alloy::network::Ethereum;
use alloy::primitives::B256;
use alloy::providers::{DynProvider, Provider, ProviderBuilder};
use alloy::rpc::types::{Header, Log};
use alloy::transports::mock::Asserter;
use std::time::Duration;
use tokio::time::Instant;
const WINDOW: Duration = Duration::from_millis(500);
fn hb(byte: u8) -> B256 {
B256::repeat_byte(byte)
}
#[test]
fn fallback_http_index_wraps_round_robin() {
assert_eq!(fallback_http_index(0, 0, 3), 0);
assert_eq!(fallback_http_index(0, 1, 3), 1);
assert_eq!(fallback_http_index(0, 2, 3), 2);
assert_eq!(fallback_http_index(0, 3, 3), 0);
assert_eq!(fallback_http_index(2, 1, 3), 0);
}
#[test]
fn fallback_http_exhausted_error_includes_hosts() {
let err = fallback_http_exhausted_error(
100,
120,
&["rpc.ankr.com".to_string(), "mainnet.infura.io".to_string()],
"timeout",
)
.to_string();
assert!(err.contains("rpc.ankr.com"));
assert!(err.contains("mainnet.infura.io"));
assert!(err.contains("timeout"));
}
#[test]
fn head_coalescer_ignores_stale_heads_before_cursor() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(42), WINDOW);
assert!(!coalescer.observe_head(41, now));
assert_eq!(coalescer.pending_range(), None);
assert_eq!(coalescer.flush_deadline(), None);
}
#[test]
fn head_coalescer_keeps_first_flush_deadline_while_heads_accumulate() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
assert!(coalescer.observe_head(100, now));
let deadline = coalescer.flush_deadline().unwrap();
assert!(coalescer.observe_head(105, now + Duration::from_millis(50)));
assert_eq!(coalescer.pending_range(), Some((100, 105)));
assert_eq!(coalescer.flush_deadline(), Some(deadline));
assert!(!coalescer.should_flush(now + WINDOW - Duration::from_millis(1)));
assert!(coalescer.should_flush(now + WINDOW));
}
#[test]
fn head_coalescer_flushes_immediately_when_pending_range_reaches_chunk_limit() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
assert!(coalescer.observe_head(MAX_BLOCKS_PER_WS_REQUEST, now));
assert_eq!(
coalescer.pending_range(),
Some((1, MAX_BLOCKS_PER_WS_REQUEST))
);
assert!(coalescer.should_flush(now));
}
#[test]
fn head_coalescer_preserves_pending_range_across_partial_cursor_advances() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(10), WINDOW);
assert!(coalescer.observe_head(25, now));
coalescer.advance_cursor(20);
assert_eq!(coalescer.pending_range(), Some((20, 25)));
assert!(coalescer.flush_deadline().is_some());
coalescer.advance_cursor(26);
assert_eq!(coalescer.pending_range(), None);
assert_eq!(coalescer.flush_deadline(), None);
assert_eq!(coalescer.block_cursor(), Some(26));
}
#[test]
fn advance_cursor_records_scanned_to_as_next_block_minus_one() {
let progress = crate::rpc::progress::ScanProgress::default();
let mut coalescer =
HeadCoalescer::new(Some(10), WINDOW).with_progress(Some(progress.clone()));
assert_eq!(progress.scanned_to(), None);
coalescer.advance_cursor(101);
assert_eq!(progress.scanned_to(), Some(100));
coalescer.advance_cursor(51);
assert_eq!(progress.scanned_to(), Some(100));
}
#[test]
fn record_tip_populates_progress_when_attached_and_is_a_no_op_otherwise() {
let progress = crate::rpc::progress::ScanProgress::default();
let with_progress = HeadCoalescer::new(None, WINDOW).with_progress(Some(progress.clone()));
with_progress.record_tip(500);
assert_eq!(progress.observed_tip(), Some(500));
let without_progress = HeadCoalescer::new(None, WINDOW);
without_progress.record_tip(500);
}
#[test]
fn head_coalescer_handles_mainnet_slow_blocks() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(1_000_000), WINDOW);
assert!(coalescer.observe_head(1_000_000, now));
assert_eq!(coalescer.pending_range(), Some((1_000_000, 1_000_000)));
assert!(!coalescer.should_flush(now + Duration::from_millis(499)));
assert!(coalescer.should_flush(now + WINDOW));
coalescer.advance_cursor(1_000_001);
assert_eq!(coalescer.pending_range(), None);
let t12s = now + Duration::from_secs(12);
assert!(coalescer.observe_head(1_000_001, t12s));
assert_eq!(coalescer.pending_range(), Some((1_000_001, 1_000_001)));
assert!(!coalescer.should_flush(t12s));
assert!(coalescer.should_flush(t12s + WINDOW));
}
#[test]
fn head_coalescer_honors_its_own_window() {
let now = Instant::now();
let short_window = Duration::from_millis(100);
let long_window = Duration::from_secs(5);
let mut short = HeadCoalescer::new(Some(1), short_window);
assert!(short.observe_head(1, now));
assert!(!short.should_flush(now + short_window - Duration::from_millis(1)));
assert!(short.should_flush(now + short_window));
let mut long = HeadCoalescer::new(Some(1), long_window);
assert!(long.observe_head(1, now));
assert!(!long.should_flush(now + long_window - Duration::from_millis(1)));
assert!(long.should_flush(now + long_window));
}
#[test]
fn head_coalescer_defer_flush_clears_deadline_only() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
assert!(coalescer.observe_head(300, now));
assert_eq!(coalescer.pending_range(), Some((1, 300)));
assert!(coalescer.flush_deadline().is_some());
coalescer.defer_flush();
assert_eq!(
coalescer.flush_deadline(),
None,
"defer_flush must clear the deadline"
);
assert_eq!(
coalescer.pending_range(),
Some((1, 300)),
"defer_flush must not touch the pending range"
);
assert!(coalescer.should_flush(now));
}
#[test]
fn coalescer_partial_defer_reserves_unproven_tail_and_rearms() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(105, now);
assert_eq!(coalescer.pending_range(), Some((100, 105)));
coalescer.advance_cursor(103);
coalescer.defer_flush();
assert_eq!(
coalescer.flush_deadline(),
None,
"defer_flush must clear the deadline after a Partial yield"
);
assert_eq!(
coalescer.pending_range(),
Some((103, 105)),
"pending range must start at watermark+1, the unproven tail"
);
assert!(coalescer.observe_head(105, now + Duration::from_millis(1)));
assert!(coalescer.flush_deadline().is_some());
}
fn log_fixture(block_number: Option<u64>) -> Log {
let value = serde_json::json!({
"address": "0x0000000000000000000000000000000000000001",
"topics": [],
"data": "0x",
"blockHash": hb(0).to_string(),
"blockNumber": block_number.map(|n| format!("0x{:x}", n)),
"transactionHash": hb(0).to_string(),
"transactionIndex": "0x0",
"logIndex": "0x0",
"removed": false
});
serde_json::from_value(value).expect("valid Log fixture")
}
#[test]
fn yield_bounds_full_when_tail_has_logs() {
let logs = vec![log_fixture(Some(101)), log_fixture(Some(102))];
assert!(matches!(yield_bounds(&logs, 102), YieldBounds::Full));
}
#[test]
fn yield_bounds_partial_stops_at_watermark() {
let logs = vec![log_fixture(Some(101))];
assert!(matches!(
yield_bounds(&logs, 102),
YieldBounds::Partial(101)
));
}
#[test]
fn yield_bounds_interior_gaps_are_proven() {
let logs = vec![log_fixture(Some(101)), log_fixture(Some(104))];
assert!(matches!(
yield_bounds(&logs, 105),
YieldBounds::Partial(104)
));
}
#[test]
fn yield_bounds_mixed_none_uses_some_only() {
let logs = vec![log_fixture(Some(101)), log_fixture(None)];
assert!(matches!(
yield_bounds(&logs, 102),
YieldBounds::Partial(101)
));
}
#[test]
fn yield_bounds_all_none_is_unattributable() {
let logs = vec![log_fixture(None), log_fixture(None)];
assert!(matches!(
yield_bounds(&logs, 999),
YieldBounds::Unattributable
));
}
#[test]
fn yield_bounds_watermark_above_range_clamps() {
let logs = vec![log_fixture(Some(103))];
assert!(matches!(yield_bounds(&logs, 102), YieldBounds::Full));
}
#[tokio::test]
async fn lazy_empty_non_capped_defers_without_tip_call() {
let asserter = Asserter::new(); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(105, Instant::now());
assert_eq!(coalescer.pending_range(), Some((100, 105)));
let mut gate = TipGate::new();
let outcome = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host",
105,
false,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(outcome, EmptyFlushOutcome::Deferred),
"expected Deferred"
);
assert_eq!(coalescer.flush_deadline(), None);
assert_eq!(
coalescer.pending_range(),
Some((100, 105)),
"cursor must stay unmoved by a deferred flush"
);
}
#[test]
fn lazy_deferred_flush_rearms_and_spans_from_unmoved_cursor() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(105, now);
assert_eq!(coalescer.pending_range(), Some((100, 105)));
coalescer.defer_flush();
assert_eq!(coalescer.flush_deadline(), None);
coalescer.observe_head(110, now + Duration::from_millis(10));
assert_eq!(
coalescer.pending_range(),
Some((100, 110)),
"deferred span must extend from the original unmoved cursor"
);
assert!(
coalescer.flush_deadline().is_some(),
"the next observe_head must re-arm the deadline"
);
}
#[tokio::test]
async fn span_cap_empty_tip_current_is_verified_empty() {
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x64")); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(100, Instant::now());
let mut gate = TipGate::new();
let outcome = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host",
100,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(outcome, EmptyFlushOutcome::VerifiedEmpty),
"expected VerifiedEmpty"
);
}
#[tokio::test]
async fn span_cap_empty_tip_behind_enters_tip_wait() {
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x63")); asserter.push_success(&serde_json::json!("0x64")); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(100, Instant::now());
let mut gate = TipGate::new();
let outcome = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host",
100,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(outcome, EmptyFlushOutcome::CaughtUp),
"expected CaughtUp"
);
}
#[tokio::test]
async fn span_cap_empty_flat_tip_stalls() {
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x5a")); asserter.push_success(&serde_json::json!("0x5a")); asserter.push_success(&serde_json::json!("0x5a")); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
)
.with_tip_wait_max_stall_cycles(2);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(100, Instant::now());
let mut gate = TipGate::new();
let outcome = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host",
100,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(
outcome,
EmptyFlushOutcome::Stalled {
to_block: 100,
reported_tip: 90,
stalled_cycles: 2,
}
),
"expected Stalled(stalled_cycles=2), got non-matching outcome"
);
}
#[tokio::test]
async fn handle_empty_flush_stale_gate_stalls_one_cycle_early_without_reset() {
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x3de")); asserter.push_failure_msg("transport blip");
for _ in 0..5 {
asserter.push_success(&serde_json::json!("0x3dc")); }
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(999, Instant::now());
let mut gate = TipGate::new();
let session_a = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host-a",
999,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(session_a, EmptyFlushOutcome::RepollFailed(_)),
"expected RepollFailed for session A"
);
let session_b = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host-b",
999,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(
session_b,
EmptyFlushOutcome::Stalled {
to_block: 999,
reported_tip: 988,
stalled_cycles: 5,
}
),
"expected Stalled(stalled_cycles=5) one cycle early from the stale gate"
);
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x3de")); asserter.push_failure_msg("transport blip");
for _ in 0..6 {
asserter.push_success(&serde_json::json!("0x3dc")); }
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(999, Instant::now());
let mut gate = TipGate::new();
let session_a = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host-a",
999,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(session_a, EmptyFlushOutcome::RepollFailed(_)),
"expected RepollFailed for session A"
);
gate.reset();
let session_b = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host-b",
999,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(
session_b,
EmptyFlushOutcome::Stalled {
to_block: 999,
reported_tip: 988,
stalled_cycles: 5,
}
),
"expected Stalled(stalled_cycles=5) reached only after the full \
6-poll budget once the gate is reset"
);
}
fn block_with_parent_at(number: u64, hash: B256, parent: B256) -> serde_json::Value {
let zero = hb(0).to_string();
serde_json::json!({
"hash": hash.to_string(),
"parentHash": parent.to_string(),
"sha3Uncles": zero,
"miner": "0x0000000000000000000000000000000000000000",
"stateRoot": zero,
"transactionsRoot": zero,
"receiptsRoot": zero,
"logsBloom": format!("0x{}", "0".repeat(512)),
"difficulty": "0x0",
"number": format!("0x{:x}", number),
"gasLimit": "0x0",
"gasUsed": "0x0",
"timestamp": "0x0",
"extraData": "0x",
"mixHash": zero,
"nonce": "0x0000000000000000",
"transactions": [],
"uncles": []
})
}
fn block_with_parent(hash: B256, parent: B256) -> serde_json::Value {
block_with_parent_at(0, hash, parent)
}
fn n_http_provider_settings(n: usize) -> ProviderSettings {
let http_providers = (0..n)
.map(|i| HttpRpcSettings::new(format!("http://unused-{i}"), 1, 100))
.collect();
ProviderSettings::new(
http_providers,
vec!["ws://127.0.0.1:9".parse().expect("static url")],
)
}
fn mock_http_provider(asserter: Asserter) -> DynProvider<Ethereum> {
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased()
}
fn seed_chain(depth: u64, lo: u64, hi: u64) -> HeadLinkage {
let mut linkage = HeadLinkage::new(depth);
for n in lo..=hi {
let parent = if n == lo {
hb((lo - 1) as u8)
} else {
hb((n - 1) as u8)
};
assert_eq!(linkage.observe(n, hb(n as u8), parent), LinkOutcome::Linked);
}
linkage
}
#[test]
fn head_linkage_disabled_is_inert() {
let mut linkage = HeadLinkage::new(0);
assert_eq!(linkage.observe(100, hb(1), hb(9)), LinkOutcome::Linked);
assert_eq!(linkage.observe(101, hb(2), hb(7)), LinkOutcome::Linked);
assert_eq!(linkage.floor(), None);
}
#[test]
fn head_linkage_clean_chain_advances_and_bounds_ring() {
let linkage = seed_chain(3, 100, 103);
assert_eq!(linkage.floor(), Some(101));
assert_eq!(linkage.hash_at(100), None);
assert_eq!(linkage.hash_at(103), Some(hb(103)));
}
#[test]
fn head_linkage_one_block_reorg_reports_parent_as_ancestor() {
let mut linkage = seed_chain(8, 100, 103);
assert_eq!(
linkage.observe(103, hb(203), hb(102)),
LinkOutcome::Reorg {
common_ancestor: 102
}
);
assert_eq!(linkage.hash_at(103), Some(hb(203)));
assert_eq!(linkage.observe(104, hb(204), hb(203)), LinkOutcome::Linked);
}
#[test]
fn head_linkage_deep_reorg_via_reemit_reports_deeper_ancestor() {
let mut linkage = seed_chain(8, 100, 105);
assert_eq!(
linkage.observe(101, hb(201), hb(100)),
LinkOutcome::Reorg {
common_ancestor: 100
}
);
}
#[test]
fn head_linkage_skipped_intermediate_heads_detach() {
let mut linkage = seed_chain(8, 100, 103);
match linkage.observe(104, hb(204), hb(203)) {
LinkOutcome::Detached {
parent_hash,
child_number,
} => {
assert_eq!(parent_hash, hb(203));
assert_eq!(child_number, 104);
}
other => panic!("expected Detached, got {other:?}"),
}
}
#[test]
fn head_linkage_parent_below_floor_has_no_recorded_parent() {
let mut linkage = seed_chain(3, 100, 105);
assert_eq!(linkage.floor(), Some(103));
assert_eq!(
linkage.observe(103, hb(203), hb(202)),
LinkOutcome::NoRecordedParent
);
}
#[test]
fn lowest_worth_recording_is_none_on_an_empty_ring() {
let linkage = HeadLinkage::new(8);
assert_eq!(linkage.lowest_worth_recording(), None);
}
#[test]
fn lowest_worth_recording_tracks_latest_at_depth() {
let linkage = seed_chain(8, 100, 105);
assert_eq!(linkage.lowest_worth_recording(), Some(98));
let deep = seed_chain(1, 100, 105);
assert_eq!(deep.lowest_worth_recording(), Some(105));
}
#[test]
fn extend_downward_records_pairs_below_the_floor() {
let mut linkage = seed_chain(8, 103, 105);
assert_eq!(linkage.floor(), Some(103));
linkage.extend_downward(&[(101, hb(101)), (102, hb(102))]);
assert_eq!(linkage.floor(), Some(101));
assert_eq!(linkage.hash_at(101), Some(hb(101)));
assert_eq!(linkage.hash_at(102), Some(hb(102)));
assert_eq!(linkage.hash_at(103), Some(hb(103)));
assert_eq!(linkage.latest(), Some((105, hb(105))));
}
#[test]
fn extend_downward_ignores_pairs_at_or_above_the_floor() {
let mut linkage = seed_chain(8, 103, 105);
let before = linkage.clone();
linkage.extend_downward(&[(103, hb(103)), (104, hb(200))]);
assert_eq!(
linkage.floor(),
before.floor(),
"no-op: nothing below the floor"
);
assert_eq!(
linkage.hash_at(104),
Some(hb(104)),
"an at-or-above pair must never overwrite an existing entry"
);
assert_eq!(linkage.ring.len(), before.ring.len());
}
#[test]
fn extend_downward_batch_not_contiguous_with_front_records_nothing() {
let mut linkage = seed_chain(8, 103, 105);
linkage.extend_downward(&[(100, hb(100))]);
assert_eq!(
linkage.floor(),
Some(103),
"a non-contiguous batch is discarded whole"
);
assert_eq!(linkage.hash_at(100), None);
}
#[test]
fn extend_downward_internal_gap_records_nothing() {
let mut linkage = seed_chain(8, 103, 105);
linkage.extend_downward(&[(100, hb(100)), (102, hb(102))]);
assert_eq!(
linkage.floor(),
Some(103),
"an internal gap is discarded whole"
);
assert_eq!(linkage.hash_at(102), None);
}
#[test]
fn extend_downward_never_exceeds_depth() {
let mut linkage = seed_chain(3, 103, 105);
linkage.extend_downward(&[(100, hb(100)), (101, hb(101)), (102, hb(102))]);
assert_eq!(
linkage.floor(),
Some(103),
"a full ring must not extend further"
);
assert_eq!(linkage.ring.len(), 3);
}
#[test]
fn extend_downward_caps_at_depth_even_when_pairs_go_deeper() {
let mut linkage = seed_chain(5, 103, 105);
linkage.extend_downward(&[(99, hb(99)), (100, hb(100)), (101, hb(101)), (102, hb(102))]);
assert_eq!(
linkage.floor(),
Some(101),
"must stop exactly at depth, not walk past it"
);
assert_eq!(
linkage.hash_at(100),
None,
"beyond depth must not be recorded"
);
assert_eq!(linkage.ring.len(), 5);
}
#[test]
fn extend_downward_and_lowest_worth_recording_are_inert_at_depth_zero() {
let mut linkage = HeadLinkage::new(0);
assert_eq!(linkage.observe(100, hb(100), hb(99)), LinkOutcome::Linked);
assert_eq!(linkage.lowest_worth_recording(), None);
linkage.extend_downward(&[(99, hb(99))]);
assert_eq!(
linkage.floor(),
None,
"disabled linkage records nothing, ever"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_walks_skipped_chain_to_common_ancestor() {
let linkage = seed_chain(8, 100, 103);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(203), hb(202)));
asserter.push_success(&block_with_parent(hb(202), hb(101)));
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(203), 104, 8).await;
assert!(matches!(err, Ok(101)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_fetched_hash_at_floor_is_reorg_beyond_ring() {
let linkage = seed_chain(2, 100, 103);
assert_eq!(linkage.floor(), Some(102));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(203), hb(202)));
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(203), 104, 2).await;
assert!(matches!(err, Err(RpcError::ReorgBeyondRing)), "got {err:?}");
}
#[test]
fn head_linkage_back_to_back_reorgs_ring_stays_coherent() {
let mut linkage = seed_chain(8, 100, 105);
assert_eq!(
linkage.observe(104, hb(204), hb(103)),
LinkOutcome::Reorg {
common_ancestor: 103
}
);
assert_eq!(linkage.hash_at(104), Some(hb(204)));
assert_eq!(linkage.observe(105, hb(205), hb(204)), LinkOutcome::Linked);
assert_eq!(
linkage.observe(105, hb(49), hb(204)),
LinkOutcome::Reorg {
common_ancestor: 104
}
);
assert_eq!(linkage.hash_at(105), Some(hb(49)));
assert_eq!(linkage.hash_at(104), Some(hb(204))); assert!(linkage.ring.len() as u64 <= 8);
}
#[test]
fn head_linkage_multi_header_same_height_duplicate_vs_replace() {
let mut linkage = seed_chain(8, 100, 104);
assert_eq!(linkage.observe(104, hb(104), hb(103)), LinkOutcome::Linked);
assert_eq!(linkage.hash_at(104), Some(hb(104)));
assert_eq!(
linkage.observe(104, hb(204), hb(103)),
LinkOutcome::Reorg {
common_ancestor: 103
}
);
assert_eq!(linkage.hash_at(104), Some(hb(204)));
}
#[test]
fn head_linkage_identical_hash_no_op() {
let mut linkage = seed_chain(8, 100, 104);
let floor_before = linkage.floor();
let depth_before = linkage.ring.len();
assert_eq!(linkage.observe(104, hb(104), hb(103)), LinkOutcome::Linked);
assert_eq!(linkage.floor(), floor_before);
assert_eq!(linkage.ring.len(), depth_before);
assert_eq!(linkage.hash_at(104), Some(hb(104)));
}
#[test]
fn head_linkage_forward_gap_reports_the_break_in_our_own_view() {
let mut linkage = seed_chain(8, 100, 103); assert_eq!(
linkage.observe(107, hb(107), hb(106)),
LinkOutcome::Gap { ring_tip: 103 }
);
assert_eq!(linkage.floor(), Some(100));
assert_eq!(linkage.hash_at(103), Some(hb(103)));
}
#[test]
fn head_linkage_idempotent_reanchor_bounds_ring() {
let mut linkage = seed_chain(5, 100, 106); assert_eq!(linkage.floor(), Some(102));
assert_eq!(
linkage.observe(105, hb(205), hb(104)),
LinkOutcome::Reorg {
common_ancestor: 104
}
);
assert_eq!(linkage.hash_at(105), Some(hb(205)));
assert_eq!(linkage.observe(106, hb(206), hb(205)), LinkOutcome::Linked);
assert_eq!(linkage.observe(107, hb(207), hb(206)), LinkOutcome::Linked);
assert!(linkage.ring.len() as u64 <= 5);
assert_eq!(linkage.hash_at(105), Some(hb(205))); assert_eq!(linkage.hash_at(102), None); }
#[tokio::test]
async fn walk_to_common_ancestor_walk_depth_1() {
let linkage = seed_chain(15, 100, 109);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(109))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 15).await;
assert!(matches!(err, Ok(109)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_walk_depth_2() {
let linkage = seed_chain(15, 100, 109);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201))); asserter.push_success(&block_with_parent(hb(201), hb(108))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 15).await;
assert!(matches!(err, Ok(108)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_walk_depth_3() {
let linkage = seed_chain(15, 100, 109);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201))); asserter.push_success(&block_with_parent(hb(201), hb(202))); asserter.push_success(&block_with_parent(hb(202), hb(107))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 15).await;
assert!(matches!(err, Ok(107)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_walk_depth_7() {
let linkage = seed_chain(15, 100, 109);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201)));
asserter.push_success(&block_with_parent(hb(201), hb(202)));
asserter.push_success(&block_with_parent(hb(202), hb(203)));
asserter.push_success(&block_with_parent(hb(203), hb(204)));
asserter.push_success(&block_with_parent(hb(204), hb(205)));
asserter.push_success(&block_with_parent(hb(205), hb(206)));
asserter.push_success(&block_with_parent(hb(206), hb(103))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 15).await;
assert!(matches!(err, Ok(103)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_deepest_in_ring_resolves() {
let linkage = seed_chain(5, 100, 104);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201))); asserter.push_success(&block_with_parent(hb(201), hb(202))); asserter.push_success(&block_with_parent(hb(202), hb(203))); asserter.push_success(&block_with_parent(hb(203), hb(100))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 105, 5).await;
assert!(matches!(err, Ok(100)), "got {err:?}");
}
#[test]
fn forward_gap_never_reaches_the_walk_whatever_its_length() {
for gap in [2u64, 1_300] {
let mut linkage = seed_chain(8, 100, 103);
let head = 103 + gap;
assert_eq!(
linkage.observe(head, hb(207), hb(206)),
LinkOutcome::Gap { ring_tip: 103 },
"a gap of {gap} must report the break in our own view, never route \
to the reorg walk"
);
assert_eq!(
linkage.latest(),
Some((103, hb(103))),
"a gap of {gap} must not mutate the ring"
);
}
}
#[tokio::test]
async fn walk_still_spends_a_fetch_per_step_when_a_divergence_reaches_it() {
let mut linkage = seed_chain(8, 100, 103);
assert_eq!(
linkage.observe(104, hb(204), hb(203)),
LinkOutcome::Detached {
parent_hash: hb(203),
child_number: 104,
}
);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(203), hb(202)));
asserter.push_success(&block_with_parent(hb(202), hb(201)));
asserter.push_success(&block_with_parent(hb(201), hb(100)));
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let ancestor =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(203), 104, 8).await;
assert!(matches!(ancestor, Ok(100)), "got {ancestor:?}");
}
#[test]
fn re_observing_the_only_recorded_head_is_not_a_divergence() {
let mut linkage = HeadLinkage::new(96);
assert_eq!(linkage.observe(500, hb(50), hb(49)), LinkOutcome::Linked);
assert_eq!(linkage.floor(), Some(500));
assert_eq!(
linkage.hash_at(499),
None,
"the parent's height is genuinely unrecorded — that is the shape \
this test exists for"
);
assert_eq!(
linkage.observe(500, hb(50), hb(49)),
LinkOutcome::Linked,
"the same block at the same height is not new information"
);
assert_eq!(linkage.latest(), Some((500, hb(50))));
assert_eq!(linkage.ring.len(), 1, "a duplicate must not grow the ring");
}
#[tokio::test]
async fn walk_at_a_one_entry_rings_floor_claims_nothing_before_a_fetch() {
let mut linkage = HeadLinkage::new(96);
assert_eq!(linkage.observe(500, hb(50), hb(49)), LinkOutcome::Linked);
assert_eq!(linkage.floor(), linkage.latest().map(|(n, _)| n));
assert_eq!(
linkage.observe(501, hb(151), hb(150)),
LinkOutcome::Detached {
parent_hash: hb(150),
child_number: 501,
}
);
let provider = mock_http_provider(Asserter::new());
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(150), 501, 96).await;
assert!(
matches!(err, Err(RpcError::Gap)),
"an unfetched disagreement at the floor must not claim a divergence; got {err:?}"
);
}
#[tokio::test]
async fn walk_at_the_floor_diverges_but_below_it_only_discontinues() {
let linkage = seed_chain(2, 100, 103);
assert_eq!(linkage.floor(), Some(102));
let settings = n_http_provider_settings(1);
let at_floor = Asserter::new();
at_floor.push_success(&block_with_parent(hb(203), hb(202)));
let err = walk_to_common_ancestor(
&mock_http_provider(at_floor),
&settings,
None,
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondRing)),
"a fetched hash disagreeing with a recorded one at the floor is a \
chain claim; got {err:?}"
);
let err = walk_to_common_ancestor(
&mock_http_provider(Asserter::new()),
&settings,
None,
&linkage,
hb(202),
102,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::Gap)),
"a height the ring never recorded cannot diverge from it; got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_loop_exhausted_is_a_gap() {
let linkage = seed_chain(3, 100, 109);
assert_eq!(linkage.floor(), Some(107));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201)));
asserter.push_success(&block_with_parent(hb(201), hb(202)));
asserter.push_success(&block_with_parent(hb(202), hb(203))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 3).await;
assert!(matches!(err, Err(RpcError::Gap)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_parent_at_floor_before_a_fetch_is_a_gap() {
let linkage = seed_chain(10, 100, 109); let asserter = Asserter::new(); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 101, 10).await;
assert!(matches!(err, Err(RpcError::Gap)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_single_provider_absent_is_unresolved() {
let linkage = seed_chain(15, 100, 109);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(1);
let http_asserter = Asserter::new();
http_asserter.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(http_asserter)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(200),
111,
15,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_transport_error_is_unresolved() {
let linkage = seed_chain(15, 100, 109);
let settings = n_http_provider_settings(1);
let http_asserter = Asserter::new();
http_asserter.push_failure_msg("transport blip");
let reused = vec![mock_http_provider(http_asserter)];
let asserter = Asserter::new();
asserter.push_failure_msg("transport blip");
let provider = mock_http_provider(asserter);
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(200),
111,
15,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_second_source_recovers_the_incident_scenario() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&block_with_parent(hb(203), hb(102))); let voter2 = Asserter::new();
voter2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(matches!(err, Ok(102)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_all_sources_absent_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let reused: Vec<_> = (0..3)
.map(|_| {
let a = Asserter::new();
a.push_success(&serde_json::json!(null));
mock_http_provider(a)
})
.collect();
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_lone_absent_answer_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_failure_msg("blip");
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_two_of_three_absent_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_n_equals_2() {
let linkage = seed_chain(8, 100, 103);
let settings = n_http_provider_settings(2);
{
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_failure_msg("blip");
let reused = vec![mock_http_provider(voter0), mock_http_provider(voter1)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
{
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(voter0), mock_http_provider(voter1)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
}
#[tokio::test]
async fn walk_to_common_ancestor_primary_transport_fail() {
let linkage = seed_chain(8, 100, 103);
let settings = n_http_provider_settings(3);
{
let ws_asserter = Asserter::new();
ws_asserter.push_failure_msg("blip");
let provider = mock_http_provider(ws_asserter);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&block_with_parent(hb(203), hb(102)));
let voter2 = Asserter::new();
voter2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(matches!(err, Ok(102)), "got {err:?}");
}
{
let ws_asserter = Asserter::new();
ws_asserter.push_failure_msg("blip");
let provider = mock_http_provider(ws_asserter);
let reused: Vec<_> = (0..3)
.map(|_| {
let a = Asserter::new();
a.push_failure_msg("blip");
mock_http_provider(a)
})
.collect();
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
}
#[tokio::test]
async fn walk_to_common_ancestor_two_recoveries_then_ring_relink() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null)); ws_asserter.push_success(&serde_json::json!(null)); let provider = mock_http_provider(ws_asserter.clone());
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null)); voter0.push_success(&serde_json::json!(null)); let voter1 = Asserter::new();
voter1.push_success(&block_with_parent(hb(205), hb(204)));
voter1.push_success(&serde_json::json!(null)); let voter2 = Asserter::new();
voter2.push_success(&block_with_parent(hb(204), hb(102)));
let reused = vec![
mock_http_provider(voter0.clone()),
mock_http_provider(voter1.clone()),
mock_http_provider(voter2.clone()),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(205),
105,
8,
)
.await;
assert!(matches!(err, Ok(102)), "got {err:?}");
assert_eq!(
ws_asserter.read_q().len(),
0,
"exactly 2 primary triggers, no third"
);
assert_eq!(
voter0.read_q().len(),
1,
"cursor carries to index 1 on step 2; index 0's queued response is unread"
);
assert_eq!(voter1.read_q().len(), 0, "index 1 tried on both steps");
assert_eq!(
voter2.read_q().len(),
0,
"index 2 reached only once, on step 2"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_ws_answered_step_leaves_cursor_undisturbed() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null)); ws_asserter.push_success(&block_with_parent(hb(205), hb(204))); ws_asserter.push_success(&serde_json::json!(null)); let provider = mock_http_provider(ws_asserter.clone());
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null)); voter0.push_success(&serde_json::json!(null)); let voter1 = Asserter::new();
voter1.push_success(&block_with_parent(hb(206), hb(205)));
voter1.push_success(&serde_json::json!(null)); let voter2 = Asserter::new();
voter2.push_success(&block_with_parent(hb(204), hb(102)));
let reused = vec![
mock_http_provider(voter0.clone()),
mock_http_provider(voter1.clone()),
mock_http_provider(voter2.clone()),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(206),
106,
8,
)
.await;
assert!(matches!(err, Ok(102)), "got {err:?}");
assert_eq!(
ws_asserter.read_q().len(),
0,
"exactly 3 primary triggers, no fourth"
);
assert_eq!(
voter0.read_q().len(),
1,
"cursor still at index 1 on step 3; index 0's queued response is unread"
);
assert_eq!(voter1.read_q().len(), 0, "index 1 tried on steps 1 and 3");
assert_eq!(
voter2.read_q().len(),
0,
"index 2 reached only once, on step 3"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_wrong_block_is_not_a_recovery() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&block_with_parent(hb(9), hb(1)));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
let Err(RpcError::ConnectionError(msg)) = err else {
panic!("got {err:?}");
};
assert!(msg.contains("blip"), "provider error text dropped: {msg}");
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let voter0 = Asserter::new();
voter0.push_success(&block_with_parent(hb(77), hb(102)));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_primary_wrong_hash_does_not_advance_the_walk() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&block_with_parent(hb(77), hb(102)));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(1);
let source = Asserter::new();
source.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(source)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"a wrong-hash primary block must never advance the walk; got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_primary_transport_error_with_absent_quorum_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_failure_msg("primary socket died");
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_reused_length_mismatch_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let lone = Asserter::new();
lone.push_success(&serde_json::json!(null));
lone.push_success(&serde_json::json!(null));
lone.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(lone)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"a short reused set must fail closed, never let one client vote \
three times; got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_zero_http_providers_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let asserter = Asserter::new(); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(0);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(203), 104, 8).await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn by_hash_tail_fetch_over_mock() {
let walk_asserter = Asserter::new();
walk_asserter.push_success(&block_with_parent_at(103, hb(103), hb(102))); walk_asserter.push_success(&block_with_parent_at(102, hb(102), hb(101))); walk_asserter.push_success(&block_with_parent_at(101, hb(101), hb(100))); let walk_provider = mock_http_provider(walk_asserter);
let walk_settings = n_http_provider_settings(1);
let pairs_desc =
collect_window_hashes(&walk_provider, &walk_settings, None, 0, hb(103), 103, 100)
.await
.expect("walk should succeed");
assert_eq!(pairs_desc.len(), 4);
assert_eq!(pairs_desc[0], (103, hb(103)));
assert_eq!(pairs_desc[1], (102, hb(102)));
assert_eq!(pairs_desc[2], (101, hb(101)));
assert_eq!(pairs_desc[3], (100, hb(100)));
let mut pairs = pairs_desc;
pairs.reverse();
let log_asserter = Asserter::new();
for n in 100u64..=103 {
log_asserter.push_success(&serde_json::json!([{
"address": "0x0000000000000000000000000000000000000001",
"topics": [],
"data": "0x",
"blockHash": hb(n as u8).to_string(),
"blockNumber": format!("0x{:x}", n),
"transactionHash": hb(0).to_string(),
"transactionIndex": "0x0",
"logIndex": "0x0",
"removed": false
}]));
}
let log_provider = ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(log_asserter)
.erased();
let settings = ProviderSettings::from_mock(log_provider);
let ranges = fetch_logs_by_hash_range(&settings, &pairs, None)
.await
.expect("log fetch should succeed");
assert_eq!(ranges.len(), 4, "expected one range per block");
for (i, range) in ranges.iter().enumerate() {
let block_n = 100u64 + i as u64;
assert_eq!(
range.from_block, block_n,
"from_block mismatch at index {i}"
);
assert_eq!(range.to_block, block_n, "to_block mismatch at index {i}");
assert_eq!(range.logs.len(), 1, "expected one log at block {block_n}");
assert_eq!(
range.logs[0].block_number,
Some(block_n),
"log block number mismatch"
);
assert_eq!(
range.tip_hash,
Some(hb(block_n as u8)),
"a by-hash range carries the hash it was fetched with"
);
}
}
#[tokio::test]
async fn by_hash_producers_stamp_the_range_with_its_canonical_tip_hash() {
let pairs = [(100u64, hb(100)), (101u64, hb(101))];
let asserter = Asserter::new();
for _ in 0..pairs.len() {
asserter.push_success(&serde_json::json!([]));
}
let tail_settings = ProviderSettings::from_mock(mock_http_provider(asserter));
let ranges = fetch_logs_by_hash_range(&tail_settings, &pairs, None)
.await
.expect("by-hash fetch should succeed");
assert_eq!(ranges.len(), 2);
for (range, &(number, hash)) in ranges.iter().zip(pairs.iter()) {
assert_eq!((range.from_block, range.to_block), (number, number));
assert_eq!(
range.tip_hash,
Some(hash),
"the restart tail knows the hash it queried with"
);
}
let flush_asserter = Asserter::new();
flush_asserter.push_success(&serde_json::json!([]));
let flush_settings = ProviderSettings::from_mock(mock_http_provider(flush_asserter));
let mut linkage = HeadLinkage::new(1);
assert_eq!(linkage.observe(300, hb(44), hb(43)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(300), WINDOW);
coalescer.observe_head(300, Instant::now());
let ranges =
flush_window_range(&mut linkage, &mut coalescer, &flush_settings, None, 0, None)
.await
.expect("flush should succeed");
assert_eq!(ranges.len(), 1);
assert_eq!(
ranges[0].tip_hash,
Some(hb(44)),
"the live flush fetches by the ring's canonical hash and carries it"
);
}
fn self_heal_bound_exceeded_fixture() -> (
ProviderSettings,
HeadLinkage,
HeadCoalescer,
Option<(u64, u64)>,
) {
let settings = ProviderSettings::from_mock(mock_http_provider(Asserter::new()));
let depth = 96u64;
let head = 10_000_000u64;
let cursor = head - (SELF_HEAL_WALK_MAX_BLOCKS + depth);
let mut linkage = HeadLinkage::new(depth);
assert_eq!(linkage.observe(head, hb(7), hb(6)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(cursor), WINDOW);
coalescer.observe_head(head, Instant::now());
let pending_before = coalescer.pending_range();
(settings, linkage, coalescer, pending_before)
}
#[tokio::test]
async fn flush_self_heal_walk_stops_at_its_length_bound() {
let (settings, mut linkage, mut coalescer, pending_before) =
self_heal_bound_exceeded_fixture();
let err = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect_err("a walk past the bound must not run");
assert!(matches!(err, RpcError::Gap), "got {err:?}");
assert_eq!(
coalescer.pending_range(),
pending_before,
"a refused flush leaves the cursor where it was — no silent gap"
);
}
#[tokio::test]
async fn self_heal_bound_error_classifies_end_stream_and_moves_no_cursor() {
let (settings, mut linkage, mut coalescer, pending_before) =
self_heal_bound_exceeded_fixture();
let err = bound_flush_time(
settings.flush_max_total_secs,
&flush_timeout_label(&settings, "ws=test-host"),
flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None),
)
.await
.expect_err("a walk past the bound must not run");
assert_eq!(
classify_flush_failure(&err),
FlushFailure::EndStream,
"a gap must reach the consumer, not another provider: {err:?}"
);
assert_eq!(
coalescer.pending_range(),
pending_before,
"the span a retry would re-run is unchanged, so a retry cannot differ"
);
}
#[test]
fn classify_flush_failure_rotates_only_on_a_transport_fault() {
for err in [
RpcError::Gap,
RpcError::ReorgBeyondRing,
RpcError::Reorg {
common_ancestor: 42,
},
] {
assert_eq!(
classify_flush_failure(&err),
FlushFailure::EndStream,
"{err:?} states a conclusion no other provider can overturn"
);
}
for err in [
RpcError::ConnectionError("stalled mid-flush".to_string()),
RpcError::SubscriptionError("dropped".to_string()),
RpcError::LogFetchError("eth_getLogs failed".to_string()),
RpcError::TransportError("reset by peer".to_string()),
RpcError::CursorPastTip {
to_block: 10,
reported_tip: 5,
},
RpcError::ProviderStalled {
to_block: 10,
reported_tip: 5,
stalled_cycles: 3,
},
RpcError::Other("unclassified".to_string()),
] {
assert_eq!(
classify_flush_failure(&err),
FlushFailure::SwitchProvider,
"{err:?} is a transport-class failure another provider may answer"
);
}
for err in [
RpcError::AllProvidersSuspended("all down".to_string()),
RpcError::AllProvidersIdle {
providers_tried: 2,
idle_secs: 30,
},
] {
assert_eq!(
classify_flush_failure(&err),
FlushFailure::SwitchProvider,
"{err:?} is self-clearing, not a permanent claim about every provider"
);
}
}
#[test]
fn flush_failure_log_line_states_the_disposition() {
let err = RpcError::Gap;
let ended = flush_failure_log_line(&err, FlushFailure::EndStream);
let rotated = flush_failure_log_line(&err, FlushFailure::SwitchProvider);
assert!(
ended.contains("ending stream"),
"an ended stream must say so: {ended}"
);
assert!(
rotated.contains("retrying the same window"),
"a retry must say so: {rotated}"
);
assert!(
!rotated.contains("next provider"),
"the retried flush's HTTP fetch never rotates off provider index \
0 — the line must not claim a rotation this call site cannot \
perform: {rotated}"
);
assert_ne!(
ended, rotated,
"one error text, two dispositions — the line must tell them apart"
);
}
#[test]
fn log_ranges_compare_and_hash_across_every_field() {
use std::collections::HashSet;
let range = LogRange {
from_block: 10,
to_block: 12,
logs: vec![],
tip_hash: Some(hb(12)),
};
let same = range.clone();
let competing_tip = LogRange {
tip_hash: Some(hb(11)),
..range.clone()
};
assert_eq!(range, same);
assert_ne!(range, competing_tip);
let mut seen = HashSet::new();
assert!(seen.insert(range));
assert!(!seen.insert(same), "equal ranges must hash alike");
assert!(seen.insert(competing_tip));
}
#[tokio::test]
async fn flush_self_heal_walk_succeeds_when_recovery_window_exceeds_the_constant_bound() {
fn hash_at_height(number: u64) -> B256 {
let mut bytes = [0u8; 32];
bytes[24..].copy_from_slice(&number.to_be_bytes());
B256::from(bytes)
}
let depth = 2_000u64;
assert!(
depth > SELF_HEAL_WALK_MAX_BLOCKS,
"the defect is inert unless the configured window is the deeper of the two"
);
let tip = 10_000_000u64;
let floor = tip - (depth - 1);
let cursor = floor - 1;
let mut linkage = HeadLinkage::new(depth);
for number in floor..=tip {
assert_eq!(
linkage.observe(number, hash_at_height(number), hash_at_height(number - 1)),
LinkOutcome::Linked
);
}
let asserter = Asserter::new();
for number in (cursor + 1..=tip).rev() {
asserter.push_success(&block_with_parent_at(
number,
hash_at_height(number),
hash_at_height(number - 1),
));
}
asserter.push_success(&serde_json::json!([]));
let settings = ProviderSettings::from_mock(mock_http_provider(asserter));
let mut coalescer = HeadCoalescer::new(Some(cursor), WINDOW);
coalescer.observe_head(cursor, Instant::now());
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect("the walk's first legitimate firing must not be refused by the bound");
assert_eq!(ranges.len(), 1);
assert_eq!(
ranges[0].tip_hash,
Some(hash_at_height(cursor)),
"the healed window is fetched by the hash the walk established"
);
}
#[test]
fn live_head_gap_event_name_is_fixed() {
assert_eq!(live_head_gap_event(), "live_head_gap");
}
#[test]
fn live_head_reorg_beyond_ring_event_name_is_fixed() {
assert_eq!(
live_head_reorg_beyond_ring_event(),
"live_head_reorg_beyond_ring"
);
}
#[test]
fn walk_failure_event_names_only_the_chain_claim() {
assert_eq!(
walk_failure_event(&RpcError::ReorgBeyondRing),
Some("live_head_reorg_beyond_ring"),
"the chain claim must carry its own event"
);
for err in [
RpcError::ConnectionError("stalled mid-walk".to_string()),
RpcError::Gap,
RpcError::Reorg {
common_ancestor: 42,
},
] {
assert_eq!(
walk_failure_event(&err),
None,
"{err:?} must keep the walk's unnamed line"
);
}
}
#[tokio::test]
async fn orphan_rejection() {
let canonical_log = serde_json::json!([{
"address": "0xCAFECAFECAFECAFECAFECAFECAFECAFECAFECAFE",
"topics": [],
"data": "0x01",
"blockHash": hb(200).to_string(),
"blockNumber": "0xc8",
"transactionHash": hb(1).to_string(),
"transactionIndex": "0x0",
"logIndex": "0x0",
"removed": false
}]);
let asserter = Asserter::new();
asserter.push_success(&canonical_log);
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let mut linkage = HeadLinkage::new(8);
linkage.observe(200, hb(200), hb(199)); let mut coalescer = HeadCoalescer::new(Some(200), WINDOW);
coalescer.observe_head(200, Instant::now());
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect("flush should succeed");
assert_eq!(ranges.len(), 1, "one range per block");
let range = &ranges[0];
assert_eq!((range.from_block, range.to_block), (200, 200));
assert_eq!(range.logs.len(), 1, "canonical log returned");
assert_eq!(
range.logs[0].block_hash,
Some(hb(200)),
"log must carry the canonical block hash, not an orphan's"
);
assert_eq!(
range.tip_hash,
Some(hb(200)),
"the live flush fetches by hash and carries it"
);
assert_eq!(coalescer.pending_range(), None);
}
#[tokio::test]
async fn hash_mismatch_null_and_transport_are_unresolved() {
{
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err = collect_window_hashes(&provider, &settings, None, 0, hb(5), 5, 1)
.await
.expect_err("null block must be an error");
assert!(
matches!(err, RpcError::ConnectionError(_)),
"null block must yield ConnectionError, got {err:?}"
);
}
{
let asserter = Asserter::new();
asserter.push_failure_msg("connection reset");
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err = collect_window_hashes(&provider, &settings, None, 0, hb(5), 5, 1)
.await
.expect_err("transport error must be an error");
assert!(
matches!(err, RpcError::ConnectionError(_)),
"transport fault must yield ConnectionError, got {err:?}"
);
}
}
#[test]
fn dormancy_at_depth_zero() {
let mut linkage = HeadLinkage::new(0);
assert!(!linkage.enabled(), "depth=0 must be disabled");
assert_eq!(
linkage.observe(100, hb(1), hb(99)),
LinkOutcome::Linked,
"depth=0 must not trigger reorg detection"
);
assert_eq!(
linkage.observe(101, hb(2), hb(55)), LinkOutcome::Linked,
"depth=0 must not trigger reorg detection on bad parent"
);
assert_eq!(
linkage.floor(),
None,
"disabled linkage must not record state"
);
}
#[test]
fn boot_tail_seeds_ring() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(
linkage.observe(105, hb(105), hb(104)),
LinkOutcome::Linked,
"bootstrap observe"
);
assert_eq!(linkage.hash_at(105), Some(hb(105)));
assert_eq!(
linkage.observe(106, hb(106), hb(105)),
LinkOutcome::Linked,
"first post-flush head must link without false reorg"
);
assert_eq!(linkage.hash_at(106), Some(hb(106)));
assert_eq!(linkage.observe(107, hb(107), hb(106)), LinkOutcome::Linked);
}
#[tokio::test]
async fn live_flush_gap_self_heals() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(105, hb(105), hb(104)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(101), WINDOW);
coalescer.observe_head(105, Instant::now());
assert_eq!(coalescer.pending_range(), Some((101, 105)));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(105, hb(105), hb(104)));
asserter.push_success(&block_with_parent_at(104, hb(104), hb(103)));
asserter.push_success(&block_with_parent_at(103, hb(103), hb(102)));
asserter.push_success(&block_with_parent_at(102, hb(102), hb(101)));
for _ in 101u64..=105 {
asserter.push_success(&serde_json::json!([]));
}
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect("flush must succeed, not spin");
assert_eq!(ranges.len(), 5, "one LogRange per block 101..=105");
for (i, range) in ranges.iter().enumerate() {
let n = 101u64 + i as u64;
assert_eq!((range.from_block, range.to_block), (n, n));
}
assert_eq!(coalescer.pending_range(), None);
}
#[tokio::test]
async fn collect_window_hashes_number_mismatch() {
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(99, hb(5), hb(4))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err = collect_window_hashes(&provider, &settings, None, 0, hb(5), 5, 4)
.await
.expect_err("number mismatch must be an error");
assert!(
matches!(err, RpcError::ConnectionError(_)),
"number mismatch must yield ConnectionError, got {err:?}"
);
}
#[tokio::test]
async fn collect_window_hashes_primary_wrong_hash_does_not_answer_the_step() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&block_with_parent_at(5, hb(77), hb(4)));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&serde_json::json!(null));
let corrob2 = Asserter::new();
corrob2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(Asserter::new()), mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 4)
.await
.expect_err("a wrong-hash primary block must never answer the step");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[tokio::test]
async fn collect_window_hashes_second_source_continues_walk() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null)); primary_asserter.push_success(&block_with_parent_at(4, hb(4), hb(3))); let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&block_with_parent_at(5, hb(5), hb(4))); let corrob2 = Asserter::new();
corrob2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(Asserter::new()), mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let pairs_desc = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 3)
.await
.expect("one second source supplying the block must let the walk continue");
assert_eq!(
pairs_desc,
vec![(5, hb(5)), (4, hb(4)), (3, hb(3))],
"got {pairs_desc:?}"
);
}
#[tokio::test]
async fn collect_window_hashes_all_sources_absent_is_unresolved() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&serde_json::json!(null));
let corrob2 = Asserter::new();
corrob2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 1)
.await
.expect_err("exhausting every provider must be an error");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[tokio::test]
async fn collect_window_hashes_lone_absent_answer_is_unresolved() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_failure_msg("blip");
let corrob2 = Asserter::new();
corrob2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 1)
.await
.expect_err("exhausting every provider must be an error");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[tokio::test]
async fn collect_window_hashes_primary_miss_does_not_shorten_fallback() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&serde_json::json!(null));
let corrob2 = Asserter::new();
corrob2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 1)
.await
.expect_err("exhausting every provider must be an error");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[tokio::test]
async fn collect_window_hashes_wrong_number_is_not_a_recovery() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&block_with_parent_at(99, hb(5), hb(4)));
let corrob2 = Asserter::new();
corrob2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 1)
.await
.expect_err("wrong-number second source must not match");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[tokio::test]
async fn flush_window_range_empty_ring_is_fail_closed() {
let asserter = Asserter::new(); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let mut linkage = HeadLinkage::new(8); let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(100, Instant::now());
let err = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect_err("empty ring must be fail-closed");
assert!(matches!(err, RpcError::Gap), "got {err:?}");
}
#[tokio::test]
async fn flush_window_range_zero_http_providers_is_fail_closed() {
let settings = n_http_provider_settings(0);
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(100, hb(100), hb(99)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(100, Instant::now());
let err = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect_err("no configured HTTP provider must be fail-closed");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[test]
fn flush_timeout_label_survives_an_empty_provider_set() {
let ws_host = "ws-provider.example.invalid";
let label = flush_timeout_label(&n_http_provider_settings(0), ws_host);
assert!(
label.contains(ws_host),
"the label must still name the WS host with no HTTP provider \
configured: {label}"
);
}
#[test]
fn flush_timeout_label_names_the_first_configured_provider() {
let label = flush_timeout_label(&n_http_provider_settings(2), "ws-host.invalid");
assert!(
label.contains("ws-host.invalid") && label.contains("unused-0"),
"the label must name the WS host and the first HTTP provider: {label}"
);
}
struct AlwaysOnSubscriber;
impl tracing::Subscriber for AlwaysOnSubscriber {
fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
fn event(&self, _event: &tracing::Event<'_>) {}
fn enter(&self, _span: &tracing::span::Id) {}
fn exit(&self, _span: &tracing::span::Id) {}
}
#[tokio::test]
async fn flush_window_range_self_heal_with_stale_ring_does_not_panic() {
let asserter = Asserter::new();
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(50, hb(50), hb(49)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(100, Instant::now());
assert_eq!(coalescer.pending_range(), Some((100, 100)));
let subscriber_guard = tracing::subscriber::set_default(AlwaysOnSubscriber);
tracing::callsite::rebuild_interest_cache();
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect(
"a stale ring must not panic — collect_window_hashes's own guard \
no-ops the walk",
);
assert!(
ranges.is_empty(),
"no hashes are resolvable above the stale ring's tip"
);
drop(subscriber_guard);
tracing::callsite::rebuild_interest_cache();
}
#[tokio::test]
async fn flush_window_range_self_heal_walk_uses_reused_http_client() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(105, hb(105), hb(104)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(101), WINDOW);
coalescer.observe_head(105, Instant::now());
assert_eq!(coalescer.pending_range(), Some((101, 105)));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(105, hb(105), hb(104)));
asserter.push_success(&block_with_parent_at(104, hb(104), hb(103)));
asserter.push_success(&block_with_parent_at(103, hb(103), hb(102)));
asserter.push_success(&block_with_parent_at(102, hb(102), hb(101)));
for _ in 101u64..=105 {
asserter.push_success(&serde_json::json!([]));
}
let reused = vec![mock_http_provider(asserter)];
let settings = ProviderSettings::new(
vec![HttpRpcSettings::new("http://unused".to_string(), 1, 100)],
vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let ranges = flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect(
"self-heal walk must resolve its client from `reused`, not rebuild via connect_http",
);
assert_eq!(ranges.len(), 5, "one LogRange per block 101..=105");
for (i, range) in ranges.iter().enumerate() {
let n = 101u64 + i as u64;
assert_eq!((range.from_block, range.to_block), (n, n));
}
}
#[tokio::test]
async fn flush_window_range_part_a_seeds_ring_down_to_the_flush_cursor() {
let mut linkage = HeadLinkage::new(5);
assert_eq!(linkage.observe(105, hb(105), hb(104)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(101), WINDOW);
coalescer.observe_head(105, Instant::now());
assert_eq!(coalescer.pending_range(), Some((101, 105)));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(105, hb(105), hb(104)));
asserter.push_success(&block_with_parent_at(104, hb(104), hb(103)));
asserter.push_success(&block_with_parent_at(103, hb(103), hb(102)));
asserter.push_success(&block_with_parent_at(102, hb(102), hb(101)));
for _ in 101u64..=105 {
asserter.push_success(&serde_json::json!([]));
}
let reused = vec![mock_http_provider(asserter)];
let settings = n_http_provider_settings(1);
let ranges = flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect("flush must succeed");
assert_eq!(ranges.len(), 5, "one LogRange per block 101..=105");
assert_eq!(
linkage.floor(),
Some(101),
"Part A must drop the floor to the flush cursor (from_block)"
);
assert_eq!(
linkage.lowest_worth_recording(),
linkage.floor(),
"depth was sized so the ring is already full — the top-up has \
nothing left to walk"
);
}
#[tokio::test]
async fn flush_window_range_part_b_deepens_ring_without_a_self_heal_walk() {
const DEPTH: u64 = 96;
const TIP: u64 = 200;
const TARGET: u64 = TIP - (DEPTH - 1);
let mut linkage = HeadLinkage::new(DEPTH);
assert_eq!(
linkage.observe(TIP, hb(TIP as u8), hb((TIP - 1) as u8)),
LinkOutcome::Linked
);
let mut steps = vec![REORG_RING_TOP_UP_STEPS; 11];
steps.push(TIP - TARGET - 11 * REORG_RING_TOP_UP_STEPS); assert_eq!(
steps.iter().sum::<u64>(),
TIP - TARGET,
"sanity: steps span the full descent"
);
let asserter = Asserter::new();
let mut floor = TIP;
for &step in &steps {
asserter.push_success(&serde_json::json!([]));
for n in ((floor - step + 1)..=floor).rev() {
asserter.push_success(&block_with_parent_at(n, hb(n as u8), hb((n - 1) as u8)));
}
floor -= step;
}
assert_eq!(floor, TARGET);
let asserter_handle = asserter.clone();
let reused = vec![mock_http_provider(asserter)];
let settings = n_http_provider_settings(1);
let mut expected_floor = TIP;
for (i, &step) in steps.iter().enumerate() {
let mut coalescer = HeadCoalescer::new(Some(TIP), WINDOW);
coalescer.observe_head(TIP, Instant::now());
assert_eq!(coalescer.pending_range(), Some((TIP, TIP)));
flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect("flush must succeed");
expected_floor -= step;
assert_eq!(
linkage.floor(),
Some(expected_floor),
"flush {i}: floor must descend by exactly this step"
);
}
assert_eq!(
steps[0], REORG_RING_TOP_UP_STEPS,
"the first flush's descent must be the full cap"
);
assert_eq!(linkage.floor(), Some(TARGET));
assert_eq!(linkage.lowest_worth_recording(), Some(TARGET));
let mut coalescer = HeadCoalescer::new(Some(TIP), WINDOW);
coalescer.observe_head(TIP, Instant::now());
asserter_handle.push_success(&serde_json::json!([])); flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect("flush must succeed even once the ring is already full");
assert_eq!(
linkage.floor(),
Some(TARGET),
"floor must stop at the accessor, not below it"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_resolves_after_ring_deepens_past_the_ancestor() {
const DEPTH: u64 = 96;
const HEAD: u64 = 200;
const ANCESTOR: u64 = 170;
const CHILD_NUMBER: u64 = HEAD + 2;
fn fork_hash(n: u64) -> B256 {
hb((50 + (n - ANCESTOR)) as u8)
}
let detached_parent = fork_hash(CHILD_NUMBER - 1);
let mut undeepened = HeadLinkage::new(DEPTH);
assert_eq!(
undeepened.observe(HEAD, hb(HEAD as u8), hb((HEAD - 1) as u8)),
LinkOutcome::Linked
);
assert_eq!(undeepened.floor(), Some(HEAD));
let undeepened_asserter = Asserter::new();
undeepened_asserter.push_success(&block_with_parent(detached_parent, fork_hash(HEAD)));
let undeepened_provider = mock_http_provider(undeepened_asserter);
let settings = n_http_provider_settings(1);
let undeepened_result = walk_to_common_ancestor(
&undeepened_provider,
&settings,
None,
&undeepened,
detached_parent,
CHILD_NUMBER,
DEPTH,
)
.await;
assert!(
matches!(undeepened_result, Err(RpcError::ReorgBeyondRing)),
"got {undeepened_result:?}"
);
let mut deepened = HeadLinkage::new(DEPTH);
assert_eq!(
deepened.observe(HEAD, hb(HEAD as u8), hb((HEAD - 1) as u8)),
LinkOutcome::Linked
);
let lowest_worth_recording = HEAD - (DEPTH - 1);
let flush_asserter = Asserter::new();
let mut floor = HEAD;
while floor > ANCESTOR {
flush_asserter.push_success(&serde_json::json!([])); let walk_from =
lowest_worth_recording.max(floor.saturating_sub(REORG_RING_TOP_UP_STEPS));
for n in ((walk_from + 1)..=floor).rev() {
flush_asserter.push_success(&block_with_parent_at(
n,
hb(n as u8),
hb((n - 1) as u8),
));
}
floor = walk_from;
}
let flush_reused = vec![mock_http_provider(flush_asserter)];
while deepened.floor().expect("bootstrapped") > ANCESTOR {
let mut coalescer = HeadCoalescer::new(Some(HEAD), WINDOW);
coalescer.observe_head(HEAD, Instant::now());
flush_window_range(
&mut deepened,
&mut coalescer,
&settings,
None,
0,
Some(&flush_reused),
)
.await
.expect("flush must succeed");
}
assert!(
deepened.floor().expect("bootstrapped") <= ANCESTOR,
"the ring must deepen past the ancestor before the reorg arrives"
);
let walk_asserter = Asserter::new();
for n in ((ANCESTOR + 1)..CHILD_NUMBER).rev() {
let parent = if n == ANCESTOR + 1 {
hb(ANCESTOR as u8)
} else {
fork_hash(n - 1)
};
walk_asserter.push_success(&block_with_parent(fork_hash(n), parent));
}
let walk_provider = mock_http_provider(walk_asserter);
let deepened_result = walk_to_common_ancestor(
&walk_provider,
&settings,
None,
&deepened,
detached_parent,
CHILD_NUMBER,
DEPTH,
)
.await;
assert!(
matches!(deepened_result, Ok(ANCESTOR)),
"got {deepened_result:?}"
);
}
#[tokio::test]
async fn flush_window_range_empty_http_providers_is_fail_closed_not_a_panic() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(100, hb(100), hb(99)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(100, Instant::now());
assert_eq!(coalescer.pending_range(), Some((100, 100)));
let settings = ProviderSettings::new(
vec![], vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let err = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect_err("zero HTTP providers must be fail-closed, not panic");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[test]
#[should_panic(expected = "resolve_http_provider: no HTTP providers configured")]
fn resolve_http_provider_panics_with_labeled_message_on_empty_http_providers() {
let settings = ProviderSettings::new(
vec![], vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
resolve_http_provider(&settings, None, 0);
}
#[test]
fn window_boundary() {
let tip: u64 = 1000;
let depth: u64 = 50;
let window_floor = tip.saturating_sub(depth);
for n in [0u64, 500, 949] {
assert!(
n < window_floor,
"block {n} should be below the window floor {window_floor}"
);
}
for n in [950u64, 975, 1000] {
assert!(
n >= window_floor,
"block {n} should be inside the by-hash window (floor {window_floor})"
);
}
let no_window_floor = tip.saturating_sub(0);
assert_eq!(no_window_floor, tip, "depth=0 must not create a window");
let huge_depth_floor = tip.saturating_sub(tip + 1);
assert_eq!(
huge_depth_floor, 0,
"depth > tip must floor at 0 (all blocks in window)"
);
}
#[tokio::test]
async fn flush_window_range_partial_failure_leaves_cursor_unmoved() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(200, hb(200), hb(199)), LinkOutcome::Linked);
assert_eq!(linkage.observe(201, hb(201), hb(200)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(200), WINDOW);
coalescer.observe_head(201, Instant::now());
assert_eq!(coalescer.pending_range(), Some((200, 201)));
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!([])); asserter.push_failure_msg("transient error");
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let result =
flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None).await;
assert!(result.is_err(), "partial failure must return Err");
assert_eq!(
coalescer.pending_range(),
Some((200, 201)),
"cursor must stay at 200 so block 200 is not silently skipped on reconnect"
);
}
#[tokio::test(start_paused = true)]
async fn bound_flush_time_disabled_at_zero_awaits_normally() {
let result: Result<u32, RpcError> = bound_flush_time(0, "unused at max_secs=0", async {
tokio::time::sleep(Duration::from_millis(50)).await;
Ok(42)
})
.await;
assert_eq!(
result.unwrap(),
42,
"max_secs=0 must not time out, however long the future takes"
);
}
#[tokio::test(start_paused = true)]
async fn bound_flush_time_fires_on_a_slow_future() {
let result: Result<u32, RpcError> = bound_flush_time(1, "ws=test-host", async {
tokio::time::sleep(Duration::from_millis(1200)).await;
Ok(42)
})
.await;
assert!(
matches!(result, Err(RpcError::ConnectionError(_))),
"a future exceeding max_secs must yield ConnectionError, got {result:?}"
);
}
#[tokio::test(start_paused = true)]
async fn bound_flush_time_timeout_names_both_providers_and_keeps_the_tail() {
let ws_host = "ws-provider.example.invalid";
let fallback_host = "fallback-provider.example.invalid";
let label = format!("ws={ws_host} fallback={fallback_host}");
let result: Result<u32, RpcError> = bound_flush_time(1, &label, async {
tokio::time::sleep(Duration::from_millis(1200)).await;
Ok(42)
})
.await;
let err = result.expect_err("a future exceeding max_secs must time out");
let display = err.to_string();
let ws_pos = display
.find(ws_host)
.expect("timeout message must name the WS host");
let fallback_pos = display
.find(fallback_host)
.expect("timeout message must name the fallback host");
assert!(
ws_pos < 100 && fallback_pos < 100,
"both hosts must land before byte 100 of the error's Display output \
so identity survives a tight cap, got ws@{ws_pos} fallback@{fallback_pos}: \
{display}"
);
let log_line = flush_failure_log_line(&err, classify_flush_failure(&err));
assert!(
log_line.contains("stalled mid-flush; switching provider"),
"the log line must carry the message complete through \
'switching provider', not cut mid-word: {log_line}"
);
}
#[tokio::test(start_paused = true)]
async fn flush_window_range_timeout_mid_top_up_drops_cursor_advance() {
use alloy::rpc::client::RpcClient;
use alloy::rpc::json_rpc::{RequestPacket, ResponsePacket};
use alloy::transports::TransportError;
use alloy::transports::TransportFut;
use alloy::transports::mock::MockTransport;
#[derive(Clone)]
struct DelayingTransport {
inner: MockTransport,
delay: Duration,
}
impl tower::Service<RequestPacket> for DelayingTransport {
type Response = ResponsePacket;
type Error = TransportError;
type Future = TransportFut<'static>;
fn poll_ready(
&mut self,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Result<(), Self::Error>> {
tower::Service::<RequestPacket>::poll_ready(&mut self.inner, cx)
}
fn call(&mut self, req: RequestPacket) -> Self::Future {
let mut inner = self.inner.clone();
let delay_this = req.method_names().any(|m| m == "eth_getBlockByHash");
let delay = self.delay;
Box::pin(async move {
if delay_this {
tokio::time::sleep(delay).await;
}
tower::Service::call(&mut inner, req).await
})
}
}
let mut linkage = HeadLinkage::new(96);
assert_eq!(linkage.observe(200, hb(200), hb(199)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(200), WINDOW);
coalescer.observe_head(200, Instant::now());
let pending_before_flush = coalescer.pending_range();
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!([])); let transport = DelayingTransport {
inner: MockTransport::new(asserter),
delay: Duration::from_millis(1_200),
};
let provider: DynProvider<Ethereum> = ProviderBuilder::new()
.disable_recommended_fillers()
.connect_client(RpcClient::new(transport, false))
.erased();
let settings = ProviderSettings::from_mock(provider);
let result = bound_flush_time(
1,
"ws=test-host fallback=test-host",
flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None),
)
.await;
assert!(
matches!(result, Err(RpcError::ConnectionError(_))),
"a top-up stalled past the flush bound must surface as a timeout, got {result:?}"
);
assert_eq!(
coalescer.pending_range(),
pending_before_flush,
"a timeout mid-top-up must leave the cursor exactly where the flush found it — \
the fetched ranges and the top-up's own walk both dropped together"
);
}
#[tokio::test(start_paused = true)]
async fn idle_head_await_returns_a_header_well_within_the_bound() {
let mut s = futures_util::stream::iter(vec![Header::default()]);
let outcome = await_head_within_idle_bound(&mut s).await;
assert!(
matches!(outcome, IdleHeadAwait::Header(_)),
"a stream with a ready header must resolve to it, not expire"
);
}
#[tokio::test(start_paused = true)]
async fn idle_head_await_distinguishes_stream_ended_from_expiry() {
let mut s = futures_util::stream::empty::<Header>();
let outcome = await_head_within_idle_bound(&mut s).await;
assert!(
matches!(outcome, IdleHeadAwait::StreamEnded),
"an ended stream must resolve to StreamEnded, not Expired"
);
}
#[tokio::test(start_paused = true)]
async fn idle_head_await_expires_on_a_silent_stream() {
let mut s = futures_util::stream::pending::<Header>();
let outcome = tokio::time::timeout(
Duration::from_secs(IDLE_HEAD_AWAIT_SECS + 60),
await_head_within_idle_bound(&mut s),
)
.await
.expect("the idle bound must expire on its own, not hang past it");
assert!(
matches!(outcome, IdleHeadAwait::Expired),
"a silently pending stream must resolve to Expired once the idle bound elapses"
);
}
#[test]
fn idle_rotation_counter_continues_below_threshold() {
let mut counter = IdleRotationCounter::new();
assert!(matches!(counter.expire(3), IdleRotation::Continue));
assert!(matches!(counter.expire(3), IdleRotation::Continue));
}
#[test]
fn idle_rotation_counter_fires_at_exactly_the_configured_provider_count() {
let mut counter = IdleRotationCounter::new();
assert!(matches!(counter.expire(3), IdleRotation::Continue));
assert!(matches!(counter.expire(3), IdleRotation::Continue));
assert!(matches!(counter.expire(3), IdleRotation::AllIdle));
}
#[test]
fn idle_rotation_counter_fires_at_one_for_a_single_provider() {
let mut counter = IdleRotationCounter::new();
assert!(matches!(counter.expire(1), IdleRotation::AllIdle));
}
#[test]
fn idle_rotation_counter_header_resets_the_count() {
let mut counter = IdleRotationCounter::new();
assert!(matches!(counter.expire(3), IdleRotation::Continue));
assert!(matches!(counter.expire(3), IdleRotation::Continue));
counter.reset();
assert!(matches!(counter.expire(3), IdleRotation::Continue));
assert!(matches!(counter.expire(3), IdleRotation::Continue));
assert!(matches!(counter.expire(3), IdleRotation::AllIdle));
}
#[tokio::test]
async fn live_loop_wiring_gap_then_reorg() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(101), WINDOW);
let mut linkage = HeadLinkage::new(8);
assert!(coalescer.observe_head(105, now));
assert_eq!(coalescer.pending_range(), Some((101, 105)));
assert_eq!(linkage.observe(105, hb(105), hb(104)), LinkOutcome::Linked);
assert_eq!(linkage.latest(), Some((105, hb(105))));
let after_window = now + WINDOW + Duration::from_millis(1);
assert!(
coalescer.should_flush(after_window),
"coalesce window expired — flush must fire"
);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(105, hb(105), hb(104)));
asserter.push_success(&block_with_parent_at(104, hb(104), hb(103)));
asserter.push_success(&block_with_parent_at(103, hb(103), hb(102)));
asserter.push_success(&block_with_parent_at(102, hb(102), hb(101)));
for _ in 101u64..=105 {
asserter.push_success(&serde_json::json!([]));
}
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect("gap self-heal must succeed");
assert_eq!(ranges.len(), 5, "one range per block 101..=105");
for (i, range) in ranges.iter().enumerate() {
let n = 101u64 + i as u64;
assert_eq!(
(range.from_block, range.to_block),
(n, n),
"range at index {i} must be block {n}"
);
}
assert_eq!(
coalescer.pending_range(),
None,
"cursor must be at 106 after flush"
);
assert_eq!(linkage.observe(106, hb(106), hb(105)), LinkOutcome::Linked);
assert_eq!(linkage.observe(107, hb(107), hb(106)), LinkOutcome::Linked);
let outcome = linkage.observe(107, hb(250), hb(106));
assert!(
matches!(
outcome,
LinkOutcome::Reorg {
common_ancestor: 106
}
),
"competing block at 107 with canonical parent must produce Reorg{{106}}, got {outcome:?}"
);
}
#[test]
fn capture_policy_notice_speaks_once_per_policy() {
let notice = CapturePolicyNotice::new();
let latest_block = 1_000u64;
let latest_first = notice.first_message(CaptureTarget::Latest, latest_block);
assert!(
latest_first.is_some(),
"first Latest resolution must announce"
);
let latest_message = latest_first.unwrap();
assert!(
!latest_message.contains("DEEP LAG"),
"Latest must not claim deep lag: {latest_message}"
);
assert!(
notice
.first_message(CaptureTarget::Latest, latest_block)
.is_none(),
"second Latest resolution must be silent"
);
assert!(
notice
.first_message(CaptureTarget::Latest, latest_block)
.is_none(),
"third Latest resolution must be silent"
);
let lag_block = 900u64;
assert!(
notice
.first_message(CaptureTarget::Lag(64), lag_block)
.is_some(),
"first Lag resolution must announce (Latest already having spoken must not \
consume Lag's independent gate)"
);
assert!(
notice
.first_message(CaptureTarget::Lag(64), lag_block)
.is_none(),
"second Lag resolution must be silent"
);
assert!(
notice
.first_message(CaptureTarget::Lag(64), lag_block)
.is_none(),
"third Lag resolution must be silent"
);
assert!(
notice
.first_message(CaptureTarget::Lag(200), lag_block)
.is_none(),
"gating is per-variant, not per-value: a distinct Lag(n) must not re-arm the gate"
);
let finalized_block = 500u64;
let finalized_first = notice.first_message(CaptureTarget::Finalized, finalized_block);
assert!(
finalized_first.is_some(),
"first Finalized resolution must announce (Latest and Lag having already \
spoken must not consume Finalized's independent gate)"
);
let finalized_message = finalized_first.unwrap();
assert!(
finalized_message.contains("DEEP LAG ACTIVE"),
"Finalized message must claim deep lag: {finalized_message}"
);
assert!(
finalized_message.contains(&finalized_block.to_string()),
"Finalized message must report the resolved block: {finalized_message}"
);
assert!(
notice
.first_message(CaptureTarget::Finalized, finalized_block)
.is_none(),
"second Finalized resolution must be silent"
);
assert!(
notice
.first_message(CaptureTarget::Finalized, finalized_block)
.is_none(),
"third Finalized resolution must be silent"
);
}
#[test]
fn resolve_http_provider_reuses_vs_rebuilds() {
let settings = ProviderSettings::new(
vec![HttpRpcSettings::new(
"http://127.0.0.1:9".to_string(),
1,
100,
)],
vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let reused = settings.build_reused_http_clients();
assert_eq!(reused.len(), 1, "one client per configured HTTP provider");
let a = resolve_http_provider(&settings, Some(&reused), 0);
let b = resolve_http_provider(&settings, Some(&reused), 0);
assert!(
std::sync::Weak::ptr_eq(&a.weak_client(), &b.weak_client()),
"Some(reused) at the same index must return the identical client"
);
let c = resolve_http_provider(&settings, None, 0);
let d = resolve_http_provider(&settings, None, 0);
assert!(
!std::sync::Weak::ptr_eq(&c.weak_client(), &d.weak_client()),
"None must rebuild a fresh client on every call, not reuse one"
);
}
#[tokio::test(start_paused = true)]
async fn fetch_concurrently_runs_items_in_parallel_up_to_bound() {
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let bound = 4usize;
let n = 10usize;
let in_flight = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let items: Vec<usize> = (0..n).collect();
let in_flight_for_fetch = in_flight.clone();
let peak_for_fetch = peak.clone();
let fetch = move |_: usize| {
let in_flight = in_flight_for_fetch.clone();
let peak = peak_for_fetch.clone();
async move {
let now = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(now, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(50)).await;
in_flight.fetch_sub(1, Ordering::SeqCst);
Ok::<usize, RpcError>(0)
}
};
let result = fetch_concurrently(items, bound, fetch).await;
assert!(result.is_ok(), "all items succeed: {result:?}");
let observed_peak = peak.load(Ordering::SeqCst);
assert!(
observed_peak > 1,
"expected concurrent dispatch (peak > 1), got peak={observed_peak}"
);
assert!(
observed_peak <= bound,
"peak in-flight {observed_peak} exceeded the bound {bound}"
);
}
#[tokio::test]
async fn fetch_concurrently_fails_fast_not_collect_then_check() {
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let bound = 3usize;
let n = 10usize;
let items: Vec<usize> = (0..n).collect();
let invoked = Arc::new(AtomicUsize::new(0));
let invoked_for_fetch = invoked.clone();
let fetch = move |i: usize| {
invoked_for_fetch.fetch_add(1, Ordering::SeqCst);
async move {
if i == 0 {
Err(RpcError::LogFetchError("early failure".to_string()))
} else {
Ok::<usize, RpcError>(i)
}
}
};
let result = fetch_concurrently(items, bound, fetch).await;
assert!(result.is_err(), "an early failure must surface as Err");
let invoked_count = invoked.load(Ordering::SeqCst);
assert!(
invoked_count < n,
"expected fewer than {n} closures invoked under fail-fast, got {invoked_count}"
);
}
#[tokio::test(start_paused = true)]
async fn fetch_concurrently_fails_fast_under_early_slow_neighbor() {
let bound = 2usize;
let items = vec![0usize, 1usize];
let long_sleep = Duration::from_secs(3600);
let fetch = move |i: usize| async move {
if i == 0 {
tokio::time::sleep(long_sleep).await;
Ok::<usize, RpcError>(0)
} else {
Err(RpcError::LogFetchError("fast failure".to_string()))
}
};
let start = Instant::now();
let result = fetch_concurrently(items, bound, fetch).await;
let elapsed = start.elapsed();
assert!(result.is_err(), "overall result must be Err");
assert!(
elapsed < long_sleep,
"Err must be observable before item 0's sleep elapses, elapsed={elapsed:?}"
);
}
#[test]
fn sort_ranges_ascending_orders_by_block_number() {
let unordered: Vec<LogRange> = [105u64, 100, 103]
.into_iter()
.map(|n| LogRange {
from_block: n,
to_block: n,
logs: vec![],
tip_hash: None,
})
.collect();
let sorted = sort_ranges_ascending(unordered);
let numbers: Vec<u64> = sorted.iter().map(|range| range.from_block).collect();
assert_eq!(numbers, vec![100, 103, 105]);
}
#[tokio::test(start_paused = true)]
async fn flush_window_range_dispatches_concurrently_and_restores_ascending_order() {
use alloy::network::Ethereum;
use alloy::providers::{DynProvider, RootProvider};
use alloy::rpc::types::Filter;
use alloy::transports::TransportResult;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Clone)]
struct DelayedLogsProvider {
stub_root: DynProvider<Ethereum>,
call_order: Arc<AtomicUsize>,
delays_by_call_order: Arc<Vec<Duration>>,
in_flight: Arc<AtomicUsize>,
peak_in_flight: Arc<AtomicUsize>,
completion_order: Arc<Mutex<Vec<usize>>>,
}
#[async_trait::async_trait]
impl Provider<Ethereum> for DelayedLogsProvider {
fn root(&self) -> &RootProvider<Ethereum> {
self.stub_root.root()
}
async fn get_logs(&self, _filter: &Filter) -> TransportResult<Vec<Log>> {
let call_index = self.call_order.fetch_add(1, Ordering::SeqCst);
let now_in_flight = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1;
self.peak_in_flight
.fetch_max(now_in_flight, Ordering::SeqCst);
let delay = self
.delays_by_call_order
.get(call_index)
.copied()
.unwrap_or(Duration::ZERO);
tokio::time::sleep(delay).await;
self.in_flight.fetch_sub(1, Ordering::SeqCst);
self.completion_order.lock().unwrap().push(call_index);
Ok(Vec::new())
}
}
let mut linkage = HeadLinkage::new(8);
for n in 300u64..=303 {
assert_eq!(
linkage.observe(n, hb(n as u8), hb((n - 1) as u8)),
LinkOutcome::Linked
);
}
let mut coalescer = HeadCoalescer::new(Some(300), WINDOW);
coalescer.observe_head(303, Instant::now());
assert_eq!(coalescer.pending_range(), Some((300, 303)));
let delays_by_call_order = vec![
Duration::from_millis(500), Duration::from_millis(10), Duration::from_millis(20), Duration::from_millis(30), ];
let peak_in_flight = Arc::new(AtomicUsize::new(0));
let completion_order = Arc::new(Mutex::new(Vec::new()));
let unused_asserter = Asserter::new(); let double = DelayedLogsProvider {
stub_root: ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(unused_asserter)
.erased(),
call_order: Arc::new(AtomicUsize::new(0)),
delays_by_call_order: Arc::new(delays_by_call_order),
in_flight: Arc::new(AtomicUsize::new(0)),
peak_in_flight: peak_in_flight.clone(),
completion_order: completion_order.clone(),
};
let reused = vec![double.erased()];
let settings = ProviderSettings::new(
vec![HttpRpcSettings::new("http://unused".to_string(), 4, 100)],
vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let ranges = flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect("flush must succeed");
assert_eq!(
peak_in_flight.load(Ordering::SeqCst),
4,
"all 4 blocks' fetches must be in flight together, not one at a time"
);
let observed_completion_order = completion_order.lock().unwrap().clone();
assert_eq!(
&observed_completion_order[0..3],
&[1, 2, 3],
"the 3 fast fetches (301-303) must resolve before the slow one (300), \
got order {observed_completion_order:?}"
);
assert_eq!(
observed_completion_order[3], 0,
"the slow fetch (300) must resolve last, proving it did not block the \
others, got order {observed_completion_order:?}"
);
assert_eq!(ranges.len(), 4);
let numbers: Vec<u64> = ranges.iter().map(|range| range.from_block).collect();
assert_eq!(numbers, vec![300, 301, 302, 303]);
}
#[tokio::test(start_paused = true)]
async fn flush_window_range_peak_in_flight_tracks_configured_max_concurrency() {
use alloy::network::Ethereum;
use alloy::providers::{DynProvider, RootProvider};
use alloy::rpc::types::Filter;
use alloy::transports::TransportResult;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Clone)]
struct PeakTrackingLogsProvider {
stub_root: DynProvider<Ethereum>,
in_flight: Arc<AtomicUsize>,
peak_in_flight: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl Provider<Ethereum> for PeakTrackingLogsProvider {
fn root(&self) -> &RootProvider<Ethereum> {
self.stub_root.root()
}
async fn get_logs(&self, _filter: &Filter) -> TransportResult<Vec<Log>> {
let now_in_flight = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1;
self.peak_in_flight
.fetch_max(now_in_flight, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(20)).await;
self.in_flight.fetch_sub(1, Ordering::SeqCst);
Ok(Vec::new())
}
}
let mut linkage = HeadLinkage::new(8);
for n in 400u64..=405 {
assert_eq!(
linkage.observe(n, hb(n as u8), hb((n - 1) as u8)),
LinkOutcome::Linked
);
}
let mut coalescer = HeadCoalescer::new(Some(400), WINDOW);
coalescer.observe_head(405, Instant::now());
assert_eq!(coalescer.pending_range(), Some((400, 405)));
let peak_in_flight = Arc::new(AtomicUsize::new(0));
let unused_asserter = Asserter::new();
let double = PeakTrackingLogsProvider {
stub_root: ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(unused_asserter)
.erased(),
in_flight: Arc::new(AtomicUsize::new(0)),
peak_in_flight: peak_in_flight.clone(),
};
let reused = vec![double.erased()];
let settings = ProviderSettings::new(
vec![HttpRpcSettings::new("http://unused".to_string(), 2, 100)],
vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let ranges = flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect("flush must succeed");
assert_eq!(
peak_in_flight.load(Ordering::SeqCst),
2,
"peak in-flight must equal the configured max_concurrency (2), not \
the pending block count (6) or any other hardcoded bound"
);
assert_eq!(ranges.len(), 6);
}
}