use std::collections::VecDeque;
use std::future::Future;
use std::sync::Once;
use std::time::Duration;
use alloy::{
network::Ethereum,
primitives::B256,
providers::{DynProvider, Provider},
rpc::types::{BlockNumberOrTag, Filter, Header, Log},
};
use async_stream::stream;
use futures_util::{StreamExt, TryStreamExt, pin_mut, stream::Stream};
use tokio::time::{Instant, sleep, sleep_until};
use tracing::{debug, warn};
use crate::rpc::cursor::{TipGate, TipWaitOutcome, next_cursor_after_batch, wait_for_tip};
use crate::rpc::error_tracking::{
ErrorCategory, ProviderErrorTracker, find_active_provider, truncate_error_short,
};
use crate::rpc::{
LogRange, RpcError,
config::{CaptureTarget, ProviderSettings},
progress::ScanProgress,
};
const MAX_BLOCKS_PER_WS_REQUEST: u64 = 300;
#[derive(Debug, Clone)]
struct HeadCoalescer {
block_cursor: Option<u64>,
latest_seen_head: Option<u64>,
flush_deadline: Option<Instant>,
window: Duration,
progress: Option<ScanProgress>,
}
impl HeadCoalescer {
fn new(block_cursor: Option<u64>, window: Duration) -> Self {
Self {
block_cursor,
latest_seen_head: None,
flush_deadline: None,
window,
progress: None,
}
}
fn with_progress(mut self, progress: Option<ScanProgress>) -> Self {
self.progress = progress;
self
}
fn record_tip(&self, tip: u64) {
if let Some(p) = &self.progress {
p.record_observed_tip(tip);
}
}
fn block_cursor(&self) -> Option<u64> {
self.block_cursor
}
fn observe_head(&mut self, head_number: u64, now: Instant) -> bool {
let from_block = self.block_cursor.unwrap_or(head_number);
if head_number < from_block {
return false;
}
self.latest_seen_head = Some(
self.latest_seen_head
.map_or(head_number, |latest| latest.max(head_number)),
);
if self.flush_deadline.is_none() {
self.flush_deadline = Some(now + self.window);
}
true
}
fn pending_range(&self) -> Option<(u64, u64)> {
let to_block = self.latest_seen_head?;
let from_block = self.block_cursor.unwrap_or(to_block);
(to_block >= from_block).then_some((from_block, to_block))
}
fn flush_deadline(&self) -> Option<Instant> {
self.pending_range()?;
self.flush_deadline
}
fn should_flush(&self, now: Instant) -> bool {
let Some((from_block, to_block)) = self.pending_range() else {
return false;
};
if to_block - from_block + 1 >= MAX_BLOCKS_PER_WS_REQUEST {
return true;
}
self.flush_deadline.is_some_and(|deadline| now >= deadline)
}
fn advance_cursor(&mut self, next_block: u64) {
self.block_cursor = Some(next_block);
if let Some(p) = &self.progress {
p.record_scanned_to(next_block.saturating_sub(1));
}
if self
.latest_seen_head
.is_some_and(|latest| next_block > latest)
{
self.latest_seen_head = None;
self.flush_deadline = None;
}
}
fn defer_flush(&mut self) {
self.flush_deadline = None;
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum LinkOutcome {
Linked,
Reorg { common_ancestor: u64 },
Detached {
parent_hash: B256,
child_number: u64,
},
BeyondFinality,
}
#[derive(Debug, Clone)]
struct HeadLinkage {
ring: VecDeque<(u64, B256)>,
depth: u64,
}
impl HeadLinkage {
fn new(depth: u64) -> Self {
Self {
ring: VecDeque::new(),
depth,
}
}
fn enabled(&self) -> bool {
self.depth > 0
}
fn floor(&self) -> Option<u64> {
self.ring.front().map(|&(number, _)| number)
}
fn hash_at(&self, number: u64) -> Option<B256> {
let &(front, _) = self.ring.front()?;
let offset = number.checked_sub(front)? as usize;
self.ring.get(offset).map(|&(_, hash)| hash)
}
fn matches(&self, number: u64, hash: B256) -> bool {
self.hash_at(number) == Some(hash)
}
fn push(&mut self, number: u64, hash: B256) {
self.ring.push_back((number, hash));
while self.ring.len() as u64 > self.depth {
self.ring.pop_front();
}
}
fn latest(&self) -> Option<(u64, B256)> {
self.ring.back().copied()
}
fn reanchor(&mut self, ancestor: u64, number: u64, hash: B256) {
while self.ring.back().is_some_and(|&(n, _)| n > ancestor) {
self.ring.pop_back();
}
self.push(number, hash);
}
fn observe(&mut self, number: u64, hash: B256, parent_hash: B256) -> LinkOutcome {
if !self.enabled() {
return LinkOutcome::Linked;
}
let Some(&(tip_number, tip_hash)) = self.ring.back() else {
self.push(number, hash);
return LinkOutcome::Linked;
};
if number == tip_number + 1 && parent_hash == tip_hash {
self.push(number, hash);
return LinkOutcome::Linked;
}
if number > tip_number + 1 {
return LinkOutcome::Detached {
parent_hash,
child_number: number,
};
}
let Some(parent_number) = number.checked_sub(1) else {
return LinkOutcome::BeyondFinality;
};
match self.hash_at(parent_number) {
Some(canonical_parent) if canonical_parent == parent_hash => {
if self.hash_at(number) == Some(hash) {
return LinkOutcome::Linked;
}
self.reanchor(parent_number, number, hash);
LinkOutcome::Reorg {
common_ancestor: parent_number,
}
}
Some(_) => LinkOutcome::Detached {
parent_hash,
child_number: number,
},
None => LinkOutcome::BeyondFinality,
}
}
}
const MAX_STEP_RESOLUTION_ROUNDS: u32 = 3;
const MIN_ABSENCE_WITNESSES: usize = 2;
enum ProviderAnswer {
Found(Box<Header>),
Absent,
Inconclusive(Option<String>),
}
impl ProviderAnswer {
fn label(&self) -> String {
match self {
ProviderAnswer::Found(_) => "found".to_string(),
ProviderAnswer::Absent => "absent".to_string(),
ProviderAnswer::Inconclusive(None) => "inconclusive(wrong identity)".to_string(),
ProviderAnswer::Inconclusive(Some(err)) => format!("inconclusive({err})"),
}
}
}
#[derive(Debug)]
enum StepFailure {
AbsenceConfirmed,
Unresolved(RpcError),
}
#[allow(clippy::too_many_arguments)]
async fn resolve_step(
caller: &str,
providers: &ProviderSettings,
reused: Option<&[DynProvider<Ethereum>]>,
n: usize,
indices: &[usize],
hash: B256,
primary_answer: Option<(usize, ProviderAnswer)>,
rounds_used: &mut u32,
steps_completed: u64,
is_match: impl Fn(&Header) -> bool,
) -> Result<Box<Header>, StepFailure> {
if *rounds_used >= MAX_STEP_RESOLUTION_ROUNDS {
warn!(
"{caller}: step-resolution budget exhausted at hash {hash} — \
{MAX_STEP_RESOLUTION_ROUNDS} round(s) already spent this walk, {steps_completed} \
step(s) completed — treating the primary's answer as transport weather"
);
return Err(StepFailure::Unresolved(RpcError::ConnectionError(format!(
"{caller}: exhausted {MAX_STEP_RESOLUTION_ROUNDS} step-resolution rounds for block \
{hash}"
))));
}
*rounds_used += 1;
let mut answers: Vec<(usize, ProviderAnswer)> =
futures_util::future::join_all(indices.iter().map(|&idx| {
let provider = resolve_http_provider(providers, reused, idx);
let is_match = &is_match;
async move {
let answer = match provider.get_block_by_hash(hash).await {
Ok(Some(block)) if is_match(&block.header) => {
ProviderAnswer::Found(Box::new(block.header))
}
Ok(Some(_)) => ProviderAnswer::Inconclusive(None),
Ok(None) => ProviderAnswer::Absent,
Err(e) => ProviderAnswer::Inconclusive(Some(truncate_error_short(e, 100))),
};
(idx, answer)
}
}))
.await;
if let Some(primary) = primary_answer {
answers.push(primary);
}
let answers_desc = answers
.iter()
.map(|(idx, answer)| format!("http[{idx}]={}", answer.label()))
.collect::<Vec<_>>()
.join(", ");
warn!(
"{caller}: resolving step at hash {hash} from other providers — {steps_completed} step(s) \
completed, round {rounds_used}/{MAX_STEP_RESOLUTION_ROUNDS}, answers [{answers_desc}]"
);
let absent = answers
.iter()
.filter(|(_, answer)| matches!(answer, ProviderAnswer::Absent))
.count();
if let Some(header) = answers.into_iter().find_map(|(_, answer)| match answer {
ProviderAnswer::Found(header) => Some(header),
_ => None,
}) {
return Ok(header);
}
if absent >= n.min(MIN_ABSENCE_WITNESSES) {
Err(StepFailure::AbsenceConfirmed)
} else {
Err(StepFailure::Unresolved(RpcError::ConnectionError(format!(
"{caller}: block {hash} unresolved after {rounds_used} round(s) — {absent} of \
{n} provider(s) reported it absent, fewer than the {} required: {answers_desc}",
n.min(MIN_ABSENCE_WITNESSES)
))))
}
}
fn provider_set_preconditions(
n: usize,
reused: Option<&[DynProvider<Ethereum>]>,
) -> Option<RpcError> {
if n == 0 {
return Some(RpcError::ConnectionError(
"no HTTP providers configured to resolve reorg-walk steps".to_string(),
));
}
if let Some(set) = reused
&& set.len() != n
{
return Some(RpcError::ConnectionError(format!(
"reused HTTP client count {} does not match configured provider count {n}",
set.len(),
)));
}
None
}
pub(crate) async fn collect_window_hashes(
provider: &DynProvider<Ethereum>,
providers: &ProviderSettings,
reused: Option<&[DynProvider<Ethereum>]>,
primary_index: usize,
tip_hash: B256,
tip_number: u64,
from_block: u64,
) -> Result<Vec<(u64, B256)>, RpcError> {
let n = providers.http_providers.len();
if let Some(err) = provider_set_preconditions(n, reused) {
return Err(err);
}
if tip_number < from_block {
return Ok(Vec::new());
}
let primary_index = primary_index % n;
let other_indices: Vec<usize> = (0..n).filter(|&idx| idx != primary_index).collect();
let mut collected: Vec<(u64, B256)> = Vec::new();
let mut current_hash = tip_hash;
let mut current_number = tip_number;
let mut rounds_used = 0u32;
let mut steps_completed = 0u64;
loop {
collected.push((current_number, current_hash));
if current_number == from_block {
break;
}
let hash = current_hash;
let primary = provider.get_block_by_hash(hash).await;
let header = match primary {
Ok(Some(block)) if block.header.number != current_number => {
return Err(RpcError::ReorgBeyondFinality);
}
Ok(Some(block)) if block.header.hash == hash => Box::new(block.header),
other => {
let primary_vote = match other {
Ok(Some(_)) => ProviderAnswer::Inconclusive(None), Ok(None) => ProviderAnswer::Absent,
Err(e) => ProviderAnswer::Inconclusive(Some(truncate_error_short(e, 100))),
};
match resolve_step(
"collect_window_hashes",
providers,
reused,
n,
&other_indices,
hash,
Some((primary_index, primary_vote)),
&mut rounds_used,
steps_completed,
|h| h.hash == hash && h.number == current_number,
)
.await
{
Ok(header) => header,
Err(StepFailure::AbsenceConfirmed) => {
return Err(RpcError::ReorgBeyondFinality);
}
Err(StepFailure::Unresolved(err)) => return Err(err),
}
}
};
let parent_hash = header.parent_hash;
let Some(parent_number) = current_number.checked_sub(1) else {
return Err(RpcError::ReorgBeyondFinality);
};
if parent_number < from_block {
break;
}
current_hash = parent_hash;
current_number = parent_number;
steps_completed += 1;
}
Ok(collected)
}
pub(crate) async fn fetch_logs_by_hash_range(
providers: &ProviderSettings,
pairs: &[(u64, B256)],
base_filter: Option<&Filter>,
) -> Result<Vec<LogRange>, RpcError> {
let mut ranges: Vec<LogRange> = Vec::with_capacity(pairs.len());
for &(number, hash) in pairs {
let query = base_filter
.cloned()
.unwrap_or_else(Filter::new)
.at_block_hash(hash);
let (_, logs) =
get_logs_with_http_fallback_retry(providers, &query, 0, number, number, None).await?;
ranges.push((number, number, logs));
}
Ok(ranges)
}
async fn bound_flush_time<F, T>(max_secs: u64, fut: F) -> Result<T, RpcError>
where
F: Future<Output = Result<T, RpcError>>,
{
if max_secs == 0 {
return fut.await;
}
match tokio::time::timeout(Duration::from_secs(max_secs), fut).await {
Ok(result) => result,
Err(_) => Err(RpcError::ConnectionError(format!(
"flush exceeded its {max_secs}s aggregate timeout — provider likely stalled \
mid-flush; switching provider"
))),
}
}
async fn flush_window_range(
linkage: &mut HeadLinkage,
coalescer: &mut HeadCoalescer,
providers: &ProviderSettings,
filter: Option<&Filter>,
provider_index: usize,
reused: Option<&[DynProvider<Ethereum>]>,
) -> Result<Vec<LogRange>, RpcError> {
let Some((from_block, pending_to_block)) = coalescer.pending_range() else {
return Ok(vec![]);
};
if providers.http_providers.is_empty() {
return Err(RpcError::ConnectionError(
"No HTTP providers configured for window flush".to_string(),
));
}
let to_block = std::cmp::min(from_block + MAX_BLOCKS_PER_WS_REQUEST - 1, pending_to_block);
let pairs: Vec<(u64, B256)> = if (from_block..=to_block).all(|n| linkage.hash_at(n).is_some()) {
(from_block..=to_block)
.map(|n| (n, linkage.hash_at(n).expect("just verified")))
.collect()
} else {
let Some((top_n, top_hash)) = linkage.latest() else {
return Err(RpcError::ReorgBeyondFinality);
};
let walk_provider = resolve_http_provider(providers, reused, provider_index);
warn!(
"flush window [{}, {}] missing ring entries — self-heal walking back up \
to {} blocks from ring anchor {} to {} ({})",
from_block,
to_block,
top_n.saturating_sub(from_block) + 1,
top_n,
from_block,
if top_n < from_block {
"ring anchor already trails the window's start — the walk \
immediately no-ops, nothing is fetched"
} else {
"uses the same client-reuse mechanism as the main fetch path when \
available, but the walk itself is sequential; not covered by the \
concurrent fetch path"
}
);
let mut pairs_desc = collect_window_hashes(
&walk_provider,
providers,
reused,
provider_index,
top_hash,
top_n,
from_block,
)
.await?;
pairs_desc.reverse();
pairs_desc
.into_iter()
.filter(|&(n, _)| n <= to_block)
.collect()
};
let last_pending = pairs.last().copied();
let bound = providers.http_settings(provider_index).max_concurrency as usize;
let fetch = move |(n, hash): (u64, B256)| async move {
let query = filter
.cloned()
.unwrap_or_else(Filter::new)
.at_block_hash(hash);
let (_, logs) =
get_logs_with_http_fallback_retry(providers, &query, provider_index, n, n, reused)
.await?;
Ok::<LogRange, RpcError>((n, n, logs))
};
let ranges = sort_ranges_ascending(fetch_concurrently(pairs, bound, fetch).await?);
if let Some((last, _)) = last_pending {
coalescer.advance_cursor(last + 1);
}
Ok(ranges)
}
#[allow(clippy::manual_async_fn)]
fn fetch_concurrently<In, Out, F, Fut>(
items: Vec<In>,
bound: usize,
fetch: F,
) -> impl Future<Output = Result<Vec<Out>, RpcError>> + Send
where
In: Send,
Out: Send,
F: Fn(In) -> Fut + Send,
Fut: Future<Output = Result<Out, RpcError>> + Send,
{
async move {
futures_util::stream::iter(items.into_iter().map(fetch).map(Ok::<Fut, RpcError>))
.try_buffer_unordered(bound.max(1))
.try_collect()
.await
}
}
fn sort_ranges_ascending(mut ranges: Vec<LogRange>) -> Vec<LogRange> {
ranges.sort_by_key(|&(from, _, _)| from);
ranges
}
async fn walk_to_common_ancestor(
provider: &DynProvider<Ethereum>,
providers: &ProviderSettings,
reused: Option<&[DynProvider<Ethereum>]>,
linkage: &HeadLinkage,
detached_parent: B256,
child_number: u64,
depth: u64,
) -> Result<u64, RpcError> {
let n = providers.http_providers.len();
if let Some(err) = provider_set_preconditions(n, reused) {
return Err(err);
}
let all_indices: Vec<usize> = (0..n).collect();
let mut parent_hash = detached_parent;
let Some(mut parent_number) = child_number.checked_sub(1) else {
return Err(RpcError::ReorgBeyondFinality);
};
let floor = linkage.floor().unwrap_or(u64::MAX);
let mut rounds_used = 0u32;
for steps_completed in 0..depth {
if linkage.matches(parent_number, parent_hash) {
return Ok(parent_number);
}
if parent_number <= floor {
return Err(RpcError::ReorgBeyondFinality);
}
let hash = parent_hash;
let header = match provider.get_block_by_hash(hash).await {
Ok(Some(block)) if block.header.hash == hash => Box::new(block.header),
_ => {
match resolve_step(
"walk_to_common_ancestor",
providers,
reused,
n,
&all_indices,
hash,
None, &mut rounds_used,
steps_completed,
|h| h.hash == hash,
)
.await
{
Ok(header) => header,
Err(StepFailure::AbsenceConfirmed) => {
return Err(RpcError::ReorgBeyondFinality);
}
Err(StepFailure::Unresolved(err)) => return Err(err),
}
}
};
parent_hash = header.parent_hash;
let Some(next) = parent_number.checked_sub(1) else {
return Err(RpcError::ReorgBeyondFinality);
};
parent_number = next;
}
Err(RpcError::ReorgBeyondFinality)
}
fn fallback_http_index(start_index: usize, attempt: usize, num_http_providers: usize) -> usize {
(start_index + attempt) % num_http_providers
}
fn fallback_http_exhausted_error(
from_block: u64,
to_block: u64,
attempted_hosts: &[String],
last_err: &str,
) -> RpcError {
RpcError::LogFetchError(format!(
"HTTP fallback exhausted for blocks {}-{} after trying providers [{}]: {}",
from_block,
to_block,
attempted_hosts.join(", "),
last_err
))
}
pub(crate) fn resolve_http_provider(
providers: &ProviderSettings,
reused: Option<&[DynProvider<Ethereum>]>,
idx: usize,
) -> DynProvider<Ethereum> {
match reused {
Some(set) => set[idx % set.len()].clone(),
None => providers.connect_http(idx),
}
}
async fn get_logs_with_http_fallback_retry(
providers: &ProviderSettings,
query_filter: &Filter,
start_index: usize,
from_block: u64,
to_block: u64,
reused: Option<&[DynProvider<Ethereum>]>,
) -> Result<(usize, Vec<alloy::rpc::types::Log>), RpcError> {
let num_http_providers = providers.http_providers.len();
let mut attempted_hosts: Vec<String> = Vec::with_capacity(num_http_providers);
let mut last_err = String::new();
for attempt in 0..num_http_providers {
let provider_idx = fallback_http_index(start_index, attempt, num_http_providers);
let provider_host = providers.http_settings(provider_idx).host();
attempted_hosts.push(provider_host.clone());
let http_provider = resolve_http_provider(providers, reused, provider_idx);
match http_provider.get_logs(query_filter).await {
Ok(logs) => return Ok((provider_idx, logs)),
Err(err) => {
last_err = err.to_string();
warn!(
"HTTP fallback {} failed for blocks {}-{}: {}",
provider_host,
from_block,
to_block,
truncate_error_short(&last_err, 100)
);
}
}
}
Err(fallback_http_exhausted_error(
from_block,
to_block,
&attempted_hosts,
&last_err,
))
}
enum YieldBounds {
Full,
Partial(u64),
Unattributable,
}
fn yield_bounds(logs: &[Log], to_block: u64) -> YieldBounds {
let Some(watermark) = logs.iter().filter_map(|log| log.block_number).max() else {
return YieldBounds::Unattributable;
};
if watermark >= to_block {
YieldBounds::Full
} else {
YieldBounds::Partial(watermark)
}
}
enum EmptyFlushOutcome {
Deferred,
VerifiedEmpty,
CaughtUp,
Stalled {
to_block: u64,
reported_tip: u64,
stalled_cycles: u32,
},
RepollFailed(String),
TipFetchFailed(String),
}
async fn handle_empty_flush<F, Fut, E>(
coalescer: &mut HeadCoalescer,
gate: &mut TipGate,
providers: &ProviderSettings,
host: &str,
to_block: u64,
span_capped: bool,
get_tip: F,
) -> EmptyFlushOutcome
where
F: Fn() -> Fut,
Fut: Future<Output = Result<u64, E>>,
E: std::fmt::Display,
{
if !span_capped {
coalescer.defer_flush();
return EmptyFlushOutcome::Deferred;
}
let reported_tip = match get_tip().await {
Ok(tip) => tip,
Err(e) => return EmptyFlushOutcome::TipFetchFailed(e.to_string()),
};
coalescer.record_tip(reported_tip);
if next_cursor_after_batch(to_block, true, reported_tip).is_err() {
match wait_for_tip(to_block, reported_tip, gate, providers, host, get_tip).await {
TipWaitOutcome::CaughtUp => EmptyFlushOutcome::CaughtUp,
TipWaitOutcome::Terminal {
to_block,
reported_tip,
stalled_cycles,
} => EmptyFlushOutcome::Stalled {
to_block,
reported_tip,
stalled_cycles,
},
TipWaitOutcome::RepollFailed(err) => EmptyFlushOutcome::RepollFailed(err),
}
} else {
EmptyFlushOutcome::VerifiedEmpty
}
}
async fn stream_heads_provider(
provider: &DynProvider<Ethereum>,
provider_host: &str,
) -> impl Stream<Item = Header> + Send {
stream! {
let head_stream = provider.subscribe_blocks().await;
match head_stream {
Ok(heads) => {
debug!("WebSocket connected, streaming headers...");
let mut s = heads.into_stream();
while let Some(header) = s.next().await {
yield header;
}
}
Err(e) => {
warn!(
"Failed to subscribe to blocks on {}: {} - retrying",
provider_host,
truncate_error_short(e, 100)
);
}
}
}
}
pub async fn stream_heads_with_logs_with_progress(
providers: &ProviderSettings,
filter: Option<&Filter>,
progress: Option<&ScanProgress>,
) -> impl Stream<Item = Result<LogRange, RpcError>> + Send {
let num_providers = providers.wss_endpoints.len();
let progress = progress.cloned();
stream! {
let mut provider_trackers: Vec<ProviderErrorTracker> = (0..num_providers)
.map(|i| {
let host = providers.wss_endpoints[i]
.host_str()
.unwrap_or("unknown")
.to_string();
ProviderErrorTracker::new(host)
})
.collect();
let reused_http_clients = providers.build_reused_http_clients();
let mut provider_index = 0;
let mut coalescer = HeadCoalescer::new(
filter.and_then(|f| f.get_from_block()),
Duration::from_millis(providers.live_head_coalesce_window_ms),
)
.with_progress(progress);
let mut linkage = HeadLinkage::new(providers.reorg_detect_depth);
let mut gate = TipGate::new();
loop {
let active_provider = find_active_provider(&mut provider_trackers, provider_index);
let Some(idx) = active_provider else {
yield Err(RpcError::AllProvidersSuspended(format!(
"All {} WebSocket providers suspended. Unable to continue streaming.",
num_providers
)));
return;
};
provider_index = idx;
let provider = match providers.connect_ws(provider_index).await {
Ok(p) => {
provider_trackers[provider_index].record_success();
p
}
Err(e) => {
let suspended = provider_trackers[provider_index].record_error();
let backoff = provider_trackers[provider_index].backoff_duration();
if suspended {
warn!(
"WebSocket provider {} suspended after connection failure: {}",
provider_trackers[provider_index].identifier(),
truncate_error_short(&e, 100)
);
} else {
warn!(
"Failed to connect to WebSocket {}: {} - retrying in {:?}",
provider_trackers[provider_index].identifier(),
truncate_error_short(&e, 100),
backoff
);
sleep(backoff).await;
}
provider_index = (provider_index + 1) % num_providers;
continue;
}
};
gate.reset();
let provider_host = provider_trackers[provider_index].identifier().to_string();
let s = stream_heads_provider(&provider, &provider_host).await;
pin_mut!(s);
let mut should_switch_provider = false;
loop {
if coalescer.should_flush(Instant::now()) {
'flush: while let Some((from_block, pending_to_block)) = coalescer.pending_range() {
let to_block =
std::cmp::min(from_block + MAX_BLOCKS_PER_WS_REQUEST - 1, pending_to_block);
if linkage.enabled() {
match bound_flush_time(
providers.flush_max_total_secs,
flush_window_range(
&mut linkage,
&mut coalescer,
providers,
filter,
provider_index,
Some(&reused_http_clients),
),
)
.await
{
Ok(ranges) => {
for range in ranges {
provider_trackers[provider_index].record_success();
yield Ok(range);
}
}
Err(e) => {
warn!(
"by-hash window flush failed: {}",
truncate_error_short(&e, 100)
);
should_switch_provider = true;
break 'flush;
}
}
continue;
}
let query_filter = filter
.cloned()
.unwrap_or_else(Filter::new)
.from_block(from_block)
.to_block(to_block);
let logs = provider.get_logs(&query_filter).await;
match logs {
Ok(logs) => {
provider_trackers[provider_index].record_success();
let watermark_bounds =
(!logs.is_empty()).then(|| yield_bounds(&logs, to_block));
if let Some(YieldBounds::Partial(watermark)) = watermark_bounds {
yield Ok((from_block, watermark, logs));
coalescer.advance_cursor(watermark + 1);
gate.reset();
coalescer.defer_flush();
break 'flush;
}
let treat_as_empty =
!matches!(watermark_bounds, Some(YieldBounds::Full));
if treat_as_empty {
let span_capped =
to_block - from_block + 1 == MAX_BLOCKS_PER_WS_REQUEST;
match handle_empty_flush(
&mut coalescer,
&mut gate,
providers,
&provider_host,
to_block,
span_capped,
|| provider.get_block_number(),
)
.await
{
EmptyFlushOutcome::Deferred => break 'flush,
EmptyFlushOutcome::VerifiedEmpty => {}
EmptyFlushOutcome::CaughtUp => continue 'flush,
EmptyFlushOutcome::Stalled {
to_block,
reported_tip,
stalled_cycles,
} => {
yield Err(RpcError::ProviderStalled {
to_block,
reported_tip,
stalled_cycles,
});
return;
}
EmptyFlushOutcome::RepollFailed(err) => {
let suspended = provider_trackers
[provider_index]
.record_error();
if suspended {
warn!(
"WebSocket provider {} suspended after tip re-poll failed during tip-wait for blocks {}-{}: {}",
provider_trackers[provider_index].identifier(),
from_block,
to_block,
truncate_error_short(&err, 100),
);
} else {
let backoff = provider_trackers
[provider_index]
.backoff_duration();
warn!(
"Tip re-poll failed during wait for blocks {}-{} from {}: {} - backing off {:?}",
from_block,
to_block,
provider_trackers[provider_index].identifier(),
truncate_error_short(&err, 100),
backoff
);
sleep(backoff).await;
}
should_switch_provider = true;
break 'flush;
}
EmptyFlushOutcome::TipFetchFailed(err_str) => {
let suspended =
provider_trackers[provider_index].record_error();
if suspended {
warn!(
"WebSocket provider {} suspended after tip fetch failed during empty-batch guard for blocks {}-{}: {}",
provider_trackers[provider_index].identifier(),
from_block, to_block,
truncate_error_short(&err_str, 100)
);
} else {
let backoff = provider_trackers[provider_index].backoff_duration();
warn!(
"Tip fetch failed during empty-batch guard for blocks {}-{} from {}: {} - backing off {:?}",
from_block, to_block,
provider_trackers[provider_index].identifier(),
truncate_error_short(&err_str, 100),
backoff
);
sleep(backoff).await;
}
should_switch_provider = true;
break 'flush;
}
}
}
let logs = if treat_as_empty { Vec::new() } else { logs };
yield Ok((from_block, to_block, logs));
coalescer.advance_cursor(to_block + 1);
gate.reset();
}
Err(e) => {
let error_msg = e.to_string();
let error_category = ErrorCategory::from_error_msg(&error_msg);
match error_category {
ErrorCategory::RangeTooWide => {
warn!(
"Range too wide for blocks {}-{}, falling back to HTTP RPC: {}",
from_block, to_block, truncate_error_short(&error_msg, 100)
);
match get_logs_with_http_fallback_retry(
providers,
&query_filter,
provider_index,
from_block,
to_block,
Some(&reused_http_clients),
)
.await
{
Ok((http_provider_idx, logs)) => {
provider_trackers[provider_index].record_success();
let watermark_bounds = (!logs.is_empty())
.then(|| yield_bounds(&logs, to_block));
if let Some(YieldBounds::Partial(watermark)) =
watermark_bounds
{
yield Ok((from_block, watermark, logs));
coalescer.advance_cursor(watermark + 1);
gate.reset();
coalescer.defer_flush();
break 'flush;
}
let treat_as_empty = !matches!(
watermark_bounds,
Some(YieldBounds::Full)
);
if treat_as_empty {
let http_provider = providers.connect_http(http_provider_idx);
let http_host = providers.http_settings(http_provider_idx).host();
match handle_empty_flush(
&mut coalescer,
&mut gate,
providers,
&http_host,
to_block,
true,
|| http_provider.get_block_number(),
)
.await
{
EmptyFlushOutcome::Deferred => {
unreachable!(
"span_capped: true never defers"
)
}
EmptyFlushOutcome::VerifiedEmpty => {}
EmptyFlushOutcome::CaughtUp => {
continue 'flush
}
EmptyFlushOutcome::Stalled {
to_block,
reported_tip,
stalled_cycles,
} => {
yield Err(
RpcError::ProviderStalled {
to_block,
reported_tip,
stalled_cycles,
},
);
return;
}
EmptyFlushOutcome::RepollFailed(err) => {
warn!(
"HTTP fallback {} tip re-poll failed during tip-wait for blocks {}-{}: {}",
http_host,
from_block,
to_block,
truncate_error_short(&err, 100),
);
should_switch_provider = true;
break 'flush;
}
EmptyFlushOutcome::TipFetchFailed(err) => {
warn!(
"HTTP fallback {} tip fetch failed during empty-batch guard for blocks {}-{}: {}",
http_host, from_block, to_block,
truncate_error_short(&err, 100)
);
should_switch_provider = true;
break 'flush;
}
}
}
let logs = if treat_as_empty {
Vec::new()
} else {
logs
};
yield Ok((from_block, to_block, logs));
coalescer.advance_cursor(to_block + 1);
gate.reset();
}
Err(http_err) => {
warn!(
"HTTP fallback retry exhausted for blocks {}-{}: {}",
from_block,
to_block,
truncate_error_short(&http_err, 100)
);
should_switch_provider = true;
break;
}
}
}
ErrorCategory::RateLimit => {
provider_trackers[provider_index].record_error();
let backoff = provider_trackers[provider_index].backoff_duration();
warn!(
"Rate limit hit on WS provider {} for blocks {}-{}: {} - backing off {:?}",
provider_trackers[provider_index].identifier(),
from_block, to_block,
truncate_error_short(&error_msg, 100),
backoff
);
sleep(backoff).await;
continue;
}
ErrorCategory::Connection | ErrorCategory::Other => {
let suspended = provider_trackers[provider_index].record_error();
if suspended {
warn!(
"WebSocket provider {} suspended after error on blocks {}-{}: {}",
provider_trackers[provider_index].identifier(),
from_block, to_block,
truncate_error_short(&error_msg, 100)
);
} else {
let backoff = provider_trackers[provider_index].backoff_duration();
warn!(
"Error fetching logs for blocks {}-{} from {}: {} - backing off {:?}",
from_block, to_block,
provider_trackers[provider_index].identifier(),
truncate_error_short(&error_msg, 100),
backoff
);
sleep(backoff).await;
}
should_switch_provider = true;
break;
}
}
}
}
}
if should_switch_provider {
break;
}
continue;
}
let maybe_header = if let Some(deadline) = coalescer.flush_deadline() {
tokio::select! {
maybe_header = s.next() => maybe_header,
_ = sleep_until(deadline) => {
continue;
}
}
} else {
s.next().await
};
match maybe_header {
Some(header) => {
let now = Instant::now();
let head_number = header.number;
if !coalescer.observe_head(head_number, now) {
let from_block = coalescer.block_cursor().unwrap_or(head_number);
warn!(
"Skipping header {} as it's before the cursor {}",
head_number, from_block
);
}
match linkage.observe(head_number, header.hash, header.parent_hash) {
LinkOutcome::Linked => {}
LinkOutcome::Reorg { common_ancestor } => {
warn!(
"reorg at head {}: common ancestor {} — ending stream for consumer recovery",
head_number, common_ancestor
);
yield Err(RpcError::Reorg { common_ancestor });
return;
}
LinkOutcome::Detached {
parent_hash,
child_number,
} => {
let err = match walk_to_common_ancestor(
&provider,
providers,
Some(&reused_http_clients),
&linkage,
parent_hash,
child_number,
providers.reorg_detect_depth,
)
.await
{
Ok(common_ancestor) => {
warn!(
"reorg at head {} (walked new chain): common ancestor {} — ending stream",
head_number, common_ancestor
);
RpcError::Reorg { common_ancestor }
}
Err(RpcError::ReorgBeyondFinality) => {
warn!(
"reorg at head {} beyond the detector ring — ending stream",
head_number
);
RpcError::ReorgBeyondFinality
}
Err(err) => {
warn!(
"reorg walk at head {} could not resolve a step from any provider — ending stream",
head_number
);
err
}
};
yield Err(err);
return;
}
LinkOutcome::BeyondFinality => {
warn!(
"reorg beyond finality at head {}: no common ancestor within the detector ring — ending stream",
head_number
);
yield Err(RpcError::ReorgBeyondFinality);
return;
}
}
}
None => break,
}
}
if !should_switch_provider {
provider_trackers[provider_index].record_error();
let backoff = provider_trackers[provider_index].backoff_duration();
warn!(
"WS connection to {} lost or stream ended - reconnecting in {:?}",
provider_trackers[provider_index].identifier(),
backoff
);
sleep(backoff).await;
}
provider_index = (provider_index + 1) % num_providers;
}
}
}
pub async fn stream_heads_with_logs(
providers: &ProviderSettings,
filter: Option<&Filter>,
) -> impl Stream<Item = Result<LogRange, RpcError>> + Send {
stream_heads_with_logs_with_progress(providers, filter, None).await
}
pub async fn last_block(providers: &ProviderSettings) -> Result<u64, RpcError> {
let num_providers = providers.http_providers.len();
if num_providers == 0 {
return Err(RpcError::ConnectionError(
"No HTTP providers configured for latest block fetch".to_string(),
));
}
let mut index: usize = 0;
loop {
let provider_index = index % num_providers;
let provider = providers.connect_http(provider_index);
match provider.get_block_number().await {
Ok(block_number) => return Ok(block_number),
Err(e) => {
let provider_host = providers.http_settings(provider_index).host();
if index >= 10 {
return Err(RpcError::ConnectionError(format!(
"Failed to fetch latest block number after {} attempts from HTTP provider {}: {}",
index, provider_host, e
)));
}
index += 1;
warn!(
"Failed to fetch latest block number from HTTP provider {} (index {}) - retrying",
provider_host, provider_index
);
let progressive_sleep = (5 * index).min(50) as u64;
sleep(Duration::from_secs(progressive_sleep)).await;
}
}
}
}
struct CapturePolicyNotice {
latest: Once,
lag: Once,
finalized: Once,
}
impl CapturePolicyNotice {
const fn new() -> Self {
Self {
latest: Once::new(),
lag: Once::new(),
finalized: Once::new(),
}
}
fn first_message(&self, target: CaptureTarget, block: u64) -> Option<String> {
let gate = match target {
CaptureTarget::Latest => &self.latest,
CaptureTarget::Lag(_) => &self.lag,
CaptureTarget::Finalized => &self.finalized,
};
let mut won = false;
gate.call_once(|| won = true);
if !won {
return None;
}
Some(match target {
CaptureTarget::Latest => format!(
"ethl capture target is `Latest` (block {}): capturing to the raw chain \
tip with NO reorg protection — a reorg can rewrite recently captured \
blocks. Reorg-prone chains should opt into `Lag`/`Finalized`.",
block,
),
CaptureTarget::Lag(n) => format!(
"ethl DEEP LAG ACTIVE (reorg-safety): capturing to tip - {} = block {}.\n\
This build AVOIDS reorgs by lagging; events within {} blocks of tip are DELAYED.",
n, block, n,
),
CaptureTarget::Finalized => format!(
"ethl DEEP LAG ACTIVE (reorg-safety): capturing to `finalized` block {}.\n\
(~2 epochs ≈ 64 blocks ≈ 13 min behind chain tip).\n\
This build AVOIDS reorgs by lagging; events within ~64 blocks of tip are DELAYED.",
block
),
})
}
}
static CAPTURE_POLICY_NOTICE: CapturePolicyNotice = CapturePolicyNotice::new();
pub async fn resolve_capture_target(settings: &ProviderSettings) -> Result<u64, RpcError> {
match settings.capture_target {
CaptureTarget::Latest => {
let tip = last_block(settings).await?;
if let Some(message) = CAPTURE_POLICY_NOTICE.first_message(settings.capture_target, tip)
{
warn!("{message}");
}
Ok(tip)
}
CaptureTarget::Lag(n) => {
let tip = last_block(settings).await?;
let target_block = tip.saturating_sub(n);
if let Some(message) =
CAPTURE_POLICY_NOTICE.first_message(settings.capture_target, target_block)
{
warn!("{message}");
}
Ok(target_block)
}
CaptureTarget::Finalized => {
let provider = settings.connect_http(0);
let result = provider
.get_block_by_number(BlockNumberOrTag::Finalized)
.await
.map_err(|e| {
RpcError::ConnectionError(format!("finalized block fetch failed: {e}"))
})?;
match result {
Some(block) => {
let finalized = block.header.number;
if let Some(message) =
CAPTURE_POLICY_NOTICE.first_message(settings.capture_target, finalized)
{
warn!("{message}");
}
Ok(finalized)
}
None => Err(RpcError::Other(
"Provider returned no finalized block; cannot determine safe capture target. \
Ensure the provider supports the `finalized` tag (mainnet only)."
.to_string(),
)),
}
}
}
}
#[cfg(test)]
mod tests {
use super::{
CapturePolicyNotice, EmptyFlushOutcome, HeadCoalescer, HeadLinkage, LinkOutcome,
MAX_BLOCKS_PER_WS_REQUEST, MAX_STEP_RESOLUTION_ROUNDS, ProviderAnswer, RpcError,
StepFailure, TipGate, YieldBounds, bound_flush_time, collect_window_hashes,
fallback_http_exhausted_error, fallback_http_index, fetch_concurrently,
fetch_logs_by_hash_range, flush_window_range, handle_empty_flush, resolve_http_provider,
resolve_step, sort_ranges_ascending, walk_to_common_ancestor, yield_bounds,
};
use crate::rpc::config::{CaptureTarget, HttpRpcSettings, ProviderSettings};
use alloy::network::Ethereum;
use alloy::primitives::B256;
use alloy::providers::{DynProvider, Provider, ProviderBuilder};
use alloy::rpc::types::Log;
use alloy::transports::mock::Asserter;
use std::time::Duration;
use tokio::time::Instant;
const WINDOW: Duration = Duration::from_millis(500);
fn hb(byte: u8) -> B256 {
B256::repeat_byte(byte)
}
#[test]
fn fallback_http_index_wraps_round_robin() {
assert_eq!(fallback_http_index(0, 0, 3), 0);
assert_eq!(fallback_http_index(0, 1, 3), 1);
assert_eq!(fallback_http_index(0, 2, 3), 2);
assert_eq!(fallback_http_index(0, 3, 3), 0);
assert_eq!(fallback_http_index(2, 1, 3), 0);
}
#[test]
fn fallback_http_exhausted_error_includes_hosts() {
let err = fallback_http_exhausted_error(
100,
120,
&["rpc.ankr.com".to_string(), "mainnet.infura.io".to_string()],
"timeout",
)
.to_string();
assert!(err.contains("rpc.ankr.com"));
assert!(err.contains("mainnet.infura.io"));
assert!(err.contains("timeout"));
}
#[test]
fn head_coalescer_ignores_stale_heads_before_cursor() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(42), WINDOW);
assert!(!coalescer.observe_head(41, now));
assert_eq!(coalescer.pending_range(), None);
assert_eq!(coalescer.flush_deadline(), None);
}
#[test]
fn head_coalescer_keeps_first_flush_deadline_while_heads_accumulate() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
assert!(coalescer.observe_head(100, now));
let deadline = coalescer.flush_deadline().unwrap();
assert!(coalescer.observe_head(105, now + Duration::from_millis(50)));
assert_eq!(coalescer.pending_range(), Some((100, 105)));
assert_eq!(coalescer.flush_deadline(), Some(deadline));
assert!(!coalescer.should_flush(now + WINDOW - Duration::from_millis(1)));
assert!(coalescer.should_flush(now + WINDOW));
}
#[test]
fn head_coalescer_flushes_immediately_when_pending_range_reaches_chunk_limit() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
assert!(coalescer.observe_head(MAX_BLOCKS_PER_WS_REQUEST, now));
assert_eq!(
coalescer.pending_range(),
Some((1, MAX_BLOCKS_PER_WS_REQUEST))
);
assert!(coalescer.should_flush(now));
}
#[test]
fn head_coalescer_preserves_pending_range_across_partial_cursor_advances() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(10), WINDOW);
assert!(coalescer.observe_head(25, now));
coalescer.advance_cursor(20);
assert_eq!(coalescer.pending_range(), Some((20, 25)));
assert!(coalescer.flush_deadline().is_some());
coalescer.advance_cursor(26);
assert_eq!(coalescer.pending_range(), None);
assert_eq!(coalescer.flush_deadline(), None);
assert_eq!(coalescer.block_cursor(), Some(26));
}
#[test]
fn advance_cursor_records_scanned_to_as_next_block_minus_one() {
let progress = crate::rpc::progress::ScanProgress::default();
let mut coalescer =
HeadCoalescer::new(Some(10), WINDOW).with_progress(Some(progress.clone()));
assert_eq!(progress.scanned_to(), None);
coalescer.advance_cursor(101);
assert_eq!(progress.scanned_to(), Some(100));
coalescer.advance_cursor(51);
assert_eq!(progress.scanned_to(), Some(100));
}
#[test]
fn record_tip_populates_progress_when_attached_and_is_a_no_op_otherwise() {
let progress = crate::rpc::progress::ScanProgress::default();
let with_progress = HeadCoalescer::new(None, WINDOW).with_progress(Some(progress.clone()));
with_progress.record_tip(500);
assert_eq!(progress.observed_tip(), Some(500));
let without_progress = HeadCoalescer::new(None, WINDOW);
without_progress.record_tip(500);
}
#[test]
fn head_coalescer_handles_mainnet_slow_blocks() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(1_000_000), WINDOW);
assert!(coalescer.observe_head(1_000_000, now));
assert_eq!(coalescer.pending_range(), Some((1_000_000, 1_000_000)));
assert!(!coalescer.should_flush(now + Duration::from_millis(499)));
assert!(coalescer.should_flush(now + WINDOW));
coalescer.advance_cursor(1_000_001);
assert_eq!(coalescer.pending_range(), None);
let t12s = now + Duration::from_secs(12);
assert!(coalescer.observe_head(1_000_001, t12s));
assert_eq!(coalescer.pending_range(), Some((1_000_001, 1_000_001)));
assert!(!coalescer.should_flush(t12s));
assert!(coalescer.should_flush(t12s + WINDOW));
}
#[test]
fn head_coalescer_honors_its_own_window() {
let now = Instant::now();
let short_window = Duration::from_millis(100);
let long_window = Duration::from_secs(5);
let mut short = HeadCoalescer::new(Some(1), short_window);
assert!(short.observe_head(1, now));
assert!(!short.should_flush(now + short_window - Duration::from_millis(1)));
assert!(short.should_flush(now + short_window));
let mut long = HeadCoalescer::new(Some(1), long_window);
assert!(long.observe_head(1, now));
assert!(!long.should_flush(now + long_window - Duration::from_millis(1)));
assert!(long.should_flush(now + long_window));
}
#[test]
fn head_coalescer_defer_flush_clears_deadline_only() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
assert!(coalescer.observe_head(300, now));
assert_eq!(coalescer.pending_range(), Some((1, 300)));
assert!(coalescer.flush_deadline().is_some());
coalescer.defer_flush();
assert_eq!(
coalescer.flush_deadline(),
None,
"defer_flush must clear the deadline"
);
assert_eq!(
coalescer.pending_range(),
Some((1, 300)),
"defer_flush must not touch the pending range"
);
assert!(coalescer.should_flush(now));
}
#[test]
fn coalescer_partial_defer_reserves_unproven_tail_and_rearms() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(105, now);
assert_eq!(coalescer.pending_range(), Some((100, 105)));
coalescer.advance_cursor(103);
coalescer.defer_flush();
assert_eq!(
coalescer.flush_deadline(),
None,
"defer_flush must clear the deadline after a Partial yield"
);
assert_eq!(
coalescer.pending_range(),
Some((103, 105)),
"pending range must start at watermark+1, the unproven tail"
);
assert!(coalescer.observe_head(105, now + Duration::from_millis(1)));
assert!(coalescer.flush_deadline().is_some());
}
fn log_fixture(block_number: Option<u64>) -> Log {
let value = serde_json::json!({
"address": "0x0000000000000000000000000000000000000001",
"topics": [],
"data": "0x",
"blockHash": hb(0).to_string(),
"blockNumber": block_number.map(|n| format!("0x{:x}", n)),
"transactionHash": hb(0).to_string(),
"transactionIndex": "0x0",
"logIndex": "0x0",
"removed": false
});
serde_json::from_value(value).expect("valid Log fixture")
}
#[test]
fn yield_bounds_full_when_tail_has_logs() {
let logs = vec![log_fixture(Some(101)), log_fixture(Some(102))];
assert!(matches!(yield_bounds(&logs, 102), YieldBounds::Full));
}
#[test]
fn yield_bounds_partial_stops_at_watermark() {
let logs = vec![log_fixture(Some(101))];
assert!(matches!(
yield_bounds(&logs, 102),
YieldBounds::Partial(101)
));
}
#[test]
fn yield_bounds_interior_gaps_are_proven() {
let logs = vec![log_fixture(Some(101)), log_fixture(Some(104))];
assert!(matches!(
yield_bounds(&logs, 105),
YieldBounds::Partial(104)
));
}
#[test]
fn yield_bounds_mixed_none_uses_some_only() {
let logs = vec![log_fixture(Some(101)), log_fixture(None)];
assert!(matches!(
yield_bounds(&logs, 102),
YieldBounds::Partial(101)
));
}
#[test]
fn yield_bounds_all_none_is_unattributable() {
let logs = vec![log_fixture(None), log_fixture(None)];
assert!(matches!(
yield_bounds(&logs, 999),
YieldBounds::Unattributable
));
}
#[test]
fn yield_bounds_watermark_above_range_clamps() {
let logs = vec![log_fixture(Some(103))];
assert!(matches!(yield_bounds(&logs, 102), YieldBounds::Full));
}
#[tokio::test]
async fn lazy_empty_non_capped_defers_without_tip_call() {
let asserter = Asserter::new(); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(105, Instant::now());
assert_eq!(coalescer.pending_range(), Some((100, 105)));
let mut gate = TipGate::new();
let outcome = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host",
105,
false,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(outcome, EmptyFlushOutcome::Deferred),
"expected Deferred"
);
assert_eq!(coalescer.flush_deadline(), None);
assert_eq!(
coalescer.pending_range(),
Some((100, 105)),
"cursor must stay unmoved by a deferred flush"
);
}
#[test]
fn lazy_deferred_flush_rearms_and_spans_from_unmoved_cursor() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(105, now);
assert_eq!(coalescer.pending_range(), Some((100, 105)));
coalescer.defer_flush();
assert_eq!(coalescer.flush_deadline(), None);
coalescer.observe_head(110, now + Duration::from_millis(10));
assert_eq!(
coalescer.pending_range(),
Some((100, 110)),
"deferred span must extend from the original unmoved cursor"
);
assert!(
coalescer.flush_deadline().is_some(),
"the next observe_head must re-arm the deadline"
);
}
#[tokio::test]
async fn span_cap_empty_tip_current_is_verified_empty() {
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x64")); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(100, Instant::now());
let mut gate = TipGate::new();
let outcome = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host",
100,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(outcome, EmptyFlushOutcome::VerifiedEmpty),
"expected VerifiedEmpty"
);
}
#[tokio::test]
async fn span_cap_empty_tip_behind_enters_tip_wait() {
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x63")); asserter.push_success(&serde_json::json!("0x64")); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(100, Instant::now());
let mut gate = TipGate::new();
let outcome = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host",
100,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(outcome, EmptyFlushOutcome::CaughtUp),
"expected CaughtUp"
);
}
#[tokio::test]
async fn span_cap_empty_flat_tip_stalls() {
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x5a")); asserter.push_success(&serde_json::json!("0x5a")); asserter.push_success(&serde_json::json!("0x5a")); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
)
.with_tip_wait_max_stall_cycles(2);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(100, Instant::now());
let mut gate = TipGate::new();
let outcome = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host",
100,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(
outcome,
EmptyFlushOutcome::Stalled {
to_block: 100,
reported_tip: 90,
stalled_cycles: 2,
}
),
"expected Stalled(stalled_cycles=2), got non-matching outcome"
);
}
#[tokio::test]
async fn handle_empty_flush_stale_gate_stalls_one_cycle_early_without_reset() {
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x3de")); asserter.push_failure_msg("transport blip");
for _ in 0..5 {
asserter.push_success(&serde_json::json!("0x3dc")); }
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(999, Instant::now());
let mut gate = TipGate::new();
let session_a = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host-a",
999,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(session_a, EmptyFlushOutcome::RepollFailed(_)),
"expected RepollFailed for session A"
);
let session_b = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host-b",
999,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(
session_b,
EmptyFlushOutcome::Stalled {
to_block: 999,
reported_tip: 988,
stalled_cycles: 5,
}
),
"expected Stalled(stalled_cycles=5) one cycle early from the stale gate"
);
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!("0x3de")); asserter.push_failure_msg("transport blip");
for _ in 0..6 {
asserter.push_success(&serde_json::json!("0x3dc")); }
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let http_provider = settings.connect_http(0);
let mut coalescer = HeadCoalescer::new(Some(1), WINDOW);
coalescer.observe_head(999, Instant::now());
let mut gate = TipGate::new();
let session_a = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host-a",
999,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(session_a, EmptyFlushOutcome::RepollFailed(_)),
"expected RepollFailed for session A"
);
gate.reset();
let session_b = handle_empty_flush(
&mut coalescer,
&mut gate,
&settings,
"host-b",
999,
true,
|| http_provider.get_block_number(),
)
.await;
assert!(
matches!(
session_b,
EmptyFlushOutcome::Stalled {
to_block: 999,
reported_tip: 988,
stalled_cycles: 5,
}
),
"expected Stalled(stalled_cycles=5) reached only after the full \
6-poll budget once the gate is reset"
);
}
fn block_with_parent_at(number: u64, hash: B256, parent: B256) -> serde_json::Value {
let zero = hb(0).to_string();
serde_json::json!({
"hash": hash.to_string(),
"parentHash": parent.to_string(),
"sha3Uncles": zero,
"miner": "0x0000000000000000000000000000000000000000",
"stateRoot": zero,
"transactionsRoot": zero,
"receiptsRoot": zero,
"logsBloom": format!("0x{}", "0".repeat(512)),
"difficulty": "0x0",
"number": format!("0x{:x}", number),
"gasLimit": "0x0",
"gasUsed": "0x0",
"timestamp": "0x0",
"extraData": "0x",
"mixHash": zero,
"nonce": "0x0000000000000000",
"transactions": [],
"uncles": []
})
}
fn block_with_parent(hash: B256, parent: B256) -> serde_json::Value {
block_with_parent_at(0, hash, parent)
}
fn n_http_provider_settings(n: usize) -> ProviderSettings {
let http_providers = (0..n)
.map(|i| HttpRpcSettings::new(format!("http://unused-{i}"), 1, 100))
.collect();
ProviderSettings::new(
http_providers,
vec!["ws://127.0.0.1:9".parse().expect("static url")],
)
}
fn mock_http_provider(asserter: Asserter) -> DynProvider<Ethereum> {
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased()
}
fn seed_chain(depth: u64, lo: u64, hi: u64) -> HeadLinkage {
let mut linkage = HeadLinkage::new(depth);
for n in lo..=hi {
let parent = if n == lo {
hb((lo - 1) as u8)
} else {
hb((n - 1) as u8)
};
assert_eq!(linkage.observe(n, hb(n as u8), parent), LinkOutcome::Linked);
}
linkage
}
#[test]
fn head_linkage_disabled_is_inert() {
let mut linkage = HeadLinkage::new(0);
assert_eq!(linkage.observe(100, hb(1), hb(9)), LinkOutcome::Linked);
assert_eq!(linkage.observe(101, hb(2), hb(7)), LinkOutcome::Linked);
assert_eq!(linkage.floor(), None);
}
#[test]
fn head_linkage_clean_chain_advances_and_bounds_ring() {
let linkage = seed_chain(3, 100, 103);
assert_eq!(linkage.floor(), Some(101));
assert_eq!(linkage.hash_at(100), None);
assert_eq!(linkage.hash_at(103), Some(hb(103)));
}
#[test]
fn head_linkage_one_block_reorg_reports_parent_as_ancestor() {
let mut linkage = seed_chain(8, 100, 103);
assert_eq!(
linkage.observe(103, hb(203), hb(102)),
LinkOutcome::Reorg {
common_ancestor: 102
}
);
assert_eq!(linkage.hash_at(103), Some(hb(203)));
assert_eq!(linkage.observe(104, hb(204), hb(203)), LinkOutcome::Linked);
}
#[test]
fn head_linkage_deep_reorg_via_reemit_reports_deeper_ancestor() {
let mut linkage = seed_chain(8, 100, 105);
assert_eq!(
linkage.observe(101, hb(201), hb(100)),
LinkOutcome::Reorg {
common_ancestor: 100
}
);
}
#[test]
fn head_linkage_skipped_intermediate_heads_detach() {
let mut linkage = seed_chain(8, 100, 103);
match linkage.observe(104, hb(204), hb(203)) {
LinkOutcome::Detached {
parent_hash,
child_number,
} => {
assert_eq!(parent_hash, hb(203));
assert_eq!(child_number, 104);
}
other => panic!("expected Detached, got {other:?}"),
}
}
#[test]
fn head_linkage_divergence_below_floor_is_beyond_finality() {
let mut linkage = seed_chain(3, 100, 105);
assert_eq!(linkage.floor(), Some(103));
assert_eq!(
linkage.observe(103, hb(203), hb(202)),
LinkOutcome::BeyondFinality
);
}
#[tokio::test]
async fn walk_to_common_ancestor_walks_skipped_chain_to_common_ancestor() {
let linkage = seed_chain(8, 100, 103);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(203), hb(202)));
asserter.push_success(&block_with_parent(hb(202), hb(101)));
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(203), 104, 8).await;
assert!(matches!(err, Ok(101)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_exhausts_ring_as_beyond_finality() {
let linkage = seed_chain(2, 100, 103);
assert_eq!(linkage.floor(), Some(102));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(203), hb(202)));
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(203), 104, 2).await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
#[test]
fn head_linkage_back_to_back_reorgs_ring_stays_coherent() {
let mut linkage = seed_chain(8, 100, 105);
assert_eq!(
linkage.observe(104, hb(204), hb(103)),
LinkOutcome::Reorg {
common_ancestor: 103
}
);
assert_eq!(linkage.hash_at(104), Some(hb(204)));
assert_eq!(linkage.observe(105, hb(205), hb(204)), LinkOutcome::Linked);
assert_eq!(
linkage.observe(105, hb(49), hb(204)),
LinkOutcome::Reorg {
common_ancestor: 104
}
);
assert_eq!(linkage.hash_at(105), Some(hb(49)));
assert_eq!(linkage.hash_at(104), Some(hb(204))); assert!(linkage.ring.len() as u64 <= 8);
}
#[test]
fn head_linkage_multi_header_same_height_duplicate_vs_replace() {
let mut linkage = seed_chain(8, 100, 104);
assert_eq!(linkage.observe(104, hb(104), hb(103)), LinkOutcome::Linked);
assert_eq!(linkage.hash_at(104), Some(hb(104)));
assert_eq!(
linkage.observe(104, hb(204), hb(103)),
LinkOutcome::Reorg {
common_ancestor: 103
}
);
assert_eq!(linkage.hash_at(104), Some(hb(204)));
}
#[test]
fn head_linkage_identical_hash_no_op() {
let mut linkage = seed_chain(8, 100, 104);
let floor_before = linkage.floor();
let depth_before = linkage.ring.len();
assert_eq!(linkage.observe(104, hb(104), hb(103)), LinkOutcome::Linked);
assert_eq!(linkage.floor(), floor_before);
assert_eq!(linkage.ring.len(), depth_before);
assert_eq!(linkage.hash_at(104), Some(hb(104)));
}
#[test]
fn head_linkage_forward_gap_becomes_detached_for_verification() {
let mut linkage = seed_chain(8, 100, 103); assert_eq!(
linkage.observe(107, hb(107), hb(106)),
LinkOutcome::Detached {
parent_hash: hb(106),
child_number: 107,
}
);
assert_eq!(linkage.floor(), Some(100));
assert_eq!(linkage.hash_at(103), Some(hb(103)));
}
#[test]
fn head_linkage_idempotent_reanchor_bounds_ring() {
let mut linkage = seed_chain(5, 100, 106); assert_eq!(linkage.floor(), Some(102));
assert_eq!(
linkage.observe(105, hb(205), hb(104)),
LinkOutcome::Reorg {
common_ancestor: 104
}
);
assert_eq!(linkage.hash_at(105), Some(hb(205)));
assert_eq!(linkage.observe(106, hb(206), hb(205)), LinkOutcome::Linked);
assert_eq!(linkage.observe(107, hb(207), hb(206)), LinkOutcome::Linked);
assert!(linkage.ring.len() as u64 <= 5);
assert_eq!(linkage.hash_at(105), Some(hb(205))); assert_eq!(linkage.hash_at(102), None); }
#[tokio::test]
async fn walk_to_common_ancestor_walk_depth_1() {
let linkage = seed_chain(15, 100, 109);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(109))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 15).await;
assert!(matches!(err, Ok(109)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_walk_depth_2() {
let linkage = seed_chain(15, 100, 109);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201))); asserter.push_success(&block_with_parent(hb(201), hb(108))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 15).await;
assert!(matches!(err, Ok(108)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_walk_depth_3() {
let linkage = seed_chain(15, 100, 109);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201))); asserter.push_success(&block_with_parent(hb(201), hb(202))); asserter.push_success(&block_with_parent(hb(202), hb(107))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 15).await;
assert!(matches!(err, Ok(107)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_walk_depth_7() {
let linkage = seed_chain(15, 100, 109);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201)));
asserter.push_success(&block_with_parent(hb(201), hb(202)));
asserter.push_success(&block_with_parent(hb(202), hb(203)));
asserter.push_success(&block_with_parent(hb(203), hb(204)));
asserter.push_success(&block_with_parent(hb(204), hb(205)));
asserter.push_success(&block_with_parent(hb(205), hb(206)));
asserter.push_success(&block_with_parent(hb(206), hb(103))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 15).await;
assert!(matches!(err, Ok(103)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_deepest_in_ring_resolves() {
let linkage = seed_chain(5, 100, 104);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201))); asserter.push_success(&block_with_parent(hb(201), hb(202))); asserter.push_success(&block_with_parent(hb(202), hb(203))); asserter.push_success(&block_with_parent(hb(203), hb(100))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 105, 5).await;
assert!(matches!(err, Ok(100)), "got {err:?}");
}
#[tokio::test]
async fn forward_gap_walks_back_and_detects_reorg() {
let linkage = seed_chain(8, 100, 103);
let mut linkage = linkage;
let outcome = linkage.observe(107, hb(207), hb(206));
assert_eq!(
outcome,
LinkOutcome::Detached {
parent_hash: hb(206),
child_number: 107,
},
"forward gap must route through Detached, not blindly re-anchor"
);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(206), hb(205)));
asserter.push_success(&block_with_parent(hb(205), hb(204)));
asserter.push_success(&block_with_parent(hb(204), hb(103)));
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(206), 107, 8).await;
assert!(matches!(err, Ok(103)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_loop_exhausted_is_beyond_finality() {
let linkage = seed_chain(3, 100, 109);
assert_eq!(linkage.floor(), Some(107));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent(hb(200), hb(201)));
asserter.push_success(&block_with_parent(hb(201), hb(202)));
asserter.push_success(&block_with_parent(hb(202), hb(203))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 111, 3).await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_parent_at_floor_is_beyond_finality() {
let linkage = seed_chain(10, 100, 109); let asserter = Asserter::new(); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(200), 101, 10).await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_single_provider_absent_is_beyond_finality() {
let linkage = seed_chain(15, 100, 109);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(1);
let http_asserter = Asserter::new();
http_asserter.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(http_asserter)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(200),
111,
15,
)
.await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_transport_error_is_unresolved() {
let linkage = seed_chain(15, 100, 109);
let settings = n_http_provider_settings(1);
let http_asserter = Asserter::new();
http_asserter.push_failure_msg("transport blip");
let reused = vec![mock_http_provider(http_asserter)];
let asserter = Asserter::new();
asserter.push_failure_msg("transport blip");
let provider = mock_http_provider(asserter);
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(200),
111,
15,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_second_source_recovers_the_incident_scenario() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&block_with_parent(hb(203), hb(102))); let voter2 = Asserter::new();
voter2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(matches!(err, Ok(102)), "got {err:?}");
}
#[tokio::test]
async fn walk_to_common_ancestor_all_sources_absent_is_beyond_finality() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let reused: Vec<_> = (0..3)
.map(|_| {
let a = Asserter::new();
a.push_success(&serde_json::json!(null));
mock_http_provider(a)
})
.collect();
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_lone_absent_answer_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_failure_msg("blip");
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_two_of_three_absent_is_beyond_finality() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_n_equals_2() {
let linkage = seed_chain(8, 100, 103);
let settings = n_http_provider_settings(2);
{
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_failure_msg("blip");
let reused = vec![mock_http_provider(voter0), mock_http_provider(voter1)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
{
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(voter0), mock_http_provider(voter1)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
}
#[tokio::test]
async fn walk_to_common_ancestor_primary_transport_fail() {
let linkage = seed_chain(8, 100, 103);
let settings = n_http_provider_settings(3);
{
let ws_asserter = Asserter::new();
ws_asserter.push_failure_msg("blip");
let provider = mock_http_provider(ws_asserter);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&block_with_parent(hb(203), hb(102)));
let voter2 = Asserter::new();
voter2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(matches!(err, Ok(102)), "got {err:?}");
}
{
let ws_asserter = Asserter::new();
ws_asserter.push_failure_msg("blip");
let provider = mock_http_provider(ws_asserter);
let reused: Vec<_> = (0..3)
.map(|_| {
let a = Asserter::new();
a.push_failure_msg("blip");
mock_http_provider(a)
})
.collect();
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
}
#[tokio::test]
async fn walk_to_common_ancestor_two_recoveries_then_ring_relink() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null)); ws_asserter.push_success(&serde_json::json!(null)); let provider = mock_http_provider(ws_asserter.clone());
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&block_with_parent(hb(205), hb(204)));
voter0.push_success(&serde_json::json!(null)); let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null)); voter1.push_success(&serde_json::json!(null)); let voter2 = Asserter::new();
voter2.push_success(&serde_json::json!(null)); voter2.push_success(&block_with_parent(hb(204), hb(102)));
let reused = vec![
mock_http_provider(voter0.clone()),
mock_http_provider(voter1.clone()),
mock_http_provider(voter2.clone()),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(205),
105,
8,
)
.await;
assert!(matches!(err, Ok(102)), "got {err:?}");
assert_eq!(
ws_asserter.read_q().len(),
0,
"exactly 2 primary triggers, no third"
);
assert_eq!(
voter0.read_q().len(),
0,
"exactly 2 rounds, no fan-out beyond them"
);
assert_eq!(
voter1.read_q().len(),
0,
"exactly 2 rounds, no fan-out beyond them"
);
assert_eq!(
voter2.read_q().len(),
0,
"exactly 2 rounds, no fan-out beyond them"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_round_budget_exhaustion_is_unresolved() {
let linkage = seed_chain(8, 100, 109);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null)); ws_asserter.push_success(&serde_json::json!(null)); ws_asserter.push_success(&serde_json::json!(null)); ws_asserter.push_success(&serde_json::json!(null)); let provider = mock_http_provider(ws_asserter.clone());
let settings = n_http_provider_settings(1);
let source = Asserter::new();
source.push_success(&block_with_parent(hb(220), hb(219))); source.push_success(&block_with_parent(hb(219), hb(218))); source.push_success(&block_with_parent(hb(218), hb(217))); source.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(source.clone())];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(220),
115,
10,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
assert_eq!(
ws_asserter.read_q().len(),
0,
"all 4 primary triggers consumed"
);
assert_eq!(
source.read_q().len(),
1,
"exactly K=3 rounds' worth of second-source calls — the 4th answer \
must still be queued, proving no 4th round was dispatched"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_wrong_block_is_not_a_recovery() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&block_with_parent(hb(9), hb(1)));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
let Err(RpcError::ConnectionError(msg)) = err else {
panic!("got {err:?}");
};
assert!(msg.contains("blip"), "provider error text dropped: {msg}");
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let voter0 = Asserter::new();
voter0.push_success(&block_with_parent(hb(9), hb(1)));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_primary_wrong_hash_does_not_advance_the_walk() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&block_with_parent(hb(77), hb(102)));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(1);
let source = Asserter::new();
source.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(source)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"a wrong-hash primary block must never advance the walk; got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_primary_transport_error_with_absent_quorum_is_beyond_finality()
{
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_failure_msg("primary socket died");
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ReorgBeyondFinality)),
"got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_reused_length_mismatch_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let ws_asserter = Asserter::new();
ws_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(ws_asserter);
let settings = n_http_provider_settings(3);
let lone = Asserter::new();
lone.push_success(&serde_json::json!(null));
lone.push_success(&serde_json::json!(null));
lone.push_success(&serde_json::json!(null));
let reused = vec![mock_http_provider(lone)];
let err = walk_to_common_ancestor(
&provider,
&settings,
Some(&reused),
&linkage,
hb(203),
104,
8,
)
.await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"a short reused set must fail closed, never let one client vote \
three times; got {err:?}"
);
}
#[tokio::test]
async fn walk_to_common_ancestor_zero_http_providers_is_unresolved() {
let linkage = seed_chain(8, 100, 103);
let asserter = Asserter::new(); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(0);
let err =
walk_to_common_ancestor(&provider, &settings, None, &linkage, hb(203), 104, 8).await;
assert!(
matches!(err, Err(RpcError::ConnectionError(_))),
"got {err:?}"
);
}
#[tokio::test]
async fn resolve_step_ws_primary_found() {
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&block_with_parent(hb(50), hb(49)));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let mut rounds_used = 0u32;
let header = resolve_step(
"test",
&settings,
Some(&reused),
3,
&[0, 1, 2],
hb(50),
None,
&mut rounds_used,
0,
|h| h.hash == hb(50),
)
.await
.expect("one matching source must supply the block");
assert_eq!(header.parent_hash, hb(49));
assert_eq!(rounds_used, 1);
}
#[tokio::test]
async fn resolve_step_ws_primary_absence_confirmed() {
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let mut rounds_used = 0u32;
let err = resolve_step(
"test",
&settings,
Some(&reused),
3,
&[0, 1, 2],
hb(50),
None,
&mut rounds_used,
0,
|h| h.hash == hb(50),
)
.await
.expect_err("2 absent answers must confirm absence");
assert!(matches!(err, StepFailure::AbsenceConfirmed));
}
#[tokio::test]
async fn resolve_step_ws_primary_unresolved() {
let settings = n_http_provider_settings(3);
let voter0 = Asserter::new();
voter0.push_success(&serde_json::json!(null));
let voter1 = Asserter::new();
voter1.push_failure_msg("blip");
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(voter0),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let mut rounds_used = 0u32;
let err = resolve_step(
"test",
&settings,
Some(&reused),
3,
&[0, 1, 2],
hb(50),
None,
&mut rounds_used,
0,
|h| h.hash == hb(50),
)
.await
.expect_err("1 absent answer must not confirm absence");
assert!(matches!(err, StepFailure::Unresolved(_)));
}
#[tokio::test]
async fn resolve_step_http_primary_found() {
let settings = n_http_provider_settings(3);
let voter1 = Asserter::new();
voter1.push_success(&block_with_parent(hb(50), hb(49)));
let voter2 = Asserter::new();
voter2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(Asserter::new()), mock_http_provider(voter1),
mock_http_provider(voter2),
];
let mut rounds_used = 0u32;
let header = resolve_step(
"test",
&settings,
Some(&reused),
3,
&[1, 2],
hb(50),
Some((0, ProviderAnswer::Absent)),
&mut rounds_used,
0,
|h| h.hash == hb(50),
)
.await
.expect("one matching source must supply the block");
assert_eq!(header.parent_hash, hb(49));
}
#[tokio::test]
async fn resolve_step_http_primary_absence_confirmed() {
let settings = n_http_provider_settings(3);
let voter1 = Asserter::new();
voter1.push_success(&serde_json::json!(null));
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let mut rounds_used = 0u32;
let err = resolve_step(
"test",
&settings,
Some(&reused),
3,
&[1, 2],
hb(50),
Some((0, ProviderAnswer::Absent)), &mut rounds_used,
0,
|h| h.hash == hb(50),
)
.await
.expect_err("primary absent + one round absent must confirm absence");
assert!(matches!(err, StepFailure::AbsenceConfirmed));
}
#[tokio::test]
async fn resolve_step_http_primary_unresolved() {
let settings = n_http_provider_settings(3);
let voter1 = Asserter::new();
voter1.push_failure_msg("blip");
let voter2 = Asserter::new();
voter2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(voter1),
mock_http_provider(voter2),
];
let mut rounds_used = 0u32;
let err = resolve_step(
"test",
&settings,
Some(&reused),
3,
&[1, 2],
hb(50),
Some((0, ProviderAnswer::Absent)), &mut rounds_used,
0,
|h| h.hash == hb(50),
)
.await
.expect_err("1 total absent answer must not confirm absence");
assert!(matches!(err, StepFailure::Unresolved(_)));
}
#[tokio::test]
async fn resolve_step_unresolved_when_round_budget_exhausted() {
let settings = n_http_provider_settings(1);
let mut rounds_used = MAX_STEP_RESOLUTION_ROUNDS; let err = resolve_step(
"test",
&settings,
None,
1,
&[0],
hb(50),
None,
&mut rounds_used,
0,
|h| h.hash == hb(50),
)
.await
.expect_err("budget exhaustion must fail closed, no dispatch");
assert!(matches!(err, StepFailure::Unresolved(_)));
assert_eq!(
rounds_used, MAX_STEP_RESOLUTION_ROUNDS,
"no round should be spent past the cap"
);
}
#[tokio::test]
async fn by_hash_tail_fetch_over_mock() {
let walk_asserter = Asserter::new();
walk_asserter.push_success(&block_with_parent_at(103, hb(103), hb(102))); walk_asserter.push_success(&block_with_parent_at(102, hb(102), hb(101))); walk_asserter.push_success(&block_with_parent_at(101, hb(101), hb(100))); let walk_provider = mock_http_provider(walk_asserter);
let walk_settings = n_http_provider_settings(1);
let pairs_desc =
collect_window_hashes(&walk_provider, &walk_settings, None, 0, hb(103), 103, 100)
.await
.expect("walk should succeed");
assert_eq!(pairs_desc.len(), 4);
assert_eq!(pairs_desc[0], (103, hb(103)));
assert_eq!(pairs_desc[1], (102, hb(102)));
assert_eq!(pairs_desc[2], (101, hb(101)));
assert_eq!(pairs_desc[3], (100, hb(100)));
let mut pairs = pairs_desc;
pairs.reverse();
let log_asserter = Asserter::new();
for n in 100u64..=103 {
log_asserter.push_success(&serde_json::json!([{
"address": "0x0000000000000000000000000000000000000001",
"topics": [],
"data": "0x",
"blockHash": hb(n as u8).to_string(),
"blockNumber": format!("0x{:x}", n),
"transactionHash": hb(0).to_string(),
"transactionIndex": "0x0",
"logIndex": "0x0",
"removed": false
}]));
}
let log_provider = ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(log_asserter)
.erased();
let settings = ProviderSettings::from_mock(log_provider);
let ranges = fetch_logs_by_hash_range(&settings, &pairs, None)
.await
.expect("log fetch should succeed");
assert_eq!(ranges.len(), 4, "expected one range per block");
for (i, (from, to, logs)) in ranges.iter().enumerate() {
let block_n = 100u64 + i as u64;
assert_eq!(*from, block_n, "from_block mismatch at index {i}");
assert_eq!(*to, block_n, "to_block mismatch at index {i}");
assert_eq!(logs.len(), 1, "expected one log at block {block_n}");
assert_eq!(
logs[0].block_number,
Some(block_n),
"log block number mismatch"
);
}
}
#[tokio::test]
async fn orphan_rejection() {
let canonical_log = serde_json::json!([{
"address": "0xCAFECAFECAFECAFECAFECAFECAFECAFECAFECAFE",
"topics": [],
"data": "0x01",
"blockHash": hb(200).to_string(),
"blockNumber": "0xc8",
"transactionHash": hb(1).to_string(),
"transactionIndex": "0x0",
"logIndex": "0x0",
"removed": false
}]);
let asserter = Asserter::new();
asserter.push_success(&canonical_log);
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let mut linkage = HeadLinkage::new(8);
linkage.observe(200, hb(200), hb(199)); let mut coalescer = HeadCoalescer::new(Some(200), WINDOW);
coalescer.observe_head(200, Instant::now());
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect("flush should succeed");
assert_eq!(ranges.len(), 1, "one range per block");
let (from, to, logs) = &ranges[0];
assert_eq!((*from, *to), (200, 200));
assert_eq!(logs.len(), 1, "canonical log returned");
assert_eq!(
logs[0].block_hash,
Some(hb(200)),
"log must carry the canonical block hash, not an orphan's"
);
assert_eq!(coalescer.pending_range(), None);
}
#[tokio::test]
async fn hash_mismatch_null_block_fail_closed() {
{
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err = collect_window_hashes(&provider, &settings, None, 0, hb(5), 5, 1)
.await
.expect_err("null block must be an error");
assert!(
matches!(err, RpcError::ReorgBeyondFinality),
"null block must yield ReorgBeyondFinality, got {err:?}"
);
}
{
let asserter = Asserter::new();
asserter.push_failure_msg("connection reset");
let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err = collect_window_hashes(&provider, &settings, None, 0, hb(5), 5, 1)
.await
.expect_err("transport error must be an error");
assert!(
matches!(err, RpcError::ConnectionError(_)),
"transport fault must yield ConnectionError, got {err:?}"
);
}
}
#[test]
fn dormancy_at_depth_zero() {
let mut linkage = HeadLinkage::new(0);
assert!(!linkage.enabled(), "depth=0 must be disabled");
assert_eq!(
linkage.observe(100, hb(1), hb(99)),
LinkOutcome::Linked,
"depth=0 must not trigger reorg detection"
);
assert_eq!(
linkage.observe(101, hb(2), hb(55)), LinkOutcome::Linked,
"depth=0 must not trigger reorg detection on bad parent"
);
assert_eq!(
linkage.floor(),
None,
"disabled linkage must not record state"
);
}
#[test]
fn boot_tail_seeds_ring() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(
linkage.observe(105, hb(105), hb(104)),
LinkOutcome::Linked,
"bootstrap observe"
);
assert_eq!(linkage.hash_at(105), Some(hb(105)));
assert_eq!(
linkage.observe(106, hb(106), hb(105)),
LinkOutcome::Linked,
"first post-flush head must link without false reorg"
);
assert_eq!(linkage.hash_at(106), Some(hb(106)));
assert_eq!(linkage.observe(107, hb(107), hb(106)), LinkOutcome::Linked);
}
#[tokio::test]
async fn live_flush_gap_self_heals() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(105, hb(105), hb(104)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(101), WINDOW);
coalescer.observe_head(105, Instant::now());
assert_eq!(coalescer.pending_range(), Some((101, 105)));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(105, hb(105), hb(104)));
asserter.push_success(&block_with_parent_at(104, hb(104), hb(103)));
asserter.push_success(&block_with_parent_at(103, hb(103), hb(102)));
asserter.push_success(&block_with_parent_at(102, hb(102), hb(101)));
for _ in 101u64..=105 {
asserter.push_success(&serde_json::json!([]));
}
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect("flush must succeed, not spin");
assert_eq!(ranges.len(), 5, "one LogRange per block 101..=105");
for (i, (from, to, _)) in ranges.iter().enumerate() {
let n = 101u64 + i as u64;
assert_eq!((*from, *to), (n, n));
}
assert_eq!(coalescer.pending_range(), None);
}
#[tokio::test]
async fn collect_window_hashes_number_mismatch() {
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(99, hb(5), hb(4))); let provider = mock_http_provider(asserter);
let settings = n_http_provider_settings(1);
let err = collect_window_hashes(&provider, &settings, None, 0, hb(5), 5, 1)
.await
.expect_err("number mismatch must be an error");
assert!(
matches!(err, RpcError::ReorgBeyondFinality),
"number mismatch must yield ReorgBeyondFinality, got {err:?}"
);
}
#[tokio::test]
async fn collect_window_hashes_second_source_continues_walk() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null)); primary_asserter.push_success(&block_with_parent_at(4, hb(4), hb(3))); let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&block_with_parent_at(5, hb(5), hb(4))); let corrob2 = Asserter::new();
corrob2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(Asserter::new()), mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let pairs_desc = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 3)
.await
.expect("one second source supplying the block must let the walk continue");
assert_eq!(
pairs_desc,
vec![(5, hb(5)), (4, hb(4)), (3, hb(3))],
"got {pairs_desc:?}"
);
}
#[tokio::test]
async fn collect_window_hashes_all_sources_absent_is_beyond_finality() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&serde_json::json!(null));
let corrob2 = Asserter::new();
corrob2.push_success(&serde_json::json!(null));
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 1)
.await
.expect_err("3 absent answers must confirm absence");
assert!(matches!(err, RpcError::ReorgBeyondFinality), "got {err:?}");
}
#[tokio::test]
async fn collect_window_hashes_lone_absent_answer_is_unresolved() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_failure_msg("blip");
let corrob2 = Asserter::new();
corrob2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 1)
.await
.expect_err("1 absent answer must not confirm absence");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[tokio::test]
async fn collect_window_hashes_primary_answer_completes_absence() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&serde_json::json!(null));
let corrob2 = Asserter::new();
corrob2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 1)
.await
.expect_err("primary absent + one second source absent must confirm absence");
assert!(matches!(err, RpcError::ReorgBeyondFinality), "got {err:?}");
}
#[tokio::test]
async fn collect_window_hashes_wrong_number_is_not_a_recovery() {
let primary_asserter = Asserter::new();
primary_asserter.push_success(&serde_json::json!(null));
let provider = mock_http_provider(primary_asserter);
let settings = n_http_provider_settings(3);
let corrob1 = Asserter::new();
corrob1.push_success(&block_with_parent_at(99, hb(5), hb(4)));
let corrob2 = Asserter::new();
corrob2.push_failure_msg("blip");
let reused = vec![
mock_http_provider(Asserter::new()),
mock_http_provider(corrob1),
mock_http_provider(corrob2),
];
let err = collect_window_hashes(&provider, &settings, Some(&reused), 0, hb(5), 5, 1)
.await
.expect_err("wrong-number second source must not match, and must not count as absent");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[tokio::test]
async fn flush_window_range_empty_ring_is_fail_closed() {
let asserter = Asserter::new(); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let mut linkage = HeadLinkage::new(8); let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(100, Instant::now());
let err = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect_err("empty ring must be fail-closed");
assert!(matches!(err, RpcError::ReorgBeyondFinality), "got {err:?}");
}
struct AlwaysOnSubscriber;
impl tracing::Subscriber for AlwaysOnSubscriber {
fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
fn event(&self, _event: &tracing::Event<'_>) {}
fn enter(&self, _span: &tracing::span::Id) {}
fn exit(&self, _span: &tracing::span::Id) {}
}
#[tokio::test]
async fn flush_window_range_self_heal_with_stale_ring_does_not_panic() {
let asserter = Asserter::new(); let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(50, hb(50), hb(49)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(100, Instant::now());
assert_eq!(coalescer.pending_range(), Some((100, 100)));
let subscriber_guard = tracing::subscriber::set_default(AlwaysOnSubscriber);
tracing::callsite::rebuild_interest_cache();
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect(
"a stale ring must not panic — collect_window_hashes's own guard \
no-ops the walk",
);
assert!(
ranges.is_empty(),
"no hashes are resolvable above the stale ring's tip"
);
drop(subscriber_guard);
tracing::callsite::rebuild_interest_cache();
}
#[tokio::test]
async fn flush_window_range_self_heal_walk_uses_reused_http_client() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(105, hb(105), hb(104)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(101), WINDOW);
coalescer.observe_head(105, Instant::now());
assert_eq!(coalescer.pending_range(), Some((101, 105)));
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(105, hb(105), hb(104)));
asserter.push_success(&block_with_parent_at(104, hb(104), hb(103)));
asserter.push_success(&block_with_parent_at(103, hb(103), hb(102)));
asserter.push_success(&block_with_parent_at(102, hb(102), hb(101)));
for _ in 101u64..=105 {
asserter.push_success(&serde_json::json!([]));
}
let reused = vec![mock_http_provider(asserter)];
let settings = ProviderSettings::new(
vec![HttpRpcSettings::new("http://unused".to_string(), 1, 100)],
vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let ranges = flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect(
"self-heal walk must resolve its client from `reused`, not rebuild via connect_http",
);
assert_eq!(ranges.len(), 5, "one LogRange per block 101..=105");
for (i, (from, to, _)) in ranges.iter().enumerate() {
let n = 101u64 + i as u64;
assert_eq!((*from, *to), (n, n));
}
}
#[tokio::test]
async fn flush_window_range_empty_http_providers_is_fail_closed_not_a_panic() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(100, hb(100), hb(99)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(100), WINDOW);
coalescer.observe_head(100, Instant::now());
assert_eq!(coalescer.pending_range(), Some((100, 100)));
let settings = ProviderSettings::new(
vec![], vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let err = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect_err("zero HTTP providers must be fail-closed, not panic");
assert!(matches!(err, RpcError::ConnectionError(_)), "got {err:?}");
}
#[test]
fn window_boundary() {
let tip: u64 = 1000;
let depth: u64 = 50;
let window_floor = tip.saturating_sub(depth);
for n in [0u64, 500, 949] {
assert!(
n < window_floor,
"block {n} should be below the window floor {window_floor}"
);
}
for n in [950u64, 975, 1000] {
assert!(
n >= window_floor,
"block {n} should be inside the by-hash window (floor {window_floor})"
);
}
let no_window_floor = tip.saturating_sub(0);
assert_eq!(no_window_floor, tip, "depth=0 must not create a window");
let huge_depth_floor = tip.saturating_sub(tip + 1);
assert_eq!(
huge_depth_floor, 0,
"depth > tip must floor at 0 (all blocks in window)"
);
}
#[tokio::test]
async fn flush_window_range_partial_failure_leaves_cursor_unmoved() {
let mut linkage = HeadLinkage::new(8);
assert_eq!(linkage.observe(200, hb(200), hb(199)), LinkOutcome::Linked);
assert_eq!(linkage.observe(201, hb(201), hb(200)), LinkOutcome::Linked);
let mut coalescer = HeadCoalescer::new(Some(200), WINDOW);
coalescer.observe_head(201, Instant::now());
assert_eq!(coalescer.pending_range(), Some((200, 201)));
let asserter = Asserter::new();
asserter.push_success(&serde_json::json!([])); asserter.push_failure_msg("transient error");
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let result =
flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None).await;
assert!(result.is_err(), "partial failure must return Err");
assert_eq!(
coalescer.pending_range(),
Some((200, 201)),
"cursor must stay at 200 so block 200 is not silently skipped on reconnect"
);
}
#[tokio::test(start_paused = true)]
async fn bound_flush_time_disabled_at_zero_awaits_normally() {
let result: Result<u32, RpcError> = bound_flush_time(0, async {
tokio::time::sleep(Duration::from_millis(50)).await;
Ok(42)
})
.await;
assert_eq!(
result.unwrap(),
42,
"max_secs=0 must not time out, however long the future takes"
);
}
#[tokio::test(start_paused = true)]
async fn bound_flush_time_fires_on_a_slow_future() {
let result: Result<u32, RpcError> = bound_flush_time(1, async {
tokio::time::sleep(Duration::from_millis(1200)).await;
Ok(42)
})
.await;
assert!(
matches!(result, Err(RpcError::ConnectionError(_))),
"a future exceeding max_secs must yield ConnectionError, got {result:?}"
);
}
#[tokio::test]
async fn live_loop_wiring_gap_then_reorg() {
let now = Instant::now();
let mut coalescer = HeadCoalescer::new(Some(101), WINDOW);
let mut linkage = HeadLinkage::new(8);
assert!(coalescer.observe_head(105, now));
assert_eq!(coalescer.pending_range(), Some((101, 105)));
assert_eq!(linkage.observe(105, hb(105), hb(104)), LinkOutcome::Linked);
assert_eq!(linkage.latest(), Some((105, hb(105))));
let after_window = now + WINDOW + Duration::from_millis(1);
assert!(
coalescer.should_flush(after_window),
"coalesce window expired — flush must fire"
);
let asserter = Asserter::new();
asserter.push_success(&block_with_parent_at(105, hb(105), hb(104)));
asserter.push_success(&block_with_parent_at(104, hb(104), hb(103)));
asserter.push_success(&block_with_parent_at(103, hb(103), hb(102)));
asserter.push_success(&block_with_parent_at(102, hb(102), hb(101)));
for _ in 101u64..=105 {
asserter.push_success(&serde_json::json!([]));
}
let settings = ProviderSettings::from_mock(
ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(asserter)
.erased(),
);
let ranges = flush_window_range(&mut linkage, &mut coalescer, &settings, None, 0, None)
.await
.expect("gap self-heal must succeed");
assert_eq!(ranges.len(), 5, "one range per block 101..=105");
for (i, &(from, to, _)) in ranges.iter().enumerate() {
let n = 101u64 + i as u64;
assert_eq!((from, to), (n, n), "range at index {i} must be block {n}");
}
assert_eq!(
coalescer.pending_range(),
None,
"cursor must be at 106 after flush"
);
assert_eq!(linkage.observe(106, hb(106), hb(105)), LinkOutcome::Linked);
assert_eq!(linkage.observe(107, hb(107), hb(106)), LinkOutcome::Linked);
let outcome = linkage.observe(107, hb(250), hb(106));
assert!(
matches!(
outcome,
LinkOutcome::Reorg {
common_ancestor: 106
}
),
"competing block at 107 with canonical parent must produce Reorg{{106}}, got {outcome:?}"
);
}
#[test]
fn capture_policy_notice_speaks_once_per_policy() {
let notice = CapturePolicyNotice::new();
let latest_block = 1_000u64;
let latest_first = notice.first_message(CaptureTarget::Latest, latest_block);
assert!(
latest_first.is_some(),
"first Latest resolution must announce"
);
let latest_message = latest_first.unwrap();
assert!(
!latest_message.contains("DEEP LAG"),
"Latest must not claim deep lag: {latest_message}"
);
assert!(
notice
.first_message(CaptureTarget::Latest, latest_block)
.is_none(),
"second Latest resolution must be silent"
);
assert!(
notice
.first_message(CaptureTarget::Latest, latest_block)
.is_none(),
"third Latest resolution must be silent"
);
let lag_block = 900u64;
assert!(
notice
.first_message(CaptureTarget::Lag(64), lag_block)
.is_some(),
"first Lag resolution must announce (Latest already having spoken must not \
consume Lag's independent gate)"
);
assert!(
notice
.first_message(CaptureTarget::Lag(64), lag_block)
.is_none(),
"second Lag resolution must be silent"
);
assert!(
notice
.first_message(CaptureTarget::Lag(64), lag_block)
.is_none(),
"third Lag resolution must be silent"
);
assert!(
notice
.first_message(CaptureTarget::Lag(200), lag_block)
.is_none(),
"gating is per-variant, not per-value: a distinct Lag(n) must not re-arm the gate"
);
let finalized_block = 500u64;
let finalized_first = notice.first_message(CaptureTarget::Finalized, finalized_block);
assert!(
finalized_first.is_some(),
"first Finalized resolution must announce (Latest and Lag having already \
spoken must not consume Finalized's independent gate)"
);
let finalized_message = finalized_first.unwrap();
assert!(
finalized_message.contains("DEEP LAG ACTIVE"),
"Finalized message must claim deep lag: {finalized_message}"
);
assert!(
finalized_message.contains(&finalized_block.to_string()),
"Finalized message must report the resolved block: {finalized_message}"
);
assert!(
notice
.first_message(CaptureTarget::Finalized, finalized_block)
.is_none(),
"second Finalized resolution must be silent"
);
assert!(
notice
.first_message(CaptureTarget::Finalized, finalized_block)
.is_none(),
"third Finalized resolution must be silent"
);
}
#[test]
fn resolve_http_provider_reuses_vs_rebuilds() {
let settings = ProviderSettings::new(
vec![HttpRpcSettings::new(
"http://127.0.0.1:9".to_string(),
1,
100,
)],
vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let reused = settings.build_reused_http_clients();
assert_eq!(reused.len(), 1, "one client per configured HTTP provider");
let a = resolve_http_provider(&settings, Some(&reused), 0);
let b = resolve_http_provider(&settings, Some(&reused), 0);
assert!(
std::sync::Weak::ptr_eq(&a.weak_client(), &b.weak_client()),
"Some(reused) at the same index must return the identical client"
);
let c = resolve_http_provider(&settings, None, 0);
let d = resolve_http_provider(&settings, None, 0);
assert!(
!std::sync::Weak::ptr_eq(&c.weak_client(), &d.weak_client()),
"None must rebuild a fresh client on every call, not reuse one"
);
}
#[tokio::test(start_paused = true)]
async fn fetch_concurrently_runs_items_in_parallel_up_to_bound() {
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let bound = 4usize;
let n = 10usize;
let in_flight = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let items: Vec<usize> = (0..n).collect();
let in_flight_for_fetch = in_flight.clone();
let peak_for_fetch = peak.clone();
let fetch = move |_: usize| {
let in_flight = in_flight_for_fetch.clone();
let peak = peak_for_fetch.clone();
async move {
let now = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(now, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(50)).await;
in_flight.fetch_sub(1, Ordering::SeqCst);
Ok::<usize, RpcError>(0)
}
};
let result = fetch_concurrently(items, bound, fetch).await;
assert!(result.is_ok(), "all items succeed: {result:?}");
let observed_peak = peak.load(Ordering::SeqCst);
assert!(
observed_peak > 1,
"expected concurrent dispatch (peak > 1), got peak={observed_peak}"
);
assert!(
observed_peak <= bound,
"peak in-flight {observed_peak} exceeded the bound {bound}"
);
}
#[tokio::test]
async fn fetch_concurrently_fails_fast_not_collect_then_check() {
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let bound = 3usize;
let n = 10usize;
let items: Vec<usize> = (0..n).collect();
let invoked = Arc::new(AtomicUsize::new(0));
let invoked_for_fetch = invoked.clone();
let fetch = move |i: usize| {
invoked_for_fetch.fetch_add(1, Ordering::SeqCst);
async move {
if i == 0 {
Err(RpcError::LogFetchError("early failure".to_string()))
} else {
Ok::<usize, RpcError>(i)
}
}
};
let result = fetch_concurrently(items, bound, fetch).await;
assert!(result.is_err(), "an early failure must surface as Err");
let invoked_count = invoked.load(Ordering::SeqCst);
assert!(
invoked_count < n,
"expected fewer than {n} closures invoked under fail-fast, got {invoked_count}"
);
}
#[tokio::test(start_paused = true)]
async fn fetch_concurrently_fails_fast_under_early_slow_neighbor() {
let bound = 2usize;
let items = vec![0usize, 1usize];
let long_sleep = Duration::from_secs(3600);
let fetch = move |i: usize| async move {
if i == 0 {
tokio::time::sleep(long_sleep).await;
Ok::<usize, RpcError>(0)
} else {
Err(RpcError::LogFetchError("fast failure".to_string()))
}
};
let start = Instant::now();
let result = fetch_concurrently(items, bound, fetch).await;
let elapsed = start.elapsed();
assert!(result.is_err(), "overall result must be Err");
assert!(
elapsed < long_sleep,
"Err must be observable before item 0's sleep elapses, elapsed={elapsed:?}"
);
}
#[test]
fn sort_ranges_ascending_orders_by_block_number() {
let unordered: Vec<(u64, u64, Vec<Log>)> =
vec![(105, 105, vec![]), (100, 100, vec![]), (103, 103, vec![])];
let sorted = sort_ranges_ascending(unordered);
let numbers: Vec<u64> = sorted.iter().map(|&(from, _, _)| from).collect();
assert_eq!(numbers, vec![100, 103, 105]);
}
#[tokio::test(start_paused = true)]
async fn flush_window_range_dispatches_concurrently_and_restores_ascending_order() {
use alloy::network::Ethereum;
use alloy::providers::{DynProvider, RootProvider};
use alloy::rpc::types::Filter;
use alloy::transports::TransportResult;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Clone)]
struct DelayedLogsProvider {
stub_root: DynProvider<Ethereum>,
call_order: Arc<AtomicUsize>,
delays_by_call_order: Arc<Vec<Duration>>,
in_flight: Arc<AtomicUsize>,
peak_in_flight: Arc<AtomicUsize>,
completion_order: Arc<Mutex<Vec<usize>>>,
}
#[async_trait::async_trait]
impl Provider<Ethereum> for DelayedLogsProvider {
fn root(&self) -> &RootProvider<Ethereum> {
self.stub_root.root()
}
async fn get_logs(&self, _filter: &Filter) -> TransportResult<Vec<Log>> {
let call_index = self.call_order.fetch_add(1, Ordering::SeqCst);
let now_in_flight = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1;
self.peak_in_flight
.fetch_max(now_in_flight, Ordering::SeqCst);
let delay = self
.delays_by_call_order
.get(call_index)
.copied()
.unwrap_or(Duration::ZERO);
tokio::time::sleep(delay).await;
self.in_flight.fetch_sub(1, Ordering::SeqCst);
self.completion_order.lock().unwrap().push(call_index);
Ok(Vec::new())
}
}
let mut linkage = HeadLinkage::new(8);
for n in 300u64..=303 {
assert_eq!(
linkage.observe(n, hb(n as u8), hb((n - 1) as u8)),
LinkOutcome::Linked
);
}
let mut coalescer = HeadCoalescer::new(Some(300), WINDOW);
coalescer.observe_head(303, Instant::now());
assert_eq!(coalescer.pending_range(), Some((300, 303)));
let delays_by_call_order = vec![
Duration::from_millis(500), Duration::from_millis(10), Duration::from_millis(20), Duration::from_millis(30), ];
let peak_in_flight = Arc::new(AtomicUsize::new(0));
let completion_order = Arc::new(Mutex::new(Vec::new()));
let unused_asserter = Asserter::new(); let double = DelayedLogsProvider {
stub_root: ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(unused_asserter)
.erased(),
call_order: Arc::new(AtomicUsize::new(0)),
delays_by_call_order: Arc::new(delays_by_call_order),
in_flight: Arc::new(AtomicUsize::new(0)),
peak_in_flight: peak_in_flight.clone(),
completion_order: completion_order.clone(),
};
let reused = vec![double.erased()];
let settings = ProviderSettings::new(
vec![HttpRpcSettings::new("http://unused".to_string(), 4, 100)],
vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let ranges = flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect("flush must succeed");
assert_eq!(
peak_in_flight.load(Ordering::SeqCst),
4,
"all 4 blocks' fetches must be in flight together, not one at a time"
);
let observed_completion_order = completion_order.lock().unwrap().clone();
assert_eq!(
&observed_completion_order[0..3],
&[1, 2, 3],
"the 3 fast fetches (301-303) must resolve before the slow one (300), \
got order {observed_completion_order:?}"
);
assert_eq!(
observed_completion_order[3], 0,
"the slow fetch (300) must resolve last, proving it did not block the \
others, got order {observed_completion_order:?}"
);
assert_eq!(ranges.len(), 4);
let numbers: Vec<u64> = ranges.iter().map(|&(from, _, _)| from).collect();
assert_eq!(numbers, vec![300, 301, 302, 303]);
}
#[tokio::test(start_paused = true)]
async fn flush_window_range_peak_in_flight_tracks_configured_max_concurrency() {
use alloy::network::Ethereum;
use alloy::providers::{DynProvider, RootProvider};
use alloy::rpc::types::Filter;
use alloy::transports::TransportResult;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Clone)]
struct PeakTrackingLogsProvider {
stub_root: DynProvider<Ethereum>,
in_flight: Arc<AtomicUsize>,
peak_in_flight: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl Provider<Ethereum> for PeakTrackingLogsProvider {
fn root(&self) -> &RootProvider<Ethereum> {
self.stub_root.root()
}
async fn get_logs(&self, _filter: &Filter) -> TransportResult<Vec<Log>> {
let now_in_flight = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1;
self.peak_in_flight
.fetch_max(now_in_flight, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(20)).await;
self.in_flight.fetch_sub(1, Ordering::SeqCst);
Ok(Vec::new())
}
}
let mut linkage = HeadLinkage::new(8);
for n in 400u64..=405 {
assert_eq!(
linkage.observe(n, hb(n as u8), hb((n - 1) as u8)),
LinkOutcome::Linked
);
}
let mut coalescer = HeadCoalescer::new(Some(400), WINDOW);
coalescer.observe_head(405, Instant::now());
assert_eq!(coalescer.pending_range(), Some((400, 405)));
let peak_in_flight = Arc::new(AtomicUsize::new(0));
let unused_asserter = Asserter::new(); let double = PeakTrackingLogsProvider {
stub_root: ProviderBuilder::new()
.disable_recommended_fillers()
.connect_mocked_client(unused_asserter)
.erased(),
in_flight: Arc::new(AtomicUsize::new(0)),
peak_in_flight: peak_in_flight.clone(),
};
let reused = vec![double.erased()];
let settings = ProviderSettings::new(
vec![HttpRpcSettings::new("http://unused".to_string(), 2, 100)],
vec!["ws://127.0.0.1:9".parse().expect("static url")],
);
let ranges = flush_window_range(
&mut linkage,
&mut coalescer,
&settings,
None,
0,
Some(&reused),
)
.await
.expect("flush must succeed");
assert_eq!(
peak_in_flight.load(Ordering::SeqCst),
2,
"peak in-flight must equal the configured max_concurrency (2), not \
the pending block count (6) or any other hardcoded bound"
);
assert_eq!(ranges.len(), 6);
}
}