use std::{
collections::HashMap,
net::IpAddr,
pin::Pin,
sync::Arc,
task::{Context, Poll},
};
use futures::{
future::{BoxFuture, FutureExt, TryFutureExt},
ready,
stream::{FuturesUnordered, Stream},
};
use pin_project::pin_project;
use tokio::{
sync::{mpsc, oneshot, Mutex},
task::JoinHandle,
};
use tower::{Service, ServiceExt};
use tracing::Instrument;
use zakura_chain::{
block::{self, HeightDiff},
chain_tip::ChainTip,
parameters::Network,
};
use zakura_network::{self as zn, PeerSocketAddr};
use zakura_state as zs;
use crate::components::{
auth_download_height::tip_child_mismatch,
download_work::has_sufficient_work,
sync::{lookahead_limit_multiplier, MIN_CONCURRENCY_LIMIT},
};
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
#[derive(Copy, Clone, Debug, thiserror::Error)]
#[error("gossiped tip child claimed height {height:?} instead of {expected_height:?}: {hash:?}")]
pub struct GossipedTipChildHeightMismatch {
pub height: block::Height,
pub expected_height: block::Height,
pub hash: block::Hash,
pub parent_recovery: bool,
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub enum AdvertiserSource {
LegacyIp(IpAddr),
Zakura(zn::zakura::ZakuraPeerId),
}
impl From<zn::PeerSource> for AdvertiserSource {
fn from(source: zn::PeerSource) -> Self {
match source {
zn::PeerSource::LegacySocket(addr) => Self::LegacyIp(addr.ip()),
zn::PeerSource::Zakura(peer_id) => Self::Zakura(peer_id),
}
}
}
impl AdvertiserSource {
fn max_in_flight(&self, global_limit: usize) -> usize {
match self {
Self::LegacyIp(_) => 1,
Self::Zakura(_) => MAX_INBOUND_BLOCK_CONCURRENCY_PER_PEER.min(global_limit),
}
}
}
fn peer_source_log_label(source: &zn::PeerSource, expose_peer_addresses: bool) -> String {
match source {
zn::PeerSource::LegacySocket(addr) if expose_peer_addresses => {
format!("legacy:{}", addr.remove_socket_addr_privacy())
}
zn::PeerSource::LegacySocket(_) => "legacy:redacted".to_string(),
zn::PeerSource::Zakura(peer_id) => format!("zakura:{peer_id:?}"),
}
}
type DownloadHandle =
JoinHandle<Result<block::Hash, (BoxError, block::Hash, Option<PeerSocketAddr>)>>;
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
enum DownloadKind {
Advertised,
ParentRecovery,
}
#[derive(Clone, Debug)]
struct DownloadTask {
hash: block::Hash,
download_source: Option<zn::PeerSource>,
advertiser: Option<AdvertiserSource>,
kind: DownloadKind,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
enum ParentRecovery {
Closed,
Attached,
Requested,
TooManyRecoveries,
SourceRecoveryInFlight,
FullQueue,
InvalidProofOfWork,
}
impl ParentRecovery {
fn metric_label(self) -> &'static str {
match self {
Self::Closed => "closed",
Self::Attached => "attached",
Self::Requested => "requested",
Self::TooManyRecoveries => "too_many_recoveries",
Self::SourceRecoveryInFlight => "source_recovery_in_flight",
Self::FullQueue => "full_queue",
Self::InvalidProofOfWork => "invalid_proof_of_work",
}
}
}
pub const MAX_INBOUND_CONCURRENCY: usize = 200;
pub const MAX_INBOUND_BLOCK_CONCURRENCY_PER_PEER: usize = 5;
pub const POISONED_GOSSIP_BLOCK_RETRY_LIMIT: usize = 3;
pub(crate) fn max_lookahead_height(
network: &Network,
tip_height: Option<block::Height>,
full_verify_concurrency_limit: usize,
) -> block::Height {
if let Some(tip_height) = tip_height {
let lookahead = HeightDiff::try_from(
full_verify_concurrency_limit * lookahead_limit_multiplier(network, tip_height),
)
.expect("fits in HeightDiff");
(tip_height + lookahead).expect("tip is much lower than Height::MAX")
} else {
let genesis_lookahead =
u32::try_from(full_verify_concurrency_limit - 1).expect("fits in u32");
block::Height(genesis_lookahead)
}
}
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum DownloadAction {
AddedToQueue,
AlreadyQueued,
FullQueue,
}
#[derive(Debug, Default)]
struct PoisonedRetryBudgets {
counts: HashMap<block::Hash, usize>,
tip: Option<block::Hash>,
}
impl PoisonedRetryBudgets {
fn consume(&mut self, hash: block::Hash, tip_hash: Option<block::Hash>) -> Option<usize> {
if self.tip != tip_hash {
self.counts.clear();
self.tip = tip_hash;
}
let spent = self.counts.entry(hash).or_default();
if *spent >= POISONED_GOSSIP_BLOCK_RETRY_LIMIT {
return None;
}
*spent += 1;
Some(*spent)
}
fn release(&mut self, hash: &block::Hash) {
self.counts.remove(hash);
}
}
pub const MAX_PARENT_RECOVERIES: usize = 4;
const MAX_PARENT_RECOVERY_CHILD_HEIGHT_ABOVE_TIP: HeightDiff = 2;
#[derive(Debug, Default)]
struct Admission {
closed: bool,
cancel_handles: HashMap<block::Hash, (oneshot::Sender<()>, Option<AdvertiserSource>)>,
source_locks: HashMap<AdvertiserSource, Arc<Mutex<()>>>,
source_counts: HashMap<AdvertiserSource, usize>,
parent_recoveries: HashMap<block::Hash, Option<AdvertiserSource>>,
}
impl Admission {
fn queue_len(&self) -> usize {
self.cancel_handles.len()
}
fn release(&mut self, hash: &block::Hash) {
self.parent_recoveries.remove(hash);
if let Some((_, Some(source))) = self.cancel_handles.remove(hash) {
let source_count = self
.source_counts
.get_mut(&source)
.expect("source count is inserted when a download task is admitted");
*source_count = source_count
.checked_sub(1)
.expect("source count is positive while a download task is admitted");
if *source_count == 0 {
self.source_counts.remove(&source);
self.source_locks.remove(&source);
}
}
}
}
#[derive(Debug)]
struct TaskSpawner<ZN, ZV, ZS> {
full_verify_concurrency_limit: usize,
network: ZN,
verifier: ZV,
state: ZS,
latest_chain_tip: zs::LatestChainTip,
chain_network: Network,
admission: Arc<std::sync::Mutex<Admission>>,
spawned_sender: mpsc::UnboundedSender<DownloadHandle>,
}
impl<ZN: Clone, ZV: Clone, ZS: Clone> Clone for TaskSpawner<ZN, ZV, ZS> {
fn clone(&self) -> Self {
Self {
full_verify_concurrency_limit: self.full_verify_concurrency_limit,
network: self.network.clone(),
verifier: self.verifier.clone(),
state: self.state.clone(),
latest_chain_tip: self.latest_chain_tip.clone(),
chain_network: self.chain_network.clone(),
admission: self.admission.clone(),
spawned_sender: self.spawned_sender.clone(),
}
}
}
#[pin_project(PinnedDrop)]
#[derive(Debug)]
pub struct Downloads<ZN, ZV, ZS>
where
ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
ZN::Future: Send,
ZV: Service<zakura_consensus::Request, Response = block::Hash, Error = BoxError>
+ Send
+ Clone
+ 'static,
ZV::Future: Send,
ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
ZS::Future: Send,
{
expose_peer_addresses: bool,
spawner: TaskSpawner<ZN, ZV, ZS>,
#[pin]
pending: FuturesUnordered<DownloadHandle>,
spawned_receiver: mpsc::UnboundedReceiver<DownloadHandle>,
poisoned_retries: PoisonedRetryBudgets,
}
#[pin_project::pinned_drop]
impl<ZN, ZV, ZS> PinnedDrop for Downloads<ZN, ZV, ZS>
where
ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
ZN::Future: Send,
ZV: Service<zakura_consensus::Request, Response = block::Hash, Error = BoxError>
+ Send
+ Clone
+ 'static,
ZV::Future: Send,
ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
ZS::Future: Send,
{
fn drop(self: Pin<&mut Self>) {
let mut admission = self.spawner.admission();
admission.closed = true;
admission.cancel_handles.clear();
}
}
impl<ZN, ZV, ZS> Stream for Downloads<ZN, ZV, ZS>
where
ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
ZN::Future: Send,
ZV: Service<zakura_consensus::Request, Response = block::Hash, Error = BoxError>
+ Send
+ Clone
+ 'static,
ZV::Future: Send,
ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
ZS::Future: Send,
{
type Item = Result<block::Hash, (BoxError, Option<PeerSocketAddr>)>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
let mut this = self.project();
while let Poll::Ready(Some(handle)) = this.spawned_receiver.poll_recv(cx) {
this.pending.push(handle);
}
if let Some(join_result) = ready!(this.pending.as_mut().poll_next(cx)) {
let result = match join_result.expect("block download and verify tasks must not panic")
{
Ok(hash) => {
this.poisoned_retries.release(&hash);
Ok(hash)
}
Err((e, _hash, advertiser_addr)) => Err((e, advertiser_addr)),
};
Poll::Ready(Some(result))
} else {
Poll::Ready(None)
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.pending.size_hint()
}
}
impl<ZN, ZV, ZS> Downloads<ZN, ZV, ZS>
where
ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
ZN::Future: Send,
ZV: Service<zakura_consensus::Request, Response = block::Hash, Error = BoxError>
+ Send
+ Clone
+ 'static,
ZV::Future: Send,
ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
ZS::Future: Send,
{
pub fn new(
full_verify_concurrency_limit: usize,
expose_peer_addresses: bool,
network: ZN,
verifier: ZV,
state: ZS,
latest_chain_tip: zs::LatestChainTip,
chain_network: Network,
) -> Self {
let full_verify_concurrency_limit =
full_verify_concurrency_limit.clamp(MIN_CONCURRENCY_LIMIT, MAX_INBOUND_CONCURRENCY);
let (spawned_sender, spawned_receiver) = mpsc::unbounded_channel();
Self {
expose_peer_addresses,
spawner: TaskSpawner {
full_verify_concurrency_limit,
network,
verifier,
state,
latest_chain_tip,
chain_network,
admission: Arc::default(),
spawned_sender,
},
pending: FuturesUnordered::new(),
spawned_receiver,
poisoned_retries: PoisonedRetryBudgets::default(),
}
}
pub fn retry_poisoned(&mut self, hash: block::Hash) -> Option<DownloadAction> {
let tip_hash = self.spawner.latest_chain_tip.best_tip_hash();
let Some(retry_attempt) = self.poisoned_retries.consume(hash, tip_hash) else {
warn!(
?hash,
retry_limit = POISONED_GOSSIP_BLOCK_RETRY_LIMIT,
"poisoned gossip block retry budget exhausted, leaving the block to the syncer"
);
metrics::counter!("gossip.poisoned.block.retry.limit.count").increment(1);
return None;
};
let action = self.download_and_verify(hash, None);
match action {
DownloadAction::AddedToQueue => {
info!(
?hash,
retry_attempt,
retry_limit = POISONED_GOSSIP_BLOCK_RETRY_LIMIT,
"gossiped block body was poisoned, re-requesting the block from another peer"
);
metrics::counter!("gossip.poisoned.block.requeued.count").increment(1);
}
DownloadAction::AlreadyQueued | DownloadAction::FullQueue => {
warn!(
?hash,
?action,
retry_attempt,
retry_limit = POISONED_GOSSIP_BLOCK_RETRY_LIMIT,
"poisoned gossip block re-request was refused, \
spending the retry budget without dispatching"
);
metrics::counter!("gossip.poisoned.block.requeue.refused.count").increment(1);
}
}
Some(action)
}
#[instrument(
skip(self, hash, download_source),
fields(hash = %hash, source = tracing::field::Empty)
)]
pub fn download_and_verify(
&mut self,
hash: block::Hash,
download_source: Option<zn::PeerSource>,
) -> DownloadAction {
let source_label = download_source
.as_ref()
.map(|source| peer_source_log_label(source, self.expose_peer_addresses))
.unwrap_or_else(|| "none".to_string());
tracing::Span::current().record("source", source_label.as_str());
let full_verify_concurrency_limit = self.spawner.full_verify_concurrency_limit;
let mut admission = self.spawner.admission();
let queue_len = admission.queue_len();
if admission.cancel_handles.contains_key(&hash) {
debug!(
?hash,
queue_len,
concurrency_limit = full_verify_concurrency_limit,
"block hash already queued for inbound download: ignored block",
);
metrics::gauge!("gossip.queued.block.count").set(queue_len as f64);
metrics::counter!("gossip.already.queued.dropped.block.hash.count").increment(1);
return DownloadAction::AlreadyQueued;
}
if queue_len >= full_verify_concurrency_limit {
debug!(
?hash,
queue_len,
concurrency_limit = full_verify_concurrency_limit,
"too many blocks queued for inbound download: ignored block",
);
metrics::gauge!("gossip.queued.block.count").set(queue_len as f64);
metrics::counter!("gossip.full.queue.dropped.block.hash.count").increment(1);
return DownloadAction::FullQueue;
}
let advertiser = download_source.clone().map(AdvertiserSource::from);
if let Some(source) = &advertiser {
let source_count = admission
.source_counts
.get(source)
.copied()
.unwrap_or_default();
let source_limit = source.max_in_flight(full_verify_concurrency_limit);
if source_count >= source_limit {
debug!(
?hash,
source_count,
source_limit,
queue_len,
concurrency_limit = full_verify_concurrency_limit,
"too many blocks queued for inbound download from one source: ignored block",
);
metrics::gauge!("gossip.queued.block.count").set(queue_len as f64);
metrics::counter!("gossip.source.queue.dropped.block.hash.count").increment(1);
return DownloadAction::FullQueue;
}
}
let download = DownloadTask {
hash,
advertiser,
download_source,
kind: DownloadKind::Advertised,
};
let task = self.spawner.spawn_download_task(&mut admission, download);
let queue_len = admission.queue_len();
drop(admission);
self.pending.push(task);
debug!(
?hash,
queue_len,
concurrency_limit = full_verify_concurrency_limit,
"queued hash for download",
);
metrics::gauge!("gossip.queued.block.count").set(queue_len as f64);
DownloadAction::AddedToQueue
}
}
impl<ZN, ZV, ZS> TaskSpawner<ZN, ZV, ZS>
where
ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
ZN::Future: Send,
ZV: Service<zakura_consensus::Request, Response = block::Hash, Error = BoxError>
+ Send
+ Clone
+ 'static,
ZV::Future: Send,
ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
ZS::Future: Send,
{
fn admission(&self) -> std::sync::MutexGuard<'_, Admission> {
self.admission
.lock()
.expect("admission is only mutated by non-panicking bookkeeping code")
}
fn recover_parent(
&self,
parent: block::Hash,
child_source: Option<zn::PeerSource>,
) -> ParentRecovery {
let mut admission = self.admission();
if admission.closed {
return ParentRecovery::Closed;
}
if admission.cancel_handles.contains_key(&parent) {
return ParentRecovery::Attached;
}
if admission.parent_recoveries.len() >= MAX_PARENT_RECOVERIES {
return ParentRecovery::TooManyRecoveries;
}
let charged_source = child_source.clone().map(AdvertiserSource::from);
if admission
.parent_recoveries
.values()
.any(|source| *source == charged_source)
{
return ParentRecovery::SourceRecoveryInFlight;
}
if admission.queue_len() >= self.full_verify_concurrency_limit {
return ParentRecovery::FullQueue;
}
let download = DownloadTask {
hash: parent,
download_source: child_source,
advertiser: None,
kind: DownloadKind::ParentRecovery,
};
let task = self.spawn_download_task(&mut admission, download);
admission.parent_recoveries.insert(parent, charged_source);
drop(admission);
let _ = self.spawned_sender.send(task);
ParentRecovery::Requested
}
fn spawn_download_task(
&self,
admission: &mut Admission,
download: DownloadTask,
) -> DownloadHandle {
let DownloadTask {
hash,
download_source,
advertiser,
kind,
} = download;
let source_lock = advertiser.as_ref().map(|source| {
*admission.source_counts.entry(source.clone()).or_default() += 1;
admission
.source_locks
.entry(source.clone())
.or_insert_with(|| Arc::new(Mutex::new(())))
.clone()
});
let (cancel_tx, mut cancel_rx) = oneshot::channel::<()>();
let spawner = self.clone();
let network = self.network.clone();
let chain_network = self.chain_network.clone();
let verifier = self.verifier.clone();
let state = self.state.clone();
let latest_chain_tip = self.latest_chain_tip.clone();
let full_verify_concurrency_limit = self.full_verify_concurrency_limit;
let fut: BoxFuture<'static, _> = async move {
match state
.clone()
.oneshot(zs::Request::AnyChainBlock(hash.into()))
.await
{
Ok(zs::Response::Block(None)) => Ok(()),
Ok(zs::Response::Block(Some(_))) => Err("already present".into()),
Ok(_) => unreachable!("wrong response"),
Err(e) => Err(e),
}
.map_err(|e| (e, None))?;
let request_hashes = std::iter::once(hash).collect();
let request = match download_source.clone() {
Some(source) => zn::Request::BlocksByHashFrom {
hashes: request_hashes,
source,
},
None => zn::Request::BlocksByHash(request_hashes),
};
let (block, advertiser_addr) = if let zn::Response::Blocks(blocks) =
network.oneshot(request).await.map_err(|e| (e, None))?
{
if blocks.len() != 1 {
return Err((
format!(
"wrong number of blocks in response to a single hash: got {}",
blocks.len()
)
.into(),
None,
));
}
let Some(available) = blocks.first().expect("just checked length").available()
else {
return Err((
"unexpected missing block status: single block failures should be errors"
.into(),
None,
));
};
available
} else {
unreachable!("wrong response to block request");
};
if block.hash() != hash {
return Err((
format!(
"peer returned block {} in response to a request for {hash}",
block.hash()
)
.into(),
advertiser_addr,
));
}
metrics::counter!("gossip.downloaded.block.count").increment(1);
let tip_height = latest_chain_tip.best_tip_height();
let best_tip = latest_chain_tip.best_tip_height_and_hash();
let max_lookahead_height =
max_lookahead_height(&chain_network, tip_height, full_verify_concurrency_limit);
let min_accepted_height = tip_height
.map(|tip_height| {
block::Height(tip_height.0.saturating_sub(zs::MAX_BLOCK_REORG_HEIGHT))
})
.unwrap_or(block::Height(0));
let block_height = block
.coinbase_height()
.ok_or_else(|| {
debug!(
?hash,
"gossiped block with no height: dropped downloaded block"
);
metrics::counter!("gossip.no.height.dropped.block.count").increment(1);
BoxError::from("gossiped block with no height")
})
.map_err(|e| (e, None))?;
if let Some(expected_height) =
tip_child_mismatch(block.header().previous_block_hash, block_height, best_tip)
{
debug!(
?hash,
?block_height,
?expected_height,
"gossiped tip child claimed the wrong coinbase height: \
dropped downloaded block"
);
metrics::counter!("gossip.tip.child.height.mismatch.count").increment(1);
Err((
BoxError::from(GossipedTipChildHeightMismatch {
height: block_height,
expected_height,
hash,
parent_recovery: kind == DownloadKind::ParentRecovery,
}),
advertiser_addr,
))?;
}
if block_height > max_lookahead_height {
debug!(
?hash,
?block_height,
?tip_height,
?max_lookahead_height,
lookahead_limit = full_verify_concurrency_limit,
"gossiped block height too far ahead of the tip: dropped downloaded block",
);
metrics::counter!("gossip.max.height.limit.dropped.block.count").increment(1);
Err("gossiped block height too far ahead").map_err(|e| (e.into(), None))?;
} else if block_height < min_accepted_height {
debug!(
?hash,
?block_height,
?tip_height,
?min_accepted_height,
behind_tip_limit = ?zs::MAX_BLOCK_REORG_HEIGHT,
"gossiped block height behind the finalized tip: dropped downloaded block",
);
metrics::counter!("gossip.min.height.limit.dropped.block.count").increment(1);
Err("gossiped block height behind the finalized tip")
.map_err(|e| (e.into(), None))?;
}
if !has_sufficient_work(
&chain_network,
tip_height,
block_height,
block.header().difficulty_threshold,
) {
metrics::counter!("gossip.insufficient_work.dropped.block.count").increment(1);
return Err((
"gossiped block target is easier than the Mainnet admission floor".into(),
advertiser_addr,
));
}
if kind == DownloadKind::Advertised {
let parent = block.header().previous_block_hash;
let one_hop_from_tip = tip_height
.and_then(|tip| tip + MAX_PARENT_RECOVERY_CHILD_HEIGHT_ABOVE_TIP)
.is_some_and(|max_height| block_height <= max_height);
let parent_is_missing = one_hop_from_tip
&& matches!(
state.oneshot(zs::Request::KnownBlock(parent)).await,
Ok(zs::Response::KnownBlock(None))
);
if parent_is_missing {
let header = *block.header();
let proof_of_work_is_valid = tokio::task::spawn_blocking(move || {
zakura_consensus::proof_of_work_is_valid(
&header,
&chain_network,
&block_height,
&hash,
)
})
.await
.unwrap_or(false);
let recovery = if proof_of_work_is_valid {
spawner.recover_parent(parent, download_source)
} else {
ParentRecovery::InvalidProofOfWork
};
debug!(
?hash,
?parent,
?block_height,
?recovery,
"gossiped block's parent is missing",
);
metrics::counter!(
"gossip.missing.parent.count",
"outcome" => recovery.metric_label(),
)
.increment(1);
}
}
let _source_guard = match source_lock {
Some(source_lock) => Some(source_lock.lock_owned().await),
None => None,
};
verifier
.oneshot(zakura_consensus::Request::Commit(block))
.await
.map(|hash| (hash, block_height))
.map_err(|e| (e, advertiser_addr))
}
.map_ok(move |(hash, height)| {
match kind {
DownloadKind::Advertised => {
info!(?height, "downloaded and verified gossiped block")
}
DownloadKind::ParentRecovery => {
info!(
?height,
"downloaded and verified missing parent of gossiped block"
)
}
}
metrics::counter!("gossip.verified.block.count").increment(1);
hash
})
.map_err(move |(e, advertiser_addr)| (e, hash, advertiser_addr))
.in_current_span()
.boxed();
let task_admission = self.admission.clone();
let task = tokio::spawn(async move {
let result = tokio::select! {
biased;
_ = &mut cancel_rx => {
trace!("task cancelled prior to completion");
metrics::counter!("gossip.cancelled.count").increment(1);
Err(("canceled".into(), hash, None))
}
verification = fut => verification,
};
task_admission
.lock()
.expect("admission is only mutated by non-panicking bookkeeping code")
.release(&hash);
result
});
assert!(
admission
.cancel_handles
.insert(hash, (cancel_tx, advertiser))
.is_none(),
"blocks are only queued once"
);
task
}
}
#[cfg(test)]
mod missing_parent_tests;
#[cfg(test)]
mod tests {
use super::*;
use futures::StreamExt as _;
use indexmap::IndexSet;
use std::{collections::HashSet, future, time::Duration};
use tower::{service_fn, util::BoxCloneService};
use zakura_chain::{block::Block, parameters::Network, serialization::ZcashDeserializeInto};
use zakura_network::InventoryResponse::Available;
use crate::components::auth_download_height::poison_coinbase_height;
type PendingNetwork = BoxCloneService<zn::Request, zn::Response, BoxError>;
type PendingVerifier = BoxCloneService<zakura_consensus::Request, block::Hash, BoxError>;
type PendingState = BoxCloneService<zs::Request, zs::Response, BoxError>;
fn hash(byte: u8) -> block::Hash {
block::Hash([byte; 32])
}
#[test]
fn peer_source_log_labels_require_explicit_opt_in() {
let source = zn::PeerSource::LegacySocket(
"192.0.2.1:8233".parse().expect("valid test peer address"),
);
assert_eq!(peer_source_log_label(&source, false), "legacy:redacted");
assert_eq!(
peer_source_log_label(&source, true),
"legacy:192.0.2.1:8233"
);
}
fn pending_downloads() -> Downloads<PendingNetwork, PendingVerifier, PendingState> {
let (_tip_sender, latest_chain_tip, _tip_change) =
zs::ChainTipSender::new(None, &Network::Mainnet);
pending_downloads_with_tip(latest_chain_tip)
}
fn pending_downloads_with_tip(
latest_chain_tip: zs::LatestChainTip,
) -> Downloads<PendingNetwork, PendingVerifier, PendingState> {
Downloads::new(
MAX_INBOUND_CONCURRENCY,
false,
BoxCloneService::new(service_fn(|_request| {
future::pending::<Result<zn::Response, BoxError>>()
})),
BoxCloneService::new(service_fn(|_request| {
future::pending::<Result<block::Hash, BoxError>>()
})),
BoxCloneService::new(service_fn(|_request| {
future::pending::<Result<zs::Response, BoxError>>()
})),
latest_chain_tip,
Network::Mainnet,
)
}
#[tokio::test]
async fn source_admission_limits_preserve_legacy_bound_and_allow_zakura_burst() {
let mut downloads = pending_downloads();
let legacy_a = zn::PeerSource::LegacySocket(([127, 0, 0, 1], 8233).into());
let legacy_same_ip = zn::PeerSource::LegacySocket(([127, 0, 0, 1], 18233).into());
let zakura_a = zn::PeerSource::Zakura(
zn::zakura::ZakuraPeerId::new(vec![7; 32]).expect("test peer id is within bounds"),
);
let zakura_b = zn::PeerSource::Zakura(
zn::zakura::ZakuraPeerId::new(vec![8; 32]).expect("test peer id is within bounds"),
);
assert_eq!(
downloads.download_and_verify(hash(1), Some(legacy_a)),
DownloadAction::AddedToQueue
);
assert_eq!(downloads.pending.len(), 1);
assert_eq!(
downloads.download_and_verify(hash(2), Some(legacy_same_ip)),
DownloadAction::FullQueue
);
assert_eq!(downloads.pending.len(), 1);
assert_eq!(
downloads.download_and_verify(hash(3), Some(zakura_a.clone())),
DownloadAction::AddedToQueue
);
assert_eq!(downloads.pending.len(), 2);
assert_eq!(
downloads.download_and_verify(hash(4), Some(zakura_a.clone())),
DownloadAction::AddedToQueue
);
assert_eq!(downloads.pending.len(), 3);
assert_eq!(
downloads.download_and_verify(hash(5), Some(zakura_a)),
DownloadAction::AddedToQueue
);
assert_eq!(downloads.pending.len(), 4);
assert_eq!(
downloads.download_and_verify(hash(6), Some(zakura_b)),
DownloadAction::AddedToQueue
);
assert_eq!(downloads.pending.len(), 5);
assert_eq!(downloads.spawner.admission().queue_len(), 5);
assert_eq!(
downloads.download_and_verify(hash(6), None),
DownloadAction::AlreadyQueued
);
}
#[tokio::test]
async fn single_zakura_source_cannot_monopolize_inbound_block_queue() {
let mut downloads = pending_downloads();
let zakura_a = zn::PeerSource::Zakura(
zn::zakura::ZakuraPeerId::new(vec![7; 32]).expect("test peer id is within bounds"),
);
let zakura_b = zn::PeerSource::Zakura(
zn::zakura::ZakuraPeerId::new(vec![8; 32]).expect("test peer id is within bounds"),
);
for index in 0..MAX_INBOUND_BLOCK_CONCURRENCY_PER_PEER {
let byte = u8::try_from(index).expect("per-peer cap fits in u8");
assert_eq!(
downloads.download_and_verify(hash(byte), Some(zakura_a.clone())),
DownloadAction::AddedToQueue
);
}
assert_eq!(
downloads.pending.len(),
MAX_INBOUND_BLOCK_CONCURRENCY_PER_PEER
);
let over_cap = hash(
u8::try_from(MAX_INBOUND_BLOCK_CONCURRENCY_PER_PEER).expect("per-peer cap fits in u8"),
);
assert_eq!(
downloads.download_and_verify(over_cap, Some(zakura_a)),
DownloadAction::FullQueue
);
assert_eq!(
downloads.pending.len(),
MAX_INBOUND_BLOCK_CONCURRENCY_PER_PEER
);
assert_eq!(
downloads.download_and_verify(hash(200), Some(zakura_b)),
DownloadAction::AddedToQueue
);
assert_eq!(
downloads.pending.len(),
MAX_INBOUND_BLOCK_CONCURRENCY_PER_PEER + 1
);
}
#[tokio::test]
async fn global_inbound_download_queue_bounds_admission() {
let (_tip_sender, latest_chain_tip, _tip_change) =
zs::ChainTipSender::new(None, &Network::Mainnet);
let mut downloads = Downloads::new(
MIN_CONCURRENCY_LIMIT,
false,
BoxCloneService::new(service_fn(|_request| {
future::pending::<Result<zn::Response, BoxError>>()
})),
BoxCloneService::new(service_fn(|_request| {
future::pending::<Result<block::Hash, BoxError>>()
})),
BoxCloneService::new(service_fn(|_request| {
future::pending::<Result<zs::Response, BoxError>>()
})),
latest_chain_tip,
Network::Mainnet,
);
for index in 0..MIN_CONCURRENCY_LIMIT {
let byte = u8::try_from(index).expect("minimum concurrency limit fits in u8");
assert_eq!(
downloads.download_and_verify(hash(byte), None),
DownloadAction::AddedToQueue
);
}
let overflow_hash = hash(
u8::try_from(MIN_CONCURRENCY_LIMIT).expect("minimum concurrency limit fits in u8"),
);
assert_eq!(
downloads.download_and_verify(overflow_hash, None),
DownloadAction::FullQueue
);
}
#[tokio::test]
async fn same_source_downloads_fetch_promptly_and_verify_in_order() -> Result<(), BoxError> {
let block_one: Arc<block::Block> =
zakura_test::vectors::BLOCK_MAINNET_1_BYTES.zcash_deserialize_into()?;
let block_two: Arc<block::Block> =
zakura_test::vectors::BLOCK_MAINNET_2_BYTES.zcash_deserialize_into()?;
let hash_one = block_one.hash();
let hash_two = block_two.hash();
let peer_id =
zn::zakura::ZakuraPeerId::new(vec![7; 32]).expect("test peer id is within bounds");
let source = zn::PeerSource::Zakura(peer_id.clone());
let blocks = Arc::new(HashMap::from([
(hash_one, block_one.clone()),
(hash_two, block_two.clone()),
]));
let (network_tx, mut network_rx) = tokio::sync::mpsc::unbounded_channel::<zn::Request>();
let network = BoxCloneService::new(service_fn(move |request: zn::Request| {
let blocks = blocks.clone();
let network_tx = network_tx.clone();
async move {
network_tx.send(request.clone())?;
let zn::Request::BlocksByHashFrom { hashes, .. } = request else {
return Err("unexpected network request".into());
};
let hash = hashes
.iter()
.next()
.copied()
.expect("download requests contain one hash");
let block = blocks
.get(&hash)
.cloned()
.expect("test network has a block for the requested hash");
Ok(zn::Response::Blocks(vec![Available((block.into(), None))]))
}
}));
let (commit_tx, mut commit_rx) = tokio::sync::mpsc::unbounded_channel();
let (release_tx, release_rx) = oneshot::channel();
let release_rx = Arc::new(Mutex::new(Some(release_rx)));
let verifier =
BoxCloneService::new(service_fn(move |request: zakura_consensus::Request| {
let commit_tx = commit_tx.clone();
let release_rx = release_rx.clone();
async move {
let zakura_consensus::Request::Commit(block) = request else {
return Err("unexpected verifier request".into());
};
let hash = block.hash();
commit_tx.send(hash)?;
if hash == hash_one {
let release_rx = release_rx
.lock()
.await
.take()
.expect("first block verifier is released once");
release_rx.await?;
}
Ok(hash)
}
}));
let (_tip_sender, latest_chain_tip, _tip_change) =
zs::ChainTipSender::new(None, &Network::Mainnet);
let mut downloads = Downloads::new(
MAX_INBOUND_CONCURRENCY,
false,
network,
verifier,
BoxCloneService::new(service_fn(|request| async move {
match request {
zs::Request::AnyChainBlock(_) => Ok(zs::Response::Block(None)),
request => Err(format!("unexpected state request: {request:?}").into()),
}
})),
latest_chain_tip,
Network::Mainnet,
);
assert_eq!(
downloads.download_and_verify(hash_one, Some(source.clone())),
DownloadAction::AddedToQueue
);
assert_eq!(
downloads.download_and_verify(hash_two, Some(source)),
DownloadAction::AddedToQueue
);
let first_request = tokio::time::timeout(Duration::from_secs(1), network_rx.recv())
.await
.expect("first same-source download starts")
.expect("network request channel is open");
assert_eq!(
first_request,
zn::Request::BlocksByHashFrom {
hashes: IndexSet::from([hash_one]),
source: zn::PeerSource::Zakura(peer_id.clone()),
}
);
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("first block reaches verifier"),
Some(hash_one)
);
let second_request = tokio::time::timeout(Duration::from_secs(1), network_rx.recv())
.await
.expect("second same-source download starts without waiting for verifier cleanup")
.expect("network request channel is open");
assert!(
tokio::time::timeout(Duration::from_millis(100), commit_rx.recv())
.await
.is_err(),
"second same-source verifier commit must wait until the first verifier finishes",
);
release_tx
.send(())
.expect("first verifier task is waiting for release");
assert_eq!(
second_request,
zn::Request::BlocksByHashFrom {
hashes: IndexSet::from([hash_two]),
source: zn::PeerSource::Zakura(peer_id),
}
);
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("second block reaches verifier after first release"),
Some(hash_two)
);
let first_result = tokio::time::timeout(Duration::from_secs(1), downloads.next())
.await
.expect("first completed download is yielded")
.expect("downloads stream is open")
.expect("first download succeeds");
let second_result = tokio::time::timeout(Duration::from_secs(1), downloads.next())
.await
.expect("second completed download is yielded")
.expect("downloads stream is open")
.expect("second download succeeds");
assert_eq!(
HashSet::from([first_result, second_result]),
HashSet::from([hash_one, hash_two])
);
assert_eq!(downloads.spawner.admission().queue_len(), 0);
let admission = downloads.spawner.admission();
assert!(admission.source_locks.is_empty());
assert!(admission.source_counts.is_empty());
Ok(())
}
#[tokio::test]
async fn zakura_advertiser_source_is_sent_to_network_download_request() {
let hash = hash(9);
let peer_id =
zn::zakura::ZakuraPeerId::new(vec![9; 32]).expect("test peer id is within bounds");
let (network_tx, mut network_rx) = tokio::sync::mpsc::unbounded_channel();
let (_tip_sender, latest_chain_tip, _tip_change) =
zs::ChainTipSender::new(None, &Network::Mainnet);
let mut downloads = Downloads::new(
MAX_INBOUND_CONCURRENCY,
false,
BoxCloneService::new(service_fn(move |request| {
let network_tx = network_tx.clone();
async move {
network_tx.send(request)?;
future::pending::<Result<zn::Response, BoxError>>().await
}
})),
BoxCloneService::new(service_fn(|_request| {
future::pending::<Result<block::Hash, BoxError>>()
})),
BoxCloneService::new(service_fn(|request| async move {
match request {
zs::Request::AnyChainBlock(_) => Ok(zs::Response::Block(None)),
request => Err(format!("unexpected state request: {request:?}").into()),
}
})),
latest_chain_tip,
Network::Mainnet,
);
assert_eq!(
downloads.download_and_verify(hash, Some(zn::PeerSource::Zakura(peer_id.clone())),),
DownloadAction::AddedToQueue
);
let poll_task = tokio::spawn(async move {
let _ = downloads.next().await;
});
let request = tokio::time::timeout(Duration::from_secs(1), network_rx.recv())
.await
.expect("network request is sent")
.expect("network request channel is open");
assert_eq!(
request,
zn::Request::BlocksByHashFrom {
hashes: IndexSet::from([hash]),
source: zn::PeerSource::Zakura(peer_id),
}
);
poll_task.abort();
}
#[tokio::test]
async fn substituted_block_response_is_rejected_and_cleans_up_requested_hash(
) -> Result<(), BoxError> {
let requested_block: Arc<block::Block> =
zakura_test::vectors::BLOCK_MAINNET_1_BYTES.zcash_deserialize_into()?;
let substitute_block: Arc<block::Block> =
zakura_test::vectors::BLOCK_MAINNET_2_BYTES.zcash_deserialize_into()?;
let requested_hash = requested_block.hash();
assert_ne!(requested_hash, substitute_block.hash());
let peer_id =
zn::zakura::ZakuraPeerId::new(vec![7; 32]).expect("test peer id is within bounds");
let source = zn::PeerSource::Zakura(peer_id);
let substitute = substitute_block.clone();
let network = BoxCloneService::new(service_fn(move |request: zn::Request| {
let substitute = substitute.clone();
async move {
let (zn::Request::BlocksByHashFrom { .. } | zn::Request::BlocksByHash(_)) = request
else {
return Err("unexpected network request".into());
};
Ok(zn::Response::Blocks(vec![Available((
substitute.into(),
None,
))]))
}
}));
let verifier = BoxCloneService::new(service_fn(
|request: zakura_consensus::Request| async move {
let zakura_consensus::Request::Commit(block) = request else {
return Err("unexpected verifier request".into());
};
Ok(block.hash())
},
));
let (_tip_sender, latest_chain_tip, _tip_change) =
zs::ChainTipSender::new(None, &Network::Mainnet);
let mut downloads = Downloads::new(
MAX_INBOUND_CONCURRENCY,
false,
network,
verifier,
BoxCloneService::new(service_fn(|request| async move {
match request {
zs::Request::AnyChainBlock(_) => Ok(zs::Response::Block(None)),
request => Err(format!("unexpected state request: {request:?}").into()),
}
})),
latest_chain_tip,
Network::Mainnet,
);
assert_eq!(
downloads.download_and_verify(requested_hash, Some(source)),
DownloadAction::AddedToQueue
);
let result = tokio::time::timeout(Duration::from_secs(1), downloads.next())
.await
.expect("download completes")
.expect("downloads stream is open");
assert!(
result.is_err(),
"a substituted block must be rejected, not accepted as {result:?}",
);
let admission = downloads.spawner.admission();
assert!(
admission.cancel_handles.is_empty(),
"queue entry under the requested hash must be removed",
);
assert!(
admission.source_counts.is_empty(),
"source accounting must be decremented for the requested hash",
);
assert!(admission.source_locks.is_empty());
assert_eq!(admission.queue_len(), 0);
Ok(())
}
fn chain_tip_at(
height: block::Height,
hash: block::Hash,
) -> (zs::ChainTipSender, zs::LatestChainTip) {
let any_block: Arc<Block> = zakura_test::vectors::BLOCK_MAINNET_1_BYTES
.zcash_deserialize_into()
.expect("test vector deserializes");
let mut tip_block: zs::ChainTipBlock =
zs::SemanticallyVerifiedBlock::from(any_block).into();
tip_block.hash = hash;
tip_block.height = height;
let (mut tip_sender, latest_chain_tip, _tip_change) =
zs::ChainTipSender::new(None, &Network::Mainnet);
tip_sender.set_finalized_tip(tip_block);
(tip_sender, latest_chain_tip)
}
fn set_tip(tip_sender: &mut zs::ChainTipSender, height: block::Height, hash: block::Hash) {
let any_block: Arc<Block> = zakura_test::vectors::BLOCK_MAINNET_1_BYTES
.zcash_deserialize_into()
.expect("test vector deserializes");
let mut tip_block: zs::ChainTipBlock =
zs::SemanticallyVerifiedBlock::from(any_block).into();
tip_block.hash = hash;
tip_block.height = height;
tip_sender.set_finalized_tip(tip_block);
}
fn downloads_returning(
block: Arc<Block>,
advertiser_addr: PeerSocketAddr,
latest_chain_tip: zs::LatestChainTip,
) -> Downloads<PendingNetwork, PendingVerifier, PendingState> {
Downloads::new(
MAX_INBOUND_CONCURRENCY,
false,
BoxCloneService::new(service_fn(move |_request| {
let block = block.clone();
async move {
Ok(zn::Response::Blocks(vec![Available((
block.into(),
Some(advertiser_addr),
))]))
}
})),
BoxCloneService::new(service_fn(|_request| {
future::pending::<Result<block::Hash, BoxError>>()
})),
BoxCloneService::new(service_fn(|request| async move {
match request {
zs::Request::AnyChainBlock(_) => Ok(zs::Response::Block(None)),
request => Err(format!("unexpected state request: {request:?}").into()),
}
})),
latest_chain_tip,
Network::Mainnet,
)
}
#[tokio::test]
async fn gossiped_tip_child_rejects_poisoned_coinbase_height() {
let canonical: Arc<Block> = zakura_test::vectors::BLOCK_MAINNET_1687107_BYTES
.zcash_deserialize_into()
.expect("test vector deserializes");
let canonical_hash = canonical.hash();
let parent_hash = canonical.header.previous_block_hash;
let poisoned = poison_coinbase_height(&canonical, block::Height(1));
assert_eq!(
poisoned.hash(),
canonical_hash,
"the poisoned body still answers for the advertised hash"
);
let advertiser_addr: PeerSocketAddr = "127.0.0.1:8233".parse().expect("valid peer address");
let (_tip_sender, latest_chain_tip) = chain_tip_at(block::Height(1_687_106), parent_hash);
let mut downloads = downloads_returning(poisoned, advertiser_addr, latest_chain_tip);
assert_eq!(
downloads.download_and_verify(
canonical_hash,
Some(zn::PeerSource::LegacySocket(advertiser_addr)),
),
DownloadAction::AddedToQueue
);
let (error, addr) = downloads
.next()
.await
.expect("the download task produces a result")
.expect_err("a poisoned tip child must be rejected");
assert_eq!(
addr,
Some(advertiser_addr),
"the advertiser must be preserved so Inbound can score the peer"
);
let mismatch = error
.downcast_ref::<GossipedTipChildHeightMismatch>()
.expect("the error must be the typed mismatch that Inbound scores");
assert_eq!(mismatch.height, block::Height(1));
assert_eq!(mismatch.expected_height, block::Height(1_687_107));
assert_eq!(mismatch.hash, canonical_hash);
}
#[tokio::test]
async fn mainnet_work_floor_rejects_gossip_before_parent_recovery() {
use zakura_chain::work::difficulty::ParameterDifficulty as _;
let floor = crate::components::download_work::MAINNET_WORK_FLOOR_HEIGHT;
for tip_height in [floor.previous().unwrap(), floor] {
for height in [floor.previous().unwrap(), floor, floor.next().unwrap()] {
if tip_height < floor && height <= floor {
continue;
}
let canonical: Arc<Block> = zakura_test::vectors::BLOCK_MAINNET_1687107_BYTES
.zcash_deserialize_into()
.expect("test vector deserializes");
let mut candidate = poison_coinbase_height(&canonical, height);
let block = Arc::make_mut(&mut candidate);
*Arc::make_mut(&mut block.transactions[0]).expiry_height_mut() = height;
let header = Arc::make_mut(&mut block.header);
header.previous_block_hash = hash(0x55);
header.difficulty_threshold =
Network::Mainnet.target_difficulty_limit().to_compact();
header.merkle_root = block.transactions.iter().map(|tx| tx.hash()).collect();
let candidate_hash = candidate.hash();
let advertiser_addr = "127.0.0.1:8233".parse().unwrap();
let (_tip_sender, latest_chain_tip) = chain_tip_at(tip_height, hash(0xAA));
let mut downloads =
downloads_returning(candidate, advertiser_addr, latest_chain_tip);
downloads.spawner.verifier = BoxCloneService::new(service_fn(|_| async {
Err::<block::Hash, BoxError>(
"weak-work block reached semantic verification".into(),
)
}));
assert_eq!(
downloads.download_and_verify(
candidate_hash,
Some(zn::PeerSource::LegacySocket(advertiser_addr)),
),
DownloadAction::AddedToQueue
);
let (error, addr) = tokio::time::timeout(Duration::from_secs(5), downloads.next())
.await
.expect("the work floor rejects without waiting on the parent")
.unwrap()
.unwrap_err();
assert!(
error.to_string().contains("Mainnet admission floor"),
"{error}"
);
assert_eq!(addr, Some(advertiser_addr));
assert!(crate::components::inbound::block_misbehavior(error, addr).is_none());
}
}
}
#[tokio::test]
async fn gossiped_non_tip_child_keeps_the_behind_tip_policy() {
let block1: Arc<Block> = zakura_test::vectors::BLOCK_MAINNET_1_BYTES
.zcash_deserialize_into()
.expect("test vector deserializes");
let block1_hash = block1.hash();
let advertiser_addr: PeerSocketAddr = "127.0.0.1:8233".parse().expect("valid peer address");
let (_tip_sender, latest_chain_tip) = chain_tip_at(block::Height(1_000_000), hash(0x11));
let mut downloads = downloads_returning(block1, advertiser_addr, latest_chain_tip);
assert_eq!(
downloads.download_and_verify(
block1_hash,
Some(zn::PeerSource::LegacySocket(advertiser_addr)),
),
DownloadAction::AddedToQueue
);
let (error, addr) = downloads
.next()
.await
.expect("the download task produces a result")
.expect_err("a block behind the finalized tip is still dropped");
assert!(
error
.downcast_ref::<GossipedTipChildHeightMismatch>()
.is_none(),
"a non-tip-child must not be reported as a height mismatch"
);
assert_eq!(addr, None, "the behind-tip policy is unchanged");
}
#[tokio::test]
async fn poisoned_retries_are_bounded_per_hash() {
let mut downloads = pending_downloads();
let poisoned_hash = hash(1);
assert_eq!(
downloads.retry_poisoned(poisoned_hash),
Some(DownloadAction::AddedToQueue),
"the first retry dispatches a replacement download"
);
for attempt in 2..=POISONED_GOSSIP_BLOCK_RETRY_LIMIT {
assert_eq!(
downloads.retry_poisoned(poisoned_hash),
Some(DownloadAction::AlreadyQueued),
"retry {attempt} is within the budget, but is refused while one is in flight"
);
}
assert!(
downloads.retry_poisoned(poisoned_hash).is_none(),
"the block is left to the syncer once the budget is exhausted"
);
assert!(
downloads.retry_poisoned(hash(2)).is_some(),
"a different hash keeps its own budget"
);
for _ in 0..3 {
assert!(
downloads.retry_poisoned(poisoned_hash).is_none(),
"an exhausted budget must not re-arm for the same hash"
);
}
}
#[tokio::test]
async fn poisoned_retry_budgets_are_scoped_to_the_tip() {
let (mut tip_sender, latest_chain_tip) = chain_tip_at(block::Height(100), hash(0xAA));
let mut downloads = pending_downloads_with_tip(latest_chain_tip);
let poisoned_hash = hash(1);
for _ in 0..POISONED_GOSSIP_BLOCK_RETRY_LIMIT {
assert!(downloads.retry_poisoned(poisoned_hash).is_some());
}
assert!(
downloads.retry_poisoned(poisoned_hash).is_none(),
"the budget is exhausted while the tip is unchanged"
);
assert_eq!(downloads.poisoned_retries.counts.len(), 1);
set_tip(&mut tip_sender, block::Height(101), hash(0xBB));
assert!(
downloads.retry_poisoned(hash(2)).is_some(),
"a child of the new tip gets a fresh budget"
);
assert!(
!downloads
.poisoned_retries
.counts
.contains_key(&poisoned_hash),
"the stale entry must be dropped when the tip moves, not held for the process life"
);
assert_eq!(downloads.poisoned_retries.counts.len(), 1);
}
#[test]
fn a_downloaded_block_releases_its_budget() {
let mut budgets = PoisonedRetryBudgets::default();
let tip = Some(hash(0xAA));
let poisoned_hash = hash(1);
for _ in 0..POISONED_GOSSIP_BLOCK_RETRY_LIMIT {
assert!(budgets.consume(poisoned_hash, tip).is_some());
}
assert_eq!(budgets.consume(poisoned_hash, tip), None);
budgets.release(&poisoned_hash);
assert_eq!(
budgets.consume(poisoned_hash, tip),
Some(1),
"the budget restarts once the block has actually been downloaded"
);
assert_eq!(
budgets.consume(hash(2), tip),
Some(1),
"releasing one hash leaves the others alone"
);
}
}