use std::{
collections::HashSet,
future::Future,
net::IpAddr,
pin::Pin,
sync::{Arc, Mutex},
task::{Context, Poll},
time::{Duration, Instant},
};
use futures::{
future::{FutureExt, TryFutureExt},
stream::Stream,
};
use tokio::sync::oneshot::{self, error::TryRecvError};
use tower::{buffer::Buffer, timeout::Timeout, util::BoxService, Service, ServiceExt};
use zakura_network::{self as zn, PeerSocketAddr};
use zakura_state::{self as zs};
use zakura_chain::{
block::{self, Block},
serialization::ZcashSerialize,
transaction::UnminedTxId,
};
use zakura_consensus::{router::RouterError, VerifyBlockError};
use zakura_network::{AddressBook, InventoryResponse};
use zakura_node_services::mempool;
use crate::BoxError;
use super::sync::{BLOCK_DOWNLOAD_TIMEOUT, BLOCK_VERIFY_TIMEOUT};
use InventoryResponse::*;
mod cached_peer_addr_response;
pub(crate) mod downloads;
use cached_peer_addr_response::CachedPeerAddrResponse;
#[cfg(test)]
mod tests;
use downloads::Downloads as BlockDownloads;
pub const MAX_INBOUND_RESPONSE_TIME: Duration = Duration::from_secs(5);
pub const GETDATA_SENT_BYTES_LIMIT: usize = 1_000_000;
pub const GETDATA_MAX_BLOCK_COUNT: usize = 16;
const ZCASHD_COMPAT_PRUNED_BLOCK_LOG_INTERVAL: Duration = Duration::from_secs(60);
pub(crate) const ZCASHD_COMPAT_PRUNED_BLOCK_ERROR: &str =
"zcashd_compat: cannot serve block because its body has been pruned; a zcashd sidecar \
requiring this block cannot continue syncing. Restore zcashd from a newer snapshot, or use \
archive storage on Zakura side https://zakura.com/snapshots/ .";
type BlockDownloadPeerSet =
Buffer<BoxService<zn::Request, zn::Response, zn::BoxError>, zn::Request>;
type State = Buffer<BoxService<zs::Request, zs::Response, zs::BoxError>, zs::Request>;
type Mempool = Buffer<BoxService<mempool::Request, mempool::Response, BoxError>, mempool::Request>;
type SemanticBlockVerifier = Buffer<
BoxService<zakura_consensus::Request, block::Hash, RouterError>,
zakura_consensus::Request,
>;
type GossipedBlockDownloads =
BlockDownloads<Timeout<BlockDownloadPeerSet>, Timeout<SemanticBlockVerifier>, State>;
#[derive(Debug)]
struct PrunedBlockNotFoundLogger {
tx_retention: Option<u32>,
peer_ips: HashSet<IpAddr>,
last_log: Mutex<Option<Instant>>,
}
impl PrunedBlockNotFoundLogger {
fn new(tx_retention: Option<u32>, peer_ips: Vec<IpAddr>) -> Self {
Self {
tx_retention,
peer_ips: peer_ips.into_iter().map(canonical_ip).collect(),
last_log: Mutex::new(None),
}
}
fn is_enabled_for(&self, source: Option<&zn::PeerSource>) -> bool {
let Some(zn::PeerSource::LegacySocket(addr)) = source else {
return false;
};
let source_ip = canonical_ip(addr.remove_socket_addr_privacy().ip());
self.tx_retention.is_some() && self.peer_ips.contains(&source_ip)
}
fn reserve_log(&self) -> Option<u32> {
self.reserve_log_at(Instant::now())
}
fn reserve_log_at(&self, now: Instant) -> Option<u32> {
let tx_retention = self.tx_retention?;
let mut last_log = self
.last_log
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if last_log.is_some_and(|last| {
now.saturating_duration_since(last) < ZCASHD_COMPAT_PRUNED_BLOCK_LOG_INTERVAL
}) {
return None;
}
*last_log = Some(now);
Some(tx_retention)
}
}
fn canonical_ip(ip: IpAddr) -> IpAddr {
match ip {
IpAddr::V6(ip) => ip
.to_ipv4_mapped()
.map(IpAddr::V4)
.unwrap_or(IpAddr::V6(ip)),
ip => ip,
}
}
async fn retained_block_height(mut state: State, hash: block::Hash) -> Option<block::Height> {
let response = state
.ready()
.await
.ok()?
.call(zs::Request::BlockHeader(hash.into()))
.await
.ok()?;
match response {
zs::Response::BlockHeader { height, .. } => Some(height),
_ => None,
}
}
fn mempool_queue_source(source: zn::PeerSource) -> mempool::QueueSource {
match source {
zn::PeerSource::LegacySocket(addr) => mempool::QueueSource::LegacySocket(*addr),
zn::PeerSource::Zakura(peer_id) => {
mempool::QueueSource::Zakura(peer_id.as_bytes().to_vec())
}
}
}
pub struct InboundSetupData {
pub address_book: Arc<std::sync::Mutex<AddressBook>>,
pub block_download_peer_set: BlockDownloadPeerSet,
pub block_verifier: SemanticBlockVerifier,
pub mempool: Mempool,
pub state: State,
pub latest_chain_tip: zs::LatestChainTip,
pub misbehavior_sender: tokio::sync::mpsc::Sender<(PeerSocketAddr, u32)>,
}
#[allow(clippy::large_enum_variant)]
pub enum Setup {
Pending {
full_verify_concurrency_limit: usize,
setup: oneshot::Receiver<InboundSetupData>,
},
Initialized {
cached_peer_addr_response: CachedPeerAddrResponse,
block_downloads: Pin<Box<GossipedBlockDownloads>>,
mempool: Mempool,
state: State,
misbehavior_sender: tokio::sync::mpsc::Sender<(PeerSocketAddr, u32)>,
},
FailedInit,
FailedRecv {
error: SharedRecvError,
},
}
#[derive(thiserror::Error, Debug, Clone)]
#[error(transparent)]
pub struct SharedRecvError(Arc<TryRecvError>);
impl From<TryRecvError> for SharedRecvError {
fn from(source: TryRecvError) -> Self {
Self(Arc::new(source))
}
}
pub struct Inbound {
setup: Setup,
expose_peer_addresses: bool,
pruned_block_not_found_logger: Arc<PrunedBlockNotFoundLogger>,
}
impl Inbound {
pub fn new(
full_verify_concurrency_limit: usize,
expose_peer_addresses: bool,
zcashd_compat_pruning_retention: Option<u32>,
zcashd_compat_peer_ips: Vec<IpAddr>,
setup: oneshot::Receiver<InboundSetupData>,
) -> Inbound {
Inbound {
setup: Setup::Pending {
full_verify_concurrency_limit,
setup,
},
expose_peer_addresses,
pruned_block_not_found_logger: Arc::new(PrunedBlockNotFoundLogger::new(
zcashd_compat_pruning_retention,
zcashd_compat_peer_ips,
)),
}
}
fn take_setup(&mut self) -> Setup {
let mut setup = Setup::FailedInit;
std::mem::swap(&mut self.setup, &mut setup);
setup
}
}
impl Service<zn::Request> for Inbound {
type Response = zn::Response;
type Error = zn::BoxError;
type Future =
Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send + 'static>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
let result;
self.setup = match self.take_setup() {
Setup::Pending {
full_verify_concurrency_limit,
mut setup,
} => match setup.try_recv() {
Ok(setup_data) => {
let InboundSetupData {
address_book,
block_download_peer_set,
block_verifier,
mempool,
state,
latest_chain_tip,
misbehavior_sender,
} = setup_data;
let cached_peer_addr_response = CachedPeerAddrResponse::new(address_book);
let block_downloads = Box::pin(BlockDownloads::new(
full_verify_concurrency_limit,
self.expose_peer_addresses,
Timeout::new(block_download_peer_set, BLOCK_DOWNLOAD_TIMEOUT),
Timeout::new(block_verifier, BLOCK_VERIFY_TIMEOUT),
state.clone(),
latest_chain_tip,
));
result = Ok(());
Setup::Initialized {
cached_peer_addr_response,
block_downloads,
mempool,
state,
misbehavior_sender,
}
}
Err(TryRecvError::Empty) => {
result = Ok(());
Setup::Pending {
full_verify_concurrency_limit,
setup,
}
}
Err(error @ TryRecvError::Closed) => {
error!(?error, "inbound setup failed");
let error: SharedRecvError = error.into();
result = Err(error.clone().into());
Setup::FailedRecv { error }
}
},
Setup::FailedInit => unreachable!("incomplete previous Inbound initialization"),
Setup::FailedRecv { error } => {
result = Err(error.clone().into());
Setup::FailedRecv { error }
}
Setup::Initialized {
cached_peer_addr_response,
mut block_downloads,
mempool,
state,
misbehavior_sender,
} => {
while let Poll::Ready(Some(result)) = block_downloads.as_mut().poll_next(cx) {
let Err((err, Some(advertiser_addr))) = result else {
continue;
};
let Ok(err) = err.downcast::<VerifyBlockError>() else {
continue;
};
if err.misbehavior_score() != 0 {
let _ =
misbehavior_sender.try_send((advertiser_addr, err.misbehavior_score()));
}
}
result = Ok(());
Setup::Initialized {
cached_peer_addr_response,
block_downloads,
mempool,
state,
misbehavior_sender,
}
}
};
if matches!(self.setup, Setup::FailedInit) {
unreachable!("incomplete Inbound initialization after poll_ready state handling");
}
Poll::Ready(result)
}
#[instrument(name = "inbound", skip(self, req))]
fn call(&mut self, req: zn::Request) -> Self::Future {
let pruned_block_not_found_logger = self.pruned_block_not_found_logger.clone();
let (cached_peer_addr_response, block_downloads, mempool, state) = match &mut self.setup {
Setup::Initialized {
cached_peer_addr_response,
block_downloads,
mempool,
state,
misbehavior_sender: _,
} => (cached_peer_addr_response, block_downloads, mempool, state),
_ => {
debug!("ignoring request from remote peer during setup");
return async { Ok(zn::Response::Nil) }.boxed();
}
};
match req {
zn::Request::Peers => {
cached_peer_addr_response.try_refresh();
let response = cached_peer_addr_response.value();
async move {
Ok(response)
}.boxed()
}
request @ (zn::Request::BlocksByHash(_)
| zn::Request::BlocksByHashFrom { .. }) => {
let (hashes, source) = match request {
zn::Request::BlocksByHash(hashes) => (hashes, None),
zn::Request::BlocksByHashFrom { hashes, source } => {
(hashes, Some(source))
}
_ => unreachable!("matched block inventory request"),
};
let log_pruned_block =
pruned_block_not_found_logger.is_enabled_for(source.as_ref());
if hashes.is_empty() {
return async { Ok(zn::Response::Nil) }.boxed();
}
let state = state.clone();
async move {
let mut blocks: Vec<InventoryResponse<(Arc<Block>, Option<PeerSocketAddr>), block::Hash>> = Vec::new();
let mut total_size = 0;
for &hash in hashes.iter().take(GETDATA_MAX_BLOCK_COUNT) {
if total_size >= GETDATA_SENT_BYTES_LIMIT {
break;
}
let response = state.clone().ready().await?.call(zs::Request::Block(hash.into())).await?;
match response {
zs::Response::Block(Some(block)) => {
total_size += block.zcash_serialized_size();
blocks.push(Available((block, None)))
},
zs::Response::Block(None) => {
if log_pruned_block {
if let Some(height) =
retained_block_height(state.clone(), hash).await
{
if let Some(tx_retention) =
pruned_block_not_found_logger.reserve_log()
{
error!(
?hash,
?height,
tx_retention,
"{ZCASHD_COMPAT_PRUNED_BLOCK_ERROR}"
);
}
}
}
blocks.push(Missing(hash))
},
_ => unreachable!("wrong response from state"),
}
}
Ok(zn::Response::Blocks(blocks))
}.boxed()
}
zn::Request::TransactionsById(req_tx_ids)
| zn::Request::TransactionsByIdFrom {
ids: req_tx_ids, ..
} => {
if req_tx_ids.is_empty() {
return async { Ok(zn::Response::Nil) }.boxed();
}
let request = mempool::Request::TransactionsById(req_tx_ids.clone());
mempool.clone().oneshot(request).map_ok(move |resp| {
let mut total_size = 0;
let transactions = match resp {
mempool::Response::Transactions(transactions) => transactions,
_ => unreachable!("Mempool component should always respond to a `TransactionsById` request with a `Transactions` response"),
};
let available_tx_ids: HashSet<UnminedTxId> = transactions.iter().map(|tx| tx.id).collect();
let missing = req_tx_ids.into_iter().filter(|tx_id| !available_tx_ids.contains(tx_id)).map(Missing);
let available = transactions.into_iter().take_while(|tx| {
let within_limit = total_size < GETDATA_SENT_BYTES_LIMIT;
total_size += tx.size;
within_limit
}).map(|tx| Available((tx, None)));
zn::Response::Transactions(available.chain(missing).collect())
}).boxed()
}
zn::Request::FindBlocks { known_blocks, stop } => {
let request = zs::Request::FindBlockHashes { known_blocks, stop };
state.clone().oneshot(request).map_ok(|resp| match resp {
zs::Response::BlockHashes(hashes) if hashes.is_empty() => zn::Response::Nil,
zs::Response::BlockHashes(hashes) => zn::Response::BlockHashes(hashes),
_ => unreachable!("zakura-state should always respond to a `FindBlockHashes` request with a `BlockHashes` response"),
})
.boxed()
}
zn::Request::FindHeaders { known_blocks, stop } => {
let request = zs::Request::FindBlockHeaders { known_blocks, stop };
state.clone().oneshot(request).map_ok(|resp| match resp {
zs::Response::BlockHeaders(headers) if headers.is_empty() => zn::Response::Nil,
zs::Response::BlockHeaders(headers) => zn::Response::BlockHeaders(headers),
_ => unreachable!("zakura-state should always respond to a `FindBlockHeaders` request with a `BlockHeaders` response"),
})
.boxed()
}
zn::Request::PushTransaction(transaction, advertiser) => {
let request = match advertiser {
Some(source) => mempool::Request::QueueFromPeer {
source: mempool_queue_source(source),
transactions: vec![transaction.into()],
},
None => mempool::Request::Queue(vec![transaction.into()]),
};
mempool
.clone()
.oneshot(request)
.map_ok(|_resp| zn::Response::Nil)
.boxed()
}
zn::Request::AdvertiseTransactionIds(transactions, advertiser) => {
let request = match advertiser {
Some(source) => mempool::Request::QueueFromPeer {
source: mempool_queue_source(source),
transactions: transactions.into_iter().map(Into::into).collect(),
},
None => mempool::Request::Queue(
transactions.into_iter().map(Into::into).collect(),
),
};
mempool
.clone()
.oneshot(request)
.map_ok(|_resp| zn::Response::Nil)
.boxed()
}
zn::Request::AdvertiseBlock(hash, advertiser) => {
block_downloads.download_and_verify(hash, advertiser);
async { Ok(zn::Response::Nil) }.boxed()
}
zn::Request::MempoolTransactionIds => {
mempool.clone().oneshot(mempool::Request::TransactionIds).map_ok(|resp| match resp {
mempool::Response::TransactionIds(transaction_ids) if transaction_ids.is_empty() => zn::Response::Nil,
mempool::Response::TransactionIds(transaction_ids) => zn::Response::TransactionIds(transaction_ids.into_iter().collect()),
_ => unreachable!("Mempool component should always respond to a `TransactionIds` request with a `TransactionIds` response"),
})
.boxed()
}
zn::Request::Ping(_) => {
unreachable!("ping requests are handled internally");
}
zn::Request::AdvertiseBlockToAll(_) => unreachable!("should always be decoded as `AdvertiseBlock` request")
}
}
}