use std::{collections::HashMap, future};
use proptest::{prop_assert, prop_assert_eq};
use super::*;
use super::{
config::{
BS_CHECKPOINT_RANGE_BYTE_FLOOR, BS_PER_BLOCK_WORST_CASE_BYTES,
DEFAULT_BS_BBR_RELIABILITY_WEIGHT_PERCENT, DEFAULT_BS_FLOOR_PEER_AVOID_COOLDOWN,
DEFAULT_BS_INITIAL_BLOCK_PROBE_REQUESTS, DEFAULT_BS_MAX_INFLIGHT_BLOCK_BYTES,
DEFAULT_BS_MAX_REORDER_LOOKAHEAD_BYTES, DEFAULT_BS_MAX_REQUESTS_WITHOUT_BLOCK_PROGRESS,
DEFAULT_BS_MAX_RESPONSE_BYTES, DEFAULT_BS_NO_PROGRESS_PEER_COOLDOWN,
DEFAULT_BS_REQUEST_TIMEOUT, MAX_BS_INFLIGHT_REQUESTS, MAX_BS_RESPONSE_BYTES,
MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES,
},
reactor::node_id_from_block_peer_id,
reorder::*,
request::*,
sequencer::*,
sequencer_task::{initial_view, SequencedBody, SequencerControlInput, SequencerTask},
state::*,
work_queue::WorkQueue,
};
use crate::zakura::{
framed_channel,
testkit::{TraceCapture, TraceValue},
ChainFrontier, FramedRecv, FramedSend, Frontier, FrontierChange, FrontierUpdate, Peer, Service,
ServicePeerSnapshot, ServiceRegistry, StreamMode, ZakuraBlockSyncCandidateState,
ZakuraSyncExchange,
};
use zakura_chain::{
fmt::HexDebug,
serialization::{ZcashDeserializeInto, ZcashSerialize},
transaction::Transaction,
transparent,
};
use zakura_test::vectors::{BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES, BLOCK_MAINNET_3_BYTES};
fn peer(byte: u8) -> ZakuraPeerId {
ZakuraPeerId::new(vec![byte; 32]).expect("test peer id is within bounds")
}
fn mainnet_block(bytes: &[u8]) -> Arc<block::Block> {
Arc::new(bytes.zcash_deserialize_into().expect("block vector parses"))
}
fn mainnet_blocks_1_to_3() -> Vec<Arc<block::Block>> {
vec![
mainnet_block(&BLOCK_MAINNET_1_BYTES),
mainnet_block(&BLOCK_MAINNET_2_BYTES),
mainnet_block(&BLOCK_MAINNET_3_BYTES),
]
}
fn raw_block_payload(block: &Arc<block::Block>) -> Arc<[u8]> {
let frame = BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("test block frame encodes");
Arc::from(frame.payload.into_boxed_slice())
}
fn forked_block(block: &Arc<block::Block>, nonce_tag: u8) -> Arc<block::Block> {
let mut fork = block.as_ref().clone();
let mut header = *fork.header;
header.nonce = HexDebug([nonce_tag; 32]);
fork.header = Arc::new(header);
Arc::new(fork)
}
fn block_with_bad_merkle_root(
block: &Arc<block::Block>,
extra_tx: &Arc<block::Block>,
) -> Arc<block::Block> {
let mut bad_block = block.as_ref().clone();
bad_block
.transactions
.push(extra_tx.transactions[0].clone());
assert_eq!(bad_block.hash(), block.hash());
assert_eq!(bad_block.coinbase_height(), block.coinbase_height());
assert_ne!(
bad_block
.transactions
.iter()
.collect::<block::merkle::Root>(),
bad_block.header.merkle_root
);
Arc::new(bad_block)
}
fn fake_sequential_blocks(count: u32) -> Vec<Arc<block::Block>> {
let template = mainnet_block(&BLOCK_MAINNET_1_BYTES);
(1..=count)
.map(|height| fake_block_at_height(&template, block::Height(height)))
.collect()
}
fn fake_blocks_in_range(start: u32, end: u32) -> Vec<Arc<block::Block>> {
let template = mainnet_block(&BLOCK_MAINNET_1_BYTES);
(start..=end)
.map(|height| fake_block_at_height(&template, block::Height(height)))
.collect()
}
fn fake_block_at_height(template: &Arc<block::Block>, height: block::Height) -> Arc<block::Block> {
let mut block = template.as_ref().clone();
let mut coinbase = block.transactions[0].clone();
let input = match Arc::make_mut(&mut coinbase) {
Transaction::V1 { inputs, .. }
| Transaction::V2 { inputs, .. }
| Transaction::V3 { inputs, .. }
| Transaction::V4 { inputs, .. }
| Transaction::V5 { inputs, .. } => &mut inputs[0],
Transaction::V6 { inputs, .. } => &mut inputs[0],
};
match input {
transparent::Input::Coinbase {
height: coinbase_height,
..
} => *coinbase_height = height,
_ => panic!("template block must start with a coinbase input"),
}
block.transactions[0] = coinbase;
let merkle_root = block.transactions.iter().collect::<block::merkle::Root>();
let mut header = *block.header;
header.merkle_root = merkle_root;
block.header = Arc::new(header);
Arc::new(block)
}
fn block_size(block: &block::Block) -> u32 {
u32::try_from(
block
.zcash_serialize_to_vec()
.expect("test block serializes")
.len(),
)
.expect("test block size fits u32")
}
fn status() -> BlockSyncStatus {
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(42),
tip_hash: block::Hash([7; 32]),
max_blocks_per_response: 16,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
}
}
fn immediate_body_download_config() -> ZakuraBlockSyncConfig {
ZakuraBlockSyncConfig {
max_blocks_per_response: MAX_BS_BLOCKS_PER_REQUEST,
..ZakuraBlockSyncConfig::default()
}
}
fn fill_loop_mechanics_config() -> ZakuraBlockSyncConfig {
ZakuraBlockSyncConfig {
initial_block_probe_requests: DEFAULT_BS_MAX_REQUESTS_WITHOUT_BLOCK_PROGRESS,
..immediate_body_download_config()
}
}
fn test_frontier(height: u32) -> Frontier {
let hash_byte = u8::try_from(height % 251).expect("height modulo 251 fits in u8");
Frontier::new(block::Height(height), block::Hash([hash_byte; 32]))
}
fn test_frontier_update(
finalized: u32,
verified_body: u32,
best_header: u32,
change: FrontierChange,
) -> FrontierUpdate {
FrontierUpdate {
frontier: ChainFrontier {
finalized: test_frontier(finalized),
verified_body: test_frontier(verified_body),
best_header: test_frontier(best_header),
},
change,
}
}
fn exchange_block_sync_startup(
initial: FrontierUpdate,
config: ZakuraBlockSyncConfig,
) -> (ZakuraSyncExchange, BlockSyncStartup) {
let exchange = ZakuraSyncExchange::new(initial, ZakuraTrace::noop());
let frontier = initial.frontier;
let startup = BlockSyncStartup::new_with_exchange(
BlockSyncFrontiers {
finalized_height: frontier.finalized.height,
verified_block_tip: frontier.verified_body.height,
verified_block_hash: frontier.verified_body.hash,
},
(frontier.best_header.height, frontier.best_header.hash),
exchange.subscribe_frontier(),
config,
);
(exchange, startup)
}
fn round_trip(message: BlockSyncMessage) {
let encoded = message.encode().expect("message encodes");
let decoded = BlockSyncMessage::decode(&encoded).expect("message decodes");
assert_eq!(decoded, message);
}
async fn next_event(events: &mut mpsc::Receiver<BlockSyncEvent>) -> BlockSyncEvent {
tokio::time::timeout(Duration::from_secs(1), events.recv())
.await
.expect("block-sync event should arrive")
.expect("block-sync event channel should stay open")
}
async fn next_action(actions: &mut mpsc::Receiver<BlockSyncAction>) -> BlockSyncAction {
tokio::time::timeout(Duration::from_secs(1), actions.recv())
.await
.expect("block-sync action should arrive")
.expect("block-sync action channel should stay open")
}
async fn wait_for_query_needed_blocks(
actions: &mut mpsc::Receiver<BlockSyncAction>,
verified_block_tip: block::Height,
best_header_tip: block::Height,
) {
let expected_from = verified_block_tip.next().unwrap_or(verified_block_tip);
loop {
match next_action(actions).await {
BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip: actual_best,
..
} if from == expected_from && actual_best == best_header_tip => return,
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before target QueryNeededBlocks: {action:?}"),
}
}
}
async fn send_inbound(inbound_tx: &FramedSend, msg: BlockSyncMessage) {
inbound_tx
.send(msg.encode_frame().expect("message encodes"))
.await
.expect("inbound frame queues onto the peer stream");
}
async fn next_outbound_message(outbound: &mut FramedRecv) -> BlockSyncMessage {
let frame = tokio::time::timeout(Duration::from_secs(1), outbound.recv())
.await
.expect("outbound frame arrives")
.expect("outbound channel is live");
BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
}
async fn wait_for_outbound_block(outbound: &mut FramedRecv) -> Arc<block::Block> {
loop {
match next_outbound_message(outbound).await {
BlockSyncMessage::Block(block) => return block,
BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {}
msg => panic!("unexpected outbound message before block: {msg:?}"),
}
}
}
async fn wait_for_outbound_blocks_done(outbound: &mut FramedRecv) -> (block::Height, u32) {
loop {
match next_outbound_message(outbound).await {
BlockSyncMessage::BlocksDone {
start_height,
returned,
} => return (start_height, returned),
BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {}
msg => panic!("unexpected outbound message before BlocksDone: {msg:?}"),
}
}
}
async fn wait_for_outbound_range_unavailable(outbound: &mut FramedRecv) -> (block::Height, u32) {
loop {
match next_outbound_message(outbound).await {
BlockSyncMessage::RangeUnavailable {
start_height,
count,
} => return (start_height, count),
BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {}
msg => panic!("unexpected outbound message before RangeUnavailable: {msg:?}"),
}
}
}
async fn wait_for_outbound_getblocks(outbound: &mut FramedRecv) -> (block::Height, u32) {
loop {
match next_outbound_message(outbound).await {
BlockSyncMessage::GetBlocks {
start_height,
count,
} => return (start_height, count),
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before GetBlocks: {msg:?}"),
}
}
}
async fn wait_for_outbound_status(outbound: &mut FramedRecv) -> BlockSyncStatus {
match next_outbound_message(outbound).await {
BlockSyncMessage::Status(status) => status,
msg => panic!("unexpected outbound message before Status: {msg:?}"),
}
}
async fn wait_for_getblocks_across(
peers: &mut [(ZakuraPeerId, &mut FramedRecv)],
) -> (ZakuraPeerId, block::Height, u32) {
use futures::stream::{FuturesUnordered, StreamExt};
loop {
let (peer_id, frame) = {
let mut recvs: FuturesUnordered<_> = peers
.iter_mut()
.map(|(peer_id, outbound)| {
let peer_id = peer_id.clone();
async move {
let frame = outbound.recv().await.expect("outbound channel is live");
(peer_id, frame)
}
})
.collect();
tokio::time::timeout(Duration::from_secs(1), recvs.next())
.await
.expect("an outbound frame arrives on some peer")
.expect("at least one peer outbound is live")
};
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height,
count,
} => return (peer_id, start_height, count),
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before GetBlocks: {msg:?}"),
}
}
}
async fn drain_parent_first_actions(
actions: &mut mpsc::Receiver<BlockSyncAction>,
verified_tip: &mut block::Height,
expected_new_fork: Option<&[Arc<block::Block>]>,
) {
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(25), actions.recv()).await
{
match action {
BlockSyncAction::SubmitBlock { block, .. } => {
let height = block
.coinbase_height()
.expect("submitted test block has height");
assert_eq!(
Some(height),
next_height(*verified_tip),
"block sync must submit only the contiguous parent-first prefix"
);
if let Some(new_fork) = expected_new_fork {
let expected_hash = match height.0 {
2 => new_fork[1].hash(),
3 => new_fork[2].hash(),
_ => panic!("unexpected post-reset submitted height: {height:?}"),
};
assert_eq!(
block.hash(),
expected_hash,
"post-reset submissions must follow the re-derived fork"
);
}
*verified_tip = height;
}
BlockSyncAction::Misbehavior {
reason: BlockSyncMisbehavior::InvalidBlock | BlockSyncMisbehavior::UnsolicitedBlock,
..
} => {}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while draining body responses: {action:?}"),
}
}
}
fn download_window() -> DownloadWindow {
DownloadWindow::new(&ZakuraBlockSyncConfig::default())
}
fn test_delivery_snapshot(now: Instant) -> DeliverySnapshot {
DeliverySnapshot {
delivered: 0,
delivered_at: now,
}
}
fn window_request(height: u32) -> OutstandingBlockRange {
let byte = u8::try_from(height).expect("test heights fit in u8");
let now = Instant::now();
OutstandingBlockRange {
request: BlockRangeRequest {
start_height: block::Height(height),
count: 1,
anchor_hash: block::Hash([byte; 32]),
estimated_bytes: 1,
expected_blocks: vec![ExpectedBlock {
height: block::Height(height),
hash: block::Hash([byte; 32]),
estimated_bytes: 1,
}],
},
queued_at: now,
deadline: now,
delivery_snapshot: test_delivery_snapshot(now),
delivered_bytes: 0,
received: ReceivedBlockTracker::default(),
}
}
fn window_request_range(start: u32, count: u32) -> OutstandingBlockRange {
let byte = u8::try_from(start).expect("test heights fit in u8");
let now = Instant::now();
OutstandingBlockRange {
request: BlockRangeRequest {
start_height: block::Height(start),
count,
anchor_hash: block::Hash([byte; 32]),
estimated_bytes: u64::from(count),
expected_blocks: (start..start + count)
.map(|height| ExpectedBlock {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("test heights fit in u8"); 32]),
estimated_bytes: 1,
})
.collect(),
},
queued_at: now,
deadline: now,
delivery_snapshot: test_delivery_snapshot(now),
delivered_bytes: 0,
received: ReceivedBlockTracker::default(),
}
}
#[test]
fn block_liveness_disconnects_silent_active_peer_after_default_timeout() {
let config = ZakuraBlockSyncConfig::default();
let timeout = config.effective_liveness_timeout();
assert_eq!(timeout, Duration::from_secs(32));
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
assert_eq!(
window.check_liveness(now + timeout - Duration::from_millis(1)),
LivenessOutcome::Ok
);
assert_eq!(
window.check_liveness(now + timeout),
LivenessOutcome::Disconnect
);
}
#[test]
fn block_liveness_disarms_a_deadline_set_without_a_request() {
let now = Instant::now();
let mut window = download_window();
assert_eq!(window.check_liveness(now), LivenessOutcome::Ok);
window.block_liveness_deadline = Some(now);
assert_eq!(window.check_liveness(now), LivenessOutcome::Disarm);
window.clear_liveness_if_idle();
assert_eq!(window.block_liveness_deadline, None);
assert_eq!(window.check_liveness(now), LivenessOutcome::Ok);
}
#[test]
fn short_response_charges_one_reliability_failure_per_request_not_per_height() {
let fresh = download_window().reliability_factor();
let one_missing = {
let mut window = download_window();
window.penalize_short_response(1);
window.reliability_factor()
};
let many_missing = {
let mut window = download_window();
window.penalize_short_response(64);
window.reliability_factor()
};
assert!(
one_missing < fresh,
"a short response must lower reliability (one goodput failure)"
);
assert_eq!(
one_missing, many_missing,
"a short response is one failure per request, independent of the missing-height count"
);
}
#[test]
fn block_liveness_progress_before_deadline_keeps_peer_alive() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let mut now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
for _ in 0..4 {
now += timeout - Duration::from_millis(1);
assert_eq!(window.check_liveness(now), LivenessOutcome::Ok);
window.note_block_progress(now, timeout);
assert_eq!(window.block_liveness_deadline, Some(now + timeout));
}
}
#[test]
fn block_liveness_disconnects_silent_peer_after_outstanding_drains() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
window.outstanding.clear();
window.disarm_liveness_after_progress_if_idle();
assert_eq!(window.block_liveness_deadline, Some(now + timeout));
assert_eq!(
window.check_liveness(now + timeout),
LivenessOutcome::Disconnect
);
}
#[test]
fn block_liveness_disarms_when_satisfied_request_drains() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
window.note_block_progress(now + Duration::from_millis(1), timeout);
window.outstanding.clear();
window.disarm_liveness_after_progress_if_idle();
assert_eq!(window.block_liveness_deadline, None);
assert_eq!(window.check_liveness(now + timeout), LivenessOutcome::Ok);
}
#[test]
fn block_liveness_uses_probe_cap_until_first_accepted_body() {
let config = ZakuraBlockSyncConfig {
initial_block_probe_requests: 1,
max_requests_without_block_progress: 8,
..ZakuraBlockSyncConfig::default()
};
let timeout = config.effective_liveness_timeout();
let now = Instant::now();
let mut window = DownloadWindow::new(&config);
assert!(!window.has_block_progress());
assert_eq!(window.no_progress_request_cap(), 1);
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
assert_eq!(window.requests_without_block_progress, 1);
assert_eq!(window.no_progress_request_cap(), 1);
window.note_block_progress(now + Duration::from_millis(1), timeout);
assert!(window.has_block_progress());
assert_eq!(window.requests_without_block_progress, 0);
assert_eq!(window.no_progress_request_cap(), 8);
}
#[test]
fn block_liveness_resuming_after_idle_gets_fresh_deadline() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
window.note_block_progress(now + Duration::from_millis(1), timeout);
window.outstanding.clear();
window.disarm_liveness_after_progress_if_idle();
let resumed = now + Duration::from_secs(60);
window.outstanding.push(window_request(2));
window.arm_liveness(resumed, timeout);
assert_eq!(window.block_liveness_deadline, Some(resumed + timeout));
}
#[test]
fn block_liveness_multi_block_range_progress_resets_each_body() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let start = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request_range(1, 3));
window.arm_liveness(start, timeout);
let first = start + Duration::from_secs(4);
window.note_block_progress(first, timeout);
assert_eq!(window.block_liveness_deadline, Some(first + timeout));
let second = first + Duration::from_secs(4);
assert_eq!(window.check_liveness(second), LivenessOutcome::Ok);
window.note_block_progress(second, timeout);
assert_eq!(window.block_liveness_deadline, Some(second + timeout));
let third = second + Duration::from_secs(4);
assert_eq!(window.check_liveness(third), LivenessOutcome::Ok);
window.note_block_progress(third, timeout);
assert_eq!(window.block_liveness_deadline, Some(third + timeout));
}
#[test]
fn view_reset_reclears_probe_streak_so_unproven_peer_can_reprobe() {
let config = ZakuraBlockSyncConfig {
initial_block_probe_requests: 1,
..ZakuraBlockSyncConfig::default()
};
let timeout = config.effective_liveness_timeout();
let now = Instant::now();
let mut window = DownloadWindow::new(&config);
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
assert_eq!(window.requests_without_block_progress, 1);
assert_eq!(window.no_progress_request_cap(), 1);
window.outstanding.clear();
window.note_view_reset();
assert_eq!(window.requests_without_block_progress, 0);
assert!(window.requests_without_block_progress < window.no_progress_request_cap());
assert!(!window.has_block_progress());
assert_eq!(
window.check_liveness(now + timeout),
LivenessOutcome::Ok,
"a reset peer must not carry a phantom liveness deadline",
);
}
#[test]
fn view_reset_preserves_proof_but_reclears_streak() {
let config = ZakuraBlockSyncConfig {
initial_block_probe_requests: 1,
max_requests_without_block_progress: 8,
..ZakuraBlockSyncConfig::default()
};
let timeout = config.effective_liveness_timeout();
let now = Instant::now();
let mut window = DownloadWindow::new(&config);
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
window.note_block_progress(now + Duration::from_millis(1), timeout);
window.outstanding.push(window_request(2));
window.arm_liveness(now + Duration::from_millis(2), timeout);
assert!(window.has_block_progress());
assert_eq!(window.no_progress_request_cap(), 8);
window.outstanding.clear();
window.note_view_reset();
assert_eq!(window.requests_without_block_progress, 0);
assert!(
window.has_block_progress(),
"reset must not un-prove a peer"
);
assert_eq!(window.no_progress_request_cap(), 8);
}
#[test]
fn backpressure_extends_liveness_instead_of_disconnecting() {
let config = ZakuraBlockSyncConfig::default();
let timeout = config.effective_liveness_timeout();
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
assert_eq!(
window.check_liveness(now + timeout),
LivenessOutcome::Disconnect,
"the deadline has expired: without backpressure this disconnects",
);
let extended_at = now + timeout;
window.extend_liveness_deadline(extended_at, timeout);
assert_eq!(window.check_liveness(extended_at), LivenessOutcome::Ok);
assert_eq!(
window.check_liveness(extended_at + timeout),
LivenessOutcome::Disconnect,
);
}
#[test]
fn work_queue_returned_height_is_contestable_by_any_peer() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(queue.in_flight_contains(block::Height(1)));
assert!(!queue.pending_contains(block::Height(1)));
queue.return_items([block::Height(1)]);
assert!(queue.pending_contains(block::Height(1)));
assert_eq!(
queue
.take_in_range(block::Height(1), block::Height(1), 1)
.len(),
1,
"a returned height must be immediately re-takable"
);
}
async fn connect_peer_with_status(
service: &BlockSyncService,
actions: &mut mpsc::Receiver<BlockSyncAction>,
byte: u8,
servable_high: block::Height,
tip_hash: block::Hash,
max_inflight_requests: u32,
max_response_bytes: u32,
) -> (ZakuraPeerId, FramedSend, FramedRecv) {
connect_peer_with_status_message(
service,
actions,
byte,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high,
tip_hash,
max_blocks_per_response: 16,
max_inflight_requests,
max_response_bytes,
},
)
.await
}
async fn connect_peer_with_status_message(
service: &BlockSyncService,
actions: &mut mpsc::Receiver<BlockSyncAction>,
byte: u8,
status: BlockSyncStatus,
) -> (ZakuraPeerId, FramedSend, FramedRecv) {
let peer = peer(byte);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
let _ = actions;
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(status)
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
(peer, inbound_tx, outbound_rx)
}
fn needed(height: u32, size: BlockSizeEstimate) -> (block::Height, block::Hash, BlockSizeEstimate) {
(block::Height(height), block::Hash([height as u8; 32]), size)
}
fn work_queue_with(
floor: u32,
items: impl IntoIterator<Item = (block::Height, block::Hash, BlockSizeEstimate)>,
) -> super::work_queue::WorkQueue {
let queue = super::work_queue::WorkQueue::new(block::Height(floor));
queue.set_estimate_floor_for_tests(1);
queue.extend(items);
queue
}
fn block_meta(block: &Arc<block::Block>) -> BlockSyncBlockMeta {
BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
}
}
#[test]
fn block_sync_config_defaults_and_round_trips() {
let default = ZakuraBlockSyncConfig::default();
assert_eq!(default.max_blocks_per_response, 1);
assert_eq!(default.max_inflight_requests, 32000);
assert_eq!(
default.max_inflight_block_bytes,
DEFAULT_BS_MAX_INFLIGHT_BLOCK_BYTES
);
assert_eq!(
default.max_reorder_lookahead_bytes,
DEFAULT_BS_MAX_REORDER_LOOKAHEAD_BYTES
);
assert_eq!(
default.floor_peer_avoid_cooldown,
DEFAULT_BS_FLOOR_PEER_AVOID_COOLDOWN
);
assert_eq!(
default.no_progress_peer_cooldown,
DEFAULT_BS_NO_PROGRESS_PEER_COOLDOWN
);
assert_eq!(
default.initial_block_probe_requests,
DEFAULT_BS_INITIAL_BLOCK_PROBE_REQUESTS,
);
assert_eq!(
default.max_requests_without_block_progress,
DEFAULT_BS_MAX_REQUESTS_WITHOUT_BLOCK_PROGRESS,
);
assert_eq!(
default.bbr_reliability_weight_percent,
DEFAULT_BS_BBR_RELIABILITY_WEIGHT_PERCENT,
);
assert_eq!(
default.effective_max_reorder_lookahead_bytes(),
DEFAULT_BS_MAX_REORDER_LOOKAHEAD_BYTES
);
assert_eq!(
default.floor_request_byte_reservation(),
u64::from(DEFAULT_BS_MAX_RESPONSE_BYTES)
);
assert_eq!(
default.effective_floor_peer_avoid_cooldown(),
DEFAULT_BS_FLOOR_PEER_AVOID_COOLDOWN
);
assert_eq!(
default.effective_no_progress_peer_cooldown(),
DEFAULT_BS_NO_PROGRESS_PEER_COOLDOWN
);
assert!(default.validate().is_ok());
assert_eq!(
default.max_submitted_block_applies,
MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES
);
assert_eq!(
default.submitted_apply_limit(),
MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES
);
assert_eq!(default.request_timeout, DEFAULT_BS_REQUEST_TIMEOUT);
let encoded = toml::to_string(&default).expect("block-sync config serializes");
let decoded: ZakuraBlockSyncConfig =
toml::from_str(&encoded).expect("block-sync config deserializes");
assert_eq!(decoded, default);
let config: crate::Config = toml::from_str(
r#"
[zakura.block_sync]
max_submitted_block_applies = 9
"#,
)
.expect("nested Zakura block-sync config deserializes");
assert_eq!(config.zakura.block_sync.max_submitted_block_applies, 9);
assert_eq!(
config.zakura.block_sync.submitted_apply_limit(),
MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES,
);
}
#[test]
fn config_validate_rejects_degenerate_values() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 0,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
max_reorder_lookahead_bytes: 0,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_CHECKPOINT_RANGE_BYTE_FLOOR - 1,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_ok());
config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_CHECKPOINT_RANGE_BYTE_FLOOR,
max_reorder_lookahead_bytes: BS_CHECKPOINT_RANGE_BYTE_FLOOR,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_ok());
config = ZakuraBlockSyncConfig {
request_timeout: Duration::ZERO,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
initial_block_probe_requests: 0,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
max_requests_without_block_progress: 0,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
bbr_reliability_weight_percent: 101,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
}
#[test]
fn config_deserialize_keeps_below_range_floor_inflight_block_bytes() {
let config: crate::Config = toml::from_str(
r#"
[zakura.block_sync]
max_inflight_block_bytes = 268435456
"#,
)
.expect("a below-range-floor max_inflight_block_bytes config still loads");
const {
assert!(268435456u64 < BS_CHECKPOINT_RANGE_BYTE_FLOOR);
}
assert_eq!(config.zakura.block_sync.max_inflight_block_bytes, 268435456);
}
#[test]
fn config_deserialize_clamps_sub_floor_request_inflight_block_bytes() {
let config: crate::Config = toml::from_str(
r#"
[zakura.block_sync]
max_inflight_block_bytes = 16777216
"#,
)
.expect("a sub-floor-request max_inflight_block_bytes config still loads");
let request_floor = config.zakura.block_sync.floor_request_byte_reservation();
assert!(
16_777_216 <= request_floor,
"test premise: the stored value is at or below one floor request"
);
assert_eq!(
config.zakura.block_sync.max_inflight_block_bytes,
request_floor + 1
);
}
#[test]
fn codec_round_trips_every_message_variant() {
round_trip(BlockSyncMessage::Status(status()));
round_trip(BlockSyncMessage::GetBlocks {
start_height: block::Height(10),
count: 3,
});
round_trip(BlockSyncMessage::Block(mainnet_block(
&BLOCK_MAINNET_1_BYTES,
)));
round_trip(BlockSyncMessage::BlocksDone {
start_height: block::Height(10),
returned: 3,
});
round_trip(BlockSyncMessage::RangeUnavailable {
start_height: block::Height(10),
count: 3,
});
}
#[test]
fn codec_round_trips_block_near_max_block_bytes() {
let block = Arc::new(zakura_chain::block::tests::generate::large_multi_transaction_block());
let serialized_len = block
.zcash_serialize_to_vec()
.expect("large test block serializes")
.len();
let max_block_bytes =
usize::try_from(block::MAX_BLOCK_BYTES).expect("max block size fits in usize");
assert!(
serialized_len <= max_block_bytes && serialized_len > max_block_bytes - 1000,
"test block should be close to the consensus cap, got {serialized_len}"
);
round_trip(BlockSyncMessage::Block(block));
}
#[test]
fn codec_rejects_malformed_discriminator_and_truncated_payload() {
assert!(matches!(
BlockSyncMessage::decode(&[99]),
Err(BlockSyncWireError::UnknownMessageType(99))
));
assert!(matches!(
BlockSyncMessage::decode(&[MSG_BS_GET_BLOCKS, 1, 0]),
Err(BlockSyncWireError::Io(_))
));
}
#[test]
fn codec_classifies_payloads_above_old_raw_stream6_cap() {
let old_max_bs_message_bytes =
usize::try_from(block::MAX_BLOCK_BYTES).expect("max block bytes fits in usize") + 1;
let payload = vec![99; old_max_bs_message_bytes + 1];
assert!(payload.len() <= MAX_BS_MESSAGE_BYTES);
assert!(matches!(
BlockSyncMessage::decode(&payload),
Err(BlockSyncWireError::UnknownMessageType(99))
));
}
#[test]
fn codec_rejects_oversized_frame_and_oversized_block() {
let oversized_payload = vec![0; MAX_BS_MESSAGE_BYTES + 1];
assert!(matches!(
BlockSyncMessage::decode(&oversized_payload),
Err(BlockSyncWireError::OversizedPayload { .. })
));
let oversized_block =
Arc::new(zakura_chain::block::tests::generate::oversized_multi_transaction_block());
assert!(matches!(
BlockSyncMessage::Block(oversized_block).encode(),
Err(BlockSyncWireError::OversizedBlock { .. })
| Err(BlockSyncWireError::OversizedPayload { .. })
));
}
#[test]
fn codec_rejects_count_and_returned_over_cap() {
let over_cap = MAX_BS_BLOCKS_PER_REQUEST + 1;
assert!(matches!(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 0,
}
.encode(),
Err(BlockSyncWireError::ZeroBlockCount)
));
let mut zero_count_get_blocks = vec![MSG_BS_GET_BLOCKS];
zero_count_get_blocks.extend_from_slice(&1u32.to_le_bytes());
zero_count_get_blocks.extend_from_slice(&0u32.to_le_bytes());
assert!(matches!(
BlockSyncMessage::decode(&zero_count_get_blocks),
Err(BlockSyncWireError::ZeroBlockCount)
));
let mut zero_count_range_unavailable = vec![MSG_BS_RANGE_UNAVAILABLE];
zero_count_range_unavailable.extend_from_slice(&1u32.to_le_bytes());
zero_count_range_unavailable.extend_from_slice(&0u32.to_le_bytes());
assert!(matches!(
BlockSyncMessage::decode(&zero_count_range_unavailable),
Err(BlockSyncWireError::ZeroBlockCount)
));
assert!(matches!(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: over_cap,
}
.encode(),
Err(BlockSyncWireError::BlockCountLimit { .. })
));
assert!(matches!(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: over_cap,
}
.encode(),
Err(BlockSyncWireError::BlockCountLimit { .. })
));
assert!(matches!(
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: over_cap,
}
.encode(),
Err(BlockSyncWireError::BlockCountLimit { .. })
));
}
#[test]
fn frame_decode_rejects_mismatched_unknown_flags_and_trailing_payload() {
let frame = Frame {
message_type: u16::from(MSG_BS_GET_BLOCKS),
flags: 1,
payload: BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 1,
}
.encode()
.expect("message encodes"),
};
assert!(matches!(
BlockSyncMessage::decode_frame(frame),
Err(BlockSyncWireError::UnsupportedFlags(1))
));
let mut payload = BlockSyncMessage::Status(status())
.encode()
.expect("message encodes");
payload.push(0);
assert!(matches!(
BlockSyncMessage::decode(&payload),
Err(BlockSyncWireError::TrailingBytes)
));
let frame = Frame {
message_type: u16::from(MSG_BS_BLOCK),
flags: 0,
payload: BlockSyncMessage::Status(status())
.encode()
.expect("message encodes"),
};
assert!(matches!(
BlockSyncMessage::decode_frame(frame),
Err(BlockSyncWireError::MismatchedFrameMessageType { .. })
));
}
#[test]
fn status_decode_clamps_peer_capacity_advertisements() {
let mut payload = Vec::new();
payload.push(MSG_BS_STATUS);
payload.extend_from_slice(&block::Height(1).0.to_le_bytes());
payload.extend_from_slice(&block::Height(2).0.to_le_bytes());
block::Hash([9; 32])
.zcash_serialize(&mut payload)
.expect("hash serializes");
payload.extend_from_slice(&u32::MAX.to_le_bytes());
payload.extend_from_slice(&u32::MAX.to_le_bytes());
payload.extend_from_slice(&u32::MAX.to_le_bytes());
let BlockSyncMessage::Status(status) =
BlockSyncMessage::decode(&payload).expect("status decodes")
else {
panic!("expected status message");
};
assert_eq!(status.max_blocks_per_response, MAX_BS_BLOCKS_PER_REQUEST);
assert_eq!(status.max_inflight_requests, MAX_BS_INFLIGHT_REQUESTS);
assert_eq!(status.max_response_bytes, MAX_BS_RESPONSE_BYTES);
}
#[test]
fn aggregate_response_cap_is_not_the_per_frame_cap() {
assert!(
MAX_BS_RESPONSE_BYTES > u32::try_from(MAX_BS_MESSAGE_BYTES).expect("frame cap fits u32"),
"range responses are multiple independently-capped block frames"
);
assert_eq!(
ZakuraBlockSyncConfig::default().advertised_max_response_bytes(),
MAX_BS_RESPONSE_BYTES
);
}
#[test]
fn work_queue_take_dedups_a_height_across_peers() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(10_000)),
needed(2, BlockSizeEstimate::Advertised(10_000)),
needed(3, BlockSizeEstimate::Advertised(10_000)),
],
);
let first = queue.take_in_range(
block::Height(1),
block::Height(3),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert_eq!(
first.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![1, 2, 3]
);
assert_eq!(queue.in_flight_len(), 3);
let second = queue.take_in_range(
block::Height(1),
block::Height(3),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert!(
second.is_empty(),
"a height taken by one peer must not be re-takable by another (dedup)"
);
}
#[test]
fn work_queue_extend_dedups_against_pending_in_flight_and_floor() {
let queue = work_queue_with(
5,
[
needed(3, BlockSizeEstimate::Advertised(100)),
needed(5, BlockSizeEstimate::Advertised(100)),
needed(6, BlockSizeEstimate::Advertised(100)),
needed(7, BlockSizeEstimate::Advertised(100)),
],
);
assert_eq!(queue.pending_len(), 2);
assert!(!queue.pending_contains(block::Height(5)));
queue.take_in_range(block::Height(6), block::Height(6), 1);
let inserted = queue.extend([
needed(6, BlockSizeEstimate::Advertised(100)), needed(7, BlockSizeEstimate::Advertised(100)), needed(8, BlockSizeEstimate::Advertised(100)), ]);
assert_eq!(inserted, 1, "only the genuinely new height is inserted");
assert!(queue.pending_contains(block::Height(8)));
assert!(!queue.pending_contains(block::Height(6)));
}
#[test]
fn work_queue_take_respects_servable_range_contiguity_and_max() {
let queue = work_queue_with(
0,
[
needed(10, BlockSizeEstimate::Advertised(100)),
needed(11, BlockSizeEstimate::Advertised(100)),
needed(12, BlockSizeEstimate::Advertised(100)),
needed(20, BlockSizeEstimate::Advertised(100)),
needed(21, BlockSizeEstimate::Advertised(100)),
],
);
let chunk = queue.take_in_range(block::Height(10), block::Height(30), 2);
assert_eq!(
chunk.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![10, 11]
);
let run = queue.take_in_range(
block::Height(12),
block::Height(30),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert_eq!(
run.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![12],
"take must stop at the first gap so the chunk is one contiguous request"
);
let high = queue.take_in_range(
block::Height(20),
block::Height(30),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert_eq!(
high.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![20, 21]
);
}
#[test]
fn work_queue_budgeted_take_respects_count_cap() {
let queue = work_queue_with(
0,
(1..=4).map(|height| needed(height, BlockSizeEstimate::Advertised(100))),
);
let taken = queue.take_in_range_budgeted(block::Height(1), block::Height(4), 2, u64::MAX);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![1, 2]
);
}
#[test]
fn work_queue_budgeted_take_respects_estimated_byte_cap() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(100)),
needed(2, BlockSizeEstimate::Advertised(150)),
needed(3, BlockSizeEstimate::Advertised(1)),
],
);
let taken = queue.take_in_range_budgeted(block::Height(1), block::Height(3), 3, 250);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![1, 2]
);
assert!(queue.pending_contains(block::Height(3)));
}
#[test]
fn work_queue_budgeted_take_stops_at_gaps() {
let queue = work_queue_with(
0,
[
needed(10, BlockSizeEstimate::Advertised(100)),
needed(11, BlockSizeEstimate::Advertised(100)),
needed(13, BlockSizeEstimate::Advertised(100)),
],
);
let taken = queue.take_in_range_budgeted(block::Height(10), block::Height(13), 3, u64::MAX);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![10, 11]
);
assert!(queue.pending_contains(block::Height(13)));
}
#[test]
fn work_queue_budgeted_take_takes_one_oversized_first_item_for_progress() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(500)),
needed(2, BlockSizeEstimate::Advertised(1)),
],
);
let taken = queue.take_in_range_budgeted(block::Height(1), block::Height(2), 2, 100);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![1]
);
assert_eq!(taken[0].1.estimated_bytes, 500);
assert!(queue.pending_contains(block::Height(2)));
}
#[test]
fn work_queue_budgeted_take_preserves_estimates_through_take_and_return() {
let queue = work_queue_with(0, [needed(10, BlockSizeEstimate::Advertised(12_345))]);
let taken = queue.take_in_range_budgeted(block::Height(10), block::Height(10), 1, 1);
assert_eq!(taken.len(), 1);
assert_eq!(taken[0].1.estimated_bytes, 12_345);
queue.return_items([block::Height(10)]);
let retaken = queue.take_in_range_budgeted(block::Height(10), block::Height(10), 1, 1);
assert_eq!(retaken.len(), 1);
assert_eq!(retaken[0].1.estimated_bytes, 12_345);
}
fn admit_grant(
config: &ZakuraBlockSyncConfig,
snapshot: super::admission::AdmissionSnapshot,
start: block::Height,
servable_high: block::Height,
response_byte_cap: u64,
) -> Option<super::admission::AdmissionGrant> {
match super::admission::admit(config, snapshot, start, servable_high, response_byte_cap) {
super::admission::AdmissionOutcome::Admit(grant) => Some(grant),
super::admission::AdmissionOutcome::LookaheadAtCap
| super::admission::AdmissionOutcome::InflightBudgetEmpty => None,
}
}
#[test]
fn admission_blocks_above_floor_at_cap_but_keeps_floor_fundable() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 40_000_000,
max_reorder_lookahead_bytes: 500,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 500,
reorder_buffered_blocks: 1,
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 40_000_000,
};
let floor = admit_grant(
&config,
snapshot,
block::Height(11),
block::Height(11),
1_000,
)
.expect("floor rescue remains admitted at the look-ahead cap");
assert_eq!(floor.priority, super::admission::RequestPriority::Floor);
assert_eq!(floor.max_request_bytes, 1_000);
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(412),
block::Height(412),
1_000
),
None,
"above-floor work stops at the look-ahead cap"
);
let under_cap = super::admission::AdmissionSnapshot {
reorder_buffered_bytes: 100,
budget_available: 40_000_000,
..snapshot
};
let above = admit_grant(
&config,
under_cap,
block::Height(412),
block::Height(412),
u64::MAX,
)
.expect("above-floor work is admitted below the cap");
assert_eq!(
above.priority,
super::admission::RequestPriority::AboveFloor
);
assert_eq!(above.max_request_bytes, 400);
}
#[test]
fn admission_counts_inflight_to_sequencer_bytes() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 200,
reorder_buffered_blocks: 1,
applying_buffered_bytes: 200,
applying_buffered_blocks: 1,
sequencer_input_queued_bytes: 600,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(412),
block::Height(412),
1_000
),
None,
"above-floor admission includes bytes already queued to the sequencer"
);
}
#[test]
fn total_resident_plateaus_under_commit_stall() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 300,
reorder_buffered_blocks: 1,
applying_buffered_bytes: 700,
applying_buffered_blocks: 1,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(412),
block::Height(412),
1_000
),
None,
"above-floor admission includes applying bytes held during a commit stall"
);
}
#[test]
fn floor_priority_request_does_not_buffer_above_floor_past_cap() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let capped = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 1_000,
reorder_buffered_blocks: 1,
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
assert_eq!(
admit_grant(
&config,
capped,
block::Height(412),
block::Height(412),
1_000
),
None,
"the above-floor tail of a floor-starting request is refused at the cap"
);
let floor = admit_grant(
&config,
capped,
block::Height(11),
block::Height(10_000),
1_000,
)
.expect("floor height remains fundable");
assert_eq!(floor.priority, super::admission::RequestPriority::Floor);
assert_eq!(floor.take_high, block::Height(411));
}
#[test]
fn outstanding_reservations_are_charged_at_wire_size() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(600),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 0,
reorder_buffered_blocks: 0,
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 1_100,
reserved_above_floor_blocks: 1,
budget_available: 64_000_000,
};
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(602),
block::Height(602),
u64::MAX
),
None,
"outstanding reservations must count against the resident budget",
);
assert!(
matches!(
super::admission::admit(
&config,
snapshot,
block::Height(601),
block::Height(601),
u64::MAX
),
super::admission::AdmissionOutcome::LookaheadAtCap
),
"an escalated floor take above the commit window is refused on reserved bytes alone",
);
let window = admit_grant(
&config,
snapshot,
block::Height(411),
block::Height(411),
1_000,
)
.expect("the commit window is exempt from the reservation charge");
assert_eq!(window.max_request_bytes, 1_000);
}
#[test]
fn floor_backpressures_when_download_floor_escalates_past_commit() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(1_000),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 0,
reorder_buffered_blocks: 0,
applying_buffered_bytes: 1_200,
applying_buffered_blocks: 990,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
assert_eq!(
super::admission::request_priority(snapshot.download_floor, block::Height(1_001)),
super::admission::RequestPriority::Floor,
);
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(1_001),
block::Height(1_001),
1_000
),
None,
"a floor request far ahead of commit is backpressured when the memory budget is full",
);
let frontier = admit_grant(
&config,
snapshot,
block::Height(11),
block::Height(11),
1_000,
)
.expect("the commit-frontier block is always fundable");
assert_eq!(frontier.max_request_bytes, 1_000);
let window_top = admit_grant(
&config,
snapshot,
block::Height(411),
block::Height(411),
1_000,
)
.expect("the top of the commit window is still fundable");
assert_eq!(window_top.max_request_bytes, 1_000);
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(412),
block::Height(412),
1_000
),
None,
"the first height above the commit window is memory-gated",
);
}
#[test]
fn exempt_take_never_spans_the_commit_window_boundary() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let full = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 0,
reorder_buffered_blocks: 0,
applying_buffered_bytes: 1_000,
applying_buffered_blocks: 10,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
let grant = admit_grant(
&config,
full,
block::Height(11),
block::Height(10_000),
1_000,
)
.expect("an in-window start stays fundable at a full gate");
assert_eq!(
grant.take_high,
block::Height(411),
"the exempt take is clamped at the commit-window top",
);
assert_eq!(grant.max_request_bytes, 1_000);
let open = super::admission::AdmissionSnapshot {
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
..full
};
let grant = admit_grant(
&config,
open,
block::Height(11),
block::Height(10_000),
1_000,
)
.expect("an in-window start is fundable with the gate open");
assert_eq!(grant.take_high, block::Height(411));
let grant = admit_grant(
&config,
open,
block::Height(412),
block::Height(10_000),
u64::MAX,
)
.expect("an above-window start is admitted below the cap");
assert_eq!(grant.take_high, block::Height(10_000));
assert_eq!(grant.max_request_bytes, 1_000);
}
#[test]
fn commit_window_stays_fundable_at_exact_floor() {
use super::config::{
BS_CHECKPOINT_RANGE_BYTE_FLOOR, MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES,
};
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_CHECKPOINT_RANGE_BYTE_FLOOR,
max_reorder_lookahead_bytes: 1,
..ZakuraBlockSyncConfig::default()
};
config.clamp_reorder_lookahead_to_floor();
let range_blocks = u32::try_from(MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES)
.expect("checkpoint range block count fits in u32");
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(range_blocks - 1),
verified_block_tip: block::Height(0),
reorder_buffered_bytes: 8_388_608,
reorder_buffered_blocks: 4,
applying_buffered_bytes: 800_000_000,
applying_buffered_blocks: u64::from(range_blocks) - 1,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 800_000_000,
reserved_above_floor_bytes: 4_000_000,
reserved_above_floor_blocks: 2,
budget_available: 2_000_000,
};
assert_eq!(
super::admission::request_priority(snapshot.download_floor, block::Height(range_blocks)),
super::admission::RequestPriority::Floor,
);
let completing = admit_grant(
&config,
snapshot,
block::Height(range_blocks),
block::Height(range_blocks),
u64::MAX,
)
.expect("the range-completing floor take must pass a full gate");
assert_eq!(
completing.priority,
super::admission::RequestPriority::Floor
);
assert_eq!(completing.max_request_bytes, 2_000_000);
assert!(matches!(
super::admission::admit(
&config,
snapshot,
block::Height(range_blocks + 1),
block::Height(range_blocks + 1),
u64::MAX
),
super::admission::AdmissionOutcome::LookaheadAtCap
));
}
#[test]
fn work_queue_force_cancel_and_owner_timeout_release_once() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
assert_eq!(budget.reserved(), 100);
let watchdog_released = queue.release_and_return_items([block::Height(1)]);
budget.release(watchdog_released);
assert_eq!(watchdog_released, 100);
assert_eq!(budget.reserved(), 0);
assert!(queue.pending_contains(block::Height(1)));
let late_owner_released = queue.release_and_return_items([block::Height(1)]);
budget.release(late_owner_released);
assert_eq!(late_owner_released, 0);
assert_eq!(budget.reserved(), 0);
assert!(queue.pending_contains(block::Height(1)));
}
#[test]
fn work_queue_reserved_bytes_counter_matches_scan_across_transitions() {
let queue = work_queue_with(
0,
(1..=6).map(|h| needed(h, BlockSizeEstimate::Advertised(100))),
);
let check = |label: &str| {
assert_eq!(
queue.reserved_bytes(),
queue.reserved_bytes_scanned(),
"reserved_bytes counter drifted from scan after {label}"
);
};
assert_eq!(queue.reserved_bytes(), 0);
check("seed");
let taken = queue.take_in_range(block::Height(1), block::Height(6), 6);
assert_eq!(taken.len(), 6);
assert_eq!(queue.mark_reserved((1..=6).map(block::Height)), 600);
assert_eq!(queue.reserved_bytes(), 600);
check("mark_reserved");
assert_eq!(
queue.release_active_reserved_height(block::Height(1)),
Some(100),
);
assert_eq!(queue.reserved_bytes(), 500);
check("release_active_reserved_height");
assert_eq!(queue.claim_received(block::Height(2)), 100);
assert_eq!(queue.reserved_bytes(), 400);
check("claim_received");
queue.release_reserved_heights([block::Height(3)]);
assert_eq!(queue.reserved_bytes(), 300);
check("release_heights");
let released = queue.release_reserved_and_return_items([block::Height(4)]);
assert_eq!(released, 100);
assert_eq!(queue.reserved_bytes(), 200);
check("release_reserved_and_return_items");
let released = queue.advance_floor(block::Height(4));
assert_eq!(released, 0, "heights <= 4 owned no live reservation");
assert!(!queue.pending_contains(block::Height(1)));
assert!(!queue.in_flight_contains(block::Height(2)));
assert_eq!(queue.reserved_bytes(), 200);
check("advance_floor");
let released = queue.reset_above(block::Height(4));
assert_eq!(released, 200);
assert_eq!(queue.reserved_bytes(), 0);
assert!(!queue.in_flight_contains(block::Height(5)));
assert!(!queue.in_flight_contains(block::Height(6)));
check("reset_above");
}
#[test]
fn work_queue_advance_floor_drops_only_committed_prefix() {
let queue = work_queue_with(
0,
(1..=10).map(|h| needed(h, BlockSizeEstimate::Advertised(100))),
);
let _ = queue.take_in_range(block::Height(1), block::Height(10), 10);
assert_eq!(queue.mark_reserved((1..=10).map(block::Height)), 1000);
let released = queue.advance_floor(block::Height(4));
assert_eq!(released, 400, "reserved bytes for the dropped 1..=4 prefix");
for h in 1..=4 {
assert!(!queue.in_flight_contains(block::Height(h)));
}
for h in 5..=10 {
assert!(queue.in_flight_contains(block::Height(h)));
}
assert_eq!(queue.reserved_bytes(), 600);
assert_eq!(queue.reserved_bytes(), queue.reserved_bytes_scanned());
assert_eq!(queue.advance_floor(block::Height(2)), 0);
assert_eq!(queue.reserved_bytes(), 600);
}
#[test]
fn receipt_releases_wire_reservation_while_body_retained() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
assert_eq!(queue.reserved_bytes(), 100);
let reserved_estimate = queue
.release_active_reserved_height(block::Height(1))
.expect("active reserved height releases at receipt");
assert_eq!(reserved_estimate, 100);
budget.release(reserved_estimate);
assert_eq!(queue.reserved_bytes(), 0);
assert_eq!(budget.reserved(), 0);
assert!(
queue.in_flight_contains(block::Height(1)),
"the received height stays claimed; the body handoff owns it"
);
}
#[test]
fn floor_watchdog_skips_received_heights() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
let reserved_estimate = queue
.release_active_reserved_height(block::Height(1))
.expect("active reserved height releases at receipt");
budget.release(reserved_estimate);
assert_eq!(budget.reserved(), 0);
let outcome = queue.release_reserved_and_return_items_detailed([block::Height(1)]);
let watchdog_released = outcome.released_bytes;
budget.release(watchdog_released);
assert_eq!(
watchdog_released, 0,
"watchdog must not release a received height owned by the sequencer handoff"
);
assert!(
queue.in_flight_contains(block::Height(1)),
"the received height must not be requeued for a re-fetch"
);
assert_eq!(outcome.released_count, 1);
assert_eq!(outcome.returned_count, 0);
assert_eq!(outcome.missing_count, 0);
assert!(queue.in_flight_contains(block::Height(1)));
assert_eq!(budget.reserved(), 0);
assert_eq!(queue.advance_floor(block::Height(1)), 0);
}
#[test]
fn late_body_does_not_resurrect_charge() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
let outcome = queue.release_reserved_and_return_items_detailed([block::Height(1)]);
let watchdog_released = outcome.released_bytes;
budget.release(watchdog_released);
assert_eq!(outcome.returned_count, 1);
assert_eq!(outcome.released_count, 0);
assert_eq!(outcome.missing_count, 0);
assert_eq!(budget.reserved(), 0);
assert!(queue.pending_contains(block::Height(1)));
let duplicate =
queue.release_reserved_and_return_items_detailed([block::Height(1), block::Height(2)]);
assert_eq!(duplicate.already_pending_count, 1);
assert_eq!(duplicate.missing_count, 1);
assert_eq!(duplicate.min_height, Some(block::Height(1)));
assert_eq!(duplicate.max_height, Some(block::Height(2)));
assert_eq!(
queue.release_active_reserved_height(block::Height(1)),
None,
"a late body cannot release a claim the watchdog already released"
);
assert_eq!(budget.reserved(), 0);
}
#[test]
fn claim_received_does_not_orphan_a_charge() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
let old_charge = queue.claim_received(block::Height(1));
budget.release(old_charge);
assert_eq!(old_charge, 100);
assert_eq!(budget.reserved(), 0);
let released = queue.release_reserved_heights([block::Height(1)]);
budget.release(released);
assert_eq!(released, 0, "a received height owns no further charge");
assert_eq!(budget.reserved(), 0);
}
#[test]
fn release_reserved_mixed_received_and_reserved_conserves_budget() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(100)),
needed(2, BlockSizeEstimate::Advertised(100)),
],
);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(2), 2);
assert_eq!(taken.len(), 2);
assert!(budget.try_reserve(200));
assert_eq!(
queue.mark_reserved([block::Height(1), block::Height(2)]),
200
);
let reserved_estimate = queue
.release_active_reserved_height(block::Height(1))
.expect("active height releases at receipt");
assert_eq!(reserved_estimate, 100);
budget.release(reserved_estimate);
assert_eq!(budget.reserved(), 100);
let released = queue.advance_floor(block::Height(2));
budget.release(released);
assert_eq!(
released, 100,
"WorkQueue releases only the still-reserved height; received bodies hold no charge"
);
assert_eq!(budget.reserved(), 0);
}
#[test]
fn release_reserved_heights_skips_received_body_owned_by_sequencer() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(100)),
needed(2, BlockSizeEstimate::Advertised(100)),
],
);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(2), 2);
assert_eq!(taken.len(), 2);
assert!(budget.try_reserve(200));
assert_eq!(
queue.mark_reserved([block::Height(1), block::Height(2)]),
200
);
let estimate = queue
.release_active_reserved_height(block::Height(1))
.expect("height 1 is reserved");
assert_eq!(estimate, 100);
budget.release(estimate);
assert_eq!(budget.reserved(), 100);
let released = queue.release_reserved_heights([block::Height(1), block::Height(2)]);
budget.release(released);
assert_eq!(
released, 100,
"release_reserved_heights frees only the still-reserved estimate, exactly once"
);
assert!(queue.in_flight_contains(block::Height(1)));
assert_eq!(budget.reserved(), 0);
assert_eq!(queue.advance_floor(block::Height(2)), 0);
assert_eq!(budget.reserved(), 0);
}
#[test]
fn work_queue_take_does_not_clamp_high_to_floor() {
let queue = work_queue_with(
0,
(100..=104)
.map(|height| needed(height, BlockSizeEstimate::Advertised(100)))
.collect::<Vec<_>>(),
);
let taken = queue.take_in_range(
block::Height(100),
block::Height(104),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![100, 101, 102, 103, 104],
"heights far above the floor are takable; the floor never clamps the take"
);
}
#[test]
fn work_queue_preserves_work_item_estimate_through_take_and_return() {
let queue = work_queue_with(0, [needed(10, BlockSizeEstimate::Advertised(12_345))]);
let taken = queue.take_in_range(block::Height(10), block::Height(10), 1);
assert_eq!(taken.len(), 1);
assert_eq!(taken[0].1.estimated_bytes, 12_345);
assert_eq!(taken[0].1.hash, block::Hash([10; 32]));
queue.return_items([block::Height(10)]);
let retaken = queue.take_in_range(block::Height(10), block::Height(10), 1);
assert_eq!(
retaken[0].1.estimated_bytes, 12_345,
"return_items must restore the stored WorkItem unchanged"
);
}
#[test]
fn work_queue_advance_floor_and_reset_above_gc_both_maps() {
let queue = work_queue_with(
0,
(1..=6)
.map(|height| needed(height, BlockSizeEstimate::Advertised(100)))
.collect::<Vec<_>>(),
);
queue.take_in_range(block::Height(1), block::Height(2), 2);
assert_eq!(queue.in_flight_len(), 2);
assert_eq!(queue.pending_len(), 4);
queue.advance_floor(block::Height(3));
assert!(!queue.in_flight_contains(block::Height(1)));
assert!(!queue.pending_contains(block::Height(3)));
assert!(queue.pending_contains(block::Height(4)));
queue.take_in_range(block::Height(4), block::Height(4), 1); queue.reset_above(block::Height(4));
assert!(queue.in_flight_contains(block::Height(4)));
assert!(!queue.pending_contains(block::Height(5)));
assert!(!queue.pending_contains(block::Height(6)));
assert_eq!(queue.pending_len(), 0);
}
#[test]
fn work_queue_height_is_in_exactly_one_set() {
let queue = work_queue_with(0, [needed(10, BlockSizeEstimate::Advertised(100))]);
let in_one_set = |height: block::Height| -> usize {
usize::from(queue.pending_contains(height)) + usize::from(queue.in_flight_contains(height))
};
let h = block::Height(10);
assert_eq!(in_one_set(h), 1, "pending only after extend");
queue.take_in_range(h, h, 1);
assert_eq!(in_one_set(h), 1, "in_flight only after take");
queue.return_items([h]);
assert_eq!(in_one_set(h), 1, "pending only after return");
queue.take_in_range(h, h, 1);
queue.advance_floor(h);
assert_eq!(in_one_set(h), 0, "gone after the floor commits past it");
}
#[tokio::test]
async fn reactor_fill_loop_saturates_multiple_slots_in_one_pass() {
let config = fill_loop_mechanics_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
41,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: block::Hash([4; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tip_tx
.send((block::Height(4), block::Hash([4; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: block::Hash([3; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
]))
.await
.expect("needed metadata queues");
let mut heights = Vec::new();
for _ in 0..4 {
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(count, 1);
heights.push(start_height.0);
}
heights.sort_unstable();
assert_eq!(heights, vec![1, 2, 3, 4]);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_suppresses_duplicate_needed_block_query_until_response() {
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let best_hash = block::Hash([4; 32]);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("header-tip event queues");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("duplicate header-tip event queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"same pending needed-block query should not be dispatched twice",
);
handle
.send(BlockSyncEvent::NeededBlocks(Vec::new()))
.await
.expect("needed-block response queues");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("header-tip event after response queues");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
reactor_task.abort();
}
#[tokio::test]
async fn reactor_suppresses_needed_block_query_when_work_already_covers_tip() {
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let best_hash = block::Hash([4; 32]);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("header-tip event queues");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=4)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("test height fits u8"); 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed-block response queues");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("covered header-tip event queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"producer should not re-query when pending/in-flight work already covers the header tip",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_fill_loop_saturates_every_peer_window_not_just_one() {
let config = fill_loop_mechanics_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let mut peer_ids = Vec::new();
let mut peer_streams = Vec::new();
for byte in [41u8, 42, 43] {
let (peer_id, inbound, outbound) = connect_peer_with_status_message(
&service,
&mut actions,
byte,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(12),
tip_hash: block::Hash([12; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
peer_ids.push(peer_id);
peer_streams.push((inbound, outbound));
}
tip_tx
.send((block::Height(12), block::Hash([12; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=12)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([height as u8; 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed metadata queues");
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = peer_ids
.iter()
.cloned()
.zip(peer_streams.iter_mut().map(|(_, outbound)| outbound))
.collect();
let mut per_peer: HashMap<ZakuraPeerId, Vec<u32>> = HashMap::new();
for _ in 0..12 {
let (peer, start_height, count) = wait_for_getblocks_across(&mut outbound_by_peer).await;
assert_eq!(count, 1);
per_peer.entry(peer).or_default().push(start_height.0);
}
assert_eq!(
per_peer.len(),
3,
"all three peers must receive requests, not just the first; got {per_peer:?}"
);
for peer_id in &peer_ids {
let issued = per_peer
.get(peer_id)
.unwrap_or_else(|| panic!("peer {peer_id:?} received no requests: {per_peer:?}"));
assert_eq!(
issued.len(),
4,
"peer {peer_id:?} window of 4 was not saturated: {per_peer:?}"
);
}
let mut all_heights: Vec<u32> = per_peer.values().flatten().copied().collect();
all_heights.sort_unstable();
assert_eq!(
all_heights,
(1..=12).collect::<Vec<_>>(),
"every needed height must be requested exactly once across peers"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_budget_constrained_issuance_rotates_across_peers() {
let config = fill_loop_mechanics_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let mut peer_inbounds = HashMap::new();
let mut peer_outbounds = Vec::new();
for byte in [0x41u8, 0x42, 0x43] {
let (peer_id, inbound, outbound) = connect_peer_with_status_message(
&service,
&mut actions,
byte,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
peer_inbounds.insert(peer_id.clone(), inbound);
peer_outbounds.push((peer_id, outbound));
}
tip_tx
.send((block::Height(1), block::Hash([1; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
}]))
.await
.expect("needed metadata queues");
let range_unavailable = BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: 1,
}
.encode_frame()
.expect("RangeUnavailable frame encodes");
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = peer_outbounds
.iter_mut()
.map(|(peer_id, outbound)| (peer_id.clone(), outbound))
.collect();
for round in 0..6 {
let (peer, start_height, count) = wait_for_getblocks_across(&mut outbound_by_peer).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1, "fanout=1 yields single-block requests");
let inbound = peer_inbounds.get(&peer).unwrap_or_else(|| {
panic!("round {round}: served peer {peer:?} must be one of the connected peers")
});
inbound
.send(range_unavailable.clone())
.await
.expect("RangeUnavailable frame queues");
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_timeout_recovery_is_local_and_healthy_peer_keeps_filling() {
let mut config = fill_loop_mechanics_config();
config.request_timeout = Duration::from_millis(400);
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 64;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let status = || BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: block::Hash([2; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
};
let (peer_a, a_in, mut a_out) =
connect_peer_with_status_message(&service, &mut actions, 0x41, status()).await;
let (peer_b, b_in, mut b_out) =
connect_peer_with_status_message(&service, &mut actions, 0x42, status()).await;
let mut inbounds = HashMap::from([(peer_a.clone(), a_in), (peer_b.clone(), b_in)]);
tip_tx
.send((block::Height(2), block::Hash([2; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
]))
.await
.expect("needed metadata queues");
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> =
vec![(peer_a.clone(), &mut a_out), (peer_b.clone(), &mut b_out)];
let first = wait_for_getblocks_across(&mut outbound_by_peer).await;
let second = wait_for_getblocks_across(&mut outbound_by_peer).await;
let mut offered: HashMap<ZakuraPeerId, block::Height> = HashMap::new();
offered.insert(first.0.clone(), first.1);
offered.insert(second.0.clone(), second.1);
assert_eq!(
offered.len(),
2,
"both peers must be offered a height in the opening pass: {offered:?}"
);
let healthy = peer_b.clone();
let healthy_in = inbounds.remove(&healthy).expect("healthy peer inbound");
let healthy_offers = tokio::time::timeout(Duration::from_secs(3), async {
let mut count = 0usize;
loop {
let (peer, start_height, count_blocks) =
wait_for_getblocks_across(&mut outbound_by_peer).await;
if peer == healthy {
count += 1;
if count >= 2 {
break count;
}
healthy_in
.send(
BlockSyncMessage::RangeUnavailable {
start_height,
count: count_blocks,
}
.encode_frame()
.expect("RangeUnavailable frame encodes"),
)
.await
.expect("RangeUnavailable frame queues");
}
}
})
.await
.expect(
"the healthy peer must keep being offered shared work while the slow peer \
is in timeout recovery; a stall here means a straggler wedged the pass",
);
assert!(
healthy_offers >= 2,
"the healthy peer was filled repeatedly despite the slow peer's timeout recovery"
);
reactor_task.abort();
}
#[tokio::test]
async fn block_liveness_disconnects_silent_peer_and_traces_reason() {
let mut capture =
TraceCapture::for_test("block_liveness_disconnects_silent_peer_and_traces_reason")
.expect("trace capture initializes");
let mut config = immediate_body_download_config();
config.request_timeout = Duration::from_millis(400);
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 64;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let mut startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
startup.trace = ZakuraTrace::new(capture.tracer(), "01");
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(0x51);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
let connection_cancel = CancellationToken::new();
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
connection_cancel.clone(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), block::Hash([1; 32])))
.expect("tip watch is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
}]))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1);
tokio::time::timeout(Duration::from_secs(3), connection_cancel.cancelled())
.await
.expect("silent active peer is disconnected by block-progress liveness");
capture.flush().await;
let reader = capture.reader().expect("trace rows load");
reader.table("block_sync").assert_row(
bs_trace::BLOCK_GET_BLOCKS_SENT,
&[
("requests_without_block_progress", TraceValue::U64(1)),
("no_progress_request_cap", TraceValue::U64(1)),
("block_progress_proven", TraceValue::U64(0)),
],
);
reader.table("block_sync").assert_row(
bs_trace::BLOCK_PEER_PROTOCOL_REJECT,
&[(
bs_trace::REASON,
TraceValue::Str("block_sync_no_block_progress"),
)],
);
reactor_task.abort();
}
#[tokio::test]
async fn block_liveness_credits_late_unmatched_body_and_keeps_peer() {
let mut config = immediate_body_download_config();
config.request_timeout = Duration::from_millis(300);
config.floor_rescue_timeout = Duration::from_millis(120);
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 64;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(0x53);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
let connection_cancel = CancellationToken::new();
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
connection_cancel.clone(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: blocks[0].hash(),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1);
tokio::time::sleep(Duration::from_millis(200)).await;
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("late block frame queues");
let submitted = tokio::time::timeout(Duration::from_secs(2), async {
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => break block.coinbase_height(),
_ => continue,
}
}
})
.await
.expect("the late unmatched body is accepted and submitted");
assert_eq!(submitted, Some(block::Height(1)));
assert!(
tokio::time::timeout(Duration::from_millis(1500), connection_cancel.cancelled())
.await
.is_err(),
"a peer that delivered an accepted (late) body must not be parked as silent",
);
reactor_task.abort();
}
#[tokio::test]
async fn peer_emits_periodic_bbr_heartbeat_while_idle() {
let mut capture = TraceCapture::for_test("peer_emits_periodic_bbr_heartbeat_while_idle")
.expect("trace capture initializes");
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let mut startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
startup.trace = ZakuraTrace::new(capture.tracer(), "01");
let (_handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, _handle.clone());
let peer = peer(0x5b);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tokio::time::sleep(Duration::from_millis(100)).await;
capture.flush().await;
let reader = capture.reader().expect("trace rows load");
reader.table("block_sync").assert_row(
bs_trace::BLOCK_PEER_BBR,
&[
("bbr_reliability_permille", TraceValue::U64(1000)),
("block_progress_proven", TraceValue::U64(0)),
("bbr_phase", TraceValue::U64(0)),
],
);
reactor_task.abort();
}
#[test]
fn work_queue_estimate_clamps_hint_between_floor_and_max_block_bytes() {
use super::work_queue::DEFAULT_BS_SIZE_FLOOR_BYTES;
let queue = super::work_queue::WorkQueue::new(block::Height(0));
queue.extend([
needed(1, BlockSizeEstimate::Unknown),
needed(2, BlockSizeEstimate::Advertised(1)), needed(3, BlockSizeEstimate::Advertised(12_345)),
needed(4, BlockSizeEstimate::Confirmed(u32::MAX)), ]);
let item = |height| {
queue
.take_in_range(block::Height(height), block::Height(height), 1)
.pop()
.expect("height present")
.1
.estimated_bytes
};
assert_eq!(item(1), block::MAX_BLOCK_BYTES);
assert_eq!(item(2), DEFAULT_BS_SIZE_FLOOR_BYTES);
assert_eq!(item(3), 12_345);
assert_eq!(item(4), block::MAX_BLOCK_BYTES);
let tuned = super::work_queue::WorkQueue::new(block::Height(0));
tuned.set_estimate_floor_for_tests(100);
tuned.extend([
needed(10, BlockSizeEstimate::Unknown),
needed(11, BlockSizeEstimate::Advertised(50)), ]);
assert_eq!(
tuned
.take_in_range(block::Height(10), block::Height(10), 1)
.pop()
.unwrap()
.1
.estimated_bytes,
block::MAX_BLOCK_BYTES
);
assert_eq!(
tuned
.take_in_range(block::Height(11), block::Height(11), 1)
.pop()
.unwrap()
.1
.estimated_bytes,
100
);
}
#[test]
fn work_queue_diagnostics_report_runs_min_and_max() {
let queue = work_queue_with(
0,
[
needed(10, BlockSizeEstimate::Advertised(100)),
needed(11, BlockSizeEstimate::Advertised(100)),
needed(12, BlockSizeEstimate::Advertised(100)),
needed(20, BlockSizeEstimate::Advertised(100)),
needed(21, BlockSizeEstimate::Advertised(100)),
],
);
assert_eq!(queue.pending_run_count(), 2);
assert_eq!(queue.pending_len(), 5);
assert_eq!(queue.min_pending(), Some(block::Height(10)));
assert_eq!(queue.max_in_flight(), None);
queue.take_in_range(block::Height(20), block::Height(21), 2);
assert_eq!(queue.pending_run_count(), 1, "the 20..=21 run was taken");
assert_eq!(queue.min_pending(), Some(block::Height(10)));
assert_eq!(queue.max_in_flight(), Some(block::Height(21)));
assert_eq!(
queue.hash_for_height(block::Height(21)),
Some(block::Hash([21; 32]))
);
assert_eq!(
queue.hash_for_height(block::Height(12)),
Some(block::Hash([12; 32]))
);
assert_eq!(queue.hash_for_height(block::Height(99)), None);
}
#[test]
fn work_queue_keeps_pending_ordered_by_height() {
let queue = super::work_queue::WorkQueue::new(block::Height(0));
queue.extend([needed(20, BlockSizeEstimate::Advertised(100))]);
queue.extend([needed(10, BlockSizeEstimate::Advertised(100))]);
assert_eq!(queue.min_pending(), Some(block::Height(10)));
let taken = queue.take_in_range(block::Height(1), block::Height(30), 1);
assert_eq!(
taken[0].0,
block::Height(10),
"lower work must be taken before higher work"
);
}
#[test]
fn reorder_drains_only_contiguous_prefix_and_reports_dropped_bytes() {
let mut reorder = ReorderBuffer::new();
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
assert_eq!(
reorder.insert(block::Height(3), block.clone(), 300, peer(0)),
ReorderInsertResult::Inserted
);
assert!(reorder.drain_contiguous_prefix(block::Height(0)).is_empty());
assert_eq!(reorder.buffered_bytes(), 300);
assert_eq!(
reorder.insert(block::Height(1), block.clone(), 100, peer(0)),
ReorderInsertResult::Inserted
);
let released = reorder.drain_contiguous_prefix(block::Height(0));
assert_eq!(
released
.iter()
.map(|drained| (drained.height, drained.bytes))
.collect::<Vec<_>>(),
vec![(block::Height(1), 100)]
);
assert_eq!(reorder.buffered_bytes(), 300);
assert_eq!(
reorder.insert(block::Height(2), block.clone(), 200, peer(0)),
ReorderInsertResult::Inserted
);
let released = reorder.drain_contiguous_prefix(block::Height(1));
assert_eq!(
released
.iter()
.map(|drained| (drained.height, drained.bytes))
.collect::<Vec<_>>(),
vec![(block::Height(2), 200), (block::Height(3), 300)]
);
assert_eq!(reorder.buffered_bytes(), 0);
assert_eq!(
reorder.insert(block::Height(2), block.clone(), 200, peer(0)),
ReorderInsertResult::Inserted
);
assert_eq!(
reorder.insert(block::Height(3), block, 300, peer(0)),
ReorderInsertResult::Inserted
);
assert_eq!(reorder.drop_from(block::Height(3)), 300);
assert_eq!(reorder.buffered_bytes(), 200);
assert_eq!(
reorder.decoded_attributed_memory_bytes(),
reorder.decoded_attributed_memory_bytes_scanned()
);
assert_eq!(reorder.drop_through(block::Height(2)), 200);
assert_eq!(reorder.buffered_bytes(), 0);
assert_eq!(reorder.decoded_attributed_memory_bytes(), 0);
assert_eq!(
reorder.insert(
block::Height(3),
mainnet_block(&BLOCK_MAINNET_1_BYTES),
300,
peer(0)
),
ReorderInsertResult::Inserted
);
assert_eq!(reorder.clear(), 300);
assert_eq!(reorder.buffered_bytes(), 0);
assert_eq!(reorder.decoded_attributed_memory_bytes(), 0);
}
fn test_sequencer(verified_tip: u32, submitted_apply_limit: usize) -> Sequencer {
Sequencer::new(block::Height(verified_tip), submitted_apply_limit)
}
#[test]
fn sequencer_accept_body_buffers_then_reports_duplicate() {
let mut seq = test_sequencer(0, 4);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let hash = block.hash();
let decoded_attributed_memory_size_bytes = block.attributed_memory_size_bytes();
assert_eq!(
seq.accept_body(block::Height(1), hash, block.clone(), 100, peer(0)),
AcceptOutcome::Buffered {
covered: block::Height(1)
}
);
assert!(seq.reorder_contains(block::Height(1)));
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
decoded_attributed_memory_size_bytes
);
assert_eq!(
seq.accept_body(block::Height(1), hash, block, 100, peer(0)),
AcceptOutcome::Redundant { release_bytes: 100 }
);
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
decoded_attributed_memory_size_bytes
);
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
seq.reorder_decoded_attributed_memory_bytes_scanned()
);
}
#[test]
fn sequencer_retains_raw_bytes_for_non_contiguous_backlog() {
let mut seq = test_sequencer(0, 4);
let blocks = mainnet_blocks_1_to_3();
let block1 = blocks[0].clone();
let block2 = blocks[1].clone();
let distinguishable_decoded_block2 = forked_block(&block2, 99);
assert_ne!(distinguishable_decoded_block2.hash(), block2.hash());
assert_eq!(
distinguishable_decoded_block2.coinbase_height(),
block2.coinbase_height()
);
let block2_body = BufferedBlockBody::from_decoded_block(
distinguishable_decoded_block2.clone(),
Some(raw_block_payload(&block2)),
);
assert_eq!(
seq.accept_buffered_body(
block::Height(2),
block2.hash(),
block2.header.previous_block_hash,
block2_body,
200,
peer(0)
),
AcceptOutcome::Buffered {
covered: block::Height(2)
}
);
assert!(seq.drain_ready_into_applying().is_empty());
assert!(seq.reorder_contains(block::Height(2)));
assert_eq!(seq.reorder_decoded_attributed_memory_bytes(), 0);
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
seq.reorder_decoded_attributed_memory_bytes_scanned()
);
assert_eq!(
seq.accept_body(
block::Height(1),
block1.hash(),
block1.clone(),
100,
peer(0),
),
AcceptOutcome::Buffered {
covered: block::Height(1)
}
);
assert_eq!(
seq.drain_ready_into_applying(),
vec![block::Height(1), block::Height(2)]
);
assert_eq!(seq.applying_hash(block::Height(2)), Some(block2.hash()));
let submitted = seq
.prepare_submit(block::Height(2))
.expect("height 2 is applying");
assert_eq!(submitted.block.hash(), block2.hash());
assert_ne!(
submitted.block.hash(),
distinguishable_decoded_block2.hash()
);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
block1
.attributed_memory_size_bytes()
.saturating_add(block2.attributed_memory_size_bytes())
);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
seq.applying_decoded_attributed_memory_bytes_scanned()
);
}
#[test]
fn sequencer_bounds_decoded_bodies_to_submission_window() {
let limit = 2;
let mut seq = test_sequencer(0, limit);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
let body =
BufferedBlockBody::from_decoded_block(block.clone(), Some(raw_block_payload(block)));
assert_eq!(
seq.accept_buffered_body(
height,
block.hash(),
block.header.previous_block_hash,
body,
100,
peer(0)
),
AcceptOutcome::Buffered { covered: height }
);
}
assert_eq!(
seq.drain_ready_into_applying(),
vec![block::Height(1), block::Height(2), block::Height(3)]
);
assert_eq!(seq.applying_len(), 3);
assert!(
seq.decoded_applying_count() <= limit,
"decoded applying bodies ({}) exceed the submission window ({limit})",
seq.decoded_applying_count(),
);
for height in seq.submittable_heights() {
seq.prepare_submit(height).expect("height is applying");
}
assert_eq!(seq.in_flight_submission_count(), limit);
assert_eq!(seq.decoded_applying_count(), limit);
seq.unsubmit(block::Height(1), 1);
assert_eq!(seq.decoded_applying_count(), limit - 1);
}
#[test]
fn sequencer_applying_counters_match_scan_across_transitions() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
let check = |seq: &Sequencer, label: &str| {
assert_eq!(
seq.applying_buffered_bytes(),
seq.applying_buffered_bytes_scanned(),
"applying_buffered_bytes drifted after {label}"
);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
seq.applying_decoded_attributed_memory_bytes_scanned(),
"applying_decoded_attributed_memory_bytes drifted after {label}"
);
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
seq.reorder_decoded_attributed_memory_bytes_scanned(),
"reorder_decoded_attributed_memory_bytes drifted after {label}"
);
assert_eq!(
seq.in_flight_submission_count(),
seq.in_flight_submission_count_scanned(),
"in_flight_submission_count drifted after {label}"
);
assert_eq!(
seq.in_flight_submission_bytes(),
seq.in_flight_submission_bytes_scanned(),
"in_flight_submission_bytes drifted after {label}"
);
assert_eq!(
seq.unsubmitted_applying_count(),
seq.applying_len() - seq.attached_submission_count_scanned(),
"unsubmitted derivation wrong after {label}"
);
};
for (i, block) in blocks.iter().enumerate() {
let height = (i + 1) as u32;
seq.accept_body(
block::Height(height),
block.hash(),
block.clone(),
100 * u64::from(height),
peer(0),
);
}
assert_eq!(
seq.drain_ready_into_applying(),
vec![block::Height(1), block::Height(2), block::Height(3)]
);
assert_eq!(seq.applying_buffered_bytes(), 600);
assert_eq!(seq.in_flight_submission_count(), 0);
check(&seq, "drain");
let item1 = seq
.prepare_submit(block::Height(1))
.expect("height 1 applying");
let item2 = seq
.prepare_submit(block::Height(2))
.expect("height 2 applying");
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(seq.in_flight_submission_bytes(), 300);
assert_eq!(seq.unsubmitted_applying_count(), 1);
check(&seq, "submit 1,2");
seq.unsubmit(block::Height(1), item1.token);
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.in_flight_submission_bytes(), 200);
check(&seq, "unsubmit 1");
seq.remove_applying(block::Height(2));
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.in_flight_submission_bytes(), 200);
assert_eq!(seq.applying_buffered_bytes(), 400);
check(&seq, "detach submitted 2");
assert!(seq.finish_submission(item2.token, item2.height, item2.hash));
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.in_flight_submission_bytes(), 0);
check(&seq, "finish detached 2");
seq.advance_verified_tip(block::Height(1), true);
assert_eq!(seq.applying_buffered_bytes(), 300);
check(&seq, "advance_verified_tip");
seq.reset_to(block::Height(0), false);
assert_eq!(seq.applying_buffered_bytes(), 0);
assert_eq!(seq.applying_decoded_attributed_memory_bytes(), 0);
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.in_flight_submission_bytes(), 0);
check(&seq, "reset");
}
#[tokio::test]
async fn sequencer_stale_checkpoint_completions_refill_full_submission_window() {
const BLOCK_COUNT: u32 = 403;
const CHANNEL_TIMEOUT: Duration = Duration::from_secs(2);
const MISSING_SUBMISSION_TIMEOUT: Duration = Duration::from_millis(100);
let submission_limit = MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES;
let submission_limit_u64 =
u64::try_from(submission_limit).expect("the checkpoint submission limit fits in u64");
let body_channel_capacity = usize::try_from(BLOCK_COUNT).expect("403 test bodies fit in usize");
assert_eq!(submission_limit, 401);
let blocks = fake_sequential_blocks(BLOCK_COUNT);
let frontiers = BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
};
let body_input_bytes = Arc::new(std::sync::atomic::AtomicU64::new(0));
let body_input_decoded_attributed_memory_bytes = Arc::new(std::sync::atomic::AtomicU64::new(0));
let (body_tx, body_rx) = mpsc::channel(body_channel_capacity);
let (control_tx, control_rx) = mpsc::unbounded_channel();
let (actions_tx, mut actions_rx) = mpsc::channel(submission_limit + 128);
let (view_tx, mut view_rx) = watch::channel(initial_view(frontiers));
let task = SequencerTask::new(
Sequencer::new(block::Height(0), submission_limit),
ByteBudget::new(1),
Arc::new(WorkQueue::new(block::Height(0))),
actions_tx,
ThroughputMeter::new(Instant::now()),
frontiers,
body_rx,
control_rx,
body_input_bytes.clone(),
body_input_decoded_attributed_memory_bytes.clone(),
view_tx,
CHANNEL_TIMEOUT,
ZakuraTrace::noop(),
);
let task = tokio::spawn(task.run());
for (index, block) in blocks.iter().enumerate() {
let height =
block::Height(u32::try_from(index + 1).expect("403 test block indices fit in u32"));
body_tx
.send(SequencedBody::new_queued(
height,
block.hash(),
block.header.previous_block_hash,
BufferedBlockBody::from_decoded_block(
block.clone(),
Some(raw_block_payload(block)),
),
u64::from(block_size(block)),
peer(1),
Instant::now(),
body_input_bytes.clone(),
body_input_decoded_attributed_memory_bytes.clone(),
))
.await
.expect("sequencer body channel remains live");
}
time::timeout(CHANNEL_TIMEOUT, async {
loop {
let view = *view_rx.borrow_and_update();
if view.applying_len == u64::from(BLOCK_COUNT)
&& view.in_flight_submission_count == submission_limit_u64
&& view.unsubmitted_applying_count == 2
{
break;
}
view_rx
.changed()
.await
.expect("sequencer view remains live");
}
})
.await
.expect("all bodies reach applying and fill the initial submission window");
let mut initial_submissions = Vec::with_capacity(submission_limit);
while initial_submissions.len() < submission_limit {
let action = time::timeout(CHANNEL_TIMEOUT, actions_rx.recv())
.await
.expect("initial submission action arrives")
.expect("sequencer action channel remains live");
match action {
BlockSyncAction::SubmitBlock { token, block } => {
initial_submissions.push((token, block));
}
action => panic!("unexpected action before initial submissions complete: {action:?}"),
}
}
for (index, (_, block)) in initial_submissions.iter().enumerate() {
assert_eq!(
block.coinbase_height(),
Some(block::Height(
u32::try_from(index + 1).expect("401 submission indices fit in u32")
)),
"initial submissions must be ordered through the full checkpoint window",
);
}
let (token_1, block_1) = &initial_submissions[0];
control_tx
.send(SequencerControlInput::ApplyFinished {
token: *token_1,
height: block::Height(1),
hash: block_1.hash(),
result: BlockApplyResult::Committed,
local_frontier: Some(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: blocks[2].hash(),
}),
})
.expect("height 1 completion queues");
let block_402 = match time::timeout(CHANNEL_TIMEOUT, actions_rx.recv())
.await
.expect("height 402 submission arrives")
.expect("sequencer action channel remains live")
{
BlockSyncAction::SubmitBlock { block, .. } => block,
action => panic!("unexpected action before height 402 submission: {action:?}"),
};
assert_eq!(block_402.coinbase_height(), Some(block::Height(402)));
for (height, (token, block)) in [
(block::Height(2), &initial_submissions[1]),
(block::Height(3), &initial_submissions[2]),
] {
control_tx
.send(SequencerControlInput::ApplyFinished {
token: *token,
height,
hash: block.hash(),
result: BlockApplyResult::Committed,
local_frontier: Some(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: blocks[2].hash(),
}),
})
.expect("stale checkpoint completion queues");
}
let submit_403 = time::timeout(MISSING_SUBMISSION_TIMEOUT, async {
loop {
let action = actions_rx
.recv()
.await
.expect("sequencer action channel remains live");
if let BlockSyncAction::SubmitBlock { block, .. } = action {
assert_eq!(block.coinbase_height(), Some(block::Height(403)));
break;
}
}
})
.await;
let diagnostic = *view_rx.borrow();
assert!(
submit_403.is_ok(),
"height 403 must refill the checkpoint submission window; \
in_flight_submission_count = {}, unsubmitted_applying_count = {}",
diagnostic.in_flight_submission_count,
diagnostic.unsubmitted_applying_count,
);
time::timeout(CHANNEL_TIMEOUT, async {
loop {
let view = *view_rx.borrow_and_update();
if view.in_flight_submission_count == 400 && view.unsubmitted_applying_count == 0 {
break;
}
view_rx
.changed()
.await
.expect("sequencer view remains live");
}
})
.await
.expect("stale completions settle with a refilled checkpoint submission window");
task.abort();
}
#[test]
fn sequencer_accept_body_rejects_at_or_below_floor() {
let mut seq = test_sequencer(5, 4);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
assert_eq!(
seq.accept_body(block::Height(5), block.hash(), block.clone(), 100, peer(0)),
AcceptOutcome::Redundant { release_bytes: 100 }
);
assert!(!seq.reorder_contains(block::Height(5)));
}
#[test]
fn sequencer_drains_contiguous_prefix_into_applying_and_advances_floor() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
seq.accept_body(
block::Height(1),
blocks[0].hash(),
blocks[0].clone(),
100,
peer(0),
);
seq.accept_body(
block::Height(3),
blocks[2].hash(),
blocks[2].clone(),
300,
peer(0),
);
assert_eq!(seq.drain_ready_into_applying(), vec![block::Height(1)]);
assert_eq!(seq.floor(), block::Height(1));
assert!(seq.applying_contains(block::Height(1)));
assert_eq!(seq.applying_len(), 1);
seq.accept_body(
block::Height(2),
blocks[1].hash(),
blocks[1].clone(),
200,
peer(0),
);
assert_eq!(
seq.drain_ready_into_applying(),
vec![block::Height(2), block::Height(3)]
);
assert_eq!(seq.floor(), block::Height(3));
assert_eq!(seq.reorder_len(), 0);
}
#[test]
fn sequencer_submits_within_window_and_rolls_back_on_unsubmit() {
let mut seq = test_sequencer(0, 2);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(height, block.hash(), block.clone(), 100, peer(0));
}
assert_eq!(seq.drain_ready_into_applying().len(), 3);
assert_eq!(
seq.submittable_heights(),
vec![block::Height(1), block::Height(2)]
);
let item1 = seq.prepare_submit(block::Height(1)).expect("applying at 1");
let item2 = seq.prepare_submit(block::Height(2)).expect("applying at 2");
assert_eq!((item1.token, item2.token), (1, 2));
assert_eq!(seq.in_flight_submission_count(), 2);
assert!(seq.submittable_heights().is_empty());
seq.unsubmit(block::Height(2), item2.token);
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.submittable_heights(), vec![block::Height(2)]);
}
#[test]
fn sequencer_completed_duplicate_releases_attached_decode_window_slot() {
let mut seq = test_sequencer(0, 1);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().take(2).enumerate() {
let height = block::Height(index as u32 + 1);
let body =
BufferedBlockBody::from_decoded_block(block.clone(), Some(raw_block_payload(block)));
seq.accept_buffered_body(
height,
block.hash(),
block.header.previous_block_hash,
body,
100,
peer(0),
);
}
seq.drain_ready_into_applying();
let item = seq
.prepare_submit(block::Height(1))
.expect("height 1 is applying");
seq.record_submitted_apply(item.height, item.hash);
assert_eq!(seq.in_flight_submission_count(), 1);
assert!(seq.submittable_heights().is_empty());
assert!(seq.finish_attached_submission(item.token, item.height, item.hash));
assert!(seq.applying_contains(block::Height(1)));
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.decoded_applying_count(), 0);
assert_eq!(seq.submittable_heights(), vec![block::Height(2)]);
}
#[test]
fn sequencer_records_and_decrements_submitted_applies() {
let mut seq = test_sequencer(0, 4);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let hash = block.hash();
assert!(!seq.has_submitted_apply(block::Height(1), hash));
seq.accept_body(block::Height(1), hash, block, 100, peer(0));
seq.drain_ready_into_applying();
let item = seq
.prepare_submit(block::Height(1))
.expect("height 1 is applying");
seq.record_submitted_apply(item.height, item.hash);
assert!(seq.has_submitted_apply(block::Height(1), hash));
assert!(seq.submitted_contains(block::Height(1)));
seq.remove_applying(item.height);
assert!(seq.finish_submission(item.token, item.height, item.hash));
assert!(!seq.has_submitted_apply(block::Height(1), hash));
assert!(!seq.submitted_contains(block::Height(1)));
}
#[test]
fn sequencer_frontier_release_keeps_in_flight_submission_charged_until_completion() {
let mut seq = test_sequencer(0, 1);
let blocks = mainnet_blocks_1_to_3();
let decoded_attributed_memory_bytes = blocks
.iter()
.map(|block| block.attributed_memory_size_bytes())
.fold(0u64, u64::saturating_add);
let remaining_decoded_attributed_memory_bytes = blocks
.iter()
.skip(1)
.map(|block| block.attributed_memory_size_bytes())
.fold(0u64, u64::saturating_add);
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(height, block.hash(), block.clone(), 100, peer(0));
}
seq.drain_ready_into_applying();
let item = seq
.prepare_submit(block::Height(1))
.expect("height 1 is applying");
seq.record_submitted_apply(item.height, item.hash);
assert!(seq.submitted_contains(block::Height(1)));
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes
);
assert!(
seq.submittable_heights().is_empty(),
"submitted-apply window is full"
);
assert_eq!(seq.release_applied_through(block::Height(1)), 100);
assert!(!seq.submitted_contains(block::Height(1)));
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes,
"detached driver-owned decoded memory remains charged"
);
assert!(
seq.submittable_heights().is_empty(),
"detached driver submission still occupies the decode window"
);
assert!(!seq.finish_submission(item.token, item.height, block::Hash([99; 32])));
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes
);
assert!(seq.finish_submission(item.token, item.height, item.hash));
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
remaining_decoded_attributed_memory_bytes
);
assert_eq!(seq.submittable_heights(), vec![block::Height(2)]);
}
#[test]
fn sequencer_advance_verified_tip_releases_bytes_and_reports_change() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
seq.accept_body(
block::Height(1),
blocks[0].hash(),
blocks[0].clone(),
100,
peer(0),
);
seq.accept_body(
block::Height(2),
blocks[1].hash(),
blocks[1].clone(),
200,
peer(0),
);
let advance = seq.advance_verified_tip(block::Height(2), false);
assert!(advance.changed);
assert_eq!(advance.release_bytes, 300);
assert_eq!(seq.verified_tip(), block::Height(2));
assert!(seq.floor() >= block::Height(2));
let advance = seq.advance_verified_tip(block::Height(2), false);
assert!(!advance.changed);
assert_eq!(advance.release_bytes, 0);
}
#[test]
fn sequencer_reset_clears_buffers_and_pins_floor_and_tip() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
seq.accept_body(
block::Height(1),
blocks[0].hash(),
blocks[0].clone(),
100,
peer(0),
);
seq.drain_ready_into_applying();
seq.accept_body(
block::Height(2),
blocks[1].hash(),
blocks[1].clone(),
200,
peer(0),
);
let released = seq.reset_to(block::Height(0), false);
assert_eq!(released, 300);
assert_eq!(seq.floor(), block::Height(0));
assert_eq!(seq.verified_tip(), block::Height(0));
assert_eq!(seq.applying_len(), 0);
assert_eq!(seq.reorder_len(), 0);
}
#[test]
fn sequencer_reset_keeps_detached_submissions_charged_until_matching_completions() {
let mut seq = test_sequencer(0, 2);
let blocks = mainnet_blocks_1_to_3();
let decoded_attributed_memory_bytes = blocks
.iter()
.take(2)
.map(|block| block.attributed_memory_size_bytes())
.fold(0u64, u64::saturating_add);
for (index, block) in blocks.iter().take(2).enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(height, block.hash(), block.clone(), 100, peer(0));
}
seq.drain_ready_into_applying();
let old_items: Vec<_> = seq
.submittable_heights()
.into_iter()
.map(|height| {
let item = seq.prepare_submit(height).expect("height is applying");
seq.record_submitted_apply(item.height, item.hash);
item
})
.collect();
assert_eq!(old_items.len(), 2);
assert_eq!(seq.in_flight_submission_count(), 2);
seq.reset_to(block::Height(0), false);
assert_eq!(seq.applying_len(), 0);
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(seq.in_flight_submission_bytes(), 200);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes,
"reset must retain detached driver-owned decoded memory charges"
);
for (index, block) in blocks.iter().take(2).enumerate() {
let replacement = forked_block(block, 100 + index as u8);
let height = block::Height(index as u32 + 1);
seq.accept_body(height, replacement.hash(), replacement, 100, peer(1));
}
seq.drain_ready_into_applying();
assert!(
seq.submittable_heights().is_empty(),
"replacement bodies must not exceed the live decode window"
);
let decoded_attributed_memory_bytes_before_completion =
seq.applying_decoded_attributed_memory_bytes();
let first = &old_items[0];
assert!(
!seq.finish_submission(first.token, first.height, block::Hash([99; 32])),
"a mismatched completion must not release a detached charge"
);
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes_before_completion
);
assert!(seq.finish_submission(first.token, first.height, first.hash));
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes_before_completion
.saturating_sub(blocks[0].attributed_memory_size_bytes())
);
assert_eq!(seq.submittable_heights().len(), 1);
let second = &old_items[1];
assert!(seq.finish_submission(second.token, second.height, second.hash));
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes_before_completion
.saturating_sub(decoded_attributed_memory_bytes)
);
assert_eq!(seq.submittable_heights().len(), 2);
}
#[test]
fn sequencer_reject_drops_successors_and_rolls_floor_back() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(height, block.hash(), block.clone(), 100, peer(0));
}
seq.drain_ready_into_applying();
assert_eq!(seq.floor(), block::Height(3));
let released = seq.release_applying_blocks_from(block::Height(2));
assert_eq!(released, 200);
assert!(seq.applying_contains(block::Height(1)));
assert!(!seq.applying_contains(block::Height(2)));
seq.reset_floor_below(block::Height(2));
assert_eq!(seq.floor(), block::Height(1));
assert_eq!(seq.release_applied_through(block::Height(1)), 100);
assert_eq!(seq.applying_len(), 0);
}
#[test]
fn sequencer_rejection_release_keeps_detached_submissions_charged() {
let mut seq = test_sequencer(0, 2);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(
height,
block.hash(),
block.clone(),
100 * (index as u64 + 1),
peer(0),
);
}
seq.drain_ready_into_applying();
let item2 = seq
.prepare_submit(block::Height(2))
.expect("height 2 applying");
let item3 = seq
.prepare_submit(block::Height(3))
.expect("height 3 applying");
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(seq.in_flight_submission_bytes(), 200 + 300);
let released = seq.release_applying_blocks_from(block::Height(2));
assert_eq!(released, 500);
assert_eq!(seq.applying_len(), 1);
assert_eq!(seq.applying_buffered_bytes(), 100);
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(seq.in_flight_submission_bytes(), 500);
assert!(seq.submittable_heights().is_empty());
assert_eq!(
seq.applying_buffered_bytes(),
seq.applying_buffered_bytes_scanned()
);
assert_eq!(
seq.in_flight_submission_count(),
seq.in_flight_submission_count_scanned()
);
assert_eq!(
seq.in_flight_submission_bytes(),
seq.in_flight_submission_bytes_scanned()
);
assert!(seq.finish_submission(item2.token, item2.height, item2.hash));
assert_eq!(seq.submittable_heights(), vec![block::Height(1)]);
assert!(seq.finish_submission(item3.token, item3.height, item3.hash));
assert_eq!(seq.in_flight_submission_count(), 0);
}
#[test]
fn sequencer_unsubmit_ignores_stale_or_mismatched_token() {
let mut seq = test_sequencer(0, 4);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
seq.accept_body(block::Height(1), block.hash(), block.clone(), 100, peer(0));
seq.drain_ready_into_applying();
let item = seq
.prepare_submit(block::Height(1))
.expect("height 1 applying");
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.in_flight_submission_bytes(), 100);
seq.unsubmit(block::Height(1), item.token + 1);
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.in_flight_submission_bytes(), 100);
assert_eq!(
seq.in_flight_submission_count(),
seq.in_flight_submission_count_scanned()
);
assert_eq!(
seq.in_flight_submission_bytes(),
seq.in_flight_submission_bytes_scanned()
);
seq.unsubmit(block::Height(1), item.token);
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.in_flight_submission_bytes(), 0);
seq.unsubmit(block::Height(1), item.token);
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.in_flight_submission_bytes(), 0);
assert_eq!(
seq.in_flight_submission_count(),
seq.in_flight_submission_count_scanned()
);
assert_eq!(
seq.in_flight_submission_bytes(),
seq.in_flight_submission_bytes_scanned()
);
}
#[test]
fn sequencer_keeps_whole_body_for_contiguous_height() {
let mut seq = test_sequencer(0, 4);
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let distinguishable_decoded_block1 = forked_block(&block1, 99);
assert_ne!(distinguishable_decoded_block1.hash(), block1.hash());
let body = BufferedBlockBody::from_decoded_block(
distinguishable_decoded_block1.clone(),
Some(raw_block_payload(&block1)),
);
assert_eq!(
seq.accept_buffered_body(
block::Height(1),
block1.hash(),
block1.header.previous_block_hash,
body,
100,
peer(0)
),
AcceptOutcome::Buffered {
covered: block::Height(1)
}
);
assert_eq!(seq.drain_ready_into_applying(), vec![block::Height(1)]);
let submitted = seq
.prepare_submit(block::Height(1))
.expect("height 1 is applying");
assert_eq!(
submitted.block.hash(),
distinguishable_decoded_block1.hash()
);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
distinguishable_decoded_block1.attributed_memory_size_bytes()
);
}
#[test]
fn reorder_fuzzes_arrival_order_as_parent_first() {
let orders = [
[1, 2, 3, 4],
[4, 3, 2, 1],
[2, 4, 1, 3],
[3, 1, 4, 2],
[2, 1, 4, 3],
];
for order in orders {
let mut reorder = ReorderBuffer::new();
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let mut tip = block::Height(0);
let mut released_all = Vec::new();
for height in order {
assert_eq!(
reorder.insert(block::Height(height), block.clone(), 100, peer(0)),
ReorderInsertResult::Inserted
);
for drained in reorder.drain_contiguous_prefix(tip) {
assert_eq!(drained.height, block::Height(tip.0 + 1));
tip = drained.height;
released_all.push(drained.height);
}
}
assert_eq!(
released_all,
vec![
block::Height(1),
block::Height(2),
block::Height(3),
block::Height(4)
]
);
assert_eq!(reorder.buffered_bytes(), 0);
}
}
const THREE_BLOCK_ESTIMATE: u64 = 1_000;
fn outstanding_three_block_range(budget: &mut ByteBudget) -> OutstandingBlockRange {
let request = BlockRangeRequest {
start_height: block::Height(1),
count: 3,
anchor_hash: block::Hash([1; 32]),
estimated_bytes: THREE_BLOCK_ESTIMATE * 3,
expected_blocks: vec![
ExpectedBlock {
height: block::Height(1),
hash: block::Hash([1; 32]),
estimated_bytes: 1_000,
},
ExpectedBlock {
height: block::Height(2),
hash: block::Hash([2; 32]),
estimated_bytes: 1_000,
},
ExpectedBlock {
height: block::Height(3),
hash: block::Hash([3; 32]),
estimated_bytes: 1_000,
},
],
};
assert!(budget.try_reserve(request.estimated_bytes));
let now = Instant::now();
OutstandingBlockRange {
request,
queued_at: now,
deadline: now,
delivery_snapshot: test_delivery_snapshot(now),
delivered_bytes: 0,
received: ReceivedBlockTracker::default(),
}
}
#[test]
fn outstanding_range_accumulates_delivered_bytes_for_bbr_sample() {
let mut budget = ByteBudget::new(THREE_BLOCK_ESTIMATE * 3);
let mut outstanding = outstanding_three_block_range(&mut budget);
for (height, bytes) in [
(block::Height(1), 700),
(block::Height(2), 800),
(block::Height(3), 900),
] {
outstanding.record_body_bytes(bytes);
outstanding.mark_received(height);
}
assert!(outstanding.is_complete());
assert_eq!(outstanding.delivered_bytes, 700 + 800 + 900);
}
#[test]
fn block_budget_ledger_releases_reserved_exactly_once() {
let mut ledger = BlockBudgetLedger::reserved(1_000);
assert!(ledger.is_reserved());
assert_eq!(ledger.reserved_charge(), 1_000);
assert_eq!(ledger.release_reserved(), 1_000);
assert!(!ledger.is_reserved());
assert_eq!(ledger.reserved_charge(), 0);
assert_eq!(ledger.release_reserved(), 0);
let mut released = BlockBudgetLedger::Released;
assert!(!released.is_reserved());
assert_eq!(released.release_reserved(), 0);
}
#[test]
fn budget_audit_catches_injected_drift() {
let mut budget = ByteBudget::new(1_000);
assert!(budget.try_reserve(400));
assert!(budget.audit(400, "test matching audit"));
assert!(budget.audit(300, "test transient handoff excess"));
assert!(
!budget.audit(500, "test injected drift"),
"audit reports a budget shortfall as drift"
);
budget.release(400);
assert!(budget.audit(0, "test released audit"));
}
#[test]
fn received_body_admission_ignores_request_budget() {
let config = ZakuraBlockSyncConfig {
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(600),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 0,
reorder_buffered_blocks: 0,
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 0,
};
assert!(super::admission::admit_received_body(
&config,
&snapshot,
block::Height(411),
2_000_000,
));
assert!(super::admission::admit_received_body(
&config,
&snapshot,
block::Height(602),
1_000,
));
assert!(!super::admission::admit_received_body(
&config,
&snapshot,
block::Height(602),
1_001,
));
}
proptest::proptest! {
#![proptest_config(proptest::test_runner::Config::with_cases(256))]
#[test]
fn admit_respects_lookahead_bounds(
reorder_bytes in 0u64..2_000,
applying_bytes in 0u64..2_000,
input_bytes in 0u64..2_000,
reserved_bytes in 0u64..2_000,
reorder_blocks in proptest::prop_oneof![
0u64..20,
proptest::strategy::Just(super::admission::LOOKAHEAD_BLOCK_HARD_CAP),
],
applying_blocks in 0u64..20,
reserved_blocks in 0u64..20,
) {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(600),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: reorder_bytes,
reorder_buffered_blocks: reorder_blocks,
applying_buffered_bytes: applying_bytes,
applying_buffered_blocks: applying_blocks,
sequencer_input_queued_bytes: input_bytes,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: reserved_bytes,
reserved_above_floor_blocks: reserved_blocks,
budget_available: 64_000_000,
};
let factor = super::admission::DESERIALIZED_MEM_FACTOR;
let estimated_resident = reorder_bytes
.saturating_add(applying_bytes)
.saturating_add(input_bytes)
.saturating_add(reserved_bytes)
.saturating_add(input_bytes.saturating_mul(factor));
let held_blocks = reorder_blocks
.saturating_add(applying_blocks)
.saturating_add(reserved_blocks);
let effective = config.effective_max_reorder_lookahead_bytes();
let above = super::admission::admit(
&config,
snapshot,
block::Height(602),
block::Height(100_000),
1_000,
);
if estimated_resident >= effective
|| held_blocks >= super::admission::LOOKAHEAD_BLOCK_HARD_CAP
|| effective - estimated_resident == 0
{
prop_assert_eq!(above, super::admission::AdmissionOutcome::LookaheadAtCap);
} else {
let remaining_wire = effective - estimated_resident;
let expected = snapshot.budget_available.min(remaining_wire).min(1_000);
match above {
super::admission::AdmissionOutcome::Admit(grant) => {
prop_assert_eq!(grant.max_request_bytes, expected);
prop_assert_eq!(grant.take_high, block::Height(100_000));
prop_assert!(
estimated_resident
.saturating_add(grant.max_request_bytes)
<= effective
);
}
other => prop_assert!(false, "expected a gated grant, got {:?}", other),
}
}
match super::admission::admit(
&config,
snapshot,
block::Height(11),
block::Height(100_000),
1_000,
) {
super::admission::AdmissionOutcome::Admit(grant) => {
prop_assert_eq!(grant.max_request_bytes, 1_000);
prop_assert_eq!(grant.take_high, block::Height(411));
}
other => prop_assert!(
false,
"the commit window remains admitted while budget is available, got {:?}",
other
),
}
}
}
proptest::proptest! {
#![proptest_config(proptest::test_runner::Config::with_cases(256))]
#[test]
fn block_budget_ledger_mirrors_byte_budget(
estimate in 1u64..1_000_000,
release_before_receipt in proptest::bool::ANY,
) {
let mut ledger = BlockBudgetLedger::reserved(estimate);
let mut budget = ByteBudget::new(u64::MAX);
prop_assert!(budget.try_reserve(estimate));
if release_before_receipt {
budget.release(ledger.release_reserved());
prop_assert_eq!(budget.reserved(), 0);
prop_assert!(!ledger.is_reserved());
}
budget.release(ledger.release_reserved());
prop_assert_eq!(budget.reserved(), 0);
prop_assert_eq!(ledger.reserved_charge(), 0);
budget.release(ledger.release_reserved());
prop_assert_eq!(budget.reserved(), 0);
}
}
#[test]
fn budget_reservation_tracks_only_unreceived_estimates() {
let estimate = THREE_BLOCK_ESTIMATE;
let max = estimate * 3;
{
let mut budget = ByteBudget::new(max);
let mut reorder = ReorderBuffer::new();
let mut outstanding = outstanding_three_block_range(&mut budget);
assert_eq!(budget.reserved(), max);
assert!(budget.reserved() <= budget.max_bytes_for_test());
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let actuals = [700u64, 800, 900];
for (index, height) in [block::Height(1), block::Height(2), block::Height(3)]
.into_iter()
.enumerate()
{
budget.release(estimate);
outstanding.mark_received(height);
assert_eq!(
reorder.insert(height, block.clone(), actuals[index], peer(0)),
ReorderInsertResult::Inserted
);
assert_eq!(budget.reserved(), outstanding.reserved_bytes());
}
assert!(outstanding.is_complete());
assert_eq!(outstanding.reserved_bytes(), 0);
assert_eq!(
budget.reserved(),
0,
"retained bodies charge nothing; the wire budget is fully free"
);
assert_eq!(reorder.buffered_bytes(), 700 + 800 + 900);
let applied_bytes: u64 = reorder
.drain_contiguous_prefix(block::Height(0))
.into_iter()
.map(|drained| drained.bytes)
.sum();
assert_eq!(applied_bytes, 700 + 800 + 900);
assert_eq!(budget.reserved(), 0);
}
{
let mut budget = ByteBudget::new(max);
let mut outstanding = outstanding_three_block_range(&mut budget);
assert_eq!(budget.reserved(), estimate * 3);
budget.release(estimate);
outstanding.mark_received(block::Height(1));
assert_eq!(outstanding.reserved_bytes(), estimate * 2);
assert_eq!(budget.reserved(), estimate * 2);
budget.release(outstanding.reserved_bytes());
assert_eq!(budget.reserved(), 0);
}
{
let budget = ByteBudget::new(max);
let mut reorder = ReorderBuffer::new();
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
assert_eq!(
reorder.insert(block::Height(1), block.clone(), 1_000, peer(0)),
ReorderInsertResult::Inserted
);
assert_eq!(
reorder.insert(block::Height(1), block, 1_000, peer(0)),
ReorderInsertResult::Duplicate
);
assert_eq!(reorder.buffered_bytes(), 1_000);
assert_eq!(budget.reserved(), 0);
assert!(budget.reserved() <= budget.max_bytes_for_test());
}
}
#[test]
fn received_tracker_handles_a_full_range_at_the_bitset_boundary() {
let count = MAX_BS_BLOCKS_PER_REQUEST;
assert_eq!(count, u128::BITS, "the cap is sized to the bitset width");
let mut outstanding = window_request_range(1, count);
assert_eq!(outstanding.reserved_bytes(), u64::from(count));
for height in 1..=count {
assert!(
!outstanding.has_received(block::Height(height)),
"height {height} should start unreceived",
);
outstanding.mark_received(block::Height(height));
assert!(
outstanding.has_received(block::Height(height)),
"height {height} (offset {}) must be markable — the bitset must cover the \
whole range",
height - 1,
);
}
assert!(
outstanding.is_complete(),
"a fully-received {count}-block range must report complete",
);
assert_eq!(
outstanding.reserved_bytes(),
0,
"every height received ⇒ no reserved bytes remain (no offset fell off the bitset)",
);
}
#[test]
fn underestimated_body_is_buffered_and_releases_only_its_estimate() {
let hint = 1_000u64;
let mut budget = ByteBudget::new(hint * 4);
let mut reorder = ReorderBuffer::new();
let request = BlockRangeRequest {
start_height: block::Height(1),
count: 1,
anchor_hash: block::Hash([1; 32]),
estimated_bytes: hint,
expected_blocks: vec![ExpectedBlock {
height: block::Height(1),
hash: block::Hash([1; 32]),
estimated_bytes: hint,
}],
};
assert!(budget.try_reserve(request.estimated_bytes));
let now = Instant::now();
let mut outstanding = OutstandingBlockRange {
request,
queued_at: now,
deadline: now,
delivery_snapshot: test_delivery_snapshot(now),
delivered_bytes: 0,
received: ReceivedBlockTracker::default(),
};
assert_eq!(budget.reserved(), hint);
let actual = hint * 50;
assert!(actual < BS_PER_BLOCK_WORST_CASE_BYTES);
assert!(actual > hint);
budget.release(hint);
outstanding.mark_received(block::Height(1));
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
assert_eq!(
reorder.insert(block::Height(1), block, actual, peer(0)),
ReorderInsertResult::Inserted,
"an underestimated body must still buffer; a received body is never dropped"
);
assert_eq!(reorder.buffered_bytes(), actual);
assert_eq!(budget.reserved(), 0);
assert_eq!(outstanding.reserved_bytes(), 0);
assert!(
budget.try_reserve(hint),
"the freed reservation funds the next request immediately"
);
let dropped = reorder.drop_through(block::Height(1));
assert_eq!(dropped, actual);
assert_eq!(reorder.buffered_bytes(), 0);
}
#[test]
fn block_sync_stream_declares_kind_capability_version_and_frame_cap() {
let stream = block_sync_streams()
.first()
.copied()
.expect("block sync declares one stream");
assert_eq!(stream.kind, ZAKURA_STREAM_BLOCK_SYNC);
assert_eq!(stream.version, ZAKURA_BLOCK_SYNC_STREAM_VERSION);
assert_eq!(stream.capability, ZAKURA_CAP_BLOCK_SYNC);
assert_eq!(stream.mode, StreamMode::Ordered);
assert_eq!(stream.frame_cap, MAX_BS_FRAME_BYTES);
}
#[tokio::test]
async fn service_registry_routes_block_sync_by_exact_capability_and_version() {
let service = Arc::new(BlockSyncService::new(ZakuraBlockSyncConfig::default()));
let registry =
ServiceRegistry::new(vec![service]).expect("block-sync service declares unique kind");
let peer = peer(1);
assert_eq!(
registry.capability_for_stream(ZAKURA_STREAM_BLOCK_SYNC, ZAKURA_BLOCK_SYNC_STREAM_VERSION),
Some(ZAKURA_CAP_BLOCK_SYNC)
);
assert!(registry
.capability_for_stream(
ZAKURA_STREAM_BLOCK_SYNC,
ZAKURA_BLOCK_SYNC_STREAM_VERSION + 1
)
.is_none());
assert_eq!(
registry
.ordered_streams_for_negotiated(ZAKURA_CAP_BLOCK_SYNC)
.iter()
.map(|stream| stream.kind)
.collect::<Vec<_>>(),
vec![ZAKURA_STREAM_BLOCK_SYNC]
);
assert!(registry.ordered_streams_for_negotiated(0).is_empty());
assert!(registry.wants_ordered_stream(
ZAKURA_STREAM_BLOCK_SYNC,
ZAKURA_CAP_BLOCK_SYNC,
&peer,
ServicePeerDirection::Inbound,
));
}
#[tokio::test(flavor = "current_thread", start_paused = true)]
async fn inert_reactor_parks_after_header_tip_watch_closes() {
let _service = BlockSyncService::new(ZakuraBlockSyncConfig::default());
let elapsed = tokio::time::timeout(Duration::from_secs(1), future::pending::<()>()).await;
assert!(
elapsed.is_err(),
"paused-time timeout only elapses if the inert reactor has no always-ready branch"
);
}
#[cfg(debug_assertions)]
#[test]
#[should_panic(expected = "state-backed block sync must have exactly one frontier source")]
fn state_backed_reactor_panics_with_two_frontier_sources() {
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let (_frontier_tx, frontier_rx) =
watch::channel(test_frontier_update(0, 0, 0, FrontierChange::Snapshot));
let mut startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
ZakuraBlockSyncConfig::default(),
);
startup.frontier_updates = Some(frontier_rx);
let (_handle, _actions, _task) = spawn_block_sync_reactor(startup);
}
#[tokio::test]
async fn add_peer_emits_events_and_round_trips_status_over_framed_path() {
let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default());
let peer = peer(2);
let cancel_token = CancellationToken::new();
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
streams,
cancel_token,
));
let session = match next_event(&mut events).await {
BlockSyncEvent::PeerConnected(session) => session,
event => panic!("expected PeerConnected, got {event:?}"),
};
assert_eq!(session.peer_id(), &peer);
assert_eq!(service.peer_count(), 1);
let _outbound_rx = outbound_rx;
inbound_tx
.send(
BlockSyncMessage::Status(status())
.encode_frame()
.expect("status frame encodes"),
)
.await
.expect("inbound status queues");
service.remove_peer(&peer, 0);
assert_eq!(service.peer_count(), 0);
assert!(session.cancel_token().is_cancelled());
}
#[tokio::test]
async fn stale_block_sync_teardown_keeps_replacement_session() {
let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default());
let peer = peer(92);
let old_conn_id = 1;
let new_conn_id = 2;
let (old_inbound_tx, old_inbound_rx) = framed_channel(4);
let (old_outbound_tx, _old_outbound_rx) = framed_channel(4);
service.add_peer(Peer::new_with_conn_id_and_direction(
old_conn_id,
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (old_inbound_rx, old_outbound_tx))]),
CancellationToken::new(),
));
assert!(matches!(
next_event(&mut events).await,
BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer
));
let (new_inbound_tx, new_inbound_rx) = framed_channel(4);
let (new_outbound_tx, _new_outbound_rx) = framed_channel(4);
service.add_peer(Peer::new_with_conn_id_and_direction(
new_conn_id,
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (new_inbound_rx, new_outbound_tx))]),
CancellationToken::new(),
));
assert!(matches!(
next_event(&mut events).await,
BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer
));
assert_eq!(service.peer_count(), 1);
let (_stale_inbound_tx, stale_inbound_rx) = framed_channel(4);
let (stale_outbound_tx, _stale_outbound_rx) = framed_channel(4);
service.add_peer(Peer::new_with_conn_id_and_direction(
old_conn_id,
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
HashMap::from([(
ZAKURA_STREAM_BLOCK_SYNC,
(stale_inbound_rx, stale_outbound_tx),
)]),
CancellationToken::new(),
));
assert_eq!(
service.peer_count(),
1,
"stale add must not overwrite the replacement block-sync session",
);
service.remove_peer(&peer, old_conn_id);
assert_eq!(service.peer_count(), 1);
drop(old_inbound_tx);
tokio::time::sleep(Duration::from_millis(50)).await;
if let Ok(Some(BlockSyncEvent::PeerDisconnected(disconnected))) =
tokio::time::timeout(Duration::from_millis(50), events.recv()).await
{
panic!("stale teardown disconnected replacement session for {disconnected:?}");
}
assert_eq!(service.peer_count(), 1);
drop(new_inbound_tx);
}
#[tokio::test]
async fn lifecycle_events_bypass_full_bounded_wire_queue() {
let mut config = ZakuraBlockSyncConfig::default();
config.peer_limits.inbound_queue_depth = 1;
let (events, _event_rx) = mpsc::channel(config.peer_limits.inbound_queue_depth);
events
.try_send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(1),
hash: block::Hash([1; 32]),
})
.expect("test fills bounded wire queue");
let (lifecycle, mut lifecycle_rx) = mpsc::unbounded_channel();
let (_peers_tx, peers) = watch::channel(ServicePeerSnapshot::new(0, 0, config.peer_limits));
let (_status_tx, status) = watch::channel(config.initial_status());
let (_candidates_tx, candidates) = watch::channel(ZakuraBlockSyncCandidateState::default());
let handle = BlockSyncHandle {
events,
lifecycle,
peers,
status,
candidates,
routine_wiring: None,
};
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer = peer(91);
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
let _inbound_tx = inbound_tx;
assert!(matches!(
tokio::time::timeout(Duration::from_secs(1), lifecycle_rx.recv())
.await
.expect("lifecycle event arrives")
.expect("lifecycle channel stays open"),
BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer
));
service.remove_peer(&peer, 0);
assert!(matches!(
tokio::time::timeout(Duration::from_secs(1), lifecycle_rx.recv())
.await
.expect("lifecycle event arrives")
.expect("lifecycle channel stays open"),
BlockSyncEvent::PeerDisconnected(disconnected) if disconnected == peer
));
}
#[tokio::test]
async fn add_peer_decode_failure_reports_malformed_and_cancels_connection() {
let config = ZakuraBlockSyncConfig::default();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, _reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(3);
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
let connection_cancel = CancellationToken::new();
service.add_peer(Peer::new(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
streams,
connection_cancel.clone(),
));
inbound_tx
.send(Frame {
message_type: u16::from(MSG_BS_STATUS),
flags: 0,
payload: Vec::new(),
})
.await
.expect("malformed inbound frame queues");
let reported = loop {
match next_action(&mut actions).await {
BlockSyncAction::Misbehavior { peer: got, reason } => break (got, reason),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before malformed-message report: {action:?}"),
}
};
assert_eq!(
reported,
(peer.clone(), BlockSyncMisbehavior::MalformedMessage)
);
tokio::time::timeout(Duration::from_secs(1), connection_cancel.cancelled())
.await
.expect("malformed frame cancels the connection");
}
#[tokio::test]
async fn registry_add_peer_requires_negotiated_block_sync_capability() {
let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default());
let registry = ServiceRegistry::new(vec![Arc::new(service)])
.expect("block-sync service declares unique kind");
let peer = peer(4);
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
registry.add_peer(Peer::new(peer, None, 0, streams, CancellationToken::new()));
drop(inbound_tx);
assert!(
tokio::time::timeout(Duration::from_millis(100), events.recv())
.await
.is_err(),
"without cap 1<<3 the registry must not deliver kind-6 streams"
);
}
#[tokio::test]
async fn wants_peer_rejects_when_configured_slot_cap_is_reached() {
let config = ZakuraBlockSyncConfig {
peer_limits: ServicePeerLimits {
max_inbound_peers: 0,
max_outbound_peers: 2,
..ServicePeerLimits::default()
},
..ZakuraBlockSyncConfig::default()
};
let (service, mut events) = BlockSyncService::new_for_test(config);
let inbound_peer = peer(5);
assert!(!service.wants_peer(
&inbound_peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Inbound
));
assert!(service.wants_peer(
&inbound_peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound
));
let mut inbound_senders = Vec::new();
for byte in 6..=7 {
let peer_id = peer(byte);
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
assert!(matches!(
next_event(&mut events).await,
BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer_id
));
inbound_senders.push(inbound_tx);
}
assert_eq!(service.peer_count(), 2);
assert!(!service.wants_peer(
&peer(8),
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound
));
let (_inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer(8),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
assert_eq!(service.peer_count(), 2);
}
#[tokio::test]
async fn reactor_drives_tip_to_getblocks_to_submit_over_framed_path() {
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(40);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let block_size = u32::try_from(
block
.zcash_serialize_to_vec()
.expect("block serializes")
.len(),
)
.expect("test block size fits u32");
tip_tx
.send((block::Height(1), block_hash))
.expect("tip watch is live");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip,
..
} => {
assert_eq!(from, block::Height(1));
if best_header_tip == block::Height(1) {
break;
}
}
action => panic!("unexpected action before query: {action:?}"),
}
}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block_hash,
size: BlockSizeEstimate::Advertised(block_size),
}]))
.await
.expect("needed metadata queues");
loop {
let frame = tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv())
.await
.expect("outbound frame arrives")
.expect("outbound channel is live");
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1);
break;
}
}
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
block: submitted, ..
} => {
assert_eq!(submitted.hash(), block_hash);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_releases_request_budget_at_receipt_not_apply() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = u64::from(block1_size);
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(41);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: blocks[1].hash(),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
]))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
let submit_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
assert_eq!(block.hash(), blocks[0].hash());
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
};
assert_eq!(handle.local_status().servable_high, block::Height(0));
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(2));
assert_eq!(count, 1);
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: submit_token,
height: block::Height(1),
hash: blocks[0].hash(),
result: BlockApplyResult::Committed,
local_frontier: Some(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}),
})
.await
.expect("apply-finished event queues");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.local_status().servable_high == block::Height(1) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("apply completion frontier advances advertised status");
reactor_task.abort();
}
#[tokio::test]
async fn reactor_does_not_requeue_held_height_reported_still_needed() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let block2_size = block_size(&blocks[1]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 4;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(73);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: blocks[1].hash(),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("body frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("re-reported needed metadata queues");
let saw_requeue_of_held_height = tokio::time::timeout(Duration::from_millis(300), async {
loop {
match outbound_rx.recv().await {
Some(frame) => {
if let BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
..
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
return true;
}
}
None => return false,
}
}
})
.await;
assert!(
saw_requeue_of_held_height.is_err(),
"a held (in_flight) height reported still needed must not be re-requested",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_buffers_body_larger_than_its_size_hint() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
41,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(1),
}]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("body queues");
let submitted = tokio::time::timeout(Duration::from_secs(1), async {
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => break block.hash(),
BlockSyncAction::Misbehavior {
reason: BlockSyncMisbehavior::SizeMismatch,
..
}
| BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
})
.await
.expect("underestimated body must still be buffered and submitted, not dropped");
assert_eq!(submitted, blocks[0].hash());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_downloads_run_ahead_of_stalled_commit() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_requests = 1;
config.max_blocks_per_response = 1;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status_message(
&service,
&mut actions,
41,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
let _submit_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
assert_eq!(block.hash(), blocks[0].hash());
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
};
tip_tx
.send((block::Height(4), block::Hash([4; 32])))
.expect("tip watch is live");
wait_for_query_needed_blocks(&mut actions, block::Height(1), block::Height(4)).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1),
"the reactor must download height 2 ahead of the stalled commit of height 1",
);
reactor_task.abort();
}
fn add_outbound_block_sync_peer(
service: &BlockSyncService,
byte: u8,
held: &mut Vec<(FramedSend, FramedRecv)>,
) -> ZakuraPeerId {
let peer_id = peer(byte);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
held.push((inbound_tx, outbound_rx));
peer_id
}
#[tokio::test]
async fn block_sync_add_peer_replaces_same_peer_even_at_full_cap() {
let mut config = immediate_body_download_config();
config.peer_limits.max_outbound_peers = 1;
config.peer_limits.max_inbound_peers = 0;
let (service, mut events) = BlockSyncService::new_for_test(config);
let mut held = Vec::new();
let peer_a = add_outbound_block_sync_peer(&service, 41, &mut held);
match next_event(&mut events).await {
BlockSyncEvent::PeerConnected(session) => assert_eq!(session.peer_id(), &peer_a),
event => panic!("expected PeerConnected for peer A, got {event:?}"),
}
assert_eq!(service.peer_count(), 1);
let _peer_b = add_outbound_block_sync_peer(&service, 42, &mut held);
let quiet = tokio::time::timeout(Duration::from_millis(100), events.recv()).await;
assert!(
quiet.is_err(),
"a new peer at a full per-direction cap must be rejected",
);
assert_eq!(service.peer_count(), 1);
let peer_a_again = add_outbound_block_sync_peer(&service, 41, &mut held);
assert_eq!(peer_a_again, peer_a);
match next_event(&mut events).await {
BlockSyncEvent::PeerConnected(session) => assert_eq!(session.peer_id(), &peer_a),
event => panic!("expected PeerConnected for replaced peer A, got {event:?}"),
}
assert_eq!(
service.peer_count(),
1,
"the replacement must not leave two sessions for the same peer",
);
}
#[tokio::test]
async fn reactor_keeps_applying_body_after_non_advancing_duplicate_result() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = u64::from(block1_size);
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
42,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
send_inbound(&inbound_tx, BlockSyncMessage::Block(blocks[0].clone())).await;
let submit_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
assert_eq!(block.hash(), blocks[0].hash());
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: submit_token,
height: block::Height(1),
hash: blocks[0].hash(),
result: BlockApplyResult::Duplicate,
local_frontier: Some(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
}),
})
.await
.expect("non-advancing duplicate completion queues");
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
]))
.await
.expect("needed metadata after duplicate queues");
let no_duplicate_request = tokio::time::timeout(Duration::from_millis(100), async {
while let Some(frame) = outbound_rx.recv().await {
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
if start_height == block::Height(1) {
panic!("non-advancing duplicate result re-requested {start_height:?}/{count}");
}
}
}
})
.await;
assert!(
no_duplicate_request.is_err(),
"reactor should keep waiting after a duplicate result that did not advance the frontier",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_keeps_active_response_when_needed_snapshot_omits_inflight_height() {
let blocks = fake_sequential_blocks(3);
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(42);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 16,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("first block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } if block.hash() == blocks[0].hash() => break,
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first submit: {action:?}"),
}
}
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("newer needed metadata queues");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("second block queues");
let mut submitted_second = false;
while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await
{
match action {
BlockSyncAction::SubmitBlock { block, .. } if block.hash() == blocks[1].hash() => {
submitted_second = true;
break;
}
BlockSyncAction::Misbehavior {
peer,
reason: BlockSyncMisbehavior::UnsolicitedBlock,
} if peer == peer_id => panic!("in-flight body was misclassified as unsolicited"),
BlockSyncAction::QueryNeededBlocks { .. } | BlockSyncAction::SubmitBlock { .. } => {}
action => panic!("unexpected action after second block: {action:?}"),
}
}
assert!(
submitted_second,
"second body from the original active response should remain correlated",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_unmatched_body_for_currently_needed_height() {
let blocks = mainnet_blocks_1_to_3();
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(142);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(2),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 16,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
let mut candidates = handle.subscribe_candidate_state();
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if candidates
.borrow_and_update()
.missing_block_bodies
.contains(&block::Height(1))
{
return;
}
candidates.changed().await.expect("candidate watch is live");
}
})
.await
.expect("producer extends the needed height into the WorkQueue");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("unmatched needed block queues");
let quiet = tokio::time::timeout(Duration::from_millis(200), async {
loop {
if let BlockSyncAction::Misbehavior { reason, .. } = next_action(&mut actions).await {
return reason;
}
}
})
.await;
assert!(
quiet.is_err(),
"unmatched body for a currently needed height should not be hard misbehavior",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_unmatched_body_for_queued_height() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let block2_size = block_size(&blocks[1]);
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
143,
block::Height(2),
blocks[1].hash(),
1,
block1_size,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1),
"the byte-capped request must cover only height 1",
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("unmatched queued block queues");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("matched block queues");
let mut submitted = Vec::new();
tokio::time::timeout(Duration::from_secs(1), async {
while submitted.len() < 2 {
if let BlockSyncAction::SubmitBlock { block, .. } = next_action(&mut actions).await {
submitted.push(block.hash());
}
}
})
.await
.expect("both bodies are submitted");
assert_eq!(submitted, vec![blocks[0].hash(), blocks[1].hash()]);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_queries_needed_blocks_above_submitted_floor() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let block2_size = block_size(&blocks[1]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 2;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(43);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 2,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2)
);
for block in blocks.iter().take(2) {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
}
let mut submitted = Vec::new();
let mut saw_refill_query = false;
while submitted.len() < 2 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
submitted.push((
block.coinbase_height().expect("test block has height"),
token,
));
}
BlockSyncAction::QueryNeededBlocks {
from: block::Height(3),
best_header_tip,
..
} => {
assert_eq!(
best_header_tip,
block::Height(3),
"missing-body query must skip already claimed contiguous bodies",
);
saw_refill_query = true;
}
BlockSyncAction::QueryNeededBlocks { from, .. } => {
panic!("missing-body query should skip claimed bodies, got from {from:?}")
}
action => panic!("unexpected action before checkpoint submissions: {action:?}"),
}
}
assert_eq!(
submitted
.iter()
.map(|(height, _token)| *height)
.collect::<Vec<_>>(),
vec![block::Height(1), block::Height(2)]
);
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: submitted[0].1,
height: block::Height(1),
hash: blocks[0].hash(),
result: BlockApplyResult::Committed,
local_frontier: Some(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}),
})
.await
.expect("apply-finished event queues");
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: submitted[1].1,
height: block::Height(2),
hash: blocks[1].hash(),
result: BlockApplyResult::Committed,
local_frontier: Some(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}),
})
.await
.expect("apply-finished event queues");
if !saw_refill_query {
let action = tokio::time::timeout(Duration::from_secs(5), actions.recv())
.await
.expect(
"bounded needed-block query should arrive after apply-finished advances the floor",
)
.expect("block-sync action channel should stay open");
match action {
BlockSyncAction::QueryNeededBlocks {
from: block::Height(3),
best_header_tip,
..
} => {
assert_eq!(
best_header_tip,
block::Height(3),
"missing-body query must skip already submitted contiguous bodies",
);
}
BlockSyncAction::QueryNeededBlocks { from, .. } => {
panic!("missing-body query should skip submitted bodies, got from {from:?}")
}
action => panic!("unexpected action before needed-block query: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_retries_submitted_body_after_apply_rejection() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_bytes = block_size(&block);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(42);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block.hash(),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), block.hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_bytes),
}]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
let submit_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
token,
block: submitted,
} => {
assert_eq!(submitted.hash(), block.hash());
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: submit_token,
height: block::Height(1),
hash: block.hash(),
result: BlockApplyResult::Rejected,
local_frontier: None,
})
.await
.expect("apply-finished event queues");
let retry_meta = vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_bytes),
}];
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count,
} => {
assert_eq!(
count, 1,
"apply rejection must release capacity and clear submitted coverage"
);
break;
}
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before retry request: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(retry_meta.clone()))
.await
.expect("needed metadata queues after rejection");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("unexpected action before retry request: {action:?}"),
}
}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_keeps_issuing_far_above_floor_with_no_near_tip_pause() {
let config = ZakuraBlockSyncConfig {
initial_block_probe_requests: DEFAULT_BS_MAX_REQUESTS_WITHOUT_BLOCK_PROGRESS,
..ZakuraBlockSyncConfig::default()
};
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
41,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: block::Hash([4; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tip_tx
.send((block::Height(1_000), block::Hash([9; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=4)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([height as u8; 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed metadata queues");
let mut heights = Vec::new();
for _ in 0..4 {
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(count, 1);
heights.push(start_height.0);
}
heights.sort_unstable();
assert_eq!(
heights,
vec![1, 2, 3, 4],
"a peer with free slots and budget must keep issuing regardless of floor distance"
);
reactor_task.abort();
}
#[tokio::test]
async fn routine_refills_after_budget_release_no_missed_wake() {
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES;
config.max_blocks_per_response = 1;
config.request_timeout = Duration::from_secs(300);
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status_message(
&service,
&mut actions,
70,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: blocks[1].hash(),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("needed metadata queues");
let (first_start, _count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(first_start, block::Height(1));
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
let token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
assert_eq!(block.coinbase_height(), Some(block::Height(1)));
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before SubmitBlock: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
token,
height: block::Height(1),
hash: blocks[0].hash(),
result: BlockApplyResult::Committed,
local_frontier: None,
})
.await
.expect("apply-finished event queues");
let (second_start, _count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(
second_start,
block::Height(2),
"freeing the request budget must wake the routine to issue the next request (no missed wake)"
);
reactor_task.abort();
}
#[tokio::test]
async fn routine_disconnect_returns_outstanding_and_releases_budget() {
let mut config = immediate_body_download_config();
config.request_timeout = Duration::from_secs(300);
config.peer_limits.max_outbound_peers = 4;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_a, _a_in, mut a_out) = connect_peer_with_status(
&service,
&mut actions,
0x71,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
let (start, _count) = wait_for_outbound_getblocks(&mut a_out).await;
assert_eq!(start, block::Height(1));
service.remove_peer(&peer_a, 0);
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if service.peer_count() == 0 {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("disconnect releases the block-sync peer slot");
let (_peer_b, _b_in, mut b_out) = connect_peer_with_status(
&service,
&mut actions,
0x72,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let offered = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let (start, _count) = wait_for_outbound_getblocks(&mut b_out).await;
if start == block::Height(1) {
return true;
}
}
})
.await
.expect("a fresh peer must be offered the re-queued height after a disconnect mid-fetch");
assert!(offered);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_reserves_size_hint_per_block_not_worst_case() {
let budget_blocks = 3u32;
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: u64::from(budget_blocks) * BS_PER_BLOCK_WORST_CASE_BYTES,
max_blocks_per_response: 16,
..ZakuraBlockSyncConfig::default()
};
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
51,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(16),
tip_hash: block::Hash([16; 32]),
max_blocks_per_response: 16,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=16)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([height as u8; 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (_start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(
count, 16,
"the request must cover the full 16-block range: the 1 KiB size hints (not a \
worst-case-per-block reservation) drive the budget, so all 16 fit under a \
{budget_blocks}-worst-case-block budget",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_packs_small_estimates_under_peer_response_byte_cap() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 4 * BS_PER_BLOCK_WORST_CASE_BYTES,
max_blocks_per_response: 4,
..ZakuraBlockSyncConfig::default()
};
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let one_worst_case_response =
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES).expect("worst-case block size fits u32");
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
52,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: block::Hash([4; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: one_worst_case_response,
},
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=4)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([height as u8; 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(
count, 4,
"small advertised estimates should pack more than the one worst-case block \
allowed by the peer response byte cap",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_tiny_estimates_pack_into_one_worst_case_budget_block() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES,
max_blocks_per_response: 4,
..ZakuraBlockSyncConfig::default()
};
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
53,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: block::Hash([4; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=4)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("test height fits u8"); 32]),
size: BlockSizeEstimate::Advertised(1),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(
count, 4,
"all 4 tiny-hint heights pack into one worst-case-block budget, because each \
reserves its 1 KiB-floored size hint rather than a worst-case share",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_zero_pause_threshold_preserves_lag_one_downloads() {
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(1),
hash: block::Hash([1; 32]),
})
.await
.expect("header-tip event queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
best_header_tip: block::Height(1),
..
}
) {}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_keeps_block_sync_peer_after_catch_up_and_reuses_later() {
let mut config = fill_loop_mechanics_config();
config.peer_limits.max_outbound_peers = 1;
let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(4), block::Hash([4; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
71,
block::Height(6),
block::Hash([6; 32]),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(3),
hash: block::Hash([3; 32]),
})
.await
.expect("header-tip event queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: block::Hash([3; 32]),
size: BlockSizeEstimate::Unknown,
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
for height in [block::Height(1), block::Height(2)] {
let hash_byte = u8::try_from(height.0).expect("test height fits in u8");
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: height,
verified_block_hash: block::Hash([hash_byte; 32]),
}))
.await
.expect("frontier event queues");
tokio::time::sleep(Duration::from_millis(50)).await;
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
assert_eq!(service.peer_count(), 1);
}
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 3,
}
.encode_frame()
.expect("BlocksDone encodes"),
)
.await
.expect("BlocksDone queues");
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: block::Hash([3; 32]),
}))
.await
.expect("caught-up frontier event queues");
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
handle.peer_snapshot().outbound_peers,
1,
"caught-up nodes must keep block-sync peers so they can serve fresh nodes",
);
assert_eq!(service.peer_count(), 1);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(6),
hash: block::Hash([6; 32]),
})
.await
.expect("later header-tip event queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(4),
best_header_tip: block::Height(6),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(5),
hash: block::Hash([5; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(6),
hash: block::Hash([6; 32]),
size: BlockSizeEstimate::Unknown,
},
]))
.await
.expect("new needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(4), 3),
"the retained block-sync peer should be reused after later header growth",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_multi_block_range_and_submits_parent_first() {
let config = ZakuraBlockSyncConfig {
max_blocks_per_response: 4,
..ZakuraBlockSyncConfig::default()
};
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(43);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
blocks
.iter()
.map(|block| BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 3);
for index in [1usize, 2, 0] {
inbound_tx
.send(
BlockSyncMessage::Block(blocks[index].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
}
let mut submitted = Vec::new();
while submitted.len() < 3 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::Misbehavior { reason, .. } => {
panic!("honest multi-block response was misclassified: {reason:?}")
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before all submits: {action:?}"),
}
}
assert_eq!(
submitted,
vec![block::Height(1), block::Height(2), block::Height(3)]
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_backpressures_inbound_body_flood_without_dropping_bodies() {
const FLOOD: u32 = 64;
let blocks = fake_sequential_blocks(FLOOD);
let tip = blocks.last().expect("flood is non-empty").clone();
let tip_height = tip.coinbase_height().expect("tip has height");
let mut config = immediate_body_download_config();
config.peer_limits.inbound_queue_depth = 1;
config.max_inflight_block_bytes = u64::MAX;
config.request_timeout = Duration::from_secs(300);
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(64);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: tip_height,
tip_hash: tip.hash(),
max_blocks_per_response: FLOOD,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((tip_height, tip.hash()))
.expect("tip watch is live");
let feed_blocks = blocks.clone();
let (feed_tx, mut feed_rx) = mpsc::unbounded_channel::<u32>();
let feeder = tokio::spawn(async move {
while let Some(height) = feed_rx.recv().await {
let frame = BlockSyncMessage::Block(feed_blocks[(height - 1) as usize].clone())
.encode_frame()
.expect("block frame encodes");
if inbound_tx.send(frame).await.is_err() {
break;
}
}
});
let metas: Vec<_> = blocks
.iter()
.map(|block| BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
})
.collect();
let drive = async {
let mut submitted = std::collections::HashSet::new();
while submitted.len() < FLOOD as usize {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let Some(frame) = frame else { break };
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
for height in start_height.0..start_height.0 + count {
feed_tx.send(height).expect("feeder task stays open");
}
}
}
action = actions.recv() => {
let Some(action) = action else { break };
match action {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(metas.clone()))
.await
.expect("needed metadata queues");
}
BlockSyncAction::SubmitBlock { block, .. } => {
submitted.insert(
block.coinbase_height().expect("submitted block has height"),
);
}
_ => {}
}
}
}
}
submitted
};
let submitted = tokio::time::timeout(Duration::from_secs(20), drive)
.await
.expect(
"flooded bodies must all reach the apply stage; a dropped inbound body wedges block sync",
);
let expected: std::collections::HashSet<_> = (1..=FLOOD).map(block::Height).collect();
assert_eq!(
submitted, expected,
"every solicited body in the flood must be submitted exactly once, with no drops"
);
feeder.abort();
reactor_task.abort();
}
#[tokio::test]
async fn reactor_restarted_at_genesis_queries_and_schedules_without_tip_change() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip,
..
} => {
assert_eq!(from, block::Height(1));
assert_eq!(best_header_tip, block::Height(3));
}
action => panic!("restart from genesis must query missing bodies, got {action:?}"),
}
let (_peer, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
67,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3),
"restart from genesis must schedule scratch body sync from height 1"
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
assert_eq!(block.coinbase_height(), Some(block::Height(1)));
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first scratch submit: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_blocks_done_after_completed_range() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
68,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 1,
}
.encode_frame()
.expect("BlocksDone encodes"),
)
.await
.expect("BlocksDone queues");
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(200), actions.recv()).await
{
if let BlockSyncAction::Misbehavior {
peer: action_peer,
reason,
} = action
{
assert_ne!(
(action_peer, reason),
(peer.clone(), BlockSyncMisbehavior::UnsolicitedDone),
"a valid terminator after a completed block response must not be scored"
);
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_retries_missing_heights_after_partial_blocks_done() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
69,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first submit: {action:?}"),
}
}
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 1,
}
.encode_frame()
.expect("BlocksDone encodes"),
)
.await
.expect("BlocksDone queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2),
"partial responses must retry the contiguous missing suffix"
);
reactor_task.abort();
}
#[tokio::test]
async fn checkpoint_hole_disconnect_retries_first_missing_height_with_fresh_peer() {
const FIRST_NEEDED: u32 = 801;
const PREFIX_END: u32 = 864;
const HOLE_START: u32 = 865;
const HOLE_END: u32 = 872;
const LAST_METADATA: u32 = 933;
const BEST_HEADER_TIP: u32 = 10_400;
let blocks = fake_blocks_in_range(FIRST_NEEDED, LAST_METADATA);
let block_at = |height: u32| -> Arc<block::Block> {
let index =
usize::try_from(height - FIRST_NEEDED).expect("test height is inside block vector");
blocks[index].clone()
};
let metas: Vec<_> = blocks.iter().map(block_meta).collect();
let prefix: std::collections::HashSet<_> =
(FIRST_NEEDED..=PREFIX_END).map(block::Height).collect();
let sparse_above_hole: std::collections::HashSet<_> = [873, 888, 905, 920]
.into_iter()
.map(block::Height)
.collect();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = u64::MAX;
config.request_timeout = Duration::from_secs(300);
config.peer_limits.max_outbound_peers = 1;
config.peer_limits.inbound_queue_depth = 128;
config.peer_limits.outbound_queue_depth = 128;
let (_tip_tx, tip_rx) = watch::channel((block::Height(BEST_HEADER_TIP), block::Hash([10; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(800),
verified_block_tip: block::Height(800),
verified_block_hash: block::Hash([8; 32]),
},
(block::Height(BEST_HEADER_TIP), block::Hash([10; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (old_peer, old_inbound, mut old_outbound) = connect_peer_with_status_message(
&service,
&mut actions,
70,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(LAST_METADATA),
tip_hash: block_at(LAST_METADATA).hash(),
max_blocks_per_response: MAX_BS_BLOCKS_PER_REQUEST,
max_inflight_requests: 8,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(metas.clone()))
.await
.expect("checkpoint metadata queues after peer connection");
let (feed_tx, mut feed_rx) = mpsc::unbounded_channel::<u32>();
let blocks_for_feeder = blocks.clone();
let feeder = tokio::spawn(async move {
while let Some(height) = feed_rx.recv().await {
let index = usize::try_from(height - FIRST_NEEDED)
.expect("fed test height is inside block vector");
let frame = BlockSyncMessage::Block(blocks_for_feeder[index].clone())
.encode_frame()
.expect("block frame encodes");
if old_inbound.send(frame).await.is_err() {
break;
}
}
});
let mut requests: Vec<(block::Height, u32)> = Vec::new();
let mut submitted = std::collections::HashSet::new();
let primed = tokio::time::timeout(Duration::from_secs(40), async {
while !prefix.is_subset(&submitted)
|| !requests.iter().any(|(start, count)| {
*start <= block::Height(HOLE_START) && start.0.saturating_add(*count) > HOLE_END
})
{
tokio::select! {
biased;
frame = old_outbound.recv() => {
let frame = frame.expect("old peer outbound is live");
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
requests.push((start_height, count));
let end_height = start_height
.0
.checked_add(count)
.expect("test request height range fits u32");
for height in start_height.0..end_height {
let height = block::Height(height);
if height.0 <= PREFIX_END || sparse_above_hole.contains(&height) {
feed_tx.send(height.0).expect("feeder task stays open");
}
}
}
}
action = actions.recv() => {
match action.expect("block-sync action channel should stay open") {
BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip,
..
} => {
assert!(from >= block::Height(801));
assert_eq!(best_header_tip, block::Height(BEST_HEADER_TIP));
handle
.send(BlockSyncEvent::NeededBlocks(metas.clone()))
.await
.expect("checkpoint metadata queues");
}
BlockSyncAction::SubmitBlock { block, .. } => {
let height =
block.coinbase_height().expect("submitted block has height");
assert!(
submitted.insert(height),
"height {height:?} was submitted more than once before the hole retry"
);
}
action => {
panic!("unexpected action while priming checkpoint hole: {action:?}")
}
}
}
}
}
})
.await;
assert!(
primed.is_ok(),
"checkpoint-hole priming timed out: requests={requests:?}, submitted={} of {}",
submitted.len(),
prefix.len()
);
assert!(
requests
.iter()
.any(|(start, count)| *start <= block::Height(HOLE_START)
&& start.0.saturating_add(*count) > HOLE_END),
"the initial in-flight requests must cover the missing checkpoint hole"
);
service.remove_peer(&old_peer, 0);
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.peer_snapshot().outbound_peers == 0 && service.peer_count() == 0 {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("disconnect releases the old block-sync peer slot");
let (_new_peer, _new_inbound, mut new_outbound) = connect_peer_with_status_message(
&service,
&mut actions,
71,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(LAST_METADATA),
tip_hash: block_at(LAST_METADATA).hash(),
max_blocks_per_response: MAX_BS_BLOCKS_PER_REQUEST,
max_inflight_requests: 8,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tokio::time::timeout(Duration::from_secs(5), async {
loop {
tokio::select! {
biased;
frame = new_outbound.recv() => {
let frame = frame.expect("new peer outbound is live");
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
assert_eq!(
start_height,
block::Height(HOLE_START),
"after a partial response disconnect, the first fresh request must fill \
the checkpoint hole instead of jumping above it or reusing a stale \
assignment"
);
assert!(count >= 1);
break;
}
}
action = actions.recv() => {
match action.expect("block-sync action channel should stay open") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(metas.clone()))
.await
.expect("post-disconnect metadata queues");
}
BlockSyncAction::SubmitBlock { block, .. } => {
let height =
block.coinbase_height().expect("submitted block has height");
assert!(
submitted.insert(height),
"height {height:?} was resubmitted before the checkpoint hole was retried"
);
}
action => {
panic!(
"unexpected action while waiting for checkpoint-hole retry: {action:?}"
)
}
}
}
}
}
})
.await
.expect("fresh peer should request the checkpoint hole after disconnect");
feeder.abort();
reactor_task.abort();
}
#[tokio::test]
async fn reactor_reset_mid_download_drops_stale_anchors_and_releases_budget() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
47,
block::Height(3),
blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(10_000),
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: blocks[2].hash(),
size: BlockSizeEstimate::Advertised(10_000),
},
]))
.await
.expect("old-fork needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("out-of-order old-fork block queues");
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}))
.await
.expect("reset event queues");
let new_fork_meta = vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([92; 32]),
size: BlockSizeEstimate::Advertised(20_000),
}];
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(2),
count,
} => {
assert_eq!(
count, 1,
"a full-budget new fork request can only be scheduled if stale bytes were released"
);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before new fork request: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(new_fork_meta.clone()))
.await
.expect("new-fork needed metadata queues");
}
action => panic!("unexpected action before new fork request: {action:?}"),
}
}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_forward_reset_preserves_submitted_successor_body() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
49,
block::Height(3),
blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("contiguous body queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[1].hash());
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first submit: {action:?}"),
}
}
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("successor body queues");
let successor_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
assert_eq!(block.hash(), blocks[2].hash());
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before successor submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}))
.await
.expect("forward reset event queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"forward reset must not re-query or re-submit the preserved successor body",
);
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: successor_token,
height: block::Height(3),
hash: blocks[2].hash(),
result: BlockApplyResult::Committed,
local_frontier: None,
})
.await
.expect("successor apply result queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"at-tip successor apply must not trigger a redundant needed-block query",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_forward_reset_preserves_future_outstanding_body() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
72,
block::Height(3),
blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}))
.await
.expect("forward reset event queues");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("successor body queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[2].hash());
break;
}
BlockSyncAction::Misbehavior { peer, reason } => {
assert_ne!(
(peer, reason),
(peer_id.clone(), BlockSyncMisbehavior::UnsolicitedBlock),
"forward reset must not drop a still-active successor request"
);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
BlockSyncAction::QueryBlocksByHeightRange { .. } => {}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_forward_reset_preserves_buffered_successor_body() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
73,
block::Height(3),
blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("successor body queues");
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(2),
returned: 2,
}
.encode_frame()
.expect("BlocksDone encodes"),
)
.await
.expect("BlocksDone queues");
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}))
.await
.expect("forward reset event queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(
block.hash(),
blocks[2].hash(),
"buffered successor must submit without a duplicate download"
);
break;
}
BlockSyncAction::Misbehavior { peer, reason } => {
assert_ne!(
(peer, reason),
(peer_id.clone(), BlockSyncMisbehavior::UnsolicitedBlock),
"forward reset must not drop a buffered successor body"
);
}
BlockSyncAction::QueryNeededBlocks { .. }
| BlockSyncAction::QueryBlocksByHeightRange { .. } => {}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_destructive_forward_reset_does_not_rerequest_same_hash_in_flight_apply() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
68,
block::Height(2),
blocks[1].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2)
);
for block in blocks.iter().take(2) {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
}
let mut submitted = Vec::new();
while submitted.len() < 2 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => submitted.push((
block.coinbase_height().expect("test block has height"),
token,
)),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submitted bodies: {action:?}"),
}
}
assert_eq!(
submitted
.iter()
.map(|(height, _)| *height)
.collect::<Vec<_>>(),
vec![block::Height(1), block::Height(2)]
);
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: block::Hash([99; 32]),
}))
.await
.expect("destructive forward reset queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(2),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("same-hash needed metadata queues");
if let Ok((re_start, re_count)) = tokio::time::timeout(
Duration::from_millis(200),
wait_for_outbound_getblocks(&mut outbound_rx),
)
.await
{
assert_eq!(
(re_start, re_count),
(block::Height(2), 1),
"any same-hash re-request must target exactly the released height"
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("same-hash body queues");
let no_resubmit = tokio::time::timeout(Duration::from_millis(200), async {
loop {
match actions.recv().await {
Some(BlockSyncAction::SubmitBlock { block, .. })
if block.hash() == blocks[1].hash() =>
{
panic!("same-hash body with a pending apply must not be re-submitted")
}
Some(_) => {}
None => break,
}
}
})
.await;
assert!(
no_resubmit.is_err(),
"a redundant same-hash body must be dropped, not re-submitted",
);
}
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: block::Hash([99; 32]),
}))
.await
.expect("fork reanchor reset queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([42; 32]),
size: BlockSizeEstimate::Advertised(block_size(&blocks[1])),
}]))
.await
.expect("new-fork needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1),
"different hash at the same height must still schedule after a fork reset"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_stale_apply_completion_after_resubmit() {
let config = immediate_body_download_config();
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(61);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block_hash,
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), block_hash))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block_hash,
size: BlockSizeEstimate::Advertised(block_size(&block)),
}]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("first body frame queues");
let stale_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
assert_eq!(block.hash(), block_hash);
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
}))
.await
.expect("reset event queues");
let reset_meta = vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block_hash,
size: BlockSizeEstimate::Advertised(block_size(&block)),
}];
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count,
} => {
assert_eq!(count, 1);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before reset re-fetch: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(reset_meta.clone()))
.await
.expect("needed metadata queues after reset");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("unexpected action before reset re-fetch: {action:?}"),
}
}
}
}
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("second body frame queues");
let current_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
assert_eq!(block.hash(), block_hash);
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before second submit: {action:?}"),
}
};
assert_ne!(stale_token, current_token);
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: stale_token,
height: block::Height(1),
hash: block_hash,
result: BlockApplyResult::Duplicate,
local_frontier: None,
})
.await
.expect("stale apply-finished event queues");
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(100), actions.recv()).await
{
if let BlockSyncAction::SubmitBlock { .. } = action {
panic!(
"stale apply completion released/re-submitted the current submission: {action:?}"
);
}
}
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: current_token,
height: block::Height(1),
hash: block_hash,
result: BlockApplyResult::Committed,
local_frontier: None,
})
.await
.expect("current apply-finished event queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"at-tip apply completion must not trigger a redundant needed-block query",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_fast_forward_reset_clears_buffered_bodies_and_releases_budget() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
50,
block::Height(4),
block::Hash([4; 32]),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(10_000),
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: blocks[2].hash(),
size: BlockSizeEstimate::Advertised(10_000),
},
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("out-of-order body queues");
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: blocks[2].hash(),
}))
.await
.expect("fast-forward reset event queues");
tip_tx
.send((block::Height(4), block::Hash([4; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(4),
best_header_tip: block::Height(4),
..
}
) {}
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
handle.peer_snapshot().outbound_peers,
1,
"caught-up reset keeps the previous block-sync peer available for serving",
);
assert_eq!(service.peer_count(), 1);
let post_reset_meta = vec![BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Advertised(20_000),
}];
handle
.send(BlockSyncEvent::NeededBlocks(post_reset_meta.clone()))
.await
.expect("post-reset needed metadata queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(4),
count,
} => {
assert_eq!(
count, 1,
"a full-budget request after fast-forward Reset requires releasing buffered bytes"
);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before fast-forward re-fetch: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(post_reset_meta.clone()))
.await
.expect("post-reset needed metadata queues");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("unexpected action before fast-forward re-fetch: {action:?}"),
}
}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_fuzzes_arrival_order_across_fork_parent_first() {
#[derive(Copy, Clone)]
enum ForkBody {
Old(usize),
New(usize),
}
let cases = vec![
(
"stale-high-before-reset",
vec![3],
vec![],
vec![ForkBody::Old(2), ForkBody::New(3), ForkBody::New(2)],
),
(
"old-prefix-before-reset",
vec![2, 1],
vec![],
vec![ForkBody::Old(3), ForkBody::New(2), ForkBody::New(3)],
),
(
"stale-before-new-needed",
vec![2],
vec![3],
vec![ForkBody::New(2), ForkBody::Old(2), ForkBody::New(3)],
),
(
"new-out-of-order-with-stale-tail",
vec![],
vec![2],
vec![ForkBody::New(3), ForkBody::Old(3), ForkBody::New(2)],
),
];
for (case, old_before_reset, old_before_new_needed, after_new_needed) in cases {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 3,
..fill_loop_mechanics_config()
};
config.peer_limits.outbound_queue_depth = 16;
let old_blocks = mainnet_blocks_1_to_3();
let new_blocks = vec![
forked_block(&old_blocks[0], 101),
forked_block(&old_blocks[1], 102),
forked_block(&old_blocks[2], 103),
];
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
51,
block::Height(4),
old_blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 3).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), old_blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
old_blocks
.iter()
.map(|block| BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
})
.collect(),
))
.await
.expect("old-fork needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3),
"{case}: old fork request schedules"
);
let mut submitted_tip = block::Height(0);
for height in old_before_reset {
inbound_tx
.send(
BlockSyncMessage::Block(old_blocks[height - 1].clone())
.encode_frame()
.expect("old-fork block encodes"),
)
.await
.expect("old-fork block queues");
drain_parent_first_actions(&mut actions, &mut submitted_tip, None).await;
}
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: new_blocks[0].hash(),
}))
.await
.expect("reset event queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
submitted_tip = block::Height(1);
tip_tx
.send((block::Height(3), new_blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
for height in old_before_new_needed {
inbound_tx
.send(
BlockSyncMessage::Block(old_blocks[height - 1].clone())
.encode_frame()
.expect("stale old-fork block encodes"),
)
.await
.expect("stale old-fork block queues");
drain_parent_first_actions(&mut actions, &mut submitted_tip, Some(&new_blocks)).await;
}
let new_fork_meta = vec![
BlockSyncBlockMeta {
height: block::Height(2),
hash: new_blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block_size(&new_blocks[1])),
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: new_blocks[2].hash(),
size: BlockSizeEstimate::Advertised(block_size(&new_blocks[2])),
},
];
handle
.send(BlockSyncEvent::NeededBlocks(new_fork_meta.clone()))
.await
.expect("new-fork needed metadata queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(2),
count,
} => {
assert_eq!(
count, 2,
"{case}: new fork request schedules after reset"
);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("{case}: unexpected outbound message before new fork re-fetch: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(new_fork_meta.clone()))
.await
.expect("new-fork needed metadata queues");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("{case}: unexpected action before new fork re-fetch: {action:?}"),
}
}
}
}
for body in after_new_needed {
let block = match body {
ForkBody::Old(height) => old_blocks[height - 1].clone(),
ForkBody::New(height) => new_blocks[height - 1].clone(),
};
inbound_tx
.send(
BlockSyncMessage::Block(block)
.encode_frame()
.expect("fork body encodes"),
)
.await
.expect("fork body queues");
drain_parent_first_actions(&mut actions, &mut submitted_tip, Some(&new_blocks)).await;
}
assert_eq!(
submitted_tip,
block::Height(3),
"{case}: new fork bodies submit parent-first through height 3"
);
handle
.send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: new_blocks[2].hash(),
}))
.await
.expect("post-submit grow event queues");
tip_tx
.send((block::Height(4), block::Hash([4; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(4),
best_header_tip: block::Height(4),
..
}
) {}
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
handle.peer_snapshot().outbound_peers,
1,
"{case}: caught-up fork handling keeps the old block-sync peer available for serving",
);
assert_eq!(service.peer_count(), 1);
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Advertised(60_000),
}]))
.await
.expect("post-fuzz needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(4), 1),
"{case}: byte budget returns to baseline after reset and submissions"
);
reactor_task.abort();
}
}
#[tokio::test]
async fn reactor_competing_fork_download_switches_to_current_header_hashes() {
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
immediate_body_download_config(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(
immediate_body_download_config(),
handle.clone(),
);
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
48,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("old fork metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}))
.await
.expect("reset event queues");
let new_fork_meta = vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([222; 32]),
size: BlockSizeEstimate::Advertised(block_size(&blocks[1])),
}];
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(2),
count,
} => {
assert_eq!(count, 1);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before new fork re-fetch: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(new_fork_meta.clone()))
.await
.expect("new fork metadata queues");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("unexpected action before new fork re-fetch: {action:?}"),
}
}
}
}
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("old-fork body encodes"),
)
.await
.expect("old-fork body queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::Misbehavior { peer, reason } => {
assert_eq!(peer, peer_id);
assert_eq!(reason, BlockSyncMisbehavior::InvalidBlock);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before stale body rejection: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_legacy_commit_dedups_inflight_request_and_reuses_budget() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES,
..fill_loop_mechanics_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
49,
block::Height(2),
blocks[1].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(10_000),
}]))
.await
.expect("first needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
handle
.send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}))
.await
.expect("legacy commit grow event queues");
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(2),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(10_000),
}]))
.await
.expect("second needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1),
"legacy commit must release the duplicate in-flight reservation"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_treats_duplicate_buffered_blocks_as_benign() {
let config = immediate_body_download_config();
let blocks = [
mainnet_block(&BLOCK_MAINNET_1_BYTES),
mainnet_block(&BLOCK_MAINNET_2_BYTES),
];
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(44);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: blocks[1].hash(),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
blocks
.iter()
.map(|block| BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 2);
for block in [&blocks[1], &blocks[1], &blocks[0]] {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
}
let mut submitted = Vec::new();
while submitted.len() < 2 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::Misbehavior { reason, .. } => {
panic!("duplicate buffered body was misclassified: {reason:?}")
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
assert_eq!(submitted, vec![block::Height(1), block::Height(2)]);
let quiet = tokio::time::timeout(Duration::from_millis(100), async {
while let Some(action) = actions.recv().await {
if let BlockSyncAction::Misbehavior { reason, .. } = action {
panic!("duplicate buffered body was misclassified after submit: {reason:?}");
}
}
})
.await;
assert!(quiet.is_err());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_rapid_status_growth_without_spam_score() {
let config = ZakuraBlockSyncConfig::default();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
drop(tip_tx);
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer_id = peer(46);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
for servable_high in [block::Height(1), block::Height(2)] {
let hash_byte = u8::try_from(servable_high.0).expect("test height fits in u8");
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high,
tip_hash: block::Hash([hash_byte; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
}
let quiet = tokio::time::timeout(Duration::from_millis(100), async {
while let Some(action) = actions.recv().await {
if let BlockSyncAction::Misbehavior { reason, .. } = action {
panic!("rapid status growth was misclassified: {reason:?}");
}
}
})
.await;
assert!(quiet.is_err());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_redundant_status_burst_without_spam_score() {
let config = ZakuraBlockSyncConfig::default();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
drop(tip_tx);
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer_id = peer(47);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
let status = BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes");
for _ in 0..3 {
inbound_tx
.send(status.clone())
.await
.expect("redundant status queues");
}
let quiet = tokio::time::timeout(Duration::from_millis(100), async {
while let Some(action) = actions.recv().await {
if let BlockSyncAction::Misbehavior { reason, .. } = action {
panic!("redundant status burst was misclassified: {reason:?}");
}
}
})
.await;
assert!(quiet.is_err());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_rejects_block_hash_mismatch_without_hard_drop_for_size_mismatch() {
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
immediate_body_download_config(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(
immediate_body_download_config(),
handle.clone(),
);
let peer = peer(41);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status queues");
tip_tx
.send((block::Height(1), block::Hash([9; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([9; 32]),
size: BlockSizeEstimate::Advertised(1),
}]))
.await
.expect("needed metadata queues");
while !matches!(
BlockSyncMessage::decode_frame(
tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv())
.await
.expect("outbound frame arrives")
.expect("outbound channel is live")
)
.expect("frame decodes"),
BlockSyncMessage::GetBlocks { .. }
) {}
inbound_tx
.send(
BlockSyncMessage::Block(mainnet_block(&BLOCK_MAINNET_1_BYTES))
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::Misbehavior { reason, .. } => {
assert_eq!(reason, BlockSyncMisbehavior::InvalidBlock);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before invalid-block report: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn scheduled_get_blocks_is_sent_once_via_session() {
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(57);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([9; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status queues");
tip_tx
.send((block::Height(1), block::Hash([9; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([9; 32]),
size: BlockSizeEstimate::Advertised(1),
}]))
.await
.expect("needed metadata queues");
let mut get_blocks = 0usize;
let mut frames = 0usize;
while frames < 16 {
match tokio::time::timeout(Duration::from_millis(300), outbound_rx.recv()).await {
Ok(Some(frame)) => {
frames += 1;
if matches!(
BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes"),
BlockSyncMessage::GetBlocks { .. }
) {
get_blocks += 1;
}
}
_ => break,
}
}
assert_eq!(
get_blocks, 1,
"one scheduled request must produce exactly one outbound GetBlocks via \
BlockSyncPeerSession; a different count would mean a mirroring path is \
(double-)sending in addition to the authoritative direct path",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_scores_peer_whose_invalid_body_is_rejected_by_consensus() {
let request_bytes: u32 = 10_000;
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
let blocks = mainnet_blocks_1_to_3();
let bad_body = block_with_bad_merkle_root(&blocks[0], &blocks[1]);
assert_eq!(bad_body.hash(), blocks[0].hash());
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (bad_peer, bad_inbound, mut bad_outbound) = connect_peer_with_status(
&service,
&mut actions,
40,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(request_bytes),
}]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut bad_outbound).await,
(block::Height(1), 1)
);
bad_inbound
.send(
BlockSyncMessage::Block(bad_body)
.encode_frame()
.expect("bad block frame encodes"),
)
.await
.expect("bad block frame queues");
let submit_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => {
assert_eq!(block.hash(), blocks[0].hash());
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before invalid body submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
token: submit_token,
height: block::Height(1),
hash: blocks[0].hash(),
result: BlockApplyResult::Rejected,
local_frontier: None,
})
.await
.expect("apply-finished event queues");
let scored = loop {
if let BlockSyncAction::Misbehavior { peer, reason } = next_action(&mut actions).await {
assert_eq!(peer, bad_peer);
assert_eq!(reason, BlockSyncMisbehavior::InvalidBlock);
break true;
}
};
assert!(
scored,
"a consensus apply rejection must score the peer that delivered the body",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_serves_committed_blocks_with_count_and_byte_clamps() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let mut config = ZakuraBlockSyncConfig {
max_blocks_per_response: 2,
max_response_bytes: block1_size,
..ZakuraBlockSyncConfig::default()
};
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: blocks[2].hash(),
},
(block::Height(4), block::Hash([4; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
60,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 10,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("GetBlocks frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => {
assert_eq!(peer, peer_id);
assert_eq!(start, block::Height(1));
assert_eq!(count, 2);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before block range query: {action:?}"),
}
}
handle
.send(BlockSyncEvent::BlockRangeResponseReady {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 2,
blocks: vec![
(
block::Height(1),
blocks[0].clone(),
usize::try_from(block1_size).expect("block size fits usize"),
),
(
block::Height(2),
blocks[1].clone(),
usize::try_from(block_size(&blocks[1])).expect("block size fits usize"),
),
],
})
.await
.expect("served block response queues");
assert_eq!(
wait_for_outbound_block(&mut outbound_rx).await.hash(),
blocks[0].hash()
);
assert_eq!(
wait_for_outbound_blocks_done(&mut outbound_rx).await,
(block::Height(1), 1),
"max_response_bytes clamps the served response to one body"
);
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(4),
count: 1,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("above-tip GetBlocks frame queues");
assert_eq!(
wait_for_outbound_range_unavailable(&mut outbound_rx).await,
(block::Height(4), 1)
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_never_serves_reorder_buffer_bodies() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
61,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(3), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
let quiet = tokio::time::timeout(Duration::from_millis(50), async {
while let Some(action) = actions.recv().await {
if matches!(action, BlockSyncAction::SubmitBlock { .. }) {
panic!("height 3 must stay buffered behind the height 2 gap");
}
}
})
.await;
assert!(quiet.is_err());
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(3),
count: 1,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("GetBlocks frame queues");
assert_eq!(
wait_for_outbound_range_unavailable(&mut outbound_rx).await,
(block::Height(3), 1),
"uncommitted reorder-buffer body must not be served"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_schedules_gap_below_buffered_reorder_run() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
63,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(3), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks { start_height, .. } => {
panic!("buffered (covered) height {start_height:?} must not be re-requested")
}
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message after buffering block: {msg:?}"),
}
}
action = tokio::time::timeout(Duration::from_millis(50), actions.recv()) => {
match action {
Ok(Some(BlockSyncAction::QueryNeededBlocks { .. })) => {}
Ok(Some(other)) => panic!("unexpected action after buffering block: {other:?}"),
Ok(None) | Err(_) => break,
}
}
}
}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("needed metadata queues");
let (got_start, _count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(
got_start,
block::Height(2),
"reactor must schedule the gap at height 2 below the buffered reorder run",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_debounces_status_advertisements_on_serving_tip_change() {
let mut config = ZakuraBlockSyncConfig {
status_refresh_interval: Duration::from_secs(60),
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(4), block::Hash([4; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
62,
block::Height(4),
block::Hash([4; 32]),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
wait_for_outbound_status(&mut outbound_rx).await;
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
}))
.await
.expect("unchanged frontier queues");
assert!(
tokio::time::timeout(Duration::from_millis(50), outbound_rx.recv())
.await
.is_err(),
"unchanged serving range must not advertise"
);
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: block::Hash([1; 32]),
}))
.await
.expect("changed frontier queues");
match next_outbound_message(&mut outbound_rx).await {
BlockSyncMessage::Status(status) => {
assert_eq!(status.servable_high, block::Height(1));
assert_eq!(handle.local_status().servable_high, block::Height(1));
}
msg => panic!("expected debounced Status after serving tip change, got {msg:?}"),
}
for height in [2, 3] {
let hash_byte = u8::try_from(height).expect("test height fits in u8");
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(height),
verified_block_hash: block::Hash([hash_byte; 32]),
}))
.await
.expect("burst frontier queues");
}
assert!(
tokio::time::timeout(Duration::from_millis(50), outbound_rx.recv())
.await
.is_err(),
"rapid serving-tip changes must be debounced to one Status per window"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_retries_status_to_peer_without_status_when_local_status_unchanged() {
let mut config = ZakuraBlockSyncConfig {
status_refresh_interval: Duration::from_millis(50),
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer = peer(63);
let (_inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer,
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
assert!(matches!(
next_outbound_message(&mut outbound_rx).await,
BlockSyncMessage::Status(_)
));
assert!(
tokio::time::timeout(Duration::from_millis(25), outbound_rx.recv())
.await
.is_err(),
"initial Status send must consume the peer refresh allowance"
);
assert!(matches!(
next_outbound_message(&mut outbound_rx).await,
BlockSyncMessage::Status(_)
));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_replies_to_first_status_when_connect_status_queue_was_full() {
let mut config = ZakuraBlockSyncConfig {
status_refresh_interval: Duration::from_millis(50),
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 1;
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(64);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(1);
outbound_tx
.try_send(
BlockSyncMessage::Status(status())
.encode_frame()
.expect("filler status frame encodes"),
)
.expect("outbound queue starts full");
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer,
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.peer_snapshot().outbound_peers == 1 {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("peer is admitted while outbound queue is full");
let _ = outbound_rx.recv().await.expect("filler frame drains");
send_inbound(&inbound_tx, BlockSyncMessage::Status(status())).await;
assert!(matches!(
next_outbound_message(&mut outbound_rx).await,
BlockSyncMessage::Status(_)
));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_does_not_ping_pong_rapid_repeated_status() {
let mut config = ZakuraBlockSyncConfig {
status_refresh_interval: Duration::from_millis(50),
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer = peer(65);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer,
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
send_inbound(&inbound_tx, BlockSyncMessage::Status(status())).await;
wait_for_outbound_status(&mut outbound_rx).await;
send_inbound(&inbound_tx, BlockSyncMessage::Status(status())).await;
assert!(
tokio::time::timeout(Duration::from_millis(25), outbound_rx.recv())
.await
.is_err(),
"a rapid second inbound Status must not trigger another Status reply"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_exchange_watch_converges_to_latest_valid_frontier() {
let initial = test_frontier_update(0, 0, 0, FrontierChange::Snapshot);
let (exchange, startup) =
exchange_block_sync_startup(initial, immediate_body_download_config());
exchange.publish_frontier(
test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced),
"test",
);
exchange.publish_frontier(
test_frontier_update(0, 0, 2, FrontierChange::HeaderAdvanced),
"test",
);
exchange.publish_frontier(
test_frontier_update(0, 0, 5, FrontierChange::HeaderAdvanced),
"test",
);
exchange.publish_frontier(
test_frontier_update(0, 0, 5, FrontierChange::HeaderAdvanced),
"test",
);
let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(5)).await;
assert_eq!(
exchange.current_frontier().frontier.best_header,
test_frontier(5)
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_exchange_progress_retries_after_empty_needed_blocks() {
let initial = test_frontier_update(0, 0, 0, FrontierChange::Snapshot);
let (exchange, startup) =
exchange_block_sync_startup(initial, immediate_body_download_config());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
exchange.publish_frontier(
test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await;
handle
.send(BlockSyncEvent::NeededBlocks(Vec::new()))
.await
.expect("empty needed-blocks event queues");
exchange.publish_frontier(
test_frontier_update(0, 0, 4, FrontierChange::HeaderAdvanced),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
reactor_task.abort();
}
#[tokio::test]
async fn reactor_exchange_body_progress_retries_after_header_tip_stops() {
let initial = test_frontier_update(0, 0, 0, FrontierChange::Snapshot);
let (exchange, startup) =
exchange_block_sync_startup(initial, immediate_body_download_config());
let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
exchange.publish_frontier(
test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await;
exchange.publish_frontier(
test_frontier_update(0, 1, 0, FrontierChange::VerifiedGrow),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(1), block::Height(3)).await;
reactor_task.abort();
}
#[tokio::test]
async fn reactor_exchange_coalesced_header_advance_catches_body_frontier_up() {
let initial = test_frontier_update(0, 0, 0, FrontierChange::Snapshot);
let (exchange, startup) =
exchange_block_sync_startup(initial, immediate_body_download_config());
exchange.publish_frontier(
test_frontier_update(0, 3, 0, FrontierChange::VerifiedGrow),
"test",
);
exchange.publish_frontier(
test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced),
"test",
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
tokio::time::timeout(Duration::from_secs(1), async {
loop {
let status = handle.local_status();
if status.servable_high == block::Height(3) {
assert_eq!(status.tip_hash, test_frontier(3).hash);
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("coalesced header update catches the body frontier up");
reactor_task.abort();
}
#[tokio::test]
async fn reactor_exchange_ignores_stale_grow_but_accepts_reset() {
let initial = test_frontier_update(0, 5, 10, FrontierChange::Snapshot);
let (exchange, startup) =
exchange_block_sync_startup(initial, immediate_body_download_config());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
wait_for_query_needed_blocks(&mut actions, block::Height(5), block::Height(10)).await;
exchange.publish_frontier(
test_frontier_update(0, 4, 10, FrontierChange::VerifiedGrow),
"test",
);
assert!(
tokio::time::timeout(Duration::from_millis(50), actions.recv())
.await
.is_err(),
"stale lower VerifiedGrow must not trigger a lower body query"
);
assert_eq!(handle.local_status().servable_high, block::Height(5));
exchange.publish_frontier(
test_frontier_update(0, 4, 0, FrontierChange::VerifiedReset),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(4), block::Height(10)).await;
assert_eq!(handle.local_status().servable_high, block::Height(4));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_preserves_successor_work_across_stale_finalized_reset() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.peer_limits.outbound_queue_depth = 16;
config.request_timeout = Duration::from_secs(300);
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(2),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
72,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(3), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::SubmitBlock { .. }
) {}
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(2),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}))
.await
.expect("stale finalized reset queues");
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("duplicate needed metadata queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
if let BlockSyncMessage::GetBlocks {
start_height: block::Height(3),
..
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
panic!("stale finalized reset made an already-submitted successor requestable again");
}
}
action = tokio::time::timeout(Duration::from_millis(100), actions.recv()) => {
match action {
Ok(Some(_)) => {}
Ok(None) | Err(_) => break,
}
}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_exchange_reanchor_lowers_only_best_header_target() {
let initial = test_frontier_update(0, 5, 10, FrontierChange::Snapshot);
let (exchange, startup) =
exchange_block_sync_startup(initial, immediate_body_download_config());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
wait_for_query_needed_blocks(&mut actions, block::Height(5), block::Height(10)).await;
exchange.publish_frontier(
test_frontier_update(0, 1, 7, FrontierChange::HeaderReanchored),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(5), block::Height(7)).await;
assert_eq!(handle.local_status().servable_high, block::Height(5));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_exchange_reanchor_requeries_while_downloads_in_flight() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes =
BS_PER_BLOCK_WORST_CASE_BYTES * u64::try_from(blocks.len()).expect("block count fits u64");
config.request_timeout = Duration::from_secs(300);
let initial = test_frontier_update(0, 0, 3, FrontierChange::Snapshot);
let (exchange, startup) = exchange_block_sync_startup(initial, config.clone());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
67,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
exchange.publish_frontier(
test_frontier_update(0, 0, 1, FrontierChange::HeaderReanchored),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await;
exchange.publish_frontier(
test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await;
reactor_task.abort();
}
#[tokio::test]
async fn reactor_exchange_reanchor_releases_stale_submitted_bodies() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes =
BS_PER_BLOCK_WORST_CASE_BYTES * u64::try_from(blocks.len()).expect("block count fits u64");
config.request_timeout = Duration::from_secs(300);
let initial = test_frontier_update(0, 0, 3, FrontierChange::Snapshot);
let (exchange, startup) = exchange_block_sync_startup(initial, config.clone());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
66,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
for block in &blocks {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
}
let mut submitted = Vec::new();
while submitted.len() < blocks.len() {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before all submitted bodies: {action:?}"),
}
}
assert_eq!(
submitted,
vec![block::Height(1), block::Height(2), block::Height(3)]
);
exchange.publish_frontier(
test_frontier_update(0, 0, 1, FrontierChange::HeaderReanchored),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await;
exchange.publish_frontier(
test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced),
"test",
);
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata after reanchor queues");
let (got_start, got_count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(got_start, block::Height(1));
assert_eq!(
got_count, 3,
"reanchored headers must release old submitted bodies and request them again",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_clamps_tiny_submitted_apply_config_above_checkpoint_range() {
let blocks = fake_sequential_blocks(4);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = u64::MAX;
config.max_submitted_block_applies = 2;
config.request_timeout = Duration::from_secs(300);
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(4), blocks[3].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
67,
block::Height(4),
blocks[3].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
let (_start, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(count, 4);
for block in &blocks {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
}
let mut submitted = Vec::new();
while submitted.len() < 4 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while collecting submissions: {action:?}"),
}
}
assert_eq!(
submitted,
vec![
block::Height(1),
block::Height(2),
block::Height(3),
block::Height(4),
],
"tiny configured submit caps are raised to the checkpoint-safe floor"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_far_ahead_header_tip_queries_only_next_refill_window() {
let best_header_tip = block::Height(50_000);
let (_tip_tx, tip_rx) = watch::channel((best_header_tip, block::Hash([50; 32])));
let config = ZakuraBlockSyncConfig {
max_blocks_per_response: 1,
max_inflight_requests: 1,
..ZakuraBlockSyncConfig::default()
};
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(best_header_tip, block::Hash([50; 32])),
tip_rx,
config,
);
let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
assert!(matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
limit: 2,
best_header_tip: block::Height(50_000),
}
));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_refill_window_advances_past_claimed_heights() {
let best_header_tip = block::Height(50_000);
let (_tip_tx, tip_rx) = watch::channel((best_header_tip, block::Hash([50; 32])));
let config = ZakuraBlockSyncConfig {
max_blocks_per_response: 1,
max_inflight_requests: 4,
..ZakuraBlockSyncConfig::default()
};
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(best_header_tip, block::Hash([50; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
assert!(matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
limit: 8,
best_header_tip: block::Height(50_000),
}
));
let metas: Vec<_> = (1..=3)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("height fits u8"); 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect();
handle
.send(BlockSyncEvent::NeededBlocks(metas))
.await
.expect("needed-blocks event queues");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(50_001),
hash: block::Hash([51; 32]),
})
.await
.expect("header-tip event queues");
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from,
limit,
best_header_tip,
} => {
assert_eq!(
from,
block::Height(4),
"refill must advance past the claimed heights, not rescan from the floor",
);
assert_eq!(limit, 8);
assert_eq!(best_header_tip, block::Height(50_001));
}
action => panic!("expected the advanced refill query, got {action:?}"),
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_stale_non_reset_frontier_updates() {
let (_tip_tx, tip_rx) = watch::channel((block::Height(3600), block::Hash([36; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(3200),
verified_block_tip: block::Height(3200),
verified_block_hash: block::Hash([32; 32]),
},
(block::Height(3600), block::Hash([36; 32])),
tip_rx,
immediate_body_download_config(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
assert!(matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(3201),
best_header_tip: block::Height(3600),
..
}
));
handle
.send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers {
finalized_height: block::Height(3200),
verified_block_tip: block::Height(2913),
verified_block_hash: block::Hash([29; 32]),
}))
.await
.expect("stale grow event queues");
assert!(
tokio::time::timeout(Duration::from_millis(50), actions.recv())
.await
.is_err(),
"stale lower grow frontier must not query from the lower height"
);
assert_eq!(handle.local_status().servable_high, block::Height(3200));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_retries_matched_range_unavailable_without_scoring_peer() {
let blocks = mainnet_blocks_1_to_3();
let mut config = fill_loop_mechanics_config();
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
63,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2)
);
inbound_tx
.send(
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: 2,
}
.encode_frame()
.expect("RangeUnavailable frame encodes"),
)
.await
.expect("RangeUnavailable frame queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2),
"matched RangeUnavailable should retry the original range without scoring the serving peer",
);
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
inbound_tx
.send(
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(2),
count: 1,
}
.encode_frame()
.expect("RangeUnavailable frame encodes"),
)
.await
.expect("unmatched RangeUnavailable frame queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks { .. } => {
panic!("unmatched RangeUnavailable must not trigger a fresh request")
}
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message after unmatched RangeUnavailable: {msg:?}"),
}
}
action = tokio::time::timeout(Duration::from_millis(50), actions.recv()) => {
match action {
Ok(Some(BlockSyncAction::Misbehavior { .. })) => {
panic!("unmatched RangeUnavailable must not score the serving peer")
}
Ok(Some(_)) => {}
Ok(None) | Err(_) => break,
}
}
}
}
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_does_not_wedge_honest_peer_under_range_unavailable_spam() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.request_timeout = Duration::from_secs(300);
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (m1, m1_in, mut m1_out) = connect_peer_with_status(
&service,
&mut actions,
0x01,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let (m2, m2_in, mut m2_out) = connect_peer_with_status(
&service,
&mut actions,
0x02,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let (h, _h_in, mut h_out) = connect_peer_with_status(
&service,
&mut actions,
0x03,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("needed metadata queues");
let range_unavailable = || {
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: 2,
}
.encode_frame()
.expect("RangeUnavailable frame encodes")
};
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = vec![
(m1.clone(), &mut m1_out),
(m2.clone(), &mut m2_out),
(h.clone(), &mut h_out),
];
let mut honest_offered = false;
for _ in 0..16 {
let (peer, _start, _count) = wait_for_getblocks_across(&mut outbound_by_peer).await;
if peer == h {
honest_offered = true;
break;
} else if peer == m1 {
m1_in
.send(range_unavailable())
.await
.expect("m1 RangeUnavailable queues");
} else if peer == m2 {
m2_in
.send(range_unavailable())
.await
.expect("m2 RangeUnavailable queues");
}
}
assert!(
honest_offered,
"honest peer must be offered the contested range after both fanout peers fail it"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_range_unavailable_retries_only_unverified_suffix() {
let blocks = mainnet_blocks_1_to_3();
let mut config = fill_loop_mechanics_config();
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
64,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2)
);
handle
.send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers {
finalized_height: block::Height(1),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}))
.await
.expect("frontier grow queues");
inbound_tx
.send(
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: 2,
}
.encode_frame()
.expect("RangeUnavailable frame encodes"),
)
.await
.expect("RangeUnavailable frame queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1),
"late RangeUnavailable must not retry the already verified prefix",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_backpressures_serving_slots_without_scoring_peer() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_requests: 1,
..ZakuraBlockSyncConfig::default()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
63,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tokio::time::sleep(Duration::from_millis(10)).await;
for _ in 0..2 {
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 1,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("GetBlocks frame queues");
}
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryBlocksByHeightRange { .. }
) {}
assert_eq!(
wait_for_outbound_range_unavailable(&mut outbound_rx).await,
(block::Height(1), 1),
"serving-slot saturation should backpressure the requester, not score it as spam",
);
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
handle
.send(BlockSyncEvent::BlockRangeResponseFinished {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 1,
returned_count: 1,
})
.await
.expect("serving slot release queues");
reactor_task.abort();
}
#[tokio::test]
async fn reactor_full_serving_queue_drops_without_disconnecting_peer() {
let blocks = mainnet_blocks_1_to_3();
let config = ZakuraBlockSyncConfig {
max_blocks_per_response: 16,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
..ZakuraBlockSyncConfig::default()
};
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: blocks[2].hash(),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(66);
let cancel = CancellationToken::new();
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(2);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
cancel.clone(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 16,
max_inflight_requests: 8,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 3,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("GetBlocks frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => {
assert_eq!(peer, peer_id);
assert_eq!(start, block::Height(1));
assert_eq!(count, 3);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before block range query: {action:?}"),
}
}
let served: Vec<_> = blocks
.iter()
.map(|block| {
(
block.coinbase_height().expect("test block has height"),
block.clone(),
usize::try_from(block_size(block)).expect("block size fits usize"),
)
})
.collect();
handle
.send(BlockSyncEvent::BlockRangeResponseReady {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 3,
blocks: served,
})
.await
.expect("served block response queues");
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
!cancel.is_cancelled(),
"a full serving queue must drop sends, not cancel the peer",
);
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
while tokio::time::timeout(
Duration::from_millis(100),
next_outbound_message(&mut outbound_rx),
)
.await
.is_ok()
{}
handle
.send(BlockSyncEvent::BlockRangeResponseFinished {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 3,
returned_count: 3,
})
.await
.expect("serving slot release queues");
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 1,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("re-request GetBlocks frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryBlocksByHeightRange { start, count, .. } => {
assert_eq!(start, block::Height(1));
assert_eq!(count, 1);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before re-request query: {action:?}"),
}
}
handle
.send(BlockSyncEvent::BlockRangeResponseReady {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 1,
blocks: vec![(
block::Height(1),
blocks[0].clone(),
usize::try_from(block_size(&blocks[0])).expect("block size fits usize"),
)],
})
.await
.expect("re-served block response queues");
assert_eq!(
wait_for_outbound_block(&mut outbound_rx).await.hash(),
blocks[0].hash(),
"serving resumes once the queue drains",
);
assert!(!cancel.is_cancelled());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_publishes_block_sync_candidate_gap() {
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(77);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
tip_tx
.send((block::Height(2), block::Hash([2; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Unknown,
},
]))
.await
.expect("needed blocks event queues");
let mut candidates = handle.subscribe_candidate_state();
let observed = tokio::time::timeout(Duration::from_secs(1), async {
loop {
candidates
.changed()
.await
.expect("candidate watch remains open");
let state = candidates.borrow().clone();
if state.missing_block_bodies == vec![block::Height(1), block::Height(2)] {
return state;
}
}
})
.await
.unwrap_or_else(|_| handle.candidate_state());
assert_eq!(
observed.missing_block_bodies,
vec![block::Height(1), block::Height(2)]
);
assert!(
observed.admitted_node_ids.is_empty(),
"a peer without block-sync status must not satisfy body-sync demand"
);
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: block::Hash([2; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status queues");
let peer_node_id =
node_id_from_block_peer_id(&peer_id).expect("test peer id is a valid node id");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
candidates
.changed()
.await
.expect("candidate watch remains open");
if candidates.borrow().admitted_node_ids == vec![peer_node_id] {
return;
}
}
})
.await
.expect("status-bearing peer is published as an admitted candidate");
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: block::Hash([2; 32]),
}))
.await
.expect("frontier event queues");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.candidate_state().missing_block_bodies.is_empty() {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("candidate state clears after the gap is gone");
reactor_task.abort();
}
#[tokio::test]
async fn oversize_body_policy_reports_size_mismatch_and_retries_without_buffering() {
let mut config = ZakuraBlockSyncConfig {
size_deviation_tolerance: 100,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 8;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(42);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer,
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status queues");
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
tip_tx
.send((block::Height(1), block.hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(1),
}]))
.await
.expect("needed metadata queues");
while !matches!(
BlockSyncMessage::decode_frame(
tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv())
.await
.expect("outbound frame arrives")
.expect("outbound channel is live")
)
.expect("frame decodes"),
BlockSyncMessage::GetBlocks { .. }
) {}
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::Misbehavior { reason, .. } => {
assert_eq!(reason, BlockSyncMisbehavior::SizeMismatch);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action during size mismatch test: {action:?}"),
}
}
let no_submit = tokio::time::timeout(Duration::from_millis(200), async {
while let Some(action) = actions.recv().await {
if matches!(action, BlockSyncAction::SubmitBlock { .. }) {
return false;
}
}
true
})
.await
.unwrap_or(true);
assert!(
no_submit,
"oversize body is not submitted after SizeMismatch"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_known_peer_unsolicited_blocks_done_is_reported_as_misbehavior() {
let config = ZakuraBlockSyncConfig::default();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
63,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(7),
returned: 1,
}
.encode_frame()
.expect("BlocksDone frame encodes"),
)
.await
.expect("BlocksDone frame queues");
let mut saw_unsolicited_done = false;
while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await
{
if let BlockSyncAction::Misbehavior { peer, reason } = action {
if peer == peer_id && reason == BlockSyncMisbehavior::UnsolicitedDone {
saw_unsolicited_done = true;
break;
}
}
}
assert!(
saw_unsolicited_done,
"a known peer's unsolicited BlocksDone with no matching outstanding request \
must be reported as Misbehavior::UnsolicitedDone (SR-6/SR-7), but the reactor \
silently tolerated it and kept the peer connected",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_unmatched_body_for_height_active_on_another_request() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer1, inbound1, mut outbound1) = connect_peer_with_status(
&service,
&mut actions,
65,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let (peer2, inbound2, mut outbound2) = connect_peer_with_status(
&service,
&mut actions,
66,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("needed metadata queues");
let (requested_peer, start_height, _count) = {
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = vec![
(peer1.clone(), &mut outbound1),
(peer2.clone(), &mut outbound2),
];
wait_for_getblocks_across(&mut outbound_by_peer).await
};
assert_eq!(start_height, block::Height(2));
handle
.send(BlockSyncEvent::NeededBlocks(Vec::new()))
.await
.expect("empty needed metadata queues");
let (late_peer, late_inbound) = if requested_peer == peer1 {
(peer2, inbound2)
} else {
(peer1, inbound1)
};
send_inbound(&late_inbound, BlockSyncMessage::Block(blocks[1].clone())).await;
let submitted = tokio::time::timeout(Duration::from_secs(1), async {
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => return block.hash(),
BlockSyncAction::QueryNeededBlocks { .. } => {}
BlockSyncAction::Misbehavior { peer, reason } => {
assert_ne!(
peer, late_peer,
"late active body was reported as {reason:?}"
);
}
action => {
panic!("unexpected action while waiting for late body submit: {action:?}")
}
}
}
})
.await
.expect("late active body is accepted and submitted");
assert_eq!(submitted, blocks[1].hash());
send_inbound(
&late_inbound,
BlockSyncMessage::BlocksDone {
start_height: block::Height(2),
returned: 1,
},
)
.await;
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(200), actions.recv()).await
{
if let BlockSyncAction::Misbehavior { peer, reason } = action {
assert_ne!(
peer, late_peer,
"late response for an active request was reported as {reason:?}"
);
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_duplicate_response_at_body_download_floor() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
65,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
let (token, hash) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => break (token, block.hash()),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while waiting for submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
token,
height: block::Height(2),
hash,
result: BlockApplyResult::Committed,
local_frontier: None,
})
.await
.expect("apply result queues");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("duplicate block frame encodes"),
)
.await
.expect("duplicate block frame queues");
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(2),
returned: 1,
}
.encode_frame()
.expect("duplicate terminator frame encodes"),
)
.await
.expect("duplicate terminator frame queues");
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(200), actions.recv()).await
{
if let BlockSyncAction::Misbehavior { peer, reason } = action {
assert_ne!(
peer, peer_id,
"duplicate response at body_download_floor was reported as {reason:?}"
);
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_matched_duplicate_response_at_body_download_floor() {
let blocks = mainnet_blocks_1_to_3();
let block2_size = block_size(&blocks[1]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 2;
let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(4), block::Hash([4; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_a, inbound_a, mut outbound_a) = connect_peer_with_status(
&service,
&mut actions,
66,
block::Height(4),
block::Hash([4; 32]),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let (peer_b, inbound_b, mut outbound_b) = connect_peer_with_status(
&service,
&mut actions,
67,
block::Height(4),
block::Hash([4; 32]),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("needed metadata queues");
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = vec![
(peer_a.clone(), &mut outbound_a),
(peer_b.clone(), &mut outbound_b),
];
let first_request = wait_for_getblocks_across(&mut outbound_by_peer).await;
assert_eq!((first_request.1, first_request.2), (block::Height(2), 1));
let (other_peer, requested_inbound, other_inbound) = if first_request.0 == peer_a {
(peer_b.clone(), &inbound_a, &inbound_b)
} else {
(peer_a.clone(), &inbound_b, &inbound_a)
};
send_inbound(
requested_inbound,
BlockSyncMessage::Block(blocks[1].clone()),
)
.await;
let (token, hash) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block } => break (token, block.hash()),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while waiting for submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
token,
height: block::Height(2),
hash,
result: BlockApplyResult::Committed,
local_frontier: None,
})
.await
.expect("apply result queues");
let _ = &other_peer;
send_inbound(other_inbound, BlockSyncMessage::Block(blocks[1].clone())).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[2]),
BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("next needed metadata queues");
let mut requested: Vec<block::Height> = Vec::new();
tokio::time::timeout(Duration::from_secs(2), async {
loop {
let (peer, start, count) = wait_for_getblocks_across(&mut outbound_by_peer).await;
assert!(
peer == peer_a || peer == peer_b,
"request should target one of the connected peers"
);
for offset in 0..count {
if let Some(height) = height_after_count(start, offset) {
requested.push(height);
}
}
if requested.contains(&block::Height(3)) && requested.contains(&block::Height(4)) {
break;
}
}
})
.await
.expect("both remaining heights are requested within the unchanged budget");
requested.sort_unstable();
let mut deduped = requested.clone();
deduped.dedup();
assert_eq!(
requested, deduped,
"a matched duplicate response at the body floor must not consume reorder budget \
(no height should be fetched twice)"
);
assert!(
requested.contains(&block::Height(3)) && requested.contains(&block::Height(4)),
"both needed heights must be fetched once the duplicate releases its transient reservation"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_scores_unsolicited_terminator_from_connected_peer() {
let config = ZakuraBlockSyncConfig::default();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config.clone(), handle.clone());
let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
64,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
send_inbound(
&inbound_tx,
BlockSyncMessage::BlocksDone {
start_height: block::Height(7),
returned: 1,
},
)
.await;
let mut saw_unsolicited_done = false;
while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await
{
if let BlockSyncAction::Misbehavior { peer, reason } = action {
if peer == peer_id && reason == BlockSyncMisbehavior::UnsolicitedDone {
saw_unsolicited_done = true;
break;
}
}
}
assert!(
saw_unsolicited_done,
"an unsolicited terminator from a connected peer is hard-scored UnsolicitedDone"
);
reactor_task.abort();
}
#[tokio::test]
async fn repeated_misbehavior_is_recorded_without_disconnecting_the_peer() {
let config = ZakuraBlockSyncConfig::default();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let probe = peer(7);
let (probe_inbound_tx, probe_inbound_rx) = framed_channel(8);
let (probe_outbound_tx, _probe_outbound_rx) = framed_channel(8);
service.add_peer(Peer::new_with_direction(
probe.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
HashMap::from([(
ZAKURA_STREAM_BLOCK_SYNC,
(probe_inbound_rx, probe_outbound_tx),
)]),
CancellationToken::new(),
));
tokio::time::timeout(Duration::from_secs(1), async {
while service.peer_count() == 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("probe peer connects");
for _ in 0..8 {
send_inbound(
&probe_inbound_tx,
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 1,
},
)
.await;
}
tokio::time::timeout(Duration::from_secs(2), async {
loop {
if let BlockSyncAction::Misbehavior { peer, reason } = next_action(&mut actions).await {
if peer == probe && reason == BlockSyncMisbehavior::GetBlocksSpam {
break;
}
}
}
})
.await
.expect("a misbehavior action is recorded for the spamming peer");
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
service.peer_count(),
1,
"misbehavior is record-only: a repeatedly-misbehaving peer must NOT be disconnected",
);
reactor_task.abort();
}