use std::{
collections::{BTreeMap, BTreeSet, HashMap},
future,
};
use proptest::{prop_assert, prop_assert_eq};
use super::*;
use super::{
config::{
BS_CHECKPOINT_RANGE_BYTE_FLOOR, BS_PER_BLOCK_WORST_CASE_BYTES,
DEFAULT_BS_BBR_RELIABILITY_WEIGHT_PERCENT, DEFAULT_BS_FLOOR_PEER_AVOID_COOLDOWN,
DEFAULT_BS_INITIAL_BLOCK_PROBE_REQUESTS, DEFAULT_BS_MAX_INFLIGHT_BLOCK_BYTES,
DEFAULT_BS_MAX_REORDER_LOOKAHEAD_BYTES, DEFAULT_BS_MAX_REQUESTS_WITHOUT_BLOCK_PROGRESS,
DEFAULT_BS_MAX_RESPONSE_BYTES, DEFAULT_BS_NO_PROGRESS_PEER_COOLDOWN,
DEFAULT_BS_REQUEST_TIMEOUT, MAX_BS_INFLIGHT_REQUESTS, MAX_BS_RESPONSE_BYTES,
MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES,
},
peer_registry::{OutstandingMeta, PeerRegistry},
reactor::{
node_id_from_block_peer_id, EMPTY_STATE_HEADER_QUIET_MIN_LAG,
EMPTY_STATE_HEADER_QUIET_PERIOD,
},
reorder::*,
request::*,
sequencer::*,
sequencer_task::{initial_view, SequencedBody, SequencerControlInput, SequencerTask},
state::*,
work_queue::WorkQueue,
};
use crate::zakura::OrderedSessionDemand;
use crate::zakura::{
framed_channel,
testkit::{await_until, TraceCapture, TraceValue},
FramedRecv, FramedSend, Peer, Service, ServicePeerSnapshot, ServiceRegistry, StreamMode,
ZakuraBlockSyncCandidateState,
};
use zakura_chain::{
fmt::HexDebug,
serialization::{ZcashDeserializeInto, ZcashSerialize},
transaction::Transaction,
transparent,
};
use zakura_test::vectors::{BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES, BLOCK_MAINNET_3_BYTES};
fn peer(byte: u8) -> ZakuraPeerId {
ZakuraPeerId::new(vec![byte; 32]).expect("test peer id is within bounds")
}
fn mainnet_block(bytes: &[u8]) -> Arc<block::Block> {
Arc::new(bytes.zcash_deserialize_into().expect("block vector parses"))
}
fn mainnet_blocks_1_to_3() -> Vec<Arc<block::Block>> {
vec![
mainnet_block(&BLOCK_MAINNET_1_BYTES),
mainnet_block(&BLOCK_MAINNET_2_BYTES),
mainnet_block(&BLOCK_MAINNET_3_BYTES),
]
}
fn raw_block_payload(block: &Arc<block::Block>) -> Arc<[u8]> {
let frame = BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("test block frame encodes");
Arc::from(frame.payload.into_boxed_slice())
}
fn forked_block(block: &Arc<block::Block>, nonce_tag: u8) -> Arc<block::Block> {
let mut fork = block.as_ref().clone();
let mut header = *fork.header;
header.nonce = HexDebug([nonce_tag; 32]);
fork.header = Arc::new(header);
Arc::new(fork)
}
fn fake_sequential_blocks(count: u32) -> Vec<Arc<block::Block>> {
let template = mainnet_block(&BLOCK_MAINNET_1_BYTES);
(1..=count)
.map(|height| fake_block_at_height(&template, block::Height(height)))
.collect()
}
fn fake_blocks_in_range(start: u32, end: u32) -> Vec<Arc<block::Block>> {
let template = mainnet_block(&BLOCK_MAINNET_1_BYTES);
(start..=end)
.map(|height| fake_block_at_height(&template, block::Height(height)))
.collect()
}
fn fake_block_at_height(template: &Arc<block::Block>, height: block::Height) -> Arc<block::Block> {
let mut block = template.as_ref().clone();
let mut coinbase = block.transactions[0].clone();
let input = match Arc::make_mut(&mut coinbase) {
Transaction::V1 { inputs, .. }
| Transaction::V2 { inputs, .. }
| Transaction::V3 { inputs, .. }
| Transaction::V4 { inputs, .. }
| Transaction::V5 { inputs, .. } => &mut inputs[0],
Transaction::V6 { inputs, .. } => &mut inputs[0],
};
match input {
transparent::Input::Coinbase {
height: coinbase_height,
..
} => *coinbase_height = height,
_ => panic!("template block must start with a coinbase input"),
}
block.transactions[0] = coinbase;
let merkle_root = block.transactions.iter().collect::<block::merkle::Root>();
let mut header = *block.header;
header.merkle_root = merkle_root;
block.header = Arc::new(header);
Arc::new(block)
}
fn block_size(block: &block::Block) -> u32 {
u32::try_from(
block
.zcash_serialize_to_vec()
.expect("test block serializes")
.len(),
)
.expect("test block size fits u32")
}
fn status() -> BlockSyncStatus {
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(42),
tip_hash: block::Hash([7; 32]),
max_blocks_per_response: 16,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
}
}
fn immediate_body_download_config() -> ZakuraBlockSyncConfig {
ZakuraBlockSyncConfig {
max_blocks_per_response: MAX_BS_BLOCKS_PER_REQUEST,
..ZakuraBlockSyncConfig::default()
}
}
fn fill_loop_mechanics_config() -> ZakuraBlockSyncConfig {
ZakuraBlockSyncConfig {
initial_block_probe_requests: DEFAULT_BS_MAX_REQUESTS_WITHOUT_BLOCK_PROGRESS,
..immediate_body_download_config()
}
}
fn alarmed_committed_snapshot(
header: zakura_header_chain::Frontier,
) -> zakura_header_chain::EngineSnapshot {
let now = chrono::Utc::now();
let verified = zakura_header_chain::Frontier::new(
block::Height(header.height.0.saturating_sub(1)),
block::Hash([0x50; 32]),
);
let alarm = zakura_header_chain::BodyUnavailableSummary {
started_at: now - chrono::Duration::minutes(12),
attempts: 10,
suppliers: 1,
supplier_set_digest: [0x51; 32],
alarmed: true,
next_probe_at: now + chrono::Duration::minutes(10),
};
zakura_header_chain::EngineSnapshot {
mode: zakura_header_chain::EngineMode::Integrated,
state_version: zakura_header_chain::StateVersion::new(7),
header_generation: zakura_header_chain::HeaderGeneration::new(8),
verified_generation: zakura_header_chain::VerifiedGeneration::new(9),
frontiers: zakura_header_chain::FrontierSet {
finalized: verified,
header_best: header,
verified_best: verified,
},
header_best_score: zakura_header_chain::ChainScore::new(
zakura_header_chain::SuffixWork::zero(),
header.hash,
),
oldest_retained_height: verified.height,
alarms: zakura_header_chain::AlarmSet {
header_best_body_unavailable: Some(alarm),
..Default::default()
},
}
}
fn committed_snapshot(
state_version: u64,
header_generation: u64,
verified_generation: u64,
finalized: zakura_header_chain::Frontier,
verified: zakura_header_chain::Frontier,
header: zakura_header_chain::Frontier,
) -> zakura_header_chain::EngineSnapshot {
zakura_header_chain::EngineSnapshot {
mode: zakura_header_chain::EngineMode::Integrated,
state_version: zakura_header_chain::StateVersion::new(state_version),
header_generation: zakura_header_chain::HeaderGeneration::new(header_generation),
verified_generation: zakura_header_chain::VerifiedGeneration::new(verified_generation),
frontiers: zakura_header_chain::FrontierSet {
finalized,
header_best: header,
verified_best: verified,
},
header_best_score: zakura_header_chain::ChainScore::new(
zakura_header_chain::SuffixWork::zero(),
header.hash,
),
oldest_retained_height: finalized.height,
alarms: zakura_header_chain::AlarmSet::default(),
}
}
fn committed_block_sync_startup(
initial: zakura_header_chain::EngineSnapshot,
config: ZakuraBlockSyncConfig,
) -> (
watch::Sender<Option<zakura_header_chain::CommittedHeaderChainView>>,
BlockSyncStartup,
) {
let frontiers = BlockSyncFrontiers {
finalized_height: initial.frontiers.finalized.height,
verified_block_tip: initial.frontiers.verified_best.height,
verified_block_hash: initial.frontiers.verified_best.hash,
};
let best_header_tip = (
initial.frontiers.header_best.height,
initial.frontiers.header_best.hash,
);
let (snapshots, committed_views) = watch::channel(Some(committed_view(initial, 0)));
let startup = BlockSyncStartup::new_with_committed_views(
frontiers,
best_header_tip,
committed_views,
config,
);
(snapshots, startup)
}
fn committed_view(
snapshot: zakura_header_chain::EngineSnapshot,
epoch: u64,
) -> zakura_header_chain::CommittedHeaderChainView {
zakura_header_chain::CommittedHeaderChainView::new(
snapshot,
zakura_header_chain::BodyWorkEpoch::new(epoch),
)
}
fn test_committed_snapshot(
state_version: u64,
header_generation: u64,
verified_generation: u64,
finalized: (u32, block::Hash),
verified: (u32, block::Hash),
header: (u32, block::Hash),
) -> zakura_header_chain::EngineSnapshot {
committed_snapshot(
state_version,
header_generation,
verified_generation,
zakura_header_chain::Frontier::new(block::Height(finalized.0), finalized.1),
zakura_header_chain::Frontier::new(block::Height(verified.0), verified.1),
zakura_header_chain::Frontier::new(block::Height(header.0), header.1),
)
}
#[test]
fn selected_extension_preserves_body_work_epoch() {
let target = block::Hash([0x80; 32]);
let old = test_committed_snapshot(
1,
2,
3,
(10, block::Hash([0x10; 32])),
(12, block::Hash([0x12; 32])),
(100, target),
);
let advanced = test_committed_snapshot(
2,
2,
4,
(11, block::Hash([0x11; 32])),
(13, block::Hash([0x13; 32])),
(100, target),
);
let old = committed_view(old, 7);
let advanced = committed_view(advanced, 7);
assert_eq!(old.body_work_epoch, advanced.body_work_epoch);
}
#[test]
fn selected_replacement_advances_body_work_epoch() {
let old = test_committed_snapshot(
1,
2,
3,
(10, block::Hash([0x10; 32])),
(12, block::Hash([0x12; 32])),
(100, block::Hash([0x80; 32])),
);
let changed_target = test_committed_snapshot(
2,
3,
4,
(10, block::Hash([0x10; 32])),
(13, block::Hash([0x13; 32])),
(100, block::Hash([0x81; 32])),
);
let old = committed_view(old, 7);
let changed_target = committed_view(changed_target, 8);
assert_ne!(old.body_work_epoch, changed_target.body_work_epoch);
}
#[tokio::test]
async fn same_epoch_verified_growth_without_successor_work_preserves_reset_epoch() {
let verified_hash = block::Hash([0x12; 32]);
let header_hash = block::Hash([0x13; 32]);
let initial = test_committed_snapshot(
1,
1,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(2, header_hash),
);
let (snapshots, startup) =
committed_block_sync_startup(initial, immediate_body_download_config());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let sequencer_view = handle
.routine_wiring
.as_ref()
.expect("the test reactor exposes routine wiring")
.view
.clone();
let reset_epoch = sequencer_view.borrow().reset_epoch;
snapshots
.send(Some(committed_view(
test_committed_snapshot(
2,
1,
2,
(0, block::Hash([0; 32])),
(1, verified_hash),
(2, header_hash),
),
0,
)))
.expect("the verified-growth view publishes");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
..
} => break,
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action after verified growth: {action:?}"),
}
}
await_until(
"verified frontier reaches height 1",
Duration::from_secs(1),
|| sequencer_view.borrow().verified_tip == block::Height(1),
)
.await
.expect("same-epoch verified growth advances the frontier");
assert_eq!(sequencer_view.borrow().reset_epoch, reset_epoch);
reactor_task.abort();
}
fn committed_block_sync_startup_at_heights(
finalized: u32,
verified: u32,
header: u32,
config: ZakuraBlockSyncConfig,
) -> (
watch::Sender<Option<zakura_header_chain::CommittedHeaderChainView>>,
BlockSyncStartup,
) {
let initial = test_committed_snapshot(
1,
1,
1,
(finalized, block::Hash([0; 32])),
(
verified,
block::Hash([u8::try_from(verified).unwrap_or(0); 32]),
),
(header, block::Hash([u8::try_from(header).unwrap_or(0); 32])),
);
committed_block_sync_startup(initial, config)
}
fn round_trip(message: BlockSyncMessage) {
let encoded = message.encode().expect("message encodes");
let decoded = BlockSyncMessage::decode(&encoded).expect("message decodes");
assert_eq!(decoded, message);
}
async fn next_event(events: &mut mpsc::Receiver<BlockSyncEvent>) -> BlockSyncEvent {
tokio::time::timeout(Duration::from_secs(1), events.recv())
.await
.expect("block-sync event should arrive")
.expect("block-sync event channel should stay open")
}
async fn next_action(actions: &mut mpsc::Receiver<BlockSyncAction>) -> BlockSyncAction {
tokio::time::timeout(Duration::from_secs(1), actions.recv())
.await
.expect("block-sync action should arrive")
.expect("block-sync action channel should stay open")
}
#[tokio::test]
async fn committed_views_are_the_sole_production_frontier_source() {
let legacy_frontiers = BlockSyncFrontiers {
finalized_height: block::Height(40),
verified_block_tip: block::Height(40),
verified_block_hash: block::Hash([0x40; 32]),
};
let legacy_header = (block::Height(50), block::Hash([0x50; 32]));
let (snapshot_tx, snapshot_rx) = watch::channel(None);
let startup = BlockSyncStartup::new_with_committed_views(
legacy_frontiers,
legacy_header,
snapshot_rx,
ZakuraBlockSyncConfig::default(),
);
assert!(startup.header_tip.is_none());
let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
match tokio::time::timeout(Duration::from_millis(50), actions.recv()).await {
Err(_) => {}
Ok(action) => panic!(
"legacy startup facts must not schedule work before semantic handoff: {action:?}"
),
}
let finalized = zakura_header_chain::Frontier::new(block::Height(0), block::Hash([0x01; 32]));
let verified = finalized;
let first_header =
zakura_header_chain::Frontier::new(block::Height(3), block::Hash([0x03; 32]));
let first = committed_snapshot(1, 1, 1, finalized, verified, first_header);
let first_view = committed_view(first.clone(), 0);
snapshot_tx
.send(Some(first_view.clone()))
.expect("the committed snapshot receiver is live");
let first_query = next_action(&mut actions).await;
let BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip,
scope,
..
} = first_query
else {
panic!("the first committed snapshot must initialize body scheduling");
};
assert_eq!(from, block::Height(1));
assert_eq!(best_header_tip, first_header.height);
assert_eq!(
scope,
zakura_header_chain::BodyWorkAuthority::for_view(&first_view)
);
let second_header =
zakura_header_chain::Frontier::new(block::Height(2), block::Hash([0x22; 32]));
let second = committed_snapshot(2, 2, 1, finalized, verified, second_header);
let second_view = committed_view(second.clone(), 1);
snapshot_tx
.send(Some(second_view.clone()))
.expect("the committed snapshot receiver is live");
let second_query = next_action(&mut actions).await;
let BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip,
scope,
..
} = second_query
else {
panic!("the reanchored committed snapshot must replace the old target");
};
assert_eq!(from, block::Height(1));
assert_eq!(best_header_tip, second_header.height);
assert_eq!(
scope,
zakura_header_chain::BodyWorkAuthority::for_view(&second_view)
);
reactor_task.abort();
}
#[tokio::test(start_paused = true)]
async fn empty_state_body_queries_wait_for_the_latest_header_quiet_period() {
let finalized = zakura_header_chain::Frontier::new(block::Height(0), block::Hash([1; 32]));
let first_header = zakura_header_chain::Frontier::new(
block::Height(EMPTY_STATE_HEADER_QUIET_MIN_LAG),
block::Hash([2; 32]),
);
let second_header = zakura_header_chain::Frontier::new(
block::Height(EMPTY_STATE_HEADER_QUIET_MIN_LAG + 250),
block::Hash([3; 32]),
);
let (snapshot_tx, snapshot_rx) = watch::channel(None);
let startup = BlockSyncStartup::new_with_committed_views(
BlockSyncFrontiers {
finalized_height: finalized.height,
verified_block_tip: finalized.height,
verified_block_hash: finalized.hash,
},
(finalized.height, finalized.hash),
snapshot_rx,
ZakuraBlockSyncConfig::default(),
);
let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
snapshot_tx
.send(Some(committed_view(
committed_snapshot(1, 1, 1, finalized, finalized, first_header),
0,
)))
.expect("the committed snapshot receiver is live");
time::advance(Duration::from_millis(1)).await;
if let Ok(action) = actions.try_recv() {
panic!("far-ahead empty-state bootstrap dispatched body work immediately: {action:?}");
}
time::advance(Duration::from_secs(10)).await;
snapshot_tx
.send(Some(committed_view(
committed_snapshot(2, 2, 1, finalized, finalized, second_header),
0,
)))
.expect("the newer committed snapshot receiver is live");
time::advance(Duration::from_millis(1)).await;
time::advance(EMPTY_STATE_HEADER_QUIET_PERIOD - Duration::from_secs(1)).await;
if let Ok(action) = actions.try_recv() {
panic!("newer header page did not restart the body-work quiet period: {action:?}");
}
time::advance(Duration::from_secs(1)).await;
let action = next_action(&mut actions).await;
assert!(matches!(
action,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
best_header_tip,
..
} if best_header_tip == second_header.height
));
reactor_task.abort();
}
#[tokio::test]
async fn past_genesis_body_queries_ignore_far_ahead_header_quiet_period() {
let finalized = zakura_header_chain::Frontier::new(block::Height(0), block::Hash([1; 32]));
let verified = zakura_header_chain::Frontier::new(block::Height(1), block::Hash([2; 32]));
let header = zakura_header_chain::Frontier::new(
block::Height(EMPTY_STATE_HEADER_QUIET_MIN_LAG + 100),
block::Hash([3; 32]),
);
let (_snapshot_tx, snapshot_rx) = watch::channel(Some(committed_view(
committed_snapshot(1, 1, 1, finalized, verified, header),
0,
)));
let startup = BlockSyncStartup::new_with_committed_views(
BlockSyncFrontiers {
finalized_height: finalized.height,
verified_block_tip: verified.height,
verified_block_hash: verified.hash,
},
(header.height, header.hash),
snapshot_rx,
ZakuraBlockSyncConfig::default(),
);
let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let action = next_action(&mut actions).await;
assert!(
matches!(
action,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip,
..
} if best_header_tip == header.height
),
"body work past genesis must start immediately even with a far-ahead header tip, got {action:?}",
);
reactor_task.abort();
}
#[test]
fn state_is_only_frontier_publisher() {
const BLOCK_DRIVER: &str =
include_str!("../../../../zakurad/src/commands/start/zakura/block_sync_driver.rs");
const HEADER_DRIVER: &str =
include_str!("../../../../zakurad/src/commands/start/zakura/header_sync_driver.rs");
const NODE_WIRING: &str = include_str!("../../../../zakurad/src/commands/start.rs");
const STATE_PUBLISHER: &str =
include_str!("../../../../zakura-state/src/service/finalized_state/header_chain.rs");
for (name, source) in [
("block-sync driver", BLOCK_DRIVER),
("header-sync driver", HEADER_DRIVER),
("node wiring", NODE_WIRING),
] {
for forbidden in [
"local_frontier",
"publish_frontier",
"publish_best_header",
"publish_header_reanchor",
"mirror_zakura_full_block_commits",
"BlockSyncEvent::HeaderTipChanged",
"BlockSyncEvent::StateFrontiersChanged",
"BlockSyncEvent::ChainTipGrow",
"BlockSyncEvent::ChainTipReset",
] {
assert!(
!source.contains(forbidden),
"{name} must not contain the retired frontier publisher `{forbidden}`"
);
}
}
assert!(
STATE_PUBLISHER.contains("pub struct Publisher")
&& STATE_PUBLISHER
.contains("fn publish(&self, snapshot: EngineSnapshot, effect: TransitionEffect)"),
"state must retain the sole committed-snapshot publisher"
);
}
#[tokio::test]
async fn operator_body_retry_is_exact_deduplicated_and_requires_a_supplier() {
let header = zakura_header_chain::Frontier::new(block::Height(10), block::Hash([0x52; 32]));
let snapshot = alarmed_committed_snapshot(header);
let view = committed_view(snapshot, 0);
let scope = zakura_header_chain::BodyWorkAuthority::for_view(&view);
let (snapshot_tx, snapshot_rx) = watch::channel(Some(view));
let mut startup = BlockSyncStartup::inert(ZakuraBlockSyncConfig::default());
startup.frontiers = BlockSyncFrontiers {
finalized_height: block::Height(9),
verified_block_tip: block::Height(9),
verified_block_hash: block::Hash([0x50; 32]),
};
startup.best_header_tip = (header.height, header.hash);
startup.committed_views = Some(snapshot_rx);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
handle
.retry_body_availability(block::Hash([0x53; 32]))
.expect("the stale operator command queues");
assert!(
tokio::time::timeout(Duration::from_millis(50), actions.recv())
.await
.is_err(),
"a stale hash must not dispatch a retry"
);
handle
.retry_body_availability(header.hash)
.expect("the no-supplier operator command queues");
assert!(
tokio::time::timeout(Duration::from_millis(50), actions.recv())
.await
.is_err(),
"an episode with no eligible supplier must not be restarted"
);
let wiring = handle
.routine_wiring
.as_ref()
.expect("the spawned reactor exposes its shared test wiring");
let supplier = peer(0x54);
let generation = wiring
.registry
.admit_session(
&supplier,
ServicePeerDirection::Outbound,
&wiring.config,
0,
std::time::Instant::now(),
)
.generation();
wiring.registry.upsert_status(
&supplier,
generation,
BlockSyncStatus {
servable_low: header.height,
servable_high: header.height,
..BlockSyncStatus::default()
},
);
handle
.retry_body_availability(header.hash)
.expect("the operator retry queues");
handle
.retry_body_availability(header.hash)
.expect("the duplicate operator retry queues");
let BlockSyncAction::RetryBodyAvailability {
expected_version,
retry,
} = next_action(&mut actions).await
else {
panic!("the exact operator command must dispatch a retry action");
};
assert_eq!(expected_version, zakura_header_chain::StateVersion::new(7));
assert_eq!(retry.hash, header.hash);
assert_eq!(retry.availability.attempts, 0);
assert_eq!(retry.availability.suppliers, 1);
assert!(!retry.availability.alarmed);
assert_eq!(
retry.availability.started_at,
retry.availability.next_probe_at
);
assert!(
wiring
.registry
.is_body_retry_avoided(&supplier, scope, header.hash, Instant::now()),
"dispatch must not reopen live admission before the durable alarm-clear snapshot"
);
assert!(
tokio::time::timeout(Duration::from_millis(50), actions.recv())
.await
.is_err(),
"duplicate commands against one durable snapshot must be coalesced"
);
let mut cleared = alarmed_committed_snapshot(header);
cleared.state_version = zakura_header_chain::StateVersion::new(8);
cleared.alarms.header_best_body_unavailable = None;
snapshot_tx
.send(Some(committed_view(cleared, 0)))
.expect("the durable alarm-clear snapshot publishes");
tokio::time::timeout(Duration::from_secs(1), async {
while wiring
.registry
.is_body_retry_avoided(&supplier, scope, header.hash, Instant::now())
{
tokio::task::yield_now().await;
}
})
.await
.expect("the committed alarm-clear edge reopens live admission");
handle
.retry_body_availability(header.hash)
.expect("the already-cleared operator command queues");
assert!(
tokio::time::timeout(Duration::from_millis(50), actions.recv())
.await
.is_err(),
"an already-cleared alarm must not start another episode"
);
drop(snapshot_tx);
reactor_task.abort();
}
#[tokio::test]
async fn newly_eligible_supplier_preserves_persistent_body_alarm_once() {
let header = zakura_header_chain::Frontier::new(block::Height(10), block::Hash([0x55; 32]));
let snapshot = alarmed_committed_snapshot(header);
let previous = snapshot
.alarms
.header_best_body_unavailable
.expect("the fixture contains a persistent body alarm");
let (_snapshot_tx, snapshot_rx) = watch::channel(Some(committed_view(snapshot, 0)));
let config = ZakuraBlockSyncConfig::default();
let mut startup = BlockSyncStartup::inert(config.clone());
startup.frontiers = BlockSyncFrontiers {
finalized_height: block::Height(9),
verified_block_tip: block::Height(9),
verified_block_hash: block::Hash([0x50; 32]),
};
startup.best_header_tip = (header.height, header.hash);
startup.committed_views = Some(snapshot_rx);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let (_supplier, inbound, _outbound) = connect_peer_with_status(
&service,
&mut actions,
0x56,
header.height,
header.hash,
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let discovery = loop {
match next_action(&mut actions).await {
BlockSyncAction::RestartBodyAvailability {
expected_version,
discovery,
} => {
assert_eq!(expected_version, zakura_header_chain::StateVersion::new(7));
break discovery;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before supplier restart: {action:?}"),
}
};
assert_eq!(discovery.hash, header.hash);
assert_eq!(discovery.availability.started_at, previous.started_at);
assert_eq!(discovery.availability.attempts, previous.attempts);
assert_eq!(discovery.availability.suppliers, 1);
assert!(discovery.availability.alarmed);
assert!(discovery.availability.next_probe_at >= discovery.availability.started_at);
assert_ne!(
discovery.availability.supplier_set_digest, previous.supplier_set_digest,
"the restart must authenticate the newly eligible supplier identity set"
);
inbound
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: header.height,
tip_hash: header.hash,
max_blocks_per_response: 16,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("the duplicate status encodes"),
)
.await
.expect("the duplicate status queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), async {
loop {
match actions.recv().await.expect("the action channel is live") {
BlockSyncAction::RestartBodyAvailability { .. } => {
panic!("one durable snapshot must dispatch at most one supplier restart")
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action after supplier restart: {action:?}"),
}
}
})
.await
.is_err(),
"a duplicate status must not dispatch another restart"
);
reactor_task.abort();
}
#[test]
fn persistent_body_alarm_metrics_expose_every_required_dimension() {
let reactor = include_str!("reactor.rs");
let metrics = reactor
.split_once("fn publish_body_unavailable_metrics")
.expect("the body-unavailable metric publisher exists")
.1
.split_once("fn clamp_served_block_count")
.expect("the metric publisher remains a focused function")
.0;
for required in [
"sync.header_chain.body_unavailable.active",
"sync.header_chain.body_unavailable",
"sync.header_chain.body_unavailable.age_seconds",
"sync.header_chain.body_unavailable.attempts",
"sync.header_chain.body_unavailable.available_suppliers",
"\"hash\" =>",
"\"height\" =>",
"summary.started_at",
"summary.attempts",
"summary.suppliers",
] {
assert!(
metrics.contains(required),
"persistent body-alarm metrics must retain `{required}`"
);
}
}
async fn wait_for_query_needed_blocks(
actions: &mut mpsc::Receiver<BlockSyncAction>,
verified_block_tip: block::Height,
best_header_tip: block::Height,
) {
let expected_from = verified_block_tip.next().unwrap_or(verified_block_tip);
loop {
match next_action(actions).await {
BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip: actual_best,
..
} if from == expected_from && actual_best == best_header_tip => return,
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before target QueryNeededBlocks: {action:?}"),
}
}
}
async fn send_inbound(inbound_tx: &FramedSend, msg: BlockSyncMessage) {
inbound_tx
.send(msg.encode_frame().expect("message encodes"))
.await
.expect("inbound frame queues onto the peer stream");
}
async fn next_outbound_message(outbound: &mut FramedRecv) -> BlockSyncMessage {
let frame = tokio::time::timeout(Duration::from_secs(1), outbound.recv())
.await
.expect("outbound frame arrives")
.expect("outbound channel is live");
BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
}
async fn wait_for_outbound_block(outbound: &mut FramedRecv) -> Arc<block::Block> {
loop {
match next_outbound_message(outbound).await {
BlockSyncMessage::Block(block) => return block,
BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {}
msg => panic!("unexpected outbound message before block: {msg:?}"),
}
}
}
async fn wait_for_outbound_blocks_done(outbound: &mut FramedRecv) -> (block::Height, u32) {
loop {
match next_outbound_message(outbound).await {
BlockSyncMessage::BlocksDone {
start_height,
returned,
} => return (start_height, returned),
BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {}
msg => panic!("unexpected outbound message before BlocksDone: {msg:?}"),
}
}
}
async fn wait_for_outbound_range_unavailable(outbound: &mut FramedRecv) -> (block::Height, u32) {
loop {
match next_outbound_message(outbound).await {
BlockSyncMessage::RangeUnavailable {
start_height,
count,
} => return (start_height, count),
BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {}
msg => panic!("unexpected outbound message before RangeUnavailable: {msg:?}"),
}
}
}
async fn wait_for_outbound_getblocks(outbound: &mut FramedRecv) -> (block::Height, u32) {
loop {
match next_outbound_message(outbound).await {
BlockSyncMessage::GetBlocks {
start_height,
count,
} => return (start_height, count),
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before GetBlocks: {msg:?}"),
}
}
}
async fn wait_for_outbound_status(outbound: &mut FramedRecv) -> BlockSyncStatus {
match next_outbound_message(outbound).await {
BlockSyncMessage::Status(status) => status,
msg => panic!("unexpected outbound message before Status: {msg:?}"),
}
}
async fn wait_for_getblocks_across(
peers: &mut [(ZakuraPeerId, &mut FramedRecv)],
) -> (ZakuraPeerId, block::Height, u32) {
use futures::stream::{FuturesUnordered, StreamExt};
loop {
let (peer_id, frame) = {
let mut recvs: FuturesUnordered<_> = peers
.iter_mut()
.map(|(peer_id, outbound)| {
let peer_id = peer_id.clone();
async move {
let frame = outbound.recv().await.expect("outbound channel is live");
(peer_id, frame)
}
})
.collect();
tokio::time::timeout(Duration::from_secs(1), recvs.next())
.await
.expect("an outbound frame arrives on some peer")
.expect("at least one peer outbound is live")
};
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height,
count,
} => return (peer_id, start_height, count),
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before GetBlocks: {msg:?}"),
}
}
}
async fn drain_parent_first_actions(
actions: &mut mpsc::Receiver<BlockSyncAction>,
verified_tip: &mut block::Height,
expected_new_fork: Option<&[Arc<block::Block>]>,
) {
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(25), actions.recv()).await
{
match action {
BlockSyncAction::SubmitBlock { block, .. } => {
let height = block
.coinbase_height()
.expect("submitted test block has height");
assert_eq!(
Some(height),
next_height(*verified_tip),
"block sync must submit only the contiguous parent-first prefix"
);
if let Some(new_fork) = expected_new_fork {
let expected_hash = match height.0 {
2 => new_fork[1].hash(),
3 => new_fork[2].hash(),
_ => panic!("unexpected post-reset submitted height: {height:?}"),
};
assert_eq!(
block.hash(),
expected_hash,
"post-reset submissions must follow the re-derived fork"
);
}
*verified_tip = height;
}
BlockSyncAction::Misbehavior {
reason:
BlockSyncMisbehavior::BodyPayloadMismatch(_)
| BlockSyncMisbehavior::InvalidBlock
| BlockSyncMisbehavior::UnsolicitedBlock,
..
} => {}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while draining body responses: {action:?}"),
}
}
}
fn download_window() -> DownloadWindow {
DownloadWindow::new(&ZakuraBlockSyncConfig::default())
}
fn test_delivery_snapshot(now: Instant) -> DeliverySnapshot {
DeliverySnapshot {
delivered: 0,
delivered_at: now,
}
}
fn window_request(height: u32) -> OutstandingBlockRange {
let byte = u8::try_from(height).expect("test heights fit in u8");
let now = Instant::now();
OutstandingBlockRange {
request: BlockRangeRequest {
owner: test_work_owner(),
start_height: block::Height(height),
count: 1,
anchor_hash: block::Hash([byte; 32]),
estimated_bytes: 1,
expected_blocks: vec![ExpectedBlock {
height: block::Height(height),
hash: block::Hash([byte; 32]),
estimated_bytes: 1,
}],
},
queued_at: now,
deadline: now,
delivery_snapshot: test_delivery_snapshot(now),
delivered_bytes: 0,
received: ReceivedBlockTracker::default(),
}
}
fn window_request_range(start: u32, count: u32) -> OutstandingBlockRange {
let byte = u8::try_from(start).expect("test heights fit in u8");
let now = Instant::now();
OutstandingBlockRange {
request: BlockRangeRequest {
owner: test_work_owner(),
start_height: block::Height(start),
count,
anchor_hash: block::Hash([byte; 32]),
estimated_bytes: u64::from(count),
expected_blocks: (start..start + count)
.map(|height| ExpectedBlock {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("test heights fit in u8"); 32]),
estimated_bytes: 1,
})
.collect(),
},
queued_at: now,
deadline: now,
delivery_snapshot: test_delivery_snapshot(now),
delivered_bytes: 0,
received: ReceivedBlockTracker::default(),
}
}
#[test]
fn block_liveness_disconnects_silent_active_peer_after_default_timeout() {
let config = ZakuraBlockSyncConfig::default();
let timeout = config.effective_liveness_timeout();
assert_eq!(timeout, Duration::from_secs(32));
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
assert_eq!(
window.check_liveness(now + timeout - Duration::from_millis(1)),
LivenessOutcome::Ok
);
assert_eq!(window.check_liveness(now + timeout), LivenessOutcome::Park);
}
#[test]
fn block_liveness_disarms_a_deadline_set_without_a_request() {
let now = Instant::now();
let mut window = download_window();
assert_eq!(window.check_liveness(now), LivenessOutcome::Ok);
window.block_liveness_deadline = Some(now);
assert_eq!(window.check_liveness(now), LivenessOutcome::Disarm);
window.clear_liveness_if_idle();
assert_eq!(window.block_liveness_deadline, None);
assert_eq!(window.check_liveness(now), LivenessOutcome::Ok);
}
#[test]
fn short_response_charges_one_reliability_failure_per_request_not_per_height() {
let fresh = download_window().reliability_factor();
let one_missing = {
let mut window = download_window();
window.penalize_short_response(1);
window.reliability_factor()
};
let many_missing = {
let mut window = download_window();
window.penalize_short_response(64);
window.reliability_factor()
};
assert!(
one_missing < fresh,
"a short response must lower reliability (one goodput failure)"
);
assert_eq!(
one_missing, many_missing,
"a short response is one failure per request, independent of the missing-height count"
);
}
#[test]
fn block_liveness_progress_before_deadline_keeps_peer_alive() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let mut now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
for _ in 0..4 {
now += timeout - Duration::from_millis(1);
assert_eq!(window.check_liveness(now), LivenessOutcome::Ok);
window.note_block_progress(now, timeout);
assert_eq!(window.block_liveness_deadline, Some(now + timeout));
}
}
#[test]
fn block_liveness_disconnects_silent_peer_after_outstanding_drains() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
window.outstanding.clear();
window.disarm_liveness_after_progress_if_idle();
assert_eq!(window.block_liveness_deadline, Some(now + timeout));
assert_eq!(window.check_liveness(now + timeout), LivenessOutcome::Park);
}
#[test]
fn block_liveness_disarms_when_satisfied_request_drains() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
window.note_block_progress(now + Duration::from_millis(1), timeout);
window.outstanding.clear();
window.disarm_liveness_after_progress_if_idle();
assert_eq!(window.block_liveness_deadline, None);
assert_eq!(window.check_liveness(now + timeout), LivenessOutcome::Ok);
}
#[test]
fn block_liveness_uses_probe_cap_until_first_accepted_body() {
let config = ZakuraBlockSyncConfig {
initial_block_probe_requests: 1,
max_requests_without_block_progress: 8,
..ZakuraBlockSyncConfig::default()
};
let timeout = config.effective_liveness_timeout();
let now = Instant::now();
let mut window = DownloadWindow::new(&config);
assert!(!window.has_block_progress());
assert_eq!(window.no_progress_request_cap(), 1);
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
assert_eq!(window.requests_without_block_progress, 1);
assert_eq!(window.no_progress_request_cap(), 1);
window.note_block_progress(now + Duration::from_millis(1), timeout);
assert!(window.has_block_progress());
assert_eq!(window.requests_without_block_progress, 0);
assert_eq!(window.no_progress_request_cap(), 8);
}
#[test]
fn block_liveness_resuming_after_idle_gets_fresh_deadline() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
window.note_block_progress(now + Duration::from_millis(1), timeout);
window.outstanding.clear();
window.disarm_liveness_after_progress_if_idle();
let resumed = now + Duration::from_secs(60);
window.outstanding.push(window_request(2));
window.arm_liveness(resumed, timeout);
assert_eq!(window.block_liveness_deadline, Some(resumed + timeout));
}
#[test]
fn block_liveness_multi_block_range_progress_resets_each_body() {
let timeout = ZakuraBlockSyncConfig::default().effective_liveness_timeout();
let start = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request_range(1, 3));
window.arm_liveness(start, timeout);
let first = start + Duration::from_secs(4);
window.note_block_progress(first, timeout);
assert_eq!(window.block_liveness_deadline, Some(first + timeout));
let second = first + Duration::from_secs(4);
assert_eq!(window.check_liveness(second), LivenessOutcome::Ok);
window.note_block_progress(second, timeout);
assert_eq!(window.block_liveness_deadline, Some(second + timeout));
let third = second + Duration::from_secs(4);
assert_eq!(window.check_liveness(third), LivenessOutcome::Ok);
window.note_block_progress(third, timeout);
assert_eq!(window.block_liveness_deadline, Some(third + timeout));
}
#[test]
fn view_reset_reclears_probe_streak_so_unproven_peer_can_reprobe() {
let config = ZakuraBlockSyncConfig {
initial_block_probe_requests: 1,
..ZakuraBlockSyncConfig::default()
};
let timeout = config.effective_liveness_timeout();
let now = Instant::now();
let mut window = DownloadWindow::new(&config);
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
assert_eq!(window.requests_without_block_progress, 1);
assert_eq!(window.no_progress_request_cap(), 1);
window.outstanding.clear();
window.note_view_reset();
assert_eq!(window.requests_without_block_progress, 0);
assert!(window.requests_without_block_progress < window.no_progress_request_cap());
assert!(!window.has_block_progress());
assert_eq!(
window.check_liveness(now + timeout),
LivenessOutcome::Ok,
"a reset peer must not carry a phantom liveness deadline",
);
}
#[test]
fn view_reset_preserves_proof_but_reclears_streak() {
let config = ZakuraBlockSyncConfig {
initial_block_probe_requests: 1,
max_requests_without_block_progress: 8,
..ZakuraBlockSyncConfig::default()
};
let timeout = config.effective_liveness_timeout();
let now = Instant::now();
let mut window = DownloadWindow::new(&config);
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
window.note_block_progress(now + Duration::from_millis(1), timeout);
window.outstanding.push(window_request(2));
window.arm_liveness(now + Duration::from_millis(2), timeout);
assert!(window.has_block_progress());
assert_eq!(window.no_progress_request_cap(), 8);
window.outstanding.clear();
window.note_view_reset();
assert_eq!(window.requests_without_block_progress, 0);
assert!(
window.has_block_progress(),
"reset must not un-prove a peer"
);
assert_eq!(window.no_progress_request_cap(), 8);
}
#[test]
fn backpressure_extends_liveness_instead_of_disconnecting() {
let config = ZakuraBlockSyncConfig::default();
let timeout = config.effective_liveness_timeout();
let now = Instant::now();
let mut window = download_window();
window.outstanding.push(window_request(1));
window.arm_liveness(now, timeout);
assert_eq!(
window.check_liveness(now + timeout),
LivenessOutcome::Park,
"the deadline has expired: without backpressure this disconnects",
);
let extended_at = now + timeout;
window.extend_liveness_deadline(extended_at, timeout);
assert_eq!(window.check_liveness(extended_at), LivenessOutcome::Ok);
assert_eq!(
window.check_liveness(extended_at + timeout),
LivenessOutcome::Park,
);
}
#[test]
fn work_queue_returned_height_is_contestable_by_any_peer() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(queue.in_flight_contains(block::Height(1)));
assert!(!queue.pending_contains(block::Height(1)));
queue.return_items([block::Height(1)]);
assert!(queue.pending_contains(block::Height(1)));
assert_eq!(
queue
.take_in_range(block::Height(1), block::Height(1), 1)
.len(),
1,
"a returned height must be immediately re-takable"
);
}
async fn connect_peer_with_status(
service: &BlockSyncService,
actions: &mut mpsc::Receiver<BlockSyncAction>,
byte: u8,
servable_high: block::Height,
tip_hash: block::Hash,
max_inflight_requests: u32,
max_response_bytes: u32,
) -> (ZakuraPeerId, FramedSend, FramedRecv) {
connect_peer_with_status_message(
service,
actions,
byte,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high,
tip_hash,
max_blocks_per_response: 16,
max_inflight_requests,
max_response_bytes,
},
)
.await
}
async fn connect_peer_with_status_message(
service: &BlockSyncService,
actions: &mut mpsc::Receiver<BlockSyncAction>,
byte: u8,
status: BlockSyncStatus,
) -> (ZakuraPeerId, FramedSend, FramedRecv) {
let peer = peer(byte);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
let _ = actions;
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(status)
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
(peer, inbound_tx, outbound_rx)
}
fn needed(height: u32, size: BlockSizeEstimate) -> (block::Height, block::Hash, BlockSizeEstimate) {
(block::Height(height), block::Hash([height as u8; 32]), size)
}
fn work_queue_with(
floor: u32,
items: impl IntoIterator<Item = (block::Height, block::Hash, BlockSizeEstimate)>,
) -> super::work_queue::WorkQueue {
let queue = super::work_queue::WorkQueue::new(block::Height(floor));
queue.set_estimate_floor_for_tests(1);
queue.extend(test_work_scope(), items);
queue
}
fn block_meta(block: &Arc<block::Block>) -> BlockSyncBlockMeta {
BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
}
}
#[test]
fn block_sync_config_defaults_and_round_trips() {
let default = ZakuraBlockSyncConfig::default();
assert_eq!(default.max_blocks_per_response, 1);
assert_eq!(default.max_inflight_requests, 32000);
assert_eq!(
default.max_inflight_block_bytes,
DEFAULT_BS_MAX_INFLIGHT_BLOCK_BYTES
);
assert_eq!(
default.max_reorder_lookahead_bytes,
DEFAULT_BS_MAX_REORDER_LOOKAHEAD_BYTES
);
assert_eq!(
default.floor_peer_avoid_cooldown,
DEFAULT_BS_FLOOR_PEER_AVOID_COOLDOWN
);
assert_eq!(
default.no_progress_peer_cooldown,
DEFAULT_BS_NO_PROGRESS_PEER_COOLDOWN
);
assert_eq!(
default.initial_block_probe_requests,
DEFAULT_BS_INITIAL_BLOCK_PROBE_REQUESTS,
);
assert_eq!(
default.max_requests_without_block_progress,
DEFAULT_BS_MAX_REQUESTS_WITHOUT_BLOCK_PROGRESS,
);
assert_eq!(
default.bbr_reliability_weight_percent,
DEFAULT_BS_BBR_RELIABILITY_WEIGHT_PERCENT,
);
assert_eq!(
default.effective_max_reorder_lookahead_bytes(),
DEFAULT_BS_MAX_REORDER_LOOKAHEAD_BYTES
);
assert_eq!(
default.floor_request_byte_reservation(),
u64::from(DEFAULT_BS_MAX_RESPONSE_BYTES)
);
assert_eq!(
default.effective_floor_peer_avoid_cooldown(),
DEFAULT_BS_FLOOR_PEER_AVOID_COOLDOWN
);
assert_eq!(
default.effective_no_progress_peer_cooldown(),
DEFAULT_BS_NO_PROGRESS_PEER_COOLDOWN
);
assert!(default.validate().is_ok());
assert_eq!(
default.max_submitted_block_applies,
MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES
);
assert_eq!(
default.submitted_apply_limit(),
MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES
);
assert_eq!(default.request_timeout, DEFAULT_BS_REQUEST_TIMEOUT);
let encoded = toml::to_string(&default).expect("block-sync config serializes");
let decoded: ZakuraBlockSyncConfig =
toml::from_str(&encoded).expect("block-sync config deserializes");
assert_eq!(decoded, default);
let config: crate::Config = toml::from_str(
r#"
[zakura.block_sync]
max_submitted_block_applies = 9
"#,
)
.expect("nested Zakura block-sync config deserializes");
assert_eq!(config.zakura.block_sync.max_submitted_block_applies, 9);
assert_eq!(
config.zakura.block_sync.submitted_apply_limit(),
MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES,
);
}
#[test]
fn config_validate_rejects_degenerate_values() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 0,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
max_reorder_lookahead_bytes: 0,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_CHECKPOINT_RANGE_BYTE_FLOOR - 1,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_ok());
config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_CHECKPOINT_RANGE_BYTE_FLOOR,
max_reorder_lookahead_bytes: BS_CHECKPOINT_RANGE_BYTE_FLOOR,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_ok());
config = ZakuraBlockSyncConfig {
request_timeout: Duration::ZERO,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
initial_block_probe_requests: 0,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
max_requests_without_block_progress: 0,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
config = ZakuraBlockSyncConfig {
bbr_reliability_weight_percent: 101,
..ZakuraBlockSyncConfig::default()
};
assert!(config.validate().is_err());
}
#[test]
fn config_deserialize_keeps_below_range_floor_inflight_block_bytes() {
let config: crate::Config = toml::from_str(
r#"
[zakura.block_sync]
max_inflight_block_bytes = 268435456
"#,
)
.expect("a below-range-floor max_inflight_block_bytes config still loads");
const {
assert!(268435456u64 < BS_CHECKPOINT_RANGE_BYTE_FLOOR);
}
assert_eq!(config.zakura.block_sync.max_inflight_block_bytes, 268435456);
}
#[test]
fn config_deserialize_clamps_sub_floor_request_inflight_block_bytes() {
let config: crate::Config = toml::from_str(
r#"
[zakura.block_sync]
max_inflight_block_bytes = 16777216
"#,
)
.expect("a sub-floor-request max_inflight_block_bytes config still loads");
let request_floor = config.zakura.block_sync.floor_request_byte_reservation();
assert!(
16_777_216 <= request_floor,
"test premise: the stored value is at or below one floor request"
);
assert_eq!(
config.zakura.block_sync.max_inflight_block_bytes,
request_floor + 1
);
}
#[test]
fn codec_round_trips_every_message_variant() {
round_trip(BlockSyncMessage::Status(status()));
round_trip(BlockSyncMessage::GetBlocks {
start_height: block::Height(10),
count: 3,
});
round_trip(BlockSyncMessage::Block(mainnet_block(
&BLOCK_MAINNET_1_BYTES,
)));
round_trip(BlockSyncMessage::BlocksDone {
start_height: block::Height(10),
returned: 3,
});
round_trip(BlockSyncMessage::RangeUnavailable {
start_height: block::Height(10),
count: 3,
});
}
#[test]
fn codec_round_trips_block_near_max_block_bytes() {
let block = Arc::new(zakura_chain::block::tests::generate::large_multi_transaction_block());
let serialized_len = block
.zcash_serialize_to_vec()
.expect("large test block serializes")
.len();
let max_block_bytes =
usize::try_from(block::MAX_BLOCK_BYTES).expect("max block size fits in usize");
assert!(
serialized_len <= max_block_bytes && serialized_len > max_block_bytes - 1000,
"test block should be close to the consensus cap, got {serialized_len}"
);
round_trip(BlockSyncMessage::Block(block));
}
#[test]
fn codec_rejects_malformed_discriminator_and_truncated_payload() {
assert!(matches!(
BlockSyncMessage::decode(&[99]),
Err(BlockSyncWireError::UnknownMessageType(99))
));
assert!(matches!(
BlockSyncMessage::decode(&[MSG_BS_GET_BLOCKS, 1, 0]),
Err(BlockSyncWireError::Io(_))
));
}
#[test]
fn codec_classifies_payloads_above_old_raw_stream6_cap() {
let old_max_bs_message_bytes =
usize::try_from(block::MAX_BLOCK_BYTES).expect("max block bytes fits in usize") + 1;
let payload = vec![99; old_max_bs_message_bytes + 1];
assert!(payload.len() <= MAX_BS_MESSAGE_BYTES);
assert!(matches!(
BlockSyncMessage::decode(&payload),
Err(BlockSyncWireError::UnknownMessageType(99))
));
}
#[test]
fn codec_rejects_oversized_frame_and_oversized_block() {
let oversized_payload = vec![0; MAX_BS_MESSAGE_BYTES + 1];
assert!(matches!(
BlockSyncMessage::decode(&oversized_payload),
Err(BlockSyncWireError::OversizedPayload { .. })
));
let oversized_block =
Arc::new(zakura_chain::block::tests::generate::oversized_multi_transaction_block());
assert!(matches!(
BlockSyncMessage::Block(oversized_block).encode(),
Err(BlockSyncWireError::OversizedBlock { .. })
| Err(BlockSyncWireError::OversizedPayload { .. })
));
}
#[test]
fn codec_rejects_count_and_returned_over_cap() {
let over_cap = MAX_BS_BLOCKS_PER_REQUEST + 1;
assert!(matches!(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 0,
}
.encode(),
Err(BlockSyncWireError::ZeroBlockCount)
));
let mut zero_count_get_blocks = vec![MSG_BS_GET_BLOCKS];
zero_count_get_blocks.extend_from_slice(&1u32.to_le_bytes());
zero_count_get_blocks.extend_from_slice(&0u32.to_le_bytes());
assert!(matches!(
BlockSyncMessage::decode(&zero_count_get_blocks),
Err(BlockSyncWireError::ZeroBlockCount)
));
let mut zero_count_range_unavailable = vec![MSG_BS_RANGE_UNAVAILABLE];
zero_count_range_unavailable.extend_from_slice(&1u32.to_le_bytes());
zero_count_range_unavailable.extend_from_slice(&0u32.to_le_bytes());
assert!(matches!(
BlockSyncMessage::decode(&zero_count_range_unavailable),
Err(BlockSyncWireError::ZeroBlockCount)
));
assert!(matches!(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: over_cap,
}
.encode(),
Err(BlockSyncWireError::BlockCountLimit { .. })
));
assert!(matches!(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: over_cap,
}
.encode(),
Err(BlockSyncWireError::BlockCountLimit { .. })
));
assert!(matches!(
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: over_cap,
}
.encode(),
Err(BlockSyncWireError::BlockCountLimit { .. })
));
}
#[test]
fn frame_decode_rejects_mismatched_unknown_flags_and_trailing_payload() {
let frame = Frame {
message_type: u16::from(MSG_BS_GET_BLOCKS),
flags: 1,
payload: BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 1,
}
.encode()
.expect("message encodes"),
};
assert!(matches!(
BlockSyncMessage::decode_frame(frame),
Err(BlockSyncWireError::UnsupportedFlags(1))
));
let mut payload = BlockSyncMessage::Status(status())
.encode()
.expect("message encodes");
payload.push(0);
assert!(matches!(
BlockSyncMessage::decode(&payload),
Err(BlockSyncWireError::TrailingBytes)
));
let frame = Frame {
message_type: u16::from(MSG_BS_BLOCK),
flags: 0,
payload: BlockSyncMessage::Status(status())
.encode()
.expect("message encodes"),
};
assert!(matches!(
BlockSyncMessage::decode_frame(frame),
Err(BlockSyncWireError::MismatchedFrameMessageType { .. })
));
}
#[test]
fn status_decode_clamps_peer_capacity_advertisements() {
let mut payload = Vec::new();
payload.push(MSG_BS_STATUS);
payload.extend_from_slice(&block::Height(1).0.to_le_bytes());
payload.extend_from_slice(&block::Height(2).0.to_le_bytes());
block::Hash([9; 32])
.zcash_serialize(&mut payload)
.expect("hash serializes");
payload.extend_from_slice(&u32::MAX.to_le_bytes());
payload.extend_from_slice(&u32::MAX.to_le_bytes());
payload.extend_from_slice(&u32::MAX.to_le_bytes());
let BlockSyncMessage::Status(status) =
BlockSyncMessage::decode(&payload).expect("status decodes")
else {
panic!("expected status message");
};
assert_eq!(status.max_blocks_per_response, MAX_BS_BLOCKS_PER_REQUEST);
assert_eq!(status.max_inflight_requests, MAX_BS_INFLIGHT_REQUESTS);
assert_eq!(status.max_response_bytes, MAX_BS_RESPONSE_BYTES);
}
#[test]
fn aggregate_response_cap_is_not_the_per_frame_cap() {
assert!(
MAX_BS_RESPONSE_BYTES > u32::try_from(MAX_BS_MESSAGE_BYTES).expect("frame cap fits u32"),
"range responses are multiple independently-capped block frames"
);
assert_eq!(
ZakuraBlockSyncConfig::default().advertised_max_response_bytes(),
MAX_BS_RESPONSE_BYTES
);
}
#[test]
fn work_queue_take_dedups_a_height_across_peers() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(10_000)),
needed(2, BlockSizeEstimate::Advertised(10_000)),
needed(3, BlockSizeEstimate::Advertised(10_000)),
],
);
let first = queue.take_in_range(
block::Height(1),
block::Height(3),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert_eq!(
first.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![1, 2, 3]
);
assert_eq!(queue.in_flight_len(), 3);
let second = queue.take_in_range(
block::Height(1),
block::Height(3),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert!(
second.is_empty(),
"a height taken by one peer must not be re-takable by another (dedup)"
);
}
#[test]
fn work_queue_extend_dedups_against_pending_in_flight_and_floor() {
let queue = work_queue_with(
5,
[
needed(3, BlockSizeEstimate::Advertised(100)),
needed(5, BlockSizeEstimate::Advertised(100)),
needed(6, BlockSizeEstimate::Advertised(100)),
needed(7, BlockSizeEstimate::Advertised(100)),
],
);
assert_eq!(queue.pending_len(), 2);
assert!(!queue.pending_contains(block::Height(5)));
queue.take_in_range(block::Height(6), block::Height(6), 1);
let inserted = queue.extend(
test_work_scope(),
[
needed(6, BlockSizeEstimate::Advertised(100)), needed(7, BlockSizeEstimate::Advertised(100)), needed(8, BlockSizeEstimate::Advertised(100)), ],
);
assert_eq!(inserted, 1, "only the genuinely new height is inserted");
assert!(queue.pending_contains(block::Height(8)));
assert!(!queue.pending_contains(block::Height(6)));
}
#[test]
fn work_queue_refreshes_queued_authority_and_preserves_registered_requests() {
let obsolete = test_work_scope();
let current = zakura_header_chain::BodyWorkAuthority {
verified_generation: zakura_header_chain::VerifiedGeneration::new(9),
..obsolete
};
let queue = WorkQueue::new(block::Height(0));
queue.set_estimate_floor_for_tests(1);
assert_eq!(
queue.extend(obsolete, [needed(1, BlockSizeEstimate::Advertised(100))]),
1
);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert_eq!(taken[0].1.scope, obsolete);
let owner = obsolete.bind(
12,
std::num::NonZeroU64::new(13).expect("thirteen is nonzero"),
);
assert_eq!(
queue.mark_reserved_for_owner(owner, [block::Height(1)]),
100
);
assert_eq!(queue.owner_for_height(block::Height(1)), Some(owner));
queue.refresh_authority(current);
assert_eq!(queue.owner_for_height(block::Height(1)), Some(owner));
assert_eq!(queue.reserved_bytes(), 100);
let returned =
queue.release_reserved_and_return_items_detailed_for_owner(owner, [block::Height(1)]);
assert_eq!(returned.released_bytes, 100);
assert_eq!(returned.returned_count, 1);
let replacement = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(replacement.len(), 1);
assert_eq!(replacement[0].0, block::Height(1));
assert_eq!(replacement[0].1.scope, current);
let replacement_owner = current.bind(
12,
std::num::NonZeroU64::new(14).expect("fourteen is nonzero"),
);
assert_eq!(replacement_owner.authority(), current);
}
#[test]
fn producer_filter_keeps_live_requests_after_same_epoch_refresh() {
let scope = test_work_scope();
let refreshed = zakura_header_chain::BodyWorkAuthority {
header: zakura_header_chain::HeaderWorkAuthority {
header_generation: zakura_header_chain::HeaderGeneration::new(
scope.header_generation.get().saturating_add(1),
),
branch: zakura_header_chain::BranchId::new(
scope.branch.anchor_hash,
block::Hash([7; 32]),
),
},
..scope
};
assert_ne!(refreshed, scope);
assert_eq!(refreshed.body_work_epoch, scope.body_work_epoch);
let owner = test_work_owner();
let height = block::Height(1);
let hash = block::Hash([1; 32]);
let queue = WorkQueue::new(block::Height(0));
queue.set_estimate_floor_for_tests(1);
assert_eq!(
queue.extend(scope, [needed(1, BlockSizeEstimate::Advertised(100))]),
1
);
assert_eq!(queue.take_in_range(height, height, 1).len(), 1);
assert_eq!(queue.mark_reserved_for_owner(owner, [height]), 100);
let config = ZakuraBlockSyncConfig::default();
let registry = PeerRegistry::new();
let peer = peer(1);
let generation = registry
.admit_session(
&peer,
ServicePeerDirection::Outbound,
&config,
0,
Instant::now(),
)
.generation();
registry.set_outstanding(
&peer,
generation,
BTreeMap::from([(
height,
OutstandingMeta {
owner,
hash,
estimated_bytes: 100,
queued_at: Instant::now(),
deadline: Instant::now(),
},
)]),
);
queue.refresh_authority(refreshed);
assert!(queue.in_flight_contains(height));
assert_eq!(queue.owner_for_height(height), Some(owner));
queue.reset_above(block::Height(0));
assert!(!queue.in_flight_contains(height));
assert!(!queue.pending_contains(height));
assert!(
registry.has_outstanding_request(height, hash),
"outstanding entries keep the issuing authority across a same-epoch refresh"
);
assert_eq!(registry.total_unreceived(), 1);
let request_floor = block::Height(0);
let producer_would_enqueue = height > request_floor
&& !queue.in_flight_contains(height)
&& !registry.has_outstanding_request(height, hash);
assert!(
!producer_would_enqueue,
"the producer filter must keep a live request from being extended again"
);
assert_eq!(
queue.extend(refreshed, [needed(1, BlockSizeEstimate::Advertised(100))]),
1,
"the height is absent from the queue, so only the outstanding filter prevents a duplicate fetch"
);
}
#[test]
fn work_queue_first_unclaimed_above_is_floor_anchored_not_frontier_anchored() {
let queue = work_queue_with(5, []);
assert_eq!(
queue.first_unclaimed_above(block::Height(5)),
Some(block::Height(6)),
);
let queue = work_queue_with(
5,
(6..=9).map(|height| needed(height, BlockSizeEstimate::Advertised(100))),
);
assert_eq!(
queue.first_unclaimed_above(block::Height(5)),
Some(block::Height(10)),
);
queue.take_in_range(block::Height(6), block::Height(9), 4);
assert_eq!(
queue.first_unclaimed_above(block::Height(5)),
Some(block::Height(10)),
);
queue.reset_above(block::Height(5));
queue.extend(
test_work_scope(),
[
needed(6, BlockSizeEstimate::Advertised(100)),
needed(8, BlockSizeEstimate::Advertised(100)),
needed(9, BlockSizeEstimate::Advertised(100)),
],
);
assert_eq!(
queue.first_unclaimed_above(block::Height(5)),
Some(block::Height(7)),
"an interior hole is the cursor even though higher heights are claimed",
);
queue.extend(
test_work_scope(),
[needed(7, BlockSizeEstimate::Advertised(100))],
);
assert_eq!(
queue.first_unclaimed_above(block::Height(5)),
Some(block::Height(10)),
);
let queue = work_queue_with(
5,
(100..=102).map(|height| needed(height, BlockSizeEstimate::Advertised(100))),
);
assert_eq!(
queue.first_unclaimed_above(block::Height(5)),
Some(block::Height(6)),
);
assert_eq!(
queue.first_unclaimed_above(block::Height(1)),
Some(block::Height(6)),
);
}
#[test]
fn work_queue_first_unclaimed_above_ignores_entries_retained_at_or_below_a_forward_reset() {
let queue = work_queue_with(
5,
(6..=20).map(|height| needed(height, BlockSizeEstimate::Advertised(100))),
);
queue.reset_above(block::Height(12));
queue.extend(
test_work_scope(),
[
needed(15, BlockSizeEstimate::Advertised(100)),
needed(16, BlockSizeEstimate::Advertised(100)),
],
);
assert_eq!(
queue.first_unclaimed_above(block::Height(12)),
Some(block::Height(13)),
"13 is unclaimed; the retained 6..=12 prefix must not hide it",
);
let queue = work_queue_with(
5,
(6..=20).map(|height| needed(height, BlockSizeEstimate::Advertised(100))),
);
queue.take_in_range(block::Height(6), block::Height(12), 7);
queue.reset_above(block::Height(12));
queue.extend(
test_work_scope(),
[needed(15, BlockSizeEstimate::Advertised(100))],
);
assert_eq!(
queue.first_unclaimed_above(block::Height(12)),
Some(block::Height(13)),
);
queue.extend(
test_work_scope(),
[
needed(13, BlockSizeEstimate::Advertised(100)),
needed(14, BlockSizeEstimate::Advertised(100)),
],
);
assert_eq!(
queue.first_unclaimed_above(block::Height(12)),
Some(block::Height(16)),
);
}
#[test]
fn work_queue_take_respects_servable_range_contiguity_and_max() {
let queue = work_queue_with(
0,
[
needed(10, BlockSizeEstimate::Advertised(100)),
needed(11, BlockSizeEstimate::Advertised(100)),
needed(12, BlockSizeEstimate::Advertised(100)),
needed(20, BlockSizeEstimate::Advertised(100)),
needed(21, BlockSizeEstimate::Advertised(100)),
],
);
let chunk = queue.take_in_range(block::Height(10), block::Height(30), 2);
assert_eq!(
chunk.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![10, 11]
);
let run = queue.take_in_range(
block::Height(12),
block::Height(30),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert_eq!(
run.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![12],
"take must stop at the first gap so the chunk is one contiguous request"
);
let high = queue.take_in_range(
block::Height(20),
block::Height(30),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert_eq!(
high.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![20, 21]
);
}
#[test]
fn work_queue_budgeted_take_respects_count_cap() {
let queue = work_queue_with(
0,
(1..=4).map(|height| needed(height, BlockSizeEstimate::Advertised(100))),
);
let taken = queue.take_in_range_budgeted(block::Height(1), block::Height(4), 2, u64::MAX);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![1, 2]
);
}
#[test]
fn work_queue_budgeted_take_respects_estimated_byte_cap() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(100)),
needed(2, BlockSizeEstimate::Advertised(150)),
needed(3, BlockSizeEstimate::Advertised(1)),
],
);
let taken = queue.take_in_range_budgeted(block::Height(1), block::Height(3), 3, 250);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![1, 2]
);
assert!(queue.pending_contains(block::Height(3)));
}
#[test]
fn work_queue_budgeted_take_stops_at_gaps() {
let queue = work_queue_with(
0,
[
needed(10, BlockSizeEstimate::Advertised(100)),
needed(11, BlockSizeEstimate::Advertised(100)),
needed(13, BlockSizeEstimate::Advertised(100)),
],
);
let taken = queue.take_in_range_budgeted(block::Height(10), block::Height(13), 3, u64::MAX);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![10, 11]
);
assert!(queue.pending_contains(block::Height(13)));
}
#[test]
fn work_queue_budgeted_take_takes_one_oversized_first_item_for_progress() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(500)),
needed(2, BlockSizeEstimate::Advertised(1)),
],
);
let taken = queue.take_in_range_budgeted(block::Height(1), block::Height(2), 2, 100);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![1]
);
assert_eq!(taken[0].1.estimated_bytes, 500);
assert!(queue.pending_contains(block::Height(2)));
}
#[test]
fn work_queue_budgeted_take_preserves_estimates_through_take_and_return() {
let queue = work_queue_with(0, [needed(10, BlockSizeEstimate::Advertised(12_345))]);
let taken = queue.take_in_range_budgeted(block::Height(10), block::Height(10), 1, 1);
assert_eq!(taken.len(), 1);
assert_eq!(taken[0].1.estimated_bytes, 12_345);
queue.return_items([block::Height(10)]);
let retaken = queue.take_in_range_budgeted(block::Height(10), block::Height(10), 1, 1);
assert_eq!(retaken.len(), 1);
assert_eq!(retaken[0].1.estimated_bytes, 12_345);
}
fn admit_grant(
config: &ZakuraBlockSyncConfig,
snapshot: super::admission::AdmissionSnapshot,
start: block::Height,
servable_high: block::Height,
response_byte_cap: u64,
) -> Option<super::admission::AdmissionGrant> {
match super::admission::admit(config, snapshot, start, servable_high, response_byte_cap) {
super::admission::AdmissionOutcome::Admit(grant) => Some(grant),
super::admission::AdmissionOutcome::LookaheadAtCap
| super::admission::AdmissionOutcome::InflightBudgetEmpty => None,
}
}
#[test]
fn admission_blocks_above_floor_at_cap_but_keeps_floor_fundable() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 40_000_000,
max_reorder_lookahead_bytes: 500,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 500,
reorder_buffered_blocks: 1,
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 40_000_000,
};
let floor = admit_grant(
&config,
snapshot,
block::Height(11),
block::Height(11),
1_000,
)
.expect("floor rescue remains admitted at the look-ahead cap");
assert_eq!(floor.priority, super::admission::RequestPriority::Floor);
assert_eq!(floor.max_request_bytes, 1_000);
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(412),
block::Height(412),
1_000
),
None,
"above-floor work stops at the look-ahead cap"
);
let under_cap = super::admission::AdmissionSnapshot {
reorder_buffered_bytes: 100,
budget_available: 40_000_000,
..snapshot
};
let above = admit_grant(
&config,
under_cap,
block::Height(412),
block::Height(412),
u64::MAX,
)
.expect("above-floor work is admitted below the cap");
assert_eq!(
above.priority,
super::admission::RequestPriority::AboveFloor
);
assert_eq!(above.max_request_bytes, 400);
}
#[test]
fn admission_counts_inflight_to_sequencer_bytes() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 200,
reorder_buffered_blocks: 1,
applying_buffered_bytes: 200,
applying_buffered_blocks: 1,
sequencer_input_queued_bytes: 600,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(412),
block::Height(412),
1_000
),
None,
"above-floor admission includes bytes already queued to the sequencer"
);
}
#[test]
fn total_resident_plateaus_under_commit_stall() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 300,
reorder_buffered_blocks: 1,
applying_buffered_bytes: 700,
applying_buffered_blocks: 1,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(412),
block::Height(412),
1_000
),
None,
"above-floor admission includes applying bytes held during a commit stall"
);
}
#[test]
fn floor_priority_request_does_not_buffer_above_floor_past_cap() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let capped = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 1_000,
reorder_buffered_blocks: 1,
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
assert_eq!(
admit_grant(
&config,
capped,
block::Height(412),
block::Height(412),
1_000
),
None,
"the above-floor tail of a floor-starting request is refused at the cap"
);
let floor = admit_grant(
&config,
capped,
block::Height(11),
block::Height(10_000),
1_000,
)
.expect("floor height remains fundable");
assert_eq!(floor.priority, super::admission::RequestPriority::Floor);
assert_eq!(floor.take_high, block::Height(411));
}
#[test]
fn outstanding_reservations_are_charged_at_wire_size() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(600),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 0,
reorder_buffered_blocks: 0,
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 1_100,
reserved_above_floor_blocks: 1,
budget_available: 64_000_000,
};
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(602),
block::Height(602),
u64::MAX
),
None,
"outstanding reservations must count against the resident budget",
);
assert!(
matches!(
super::admission::admit(
&config,
snapshot,
block::Height(601),
block::Height(601),
u64::MAX
),
super::admission::AdmissionOutcome::LookaheadAtCap
),
"an escalated floor take above the commit window is refused on reserved bytes alone",
);
let window = admit_grant(
&config,
snapshot,
block::Height(411),
block::Height(411),
1_000,
)
.expect("the commit window is exempt from the reservation charge");
assert_eq!(window.max_request_bytes, 1_000);
}
#[test]
fn floor_backpressures_when_download_floor_escalates_past_commit() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(1_000),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 0,
reorder_buffered_blocks: 0,
applying_buffered_bytes: 1_200,
applying_buffered_blocks: 990,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
assert_eq!(
super::admission::request_priority(snapshot.download_floor, block::Height(1_001)),
super::admission::RequestPriority::Floor,
);
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(1_001),
block::Height(1_001),
1_000
),
None,
"a floor request far ahead of commit is backpressured when the memory budget is full",
);
let frontier = admit_grant(
&config,
snapshot,
block::Height(11),
block::Height(11),
1_000,
)
.expect("the commit-frontier block is always fundable");
assert_eq!(frontier.max_request_bytes, 1_000);
let window_top = admit_grant(
&config,
snapshot,
block::Height(411),
block::Height(411),
1_000,
)
.expect("the top of the commit window is still fundable");
assert_eq!(window_top.max_request_bytes, 1_000);
assert_eq!(
admit_grant(
&config,
snapshot,
block::Height(412),
block::Height(412),
1_000
),
None,
"the first height above the commit window is memory-gated",
);
}
#[test]
fn exempt_take_never_spans_the_commit_window_boundary() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let full = super::admission::AdmissionSnapshot {
download_floor: block::Height(10),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 0,
reorder_buffered_blocks: 0,
applying_buffered_bytes: 1_000,
applying_buffered_blocks: 10,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 64_000_000,
};
let grant = admit_grant(
&config,
full,
block::Height(11),
block::Height(10_000),
1_000,
)
.expect("an in-window start stays fundable at a full gate");
assert_eq!(
grant.take_high,
block::Height(411),
"the exempt take is clamped at the commit-window top",
);
assert_eq!(grant.max_request_bytes, 1_000);
let open = super::admission::AdmissionSnapshot {
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
..full
};
let grant = admit_grant(
&config,
open,
block::Height(11),
block::Height(10_000),
1_000,
)
.expect("an in-window start is fundable with the gate open");
assert_eq!(grant.take_high, block::Height(411));
let grant = admit_grant(
&config,
open,
block::Height(412),
block::Height(10_000),
u64::MAX,
)
.expect("an above-window start is admitted below the cap");
assert_eq!(grant.take_high, block::Height(10_000));
assert_eq!(grant.max_request_bytes, 1_000);
}
#[test]
fn commit_window_stays_fundable_at_exact_floor() {
use super::config::{
BS_CHECKPOINT_RANGE_BYTE_FLOOR, MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES,
};
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_CHECKPOINT_RANGE_BYTE_FLOOR,
max_reorder_lookahead_bytes: 1,
..ZakuraBlockSyncConfig::default()
};
config.clamp_reorder_lookahead_to_floor();
let range_blocks = u32::try_from(MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES)
.expect("checkpoint range block count fits in u32");
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(range_blocks - 1),
verified_block_tip: block::Height(0),
reorder_buffered_bytes: 8_388_608,
reorder_buffered_blocks: 4,
applying_buffered_bytes: 800_000_000,
applying_buffered_blocks: u64::from(range_blocks) - 1,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 800_000_000,
reserved_above_floor_bytes: 4_000_000,
reserved_above_floor_blocks: 2,
budget_available: 2_000_000,
};
assert_eq!(
super::admission::request_priority(snapshot.download_floor, block::Height(range_blocks)),
super::admission::RequestPriority::Floor,
);
let completing = admit_grant(
&config,
snapshot,
block::Height(range_blocks),
block::Height(range_blocks),
u64::MAX,
)
.expect("the range-completing floor take must pass a full gate");
assert_eq!(
completing.priority,
super::admission::RequestPriority::Floor
);
assert_eq!(completing.max_request_bytes, 2_000_000);
assert!(matches!(
super::admission::admit(
&config,
snapshot,
block::Height(range_blocks + 1),
block::Height(range_blocks + 1),
u64::MAX
),
super::admission::AdmissionOutcome::LookaheadAtCap
));
}
#[test]
fn work_queue_force_cancel_and_owner_timeout_release_once() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
assert_eq!(budget.reserved(), 100);
let watchdog_released = queue.release_and_return_items([block::Height(1)]);
budget.release(watchdog_released);
assert_eq!(watchdog_released, 100);
assert_eq!(budget.reserved(), 0);
assert!(queue.pending_contains(block::Height(1)));
let late_owner_released = queue.release_and_return_items([block::Height(1)]);
budget.release(late_owner_released);
assert_eq!(late_owner_released, 0);
assert_eq!(budget.reserved(), 0);
assert!(queue.pending_contains(block::Height(1)));
}
#[test]
fn work_queue_reserved_bytes_counter_matches_scan_across_transitions() {
let queue = work_queue_with(
0,
(1..=6).map(|h| needed(h, BlockSizeEstimate::Advertised(100))),
);
let check = |label: &str| {
assert_eq!(
queue.reserved_bytes(),
queue.reserved_bytes_scanned(),
"reserved_bytes counter drifted from scan after {label}"
);
};
assert_eq!(queue.reserved_bytes(), 0);
check("seed");
let taken = queue.take_in_range(block::Height(1), block::Height(6), 6);
assert_eq!(taken.len(), 6);
assert_eq!(queue.mark_reserved((1..=6).map(block::Height)), 600);
assert_eq!(queue.reserved_bytes(), 600);
check("mark_reserved");
assert_eq!(
queue.release_active_reserved_height(block::Height(1)),
Some(100),
);
assert_eq!(queue.reserved_bytes(), 500);
check("release_active_reserved_height");
assert_eq!(queue.claim_received(block::Height(2)), 100);
assert_eq!(queue.reserved_bytes(), 400);
check("claim_received");
queue.release_reserved_heights([block::Height(3)]);
assert_eq!(queue.reserved_bytes(), 300);
check("release_heights");
let released = queue.release_reserved_and_return_items([block::Height(4)]);
assert_eq!(released, 100);
assert_eq!(queue.reserved_bytes(), 200);
check("release_reserved_and_return_items");
let released = queue.advance_floor(block::Height(4));
assert_eq!(released, 0, "heights <= 4 owned no live reservation");
assert!(!queue.pending_contains(block::Height(1)));
assert!(!queue.in_flight_contains(block::Height(2)));
assert_eq!(queue.reserved_bytes(), 200);
check("advance_floor");
let released = queue.reset_above(block::Height(4));
assert_eq!(released, 200);
assert_eq!(queue.reserved_bytes(), 0);
assert!(!queue.in_flight_contains(block::Height(5)));
assert!(!queue.in_flight_contains(block::Height(6)));
check("reset_above");
}
#[test]
fn work_queue_advance_floor_drops_only_committed_prefix() {
let queue = work_queue_with(
0,
(1..=10).map(|h| needed(h, BlockSizeEstimate::Advertised(100))),
);
let _ = queue.take_in_range(block::Height(1), block::Height(10), 10);
assert_eq!(queue.mark_reserved((1..=10).map(block::Height)), 1000);
let released = queue.advance_floor(block::Height(4));
assert_eq!(released, 400, "reserved bytes for the dropped 1..=4 prefix");
for h in 1..=4 {
assert!(!queue.in_flight_contains(block::Height(h)));
}
for h in 5..=10 {
assert!(queue.in_flight_contains(block::Height(h)));
}
assert_eq!(queue.reserved_bytes(), 600);
assert_eq!(queue.reserved_bytes(), queue.reserved_bytes_scanned());
assert_eq!(queue.advance_floor(block::Height(2)), 0);
assert_eq!(queue.reserved_bytes(), 600);
}
#[test]
fn receipt_releases_wire_reservation_while_body_retained() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
assert_eq!(queue.reserved_bytes(), 100);
let reserved_estimate = queue
.release_active_reserved_height(block::Height(1))
.expect("active reserved height releases at receipt");
assert_eq!(reserved_estimate, 100);
budget.release(reserved_estimate);
assert_eq!(queue.reserved_bytes(), 0);
assert_eq!(budget.reserved(), 0);
assert!(
queue.in_flight_contains(block::Height(1)),
"the received height stays claimed; the body handoff owns it"
);
}
#[test]
fn floor_watchdog_skips_received_heights() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
let reserved_estimate = queue
.release_active_reserved_height(block::Height(1))
.expect("active reserved height releases at receipt");
budget.release(reserved_estimate);
assert_eq!(budget.reserved(), 0);
let outcome = queue.release_reserved_and_return_items_detailed([block::Height(1)]);
let watchdog_released = outcome.released_bytes;
budget.release(watchdog_released);
assert_eq!(
watchdog_released, 0,
"watchdog must not release a received height owned by the sequencer handoff"
);
assert!(
queue.in_flight_contains(block::Height(1)),
"the received height must not be requeued for a re-fetch"
);
assert_eq!(outcome.released_count, 1);
assert_eq!(outcome.returned_count, 0);
assert_eq!(outcome.missing_count, 0);
assert!(queue.in_flight_contains(block::Height(1)));
assert_eq!(budget.reserved(), 0);
assert_eq!(queue.advance_floor(block::Height(1)), 0);
}
#[test]
fn late_body_does_not_resurrect_charge() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
let outcome = queue.release_reserved_and_return_items_detailed([block::Height(1)]);
let watchdog_released = outcome.released_bytes;
budget.release(watchdog_released);
assert_eq!(outcome.returned_count, 1);
assert_eq!(outcome.released_count, 0);
assert_eq!(outcome.missing_count, 0);
assert_eq!(budget.reserved(), 0);
assert!(queue.pending_contains(block::Height(1)));
let duplicate =
queue.release_reserved_and_return_items_detailed([block::Height(1), block::Height(2)]);
assert_eq!(duplicate.already_pending_count, 1);
assert_eq!(duplicate.missing_count, 1);
assert_eq!(duplicate.min_height, Some(block::Height(1)));
assert_eq!(duplicate.max_height, Some(block::Height(2)));
assert_eq!(
queue.release_active_reserved_height(block::Height(1)),
None,
"a late body cannot release a claim the watchdog already released"
);
assert_eq!(budget.reserved(), 0);
}
#[test]
fn claim_received_does_not_orphan_a_charge() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(1), 1);
assert_eq!(taken.len(), 1);
assert!(budget.try_reserve(100));
assert_eq!(queue.mark_reserved([block::Height(1)]), 100);
let old_charge = queue.claim_received(block::Height(1));
budget.release(old_charge);
assert_eq!(old_charge, 100);
assert_eq!(budget.reserved(), 0);
let released = queue.release_reserved_heights([block::Height(1)]);
budget.release(released);
assert_eq!(released, 0, "a received height owns no further charge");
assert_eq!(budget.reserved(), 0);
}
#[test]
fn release_reserved_mixed_received_and_reserved_conserves_budget() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(100)),
needed(2, BlockSizeEstimate::Advertised(100)),
],
);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(2), 2);
assert_eq!(taken.len(), 2);
assert!(budget.try_reserve(200));
assert_eq!(
queue.mark_reserved([block::Height(1), block::Height(2)]),
200
);
let reserved_estimate = queue
.release_active_reserved_height(block::Height(1))
.expect("active height releases at receipt");
assert_eq!(reserved_estimate, 100);
budget.release(reserved_estimate);
assert_eq!(budget.reserved(), 100);
let released = queue.advance_floor(block::Height(2));
budget.release(released);
assert_eq!(
released, 100,
"WorkQueue releases only the still-reserved height; received bodies hold no charge"
);
assert_eq!(budget.reserved(), 0);
}
#[test]
fn release_reserved_heights_skips_received_body_owned_by_sequencer() {
let queue = work_queue_with(
0,
[
needed(1, BlockSizeEstimate::Advertised(100)),
needed(2, BlockSizeEstimate::Advertised(100)),
],
);
let mut budget = ByteBudget::new(1_000);
let taken = queue.take_in_range(block::Height(1), block::Height(2), 2);
assert_eq!(taken.len(), 2);
assert!(budget.try_reserve(200));
assert_eq!(
queue.mark_reserved([block::Height(1), block::Height(2)]),
200
);
let estimate = queue
.release_active_reserved_height(block::Height(1))
.expect("height 1 is reserved");
assert_eq!(estimate, 100);
budget.release(estimate);
assert_eq!(budget.reserved(), 100);
let released = queue.release_reserved_heights([block::Height(1), block::Height(2)]);
budget.release(released);
assert_eq!(
released, 100,
"release_reserved_heights frees only the still-reserved estimate, exactly once"
);
assert!(queue.in_flight_contains(block::Height(1)));
assert_eq!(budget.reserved(), 0);
assert_eq!(queue.advance_floor(block::Height(2)), 0);
assert_eq!(budget.reserved(), 0);
}
#[test]
fn work_queue_take_does_not_clamp_high_to_floor() {
let queue = work_queue_with(
0,
(100..=104)
.map(|height| needed(height, BlockSizeEstimate::Advertised(100)))
.collect::<Vec<_>>(),
);
let taken = queue.take_in_range(
block::Height(100),
block::Height(104),
MAX_BS_BLOCKS_PER_REQUEST as usize,
);
assert_eq!(
taken.iter().map(|(height, _)| height.0).collect::<Vec<_>>(),
vec![100, 101, 102, 103, 104],
"heights far above the floor are takable; the floor never clamps the take"
);
}
#[test]
fn work_queue_preserves_work_item_estimate_through_take_and_return() {
let queue = work_queue_with(0, [needed(10, BlockSizeEstimate::Advertised(12_345))]);
let taken = queue.take_in_range(block::Height(10), block::Height(10), 1);
assert_eq!(taken.len(), 1);
assert_eq!(taken[0].1.estimated_bytes, 12_345);
assert_eq!(taken[0].1.hash, block::Hash([10; 32]));
queue.return_items([block::Height(10)]);
let retaken = queue.take_in_range(block::Height(10), block::Height(10), 1);
assert_eq!(
retaken[0].1.estimated_bytes, 12_345,
"return_items must restore the stored WorkItem unchanged"
);
}
#[test]
fn work_queue_advance_floor_and_reset_above_gc_both_maps() {
let queue = work_queue_with(
0,
(1..=6)
.map(|height| needed(height, BlockSizeEstimate::Advertised(100)))
.collect::<Vec<_>>(),
);
queue.take_in_range(block::Height(1), block::Height(2), 2);
assert_eq!(queue.in_flight_len(), 2);
assert_eq!(queue.pending_len(), 4);
queue.advance_floor(block::Height(3));
assert!(!queue.in_flight_contains(block::Height(1)));
assert!(!queue.pending_contains(block::Height(3)));
assert!(queue.pending_contains(block::Height(4)));
queue.take_in_range(block::Height(4), block::Height(4), 1); queue.reset_above(block::Height(4));
assert!(queue.in_flight_contains(block::Height(4)));
assert!(!queue.pending_contains(block::Height(5)));
assert!(!queue.pending_contains(block::Height(6)));
assert_eq!(queue.pending_len(), 0);
}
#[test]
fn work_queue_height_is_in_exactly_one_set() {
let queue = work_queue_with(0, [needed(10, BlockSizeEstimate::Advertised(100))]);
let in_one_set = |height: block::Height| -> usize {
usize::from(queue.pending_contains(height)) + usize::from(queue.in_flight_contains(height))
};
let h = block::Height(10);
assert_eq!(in_one_set(h), 1, "pending only after extend");
queue.take_in_range(h, h, 1);
assert_eq!(in_one_set(h), 1, "in_flight only after take");
queue.return_items([h]);
assert_eq!(in_one_set(h), 1, "pending only after return");
queue.take_in_range(h, h, 1);
queue.advance_floor(h);
assert_eq!(in_one_set(h), 0, "gone after the floor commits past it");
}
#[tokio::test]
async fn reactor_fill_loop_saturates_multiple_slots_in_one_pass() {
let config = fill_loop_mechanics_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
41,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: block::Hash([4; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tip_tx
.send((block::Height(4), block::Hash([4; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: block::Hash([3; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
]))
.await
.expect("needed metadata queues");
let mut heights = Vec::new();
for _ in 0..4 {
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(count, 1);
heights.push(start_height.0);
}
heights.sort_unstable();
assert_eq!(heights, vec![1, 2, 3, 4]);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_suppresses_duplicate_needed_block_query_until_response() {
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let best_hash = block::Hash([4; 32]);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("header-tip event queues");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("duplicate header-tip event queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"same pending needed-block query should not be dispatched twice",
);
handle
.send(BlockSyncEvent::NeededBlocks(Vec::new()))
.await
.expect("needed-block response queues");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("header-tip event after response queues");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
reactor_task.abort();
}
#[tokio::test]
async fn reactor_requeries_a_height_stranded_below_higher_claimed_work() {
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let best_hash = block::Hash([10; 32]);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(10),
hash: best_hash,
})
.await
.expect("header-tip event queues");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(10)).await;
handle
.send(BlockSyncEvent::NeededBlocks(
(5..=8)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("test height fits u8"); 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed-block response queues");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(10),
hash: best_hash,
})
.await
.expect("header-tip event after response queues");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(10)).await;
reactor_task.abort();
}
#[tokio::test]
async fn reactor_suppresses_needed_block_query_when_work_already_covers_tip() {
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let best_hash = block::Hash([4; 32]);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("header-tip event queues");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=4)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("test height fits u8"); 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed-block response queues");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(4),
hash: best_hash,
})
.await
.expect("covered header-tip event queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"producer should not re-query when pending/in-flight work already covers the header tip",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_fill_loop_saturates_every_peer_window_not_just_one() {
let config = fill_loop_mechanics_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let mut peer_ids = Vec::new();
let mut peer_streams = Vec::new();
for byte in [41u8, 42, 43] {
let (peer_id, inbound, outbound) = connect_peer_with_status_message(
&service,
&mut actions,
byte,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(12),
tip_hash: block::Hash([12; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
peer_ids.push(peer_id);
peer_streams.push((inbound, outbound));
}
tip_tx
.send((block::Height(12), block::Hash([12; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=12)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([height as u8; 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed metadata queues");
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = peer_ids
.iter()
.cloned()
.zip(peer_streams.iter_mut().map(|(_, outbound)| outbound))
.collect();
let mut per_peer: HashMap<ZakuraPeerId, Vec<u32>> = HashMap::new();
for _ in 0..12 {
let (peer, start_height, count) = wait_for_getblocks_across(&mut outbound_by_peer).await;
assert_eq!(count, 1);
per_peer.entry(peer).or_default().push(start_height.0);
}
assert_eq!(
per_peer.len(),
3,
"all three peers must receive requests, not just the first; got {per_peer:?}"
);
for peer_id in &peer_ids {
let issued = per_peer
.get(peer_id)
.unwrap_or_else(|| panic!("peer {peer_id:?} received no requests: {per_peer:?}"));
assert_eq!(
issued.len(),
4,
"peer {peer_id:?} window of 4 was not saturated: {per_peer:?}"
);
}
let mut all_heights: Vec<u32> = per_peer.values().flatten().copied().collect();
all_heights.sort_unstable();
assert_eq!(
all_heights,
(1..=12).collect::<Vec<_>>(),
"every needed height must be requested exactly once across peers"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_budget_constrained_issuance_rotates_across_peers() {
let config = fill_loop_mechanics_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let mut peer_inbounds = HashMap::new();
let mut peer_outbounds = Vec::new();
for byte in [0x41u8, 0x42, 0x43] {
let (peer_id, inbound, outbound) = connect_peer_with_status_message(
&service,
&mut actions,
byte,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
peer_inbounds.insert(peer_id.clone(), inbound);
peer_outbounds.push((peer_id, outbound));
}
tip_tx
.send((block::Height(1), block::Hash([1; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
}]))
.await
.expect("needed metadata queues");
let range_unavailable = BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: 1,
}
.encode_frame()
.expect("RangeUnavailable frame encodes");
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = peer_outbounds
.iter_mut()
.map(|(peer_id, outbound)| (peer_id.clone(), outbound))
.collect();
for round in 0..6 {
let (peer, start_height, count) = wait_for_getblocks_across(&mut outbound_by_peer).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1, "fanout=1 yields single-block requests");
let inbound = peer_inbounds.get(&peer).unwrap_or_else(|| {
panic!("round {round}: served peer {peer:?} must be one of the connected peers")
});
inbound
.send(range_unavailable.clone())
.await
.expect("RangeUnavailable frame queues");
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_timeout_recovery_is_local_and_healthy_peer_keeps_filling() {
let mut config = fill_loop_mechanics_config();
config.request_timeout = Duration::from_millis(400);
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 64;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let status = || BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: block::Hash([2; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
};
let (peer_a, a_in, mut a_out) =
connect_peer_with_status_message(&service, &mut actions, 0x41, status()).await;
let (peer_b, b_in, mut b_out) =
connect_peer_with_status_message(&service, &mut actions, 0x42, status()).await;
let mut inbounds = HashMap::from([(peer_a.clone(), a_in), (peer_b.clone(), b_in)]);
tip_tx
.send((block::Height(2), block::Hash([2; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Advertised(1_000),
},
]))
.await
.expect("needed metadata queues");
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> =
vec![(peer_a.clone(), &mut a_out), (peer_b.clone(), &mut b_out)];
let first = wait_for_getblocks_across(&mut outbound_by_peer).await;
let second = wait_for_getblocks_across(&mut outbound_by_peer).await;
let mut offered: HashMap<ZakuraPeerId, block::Height> = HashMap::new();
offered.insert(first.0.clone(), first.1);
offered.insert(second.0.clone(), second.1);
assert_eq!(
offered.len(),
2,
"both peers must be offered a height in the opening pass: {offered:?}"
);
let healthy = peer_b.clone();
let healthy_in = inbounds.remove(&healthy).expect("healthy peer inbound");
let healthy_offers = tokio::time::timeout(Duration::from_secs(3), async {
let mut count = 0usize;
loop {
let (peer, start_height, count_blocks) =
wait_for_getblocks_across(&mut outbound_by_peer).await;
if peer == healthy {
count += 1;
if count >= 2 {
break count;
}
healthy_in
.send(
BlockSyncMessage::RangeUnavailable {
start_height,
count: count_blocks,
}
.encode_frame()
.expect("RangeUnavailable frame encodes"),
)
.await
.expect("RangeUnavailable frame queues");
}
}
})
.await
.expect(
"the healthy peer must keep being offered shared work while the slow peer \
is in timeout recovery; a stall here means a straggler wedged the pass",
);
assert!(
healthy_offers >= 2,
"the healthy peer was filled repeatedly despite the slow peer's timeout recovery"
);
reactor_task.abort();
}
#[tokio::test]
async fn block_liveness_parks_silent_peer_and_traces_reason() {
let mut capture = TraceCapture::for_test("block_liveness_parks_silent_peer_and_traces_reason")
.expect("trace capture initializes");
let mut config = immediate_body_download_config();
config.request_timeout = Duration::from_millis(400);
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 64;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let mut startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
startup.trace = ZakuraTrace::new(capture.tracer(), "01");
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(0x51);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
let connection_cancel = CancellationToken::new();
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
connection_cancel.clone(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), block::Hash([1; 32])))
.expect("tip watch is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
}]))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1);
await_until(
"silent block-sync session is locally parked",
Duration::from_secs(3),
|| {
handle.peer_snapshot().outbound_peers == 0
&& !service.wants_peer(&peer, ZAKURA_CAP_BLOCK_SYNC, ServicePeerDirection::Outbound)
},
)
.await
.expect("silent active peer is parked by block-progress liveness");
assert!(
!connection_cancel.is_cancelled(),
"a first no-progress stall preserves the shared connection",
);
capture.flush().await;
let reader = capture.reader().expect("trace rows load");
reader.table("block_sync").assert_row(
bs_trace::BLOCK_GET_BLOCKS_SENT,
&[
("requests_without_block_progress", TraceValue::U64(1)),
("no_progress_request_cap", TraceValue::U64(1)),
("block_progress_proven", TraceValue::U64(0)),
],
);
reader.table("block_sync").assert_row(
bs_trace::BLOCK_PEER_PARKED,
&[(
bs_trace::REASON,
TraceValue::Str("block_sync_no_block_progress"),
)],
);
reactor_task.abort();
}
#[tokio::test]
async fn late_unowned_body_is_rejected_and_the_session_is_parked() {
let mut config = immediate_body_download_config();
config.request_timeout = Duration::from_millis(300);
config.floor_rescue_timeout = Duration::from_millis(120);
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 64;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(0x53);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
let connection_cancel = CancellationToken::new();
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
connection_cancel.clone(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: blocks[0].hash(),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1);
tokio::time::sleep(Duration::from_millis(200)).await;
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("late block frame queues");
assert!(
tokio::time::timeout(Duration::from_millis(200), async {
loop {
if matches!(
next_action(&mut actions).await,
BlockSyncAction::SubmitBlock { .. }
) {
break;
}
}
})
.await
.is_err(),
"a completion whose request owner retired must not reach the verifier",
);
await_until(
"late unowned body does not prevent the session park",
Duration::from_millis(1500),
|| {
handle.peer_snapshot().outbound_peers == 0
&& !service.wants_peer(&peer, ZAKURA_CAP_BLOCK_SYNC, ServicePeerDirection::Outbound)
},
)
.await
.expect("an unproven peer is still parked at its liveness deadline");
assert!(
!connection_cancel.is_cancelled(),
"a first no-progress stall preserves the shared connection",
);
reactor_task.abort();
}
#[tokio::test]
async fn peer_emits_periodic_bbr_heartbeat_while_idle() {
let mut capture = TraceCapture::for_test("peer_emits_periodic_bbr_heartbeat_while_idle")
.expect("trace capture initializes");
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let mut startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
startup.trace = ZakuraTrace::new(capture.tracer(), "01");
let (_handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, _handle.clone());
let peer = peer(0x5b);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tokio::time::sleep(Duration::from_millis(100)).await;
capture.flush().await;
let reader = capture.reader().expect("trace rows load");
reader.table("block_sync").assert_row(
bs_trace::BLOCK_PEER_BBR,
&[
("bbr_reliability_permille", TraceValue::U64(1000)),
("block_progress_proven", TraceValue::U64(0)),
("bbr_phase", TraceValue::U64(0)),
],
);
reactor_task.abort();
}
#[test]
fn work_queue_estimate_clamps_hint_between_floor_and_max_block_bytes() {
use super::work_queue::DEFAULT_BS_SIZE_FLOOR_BYTES;
let queue = super::work_queue::WorkQueue::new(block::Height(0));
queue.extend(
test_work_scope(),
[
needed(1, BlockSizeEstimate::Unknown),
needed(2, BlockSizeEstimate::Advertised(1)), needed(3, BlockSizeEstimate::Advertised(12_345)),
needed(4, BlockSizeEstimate::Confirmed(u32::MAX)), ],
);
let item = |height| {
queue
.take_in_range(block::Height(height), block::Height(height), 1)
.pop()
.expect("height present")
.1
.estimated_bytes
};
assert_eq!(item(1), block::MAX_BLOCK_BYTES);
assert_eq!(item(2), DEFAULT_BS_SIZE_FLOOR_BYTES);
assert_eq!(item(3), 12_345);
assert_eq!(item(4), block::MAX_BLOCK_BYTES);
let tuned = super::work_queue::WorkQueue::new(block::Height(0));
tuned.set_estimate_floor_for_tests(100);
tuned.extend(
test_work_scope(),
[
needed(10, BlockSizeEstimate::Unknown),
needed(11, BlockSizeEstimate::Advertised(50)), ],
);
assert_eq!(
tuned
.take_in_range(block::Height(10), block::Height(10), 1)
.pop()
.unwrap()
.1
.estimated_bytes,
block::MAX_BLOCK_BYTES
);
assert_eq!(
tuned
.take_in_range(block::Height(11), block::Height(11), 1)
.pop()
.unwrap()
.1
.estimated_bytes,
100
);
}
#[test]
fn work_queue_diagnostics_report_runs_min_and_max() {
let queue = work_queue_with(
0,
[
needed(10, BlockSizeEstimate::Advertised(100)),
needed(11, BlockSizeEstimate::Advertised(100)),
needed(12, BlockSizeEstimate::Advertised(100)),
needed(20, BlockSizeEstimate::Advertised(100)),
needed(21, BlockSizeEstimate::Advertised(100)),
],
);
assert_eq!(queue.pending_run_count(), 2);
assert_eq!(queue.pending_len(), 5);
assert_eq!(queue.min_pending(), Some(block::Height(10)));
assert_eq!(queue.max_in_flight(), None);
queue.take_in_range(block::Height(20), block::Height(21), 2);
assert_eq!(queue.pending_run_count(), 1, "the 20..=21 run was taken");
assert_eq!(queue.min_pending(), Some(block::Height(10)));
assert_eq!(queue.max_in_flight(), Some(block::Height(21)));
assert_eq!(
queue.hash_for_height(block::Height(21)),
Some(block::Hash([21; 32]))
);
assert_eq!(
queue.hash_for_height(block::Height(12)),
Some(block::Hash([12; 32]))
);
assert_eq!(queue.hash_for_height(block::Height(99)), None);
}
#[test]
fn work_queue_keeps_pending_ordered_by_height() {
let queue = super::work_queue::WorkQueue::new(block::Height(0));
queue.extend(
test_work_scope(),
[needed(20, BlockSizeEstimate::Advertised(100))],
);
queue.extend(
test_work_scope(),
[needed(10, BlockSizeEstimate::Advertised(100))],
);
assert_eq!(queue.min_pending(), Some(block::Height(10)));
let taken = queue.take_in_range(block::Height(1), block::Height(30), 1);
assert_eq!(
taken[0].0,
block::Height(10),
"lower work must be taken before higher work"
);
}
#[test]
fn reorder_drains_only_contiguous_prefix_and_reports_dropped_bytes() {
let mut reorder = ReorderBuffer::new();
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
assert_eq!(
reorder.insert(block::Height(3), block.clone(), 300, peer(0)),
ReorderInsertResult::Inserted
);
assert!(reorder.drain_contiguous_prefix(block::Height(0)).is_empty());
assert_eq!(reorder.buffered_bytes(), 300);
assert_eq!(
reorder.insert(block::Height(1), block.clone(), 100, peer(0)),
ReorderInsertResult::Inserted
);
let released = reorder.drain_contiguous_prefix(block::Height(0));
assert_eq!(
released
.iter()
.map(|drained| (drained.height, drained.bytes))
.collect::<Vec<_>>(),
vec![(block::Height(1), 100)]
);
assert_eq!(reorder.buffered_bytes(), 300);
assert_eq!(
reorder.insert(block::Height(2), block.clone(), 200, peer(0)),
ReorderInsertResult::Inserted
);
let released = reorder.drain_contiguous_prefix(block::Height(1));
assert_eq!(
released
.iter()
.map(|drained| (drained.height, drained.bytes))
.collect::<Vec<_>>(),
vec![(block::Height(2), 200), (block::Height(3), 300)]
);
assert_eq!(reorder.buffered_bytes(), 0);
assert_eq!(
reorder.insert(block::Height(2), block.clone(), 200, peer(0)),
ReorderInsertResult::Inserted
);
assert_eq!(
reorder.insert(block::Height(3), block, 300, peer(0)),
ReorderInsertResult::Inserted
);
assert_eq!(reorder.drop_from(block::Height(3)), 300);
assert_eq!(reorder.buffered_bytes(), 200);
assert_eq!(
reorder.decoded_attributed_memory_bytes(),
reorder.decoded_attributed_memory_bytes_scanned()
);
assert_eq!(reorder.drop_through(block::Height(2)), 200);
assert_eq!(reorder.buffered_bytes(), 0);
assert_eq!(reorder.decoded_attributed_memory_bytes(), 0);
assert_eq!(
reorder.insert(
block::Height(3),
mainnet_block(&BLOCK_MAINNET_1_BYTES),
300,
peer(0)
),
ReorderInsertResult::Inserted
);
assert_eq!(reorder.clear(), 300);
assert_eq!(reorder.buffered_bytes(), 0);
assert_eq!(reorder.decoded_attributed_memory_bytes(), 0);
}
fn test_sequencer(verified_tip: u32, submitted_apply_limit: usize) -> Sequencer {
Sequencer::new(block::Height(verified_tip), submitted_apply_limit)
}
#[test]
fn sequencer_accept_body_buffers_then_reports_duplicate() {
let mut seq = test_sequencer(0, 4);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let hash = block.hash();
let decoded_attributed_memory_size_bytes = block.attributed_memory_size_bytes();
assert_eq!(
seq.accept_body(block::Height(1), hash, block.clone(), 100, peer(0)),
AcceptOutcome::Buffered {
covered: block::Height(1)
}
);
assert!(seq.reorder_contains(block::Height(1)));
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
decoded_attributed_memory_size_bytes
);
assert_eq!(
seq.accept_body(block::Height(1), hash, block, 100, peer(0)),
AcceptOutcome::Redundant { release_bytes: 100 }
);
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
decoded_attributed_memory_size_bytes
);
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
seq.reorder_decoded_attributed_memory_bytes_scanned()
);
}
#[test]
fn sequencer_retains_raw_bytes_for_non_contiguous_backlog() {
let mut seq = test_sequencer(0, 4);
let blocks = mainnet_blocks_1_to_3();
let block1 = blocks[0].clone();
let block2 = blocks[1].clone();
let distinguishable_decoded_block2 = forked_block(&block2, 99);
assert_ne!(distinguishable_decoded_block2.hash(), block2.hash());
assert_eq!(
distinguishable_decoded_block2.coinbase_height(),
block2.coinbase_height()
);
let block2_body = BufferedBlockBody::from_decoded_block(
distinguishable_decoded_block2.clone(),
Some(raw_block_payload(&block2)),
);
assert_eq!(
seq.accept_buffered_body(
test_work_owner(),
zakura_header_chain::SourceId::from_digest([1; 32]),
block::Height(2),
block2.hash(),
block2.header.previous_block_hash,
block2_body,
200,
peer(0)
),
AcceptOutcome::Buffered {
covered: block::Height(2)
}
);
assert!(seq.drain_ready_into_applying().is_empty());
assert!(seq.reorder_contains(block::Height(2)));
assert_eq!(seq.reorder_decoded_attributed_memory_bytes(), 0);
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
seq.reorder_decoded_attributed_memory_bytes_scanned()
);
assert_eq!(
seq.accept_body(
block::Height(1),
block1.hash(),
block1.clone(),
100,
peer(0),
),
AcceptOutcome::Buffered {
covered: block::Height(1)
}
);
assert_eq!(
seq.drain_ready_into_applying(),
vec![block::Height(1), block::Height(2)]
);
assert_eq!(seq.applying_hash(block::Height(2)), Some(block2.hash()));
let submitted = seq
.prepare_submit(block::Height(2))
.expect("height 2 is applying");
assert_eq!(submitted.block.hash(), block2.hash());
assert_ne!(
submitted.block.hash(),
distinguishable_decoded_block2.hash()
);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
block1
.attributed_memory_size_bytes()
.saturating_add(block2.attributed_memory_size_bytes())
);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
seq.applying_decoded_attributed_memory_bytes_scanned()
);
}
#[test]
fn sequencer_bounds_decoded_bodies_to_submission_window() {
let limit = 2;
let mut seq = test_sequencer(0, limit);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
let body =
BufferedBlockBody::from_decoded_block(block.clone(), Some(raw_block_payload(block)));
assert_eq!(
seq.accept_buffered_body(
test_work_owner(),
zakura_header_chain::SourceId::from_digest([1; 32]),
height,
block.hash(),
block.header.previous_block_hash,
body,
100,
peer(0)
),
AcceptOutcome::Buffered { covered: height }
);
}
assert_eq!(
seq.drain_ready_into_applying(),
vec![block::Height(1), block::Height(2), block::Height(3)]
);
assert_eq!(seq.applying_len(), 3);
assert!(
seq.decoded_applying_count() <= limit,
"decoded applying bodies ({}) exceed the submission window ({limit})",
seq.decoded_applying_count(),
);
for height in seq.submittable_heights() {
seq.prepare_submit(height).expect("height is applying");
}
assert_eq!(seq.in_flight_submission_count(), limit);
assert_eq!(seq.decoded_applying_count(), limit);
seq.unsubmit(block::Height(1), 1);
assert_eq!(seq.decoded_applying_count(), limit - 1);
}
#[test]
fn sequencer_applying_counters_match_scan_across_transitions() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
let check = |seq: &Sequencer, label: &str| {
assert_eq!(
seq.applying_buffered_bytes(),
seq.applying_buffered_bytes_scanned(),
"applying_buffered_bytes drifted after {label}"
);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
seq.applying_decoded_attributed_memory_bytes_scanned(),
"applying_decoded_attributed_memory_bytes drifted after {label}"
);
assert_eq!(
seq.reorder_decoded_attributed_memory_bytes(),
seq.reorder_decoded_attributed_memory_bytes_scanned(),
"reorder_decoded_attributed_memory_bytes drifted after {label}"
);
assert_eq!(
seq.in_flight_submission_count(),
seq.in_flight_submission_count_scanned(),
"in_flight_submission_count drifted after {label}"
);
assert_eq!(
seq.in_flight_submission_bytes(),
seq.in_flight_submission_bytes_scanned(),
"in_flight_submission_bytes drifted after {label}"
);
assert_eq!(
seq.unsubmitted_applying_count(),
seq.applying_len() - seq.attached_submission_count_scanned(),
"unsubmitted derivation wrong after {label}"
);
};
for (i, block) in blocks.iter().enumerate() {
let height = (i + 1) as u32;
seq.accept_body(
block::Height(height),
block.hash(),
block.clone(),
100 * u64::from(height),
peer(0),
);
}
assert_eq!(
seq.drain_ready_into_applying(),
vec![block::Height(1), block::Height(2), block::Height(3)]
);
assert_eq!(seq.applying_buffered_bytes(), 600);
assert_eq!(seq.in_flight_submission_count(), 0);
check(&seq, "drain");
let item1 = seq
.prepare_submit(block::Height(1))
.expect("height 1 applying");
let item2 = seq
.prepare_submit(block::Height(2))
.expect("height 2 applying");
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(seq.in_flight_submission_bytes(), 300);
assert_eq!(seq.unsubmitted_applying_count(), 1);
check(&seq, "submit 1,2");
seq.unsubmit(block::Height(1), item1.token);
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.in_flight_submission_bytes(), 200);
check(&seq, "unsubmit 1");
seq.remove_applying(block::Height(2));
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.in_flight_submission_bytes(), 200);
assert_eq!(seq.applying_buffered_bytes(), 400);
check(&seq, "detach submitted 2");
assert!(seq.finish_submission(
item2.owner,
item2.source,
item2.token,
item2.height,
item2.hash
));
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.in_flight_submission_bytes(), 0);
check(&seq, "finish detached 2");
seq.advance_verified_tip(block::Height(1), true);
assert_eq!(seq.applying_buffered_bytes(), 300);
check(&seq, "advance_verified_tip");
seq.reset_to(block::Height(0), false);
assert_eq!(seq.applying_buffered_bytes(), 0);
assert_eq!(seq.applying_decoded_attributed_memory_bytes(), 0);
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.in_flight_submission_bytes(), 0);
check(&seq, "reset");
}
#[tokio::test]
async fn sequencer_stale_checkpoint_completions_refill_full_submission_window() {
const BLOCK_COUNT: u32 = 403;
const CHANNEL_TIMEOUT: Duration = Duration::from_secs(2);
const MISSING_SUBMISSION_TIMEOUT: Duration = Duration::from_millis(100);
let submission_limit = MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES;
let submission_limit_u64 =
u64::try_from(submission_limit).expect("the checkpoint submission limit fits in u64");
let body_channel_capacity = usize::try_from(BLOCK_COUNT).expect("403 test bodies fit in usize");
assert_eq!(submission_limit, 401);
let blocks = fake_sequential_blocks(BLOCK_COUNT);
let frontiers = BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
};
let body_input_bytes = Arc::new(std::sync::atomic::AtomicU64::new(0));
let body_input_decoded_attributed_memory_bytes = Arc::new(std::sync::atomic::AtomicU64::new(0));
let (body_tx, body_rx) = mpsc::channel(body_channel_capacity);
let (control_tx, control_rx) = mpsc::unbounded_channel();
let (actions_tx, mut actions_rx) = mpsc::channel(submission_limit + 128);
let (view_tx, mut view_rx) = watch::channel(initial_view(frontiers));
let task = SequencerTask::new(
Sequencer::new(block::Height(0), submission_limit),
ByteBudget::new(1),
Arc::new(WorkQueue::new(block::Height(0))),
Arc::new(PeerRegistry::new()),
actions_tx,
ThroughputMeter::new(Instant::now()),
frontiers,
Some(test_work_scope()),
crate::zakura::header_sync::SeededRetryJitter::new([0; 32]),
body_rx,
control_rx,
body_input_bytes.clone(),
body_input_decoded_attributed_memory_bytes.clone(),
view_tx,
CHANNEL_TIMEOUT,
ZakuraTrace::noop(),
);
let task = tokio::spawn(task.run());
for (index, block) in blocks.iter().enumerate() {
let height =
block::Height(u32::try_from(index + 1).expect("403 test block indices fit in u32"));
body_tx
.send(SequencedBody::new_queued(
test_work_owner(),
zakura_header_chain::SourceId::from_digest([1; 32]),
height,
block.hash(),
block.header.previous_block_hash,
BufferedBlockBody::from_decoded_block(
block.clone(),
Some(raw_block_payload(block)),
),
u64::from(block_size(block)),
peer(1),
Instant::now(),
body_input_bytes.clone(),
body_input_decoded_attributed_memory_bytes.clone(),
))
.await
.expect("sequencer body channel remains live");
}
time::timeout(CHANNEL_TIMEOUT, async {
loop {
let view = *view_rx.borrow_and_update();
if view.applying_len == u64::from(BLOCK_COUNT)
&& view.in_flight_submission_count == submission_limit_u64
&& view.unsubmitted_applying_count == 2
{
break;
}
view_rx
.changed()
.await
.expect("sequencer view remains live");
}
})
.await
.expect("all bodies reach applying and fill the initial submission window");
let mut initial_submissions = Vec::with_capacity(submission_limit);
while initial_submissions.len() < submission_limit {
let action = time::timeout(CHANNEL_TIMEOUT, actions_rx.recv())
.await
.expect("initial submission action arrives")
.expect("sequencer action channel remains live");
match action {
BlockSyncAction::SubmitBlock { token, block, .. } => {
initial_submissions.push((token, block));
}
action => panic!("unexpected action before initial submissions complete: {action:?}"),
}
}
for (index, (_, block)) in initial_submissions.iter().enumerate() {
assert_eq!(
block.coinbase_height(),
Some(block::Height(
u32::try_from(index + 1).expect("401 submission indices fit in u32")
)),
"initial submissions must be ordered through the full checkpoint window",
);
}
let (token_1, block_1) = &initial_submissions[0];
control_tx
.send(SequencerControlInput::ApplyFinished {
owner: Box::new(test_work_owner()),
source: zakura_header_chain::SourceId::from_digest([1; 32]),
token: *token_1,
height: block::Height(1),
hash: block_1.hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
eligible_sources: BTreeSet::new(),
persisted_availability: None,
semantic_completion: Some((
test_work_owner(),
zakura_header_chain::StateVersion::default(),
)),
})
.expect("height 1 completion queues");
let block_402 = match time::timeout(CHANNEL_TIMEOUT, actions_rx.recv())
.await
.expect("height 402 submission arrives")
.expect("sequencer action channel remains live")
{
BlockSyncAction::SubmitBlock { block, .. } => block,
action => panic!("unexpected action before height 402 submission: {action:?}"),
};
assert_eq!(block_402.coinbase_height(), Some(block::Height(402)));
for (height, (token, block)) in [
(block::Height(2), &initial_submissions[1]),
(block::Height(3), &initial_submissions[2]),
] {
control_tx
.send(SequencerControlInput::ApplyFinished {
owner: Box::new(test_work_owner()),
source: zakura_header_chain::SourceId::from_digest([1; 32]),
token: *token,
height,
hash: block.hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
eligible_sources: BTreeSet::new(),
persisted_availability: None,
semantic_completion: Some((
test_work_owner(),
zakura_header_chain::StateVersion::default(),
)),
})
.expect("stale checkpoint completion queues");
}
let submit_403 = time::timeout(MISSING_SUBMISSION_TIMEOUT, async {
loop {
let action = actions_rx
.recv()
.await
.expect("sequencer action channel remains live");
if let BlockSyncAction::SubmitBlock { block, .. } = action {
assert_eq!(block.coinbase_height(), Some(block::Height(403)));
break;
}
}
})
.await;
let diagnostic = *view_rx.borrow();
assert!(
submit_403.is_ok(),
"height 403 must refill the checkpoint submission window; \
in_flight_submission_count = {}, unsubmitted_applying_count = {}",
diagnostic.in_flight_submission_count,
diagnostic.unsubmitted_applying_count,
);
time::timeout(CHANNEL_TIMEOUT, async {
loop {
let view = *view_rx.borrow_and_update();
if view.in_flight_submission_count == 400 && view.unsubmitted_applying_count == 0 {
break;
}
view_rx
.changed()
.await
.expect("sequencer view remains live");
}
})
.await
.expect("stale completions settle with a refilled checkpoint submission window");
task.abort();
}
#[tokio::test]
async fn detached_checkpoint_duplicates_refill_current_submission_window() {
const BLOCK_COUNT: u32 = 402;
const DETACHED_PREFIX: usize = 150;
const CHANNEL_TIMEOUT: Duration = Duration::from_secs(2);
let submission_limit = MIN_BS_CHECKPOINT_SUBMITTED_BLOCK_APPLIES;
assert_eq!(submission_limit, 401);
let body_channel_capacity = usize::try_from(BLOCK_COUNT).expect("402 test bodies fit in usize");
let blocks = fake_sequential_blocks(BLOCK_COUNT);
let frontiers = BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
};
let body_input_bytes = Arc::new(std::sync::atomic::AtomicU64::new(0));
let body_input_decoded_attributed_memory_bytes = Arc::new(std::sync::atomic::AtomicU64::new(0));
let (body_tx, body_rx) = mpsc::channel(body_channel_capacity);
let (control_tx, control_rx) = mpsc::unbounded_channel();
let (actions_tx, mut actions_rx) = mpsc::channel(submission_limit + DETACHED_PREFIX);
let (view_tx, mut view_rx) = watch::channel(initial_view(frontiers));
let task = SequencerTask::new(
Sequencer::new(block::Height(0), submission_limit),
ByteBudget::new(1),
Arc::new(WorkQueue::new(block::Height(0))),
Arc::new(PeerRegistry::new()),
actions_tx,
ThroughputMeter::new(Instant::now()),
frontiers,
Some(test_work_scope()),
crate::zakura::header_sync::SeededRetryJitter::new([0; 32]),
body_rx,
control_rx,
body_input_bytes.clone(),
body_input_decoded_attributed_memory_bytes.clone(),
view_tx,
CHANNEL_TIMEOUT,
ZakuraTrace::noop(),
);
let task = tokio::spawn(task.run());
for (index, block) in blocks.iter().take(DETACHED_PREFIX).enumerate() {
let height =
block::Height(u32::try_from(index + 1).expect("detached prefix indices fit in u32"));
body_tx
.send(SequencedBody::new_queued(
test_work_owner(),
zakura_header_chain::SourceId::from_digest([1; 32]),
height,
block.hash(),
block.header.previous_block_hash,
BufferedBlockBody::from_decoded_block(
block.clone(),
Some(raw_block_payload(block)),
),
u64::from(block_size(block)),
peer(1),
Instant::now(),
body_input_bytes.clone(),
body_input_decoded_attributed_memory_bytes.clone(),
))
.await
.expect("initial checkpoint bodies queue");
}
let mut detached_submissions = Vec::with_capacity(DETACHED_PREFIX);
while detached_submissions.len() < DETACHED_PREFIX {
let action = time::timeout(CHANNEL_TIMEOUT, actions_rx.recv())
.await
.expect("initial checkpoint submission arrives")
.expect("sequencer action channel remains live");
match action {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => detached_submissions.push((owner, source, token, block)),
action => panic!("unexpected initial checkpoint action: {action:?}"),
}
}
let mut changed_authority = test_work_scope();
changed_authority.body_work_epoch = zakura_header_chain::BodyWorkEpoch::new(1);
control_tx
.send(SequencerControlInput::BodyWorkEpochChanged {
authority: changed_authority,
frontiers,
})
.expect("scope transition queues");
time::timeout(CHANNEL_TIMEOUT, async {
loop {
let view = *view_rx.borrow_and_update();
if view.applying_len == 0
&& view.in_flight_submission_count
== u64::try_from(DETACHED_PREFIX).expect("detached prefix fits u64")
{
break;
}
view_rx
.changed()
.await
.expect("sequencer view remains live");
}
})
.await
.expect("scope transition detaches the old verifier submissions");
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(u32::try_from(index + 1).expect("402 indices fit in u32"));
body_tx
.send(SequencedBody::new_queued(
changed_authority.bind(test_work_owner().session_id, test_work_owner().request_id),
zakura_header_chain::SourceId::from_digest([1; 32]),
height,
block.hash(),
block.header.previous_block_hash,
BufferedBlockBody::from_decoded_block(
block.clone(),
Some(raw_block_payload(block)),
),
u64::from(block_size(block)),
peer(1),
Instant::now(),
body_input_bytes.clone(),
body_input_decoded_attributed_memory_bytes.clone(),
))
.await
.expect("current checkpoint bodies queue");
}
let current_initial = submission_limit - DETACHED_PREFIX;
for expected_height in 1..=current_initial {
let action = time::timeout(CHANNEL_TIMEOUT, actions_rx.recv())
.await
.expect("current checkpoint submission arrives")
.expect("sequencer action channel remains live");
let BlockSyncAction::SubmitBlock { block, .. } = action else {
panic!("unexpected current checkpoint action: {action:?}");
};
assert_eq!(
block.coinbase_height(),
Some(block::Height(
u32::try_from(expected_height).expect("submission height fits u32")
))
);
}
for (owner, source, token, block) in detached_submissions {
let height = block.coinbase_height().expect("test block has height");
control_tx
.send(SequencerControlInput::ApplyFinished {
owner: Box::new(owner),
source,
token,
height,
hash: block.hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Duplicate),
eligible_sources: BTreeSet::new(),
persisted_availability: None,
semantic_completion: None,
})
.expect("detached duplicate completion queues");
}
for expected_height in (current_initial + 1)..=submission_limit {
let action = time::timeout(CHANNEL_TIMEOUT, actions_rx.recv())
.await
.expect("refill submission arrives")
.expect("sequencer action channel remains live");
let BlockSyncAction::SubmitBlock { block, .. } = action else {
panic!("unexpected refill action: {action:?}");
};
assert_eq!(
block.coinbase_height(),
Some(block::Height(
u32::try_from(expected_height).expect("refill height fits u32")
)),
"each stale duplicate completion must refill one current checkpoint slot",
);
}
time::timeout(CHANNEL_TIMEOUT, async {
loop {
let view = *view_rx.borrow_and_update();
if view.in_flight_submission_count
== u64::try_from(submission_limit).expect("submission limit fits u64")
&& view.unsubmitted_applying_count == 1
{
break;
}
view_rx
.changed()
.await
.expect("sequencer view remains live");
}
})
.await
.expect("detached completions refill the full checkpoint submission window");
task.abort();
}
#[test]
fn sequencer_accept_body_rejects_at_or_below_floor() {
let mut seq = test_sequencer(5, 4);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
assert_eq!(
seq.accept_body(block::Height(5), block.hash(), block.clone(), 100, peer(0)),
AcceptOutcome::Redundant { release_bytes: 100 }
);
assert!(!seq.reorder_contains(block::Height(5)));
}
#[test]
fn sequencer_drains_contiguous_prefix_into_applying_and_advances_floor() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
seq.accept_body(
block::Height(1),
blocks[0].hash(),
blocks[0].clone(),
100,
peer(0),
);
seq.accept_body(
block::Height(3),
blocks[2].hash(),
blocks[2].clone(),
300,
peer(0),
);
assert_eq!(seq.drain_ready_into_applying(), vec![block::Height(1)]);
assert_eq!(seq.floor(), block::Height(1));
assert!(seq.applying_contains(block::Height(1)));
assert_eq!(seq.applying_len(), 1);
seq.accept_body(
block::Height(2),
blocks[1].hash(),
blocks[1].clone(),
200,
peer(0),
);
assert_eq!(
seq.drain_ready_into_applying(),
vec![block::Height(2), block::Height(3)]
);
assert_eq!(seq.floor(), block::Height(3));
assert_eq!(seq.reorder_len(), 0);
}
#[test]
fn sequencer_submits_within_window_and_rolls_back_on_unsubmit() {
let mut seq = test_sequencer(0, 2);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(height, block.hash(), block.clone(), 100, peer(0));
}
assert_eq!(seq.drain_ready_into_applying().len(), 3);
assert_eq!(
seq.submittable_heights(),
vec![block::Height(1), block::Height(2)]
);
let item1 = seq.prepare_submit(block::Height(1)).expect("applying at 1");
let item2 = seq.prepare_submit(block::Height(2)).expect("applying at 2");
assert_eq!((item1.token, item2.token), (1, 2));
assert_eq!(seq.in_flight_submission_count(), 2);
assert!(seq.submittable_heights().is_empty());
seq.unsubmit(block::Height(2), item2.token);
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.submittable_heights(), vec![block::Height(2)]);
}
#[test]
fn sequencer_completed_duplicate_releases_attached_decode_window_slot() {
let mut seq = test_sequencer(0, 1);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().take(2).enumerate() {
let height = block::Height(index as u32 + 1);
let body =
BufferedBlockBody::from_decoded_block(block.clone(), Some(raw_block_payload(block)));
seq.accept_buffered_body(
test_work_owner(),
zakura_header_chain::SourceId::from_digest([1; 32]),
height,
block.hash(),
block.header.previous_block_hash,
body,
100,
peer(0),
);
}
seq.drain_ready_into_applying();
let item = seq
.prepare_submit(block::Height(1))
.expect("height 1 is applying");
seq.record_submitted_apply(item.height, item.hash);
assert_eq!(seq.in_flight_submission_count(), 1);
assert!(seq.submittable_heights().is_empty());
assert!(seq.finish_attached_submission(
item.owner,
item.source,
item.token,
item.height,
item.hash
));
assert!(seq.applying_contains(block::Height(1)));
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.decoded_applying_count(), 0);
assert_eq!(seq.submittable_heights(), vec![block::Height(2)]);
}
#[test]
fn sequencer_records_and_decrements_submitted_applies() {
let mut seq = test_sequencer(0, 4);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let hash = block.hash();
assert!(!seq.has_submitted_apply(block::Height(1), hash));
seq.accept_body(block::Height(1), hash, block, 100, peer(0));
seq.drain_ready_into_applying();
let item = seq
.prepare_submit(block::Height(1))
.expect("height 1 is applying");
seq.record_submitted_apply(item.height, item.hash);
assert!(seq.has_submitted_apply(block::Height(1), hash));
assert!(seq.submitted_contains(block::Height(1)));
seq.remove_applying(item.height);
assert!(seq.finish_submission(item.owner, item.source, item.token, item.height, item.hash));
assert!(!seq.has_submitted_apply(block::Height(1), hash));
assert!(!seq.submitted_contains(block::Height(1)));
}
#[test]
fn sequencer_frontier_release_keeps_in_flight_submission_charged_until_completion() {
let mut seq = test_sequencer(0, 1);
let blocks = mainnet_blocks_1_to_3();
let decoded_attributed_memory_bytes = blocks
.iter()
.map(|block| block.attributed_memory_size_bytes())
.fold(0u64, u64::saturating_add);
let remaining_decoded_attributed_memory_bytes = blocks
.iter()
.skip(1)
.map(|block| block.attributed_memory_size_bytes())
.fold(0u64, u64::saturating_add);
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(height, block.hash(), block.clone(), 100, peer(0));
}
seq.drain_ready_into_applying();
let item = seq
.prepare_submit(block::Height(1))
.expect("height 1 is applying");
seq.record_submitted_apply(item.height, item.hash);
assert!(seq.submitted_contains(block::Height(1)));
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes
);
assert!(
seq.submittable_heights().is_empty(),
"submitted-apply window is full"
);
assert_eq!(seq.release_applied_through(block::Height(1)), 100);
assert!(!seq.submitted_contains(block::Height(1)));
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes,
"detached driver-owned decoded memory remains charged"
);
assert!(
seq.submittable_heights().is_empty(),
"detached driver submission still occupies the decode window"
);
assert!(!seq.finish_submission(
item.owner,
item.source,
item.token,
item.height,
block::Hash([99; 32])
));
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes
);
let wrong_owner = zakura_header_chain::BodyWorkOwner {
request_id: std::num::NonZeroU64::new(item.owner.request_id.get().saturating_add(1))
.expect("incremented test request ID is nonzero"),
..item.owner
};
assert!(!seq.finish_submission(wrong_owner, item.source, item.token, item.height, item.hash));
assert!(!seq.finish_submission(
item.owner,
zakura_header_chain::SourceId::from_digest([99; 32]),
item.token,
item.height,
item.hash
));
assert_eq!(seq.in_flight_submission_count(), 1);
assert!(seq.finish_submission(item.owner, item.source, item.token, item.height, item.hash));
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
remaining_decoded_attributed_memory_bytes
);
assert_eq!(seq.submittable_heights(), vec![block::Height(2)]);
}
#[test]
fn sequencer_advance_verified_tip_releases_bytes_and_reports_change() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
seq.accept_body(
block::Height(1),
blocks[0].hash(),
blocks[0].clone(),
100,
peer(0),
);
seq.accept_body(
block::Height(2),
blocks[1].hash(),
blocks[1].clone(),
200,
peer(0),
);
let advance = seq.advance_verified_tip(block::Height(2), false);
assert!(advance.changed);
assert_eq!(advance.release_bytes, 300);
assert_eq!(seq.verified_tip(), block::Height(2));
assert!(seq.floor() >= block::Height(2));
let advance = seq.advance_verified_tip(block::Height(2), false);
assert!(!advance.changed);
assert_eq!(advance.release_bytes, 0);
}
#[test]
fn sequencer_reset_clears_buffers_and_pins_floor_and_tip() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
seq.accept_body(
block::Height(1),
blocks[0].hash(),
blocks[0].clone(),
100,
peer(0),
);
seq.drain_ready_into_applying();
seq.accept_body(
block::Height(2),
blocks[1].hash(),
blocks[1].clone(),
200,
peer(0),
);
let released = seq.reset_to(block::Height(0), false);
assert_eq!(released, 300);
assert_eq!(seq.floor(), block::Height(0));
assert_eq!(seq.verified_tip(), block::Height(0));
assert_eq!(seq.applying_len(), 0);
assert_eq!(seq.reorder_len(), 0);
}
#[test]
fn sequencer_reset_keeps_detached_submissions_charged_until_matching_completions() {
let mut seq = test_sequencer(0, 2);
let blocks = mainnet_blocks_1_to_3();
let decoded_attributed_memory_bytes = blocks
.iter()
.take(2)
.map(|block| block.attributed_memory_size_bytes())
.fold(0u64, u64::saturating_add);
for (index, block) in blocks.iter().take(2).enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(height, block.hash(), block.clone(), 100, peer(0));
}
seq.drain_ready_into_applying();
let old_items: Vec<_> = seq
.submittable_heights()
.into_iter()
.map(|height| {
let item = seq.prepare_submit(height).expect("height is applying");
seq.record_submitted_apply(item.height, item.hash);
item
})
.collect();
assert_eq!(old_items.len(), 2);
assert_eq!(seq.in_flight_submission_count(), 2);
seq.reset_to(block::Height(0), false);
assert_eq!(seq.applying_len(), 0);
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(seq.in_flight_submission_bytes(), 200);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes,
"reset must retain detached driver-owned decoded memory charges"
);
for (index, block) in blocks.iter().take(2).enumerate() {
let replacement = forked_block(block, 100 + index as u8);
let height = block::Height(index as u32 + 1);
seq.accept_body(height, replacement.hash(), replacement, 100, peer(1));
}
seq.drain_ready_into_applying();
assert!(
seq.submittable_heights().is_empty(),
"replacement bodies must not exceed the live decode window"
);
let decoded_attributed_memory_bytes_before_completion =
seq.applying_decoded_attributed_memory_bytes();
let first = &old_items[0];
assert!(
!seq.finish_submission(
first.owner,
first.source,
first.token,
first.height,
block::Hash([99; 32])
),
"a mismatched completion must not release a detached charge"
);
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes_before_completion
);
assert!(seq.finish_submission(
first.owner,
first.source,
first.token,
first.height,
first.hash
));
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes_before_completion
.saturating_sub(blocks[0].attributed_memory_size_bytes())
);
assert_eq!(seq.submittable_heights().len(), 1);
let second = &old_items[1];
assert!(seq.finish_submission(
second.owner,
second.source,
second.token,
second.height,
second.hash
));
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
decoded_attributed_memory_bytes_before_completion
.saturating_sub(decoded_attributed_memory_bytes)
);
assert_eq!(seq.submittable_heights().len(), 2);
}
#[test]
fn sequencer_reject_drops_successors_and_rolls_floor_back() {
let mut seq = test_sequencer(0, 8);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(height, block.hash(), block.clone(), 100, peer(0));
}
seq.drain_ready_into_applying();
assert_eq!(seq.floor(), block::Height(3));
let released = seq.release_applying_blocks_from(block::Height(2));
assert_eq!(released, 200);
assert!(seq.applying_contains(block::Height(1)));
assert!(!seq.applying_contains(block::Height(2)));
seq.reset_floor_below(block::Height(2));
assert_eq!(seq.floor(), block::Height(1));
assert_eq!(seq.release_applied_through(block::Height(1)), 100);
assert_eq!(seq.applying_len(), 0);
}
#[test]
fn sequencer_rejection_release_keeps_detached_submissions_charged() {
let mut seq = test_sequencer(0, 2);
let blocks = mainnet_blocks_1_to_3();
for (index, block) in blocks.iter().enumerate() {
let height = block::Height(index as u32 + 1);
seq.accept_body(
height,
block.hash(),
block.clone(),
100 * (index as u64 + 1),
peer(0),
);
}
seq.drain_ready_into_applying();
let item2 = seq
.prepare_submit(block::Height(2))
.expect("height 2 applying");
let item3 = seq
.prepare_submit(block::Height(3))
.expect("height 3 applying");
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(seq.in_flight_submission_bytes(), 200 + 300);
let released = seq.release_applying_blocks_from(block::Height(2));
assert_eq!(released, 500);
assert_eq!(seq.applying_len(), 1);
assert_eq!(seq.applying_buffered_bytes(), 100);
assert_eq!(seq.in_flight_submission_count(), 2);
assert_eq!(seq.in_flight_submission_bytes(), 500);
assert!(seq.submittable_heights().is_empty());
assert_eq!(
seq.applying_buffered_bytes(),
seq.applying_buffered_bytes_scanned()
);
assert_eq!(
seq.in_flight_submission_count(),
seq.in_flight_submission_count_scanned()
);
assert_eq!(
seq.in_flight_submission_bytes(),
seq.in_flight_submission_bytes_scanned()
);
assert!(seq.finish_submission(
item2.owner,
item2.source,
item2.token,
item2.height,
item2.hash
));
assert_eq!(seq.submittable_heights(), vec![block::Height(1)]);
assert!(seq.finish_submission(
item3.owner,
item3.source,
item3.token,
item3.height,
item3.hash
));
assert_eq!(seq.in_flight_submission_count(), 0);
}
#[test]
fn sequencer_unsubmit_ignores_stale_or_mismatched_token() {
let mut seq = test_sequencer(0, 4);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
seq.accept_body(block::Height(1), block.hash(), block.clone(), 100, peer(0));
seq.drain_ready_into_applying();
let item = seq
.prepare_submit(block::Height(1))
.expect("height 1 applying");
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.in_flight_submission_bytes(), 100);
seq.unsubmit(block::Height(1), item.token + 1);
assert_eq!(seq.in_flight_submission_count(), 1);
assert_eq!(seq.in_flight_submission_bytes(), 100);
assert_eq!(
seq.in_flight_submission_count(),
seq.in_flight_submission_count_scanned()
);
assert_eq!(
seq.in_flight_submission_bytes(),
seq.in_flight_submission_bytes_scanned()
);
seq.unsubmit(block::Height(1), item.token);
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.in_flight_submission_bytes(), 0);
seq.unsubmit(block::Height(1), item.token);
assert_eq!(seq.in_flight_submission_count(), 0);
assert_eq!(seq.in_flight_submission_bytes(), 0);
assert_eq!(
seq.in_flight_submission_count(),
seq.in_flight_submission_count_scanned()
);
assert_eq!(
seq.in_flight_submission_bytes(),
seq.in_flight_submission_bytes_scanned()
);
}
#[test]
fn sequencer_keeps_whole_body_for_contiguous_height() {
let mut seq = test_sequencer(0, 4);
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let distinguishable_decoded_block1 = forked_block(&block1, 99);
assert_ne!(distinguishable_decoded_block1.hash(), block1.hash());
let body = BufferedBlockBody::from_decoded_block(
distinguishable_decoded_block1.clone(),
Some(raw_block_payload(&block1)),
);
assert_eq!(
seq.accept_buffered_body(
test_work_owner(),
zakura_header_chain::SourceId::from_digest([1; 32]),
block::Height(1),
block1.hash(),
block1.header.previous_block_hash,
body,
100,
peer(0)
),
AcceptOutcome::Buffered {
covered: block::Height(1)
}
);
assert_eq!(seq.drain_ready_into_applying(), vec![block::Height(1)]);
let submitted = seq
.prepare_submit(block::Height(1))
.expect("height 1 is applying");
assert_eq!(
submitted.block.hash(),
distinguishable_decoded_block1.hash()
);
assert_eq!(
seq.applying_decoded_attributed_memory_bytes(),
distinguishable_decoded_block1.attributed_memory_size_bytes()
);
}
#[test]
fn reorder_fuzzes_arrival_order_as_parent_first() {
let orders = [
[1, 2, 3, 4],
[4, 3, 2, 1],
[2, 4, 1, 3],
[3, 1, 4, 2],
[2, 1, 4, 3],
];
for order in orders {
let mut reorder = ReorderBuffer::new();
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let mut tip = block::Height(0);
let mut released_all = Vec::new();
for height in order {
assert_eq!(
reorder.insert(block::Height(height), block.clone(), 100, peer(0)),
ReorderInsertResult::Inserted
);
for drained in reorder.drain_contiguous_prefix(tip) {
assert_eq!(drained.height, block::Height(tip.0 + 1));
tip = drained.height;
released_all.push(drained.height);
}
}
assert_eq!(
released_all,
vec![
block::Height(1),
block::Height(2),
block::Height(3),
block::Height(4)
]
);
assert_eq!(reorder.buffered_bytes(), 0);
}
}
const THREE_BLOCK_ESTIMATE: u64 = 1_000;
fn outstanding_three_block_range(budget: &mut ByteBudget) -> OutstandingBlockRange {
let request = BlockRangeRequest {
owner: test_work_owner(),
start_height: block::Height(1),
count: 3,
anchor_hash: block::Hash([1; 32]),
estimated_bytes: THREE_BLOCK_ESTIMATE * 3,
expected_blocks: vec![
ExpectedBlock {
height: block::Height(1),
hash: block::Hash([1; 32]),
estimated_bytes: 1_000,
},
ExpectedBlock {
height: block::Height(2),
hash: block::Hash([2; 32]),
estimated_bytes: 1_000,
},
ExpectedBlock {
height: block::Height(3),
hash: block::Hash([3; 32]),
estimated_bytes: 1_000,
},
],
};
assert!(budget.try_reserve(request.estimated_bytes));
let now = Instant::now();
OutstandingBlockRange {
request,
queued_at: now,
deadline: now,
delivery_snapshot: test_delivery_snapshot(now),
delivered_bytes: 0,
received: ReceivedBlockTracker::default(),
}
}
#[test]
fn outstanding_range_accumulates_delivered_bytes_for_bbr_sample() {
let mut budget = ByteBudget::new(THREE_BLOCK_ESTIMATE * 3);
let mut outstanding = outstanding_three_block_range(&mut budget);
for (height, bytes) in [
(block::Height(1), 700),
(block::Height(2), 800),
(block::Height(3), 900),
] {
outstanding.record_body_bytes(bytes);
outstanding.mark_received(height);
}
assert!(outstanding.is_complete());
assert_eq!(outstanding.delivered_bytes, 700 + 800 + 900);
}
#[test]
fn block_budget_ledger_releases_reserved_exactly_once() {
let mut ledger = BlockBudgetLedger::reserved(1_000);
assert!(ledger.is_reserved());
assert_eq!(ledger.reserved_charge(), 1_000);
assert_eq!(ledger.release_reserved(), 1_000);
assert!(!ledger.is_reserved());
assert_eq!(ledger.reserved_charge(), 0);
assert_eq!(ledger.release_reserved(), 0);
let mut released = BlockBudgetLedger::Released;
assert!(!released.is_reserved());
assert_eq!(released.release_reserved(), 0);
}
#[test]
fn budget_audit_catches_injected_drift() {
let mut budget = ByteBudget::new(1_000);
assert!(budget.try_reserve(400));
assert!(budget.audit(400, "test matching audit"));
assert!(budget.audit(300, "test transient handoff excess"));
assert!(
!budget.audit(500, "test injected drift"),
"audit reports a budget shortfall as drift"
);
budget.release(400);
assert!(budget.audit(0, "test released audit"));
}
#[test]
fn received_body_admission_ignores_request_budget() {
let config = ZakuraBlockSyncConfig {
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(600),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: 0,
reorder_buffered_blocks: 0,
applying_buffered_bytes: 0,
applying_buffered_blocks: 0,
sequencer_input_queued_bytes: 0,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: 0,
reserved_above_floor_blocks: 0,
budget_available: 0,
};
assert!(super::admission::admit_received_body(
&config,
&snapshot,
block::Height(411),
2_000_000,
));
assert!(super::admission::admit_received_body(
&config,
&snapshot,
block::Height(602),
1_000,
));
assert!(!super::admission::admit_received_body(
&config,
&snapshot,
block::Height(602),
1_001,
));
}
proptest::proptest! {
#![proptest_config(proptest::test_runner::Config::with_cases(256))]
#[test]
fn admit_respects_lookahead_bounds(
reorder_bytes in 0u64..2_000,
applying_bytes in 0u64..2_000,
input_bytes in 0u64..2_000,
reserved_bytes in 0u64..2_000,
reorder_blocks in proptest::prop_oneof![
0u64..20,
proptest::strategy::Just(super::admission::LOOKAHEAD_BLOCK_HARD_CAP),
],
applying_blocks in 0u64..20,
reserved_blocks in 0u64..20,
) {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 64_000_000,
max_reorder_lookahead_bytes: 1_000,
..ZakuraBlockSyncConfig::default()
};
let snapshot = super::admission::AdmissionSnapshot {
download_floor: block::Height(600),
verified_block_tip: block::Height(10),
reorder_buffered_bytes: reorder_bytes,
reorder_buffered_blocks: reorder_blocks,
applying_buffered_bytes: applying_bytes,
applying_buffered_blocks: applying_blocks,
sequencer_input_queued_bytes: input_bytes,
in_flight_submission_bytes: 0,
reserved_above_floor_bytes: reserved_bytes,
reserved_above_floor_blocks: reserved_blocks,
budget_available: 64_000_000,
};
let factor = super::admission::DESERIALIZED_MEM_FACTOR;
let estimated_resident = reorder_bytes
.saturating_add(applying_bytes)
.saturating_add(input_bytes)
.saturating_add(reserved_bytes)
.saturating_add(input_bytes.saturating_mul(factor));
let held_blocks = reorder_blocks
.saturating_add(applying_blocks)
.saturating_add(reserved_blocks);
let effective = config.effective_max_reorder_lookahead_bytes();
let above = super::admission::admit(
&config,
snapshot,
block::Height(602),
block::Height(100_000),
1_000,
);
if estimated_resident >= effective
|| held_blocks >= super::admission::LOOKAHEAD_BLOCK_HARD_CAP
|| effective - estimated_resident == 0
{
prop_assert_eq!(above, super::admission::AdmissionOutcome::LookaheadAtCap);
} else {
let remaining_wire = effective - estimated_resident;
let expected = snapshot.budget_available.min(remaining_wire).min(1_000);
match above {
super::admission::AdmissionOutcome::Admit(grant) => {
prop_assert_eq!(grant.max_request_bytes, expected);
prop_assert_eq!(grant.take_high, block::Height(100_000));
prop_assert!(
estimated_resident
.saturating_add(grant.max_request_bytes)
<= effective
);
}
other => prop_assert!(false, "expected a gated grant, got {:?}", other),
}
}
match super::admission::admit(
&config,
snapshot,
block::Height(11),
block::Height(100_000),
1_000,
) {
super::admission::AdmissionOutcome::Admit(grant) => {
prop_assert_eq!(grant.max_request_bytes, 1_000);
prop_assert_eq!(grant.take_high, block::Height(411));
}
other => prop_assert!(
false,
"the commit window remains admitted while budget is available, got {:?}",
other
),
}
}
}
proptest::proptest! {
#![proptest_config(proptest::test_runner::Config::with_cases(256))]
#[test]
fn block_budget_ledger_mirrors_byte_budget(
estimate in 1u64..1_000_000,
release_before_receipt in proptest::bool::ANY,
) {
let mut ledger = BlockBudgetLedger::reserved(estimate);
let mut budget = ByteBudget::new(u64::MAX);
prop_assert!(budget.try_reserve(estimate));
if release_before_receipt {
budget.release(ledger.release_reserved());
prop_assert_eq!(budget.reserved(), 0);
prop_assert!(!ledger.is_reserved());
}
budget.release(ledger.release_reserved());
prop_assert_eq!(budget.reserved(), 0);
prop_assert_eq!(ledger.reserved_charge(), 0);
budget.release(ledger.release_reserved());
prop_assert_eq!(budget.reserved(), 0);
}
}
#[test]
fn budget_reservation_tracks_only_unreceived_estimates() {
let estimate = THREE_BLOCK_ESTIMATE;
let max = estimate * 3;
{
let mut budget = ByteBudget::new(max);
let mut reorder = ReorderBuffer::new();
let mut outstanding = outstanding_three_block_range(&mut budget);
assert_eq!(budget.reserved(), max);
assert!(budget.reserved() <= budget.max_bytes_for_test());
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let actuals = [700u64, 800, 900];
for (index, height) in [block::Height(1), block::Height(2), block::Height(3)]
.into_iter()
.enumerate()
{
budget.release(estimate);
outstanding.mark_received(height);
assert_eq!(
reorder.insert(height, block.clone(), actuals[index], peer(0)),
ReorderInsertResult::Inserted
);
assert_eq!(budget.reserved(), outstanding.reserved_bytes());
}
assert!(outstanding.is_complete());
assert_eq!(outstanding.reserved_bytes(), 0);
assert_eq!(
budget.reserved(),
0,
"retained bodies charge nothing; the wire budget is fully free"
);
assert_eq!(reorder.buffered_bytes(), 700 + 800 + 900);
let applied_bytes: u64 = reorder
.drain_contiguous_prefix(block::Height(0))
.into_iter()
.map(|drained| drained.bytes)
.sum();
assert_eq!(applied_bytes, 700 + 800 + 900);
assert_eq!(budget.reserved(), 0);
}
{
let mut budget = ByteBudget::new(max);
let mut outstanding = outstanding_three_block_range(&mut budget);
assert_eq!(budget.reserved(), estimate * 3);
budget.release(estimate);
outstanding.mark_received(block::Height(1));
assert_eq!(outstanding.reserved_bytes(), estimate * 2);
assert_eq!(budget.reserved(), estimate * 2);
budget.release(outstanding.reserved_bytes());
assert_eq!(budget.reserved(), 0);
}
{
let budget = ByteBudget::new(max);
let mut reorder = ReorderBuffer::new();
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
assert_eq!(
reorder.insert(block::Height(1), block.clone(), 1_000, peer(0)),
ReorderInsertResult::Inserted
);
assert_eq!(
reorder.insert(block::Height(1), block, 1_000, peer(0)),
ReorderInsertResult::Duplicate
);
assert_eq!(reorder.buffered_bytes(), 1_000);
assert_eq!(budget.reserved(), 0);
assert!(budget.reserved() <= budget.max_bytes_for_test());
}
}
#[test]
fn received_tracker_handles_a_full_range_at_the_bitset_boundary() {
let count = MAX_BS_BLOCKS_PER_REQUEST;
assert_eq!(count, u128::BITS, "the cap is sized to the bitset width");
let mut outstanding = window_request_range(1, count);
assert_eq!(outstanding.reserved_bytes(), u64::from(count));
for height in 1..=count {
assert!(
!outstanding.has_received(block::Height(height)),
"height {height} should start unreceived",
);
outstanding.mark_received(block::Height(height));
assert!(
outstanding.has_received(block::Height(height)),
"height {height} (offset {}) must be markable — the bitset must cover the \
whole range",
height - 1,
);
}
assert!(
outstanding.is_complete(),
"a fully-received {count}-block range must report complete",
);
assert_eq!(
outstanding.reserved_bytes(),
0,
"every height received ⇒ no reserved bytes remain (no offset fell off the bitset)",
);
}
#[test]
fn underestimated_body_is_buffered_and_releases_only_its_estimate() {
let hint = 1_000u64;
let mut budget = ByteBudget::new(hint * 4);
let mut reorder = ReorderBuffer::new();
let request = BlockRangeRequest {
owner: test_work_owner(),
start_height: block::Height(1),
count: 1,
anchor_hash: block::Hash([1; 32]),
estimated_bytes: hint,
expected_blocks: vec![ExpectedBlock {
height: block::Height(1),
hash: block::Hash([1; 32]),
estimated_bytes: hint,
}],
};
assert!(budget.try_reserve(request.estimated_bytes));
let now = Instant::now();
let mut outstanding = OutstandingBlockRange {
request,
queued_at: now,
deadline: now,
delivery_snapshot: test_delivery_snapshot(now),
delivered_bytes: 0,
received: ReceivedBlockTracker::default(),
};
assert_eq!(budget.reserved(), hint);
let actual = hint * 50;
assert!(actual < BS_PER_BLOCK_WORST_CASE_BYTES);
assert!(actual > hint);
budget.release(hint);
outstanding.mark_received(block::Height(1));
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
assert_eq!(
reorder.insert(block::Height(1), block, actual, peer(0)),
ReorderInsertResult::Inserted,
"an underestimated body must still buffer; a received body is never dropped"
);
assert_eq!(reorder.buffered_bytes(), actual);
assert_eq!(budget.reserved(), 0);
assert_eq!(outstanding.reserved_bytes(), 0);
assert!(
budget.try_reserve(hint),
"the freed reservation funds the next request immediately"
);
let dropped = reorder.drop_through(block::Height(1));
assert_eq!(dropped, actual);
assert_eq!(reorder.buffered_bytes(), 0);
}
#[test]
fn block_sync_stream_declares_kind_capability_version_and_frame_cap() {
let stream = block_sync_streams()
.first()
.copied()
.expect("block sync declares one stream");
assert_eq!(stream.kind, ZAKURA_STREAM_BLOCK_SYNC);
assert_eq!(stream.version, ZAKURA_BLOCK_SYNC_STREAM_VERSION);
assert_eq!(stream.capability, ZAKURA_CAP_BLOCK_SYNC);
assert_eq!(stream.mode, StreamMode::Ordered);
assert_eq!(stream.frame_cap, MAX_BS_FRAME_BYTES);
}
#[tokio::test]
async fn service_registry_routes_block_sync_by_exact_capability_and_version() {
let service = Arc::new(BlockSyncService::new(ZakuraBlockSyncConfig::default()));
let registry =
ServiceRegistry::new(vec![service]).expect("block-sync service declares unique kind");
let peer = peer(1);
assert_eq!(
registry.capability_for_stream(ZAKURA_STREAM_BLOCK_SYNC, ZAKURA_BLOCK_SYNC_STREAM_VERSION),
Some(ZAKURA_CAP_BLOCK_SYNC)
);
assert!(registry
.capability_for_stream(
ZAKURA_STREAM_BLOCK_SYNC,
ZAKURA_BLOCK_SYNC_STREAM_VERSION + 1
)
.is_none());
assert_eq!(
registry
.ordered_streams_for_negotiated(ZAKURA_CAP_BLOCK_SYNC)
.iter()
.map(|stream| stream.kind)
.collect::<Vec<_>>(),
vec![ZAKURA_STREAM_BLOCK_SYNC]
);
assert!(registry.ordered_streams_for_negotiated(0).is_empty());
assert!(registry.wants_ordered_stream(
ZAKURA_STREAM_BLOCK_SYNC,
ZAKURA_CAP_BLOCK_SYNC,
&peer,
ServicePeerDirection::Inbound,
));
}
#[tokio::test(flavor = "current_thread", start_paused = true)]
async fn inert_reactor_parks_after_header_tip_watch_closes() {
let _service = BlockSyncService::new(ZakuraBlockSyncConfig::default());
let elapsed = tokio::time::timeout(Duration::from_secs(1), future::pending::<()>()).await;
assert!(
elapsed.is_err(),
"paused-time timeout only elapses if the inert reactor has no always-ready branch"
);
}
#[tokio::test]
async fn add_peer_emits_events_and_round_trips_status_over_framed_path() {
let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default());
let peer = peer(2);
let cancel_token = CancellationToken::new();
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
streams,
cancel_token,
));
let session = match next_event(&mut events).await {
BlockSyncEvent::PeerConnected(session) => session,
event => panic!("expected PeerConnected, got {event:?}"),
};
assert_eq!(session.peer_id(), &peer);
assert_eq!(service.peer_count(), 1);
let _outbound_rx = outbound_rx;
inbound_tx
.send(
BlockSyncMessage::Status(status())
.encode_frame()
.expect("status frame encodes"),
)
.await
.expect("inbound status queues");
service.remove_peer(&peer, 0);
assert_eq!(service.peer_count(), 0);
assert!(session.cancel_token().is_cancelled());
}
#[tokio::test]
async fn stale_block_sync_teardown_keeps_replacement_session() {
let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default());
let peer = peer(92);
let old_conn_id = 1;
let new_conn_id = 2;
let (old_inbound_tx, old_inbound_rx) = framed_channel(4);
let (old_outbound_tx, _old_outbound_rx) = framed_channel(4);
service.add_peer(Peer::new_with_conn_id_and_direction(
old_conn_id,
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (old_inbound_rx, old_outbound_tx))]),
CancellationToken::new(),
));
assert!(matches!(
next_event(&mut events).await,
BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer
));
let (new_inbound_tx, new_inbound_rx) = framed_channel(4);
let (new_outbound_tx, _new_outbound_rx) = framed_channel(4);
service.add_peer(Peer::new_with_conn_id_and_direction(
new_conn_id,
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (new_inbound_rx, new_outbound_tx))]),
CancellationToken::new(),
));
assert!(matches!(
next_event(&mut events).await,
BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer
));
assert_eq!(service.peer_count(), 1);
let (_stale_inbound_tx, stale_inbound_rx) = framed_channel(4);
let (stale_outbound_tx, _stale_outbound_rx) = framed_channel(4);
service.add_peer(Peer::new_with_conn_id_and_direction(
old_conn_id,
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
HashMap::from([(
ZAKURA_STREAM_BLOCK_SYNC,
(stale_inbound_rx, stale_outbound_tx),
)]),
CancellationToken::new(),
));
assert_eq!(
service.peer_count(),
1,
"stale add must not overwrite the replacement block-sync session",
);
service.remove_peer(&peer, old_conn_id);
assert_eq!(service.peer_count(), 1);
drop(old_inbound_tx);
tokio::time::sleep(Duration::from_millis(50)).await;
if let Ok(Some(BlockSyncEvent::PeerDisconnected(disconnected))) =
tokio::time::timeout(Duration::from_millis(50), events.recv()).await
{
panic!("stale teardown disconnected replacement session for {disconnected:?}");
}
assert_eq!(service.peer_count(), 1);
drop(new_inbound_tx);
}
#[tokio::test]
async fn lifecycle_events_bypass_full_bounded_wire_queue() {
let mut config = ZakuraBlockSyncConfig::default();
config.peer_limits.inbound_queue_depth = 1;
let (events, _event_rx) = mpsc::channel(config.peer_limits.inbound_queue_depth);
events
.try_send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(1),
hash: block::Hash([1; 32]),
})
.expect("test fills bounded wire queue");
let (lifecycle, mut lifecycle_rx) = mpsc::unbounded_channel();
let (_peers_tx, peers) = watch::channel(ServicePeerSnapshot::new(0, 0, config.peer_limits));
let (_status_tx, status) = watch::channel(config.initial_status());
let (_candidates_tx, candidates) = watch::channel(ZakuraBlockSyncCandidateState::default());
let handle = BlockSyncHandle {
events,
lifecycle,
peers,
status,
candidates,
routine_wiring: None,
};
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer = peer(91);
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
let _inbound_tx = inbound_tx;
assert!(matches!(
tokio::time::timeout(Duration::from_secs(1), lifecycle_rx.recv())
.await
.expect("lifecycle event arrives")
.expect("lifecycle channel stays open"),
BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer
));
service.remove_peer(&peer, 0);
assert!(matches!(
tokio::time::timeout(Duration::from_secs(1), lifecycle_rx.recv())
.await
.expect("lifecycle event arrives")
.expect("lifecycle channel stays open"),
BlockSyncEvent::PeerDisconnected(disconnected) if disconnected == peer
));
}
#[tokio::test]
async fn add_peer_decode_failure_reports_malformed_and_cancels_connection() {
let config = ZakuraBlockSyncConfig::default();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, _reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(3);
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
let connection_cancel = CancellationToken::new();
service.add_peer(Peer::new(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
streams,
connection_cancel.clone(),
));
inbound_tx
.send(Frame {
message_type: u16::from(MSG_BS_STATUS),
flags: 0,
payload: Vec::new(),
})
.await
.expect("malformed inbound frame queues");
let reported = loop {
match next_action(&mut actions).await {
BlockSyncAction::Misbehavior { peer: got, reason } => break (got, reason),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before malformed-message report: {action:?}"),
}
};
assert_eq!(
reported,
(peer.clone(), BlockSyncMisbehavior::MalformedMessage)
);
tokio::time::timeout(Duration::from_secs(1), connection_cancel.cancelled())
.await
.expect("malformed frame cancels the connection");
}
#[tokio::test]
async fn registry_add_peer_requires_negotiated_block_sync_capability() {
let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default());
let registry = ServiceRegistry::new(vec![Arc::new(service)])
.expect("block-sync service declares unique kind");
let peer = peer(4);
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
registry.add_peer(Peer::new(peer, None, 0, streams, CancellationToken::new()));
drop(inbound_tx);
assert!(
tokio::time::timeout(Duration::from_millis(100), events.recv())
.await
.is_err(),
"without cap 1<<3 the registry must not deliver kind-6 streams"
);
}
#[tokio::test]
async fn wants_peer_rejects_when_configured_slot_cap_is_reached() {
let config = ZakuraBlockSyncConfig {
peer_limits: ServicePeerLimits {
max_inbound_peers: 0,
max_outbound_peers: 2,
..ServicePeerLimits::default()
},
..ZakuraBlockSyncConfig::default()
};
let (service, mut events) = BlockSyncService::new_for_test(config);
let inbound_peer = peer(5);
assert!(!service.wants_peer(
&inbound_peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Inbound
));
assert!(service.wants_peer(
&inbound_peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound
));
let mut inbound_senders = Vec::new();
for byte in 6..=7 {
let peer_id = peer(byte);
let (inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
assert!(matches!(
next_event(&mut events).await,
BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer_id
));
inbound_senders.push(inbound_tx);
}
assert_eq!(service.peer_count(), 2);
assert!(!service.wants_peer(
&peer(8),
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound
));
let (_inbound_tx, inbound_rx) = framed_channel(4);
let (outbound_tx, _outbound_rx) = framed_channel(4);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer(8),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
assert_eq!(service.peer_count(), 2);
}
#[tokio::test]
async fn reactor_drives_tip_to_getblocks_to_submit_over_framed_path() {
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(40);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let block_size = u32::try_from(
block
.zcash_serialize_to_vec()
.expect("block serializes")
.len(),
)
.expect("test block size fits u32");
tip_tx
.send((block::Height(1), block_hash))
.expect("tip watch is live");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip,
..
} => {
assert_eq!(from, block::Height(1));
if best_header_tip == block::Height(1) {
break;
}
}
action => panic!("unexpected action before query: {action:?}"),
}
}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block_hash,
size: BlockSizeEstimate::Advertised(block_size),
}]))
.await
.expect("needed metadata queues");
loop {
let frame = tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv())
.await
.expect("outbound frame arrives")
.expect("outbound channel is live");
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1);
break;
}
}
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
block: submitted, ..
} => {
assert_eq!(submitted.hash(), block_hash);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_releases_request_budget_at_receipt_not_apply() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = u64::from(block1_size);
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(41);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: blocks[1].hash(),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
]))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 1);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
let (submit_owner, submit_source, submit_token) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => {
assert_eq!(block.hash(), blocks[0].hash());
break (owner, source, token);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
};
assert_eq!(handle.local_status().servable_high, block::Height(0));
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(2));
assert_eq!(count, 1);
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: submit_owner,
source: submit_source,
token: submit_token,
height: block::Height(1),
hash: blocks[0].hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("apply-finished event queues");
tokio::task::yield_now().await;
assert_eq!(
handle.local_status().servable_high,
block::Height(0),
"apply completion releases capacity but cannot publish a frontier before state commits it"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_does_not_requeue_held_height_reported_still_needed() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let block2_size = block_size(&blocks[1]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 4;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(73);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: blocks[1].hash(),
max_blocks_per_response: 1,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("body frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("re-reported needed metadata queues");
let saw_requeue_of_held_height = tokio::time::timeout(Duration::from_millis(300), async {
loop {
match outbound_rx.recv().await {
Some(frame) => {
if let BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
..
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
return true;
}
}
None => return false,
}
}
})
.await;
assert!(
saw_requeue_of_held_height.is_err(),
"a held (in_flight) height reported still needed must not be re-requested",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_buffers_body_larger_than_its_size_hint() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
41,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(1),
}]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("body queues");
let submitted = tokio::time::timeout(Duration::from_secs(1), async {
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => break block.hash(),
BlockSyncAction::Misbehavior {
reason: BlockSyncMisbehavior::SizeMismatch,
..
}
| BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
})
.await
.expect("underestimated body must still be buffered and submitted, not dropped");
assert_eq!(submitted, blocks[0].hash());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_downloads_run_ahead_of_stalled_commit() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_requests = 1;
config.max_blocks_per_response = 1;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status_message(
&service,
&mut actions,
41,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
let _submit_token = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
break token;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
};
tip_tx
.send((block::Height(4), block::Hash([4; 32])))
.expect("tip watch is live");
wait_for_query_needed_blocks(&mut actions, block::Height(1), block::Height(4)).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1),
"the reactor must download height 2 ahead of the stalled commit of height 1",
);
reactor_task.abort();
}
fn add_outbound_block_sync_peer(
service: &BlockSyncService,
byte: u8,
held: &mut Vec<(FramedSend, FramedRecv)>,
) -> ZakuraPeerId {
let peer_id = peer(byte);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
held.push((inbound_tx, outbound_rx));
peer_id
}
#[tokio::test]
async fn block_sync_add_peer_replaces_same_peer_even_at_full_cap() {
let mut config = immediate_body_download_config();
config.peer_limits.max_outbound_peers = 1;
config.peer_limits.max_inbound_peers = 0;
let (service, mut events) = BlockSyncService::new_for_test(config);
let mut held = Vec::new();
let peer_a = add_outbound_block_sync_peer(&service, 41, &mut held);
match next_event(&mut events).await {
BlockSyncEvent::PeerConnected(session) => assert_eq!(session.peer_id(), &peer_a),
event => panic!("expected PeerConnected for peer A, got {event:?}"),
}
assert_eq!(service.peer_count(), 1);
let _peer_b = add_outbound_block_sync_peer(&service, 42, &mut held);
let quiet = tokio::time::timeout(Duration::from_millis(100), events.recv()).await;
assert!(
quiet.is_err(),
"a new peer at a full per-direction cap must be rejected",
);
assert_eq!(service.peer_count(), 1);
let peer_a_again = add_outbound_block_sync_peer(&service, 41, &mut held);
assert_eq!(peer_a_again, peer_a);
match next_event(&mut events).await {
BlockSyncEvent::PeerConnected(session) => assert_eq!(session.peer_id(), &peer_a),
event => panic!("expected PeerConnected for replaced peer A, got {event:?}"),
}
assert_eq!(
service.peer_count(),
1,
"the replacement must not leave two sessions for the same peer",
);
}
#[tokio::test]
async fn reactor_keeps_applying_body_after_non_advancing_duplicate_result() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = u64::from(block1_size);
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
42,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
send_inbound(&inbound_tx, BlockSyncMessage::Block(blocks[0].clone())).await;
let (submit_owner, submit_source, submit_token) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => {
assert_eq!(block.hash(), blocks[0].hash());
break (owner, source, token);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: submit_owner,
source: submit_source,
token: submit_token,
height: block::Height(1),
hash: blocks[0].hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Duplicate),
})
.await
.expect("non-advancing duplicate completion queues");
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
]))
.await
.expect("needed metadata after duplicate queues");
let no_duplicate_request = tokio::time::timeout(Duration::from_millis(100), async {
while let Some(frame) = outbound_rx.recv().await {
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
if start_height == block::Height(1) {
panic!("non-advancing duplicate result re-requested {start_height:?}/{count}");
}
}
}
})
.await;
assert!(
no_duplicate_request.is_err(),
"reactor should keep waiting after a duplicate result that did not advance the frontier",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_keeps_active_response_when_needed_snapshot_omits_inflight_height() {
let blocks = fake_sequential_blocks(3);
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(42);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 16,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("first block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } if block.hash() == blocks[0].hash() => break,
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first submit: {action:?}"),
}
}
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("newer needed metadata queues");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("second block queues");
let mut submitted_second = false;
while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await
{
match action {
BlockSyncAction::SubmitBlock { block, .. } if block.hash() == blocks[1].hash() => {
submitted_second = true;
break;
}
BlockSyncAction::Misbehavior {
peer,
reason: BlockSyncMisbehavior::UnsolicitedBlock,
} if peer == peer_id => panic!("in-flight body was misclassified as unsolicited"),
BlockSyncAction::QueryNeededBlocks { .. } | BlockSyncAction::SubmitBlock { .. } => {}
action => panic!("unexpected action after second block: {action:?}"),
}
}
assert!(
submitted_second,
"second body from the original active response should remain correlated",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_unmatched_body_for_currently_needed_height() {
let blocks = mainnet_blocks_1_to_3();
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(142);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(2),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 16,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
let mut candidates = handle.subscribe_candidate_state();
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if candidates
.borrow_and_update()
.missing_block_bodies
.contains(&block::Height(1))
{
return;
}
candidates.changed().await.expect("candidate watch is live");
}
})
.await
.expect("producer extends the needed height into the WorkQueue");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("unmatched needed block queues");
let quiet = tokio::time::timeout(Duration::from_millis(200), async {
loop {
if let BlockSyncAction::Misbehavior { reason, .. } = next_action(&mut actions).await {
return reason;
}
}
})
.await;
assert!(
quiet.is_err(),
"unmatched body for a currently needed height should not be hard misbehavior",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_rejects_unmatched_body_for_ownerless_queued_height() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let block2_size = block_size(&blocks[1]);
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
143,
block::Height(2),
blocks[1].hash(),
1,
block1_size,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1),
"the byte-capped request must cover only height 1",
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("unmatched queued block queues");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("matched block queues");
let submitted = loop {
if let BlockSyncAction::SubmitBlock { block, .. } = next_action(&mut actions).await {
break block.hash();
}
};
assert_eq!(submitted, blocks[0].hash());
assert!(
tokio::time::timeout(Duration::from_millis(200), async {
loop {
if matches!(
next_action(&mut actions).await,
BlockSyncAction::SubmitBlock { .. }
) {
break;
}
}
})
.await
.is_err(),
"the ownerless height-2 body must not be submitted",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_queries_needed_blocks_above_submitted_floor() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let block2_size = block_size(&blocks[1]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 2;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(43);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 2,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(block1_size),
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2)
);
for block in blocks.iter().take(2) {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
}
let mut submitted = Vec::new();
let mut saw_refill_query = false;
while submitted.len() < 2 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => {
submitted.push((
block.coinbase_height().expect("test block has height"),
owner,
source,
token,
));
}
BlockSyncAction::QueryNeededBlocks {
from: block::Height(3),
best_header_tip,
..
} => {
assert_eq!(
best_header_tip,
block::Height(3),
"missing-body query must skip already claimed contiguous bodies",
);
saw_refill_query = true;
}
BlockSyncAction::QueryNeededBlocks { from, .. } => {
panic!("missing-body query should skip claimed bodies, got from {from:?}")
}
action => panic!("unexpected action before checkpoint submissions: {action:?}"),
}
}
assert_eq!(
submitted
.iter()
.map(|(height, ..)| *height)
.collect::<Vec<_>>(),
vec![block::Height(1), block::Height(2)]
);
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: submitted[0].1,
source: submitted[0].2,
token: submitted[0].3,
height: block::Height(1),
hash: blocks[0].hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("apply-finished event queues");
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: submitted[1].1,
source: submitted[1].2,
token: submitted[1].3,
height: block::Height(2),
hash: blocks[1].hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("apply-finished event queues");
if !saw_refill_query {
let action = tokio::time::timeout(Duration::from_secs(5), actions.recv())
.await
.expect(
"bounded needed-block query should arrive after apply-finished advances the floor",
)
.expect("block-sync action channel should stay open");
match action {
BlockSyncAction::QueryNeededBlocks {
from: block::Height(3),
best_header_tip,
..
} => {
assert_eq!(
best_header_tip,
block::Height(3),
"missing-body query must skip already submitted contiguous bodies",
);
}
BlockSyncAction::QueryNeededBlocks { from, .. } => {
panic!("missing-body query should skip submitted bodies, got from {from:?}")
}
action => panic!("unexpected action before needed-block query: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_retries_unavailable_body_without_scoring_its_supplier() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_bytes = block_size(&block);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(42);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block.hash(),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), block.hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_bytes),
}]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
let (submit_owner, submit_source, submit_token) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block: submitted,
} => {
assert_eq!(submitted.hash(), block.hash());
break (owner, source, token);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: submit_owner,
source: submit_source,
token: submit_token,
height: block::Height(1),
hash: block.hash(),
outcome: BlockApplyOutcome::retryable(zakura_header_chain::TransientBodyFailure {
hash: block.hash(),
evidence: zakura_header_chain::EvidenceId::from_digest([0xa5; 32]),
kind: zakura_header_chain::TransientBodyFailureKind::VerifierUnavailable,
availability: zakura_header_chain::BodyUnavailableSummary::default(),
}),
})
.await
.expect("apply-finished event queues");
let retry_meta = vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_bytes),
}];
let mut saw_retry_persistence = false;
assert!(
tokio::time::timeout(Duration::from_millis(700), async {
loop {
tokio::select! {
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks { .. } => {
panic!("the failed supplier must respect the first retry deadline")
}
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message during retry backoff: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(retry_meta.clone()))
.await
.expect("needed metadata queues during retry backoff");
}
BlockSyncAction::RecordBodyUnavailable {
expected_version,
failure,
} => {
assert_eq!(expected_version, zakura_header_chain::StateVersion::default());
assert_eq!(failure.hash, block.hash());
assert_eq!(failure.availability.attempts, 1);
saw_retry_persistence = true;
}
BlockSyncAction::Misbehavior { .. } => {
panic!("a local/transient unavailable result must not score its supplier")
}
action => panic!("unexpected action during retry backoff: {action:?}"),
}
}
}
}
})
.await
.is_err(),
"first retry backoff must remain active for at least 700ms"
);
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count,
} => {
assert_eq!(
count, 1,
"unavailable apply must release capacity and clear submitted coverage"
);
break;
}
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before retry request: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(retry_meta.clone()))
.await
.expect("needed metadata queues after unavailable result");
}
BlockSyncAction::RecordBodyUnavailable {
expected_version,
failure,
} => {
assert_eq!(expected_version, zakura_header_chain::StateVersion::default());
assert_eq!(failure.hash, block.hash());
assert_eq!(
failure.kind,
zakura_header_chain::TransientBodyFailureKind::VerifierUnavailable
);
assert_eq!(failure.availability.attempts, 1);
assert_eq!(failure.availability.suppliers, 1);
assert!(!failure.availability.alarmed);
saw_retry_persistence = true;
}
BlockSyncAction::Misbehavior { .. } => {
panic!("a local/transient unavailable result must not score its supplier")
}
action => panic!("unexpected action before retry request: {action:?}"),
}
}
}
}
assert!(
saw_retry_persistence,
"the completion-gated transient result must persist its exact retry summary"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_keeps_issuing_far_above_floor_with_no_near_tip_pause() {
let config = ZakuraBlockSyncConfig {
initial_block_probe_requests: DEFAULT_BS_MAX_REQUESTS_WITHOUT_BLOCK_PROGRESS,
..ZakuraBlockSyncConfig::default()
};
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
41,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: block::Hash([4; 32]),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tip_tx
.send((block::Height(1_000), block::Hash([9; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=4)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([height as u8; 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed metadata queues");
let mut heights = Vec::new();
for _ in 0..4 {
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(count, 1);
heights.push(start_height.0);
}
heights.sort_unstable();
assert_eq!(
heights,
vec![1, 2, 3, 4],
"a peer with free slots and budget must keep issuing regardless of floor distance"
);
reactor_task.abort();
}
#[tokio::test]
async fn routine_refills_after_budget_release_no_missed_wake() {
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES;
config.max_blocks_per_response = 1;
config.request_timeout = Duration::from_secs(300);
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status_message(
&service,
&mut actions,
70,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: blocks[1].hash(),
max_blocks_per_response: 1,
max_inflight_requests: 4,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("needed metadata queues");
let (first_start, _count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(first_start, block::Height(1));
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
let (owner, source, token) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => {
assert_eq!(block.coinbase_height(), Some(block::Height(1)));
break (owner, source, token);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before SubmitBlock: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner,
source,
token,
height: block::Height(1),
hash: blocks[0].hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("apply-finished event queues");
let (second_start, _count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(
second_start,
block::Height(2),
"freeing the request budget must wake the routine to issue the next request (no missed wake)"
);
reactor_task.abort();
}
#[tokio::test]
async fn routine_disconnect_returns_outstanding_and_releases_budget() {
let mut config = immediate_body_download_config();
config.request_timeout = Duration::from_secs(300);
config.peer_limits.max_outbound_peers = 4;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_a, _a_in, mut a_out) = connect_peer_with_status(
&service,
&mut actions,
0x71,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
let (start, _count) = wait_for_outbound_getblocks(&mut a_out).await;
assert_eq!(start, block::Height(1));
service.remove_peer(&peer_a, 0);
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if service.peer_count() == 0 {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("disconnect releases the block-sync peer slot");
let (_peer_b, _b_in, mut b_out) = connect_peer_with_status(
&service,
&mut actions,
0x72,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let offered = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let (start, _count) = wait_for_outbound_getblocks(&mut b_out).await;
if start == block::Height(1) {
return true;
}
}
})
.await
.expect("a fresh peer must be offered the re-queued height after a disconnect mid-fetch");
assert!(offered);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_reserves_size_hint_per_block_not_worst_case() {
let budget_blocks = 3u32;
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: u64::from(budget_blocks) * BS_PER_BLOCK_WORST_CASE_BYTES,
max_blocks_per_response: 16,
..ZakuraBlockSyncConfig::default()
};
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
51,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(16),
tip_hash: block::Hash([16; 32]),
max_blocks_per_response: 16,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=16)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([height as u8; 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (_start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(
count, 16,
"the request must cover the full 16-block range: the 1 KiB size hints (not a \
worst-case-per-block reservation) drive the budget, so all 16 fit under a \
{budget_blocks}-worst-case-block budget",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_packs_small_estimates_under_peer_response_byte_cap() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: 4 * BS_PER_BLOCK_WORST_CASE_BYTES,
max_blocks_per_response: 4,
..ZakuraBlockSyncConfig::default()
};
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let one_worst_case_response =
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES).expect("worst-case block size fits u32");
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
52,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: block::Hash([4; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: one_worst_case_response,
},
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=4)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([height as u8; 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(
count, 4,
"small advertised estimates should pack more than the one worst-case block \
allowed by the peer response byte cap",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_tiny_estimates_pack_into_one_worst_case_budget_block() {
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES,
max_blocks_per_response: 4,
..ZakuraBlockSyncConfig::default()
};
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound, mut outbound) = connect_peer_with_status_message(
&service,
&mut actions,
53,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: block::Hash([4; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
(1..=4)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("test height fits u8"); 32]),
size: BlockSizeEstimate::Advertised(1),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(
count, 4,
"all 4 tiny-hint heights pack into one worst-case-block budget, because each \
reserves its 1 KiB-floored size hint rather than a worst-case share",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_zero_pause_threshold_preserves_lag_one_downloads() {
let config = immediate_body_download_config();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(1),
hash: block::Hash([1; 32]),
})
.await
.expect("header-tip event queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
best_header_tip: block::Height(1),
..
}
) {}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_keeps_block_sync_peer_after_catch_up_and_reuses_later() {
let mut config = fill_loop_mechanics_config();
config.peer_limits.max_outbound_peers = 1;
let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(4), block::Hash([4; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
71,
block::Height(6),
block::Hash([6; 32]),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(3),
hash: block::Hash([3; 32]),
})
.await
.expect("header-tip event queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: block::Hash([3; 32]),
size: BlockSizeEstimate::Unknown,
},
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
for height in [block::Height(1), block::Height(2)] {
let hash_byte = u8::try_from(height.0).expect("test height fits in u8");
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: height,
verified_block_hash: block::Hash([hash_byte; 32]),
}))
.await
.expect("frontier event queues");
tokio::time::sleep(Duration::from_millis(50)).await;
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
assert_eq!(service.peer_count(), 1);
}
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 3,
}
.encode_frame()
.expect("BlocksDone encodes"),
)
.await
.expect("BlocksDone queues");
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: block::Hash([3; 32]),
}))
.await
.expect("caught-up frontier event queues");
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
handle.peer_snapshot().outbound_peers,
1,
"caught-up nodes must keep block-sync peers so they can serve fresh nodes",
);
assert_eq!(service.peer_count(), 1);
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(6),
hash: block::Hash([6; 32]),
})
.await
.expect("later header-tip event queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(4),
best_header_tip: block::Height(6),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(5),
hash: block::Hash([5; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(6),
hash: block::Hash([6; 32]),
size: BlockSizeEstimate::Unknown,
},
]))
.await
.expect("new needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(4), 3),
"the retained block-sync peer should be reused after later header growth",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_multi_block_range_and_submits_parent_first() {
let config = ZakuraBlockSyncConfig {
max_blocks_per_response: 4,
..ZakuraBlockSyncConfig::default()
};
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(43);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
blocks
.iter()
.map(|block| BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 3);
for index in [1usize, 2, 0] {
inbound_tx
.send(
BlockSyncMessage::Block(blocks[index].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
}
let mut submitted = Vec::new();
while submitted.len() < 3 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::Misbehavior { reason, .. } => {
panic!("honest multi-block response was misclassified: {reason:?}")
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before all submits: {action:?}"),
}
}
assert_eq!(
submitted,
vec![block::Height(1), block::Height(2), block::Height(3)]
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_backpressures_inbound_body_flood_without_dropping_bodies() {
const FLOOD: u32 = 64;
let blocks = fake_sequential_blocks(FLOOD);
let tip = blocks.last().expect("flood is non-empty").clone();
let tip_height = tip.coinbase_height().expect("tip has height");
let mut config = immediate_body_download_config();
config.peer_limits.inbound_queue_depth = 1;
config.max_inflight_block_bytes = u64::MAX;
config.request_timeout = Duration::from_secs(300);
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(64);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: tip_height,
tip_hash: tip.hash(),
max_blocks_per_response: FLOOD,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((tip_height, tip.hash()))
.expect("tip watch is live");
let feed_blocks = blocks.clone();
let (feed_tx, mut feed_rx) = mpsc::unbounded_channel::<u32>();
let feeder = tokio::spawn(async move {
while let Some(height) = feed_rx.recv().await {
let frame = BlockSyncMessage::Block(feed_blocks[(height - 1) as usize].clone())
.encode_frame()
.expect("block frame encodes");
if inbound_tx.send(frame).await.is_err() {
break;
}
}
});
let metas: Vec<_> = blocks
.iter()
.map(|block| BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
})
.collect();
let drive = async {
let mut submitted = std::collections::HashSet::new();
while submitted.len() < FLOOD as usize {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let Some(frame) = frame else { break };
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
for height in start_height.0..start_height.0 + count {
feed_tx.send(height).expect("feeder task stays open");
}
}
}
action = actions.recv() => {
let Some(action) = action else { break };
match action {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(metas.clone()))
.await
.expect("needed metadata queues");
}
BlockSyncAction::SubmitBlock { block, .. } => {
submitted.insert(
block.coinbase_height().expect("submitted block has height"),
);
}
_ => {}
}
}
}
}
submitted
};
let submitted = tokio::time::timeout(Duration::from_secs(20), drive)
.await
.expect(
"flooded bodies must all reach the apply stage; a dropped inbound body wedges block sync",
);
let expected: std::collections::HashSet<_> = (1..=FLOOD).map(block::Height).collect();
assert_eq!(
submitted, expected,
"every solicited body in the flood must be submitted exactly once, with no drops"
);
feeder.abort();
reactor_task.abort();
}
#[tokio::test]
async fn reactor_restarted_at_genesis_queries_and_schedules_without_tip_change() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip,
..
} => {
assert_eq!(from, block::Height(1));
assert_eq!(best_header_tip, block::Height(3));
}
action => panic!("restart from genesis must query missing bodies, got {action:?}"),
}
let (_peer, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
67,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3),
"restart from genesis must schedule scratch body sync from height 1"
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
assert_eq!(block.coinbase_height(), Some(block::Height(1)));
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first scratch submit: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_blocks_done_after_completed_range() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
68,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 1,
}
.encode_frame()
.expect("BlocksDone encodes"),
)
.await
.expect("BlocksDone queues");
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(200), actions.recv()).await
{
if let BlockSyncAction::Misbehavior {
peer: action_peer,
reason,
} = action
{
assert_ne!(
(action_peer, reason),
(peer.clone(), BlockSyncMisbehavior::UnsolicitedDone),
"a valid terminator after a completed block response must not be scored"
);
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_retries_missing_heights_after_partial_blocks_done() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
69,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[0].hash());
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first submit: {action:?}"),
}
}
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 1,
}
.encode_frame()
.expect("BlocksDone encodes"),
)
.await
.expect("BlocksDone queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2),
"partial responses must retry the contiguous missing suffix"
);
reactor_task.abort();
}
#[tokio::test]
async fn checkpoint_hole_disconnect_retries_first_missing_height_with_fresh_peer() {
const FIRST_NEEDED: u32 = 801;
const PREFIX_END: u32 = 864;
const HOLE_START: u32 = 865;
const HOLE_END: u32 = 872;
const LAST_METADATA: u32 = 933;
const BEST_HEADER_TIP: u32 = 10_400;
let blocks = fake_blocks_in_range(FIRST_NEEDED, LAST_METADATA);
let block_at = |height: u32| -> Arc<block::Block> {
let index =
usize::try_from(height - FIRST_NEEDED).expect("test height is inside block vector");
blocks[index].clone()
};
let metas: Vec<_> = blocks.iter().map(block_meta).collect();
let prefix: std::collections::HashSet<_> =
(FIRST_NEEDED..=PREFIX_END).map(block::Height).collect();
let sparse_above_hole: std::collections::HashSet<_> = [873, 888, 905, 920]
.into_iter()
.map(block::Height)
.collect();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = u64::MAX;
config.request_timeout = Duration::from_secs(300);
config.peer_limits.max_outbound_peers = 1;
config.peer_limits.inbound_queue_depth = 128;
config.peer_limits.outbound_queue_depth = 128;
let (_tip_tx, tip_rx) = watch::channel((block::Height(BEST_HEADER_TIP), block::Hash([10; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(800),
verified_block_tip: block::Height(800),
verified_block_hash: block::Hash([8; 32]),
},
(block::Height(BEST_HEADER_TIP), block::Hash([10; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (old_peer, old_inbound, mut old_outbound) = connect_peer_with_status_message(
&service,
&mut actions,
70,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(LAST_METADATA),
tip_hash: block_at(LAST_METADATA).hash(),
max_blocks_per_response: MAX_BS_BLOCKS_PER_REQUEST,
max_inflight_requests: 8,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(metas.clone()))
.await
.expect("checkpoint metadata queues after peer connection");
let (feed_tx, mut feed_rx) = mpsc::unbounded_channel::<u32>();
let blocks_for_feeder = blocks.clone();
let feeder = tokio::spawn(async move {
while let Some(height) = feed_rx.recv().await {
let index = usize::try_from(height - FIRST_NEEDED)
.expect("fed test height is inside block vector");
let frame = BlockSyncMessage::Block(blocks_for_feeder[index].clone())
.encode_frame()
.expect("block frame encodes");
if old_inbound.send(frame).await.is_err() {
break;
}
}
});
let mut requests: Vec<(block::Height, u32)> = Vec::new();
let mut submitted = std::collections::HashSet::new();
let primed = tokio::time::timeout(Duration::from_secs(40), async {
while !prefix.is_subset(&submitted)
|| !requests.iter().any(|(start, count)| {
*start <= block::Height(HOLE_START) && start.0.saturating_add(*count) > HOLE_END
})
{
tokio::select! {
biased;
frame = old_outbound.recv() => {
let frame = frame.expect("old peer outbound is live");
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
requests.push((start_height, count));
let end_height = start_height
.0
.checked_add(count)
.expect("test request height range fits u32");
for height in start_height.0..end_height {
let height = block::Height(height);
if height.0 <= PREFIX_END || sparse_above_hole.contains(&height) {
feed_tx.send(height.0).expect("feeder task stays open");
}
}
}
}
action = actions.recv() => {
match action.expect("block-sync action channel should stay open") {
BlockSyncAction::QueryNeededBlocks {
from,
best_header_tip,
..
} => {
assert!(from >= block::Height(801));
assert_eq!(best_header_tip, block::Height(BEST_HEADER_TIP));
handle
.send(BlockSyncEvent::NeededBlocks(metas.clone()))
.await
.expect("checkpoint metadata queues");
}
BlockSyncAction::SubmitBlock { block, .. } => {
let height =
block.coinbase_height().expect("submitted block has height");
assert!(
submitted.insert(height),
"height {height:?} was submitted more than once before the hole retry"
);
}
action => {
panic!("unexpected action while priming checkpoint hole: {action:?}")
}
}
}
}
}
})
.await;
assert!(
primed.is_ok(),
"checkpoint-hole priming timed out: requests={requests:?}, submitted={} of {}",
submitted.len(),
prefix.len()
);
assert!(
requests
.iter()
.any(|(start, count)| *start <= block::Height(HOLE_START)
&& start.0.saturating_add(*count) > HOLE_END),
"the initial in-flight requests must cover the missing checkpoint hole"
);
service.remove_peer(&old_peer, 0);
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.peer_snapshot().outbound_peers == 0 && service.peer_count() == 0 {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("disconnect releases the old block-sync peer slot");
let (_new_peer, _new_inbound, mut new_outbound) = connect_peer_with_status_message(
&service,
&mut actions,
71,
BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(LAST_METADATA),
tip_hash: block_at(LAST_METADATA).hash(),
max_blocks_per_response: MAX_BS_BLOCKS_PER_REQUEST,
max_inflight_requests: 8,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
},
)
.await;
tokio::time::timeout(Duration::from_secs(5), async {
loop {
tokio::select! {
biased;
frame = new_outbound.recv() => {
let frame = frame.expect("new peer outbound is live");
if let BlockSyncMessage::GetBlocks {
start_height,
count,
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
assert_eq!(
start_height,
block::Height(HOLE_START),
"after a partial response disconnect, the first fresh request must fill \
the checkpoint hole instead of jumping above it or reusing a stale \
assignment"
);
assert!(count >= 1);
break;
}
}
action = actions.recv() => {
match action.expect("block-sync action channel should stay open") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(metas.clone()))
.await
.expect("post-disconnect metadata queues");
}
BlockSyncAction::SubmitBlock { block, .. } => {
let height =
block.coinbase_height().expect("submitted block has height");
assert!(
submitted.insert(height),
"height {height:?} was resubmitted before the checkpoint hole was retried"
);
}
action => {
panic!(
"unexpected action while waiting for checkpoint-hole retry: {action:?}"
)
}
}
}
}
}
})
.await
.expect("fresh peer should request the checkpoint hole after disconnect");
feeder.abort();
reactor_task.abort();
}
#[tokio::test]
async fn reactor_reset_mid_download_drops_stale_anchors_and_releases_budget() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
47,
block::Height(3),
blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(10_000),
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: blocks[2].hash(),
size: BlockSizeEstimate::Advertised(10_000),
},
]))
.await
.expect("old-fork needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("out-of-order old-fork block queues");
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}))
.await
.expect("reset event queues");
let new_fork_meta = vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([92; 32]),
size: BlockSizeEstimate::Advertised(20_000),
}];
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(2),
count,
} => {
assert_eq!(
count, 1,
"a full-budget new fork request can only be scheduled if stale bytes were released"
);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before new fork request: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(new_fork_meta.clone()))
.await
.expect("new-fork needed metadata queues");
}
action => panic!("unexpected action before new fork request: {action:?}"),
}
}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_forward_reset_preserves_submitted_successor_body() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
49,
block::Height(3),
blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("contiguous body queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[1].hash());
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first submit: {action:?}"),
}
}
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("successor body queues");
let (successor_owner, successor_source, successor_token) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => {
assert_eq!(block.hash(), blocks[2].hash());
break (owner, source, token);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before successor submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}))
.await
.expect("forward reset event queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"forward reset must not re-query or re-submit the preserved successor body",
);
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: successor_owner,
source: successor_source,
token: successor_token,
height: block::Height(3),
hash: blocks[2].hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("successor apply result queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"at-tip successor apply must not trigger a redundant needed-block query",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_forward_reset_preserves_future_outstanding_body() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
72,
block::Height(3),
blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}))
.await
.expect("forward reset event queues");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("successor body queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(block.hash(), blocks[2].hash());
break;
}
BlockSyncAction::Misbehavior { peer, reason } => {
assert_ne!(
(peer, reason),
(peer_id.clone(), BlockSyncMisbehavior::UnsolicitedBlock),
"forward reset must not drop a still-active successor request"
);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
BlockSyncAction::QueryBlocksByHeightRange { .. } => {}
BlockSyncAction::RecordBodyUnavailable { .. }
| BlockSyncAction::RecordBodyInvalid { .. }
| BlockSyncAction::RestartBodyAvailability { .. }
| BlockSyncAction::RetryBodyAvailability { .. } => {}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_forward_reset_preserves_buffered_successor_body() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
73,
block::Height(3),
blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("successor body queues");
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(2),
returned: 2,
}
.encode_frame()
.expect("BlocksDone encodes"),
)
.await
.expect("BlocksDone queues");
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}))
.await
.expect("forward reset event queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => {
assert_eq!(
block.hash(),
blocks[2].hash(),
"buffered successor must submit without a duplicate download"
);
break;
}
BlockSyncAction::Misbehavior { peer, reason } => {
assert_ne!(
(peer, reason),
(peer_id.clone(), BlockSyncMisbehavior::UnsolicitedBlock),
"forward reset must not drop a buffered successor body"
);
}
BlockSyncAction::QueryNeededBlocks { .. }
| BlockSyncAction::QueryBlocksByHeightRange { .. }
| BlockSyncAction::RecordBodyUnavailable { .. }
| BlockSyncAction::RecordBodyInvalid { .. }
| BlockSyncAction::RestartBodyAvailability { .. }
| BlockSyncAction::RetryBodyAvailability { .. } => {}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_destructive_forward_reset_does_not_rerequest_same_hash_in_flight_apply() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
68,
block::Height(2),
blocks[1].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2)
);
for block in blocks.iter().take(2) {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
}
let mut submitted = Vec::new();
while submitted.len() < 2 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { token, block, .. } => submitted.push((
block.coinbase_height().expect("test block has height"),
token,
)),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submitted bodies: {action:?}"),
}
}
assert_eq!(
submitted
.iter()
.map(|(height, _)| *height)
.collect::<Vec<_>>(),
vec![block::Height(1), block::Height(2)]
);
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: block::Hash([99; 32]),
}))
.await
.expect("destructive forward reset queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(2),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("same-hash needed metadata queues");
if let Ok((re_start, re_count)) = tokio::time::timeout(
Duration::from_millis(200),
wait_for_outbound_getblocks(&mut outbound_rx),
)
.await
{
assert_eq!(
(re_start, re_count),
(block::Height(2), 1),
"any same-hash re-request must target exactly the released height"
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("same-hash body queues");
let no_resubmit = tokio::time::timeout(Duration::from_millis(200), async {
loop {
match actions.recv().await {
Some(BlockSyncAction::SubmitBlock { block, .. })
if block.hash() == blocks[1].hash() =>
{
panic!("same-hash body with a pending apply must not be re-submitted")
}
Some(_) => {}
None => break,
}
}
})
.await;
assert!(
no_resubmit.is_err(),
"a redundant same-hash body must be dropped, not re-submitted",
);
}
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: block::Hash([99; 32]),
}))
.await
.expect("fork reanchor reset queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([42; 32]),
size: BlockSizeEstimate::Advertised(block_size(&blocks[1])),
}]))
.await
.expect("new-fork needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1),
"different hash at the same height must still schedule after a fork reset"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_stale_apply_completion_after_resubmit() {
let config = immediate_body_download_config();
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(61);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block_hash,
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(1), block_hash))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block_hash,
size: BlockSizeEstimate::Advertised(block_size(&block)),
}]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("first body frame queues");
let (stale_owner, stale_source, stale_token) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => {
assert_eq!(block.hash(), block_hash);
break (owner, source, token);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before first submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
}))
.await
.expect("reset event queues");
let reset_meta = vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block_hash,
size: BlockSizeEstimate::Advertised(block_size(&block)),
}];
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count,
} => {
assert_eq!(count, 1);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before reset re-fetch: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(reset_meta.clone()))
.await
.expect("needed metadata queues after reset");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("unexpected action before reset re-fetch: {action:?}"),
}
}
}
}
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("second body frame queues");
let (current_owner, current_source, current_token) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => {
assert_eq!(block.hash(), block_hash);
break (owner, source, token);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before second submit: {action:?}"),
}
};
assert_ne!(stale_token, current_token);
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: stale_owner,
source: stale_source,
token: stale_token,
height: block::Height(1),
hash: block_hash,
outcome: test_block_apply_outcome(BlockApplyResult::Duplicate),
})
.await
.expect("stale apply-finished event queues");
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(100), actions.recv()).await
{
if let BlockSyncAction::SubmitBlock { .. } = action {
panic!(
"stale apply completion released/re-submitted the current submission: {action:?}"
);
}
}
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: stale_owner,
source: current_source,
token: current_token,
height: block::Height(1),
hash: block_hash,
outcome: test_block_apply_outcome(BlockApplyResult::Duplicate),
})
.await
.expect("owner-mismatched apply-finished event queues");
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: current_owner,
source: zakura_header_chain::SourceId::from_digest([0xfe; 32]),
token: current_token,
height: block::Height(1),
hash: block_hash,
outcome: test_block_apply_outcome(BlockApplyResult::Duplicate),
})
.await
.expect("source-mismatched apply-finished event queues");
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(100), actions.recv()).await
{
if let BlockSyncAction::SubmitBlock { .. } = action {
panic!("owner/source mismatch released the current submission: {action:?}");
}
}
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: current_owner,
source: current_source,
token: current_token,
height: block::Height(1),
hash: block_hash,
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("current apply-finished event queues");
assert!(
tokio::time::timeout(Duration::from_millis(100), actions.recv())
.await
.is_err(),
"at-tip apply completion must not trigger a redundant needed-block query",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_fast_forward_reset_clears_buffered_bodies_and_releases_budget() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
50,
block::Height(4),
block::Hash([4; 32]),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 2).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(10_000),
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: blocks[2].hash(),
size: BlockSizeEstimate::Advertised(10_000),
},
]))
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("out-of-order body queues");
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: blocks[2].hash(),
}))
.await
.expect("fast-forward reset event queues");
tip_tx
.send((block::Height(4), block::Hash([4; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(4),
best_header_tip: block::Height(4),
..
}
) {}
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
handle.peer_snapshot().outbound_peers,
1,
"caught-up reset keeps the previous block-sync peer available for serving",
);
assert_eq!(service.peer_count(), 1);
let post_reset_meta = vec![BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Advertised(20_000),
}];
handle
.send(BlockSyncEvent::NeededBlocks(post_reset_meta.clone()))
.await
.expect("post-reset needed metadata queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(4),
count,
} => {
assert_eq!(
count, 1,
"a full-budget request after fast-forward Reset requires releasing buffered bytes"
);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before fast-forward re-fetch: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(post_reset_meta.clone()))
.await
.expect("post-reset needed metadata queues");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("unexpected action before fast-forward re-fetch: {action:?}"),
}
}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_fuzzes_arrival_order_across_fork_parent_first() {
#[derive(Copy, Clone)]
enum ForkBody {
Old(usize),
New(usize),
}
let cases = vec![
(
"stale-high-before-reset",
vec![3],
vec![],
vec![ForkBody::Old(2), ForkBody::New(3), ForkBody::New(2)],
),
(
"old-prefix-before-reset",
vec![2, 1],
vec![],
vec![ForkBody::Old(3), ForkBody::New(2), ForkBody::New(3)],
),
(
"stale-before-new-needed",
vec![2],
vec![3],
vec![ForkBody::New(2), ForkBody::Old(2), ForkBody::New(3)],
),
(
"new-out-of-order-with-stale-tail",
vec![],
vec![2],
vec![ForkBody::New(3), ForkBody::Old(3), ForkBody::New(2)],
),
];
for (case, old_before_reset, old_before_new_needed, after_new_needed) in cases {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 3,
..fill_loop_mechanics_config()
};
config.peer_limits.outbound_queue_depth = 16;
let old_blocks = mainnet_blocks_1_to_3();
let new_blocks = vec![
forked_block(&old_blocks[0], 101),
forked_block(&old_blocks[1], 102),
forked_block(&old_blocks[2], 103),
];
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
51,
block::Height(4),
old_blocks[2].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES * 3).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(3), old_blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
best_header_tip: block::Height(3),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
old_blocks
.iter()
.map(|block| BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
})
.collect(),
))
.await
.expect("old-fork needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3),
"{case}: old fork request schedules"
);
let mut submitted_tip = block::Height(0);
for height in old_before_reset {
inbound_tx
.send(
BlockSyncMessage::Block(old_blocks[height - 1].clone())
.encode_frame()
.expect("old-fork block encodes"),
)
.await
.expect("old-fork block queues");
drain_parent_first_actions(&mut actions, &mut submitted_tip, None).await;
}
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: new_blocks[0].hash(),
}))
.await
.expect("reset event queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
submitted_tip = block::Height(1);
tip_tx
.send((block::Height(3), new_blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(3),
..
}
) {}
for height in old_before_new_needed {
inbound_tx
.send(
BlockSyncMessage::Block(old_blocks[height - 1].clone())
.encode_frame()
.expect("stale old-fork block encodes"),
)
.await
.expect("stale old-fork block queues");
drain_parent_first_actions(&mut actions, &mut submitted_tip, Some(&new_blocks)).await;
}
let new_fork_meta = vec![
BlockSyncBlockMeta {
height: block::Height(2),
hash: new_blocks[1].hash(),
size: BlockSizeEstimate::Advertised(block_size(&new_blocks[1])),
},
BlockSyncBlockMeta {
height: block::Height(3),
hash: new_blocks[2].hash(),
size: BlockSizeEstimate::Advertised(block_size(&new_blocks[2])),
},
];
handle
.send(BlockSyncEvent::NeededBlocks(new_fork_meta.clone()))
.await
.expect("new-fork needed metadata queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(2),
count,
} => {
assert_eq!(
count, 2,
"{case}: new fork request schedules after reset"
);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("{case}: unexpected outbound message before new fork re-fetch: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(new_fork_meta.clone()))
.await
.expect("new-fork needed metadata queues");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("{case}: unexpected action before new fork re-fetch: {action:?}"),
}
}
}
}
for body in after_new_needed {
let block = match body {
ForkBody::Old(height) => old_blocks[height - 1].clone(),
ForkBody::New(height) => new_blocks[height - 1].clone(),
};
inbound_tx
.send(
BlockSyncMessage::Block(block)
.encode_frame()
.expect("fork body encodes"),
)
.await
.expect("fork body queues");
drain_parent_first_actions(&mut actions, &mut submitted_tip, Some(&new_blocks)).await;
}
assert_eq!(
submitted_tip,
block::Height(3),
"{case}: new fork bodies submit parent-first through height 3"
);
handle
.send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: new_blocks[2].hash(),
}))
.await
.expect("post-submit grow event queues");
tip_tx
.send((block::Height(4), block::Hash([4; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(4),
best_header_tip: block::Height(4),
..
}
) {}
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
handle.peer_snapshot().outbound_peers,
1,
"{case}: caught-up fork handling keeps the old block-sync peer available for serving",
);
assert_eq!(service.peer_count(), 1);
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Advertised(60_000),
}]))
.await
.expect("post-fuzz needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(4), 1),
"{case}: byte budget returns to baseline after reset and submissions"
);
reactor_task.abort();
}
}
#[tokio::test]
async fn reactor_competing_fork_download_switches_to_current_header_hashes() {
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
immediate_body_download_config(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(
immediate_body_download_config(),
handle.clone(),
);
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
48,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(3), blocks[2].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("old fork metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}))
.await
.expect("reset event queues");
let new_fork_meta = vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([222; 32]),
size: BlockSizeEstimate::Advertised(block_size(&blocks[1])),
}];
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks {
start_height: block::Height(2),
count,
} => {
assert_eq!(count, 1);
break;
}
BlockSyncMessage::GetBlocks { .. } | BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message before new fork re-fetch: {msg:?}"),
}
}
action = actions.recv() => {
match action.expect("action channel is live") {
BlockSyncAction::QueryNeededBlocks { .. } => {
handle
.send(BlockSyncEvent::NeededBlocks(new_fork_meta.clone()))
.await
.expect("new fork metadata queues");
}
BlockSyncAction::Misbehavior { .. } => {}
action => panic!("unexpected action before new fork re-fetch: {action:?}"),
}
}
}
}
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("old-fork body encodes"),
)
.await
.expect("old-fork body queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::Misbehavior { peer, reason } => {
assert_eq!(peer, peer_id);
assert!(matches!(
reason,
BlockSyncMisbehavior::BodyPayloadMismatch(
zakura_header_chain::BodyPayloadMismatch {
requested,
delivered,
kind: zakura_header_chain::BodyCommitmentKind::HeaderHash,
source,
..
}
) if requested == block::Hash([222; 32])
&& delivered == blocks[1].hash()
&& source == zakura_header_chain::SourceId::from_digest([48; 32])
));
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before stale body rejection: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_legacy_commit_dedups_inflight_request_and_reuses_budget() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES,
..fill_loop_mechanics_config()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
49,
block::Height(2),
blocks[1].hash(),
1,
u32::try_from(BS_PER_BLOCK_WORST_CASE_BYTES).unwrap_or(u32::MAX),
)
.await;
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(10_000),
}]))
.await
.expect("first needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
handle
.send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}))
.await
.expect("legacy commit grow event queues");
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
best_header_tip: block::Height(2),
..
}
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: blocks[1].hash(),
size: BlockSizeEstimate::Advertised(10_000),
}]))
.await
.expect("second needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1),
"legacy commit must release the duplicate in-flight reservation"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_treats_duplicate_buffered_blocks_as_benign() {
let config = immediate_body_download_config();
let blocks = [
mainnet_block(&BLOCK_MAINNET_1_BYTES),
mainnet_block(&BLOCK_MAINNET_2_BYTES),
];
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(44);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: blocks[1].hash(),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
tip_tx
.send((block::Height(2), blocks[1].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(
blocks
.iter()
.map(|block| BlockSyncBlockMeta {
height: block.coinbase_height().expect("test block has height"),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(block_size(block)),
})
.collect(),
))
.await
.expect("needed metadata queues");
let (start_height, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 2);
for block in [&blocks[1], &blocks[1], &blocks[0]] {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
}
let mut submitted = Vec::new();
while submitted.len() < 2 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::Misbehavior { reason, .. } => {
panic!("duplicate buffered body was misclassified: {reason:?}")
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before submit: {action:?}"),
}
}
assert_eq!(submitted, vec![block::Height(1), block::Height(2)]);
let quiet = tokio::time::timeout(Duration::from_millis(100), async {
while let Some(action) = actions.recv().await {
if let BlockSyncAction::Misbehavior { reason, .. } = action {
panic!("duplicate buffered body was misclassified after submit: {reason:?}");
}
}
})
.await;
assert!(quiet.is_err());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_rapid_status_growth_without_spam_score() {
let config = ZakuraBlockSyncConfig::default();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
drop(tip_tx);
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer_id = peer(46);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
for servable_high in [block::Height(1), block::Height(2)] {
let hash_byte = u8::try_from(servable_high.0).expect("test height fits in u8");
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high,
tip_hash: block::Hash([hash_byte; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
}
let quiet = tokio::time::timeout(Duration::from_millis(100), async {
while let Some(action) = actions.recv().await {
if let BlockSyncAction::Misbehavior { reason, .. } = action {
panic!("rapid status growth was misclassified: {reason:?}");
}
}
})
.await;
assert!(quiet.is_err());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_redundant_status_burst_without_spam_score() {
let config = ZakuraBlockSyncConfig::default();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
drop(tip_tx);
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer_id = peer(47);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
let status = BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes");
for _ in 0..3 {
inbound_tx
.send(status.clone())
.await
.expect("redundant status queues");
}
let quiet = tokio::time::timeout(Duration::from_millis(100), async {
while let Some(action) = actions.recv().await {
if let BlockSyncAction::Misbehavior { reason, .. } = action {
panic!("redundant status burst was misclassified: {reason:?}");
}
}
})
.await;
assert!(quiet.is_err());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_rejects_block_hash_mismatch_without_hard_drop_for_size_mismatch() {
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
immediate_body_download_config(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(
immediate_body_download_config(),
handle.clone(),
);
let peer = peer(41);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status queues");
tip_tx
.send((block::Height(1), block::Hash([9; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([9; 32]),
size: BlockSizeEstimate::Advertised(1),
}]))
.await
.expect("needed metadata queues");
while !matches!(
BlockSyncMessage::decode_frame(
tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv())
.await
.expect("outbound frame arrives")
.expect("outbound channel is live")
)
.expect("frame decodes"),
BlockSyncMessage::GetBlocks { .. }
) {}
inbound_tx
.send(
BlockSyncMessage::Block(mainnet_block(&BLOCK_MAINNET_1_BYTES))
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::Misbehavior { reason, .. } => {
assert!(matches!(
reason,
BlockSyncMisbehavior::BodyPayloadMismatch(
zakura_header_chain::BodyPayloadMismatch {
requested,
delivered,
kind: zakura_header_chain::BodyCommitmentKind::HeaderHash,
source,
..
}
) if requested == block::Hash([9; 32])
&& delivered == mainnet_block(&BLOCK_MAINNET_1_BYTES).hash()
&& source == zakura_header_chain::SourceId::from_digest([41; 32])
));
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before invalid-block report: {action:?}"),
}
}
reactor_task.abort();
}
#[tokio::test]
async fn scheduled_get_blocks_is_sent_once_via_session() {
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(57);
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([9; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status queues");
tip_tx
.send((block::Height(1), block::Hash([9; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([9; 32]),
size: BlockSizeEstimate::Advertised(1),
}]))
.await
.expect("needed metadata queues");
let mut get_blocks = 0usize;
let mut frames = 0usize;
while frames < 16 {
match tokio::time::timeout(Duration::from_millis(300), outbound_rx.recv()).await {
Ok(Some(frame)) => {
frames += 1;
if matches!(
BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes"),
BlockSyncMessage::GetBlocks { .. }
) {
get_blocks += 1;
}
}
_ => break,
}
}
assert_eq!(
get_blocks, 1,
"one scheduled request must produce exactly one outbound GetBlocks via \
BlockSyncPeerSession; a different count would mean a mirroring path is \
(double-)sending in addition to the authoritative direct path",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_scores_exact_supplier_for_commitment_matching_consensus_invalidity() {
let request_bytes: u32 = 10_000;
let config = ZakuraBlockSyncConfig {
max_inflight_block_bytes: BS_PER_BLOCK_WORST_CASE_BYTES * 2,
..immediate_body_download_config()
};
let blocks = mainnet_blocks_1_to_3();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (bad_peer, bad_inbound, mut bad_outbound) = connect_peer_with_status(
&service,
&mut actions,
40,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tip_tx
.send((block::Height(1), blocks[0].hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: blocks[0].hash(),
size: BlockSizeEstimate::Advertised(request_bytes),
}]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut bad_outbound).await,
(block::Height(1), 1)
);
bad_inbound
.send(
BlockSyncMessage::Block(blocks[0].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
let (submit_owner, submit_source, submit_token) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => {
assert_eq!(block.hash(), blocks[0].hash());
break (owner, source, token);
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before invalid body submit: {action:?}"),
}
};
let invalid = zakura_header_chain::ConsensusBodyInvalid {
hash: blocks[0].hash(),
evidence: zakura_header_chain::EvidenceId::from_digest([0xa5; 32]),
rule: zakura_header_chain::BodyRuleId::new("test.consensus_invalid"),
source: submit_source,
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner: submit_owner,
source: submit_source,
token: submit_token,
height: block::Height(1),
hash: blocks[0].hash(),
outcome: BlockApplyOutcome::consensus_invalid(invalid.clone()),
})
.await
.expect("apply-finished event queues");
let scored = loop {
if let BlockSyncAction::Misbehavior { peer, reason } = next_action(&mut actions).await {
assert_eq!(peer, bad_peer);
assert_eq!(
reason,
BlockSyncMisbehavior::ConsensusBodyInvalid(invalid.clone())
);
break true;
}
};
assert!(
scored,
"a consensus apply rejection must score the peer that delivered the body",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_serves_committed_blocks_with_count_and_byte_clamps() {
let blocks = mainnet_blocks_1_to_3();
let block1_size = block_size(&blocks[0]);
let mut config = ZakuraBlockSyncConfig {
max_blocks_per_response: 2,
max_response_bytes: block1_size,
..ZakuraBlockSyncConfig::default()
};
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: blocks[2].hash(),
},
(block::Height(4), block::Hash([4; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
60,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 10,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("GetBlocks frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => {
assert_eq!(peer, peer_id);
assert_eq!(start, block::Height(1));
assert_eq!(count, 2);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before block range query: {action:?}"),
}
}
handle
.send(BlockSyncEvent::BlockRangeResponseReady {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 2,
blocks: vec![
(
block::Height(1),
blocks[0].clone(),
usize::try_from(block1_size).expect("block size fits usize"),
),
(
block::Height(2),
blocks[1].clone(),
usize::try_from(block_size(&blocks[1])).expect("block size fits usize"),
),
],
})
.await
.expect("served block response queues");
assert_eq!(
wait_for_outbound_block(&mut outbound_rx).await.hash(),
blocks[0].hash()
);
assert_eq!(
wait_for_outbound_blocks_done(&mut outbound_rx).await,
(block::Height(1), 1),
"max_response_bytes clamps the served response to one body"
);
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(4),
count: 1,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("above-tip GetBlocks frame queues");
assert_eq!(
wait_for_outbound_range_unavailable(&mut outbound_rx).await,
(block::Height(4), 1)
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_never_serves_reorder_buffer_bodies() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
61,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(3), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
let quiet = tokio::time::timeout(Duration::from_millis(50), async {
while let Some(action) = actions.recv().await {
if matches!(action, BlockSyncAction::SubmitBlock { .. }) {
panic!("height 3 must stay buffered behind the height 2 gap");
}
}
})
.await;
assert!(quiet.is_err());
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(3),
count: 1,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("GetBlocks frame queues");
assert_eq!(
wait_for_outbound_range_unavailable(&mut outbound_rx).await,
(block::Height(3), 1),
"uncommitted reorder-buffer body must not be served"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_schedules_gap_below_buffered_reorder_run() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
63,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(3), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks { start_height, .. } => {
panic!("buffered (covered) height {start_height:?} must not be re-requested")
}
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message after buffering block: {msg:?}"),
}
}
action = tokio::time::timeout(Duration::from_millis(50), actions.recv()) => {
match action {
Ok(Some(BlockSyncAction::QueryNeededBlocks { .. })) => {}
Ok(Some(other)) => panic!("unexpected action after buffering block: {other:?}"),
Ok(None) | Err(_) => break,
}
}
}
}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[1]),
block_meta(&blocks[2]),
]))
.await
.expect("needed metadata queues");
let (got_start, _count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(
got_start,
block::Height(2),
"reactor must schedule the gap at height 2 below the buffered reorder run",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_debounces_status_advertisements_on_serving_tip_change() {
let mut config = ZakuraBlockSyncConfig {
status_refresh_interval: Duration::from_millis(100),
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(4), block::Hash([4; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
62,
block::Height(4),
block::Hash([4; 32]),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
wait_for_outbound_status(&mut outbound_rx).await;
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
}))
.await
.expect("unchanged frontier queues");
assert!(
tokio::time::timeout(Duration::from_millis(50), outbound_rx.recv())
.await
.is_err(),
"unchanged serving range must not advertise"
);
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: block::Hash([1; 32]),
}))
.await
.expect("changed frontier queues");
match next_outbound_message(&mut outbound_rx).await {
BlockSyncMessage::Status(status) => {
assert_eq!(status.servable_high, block::Height(1));
assert_eq!(handle.local_status().servable_high, block::Height(1));
}
msg => panic!("expected debounced Status after serving tip change, got {msg:?}"),
}
for height in [2, 3] {
let hash_byte = u8::try_from(height).expect("test height fits in u8");
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(height),
verified_block_hash: block::Hash([hash_byte; 32]),
}))
.await
.expect("burst frontier queues");
}
assert!(
tokio::time::timeout(Duration::from_millis(50), outbound_rx.recv())
.await
.is_err(),
"rapid serving-tip changes must be debounced to one Status per window"
);
match next_outbound_message(&mut outbound_rx).await {
BlockSyncMessage::Status(status) => {
assert_eq!(status.servable_high, block::Height(3));
assert_eq!(handle.local_status().servable_high, block::Height(3));
}
msg => panic!("expected coalesced Status after debounce window, got {msg:?}"),
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_retries_status_to_peer_without_status_when_local_status_unchanged() {
let mut config = ZakuraBlockSyncConfig {
status_refresh_interval: Duration::from_millis(50),
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer = peer(63);
let (_inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer,
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
assert!(matches!(
next_outbound_message(&mut outbound_rx).await,
BlockSyncMessage::Status(_)
));
assert!(
tokio::time::timeout(Duration::from_millis(25), outbound_rx.recv())
.await
.is_err(),
"initial Status send must consume the peer refresh allowance"
);
assert!(matches!(
next_outbound_message(&mut outbound_rx).await,
BlockSyncMessage::Status(_)
));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_replies_to_first_status_when_connect_status_queue_was_full() {
let mut config = ZakuraBlockSyncConfig {
status_refresh_interval: Duration::from_millis(50),
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 1;
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(64);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(1);
outbound_tx
.try_send(
BlockSyncMessage::Status(status())
.encode_frame()
.expect("filler status frame encodes"),
)
.expect("outbound queue starts full");
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer,
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.peer_snapshot().outbound_peers == 1 {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("peer is admitted while outbound queue is full");
let _ = outbound_rx.recv().await.expect("filler frame drains");
send_inbound(&inbound_tx, BlockSyncMessage::Status(status())).await;
assert!(matches!(
next_outbound_message(&mut outbound_rx).await,
BlockSyncMessage::Status(_)
));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_does_not_ping_pong_rapid_repeated_status() {
let mut config = ZakuraBlockSyncConfig {
status_refresh_interval: Duration::from_millis(50),
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle);
let peer = peer(65);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(16);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer,
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
send_inbound(&inbound_tx, BlockSyncMessage::Status(status())).await;
wait_for_outbound_status(&mut outbound_rx).await;
send_inbound(&inbound_tx, BlockSyncMessage::Status(status())).await;
assert!(
tokio::time::timeout(Duration::from_millis(25), outbound_rx.recv())
.await
.is_err(),
"a rapid second inbound Status must not trigger another Status reply"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_preserves_successor_work_across_stale_finalized_reset() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.peer_limits.outbound_queue_depth = 16;
config.request_timeout = Duration::from_secs(300);
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(2),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
72,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(3), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[2].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::SubmitBlock { .. }
) {}
handle
.send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers {
finalized_height: block::Height(2),
verified_block_tip: block::Height(2),
verified_block_hash: blocks[1].hash(),
}))
.await
.expect("stale finalized reset queues");
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])]))
.await
.expect("duplicate needed metadata queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
if let BlockSyncMessage::GetBlocks {
start_height: block::Height(3),
..
} = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes")
{
panic!("stale finalized reset made an already-submitted successor requestable again");
}
}
action = tokio::time::timeout(Duration::from_millis(100), actions.recv()) => {
match action {
Ok(Some(_)) => {}
Ok(None) | Err(_) => break,
}
}
}
}
reactor_task.abort();
}
#[tokio::test]
async fn committed_reanchor_lowers_only_best_header_target() {
let (snapshots, startup) =
committed_block_sync_startup_at_heights(0, 5, 10, immediate_body_download_config());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
wait_for_query_needed_blocks(&mut actions, block::Height(5), block::Height(10)).await;
snapshots
.send(Some(committed_view(
test_committed_snapshot(
2,
2,
1,
(0, block::Hash([0; 32])),
(5, block::Hash([5; 32])),
(7, block::Hash([7; 32])),
),
1,
)))
.expect("the committed snapshot receiver is live");
wait_for_query_needed_blocks(&mut actions, block::Height(5), block::Height(7)).await;
assert_eq!(handle.local_status().servable_high, block::Height(5));
reactor_task.abort();
}
#[tokio::test]
async fn committed_reanchor_requeries_while_downloads_in_flight() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes =
BS_PER_BLOCK_WORST_CASE_BYTES * u64::try_from(blocks.len()).expect("block count fits u64");
config.request_timeout = Duration::from_secs(300);
let initial = test_committed_snapshot(
1,
1,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(3, blocks[2].hash()),
);
let (snapshots, startup) = committed_block_sync_startup(initial, config.clone());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, _inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
67,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
snapshots
.send(Some(committed_view(
test_committed_snapshot(
2,
2,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(1, blocks[0].hash()),
),
1,
)))
.expect("the committed snapshot receiver is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await;
snapshots
.send(Some(committed_view(
test_committed_snapshot(
3,
3,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(3, blocks[2].hash()),
),
1,
)))
.expect("the committed snapshot receiver is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await;
reactor_task.abort();
}
#[tokio::test]
async fn epoch_change_supersedes_stale_floor_query() {
let blocks = mainnet_blocks_1_to_3();
let old_target_hash = block::Hash([0x44; 32]);
let new_target_hash = block::Hash([0x45; 32]);
let config = immediate_body_download_config();
let initial = test_committed_snapshot(
1,
1,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(4, old_target_hash),
);
let (snapshots, startup) = committed_block_sync_startup(initial, config.clone());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let BlockSyncAction::QueryNeededBlocks {
query_id,
scope,
from: block::Height(1),
..
} = next_action(&mut actions).await
else {
panic!("startup must query from the verified tip");
};
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
69,
block::Height(4),
old_target_hash,
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::ScopedNeededBlocks {
query_id,
scope,
body_anchor: zakura_header_chain::Frontier::new(block::Height(0), block::Hash([0; 32])),
blocks: blocks.iter().map(block_meta).collect(),
})
.await
.expect("initial needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
for block in &blocks {
send_inbound(&inbound_tx, BlockSyncMessage::Block(block.clone())).await;
}
send_inbound(
&inbound_tx,
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 3,
},
)
.await;
let mut submitted = 0usize;
while submitted < blocks.len() {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { .. } => submitted += 1,
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while advancing the floor: {action:?}"),
}
}
let sequencer_view = handle
.routine_wiring
.as_ref()
.expect("the test reactor exposes routine wiring")
.view
.clone();
await_until(
"download floor reaches height 3",
Duration::from_secs(1),
|| sequencer_view.borrow().download_floor == block::Height(3),
)
.await
.expect("unresolved submissions advance the download floor");
let old_reset_epoch = sequencer_view.borrow().reset_epoch;
let changed_view = committed_view(
test_committed_snapshot(
2,
2,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(4, new_target_hash),
),
1,
);
let changed_scope = zakura_header_chain::BodyWorkAuthority::for_view(&changed_view);
snapshots
.send(Some(changed_view))
.expect("the invalidating committed view publishes");
let stale_floor_query = loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
query_id,
from: block::Height(4),
scope,
..
} if scope == changed_scope => break (query_id, scope),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before the stale-floor query: {action:?}"),
}
};
let replacement_query = loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
query_id,
from: block::Height(1),
scope,
..
} if scope == changed_scope => break (query_id, scope),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before the replacement query: {action:?}"),
}
};
assert_ne!(stale_floor_query.0, replacement_query.0);
assert!(sequencer_view.borrow().reset_epoch > old_reset_epoch);
send_inbound(
&inbound_tx,
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(4),
tip_hash: new_target_hash,
max_blocks_per_response: 16,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
}),
)
.await;
handle
.send(BlockSyncEvent::ScopedNeededBlocks {
query_id: stale_floor_query.0,
scope: stale_floor_query.1,
body_anchor: zakura_header_chain::Frontier::new(block::Height(0), block::Hash([0; 32])),
blocks: vec![block_meta(&blocks[2])],
})
.await
.expect("stale query completion queues");
handle
.send(BlockSyncEvent::ScopedNeededBlocks {
query_id: replacement_query.0,
scope: replacement_query.1,
body_anchor: zakura_header_chain::Frontier::new(block::Height(0), block::Hash([0; 32])),
blocks: vec![block_meta(&blocks[0])],
})
.await
.expect("replacement query completion queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 1)
);
reactor_task.abort();
}
#[tokio::test]
async fn header_extension_preserves_checkpoint_pipeline() {
let blocks = mainnet_blocks_1_to_3();
let old_target_hash = blocks[2].hash();
let new_target_hash = block::Hash([0x47; 32]);
let config = immediate_body_download_config();
let initial = test_committed_snapshot(
1,
1,
1,
(0, block::Hash([0; 32])),
(1, blocks[0].hash()),
(3, old_target_hash),
);
let (snapshots, startup) = committed_block_sync_startup(initial, config.clone());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let BlockSyncAction::QueryNeededBlocks {
query_id, scope, ..
} = next_action(&mut actions).await
else {
panic!("startup must query from the verified tip");
};
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
70,
block::Height(3),
old_target_hash,
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::ScopedNeededBlocks {
query_id,
scope,
body_anchor: zakura_header_chain::Frontier::new(block::Height(1), blocks[0].hash()),
blocks: blocks[1..].iter().map(block_meta).collect(),
})
.await
.expect("initial metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 2)
);
for block in &blocks[1..] {
send_inbound(&inbound_tx, BlockSyncMessage::Block(block.clone())).await;
}
send_inbound(
&inbound_tx,
BlockSyncMessage::BlocksDone {
start_height: block::Height(2),
returned: 2,
},
)
.await;
let mut submissions = Vec::new();
while submissions.len() < 2 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => submissions.push((owner, source, token, block)),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while filling the checkpoint pipeline: {action:?}"),
}
}
let sequencer_view = handle
.routine_wiring
.as_ref()
.expect("the test reactor exposes routine wiring")
.view
.clone();
await_until(
"download floor reaches height 3",
Duration::from_secs(1),
|| sequencer_view.borrow().download_floor == block::Height(3),
)
.await
.expect("unresolved submissions advance the floor");
let reset_epoch = sequencer_view.borrow().reset_epoch;
let extension = committed_view(
test_committed_snapshot(
2,
2,
1,
(0, block::Hash([0; 32])),
(1, blocks[0].hash()),
(2_010, new_target_hash),
),
0,
);
let extension_scope = zakura_header_chain::BodyWorkAuthority::for_view(&extension);
snapshots
.send(Some(extension))
.expect("the 2,007-header extension publishes");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from: block::Height(4),
scope,
..
} if scope == extension_scope => break,
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action after the compatible extension: {action:?}"),
}
}
assert_eq!(sequencer_view.borrow().reset_epoch, reset_epoch);
assert_eq!(sequencer_view.borrow().download_floor, block::Height(3));
assert_eq!(sequencer_view.borrow().applying_len, 2);
let checkpoint = committed_view(
test_committed_snapshot(
3,
3,
2,
(1, blocks[0].hash()),
(1, blocks[0].hash()),
(2_010, new_target_hash),
),
0,
);
let checkpoint_scope = zakura_header_chain::BodyWorkAuthority::for_view(&checkpoint);
snapshots
.send(Some(checkpoint))
.expect("checkpoint finality publishes after the extension");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from: block::Height(4),
scope,
..
} if scope == checkpoint_scope => break,
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action after checkpoint finality: {action:?}"),
}
}
assert_eq!(sequencer_view.borrow().reset_epoch, reset_epoch);
assert_eq!(sequencer_view.borrow().download_floor, block::Height(3));
assert_eq!(sequencer_view.borrow().applying_len, 2);
assert_eq!(sequencer_view.borrow().in_flight_submission_count, 2);
for (owner, source, token, block) in submissions {
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner,
source,
token,
height: block
.coinbase_height()
.expect("the fixture block has a height"),
hash: block.hash(),
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("the compatible completion queues");
}
await_until(
"checkpoint submissions complete",
Duration::from_secs(1),
|| sequencer_view.borrow().applying_len == 0,
)
.await
.expect("the preserved checkpoint pipeline completes without a gap");
assert_eq!(sequencer_view.borrow().reset_epoch, reset_epoch);
reactor_task.abort();
}
#[tokio::test]
async fn committed_body_progress_preserves_same_target_native_response() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes =
BS_PER_BLOCK_WORST_CASE_BYTES * u64::try_from(blocks.len()).expect("block count fits u64");
config.request_timeout = Duration::from_secs(300);
let initial = test_committed_snapshot(
1,
1,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(3, blocks[2].hash()),
);
let (snapshots, startup) = committed_block_sync_startup(initial, config.clone());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
68,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
snapshots
.send(Some(committed_view(
test_committed_snapshot(
2,
1,
2,
(0, block::Hash([0; 32])),
(1, blocks[0].hash()),
(3, blocks[2].hash()),
),
0,
)))
.expect("the committed snapshot receiver is live");
await_until(
"committed frontier reaches height 1",
Duration::from_secs(1),
|| handle.local_status().servable_high >= block::Height(1),
)
.await
.expect("committed frontier should advance");
for block in &blocks {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
}
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(1),
returned: 3,
}
.encode_frame()
.expect("BlocksDone frame encodes"),
)
.await
.expect("BlocksDone frame queues");
let mut submitted = Vec::new();
while submitted.len() < 2 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while preserving response: {action:?}"),
}
}
assert_eq!(submitted, vec![block::Height(2), block::Height(3)]);
reactor_task.abort();
}
#[tokio::test]
async fn committed_reanchor_releases_stale_submitted_bodies() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes =
BS_PER_BLOCK_WORST_CASE_BYTES * u64::try_from(blocks.len()).expect("block count fits u64");
config.request_timeout = Duration::from_secs(300);
let initial = test_committed_snapshot(
1,
1,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(3, blocks[2].hash()),
);
let (snapshots, startup) = committed_block_sync_startup(initial, config.clone());
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
66,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 3)
);
for block in &blocks {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
}
let mut submitted = Vec::new();
while submitted.len() < blocks.len() {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before all submitted bodies: {action:?}"),
}
}
assert_eq!(
submitted,
vec![block::Height(1), block::Height(2), block::Height(3)]
);
snapshots
.send(Some(committed_view(
test_committed_snapshot(
2,
2,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(1, blocks[0].hash()),
),
1,
)))
.expect("the committed snapshot receiver is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await;
snapshots
.send(Some(committed_view(
test_committed_snapshot(
3,
3,
1,
(0, block::Hash([0; 32])),
(0, block::Hash([0; 32])),
(3, blocks[2].hash()),
),
1,
)))
.expect("the committed snapshot receiver is live");
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata after reanchor queues");
let (got_start, got_count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(got_start, block::Height(1));
assert_eq!(
got_count, 3,
"reanchored headers must release old submitted bodies and request them again",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_clamps_tiny_submitted_apply_config_above_checkpoint_range() {
let blocks = fake_sequential_blocks(4);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = u64::MAX;
config.max_submitted_block_applies = 2;
config.request_timeout = Duration::from_secs(300);
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(4), blocks[3].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await;
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
67,
block::Height(4),
blocks[3].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(
blocks.iter().map(block_meta).collect(),
))
.await
.expect("needed metadata queues");
let (_start, count) = wait_for_outbound_getblocks(&mut outbound_rx).await;
assert_eq!(count, 4);
for block in &blocks {
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
}
let mut submitted = Vec::new();
while submitted.len() < 4 {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => submitted.push(
block
.coinbase_height()
.expect("submitted test block has height"),
),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while collecting submissions: {action:?}"),
}
}
assert_eq!(
submitted,
vec![
block::Height(1),
block::Height(2),
block::Height(3),
block::Height(4),
],
"tiny configured submit caps are raised to the checkpoint-safe floor"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_far_ahead_header_tip_queries_only_next_refill_window() {
let best_header_tip = block::Height(50_000);
let (_tip_tx, tip_rx) = watch::channel((best_header_tip, block::Hash([50; 32])));
let config = ZakuraBlockSyncConfig {
max_blocks_per_response: 1,
max_inflight_requests: 1,
..ZakuraBlockSyncConfig::default()
};
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(best_header_tip, block::Hash([50; 32])),
tip_rx,
config,
);
let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
assert!(matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
limit: 2,
best_header_tip: block::Height(50_000),
..
}
));
reactor_task.abort();
}
#[tokio::test]
async fn selected_body_fork_reanchors_before_scheduling_successors() {
let best_header_tip = block::Height(4);
let (_tip_tx, tip_rx) = watch::channel((best_header_tip, block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: block::Hash([0xf2; 32]),
},
(best_header_tip, block::Hash([4; 32])),
tip_rx,
ZakuraBlockSyncConfig::default(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let BlockSyncAction::QueryNeededBlocks {
query_id,
scope,
from: block::Height(3),
..
} = next_action(&mut actions).await
else {
panic!("startup must query above the stale full-state fork tip");
};
handle
.send(BlockSyncEvent::ScopedNeededBlocks {
query_id,
scope,
body_anchor: zakura_header_chain::Frontier::new(block::Height(1), block::Hash([1; 32])),
blocks: vec![BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Unknown,
}],
})
.await
.expect("fork-aware metadata completion queues");
assert!(matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(2),
..
}
));
reactor_task.abort();
}
#[tokio::test]
async fn stale_needed_block_completion_cannot_clear_a_newer_query() {
let best_header_tip = block::Height(10);
let (_tip_tx, tip_rx) = watch::channel((best_header_tip, block::Hash([10; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(best_header_tip, block::Hash([10; 32])),
tip_rx,
ZakuraBlockSyncConfig::default(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let BlockSyncAction::QueryNeededBlocks {
query_id: first_query_id,
scope: first_scope,
..
} = next_action(&mut actions).await
else {
panic!("startup must query needed blocks");
};
handle
.send(BlockSyncEvent::NeededBlocks(Vec::new()))
.await
.expect("test fixture clears the first pending query");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(11),
hash: block::Hash([11; 32]),
})
.await
.expect("header-tip event queues");
let BlockSyncAction::QueryNeededBlocks {
query_id: second_query_id,
scope: second_scope,
..
} = next_action(&mut actions).await
else {
panic!("new header tip must query needed blocks");
};
assert_ne!(first_query_id, second_query_id);
handle
.send(BlockSyncEvent::ScopedNeededBlocks {
query_id: first_query_id,
scope: first_scope,
body_anchor: zakura_header_chain::Frontier::new(block::Height(0), block::Hash([0; 32])),
blocks: Vec::new(),
})
.await
.expect("stale completion queues");
handle
.send(BlockSyncEvent::ScopedNeededBlocks {
query_id: second_query_id,
scope: second_scope,
body_anchor: zakura_header_chain::Frontier::new(block::Height(0), block::Hash([0; 32])),
blocks: vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
}],
})
.await
.expect("matched query completion queues");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(12),
hash: block::Hash([12; 32]),
})
.await
.expect("second header-tip event queues");
assert!(matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
query_id,
from: block::Height(2),
..
} if query_id != first_query_id && query_id != second_query_id
));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_refill_window_advances_past_claimed_heights() {
let best_header_tip = block::Height(50_000);
let (_tip_tx, tip_rx) = watch::channel((best_header_tip, block::Hash([50; 32])));
let config = ZakuraBlockSyncConfig {
max_blocks_per_response: 1,
max_inflight_requests: 4,
..ZakuraBlockSyncConfig::default()
};
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(best_header_tip, block::Hash([50; 32])),
tip_rx,
config,
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
assert!(matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
limit: 8,
best_header_tip: block::Height(50_000),
..
}
));
let metas: Vec<_> = (1..=3)
.map(|height| BlockSyncBlockMeta {
height: block::Height(height),
hash: block::Hash([u8::try_from(height).expect("height fits u8"); 32]),
size: BlockSizeEstimate::Advertised(1_000),
})
.collect();
handle
.send(BlockSyncEvent::NeededBlocks(metas))
.await
.expect("needed-blocks event queues");
handle
.send(BlockSyncEvent::HeaderTipChanged {
height: block::Height(50_001),
hash: block::Hash([51; 32]),
})
.await
.expect("header-tip event queues");
match next_action(&mut actions).await {
BlockSyncAction::QueryNeededBlocks {
from,
limit,
best_header_tip,
..
} => {
assert_eq!(
from,
block::Height(4),
"refill must advance past the claimed heights, not rescan from the floor",
);
assert_eq!(limit, 8);
assert_eq!(best_header_tip, block::Height(50_001));
}
action => panic!("expected the advanced refill query, got {action:?}"),
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_stale_non_reset_frontier_updates() {
let (_tip_tx, tip_rx) = watch::channel((block::Height(3600), block::Hash([36; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(3200),
verified_block_tip: block::Height(3200),
verified_block_hash: block::Hash([32; 32]),
},
(block::Height(3600), block::Hash([36; 32])),
tip_rx,
immediate_body_download_config(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
assert!(matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(3201),
best_header_tip: block::Height(3600),
..
}
));
handle
.send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers {
finalized_height: block::Height(3200),
verified_block_tip: block::Height(2913),
verified_block_hash: block::Hash([29; 32]),
}))
.await
.expect("stale grow event queues");
assert!(
tokio::time::timeout(Duration::from_millis(50), actions.recv())
.await
.is_err(),
"stale lower grow frontier must not query from the lower height"
);
assert_eq!(handle.local_status().servable_high, block::Height(3200));
reactor_task.abort();
}
#[tokio::test]
async fn reactor_retries_matched_range_unavailable_without_scoring_peer() {
let blocks = mainnet_blocks_1_to_3();
let mut config = fill_loop_mechanics_config();
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
63,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2)
);
inbound_tx
.send(
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: 2,
}
.encode_frame()
.expect("RangeUnavailable frame encodes"),
)
.await
.expect("RangeUnavailable frame queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2),
"matched RangeUnavailable should retry the original range without scoring the serving peer",
);
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
inbound_tx
.send(
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(2),
count: 1,
}
.encode_frame()
.expect("RangeUnavailable frame encodes"),
)
.await
.expect("unmatched RangeUnavailable frame queues");
loop {
tokio::select! {
biased;
frame = outbound_rx.recv() => {
let frame = frame.expect("outbound channel is live");
match BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") {
BlockSyncMessage::GetBlocks { .. } => {
panic!("unmatched RangeUnavailable must not trigger a fresh request")
}
BlockSyncMessage::Status(_) => {}
msg => panic!("unexpected outbound message after unmatched RangeUnavailable: {msg:?}"),
}
}
action = tokio::time::timeout(Duration::from_millis(50), actions.recv()) => {
match action {
Ok(Some(BlockSyncAction::Misbehavior { .. })) => {
panic!("unmatched RangeUnavailable must not score the serving peer")
}
Ok(Some(_)) => {}
Ok(None) | Err(_) => break,
}
}
}
}
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_does_not_wedge_honest_peer_under_range_unavailable_spam() {
let blocks = mainnet_blocks_1_to_3();
let mut config = immediate_body_download_config();
config.request_timeout = Duration::from_secs(300);
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (m1, m1_in, mut m1_out) = connect_peer_with_status(
&service,
&mut actions,
0x01,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let (m2, m2_in, mut m2_out) = connect_peer_with_status(
&service,
&mut actions,
0x02,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let (h, _h_in, mut h_out) = connect_peer_with_status(
&service,
&mut actions,
0x03,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("needed metadata queues");
let range_unavailable = || {
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: 2,
}
.encode_frame()
.expect("RangeUnavailable frame encodes")
};
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = vec![
(m1.clone(), &mut m1_out),
(m2.clone(), &mut m2_out),
(h.clone(), &mut h_out),
];
let mut honest_offered = false;
for _ in 0..16 {
let (peer, _start, _count) = wait_for_getblocks_across(&mut outbound_by_peer).await;
if peer == h {
honest_offered = true;
break;
} else if peer == m1 {
m1_in
.send(range_unavailable())
.await
.expect("m1 RangeUnavailable queues");
} else if peer == m2 {
m2_in
.send(range_unavailable())
.await
.expect("m2 RangeUnavailable queues");
}
}
assert!(
honest_offered,
"honest peer must be offered the contested range after both fanout peers fail it"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_range_unavailable_retries_only_unverified_suffix() {
let blocks = mainnet_blocks_1_to_3();
let mut config = fill_loop_mechanics_config();
config.peer_limits.outbound_queue_depth = 16;
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (_peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
64,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[0]),
block_meta(&blocks[1]),
]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(1), 2)
);
handle
.send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers {
finalized_height: block::Height(1),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
}))
.await
.expect("frontier grow queues");
inbound_tx
.send(
BlockSyncMessage::RangeUnavailable {
start_height: block::Height(1),
count: 2,
}
.encode_frame()
.expect("RangeUnavailable frame encodes"),
)
.await
.expect("RangeUnavailable frame queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1),
"late RangeUnavailable must not retry the already verified prefix",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_backpressures_serving_slots_without_scoring_peer() {
let mut config = ZakuraBlockSyncConfig {
max_inflight_requests: 1,
..ZakuraBlockSyncConfig::default()
};
config.peer_limits.outbound_queue_depth = 16;
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
63,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
tokio::time::sleep(Duration::from_millis(10)).await;
for _ in 0..2 {
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 1,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("GetBlocks frame queues");
}
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryBlocksByHeightRange { .. }
) {}
assert_eq!(
wait_for_outbound_range_unavailable(&mut outbound_rx).await,
(block::Height(1), 1),
"serving-slot saturation should backpressure the requester, not score it as spam",
);
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
handle
.send(BlockSyncEvent::BlockRangeResponseFinished {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 1,
returned_count: 1,
})
.await
.expect("serving slot release queues");
reactor_task.abort();
}
#[tokio::test]
async fn reactor_full_serving_queue_drops_without_disconnecting_peer() {
let blocks = mainnet_blocks_1_to_3();
let config = ZakuraBlockSyncConfig {
max_blocks_per_response: 16,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
..ZakuraBlockSyncConfig::default()
};
let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(3),
verified_block_hash: blocks[2].hash(),
},
(block::Height(3), blocks[2].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(66);
let cancel = CancellationToken::new();
let (inbound_tx, inbound_rx) = framed_channel(16);
let (outbound_tx, mut outbound_rx) = framed_channel(2);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
cancel.clone(),
));
wait_for_outbound_status(&mut outbound_rx).await;
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(3),
tip_hash: blocks[2].hash(),
max_blocks_per_response: 16,
max_inflight_requests: 8,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status frame queues");
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 3,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("GetBlocks frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => {
assert_eq!(peer, peer_id);
assert_eq!(start, block::Height(1));
assert_eq!(count, 3);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before block range query: {action:?}"),
}
}
let served: Vec<_> = blocks
.iter()
.map(|block| {
(
block.coinbase_height().expect("test block has height"),
block.clone(),
usize::try_from(block_size(block)).expect("block size fits usize"),
)
})
.collect();
handle
.send(BlockSyncEvent::BlockRangeResponseReady {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 3,
blocks: served,
})
.await
.expect("served block response queues");
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
!cancel.is_cancelled(),
"a full serving queue must drop sends, not cancel the peer",
);
assert_eq!(handle.peer_snapshot().outbound_peers, 1);
while tokio::time::timeout(
Duration::from_millis(100),
next_outbound_message(&mut outbound_rx),
)
.await
.is_ok()
{}
handle
.send(BlockSyncEvent::BlockRangeResponseFinished {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 3,
returned_count: 3,
})
.await
.expect("serving slot release queues");
inbound_tx
.send(
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 1,
}
.encode_frame()
.expect("GetBlocks frame encodes"),
)
.await
.expect("re-request GetBlocks frame queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::QueryBlocksByHeightRange { start, count, .. } => {
assert_eq!(start, block::Height(1));
assert_eq!(count, 1);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action before re-request query: {action:?}"),
}
}
handle
.send(BlockSyncEvent::BlockRangeResponseReady {
peer: peer_id.clone(),
start_height: block::Height(1),
requested_count: 1,
blocks: vec![(
block::Height(1),
blocks[0].clone(),
usize::try_from(block_size(&blocks[0])).expect("block size fits usize"),
)],
})
.await
.expect("re-served block response queues");
assert_eq!(
wait_for_outbound_block(&mut outbound_rx).await.hash(),
blocks[0].hash(),
"serving resumes once the queue drains",
);
assert!(!cancel.is_cancelled());
reactor_task.abort();
}
#[tokio::test]
async fn reactor_publishes_block_sync_candidate_gap() {
let config = immediate_body_download_config();
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer_id = peer(77);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer_id.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
wait_for_outbound_status(&mut outbound_rx).await;
tip_tx
.send((block::Height(2), block::Hash([2; 32])))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![
BlockSyncBlockMeta {
height: block::Height(2),
hash: block::Hash([2; 32]),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Unknown,
},
]))
.await
.expect("needed blocks event queues");
let mut candidates = handle.subscribe_candidate_state();
let observed = tokio::time::timeout(Duration::from_secs(1), async {
loop {
candidates
.changed()
.await
.expect("candidate watch remains open");
let state = candidates.borrow().clone();
if state.missing_block_bodies == vec![block::Height(1), block::Height(2)] {
return state;
}
}
})
.await
.unwrap_or_else(|_| handle.candidate_state());
assert_eq!(
observed.missing_block_bodies,
vec![block::Height(1), block::Height(2)]
);
assert!(
observed.admitted_node_ids.is_empty(),
"a peer without block-sync status must not satisfy body-sync demand"
);
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(2),
tip_hash: block::Hash([2; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status queues");
let peer_node_id =
node_id_from_block_peer_id(&peer_id).expect("test peer id is a valid node id");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
candidates
.changed()
.await
.expect("candidate watch remains open");
if candidates.borrow().admitted_node_ids == vec![peer_node_id] {
return;
}
}
})
.await
.expect("status-bearing peer is published as an admitted candidate");
handle
.send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(2),
verified_block_hash: block::Hash([2; 32]),
}))
.await
.expect("frontier event queues");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.candidate_state().missing_block_bodies.is_empty() {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("candidate state clears after the gap is gone");
reactor_task.abort();
}
#[tokio::test]
async fn oversize_body_policy_reports_size_mismatch_and_retries_without_buffering() {
let mut config = ZakuraBlockSyncConfig {
size_deviation_tolerance: 100,
..immediate_body_download_config()
};
config.peer_limits.outbound_queue_depth = 8;
let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(42);
let (inbound_tx, inbound_rx) = framed_channel(8);
let (outbound_tx, mut outbound_rx) = framed_channel(8);
let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]);
service.add_peer(Peer::new_with_direction(
peer,
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
streams,
CancellationToken::new(),
));
inbound_tx
.send(
BlockSyncMessage::Status(BlockSyncStatus {
servable_low: block::Height(1),
servable_high: block::Height(1),
tip_hash: block::Hash([1; 32]),
max_blocks_per_response: 4,
max_inflight_requests: 1,
max_response_bytes: MAX_BS_RESPONSE_BYTES,
})
.encode_frame()
.expect("status encodes"),
)
.await
.expect("status queues");
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
tip_tx
.send((block::Height(1), block.hash()))
.expect("tip watch is live");
while !matches!(
next_action(&mut actions).await,
BlockSyncAction::QueryNeededBlocks { .. }
) {}
handle
.send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block.hash(),
size: BlockSizeEstimate::Advertised(1),
}]))
.await
.expect("needed metadata queues");
while !matches!(
BlockSyncMessage::decode_frame(
tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv())
.await
.expect("outbound frame arrives")
.expect("outbound channel is live")
)
.expect("frame decodes"),
BlockSyncMessage::GetBlocks { .. }
) {}
inbound_tx
.send(
BlockSyncMessage::Block(block.clone())
.encode_frame()
.expect("block encodes"),
)
.await
.expect("block queues");
loop {
match next_action(&mut actions).await {
BlockSyncAction::Misbehavior { reason, .. } => {
assert_eq!(reason, BlockSyncMisbehavior::SizeMismatch);
break;
}
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action during size mismatch test: {action:?}"),
}
}
let no_submit = tokio::time::timeout(Duration::from_millis(200), async {
while let Some(action) = actions.recv().await {
if matches!(action, BlockSyncAction::SubmitBlock { .. }) {
return false;
}
}
true
})
.await
.unwrap_or(true);
assert!(
no_submit,
"oversize body is not submitted after SizeMismatch"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_known_peer_unsolicited_blocks_done_is_reported_as_misbehavior() {
let config = ZakuraBlockSyncConfig::default();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
63,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(7),
returned: 1,
}
.encode_frame()
.expect("BlocksDone frame encodes"),
)
.await
.expect("BlocksDone frame queues");
let mut saw_unsolicited_done = false;
while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await
{
if let BlockSyncAction::Misbehavior { peer, reason } = action {
if peer == peer_id && reason == BlockSyncMisbehavior::UnsolicitedDone {
saw_unsolicited_done = true;
break;
}
}
}
assert!(
saw_unsolicited_done,
"a known peer's unsolicited BlocksDone with no matching outstanding request \
must be reported as Misbehavior::UnsolicitedDone (SR-6/SR-7), but the reactor \
silently tolerated it and kept the peer connected",
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_accepts_unmatched_body_for_height_active_on_another_request() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer1, inbound1, mut outbound1) = connect_peer_with_status(
&service,
&mut actions,
65,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let (peer2, inbound2, mut outbound2) = connect_peer_with_status(
&service,
&mut actions,
66,
block::Height(3),
blocks[2].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("needed metadata queues");
let (requested_peer, start_height, _count) = {
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = vec![
(peer1.clone(), &mut outbound1),
(peer2.clone(), &mut outbound2),
];
wait_for_getblocks_across(&mut outbound_by_peer).await
};
assert_eq!(start_height, block::Height(2));
handle
.send(BlockSyncEvent::NeededBlocks(Vec::new()))
.await
.expect("empty needed metadata queues");
let (late_peer, late_inbound) = if requested_peer == peer1 {
(peer2, inbound2)
} else {
(peer1, inbound1)
};
send_inbound(&late_inbound, BlockSyncMessage::Block(blocks[1].clone())).await;
let submitted = tokio::time::timeout(Duration::from_secs(1), async {
loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock { block, .. } => return block.hash(),
BlockSyncAction::QueryNeededBlocks { .. } => {}
BlockSyncAction::Misbehavior { peer, reason } => {
assert_ne!(
peer, late_peer,
"late active body was reported as {reason:?}"
);
}
action => {
panic!("unexpected action while waiting for late body submit: {action:?}")
}
}
}
})
.await
.expect("late active body is accepted and submitted");
assert_eq!(submitted, blocks[1].hash());
send_inbound(
&late_inbound,
BlockSyncMessage::BlocksDone {
start_height: block::Height(2),
returned: 1,
},
)
.await;
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(200), actions.recv()).await
{
if let BlockSyncAction::Misbehavior { peer, reason } = action {
assert_ne!(
peer, late_peer,
"late response for an active request was reported as {reason:?}"
);
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_duplicate_response_at_body_download_floor() {
let config = immediate_body_download_config();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(2), blocks[1].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
65,
block::Height(2),
blocks[1].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("needed metadata queues");
assert_eq!(
wait_for_outbound_getblocks(&mut outbound_rx).await,
(block::Height(2), 1)
);
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("block frame encodes"),
)
.await
.expect("block frame queues");
let (owner, source, token, hash) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => break (owner, source, token, block.hash()),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while waiting for submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner,
source,
token,
height: block::Height(2),
hash,
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("apply result queues");
inbound_tx
.send(
BlockSyncMessage::Block(blocks[1].clone())
.encode_frame()
.expect("duplicate block frame encodes"),
)
.await
.expect("duplicate block frame queues");
inbound_tx
.send(
BlockSyncMessage::BlocksDone {
start_height: block::Height(2),
returned: 1,
}
.encode_frame()
.expect("duplicate terminator frame encodes"),
)
.await
.expect("duplicate terminator frame queues");
while let Ok(Some(action)) =
tokio::time::timeout(Duration::from_millis(200), actions.recv()).await
{
if let BlockSyncAction::Misbehavior { peer, reason } = action {
assert_ne!(
peer, peer_id,
"duplicate response at body_download_floor was reported as {reason:?}"
);
}
}
reactor_task.abort();
}
#[tokio::test]
async fn reactor_ignores_matched_duplicate_response_at_body_download_floor() {
let blocks = mainnet_blocks_1_to_3();
let block2_size = block_size(&blocks[1]);
let mut config = immediate_body_download_config();
config.max_inflight_block_bytes = BS_PER_BLOCK_WORST_CASE_BYTES * 2;
let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(4), block::Hash([4; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let (peer_a, inbound_a, mut outbound_a) = connect_peer_with_status(
&service,
&mut actions,
66,
block::Height(4),
block::Hash([4; 32]),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
let (peer_b, inbound_b, mut outbound_b) = connect_peer_with_status(
&service,
&mut actions,
67,
block::Height(4),
block::Hash([4; 32]),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])]))
.await
.expect("needed metadata queues");
let mut outbound_by_peer: Vec<(ZakuraPeerId, &mut FramedRecv)> = vec![
(peer_a.clone(), &mut outbound_a),
(peer_b.clone(), &mut outbound_b),
];
let first_request = wait_for_getblocks_across(&mut outbound_by_peer).await;
assert_eq!((first_request.1, first_request.2), (block::Height(2), 1));
let (other_peer, requested_inbound, other_inbound) = if first_request.0 == peer_a {
(peer_b.clone(), &inbound_a, &inbound_b)
} else {
(peer_a.clone(), &inbound_b, &inbound_a)
};
send_inbound(
requested_inbound,
BlockSyncMessage::Block(blocks[1].clone()),
)
.await;
let (owner, source, token, hash) = loop {
match next_action(&mut actions).await {
BlockSyncAction::SubmitBlock {
owner,
source,
token,
block,
} => break (owner, source, token, block.hash()),
BlockSyncAction::QueryNeededBlocks { .. } => {}
action => panic!("unexpected action while waiting for submit: {action:?}"),
}
};
handle
.send(BlockSyncEvent::BlockApplyFinished {
owner,
source,
token,
height: block::Height(2),
hash,
outcome: test_block_apply_outcome(BlockApplyResult::Committed),
})
.await
.expect("apply result queues");
let _ = &other_peer;
send_inbound(other_inbound, BlockSyncMessage::Block(blocks[1].clone())).await;
handle
.send(BlockSyncEvent::NeededBlocks(vec![
block_meta(&blocks[2]),
BlockSyncBlockMeta {
height: block::Height(4),
hash: block::Hash([4; 32]),
size: BlockSizeEstimate::Advertised(block2_size),
},
]))
.await
.expect("next needed metadata queues");
let mut requested: Vec<block::Height> = Vec::new();
tokio::time::timeout(Duration::from_secs(2), async {
loop {
let (peer, start, count) = wait_for_getblocks_across(&mut outbound_by_peer).await;
assert!(
peer == peer_a || peer == peer_b,
"request should target one of the connected peers"
);
for offset in 0..count {
if let Some(height) = height_after_count(start, offset) {
requested.push(height);
}
}
if requested.contains(&block::Height(3)) && requested.contains(&block::Height(4)) {
break;
}
}
})
.await
.expect("both remaining heights are requested within the unchanged budget");
requested.sort_unstable();
let mut deduped = requested.clone();
deduped.dedup();
assert_eq!(
requested, deduped,
"a matched duplicate response at the body floor must not consume reorder budget \
(no height should be fetched twice)"
);
assert!(
requested.contains(&block::Height(3)) && requested.contains(&block::Height(4)),
"both needed heights must be fetched once the duplicate releases its transient reservation"
);
reactor_task.abort();
}
#[tokio::test]
async fn reactor_scores_unsolicited_terminator_from_connected_peer() {
let config = ZakuraBlockSyncConfig::default();
let blocks = mainnet_blocks_1_to_3();
let (_tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash()));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(1),
verified_block_hash: blocks[0].hash(),
},
(block::Height(1), blocks[0].hash()),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config.clone(), handle.clone());
let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status(
&service,
&mut actions,
64,
block::Height(1),
blocks[0].hash(),
1,
MAX_BS_RESPONSE_BYTES,
)
.await;
send_inbound(
&inbound_tx,
BlockSyncMessage::BlocksDone {
start_height: block::Height(7),
returned: 1,
},
)
.await;
let mut saw_unsolicited_done = false;
while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await
{
if let BlockSyncAction::Misbehavior { peer, reason } = action {
if peer == peer_id && reason == BlockSyncMisbehavior::UnsolicitedDone {
saw_unsolicited_done = true;
break;
}
}
}
assert!(
saw_unsolicited_done,
"an unsolicited terminator from a connected peer is hard-scored UnsolicitedDone"
);
reactor_task.abort();
}
#[tokio::test]
async fn repeated_misbehavior_is_recorded_without_disconnecting_the_peer() {
let config = ZakuraBlockSyncConfig::default();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let probe = peer(7);
let (probe_inbound_tx, probe_inbound_rx) = framed_channel(8);
let (probe_outbound_tx, _probe_outbound_rx) = framed_channel(8);
service.add_peer(Peer::new_with_direction(
probe.clone(),
None,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
HashMap::from([(
ZAKURA_STREAM_BLOCK_SYNC,
(probe_inbound_rx, probe_outbound_tx),
)]),
CancellationToken::new(),
));
tokio::time::timeout(Duration::from_secs(1), async {
while service.peer_count() == 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("probe peer connects");
for _ in 0..8 {
send_inbound(
&probe_inbound_tx,
BlockSyncMessage::GetBlocks {
start_height: block::Height(1),
count: 1,
},
)
.await;
}
tokio::time::timeout(Duration::from_secs(2), async {
loop {
if let BlockSyncAction::Misbehavior { peer, reason } = next_action(&mut actions).await {
if peer == probe && reason == BlockSyncMisbehavior::GetBlocksSpam {
break;
}
}
}
})
.await
.expect("a misbehavior action is recorded for the spamming peer");
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
service.peer_count(),
1,
"misbehavior is record-only: a repeatedly-misbehaving peer must NOT be disconnected",
);
reactor_task.abort();
}
#[tokio::test]
async fn parked_connection_cleanup_allows_a_fresh_connection_after_cooldown() {
let config = ZakuraBlockSyncConfig::default();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(93);
let old_conn_id = 7;
let new_conn_id = 8;
handle.park_session_for_test(&peer, old_conn_id, Duration::ZERO);
assert!(matches!(
service.ordered_session_demand(
old_conn_id,
&peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
),
OrderedSessionDemand::WaitForChange(_),
));
service.remove_peer(&peer, old_conn_id);
assert!(matches!(
service.ordered_session_demand(
new_conn_id,
&peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
),
OrderedSessionDemand::OpenNow
));
reactor_task.abort();
}
#[tokio::test]
async fn same_connection_block_sync_session_waits_at_tip_then_reopens_for_new_work() {
let config = ZakuraBlockSyncConfig::default();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone());
let peer = peer(9);
let conn_id = 17;
handle.park_session_for_test(&peer, conn_id, Duration::ZERO);
let demand = service.ordered_session_demand(
conn_id,
&peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
);
let OrderedSessionDemand::WaitForChange(changed) = demand else {
panic!("a locally parked session must stay absent while block sync is at tip");
};
handle
.send_control(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta {
height: block::Height(1),
hash: block::Hash([1; 32]),
size: BlockSizeEstimate::Advertised(1_000),
}]))
.expect("the block-sync reactor is live");
tokio::time::timeout(Duration::from_secs(1), changed)
.await
.expect("new block work wakes the parked session demand");
assert!(matches!(
service.ordered_session_demand(
conn_id,
&peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
),
OrderedSessionDemand::OpenNow,
));
reactor_task.abort();
}
#[tokio::test]
async fn serving_only_coordinator_demand_keeps_block_session_available_during_fallback() {
use zakura_node_services::sync_lifecycle::{
BlockServiceDemand, HeaderServiceDemand, LifecycleEpoch, SyncServiceDemand,
};
let config = ZakuraBlockSyncConfig::default();
let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32])));
let startup = BlockSyncStartup::new(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
config.clone(),
);
let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup);
let applying = SyncServiceDemand {
header: HeaderServiceDemand::Enabled {
capability_epoch: LifecycleEpoch::new(1),
},
block: BlockServiceDemand::ServingAndApplying {
apply_epoch: LifecycleEpoch::new(2),
},
};
let (demand_tx, demand_rx) = watch::channel(applying);
let service = BlockSyncService::new_with_handle_for_test(config, handle.clone())
.with_service_demand(Some(demand_rx));
let peer = peer(10);
let conn_id = 18;
handle.park_session_for_test(&peer, conn_id, Duration::ZERO);
let OrderedSessionDemand::WaitForChange(changed) = service.ordered_session_demand(
conn_id,
&peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
) else {
panic!("native applying demand waits at tip after a local session park");
};
demand_tx
.send(SyncServiceDemand {
block: BlockServiceDemand::ServingOnly {
apply_epoch: LifecycleEpoch::new(2),
},
..applying
})
.expect("the block service retains its coordinator demand receiver");
tokio::time::timeout(Duration::from_secs(1), changed)
.await
.expect("fallback service demand wakes the parked ordered session");
assert!(matches!(
service.ordered_session_demand(
conn_id,
&peer,
ZAKURA_CAP_BLOCK_SYNC,
ServicePeerDirection::Outbound,
),
OrderedSessionDemand::OpenNow,
));
reactor_task.abort();
}