use super::*;
use std::{
collections::VecDeque,
future,
sync::{
atomic::{AtomicBool, AtomicUsize, Ordering},
Arc, Mutex,
},
time::Duration,
};
use futures::stream::{FuturesUnordered, StreamExt};
use tokio::sync::mpsc;
use tower::{
service_fn,
util::{BoxCloneService, BoxService},
Service, ServiceExt,
};
use zakura_chain::serialization::ZcashDeserializeInto;
use zakura_chain::{block, orchard, parallel::commitment_aux::BlockCommitmentRoots, sapling};
use zakura_network::zakura::testkit::{TraceCapture, TraceValue};
use zakura_network::zakura::{
commit_state_trace as cs_trace, BlockApplyResult, BlockSizeEstimate, BlockSyncAction,
BlockSyncBlockMeta, BlockSyncEvent, BlockSyncFrontiers, BlockSyncMisbehavior, BLOCK_SYNC_TABLE,
COMMIT_STATE_TABLE, DEFAULT_HS_RANGE,
};
use zakura_network::P2pStack;
use zakura_test::vectors::{
BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES, BLOCK_MAINNET_3_BYTES, BLOCK_MAINNET_4_BYTES,
BLOCK_MAINNET_5_BYTES, BLOCK_MAINNET_GENESIS_BYTES,
};
use super::zakura::{
abandoned_block_apply_finished_event, apply_block_sync_body, block_apply_class,
block_roots_cover_range, block_sync_missing_body_window, block_sync_needed_blocks_from_state,
block_verify_error_class, block_verify_error_diagnostic, coalesce_ready_needed_block_queries,
coalesce_stale_needed_block_queries, commit_block_sync_body, drive_block_sync_actions,
query_block_sync_frontiers, query_block_sync_needed_blocks, root_covered_query_best_header_tip,
verified_block_tip_from_state, BlockApplyClass, BlocksyncThroughputProbe,
ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, ZAKURA_BLOCK_SYNC_MISSING_BODY_WINDOW,
};
fn needed_blocks_query(
query_id: u64,
from: block::Height,
limit: u32,
best_header_tip: block::Height,
) -> BlockSyncAction {
BlockSyncAction::QueryNeededBlocks {
query_id: std::num::NonZeroU64::new(query_id).expect("test query ID is nonzero"),
from,
limit,
best_header_tip,
scope: zakura_header_chain::BodyWorkAuthority {
header: zakura_header_chain::HeaderWorkAuthority {
header_generation: zakura_header_chain::HeaderGeneration::new(query_id),
branch: zakura_header_chain::BranchId::new(
block::Hash([0; 32]),
block::Hash([0; 32]),
),
},
verified_generation: zakura_header_chain::VerifiedGeneration::new(query_id),
body_work_epoch: zakura_header_chain::BodyWorkEpoch::default(),
},
}
}
fn mainnet_block(bytes: &[u8]) -> Arc<block::Block> {
Arc::new(bytes.zcash_deserialize_into().expect("block vector parses"))
}
async fn wait_for_header_snapshot(
snapshots: &mut watch::Receiver<Option<zakura_header_chain::EngineSnapshot>>,
height: block::Height,
) -> zakura_header_chain::EngineSnapshot {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if let Some(snapshot) = snapshots.borrow().clone() {
if snapshot.frontiers.header_best.height == height {
return snapshot;
}
}
snapshots
.changed()
.await
.expect("the durable header snapshot publisher remains live");
}
})
.await
.expect("the durable header snapshot reaches the expected height")
}
fn durable_header_entry(
height: block::Height,
block: &Arc<block::Block>,
serialized_size: usize,
) -> zakura_node_services::header_chain::TargetEntry {
zakura_node_services::header_chain::TargetEntry {
header: block.header.clone(),
body_size: u32::try_from(serialized_size).expect("mainnet block vectors fit in u32"),
tree_aux: Some(zakura_header_chain::TreeAuxRecordV1 {
height,
sapling_root: Default::default(),
orchard_root: Default::default(),
ironwood_root: Default::default(),
sapling_tx_count: 0,
orchard_tx_count: 0,
ironwood_tx_count: 0,
auth_data_root: [0; 32].into(),
}),
}
}
async fn apply_durable_header_suffix(
startup: &zakura_network::zakura::ZakuraHeaderSyncDriverStartup,
snapshot: &zakura_header_chain::EngineSnapshot,
common_ancestor: zakura_header_chain::Frontier,
blocks: &[(block::Height, Arc<block::Block>, usize)],
session_id: u64,
request_id: u64,
) {
let (target_height, target_block, _) = blocks.last().expect("the durable suffix is non-empty");
let target = zakura_header_chain::Frontier::new(*target_height, target_block.hash());
let scope = zakura_header_chain::HeaderWorkAuthority::for_target(snapshot, target_block.hash());
let owner = scope.bind(
session_id,
std::num::NonZeroU64::new(request_id).expect("the test request ID is nonzero"),
);
let source = zakura_header_chain::SourceId::from_digest([0xd5; 32]);
let prepared = startup
.header_chain_port
.prepare_header_target(zakura_node_services::header_chain::PrepareHeaderTarget {
source,
owner: owner.into(),
common_ancestor,
target,
entries: blocks
.iter()
.map(|(height, block, size)| durable_header_entry(*height, block, *size))
.collect::<Vec<_>>()
.try_into()
.expect("the durable suffix fits the header transition cap"),
completion: zakura_header_chain::TargetCompletion::TargetComplete { common_ancestor },
})
.await
.expect("the canonical durable suffix prepares");
startup
.header_chain_port
.apply_header_target(prepared)
.await
.expect("the canonical durable suffix applies");
}
#[tokio::test(flavor = "multi_thread")]
async fn durable_header_sync_restart_requests_only_missing_suffix() {
const PHASE_ENV: &str = "ZAKURA_DURABLE_HEADER_RESTART_PHASE";
const STATE_DIR_ENV: &str = "ZAKURA_DURABLE_HEADER_RESTART_STATE_DIR";
let phase = std::env::var(PHASE_ENV).ok();
if phase.is_none() {
let temp_dir = tempfile::tempdir().expect("the durable state tempdir is created");
for phase in ["persist", "reopen"] {
let status = std::process::Command::new(
std::env::current_exe().expect("the current test executable has a path"),
)
.arg(
"commands::start::zakura_header_sync_driver_tests::\
durable_header_sync_restart_requests_only_missing_suffix",
)
.arg("--exact")
.arg("--nocapture")
.env(PHASE_ENV, phase)
.env(STATE_DIR_ENV, temp_dir.path())
.status()
.expect("the durable restart phase starts");
assert!(
status.success(),
"the durable restart {phase} phase succeeds"
);
}
return;
}
let state_dir = std::path::PathBuf::from(
std::env::var_os(STATE_DIR_ENV).expect("the parent test supplies the durable state path"),
);
let state_config = zakura_state::Config {
cache_dir: state_dir,
ephemeral: false,
debug_skip_non_finalized_state_backup_task: true,
enable_zakura_header_seed_from_committed_blocks: true,
..Default::default()
};
let network = zakura_chain::parameters::Network::Mainnet;
let genesis = mainnet_block(&BLOCK_MAINNET_GENESIS_BYTES);
let blocks = [
mainnet_block(&BLOCK_MAINNET_1_BYTES),
mainnet_block(&BLOCK_MAINNET_2_BYTES),
mainnet_block(&BLOCK_MAINNET_3_BYTES),
mainnet_block(&BLOCK_MAINNET_4_BYTES),
mainnet_block(&BLOCK_MAINNET_5_BYTES),
];
if phase.as_deref() == Some("persist") {
let (
zakura_state::InitializedState {
state: mut state_service,
read_state,
latest_chain_tip: latest_tip,
chain_tip_change: tip_change,
..
},
header_chain_authority,
) = zakura_state::init_with_header_chain_body_evidence(
state_config.clone(),
&network,
block::Height(0),
2,
)
.await
.expect("persistent test state initialization succeeds");
let committed = state_service
.ready()
.await
.expect("the persistent state is ready for genesis")
.call(zakura_state::Request::CommitCheckpointVerifiedBlock(
genesis.clone().into(),
))
.await
.expect("genesis commits to the persistent state");
assert!(matches!(
committed,
zakura_state::Response::Committed(hash) if hash == genesis.hash()
));
state_service
.ready()
.await
.expect("the persistent state performs semantic handoff");
let state = {
let state_service = Arc::new(tokio::sync::Mutex::new(state_service));
service_fn(move |request| {
let state_service = state_service.clone();
async move {
state_service
.lock()
.await
.ready()
.await?
.call(request)
.await
}
})
};
let coordinator = zakura::SyncCoordinator::new();
let startup = zakura_header_sync_driver_startup(
state.clone(),
read_state.clone(),
header_chain_authority,
&network,
&coordinator,
)
.await
.expect("the first durable header driver starts");
let mut snapshots = startup.header_chain_port.selected_chain();
let genesis_snapshot = wait_for_header_snapshot(&mut snapshots, block::Height(0)).await;
let suffix: Vec<_> = blocks[..4]
.iter()
.enumerate()
.map(|(index, block)| {
(
block::Height(u32::try_from(index + 1).expect("fixture height fits u32")),
block.clone(),
[
BLOCK_MAINNET_1_BYTES.len(),
BLOCK_MAINNET_2_BYTES.len(),
BLOCK_MAINNET_3_BYTES.len(),
BLOCK_MAINNET_4_BYTES.len(),
][index],
)
})
.collect();
apply_durable_header_suffix(
&startup,
&genesis_snapshot,
zakura_header_chain::Frontier::new(block::Height(0), genesis.hash()),
&suffix,
1,
1,
)
.await;
let committed = wait_for_header_snapshot(&mut snapshots, block::Height(4)).await;
assert_eq!(committed.frontiers.header_best.hash, blocks[3].hash());
drop(suffix);
drop(snapshots);
drop(startup);
drop(read_state);
drop(state);
drop(tip_change);
drop(latest_tip);
}
if phase.as_deref() == Some("reopen") {
let (
zakura_state::InitializedState {
state: mut state_service,
read_state,
latest_chain_tip: latest_tip,
chain_tip_change: tip_change,
..
},
header_chain_authority,
) = zakura_state::init_with_header_chain_body_evidence(
state_config,
&network,
block::Height(0),
2,
)
.await
.expect("persistent test state reopens");
state_service
.ready()
.await
.expect("the reopened state performs semantic handoff");
let state = {
let state_service = Arc::new(tokio::sync::Mutex::new(state_service));
service_fn(move |request| {
let state_service = state_service.clone();
async move {
state_service
.lock()
.await
.ready()
.await?
.call(request)
.await
}
})
};
let coordinator = zakura::SyncCoordinator::new();
let startup = zakura_header_sync_driver_startup(
state.clone(),
read_state.clone(),
header_chain_authority,
&network,
&coordinator,
)
.await
.expect("the restarted durable header driver starts");
assert_eq!(
startup.best_header_tip,
Some((block::Height(4), blocks[3].hash()))
);
let locator = startup
.header_chain_port
.continuation_locator(zakura_node_services::header_chain::LocatorReadPermit::new(
Arc::new(tokio::sync::Semaphore::new(1))
.try_acquire_owned()
.unwrap(),
))
.await
.expect("the restarted continuation locator query succeeds")
.expect("the restarted state has a selected-path locator");
assert_eq!(
locator.entries().first().copied(),
Some(zakura_header_chain::Frontier::new(
block::Height(4),
blocks[3].hash()
)),
"the restart requests only the missing suffix after block 4",
);
let mut snapshots = startup.committed_snapshots.clone();
let block_four_snapshot = wait_for_header_snapshot(&mut snapshots, block::Height(4)).await;
apply_durable_header_suffix(
&startup,
&block_four_snapshot,
zakura_header_chain::Frontier::new(block::Height(4), blocks[3].hash()),
&[(
block::Height(5),
blocks[4].clone(),
BLOCK_MAINNET_5_BYTES.len(),
)],
2,
1,
)
.await;
let committed = wait_for_header_snapshot(&mut snapshots, block::Height(5)).await;
assert_eq!(committed.frontiers.header_best.hash, blocks[4].hash());
assert_eq!(locator.entries()[0].hash, blocks[3].hash());
drop(snapshots);
drop(startup);
drop(read_state);
drop(state);
drop(tip_change);
drop(latest_tip);
}
}
fn test_block_work_owner() -> zakura_header_chain::BodyWorkOwner {
zakura_header_chain::BodyWorkAuthority {
header: zakura_header_chain::HeaderWorkAuthority {
header_generation: zakura_header_chain::HeaderGeneration::new(1),
branch: zakura_header_chain::BranchId::new(block::Hash([0; 32]), block::Hash([1; 32])),
},
verified_generation: zakura_header_chain::VerifiedGeneration::new(1),
body_work_epoch: zakura_header_chain::BodyWorkEpoch::default(),
}
.bind(
1,
std::num::NonZeroU64::new(1).expect("test request ID is nonzero"),
)
}
fn test_block_source() -> zakura_header_chain::SourceId {
zakura_header_chain::SourceId::from_digest([2; 32])
}
fn root_at(height: block::Height) -> BlockCommitmentRoots {
BlockCommitmentRoots {
height,
sapling_root: sapling::tree::NoteCommitmentTree::default().root(),
orchard_root: orchard::tree::NoteCommitmentTree::default().root(),
ironwood_root: zakura_chain::ironwood::tree::NoteCommitmentTree::default().root(),
sapling_tx: 0,
orchard_tx: 0,
ironwood_tx: 0,
auth_data_root: zakura_chain::block::merkle::AuthDataRoot::from([0u8; 32]),
}
}
fn block_sync_startup_with_snapshot(
frontiers: BlockSyncFrontiers,
best_header_tip: (block::Height, block::Hash),
header_tip: tokio::sync::watch::Receiver<(block::Height, block::Hash)>,
config: zakura_network::zakura::ZakuraBlockSyncConfig,
) -> zakura_network::zakura::BlockSyncStartup {
let mut startup = zakura_network::zakura::BlockSyncStartup::new(
frontiers,
best_header_tip,
header_tip,
(config)
.decoded_block_pool(10_000)
.expect("fixture decoded allowance fits"),
config,
);
startup.committed_views = Some(test_committed_views(frontiers, best_header_tip));
startup
}
fn block_sync_startup_for_test() -> zakura_network::zakura::BlockSyncStartup {
let (tip_tx, tip_rx) = tokio::sync::watch::channel((block::Height(0), block::Hash([0; 32])));
drop(tip_tx);
block_sync_startup_with_snapshot(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(0), block::Hash([0; 32])),
tip_rx,
zakura_network::zakura::ZakuraBlockSyncConfig::default(),
)
}
fn test_committed_views(
frontiers: BlockSyncFrontiers,
best_header_tip: (block::Height, block::Hash),
) -> tokio::sync::watch::Receiver<Option<zakura_header_chain::CommittedHeaderChainView>> {
let finalized = zakura_header_chain::Frontier::new(
frontiers.finalized_height,
frontiers.verified_block_hash,
);
let verified = zakura_header_chain::Frontier::new(
frontiers.verified_block_tip,
frontiers.verified_block_hash,
);
let header_best = zakura_header_chain::Frontier::new(best_header_tip.0, best_header_tip.1);
let snapshot = zakura_header_chain::EngineSnapshot {
mode: zakura_header_chain::EngineMode::Integrated,
state_version: zakura_header_chain::StateVersion::new(1),
header_generation: zakura_header_chain::HeaderGeneration::new(1),
verified_generation: zakura_header_chain::VerifiedGeneration::new(1),
frontiers: zakura_header_chain::FrontierSet {
finalized,
header_best,
verified_best: verified,
},
header_best_score: zakura_header_chain::ChainScore::new(
zakura_header_chain::SuffixWork::zero(),
header_best.hash,
),
oldest_retained_height: finalized.height,
alarms: Default::default(),
};
let (snapshot_tx, snapshot_rx) =
tokio::sync::watch::channel(Some(zakura_header_chain::CommittedHeaderChainView::new(
snapshot,
zakura_header_chain::BodyWorkEpoch::default(),
)));
drop(snapshot_tx);
snapshot_rx
}
fn test_zakura_peer(byte: u8) -> zakura_network::zakura::ZakuraPeerId {
zakura_network::zakura::ZakuraPeerId::new(vec![byte; 32]).expect("test peer id is valid")
}
fn fallback_read_state(
blocks: Vec<Arc<block::Block>>,
query_seen: Option<Arc<Mutex<Option<oneshot::Sender<()>>>>>,
) -> BoxCloneService<zakura_state::ReadRequest, zakura_state::ReadResponse, zakura_state::BoxError>
{
BoxCloneService::new(service_fn(move |request: zakura_state::ReadRequest| {
let blocks = blocks.clone();
let query_seen = query_seen.clone();
async move {
match request {
zakura_state::ReadRequest::MissingBlockBodyMetadata { .. } => {
if let Some(query_seen) = query_seen {
if let Some(query_seen) = query_seen
.lock()
.expect("query signal mutex is not poisoned")
.take()
{
let _ = query_seen.send(());
}
}
Ok(zakura_state::ReadResponse::MissingBlockBodyMetadata(
zakura_state::BlockSyncBodyMetadata {
anchor: zakura_header_chain::Frontier::new(
block::Height(0),
block::Hash([0; 32]),
),
blocks: Vec::new(),
},
))
}
zakura_state::ReadRequest::FinalizedTip => {
let tip = blocks
.iter()
.filter_map(|block| Some((block.coinbase_height()?, block.hash())))
.max_by_key(|(height, _hash)| *height);
Ok(zakura_state::ReadResponse::FinalizedTip(tip))
}
zakura_state::ReadRequest::Tip => {
let tip = blocks
.iter()
.filter_map(|block| Some((block.coinbase_height()?, block.hash())))
.max_by_key(|(height, _hash)| *height);
Ok(zakura_state::ReadResponse::Tip(tip))
}
request => {
panic!("unexpected read request in fallback driver test: {request:?}")
}
}
}
}))
}
fn counting_verifier(
commit_count: Arc<AtomicUsize>,
release_first: Option<Arc<tokio::sync::Notify>>,
) -> BoxCloneService<zakura_consensus::Request, block::Hash, zakura_consensus::BoxError> {
BoxCloneService::new(service_fn(move |request: zakura_consensus::Request| {
let commit_count = commit_count.clone();
let release_first = release_first.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
let height = block.coinbase_height().expect("test block has height");
commit_count.fetch_add(1, Ordering::SeqCst);
if height == block::Height(1) {
if let Some(release_first) = release_first {
release_first.notified().await;
}
}
Ok::<_, zakura_consensus::BoxError>(block.hash())
}
request => {
panic!("unexpected consensus request in fallback driver test: {request:?}")
}
}
}
}))
}
fn height_reporting_verifier(
commit_count: Arc<AtomicUsize>,
release_first: Option<Arc<tokio::sync::Notify>>,
) -> (
BoxCloneService<zakura_consensus::Request, block::Hash, zakura_consensus::BoxError>,
mpsc::Receiver<block::Height>,
) {
let (commit_tx, commit_rx) = mpsc::channel(8);
let verifier = service_fn(move |request: zakura_consensus::Request| {
let commit_tx = commit_tx.clone();
let commit_count = commit_count.clone();
let release_first = release_first.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
let height = block.coinbase_height().expect("test block has height");
commit_count.fetch_add(1, Ordering::SeqCst);
commit_tx
.send(height)
.await
.expect("test commit receiver stays open");
if height == block::Height(1) {
if let Some(release_first) = release_first {
release_first.notified().await;
}
}
Ok::<_, zakura_consensus::BoxError>(block.hash())
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
(BoxCloneService::new(verifier), commit_rx)
}
fn commit_channel_verifier() -> (
BoxCloneService<zakura_consensus::Request, block::Hash, zakura_consensus::BoxError>,
mpsc::Receiver<block::Hash>,
) {
let (commit_tx, commit_rx) = mpsc::channel(8);
let verifier = service_fn(move |request: zakura_consensus::Request| {
let commit_tx = commit_tx.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
let hash = block.hash();
commit_tx
.send(hash)
.await
.expect("test commit receiver stays open");
Ok::<_, zakura_consensus::BoxError>(hash)
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
(BoxCloneService::new(verifier), commit_rx)
}
fn fixed_tip_read_state(
height: block::Height,
hash: block::Hash,
) -> BoxCloneService<zakura_state::ReadRequest, zakura_state::ReadResponse, zakura_state::BoxError>
{
BoxCloneService::new(service_fn(
move |request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::FinalizedTip => Ok::<_, zakura_state::BoxError>(
zakura_state::ReadResponse::FinalizedTip(Some((height, hash))),
),
zakura_state::ReadRequest::Tip => {
Ok(zakura_state::ReadResponse::Tip(Some((height, hash))))
}
request => panic!("unexpected read request: {request:?}"),
}
},
))
}
fn panicking_read_state(
context: &'static str,
) -> BoxCloneService<zakura_state::ReadRequest, zakura_state::ReadResponse, zakura_state::BoxError>
{
BoxCloneService::new(service_fn(
move |request: zakura_state::ReadRequest| async move {
panic!("unexpected read request while {context}: {request:?}");
#[allow(unreachable_code)]
Ok::<_, zakura_state::BoxError>(zakura_state::ReadResponse::Tip(None))
},
))
}
fn panicking_verifier(
context: &'static str,
) -> BoxCloneService<zakura_consensus::Request, block::Hash, zakura_consensus::BoxError> {
BoxCloneService::new(service_fn(
move |request: zakura_consensus::Request| async move {
panic!("unexpected verifier request while {context}: {request:?}");
#[allow(unreachable_code)]
Ok::<_, zakura_consensus::BoxError>(block::Hash([0; 32]))
},
))
}
struct DriverParams {
header_chain_write: Option<
BoxCloneService<zakura_state::Request, zakura_state::Response, zakura_state::BoxError>,
>,
body_evidence_authority: Option<zakura_state::HeaderChainBodyEvidenceAuthority>,
max_checkpoint_height: block::Height,
checkpoint_apply_limit: usize,
full_apply_limit: usize,
combined_apply_limit: usize,
trace: zakura_network::zakura::ZakuraTrace,
throughput_probe: Option<BlocksyncThroughputProbe>,
handoff: Arc<super::zakura::SyncCoordinator>,
}
impl Default for DriverParams {
fn default() -> Self {
Self {
header_chain_write: None,
body_evidence_authority: None,
max_checkpoint_height: block::Height::MAX,
checkpoint_apply_limit: sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT,
full_apply_limit: sync::MIN_CONCURRENCY_LIMIT,
combined_apply_limit: sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT,
trace: zakura_network::zakura::ZakuraTrace::noop(),
throughput_probe: None,
handoff: super::zakura::SyncCoordinator::new(),
}
}
}
impl DriverParams {
fn spawn<ReadState, BlockVerifier>(
self,
action_rx: mpsc::Receiver<BlockSyncAction>,
block_sync: zakura_network::zakura::BlockSyncHandle,
latest_chain_tip: impl zakura_chain::chain_tip::ChainTip + Clone + Send + Sync + 'static,
read_state: ReadState,
block_verifier: BlockVerifier,
) -> (tokio::task::JoinHandle<()>, oneshot::Sender<()>)
where
ReadState: Service<
zakura_state::ReadRequest,
Response = zakura_state::ReadResponse,
Error = zakura_state::BoxError,
> + Clone
+ Send
+ 'static,
ReadState::Future: Send + 'static,
BlockVerifier:
Service<zakura_consensus::Request, Response = block::Hash> + Clone + Send + 'static,
BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static,
BlockVerifier::Future: Send + 'static,
{
let (shutdown_tx, shutdown_rx) = oneshot::channel();
let driver = tokio::spawn(drive_block_sync_actions(
action_rx,
zakura_network::zakura::ZakuraSupervisorHandle::new(1),
None,
block_sync,
latest_chain_tip,
read_state,
self.header_chain_write,
self.body_evidence_authority,
block_verifier,
self.max_checkpoint_height,
self.checkpoint_apply_limit,
self.full_apply_limit,
self.combined_apply_limit,
self.trace,
self.throughput_probe,
self.handoff,
async move {
let _ = shutdown_rx.await;
},
));
(driver, shutdown_tx)
}
}
async fn wait_for_query_seen(query_seen_rx: oneshot::Receiver<()>) {
tokio::time::timeout(Duration::from_secs(1), query_seen_rx)
.await
.expect("driver handles the barrier query")
.expect("query signal sender remains live");
}
fn assert_abandoned_apply_trace_rows(
rows: &[&serde_json::Value],
tokens: impl IntoIterator<Item = u64>,
) {
for token in tokens {
assert!(
rows.iter().any(|row| {
row.get("event").and_then(serde_json::Value::as_str)
== Some(cs_trace::REACTOR_EVENT_SENT)
&& row
.get(cs_trace::ACTION)
.and_then(serde_json::Value::as_str)
== Some("block_apply_finished")
&& row
.get(cs_trace::APPLY_TOKEN)
.and_then(serde_json::Value::as_u64)
== Some(token)
&& row
.get(cs_trace::RESULT)
.and_then(serde_json::Value::as_str)
== Some("unavailable")
}),
"missing abandoned apply trace row for token {token}; rows: {rows:?}",
);
}
}
#[test]
fn zakura_block_sync_replaces_chain_sync_when_v2_p2p_is_enabled() {
let mut config = zakura_network::Config::for_test(P2pStack::Dual);
assert!(use_zakura_block_sync(&config));
config.p2p_stack = P2pStack::Legacy;
assert!(!use_zakura_block_sync(&config));
}
#[test]
fn startup_root_backfill_gate_requires_complete_root_coverage() {
let start = block::Height(10);
let complete_roots = [
root_at(block::Height(10)),
root_at(block::Height(11)),
root_at(block::Height(12)),
];
assert!(block_roots_cover_range(start, 3, &complete_roots));
assert!(!block_roots_cover_range(start, 3, &complete_roots[..2]));
let roots_with_gap = [
root_at(block::Height(10)),
root_at(block::Height(12)),
root_at(block::Height(13)),
];
assert!(!block_roots_cover_range(start, 3, &roots_with_gap));
}
#[tokio::test]
async fn query_best_header_tip_is_capped_when_roots_are_missing() {
let verified_tip = (block::Height(0), block::Hash([0; 32]));
let durable_header_tip = (block::Height(2), block::Hash([2; 32]));
let read_state = service_fn(move |request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::Tip => {
Ok::<_, zakura_state::BoxError>(zakura_state::ReadResponse::Tip(Some(verified_tip)))
}
zakura_state::ReadRequest::BlockRoots {
start_height,
count,
} => {
assert_eq!(start_height, block::Height(1));
assert_eq!(count, 2);
Ok(zakura_state::ReadResponse::BlockRoots(Vec::new()))
}
request => panic!("unexpected read request: {request:?}"),
}
});
assert_eq!(
root_covered_query_best_header_tip(read_state, durable_header_tip)
.await
.expect("capped query succeeds"),
verified_tip
);
}
#[test]
fn block_verify_error_classifier_detects_router_and_block_errors() {
use zakura_header_chain::{BodyRuleId, BodyVerificationClass};
let hash = block::Hash([1; 32]);
let duplicate_block_error = zakura_consensus::VerifyBlockError::Block {
source: zakura_consensus::BlockError::AlreadyInChain(
hash,
zakura_state::KnownBlock::BestChain,
),
};
assert_eq!(
block_verify_error_class(&duplicate_block_error),
BodyVerificationClass::Duplicate
);
let duplicate_router_error = zakura_consensus::RouterError::Block {
source: Box::new(zakura_consensus::VerifyBlockError::Block {
source: zakura_consensus::BlockError::AlreadyInChain(
hash,
zakura_state::KnownBlock::BestChain,
),
}),
};
assert_eq!(
block_verify_error_class(&duplicate_router_error),
BodyVerificationClass::Duplicate
);
let invalid_block_error = zakura_consensus::VerifyBlockError::Block {
source: zakura_consensus::BlockError::NoTransactions,
};
assert_eq!(
block_verify_error_class(&invalid_block_error),
BodyVerificationClass::ConsensusInvalid(BodyRuleId::new("block.no_transactions"))
);
let missing_context_error = zakura_consensus::RouterError::Block {
source: Box::new(zakura_consensus::VerifyBlockError::Commit(
zakura_state::CommitBlockError::HeaderChainError {
error: "diagnostic sentinel".to_string(),
},
)),
};
let diagnostic = block_verify_error_diagnostic(&missing_context_error)
.expect("known verifier errors have a concise diagnostic chain");
assert!(diagnostic.contains("diagnostic sentinel"), "{diagnostic}");
}
#[tokio::test]
async fn block_commit_maps_unknown_local_errors_to_unavailable() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let owner = test_block_work_owner();
let source = test_block_source();
let unavailable = service_fn(|request: zakura_consensus::Request| async move {
match request {
zakura_consensus::Request::Commit(_) => {
Err::<block::Hash, zakura_consensus::BoxError>("local verifier unavailable".into())
}
request => panic!("unexpected consensus request: {request:?}"),
}
});
let unavailable_outcome = commit_block_sync_body(
unavailable,
owner,
source,
block.clone().into(),
BlockApplyClass::Full,
)
.await;
assert!(matches!(
unavailable_outcome.verification(),
zakura_header_chain::BodyVerificationOutcome::Retryable(
zakura_header_chain::TransientBodyFailure {
kind: zakura_header_chain::TransientBodyFailureKind::VerifierUnavailable,
..
}
)
));
let invalid = service_fn(|request: zakura_consensus::Request| async move {
match request {
zakura_consensus::Request::Commit(_) => {
Err::<block::Hash, zakura_consensus::VerifyBlockError>(
zakura_consensus::VerifyBlockError::Block {
source: zakura_consensus::BlockError::NoTransactions,
},
)
}
request => panic!("unexpected consensus request: {request:?}"),
}
});
let invalid_outcome =
commit_block_sync_body(invalid, owner, source, block.into(), BlockApplyClass::Full).await;
assert!(matches!(
invalid_outcome.verification(),
zakura_header_chain::BodyVerificationOutcome::ConsensusInvalid(
zakura_header_chain::ConsensusBodyInvalid { rule, .. }
)
if rule == &zakura_header_chain::BodyRuleId::new("block.no_transactions")
));
}
#[test]
fn block_sync_missing_body_window_stays_inside_body_sync_bound() {
assert_eq!(
block_sync_missing_body_window(block::Height(11), block::Height(10), 10),
None
);
assert_eq!(
block_sync_missing_body_window(block::Height(11), block::Height(12), 10),
Some((block::Height(11), 2))
);
assert_eq!(
block_sync_missing_body_window(
block::Height(11),
block::Height(10 + ZAKURA_BLOCK_SYNC_MISSING_BODY_WINDOW + 100),
ZAKURA_BLOCK_SYNC_MISSING_BODY_WINDOW + 100,
),
Some((block::Height(11), ZAKURA_BLOCK_SYNC_MISSING_BODY_WINDOW))
);
assert_eq!(
block_sync_missing_body_window(block::Height(u32::MAX), block::Height(u32::MAX), 10),
Some((block::Height(u32::MAX), 1))
);
assert_eq!(
block_sync_missing_body_window(block::Height(u32::MAX), block::Height(u32::MAX), 0),
None
);
}
#[test]
fn block_sync_needed_blocks_align_missing_hashes_and_size_hints() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let needed = block_sync_needed_blocks_from_state(vec![
(block::Height(1), block1.hash(), Some(0)),
(block::Height(2), block2.hash(), Some(42)),
]);
assert_eq!(
needed,
vec![
BlockSyncBlockMeta {
height: block::Height(1),
hash: block1.hash(),
size: BlockSizeEstimate::Unknown,
},
BlockSyncBlockMeta {
height: block::Height(2),
hash: block2.hash(),
size: BlockSizeEstimate::Advertised(42),
},
]
);
}
#[tokio::test]
async fn block_sync_needed_blocks_chunks_state_range_reads() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let hash = block.hash();
let requests = Arc::new(Mutex::new(Vec::new()));
let read_state = {
let requests = Arc::clone(&requests);
service_fn(move |request| {
let requests = Arc::clone(&requests);
async move {
match request {
zakura_state::ReadRequest::MissingBlockBodyMetadata { from, limit } => {
requests
.lock()
.expect("request capture mutex is not poisoned")
.push(("metadata", from, limit));
let blocks = (0..limit)
.filter_map(|offset| {
from.0
.checked_add(offset)
.map(|height| (block::Height(height), hash, Some(32)))
})
.collect();
Ok::<_, zakura_state::BoxError>(
zakura_state::ReadResponse::MissingBlockBodyMetadata(
zakura_state::BlockSyncBodyMetadata {
anchor: zakura_header_chain::Frontier::new(
block::Height(0),
block::Hash([0; 32]),
),
blocks,
},
),
)
}
request => panic!("unexpected read request: {request:?}"),
}
}
})
};
let count = zakura_state::constants::MAX_HEADER_SYNC_HEIGHT_RANGE + 2;
let (_anchor, needed) = query_block_sync_needed_blocks(read_state, block::Height(1), count)
.await
.expect("mock read state succeeds");
assert_eq!(
needed.len(),
usize::try_from(count).expect("test count fits usize")
);
assert_eq!(
requests
.lock()
.expect("request capture mutex is not poisoned")
.as_slice(),
&[
(
"metadata",
block::Height(1),
zakura_state::constants::MAX_HEADER_SYNC_HEIGHT_RANGE,
),
("metadata", block::Height(4001), 2),
]
);
}
#[test]
fn verified_block_tip_from_state_prefers_highest_frontier_with_matching_hash() {
let empty = (block::Height(0), block::Hash([0; 32]));
assert_eq!(
verified_block_tip_from_state(
Some((block::Height(2800), block::Hash([28; 32]))),
Some((block::Height(2561), block::Hash([25; 32]))),
empty,
),
(block::Height(2800), block::Hash([28; 32]))
);
assert_eq!(
verified_block_tip_from_state(
Some((block::Height(2400), block::Hash([24; 32]))),
Some((block::Height(2801), block::Hash([29; 32]))),
empty,
),
(block::Height(2801), block::Hash([29; 32]))
);
assert_eq!(
verified_block_tip_from_state(None, None, empty),
(block::Height(0), block::Hash([0; 32]))
);
assert_eq!(
verified_block_tip_from_state(None, Some((block::Height(3), block::Hash([3; 32]))), empty),
(block::Height(3), block::Hash([3; 32]))
);
}
#[test]
fn block_apply_class_checkpoint_boundary_is_inclusive() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
assert_eq!(
block_apply_class(&block1, block::Height(1)),
BlockApplyClass::Checkpoint
);
assert_eq!(
block_apply_class(&block2, block::Height(1)),
BlockApplyClass::Full
);
}
#[test]
fn zakura_checkpoint_static_limits_cover_checkpoint_gap() {
const {
assert!(
sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT >= zakura_consensus::MAX_CHECKPOINT_HEIGHT_GAP
);
}
assert!(
usize::try_from(DEFAULT_HS_RANGE)
.expect("DEFAULT_HS_RANGE fits usize on supported targets")
>= zakura_consensus::MAX_CHECKPOINT_HEIGHT_GAP
);
assert!(
usize::try_from(zakura_state::MAX_BLOCK_REORG_HEIGHT)
.expect("MAX_BLOCK_REORG_HEIGHT fits usize on supported targets")
>= zakura_consensus::MAX_CHECKPOINT_HEIGHT_GAP
);
}
#[tokio::test]
async fn block_sync_driver_coalesces_stale_needed_queries() {
let (action_tx, mut action_rx) = mpsc::channel(8);
action_tx
.send(needed_blocks_query(
1,
block::Height(1),
1,
block::Height(1),
))
.await
.expect("first query queues");
action_tx
.send(needed_blocks_query(
2,
block::Height(1),
2,
block::Height(2),
))
.await
.expect("stale query queues");
let deferred_peer =
zakura_network::zakura::ZakuraPeerId::new(vec![7; 32]).expect("test peer id is valid");
action_tx
.send(BlockSyncAction::Misbehavior {
peer: deferred_peer.clone(),
reason: BlockSyncMisbehavior::StatusSpam,
})
.await
.expect("non-query action queues");
action_tx
.send(needed_blocks_query(
3,
block::Height(3),
6,
block::Height(8),
))
.await
.expect("latest query queues");
let first = action_rx.recv().await.expect("first action remains queued");
let mut deferred_actions = VecDeque::new();
let action = coalesce_stale_needed_block_queries(first, &mut action_rx, &mut deferred_actions);
assert!(matches!(
action,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(3),
limit: 6,
best_header_tip: block::Height(8),
..
}
));
assert!(matches!(
deferred_actions.pop_front(),
Some(BlockSyncAction::Misbehavior {
peer,
reason: BlockSyncMisbehavior::StatusSpam,
}) if peer == deferred_peer
));
assert!(deferred_actions.is_empty());
assert!(action_rx.try_recv().is_err());
}
#[tokio::test]
async fn block_sync_driver_prioritizes_ready_submit_over_needed_query() {
let (action_tx, mut action_rx) = mpsc::channel(8);
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 7,
block: block.into(),
})
.await
.expect("submit action queues");
action_tx
.send(needed_blocks_query(
1,
block::Height(1),
8,
block::Height(8),
))
.await
.expect("query action queues");
let mut deferred_actions = VecDeque::new();
assert!(coalesce_ready_needed_block_queries(&mut action_rx, &mut deferred_actions).is_none());
assert!(matches!(
deferred_actions.pop_front(),
Some(BlockSyncAction::SubmitBlock { token: 7, .. })
));
assert!(matches!(
deferred_actions.pop_front(),
Some(BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
limit: 8,
best_header_tip: block::Height(8),
..
})
));
assert!(deferred_actions.is_empty());
assert!(action_rx.try_recv().is_err());
}
#[tokio::test]
async fn block_sync_driver_does_not_refill_a_nonempty_deferred_queue() {
let (action_tx, mut action_rx) = mpsc::channel(2);
for marker in [1, 2] {
action_tx
.send(BlockSyncAction::Misbehavior {
peer: test_zakura_peer(marker),
reason: BlockSyncMisbehavior::StatusSpam,
})
.await
.expect("the initial bounded action batch queues");
}
let mut deferred_actions = VecDeque::new();
assert!(coalesce_ready_needed_block_queries(&mut action_rx, &mut deferred_actions).is_none());
assert_eq!(deferred_actions.len(), 2);
let _dispatched = deferred_actions.pop_front();
action_tx
.send(BlockSyncAction::Misbehavior {
peer: test_zakura_peer(3),
reason: BlockSyncMisbehavior::StatusSpam,
})
.await
.expect("the bounded receiver refills while deferred work remains");
assert!(coalesce_ready_needed_block_queries(&mut action_rx, &mut deferred_actions).is_none());
assert_eq!(
deferred_actions.len(),
1,
"a nonempty deferred queue must apply backpressure to the receiver"
);
assert!(matches!(
action_rx.try_recv(),
Ok(BlockSyncAction::Misbehavior { peer, .. }) if peer == test_zakura_peer(3)
));
}
#[tokio::test]
async fn block_sync_driver_persists_exact_body_retry_evidence() {
let (action_tx, action_rx) = mpsc::channel(1);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let read_state = panicking_read_state("persisting body retry evidence");
let verifier = panicking_verifier("persisting body retry evidence");
let (request_tx, mut request_rx) = mpsc::channel(1);
let header_chain_write =
BoxCloneService::new(service_fn(move |request: zakura_state::Request| {
let request_tx = request_tx.clone();
async move {
let zakura_state::Request::RecordHeaderChainBodyUnavailable { prepared } = request
else {
panic!("unexpected state write while persisting body retry evidence")
};
request_tx
.send(prepared)
.await
.expect("body retry request receiver stays open");
Err::<zakura_state::Response, zakura_state::BoxError>(
"test writer stops after recording the request".into(),
)
}
}));
let body_evidence_authority = zakura_state::HeaderChainBodyEvidenceAuthority::new_test();
let (driver, shutdown_tx) = DriverParams {
header_chain_write: Some(header_chain_write),
body_evidence_authority: Some(body_evidence_authority.clone()),
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
let expected_version = zakura_header_chain::StateVersion::new(7);
let failure = zakura_header_chain::TransientBodyFailure {
hash: block::Hash([3; 32]),
evidence: zakura_header_chain::EvidenceId::from_digest([4; 32]),
kind: zakura_header_chain::TransientBodyFailureKind::Storage,
availability: zakura_header_chain::BodyUnavailableSummary {
attempts: 3,
suppliers: 2,
alarmed: false,
..Default::default()
},
};
action_tx
.send(BlockSyncAction::RecordBodyUnavailable {
expected_version,
failure,
})
.await
.expect("body retry persistence action queues");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), request_rx.recv())
.await
.expect("state writer receives body retry evidence"),
Some(body_evidence_authority.from_registered_attempt(expected_version, failure,))
);
let _ = shutdown_tx.send(());
driver.await.expect("driver exits after shutdown");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_persists_consensus_invalid_body_evidence() {
let (action_tx, action_rx) = mpsc::channel(1);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let read_state = panicking_read_state("persisting invalid-body evidence");
let verifier = panicking_verifier("persisting invalid-body evidence");
let (request_tx, mut request_rx) = mpsc::channel(1);
let header_chain_write =
BoxCloneService::new(service_fn(move |request: zakura_state::Request| {
let request_tx = request_tx.clone();
async move {
let zakura_state::Request::RecordHeaderChainBodyInvalid { prepared } = request
else {
panic!("unexpected state write while persisting invalid-body evidence")
};
request_tx
.send(prepared)
.await
.expect("invalid-body request receiver stays open");
Ok::<_, zakura_state::BoxError>(
zakura_state::Response::HeaderChainBodyInvalidRecorded(
zakura_header_chain::ApplyResult::Committed,
),
)
}
}));
let body_evidence_authority = zakura_state::HeaderChainBodyEvidenceAuthority::new_test();
let (driver, shutdown_tx) = DriverParams {
header_chain_write: Some(header_chain_write),
body_evidence_authority: Some(body_evidence_authority.clone()),
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
let expected_version = zakura_header_chain::StateVersion::new(7);
let invalid = zakura_header_chain::ConsensusBodyInvalid {
hash: block::Hash([3; 32]),
evidence: zakura_header_chain::EvidenceId::from_digest([4; 32]),
rule: zakura_header_chain::BodyRuleId::new("test.commitment_matching_invalid"),
source: zakura_header_chain::SourceId::from_digest([5; 32]),
};
action_tx
.send(BlockSyncAction::RecordBodyInvalid {
expected_version,
invalid: invalid.clone(),
})
.await
.expect("invalid-body persistence action queues");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), request_rx.recv())
.await
.expect("state writer receives invalid-body evidence"),
Some(body_evidence_authority.from_full_verifier(expected_version, invalid))
);
let _ = shutdown_tx.send(());
driver.await.expect("driver exits after shutdown");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_refreshes_stale_consensus_invalid_body_evidence() {
let (action_tx, action_rx) = mpsc::channel(1);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let read_state = panicking_read_state("refreshing invalid-body evidence");
let verifier = panicking_verifier("refreshing invalid-body evidence");
let (request_tx, mut request_rx) = mpsc::channel(2);
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let attempts_for_service = attempts.clone();
let current_version = zakura_header_chain::StateVersion::new(11);
let header_chain_write =
BoxCloneService::new(service_fn(move |request: zakura_state::Request| {
let request_tx = request_tx.clone();
let attempts = attempts_for_service.clone();
async move {
let zakura_state::Request::RecordHeaderChainBodyInvalid { prepared } = request
else {
panic!("unexpected state write while refreshing invalid-body evidence")
};
request_tx
.send(prepared)
.await
.expect("invalid-body request receiver stays open");
let attempt = attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if attempt == 0 {
return Ok(zakura_state::Response::HeaderChainBodyInvalidRecorded(
zakura_header_chain::ApplyResult::Stale(
zakura_header_chain::StaleReceipt {
current_version,
branch: None,
},
),
));
}
Ok(zakura_state::Response::HeaderChainBodyInvalidRecorded(
zakura_header_chain::ApplyResult::Committed,
))
}
}));
let body_evidence_authority = zakura_state::HeaderChainBodyEvidenceAuthority::new_test();
let (driver, shutdown_tx) = DriverParams {
header_chain_write: Some(header_chain_write),
body_evidence_authority: Some(body_evidence_authority.clone()),
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
let stale_version = zakura_header_chain::StateVersion::new(7);
let invalid = zakura_header_chain::ConsensusBodyInvalid {
hash: block::Hash([6; 32]),
evidence: zakura_header_chain::EvidenceId::from_digest([7; 32]),
rule: zakura_header_chain::BodyRuleId::new("test.stale_refresh_invalid"),
source: zakura_header_chain::SourceId::from_digest([8; 32]),
};
action_tx
.send(BlockSyncAction::RecordBodyInvalid {
expected_version: stale_version,
invalid: invalid.clone(),
})
.await
.expect("invalid-body persistence action queues");
let first = tokio::time::timeout(Duration::from_secs(1), request_rx.recv())
.await
.expect("state writer receives the stale invalid-body attempt")
.expect("stale invalid-body request arrives");
assert_eq!(
first,
body_evidence_authority.from_full_verifier(stale_version, invalid.clone())
);
let second = tokio::time::timeout(Duration::from_secs(1), request_rx.recv())
.await
.expect("state writer receives the refreshed invalid-body attempt")
.expect("refreshed invalid-body request arrives");
assert_eq!(
second,
body_evidence_authority.from_full_verifier(current_version, invalid)
);
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 2);
let _ = shutdown_tx.send(());
driver.await.expect("driver exits after shutdown");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_fail_closes_when_consensus_invalid_body_persistence_errors() {
let (action_tx, action_rx) = mpsc::channel(1);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let read_state = panicking_read_state("fail-closing invalid-body evidence");
let verifier = panicking_verifier("fail-closing invalid-body evidence");
let header_chain_write =
BoxCloneService::new(service_fn(|request: zakura_state::Request| async move {
let zakura_state::Request::RecordHeaderChainBodyInvalid { .. } = request else {
panic!("unexpected state write while fail-closing invalid-body evidence")
};
Err::<zakura_state::Response, zakura_state::BoxError>(
"injected invalid-body persistence failure".into(),
)
}));
let body_evidence_authority = zakura_state::HeaderChainBodyEvidenceAuthority::new_test();
let (driver, _shutdown_tx) = DriverParams {
header_chain_write: Some(header_chain_write),
body_evidence_authority: Some(body_evidence_authority),
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::RecordBodyInvalid {
expected_version: zakura_header_chain::StateVersion::new(7),
invalid: zakura_header_chain::ConsensusBodyInvalid {
hash: block::Hash([9; 32]),
evidence: zakura_header_chain::EvidenceId::from_digest([10; 32]),
rule: zakura_header_chain::BodyRuleId::new("test.fail_closed_invalid"),
source: zakura_header_chain::SourceId::from_digest([11; 32]),
},
})
.await
.expect("invalid-body persistence action queues");
tokio::time::timeout(Duration::from_secs(2), driver)
.await
.expect("driver fail-closes without waiting for external shutdown")
.expect("driver task joins after fail-closed shutdown");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_answers_needed_block_queries_from_state() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let expected_metadata = Arc::new(vec![
(block::Height(1), block1.hash(), Some(0)),
(block::Height(2), block2.hash(), Some(42)),
]);
let read_requests = Arc::new(Mutex::new(Vec::new()));
let read_requests_for_service = read_requests.clone();
let read_metadata = expected_metadata.clone();
let block1_hash = block1.hash();
let read_state = service_fn(move |request: zakura_state::ReadRequest| {
let read_requests = read_requests_for_service.clone();
let read_metadata = read_metadata.clone();
async move {
read_requests
.lock()
.expect("test read request log is not poisoned")
.push(request.clone());
match request {
zakura_state::ReadRequest::MissingBlockBodyMetadata { from, limit } => {
assert_eq!(from, block::Height(1));
assert_eq!(limit, 2);
Ok(zakura_state::ReadResponse::MissingBlockBodyMetadata(
zakura_state::BlockSyncBodyMetadata {
anchor: zakura_header_chain::Frontier::new(
block::Height(0),
block::Hash([0; 32]),
),
blocks: (*read_metadata).clone(),
},
))
}
zakura_state::ReadRequest::FinalizedTip => {
Ok(zakura_state::ReadResponse::FinalizedTip(None))
}
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block::Height(1),
block1_hash,
)))),
request => panic!("unexpected read request: {request:?}"),
}
}
});
let (verifier, mut commit_rx) = commit_channel_verifier();
let (driver, shutdown_tx) = DriverParams::default().spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(needed_blocks_query(
1,
block::Height(1),
2,
block::Height(2),
))
.await
.expect("driver action channel stays open");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if !read_requests
.lock()
.expect("test read request log is not poisoned")
.is_empty()
{
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("driver answers QueryNeededBlocks through state reads");
assert_eq!(
read_requests
.lock()
.expect("test read request log is not poisoned")
.as_slice(),
&[zakura_state::ReadRequest::MissingBlockBodyMetadata {
from: block::Height(1),
limit: 2,
},]
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("commit arrives after query"),
Some(block1.hash())
);
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_reports_failed_needed_query_for_retry() {
let (action_tx, action_rx) = mpsc::channel(8);
let (tip_tx, tip_rx) = watch::channel((block::Height(2), block::Hash([2; 32])));
drop(tip_tx);
let startup = block_sync_startup_with_snapshot(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), block::Hash([2; 32])),
tip_rx,
zakura_network::zakura::ZakuraBlockSyncConfig::default(),
);
let (block_sync, mut reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let first_query = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv())
.await
.expect("startup needed-body query arrives")
.expect("reactor action channel stays open");
let (first_query_id, first_scope) = match &first_query {
BlockSyncAction::QueryNeededBlocks {
query_id, scope, ..
} => (*query_id, *scope),
action => panic!("expected startup needed-body query, got {action:?}"),
};
let read_state = service_fn(|request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::MissingBlockBodyMetadata { .. } => {
Err::<zakura_state::ReadResponse, zakura_state::BoxError>(
std::io::Error::other("simulated needed-body query failure").into(),
)
}
request => panic!("unexpected read request: {request:?}"),
}
});
let (verifier, _commit_rx) = commit_channel_verifier();
let (driver, shutdown_tx) = DriverParams::default().spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(first_query)
.await
.expect("driver action channel stays open");
let retry = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv())
.await
.expect("failed needed-body query retries")
.expect("reactor action channel stays open");
assert!(matches!(
retry,
BlockSyncAction::QueryNeededBlocks {
query_id,
scope,
..
} if query_id != first_query_id && scope == first_scope
));
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_stops_when_needed_query_failure_receiver_closes() {
let (action_tx, action_rx) = mpsc::channel(1);
let (tip_tx, tip_rx) = watch::channel((block::Height(2), block::Hash([2; 32])));
drop(tip_tx);
let startup = block_sync_startup_with_snapshot(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(2), block::Hash([2; 32])),
tip_rx,
zakura_network::zakura::ZakuraBlockSyncConfig::default(),
);
let (block_sync, mut reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let first_query = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv())
.await
.expect("startup needed-body query arrives")
.expect("reactor action channel stays open");
reactor_task.abort();
reactor_task
.await
.expect_err("the test aborts the block-sync reactor");
let read_state = service_fn(|request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::MissingBlockBodyMetadata { .. } => {
Err::<zakura_state::ReadResponse, zakura_state::BoxError>(
std::io::Error::other("simulated needed-body query failure").into(),
)
}
request => panic!("unexpected read request: {request:?}"),
}
});
let (verifier, _commit_rx) = commit_channel_verifier();
let (driver, _shutdown_tx) = DriverParams::default().spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(first_query)
.await
.expect("driver action channel stays open");
tokio::time::timeout(Duration::from_secs(1), driver)
.await
.expect("closed completion receiver stops the driver")
.expect("driver task exits without panicking");
}
#[tokio::test]
async fn block_sync_driver_throughput_probe_advances_without_consensus_commit() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let (_tip_tx, tip_rx) = tokio::sync::watch::channel((block::Height(3), block::Hash([3; 32])));
let mut capture =
TraceCapture::for_test("block_sync_driver_throughput_probe_advances").unwrap();
let trace = zakura_network::zakura::ZakuraTrace::new(capture.tracer(), "01");
let initial_frontiers = BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
};
let mut startup = block_sync_startup_with_snapshot(
initial_frontiers,
(block::Height(3), block::Hash([3; 32])),
tip_rx,
zakura_network::zakura::ZakuraBlockSyncConfig::default(),
);
startup.trace = trace.clone();
let (block_sync, mut reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let (probe, mut completion_rx) =
BlocksyncThroughputProbe::new(initial_frontiers, block::Height(2));
let read_requests = Arc::new(Mutex::new(Vec::new()));
let read_requests_for_service = read_requests.clone();
let read_block1 = block1.clone();
let read_block2 = block2.clone();
let read_state = service_fn(move |request: zakura_state::ReadRequest| {
let read_requests = read_requests_for_service.clone();
let read_block1 = read_block1.clone();
let read_block2 = read_block2.clone();
async move {
read_requests
.lock()
.expect("test read request log is not poisoned")
.push(request.clone());
match request {
zakura_state::ReadRequest::MissingBlockBodyMetadata { from, limit } => {
assert_eq!(from, block::Height(1));
assert_eq!(limit, 3);
Ok(zakura_state::ReadResponse::MissingBlockBodyMetadata(
zakura_state::BlockSyncBodyMetadata {
anchor: zakura_header_chain::Frontier::new(
block::Height(0),
block::Hash([0; 32]),
),
blocks: vec![
(block::Height(1), read_block1.hash(), None),
(block::Height(2), read_block2.hash(), None),
],
},
))
}
request => panic!("unexpected read request in throughput probe: {request:?}"),
}
}
});
let commit_count = Arc::new(AtomicUsize::new(0));
let verifier = counting_verifier(commit_count.clone(), None);
let (driver, shutdown_tx) = DriverParams {
trace: trace.clone(),
throughput_probe: Some(probe),
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
let startup_action = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv())
.await
.expect("reactor emits startup body query")
.expect("reactor action channel stays open");
action_tx
.send(startup_action)
.await
.expect("driver action channel stays open");
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if !read_requests
.lock()
.expect("test read request log is not poisoned")
.is_empty()
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("driver records expected throughput metadata from state");
assert_eq!(
read_requests
.lock()
.expect("test read request log is not poisoned")
.as_slice(),
&[zakura_state::ReadRequest::MissingBlockBodyMetadata {
from: block::Height(1),
limit: 3,
},]
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block2.clone().into(),
})
.await
.expect("driver action channel stays open");
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.clone().into(),
})
.await
.expect("driver action channel stays open");
let summary = tokio::time::timeout(Duration::from_secs(1), &mut completion_rx)
.await
.expect("throughput probe completes after second body")
.expect("completion sender remains live");
assert_eq!(summary.verified_block_tip, block::Height(2));
assert_eq!(commit_count.load(Ordering::SeqCst), 0);
capture.flush().await;
let reader = capture.reader().unwrap();
let block_sync_trace = reader.table(BLOCK_SYNC_TABLE.table());
let chain_tip_grow_events = block_sync_trace
.rows()
.into_iter()
.filter(|row| {
row.get("event").and_then(serde_json::Value::as_str) == Some("block_event_received")
&& row.get("kind").and_then(serde_json::Value::as_str) == Some("chain_tip_grow")
})
.count();
let apply_finished_events = block_sync_trace
.rows()
.into_iter()
.filter(|row| {
row.get("event").and_then(serde_json::Value::as_str) == Some("block_event_received")
&& row.get("kind").and_then(serde_json::Value::as_str)
== Some("block_apply_finished")
})
.count();
assert_eq!(
chain_tip_grow_events, 0,
"throughput probe should advance via apply-finished local frontier only"
);
assert!(
apply_finished_events >= 2,
"throughput probe should still send apply-finished events"
);
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
let _ = capture.finish().await.unwrap();
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_commits_parent_first_and_ignores_outbound_actions() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let (verifier, mut commit_rx) = commit_channel_verifier();
let read_state = fixed_tip_read_state(block::Height(2), block2.hash());
let (driver, shutdown_tx) = DriverParams::default().spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
let peer =
zakura_network::zakura::ZakuraPeerId::new(vec![8; 32]).expect("test peer id is valid");
action_tx
.send(BlockSyncAction::Misbehavior {
peer,
reason: zakura_network::zakura::BlockSyncMisbehavior::InvalidBlock,
})
.await
.expect("driver action channel stays open");
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.clone().into(),
})
.await
.expect("driver action channel stays open");
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block2.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("first commit arrives"),
Some(block1.hash())
);
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("second commit arrives"),
Some(block2.hash())
);
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_submits_checkpoint_blocks_without_waiting_for_first_commit() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let release_first = Arc::new(tokio::sync::Notify::new());
let (verifier, mut commit_rx) =
height_reporting_verifier(Arc::new(AtomicUsize::new(0)), Some(release_first.clone()));
let read_state = fixed_tip_read_state(block::Height(2), block2.hash());
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(2),
checkpoint_apply_limit: 2,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("first checkpoint commit starts"),
Some(block::Height(1))
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block2.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("second checkpoint commit starts while first is pending"),
Some(block::Height(2))
);
release_first.notify_waiters();
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_combined_apply_limit_preserves_checkpoint_window() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let release_first = Arc::new(tokio::sync::Notify::new());
let (verifier, mut commit_rx) =
height_reporting_verifier(Arc::new(AtomicUsize::new(0)), Some(release_first.clone()));
let read_state = fixed_tip_read_state(block::Height(2), block2.hash());
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(2),
combined_apply_limit: 1,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("first checkpoint commit starts"),
Some(block::Height(1))
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block2.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("second checkpoint commit starts despite the smaller combined cap"),
Some(block::Height(2)),
"checkpoint applies need enough depth to complete a checkpoint verifier window",
);
release_first.notify_waiters();
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_combined_apply_limit_binds_full_applies() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let release_first = Arc::new(tokio::sync::Notify::new());
let (verifier, mut commit_rx) =
height_reporting_verifier(Arc::new(AtomicUsize::new(0)), Some(release_first.clone()));
let read_state = fixed_tip_read_state(block::Height(2), block2.hash());
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(0),
checkpoint_apply_limit: 2,
full_apply_limit: 2,
combined_apply_limit: 1,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("first checkpoint commit starts"),
Some(block::Height(1))
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block2.clone().into(),
})
.await
.expect("driver action channel stays open");
assert!(
tokio::time::timeout(Duration::from_millis(50), commit_rx.recv())
.await
.is_err(),
"combined apply cap must bind full applies",
);
release_first.notify_one();
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("second checkpoint commit starts after combined cap has room"),
Some(block::Height(2)),
);
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn fallback_yield_abandons_new_submit_blocks_without_verifying() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let mut capture =
TraceCapture::for_test("fallback_yield_abandons_new_submit_blocks_without_verifying")
.unwrap();
let trace = zakura_network::zakura::ZakuraTrace::new(capture.tracer(), "01");
let mut startup = block_sync_startup_for_test();
startup.trace = trace.clone();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let commit_count = Arc::new(AtomicUsize::new(0));
let verifier = counting_verifier(commit_count.clone(), None);
let handoff = super::zakura::SyncCoordinator::new();
let _fallback_lease = handoff
.acquire_legacy_fallback(Duration::from_secs(1))
.await
.expect("idle native applies yield to fallback");
let (query_seen_tx, query_seen_rx) = oneshot::channel();
let query_seen = Arc::new(Mutex::new(Some(query_seen_tx)));
let read_state = fallback_read_state(vec![block.clone()], Some(query_seen));
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(0),
full_apply_limit: 1,
combined_apply_limit: 1,
trace: trace.clone(),
handoff,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 77,
block: block.clone().into(),
})
.await
.expect("driver action channel stays open");
action_tx
.send(needed_blocks_query(
1,
block::Height(1),
1,
block::Height(1),
))
.await
.expect("driver action channel stays open");
wait_for_query_seen(query_seen_rx).await;
assert_eq!(
commit_count.load(Ordering::SeqCst),
0,
"post-fallback submissions must not call the verifier"
);
capture.flush().await;
let reader = capture.reader().unwrap();
let commit_state = reader.table(COMMIT_STATE_TABLE.table());
let rows = commit_state.rows();
assert_abandoned_apply_trace_rows(&rows, [77]);
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn fallback_yield_releases_queued_submit_blocks() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let mut capture =
TraceCapture::for_test("fallback_yield_releases_queued_submit_blocks").unwrap();
let trace = zakura_network::zakura::ZakuraTrace::new(capture.tracer(), "01");
let mut startup = block_sync_startup_for_test();
startup.trace = trace.clone();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let commit_count = Arc::new(AtomicUsize::new(0));
let release_first = Arc::new(tokio::sync::Notify::new());
let verifier = counting_verifier(commit_count.clone(), Some(release_first.clone()));
let (query_seen_tx, query_seen_rx) = oneshot::channel();
let query_seen = Arc::new(Mutex::new(Some(query_seen_tx)));
let read_state = fallback_read_state(vec![block1.clone(), block2.clone()], Some(query_seen));
let handoff = super::zakura::SyncCoordinator::new();
let drain_handoff = handoff.clone();
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(0),
full_apply_limit: 1,
combined_apply_limit: 1,
trace: trace.clone(),
handoff,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.clone().into(),
})
.await
.expect("driver action channel stays open");
tokio::time::timeout(Duration::from_secs(1), async {
while commit_count.load(Ordering::SeqCst) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("first commit starts");
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block2.clone().into(),
})
.await
.expect("second block queues behind the full apply limit");
let drain = tokio::spawn(async move {
drain_handoff
.acquire_legacy_fallback(Duration::from_secs(5))
.await
});
let _fallback_lease = tokio::time::timeout(Duration::from_secs(1), drain)
.await
.expect("fallback does not wait for the in-flight apply")
.expect("fallback drain task exits")
.expect("fallback acquires the lease after transferring the apply");
release_first.notify_waiters();
action_tx
.send(needed_blocks_query(
1,
block::Height(1),
1,
block::Height(1),
))
.await
.expect("driver action channel stays open");
wait_for_query_seen(query_seen_rx).await;
assert_eq!(
commit_count.load(Ordering::SeqCst),
1,
"fallback must not start the queued body after yielding"
);
capture.flush().await;
let reader = capture.reader().unwrap();
let commit_state = reader.table(COMMIT_STATE_TABLE.table());
let rows = commit_state.rows();
assert_abandoned_apply_trace_rows(&rows, [1, 2]);
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn fallback_yield_handles_submit_storm_without_restarting_applies() {
const SUBMIT_COUNT: u64 = 128;
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(256);
let mut capture =
TraceCapture::for_test("fallback_yield_handles_submit_storm_without_restarting_applies")
.unwrap();
let trace = zakura_network::zakura::ZakuraTrace::new(capture.tracer(), "01");
let mut startup = block_sync_startup_for_test();
startup.trace = trace.clone();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let commit_count = Arc::new(AtomicUsize::new(0));
let verifier = counting_verifier(commit_count.clone(), None);
let (query_seen_tx, query_seen_rx) = oneshot::channel();
let query_seen = Arc::new(Mutex::new(Some(query_seen_tx)));
let read_state = fallback_read_state(vec![block1.clone(), block2.clone()], Some(query_seen));
let handoff = super::zakura::SyncCoordinator::new();
let _fallback_lease = handoff
.acquire_legacy_fallback(Duration::from_secs(1))
.await
.expect("idle native applies yield to fallback");
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(0),
full_apply_limit: 1,
combined_apply_limit: 1,
trace: trace.clone(),
handoff,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
for token in 1..=SUBMIT_COUNT {
let block = if token % 2 == 0 {
block2.clone()
} else {
block1.clone()
};
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token,
block: block.into(),
})
.await
.expect("driver action channel stays open during submit storm");
}
action_tx
.send(needed_blocks_query(
1,
block::Height(1),
1,
block::Height(1),
))
.await
.expect("driver action channel handles the barrier query after submit storm");
wait_for_query_seen(query_seen_rx).await;
assert_eq!(
commit_count.load(Ordering::SeqCst),
0,
"post-fallback submit storm must not restart Zakura applies"
);
capture.flush().await;
let reader = capture.reader().unwrap();
let commit_state = reader.table(COMMIT_STATE_TABLE.table());
let rows = commit_state.rows();
assert_abandoned_apply_trace_rows(&rows, 1..=SUBMIT_COUNT);
let _ = shutdown_tx.send(());
tokio::time::timeout(Duration::from_secs(1), driver)
.await
.expect("driver exits after post-fallback submit storm")
.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_shutdown_drains_in_flight_apply_without_starting_queued_apply() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let release_first = Arc::new(tokio::sync::Notify::new());
let commit_count = Arc::new(AtomicUsize::new(0));
let (verifier, mut commit_rx) =
height_reporting_verifier(commit_count.clone(), Some(release_first.clone()));
let read_state = fixed_tip_read_state(block::Height(1), block1.hash());
let (mut driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(0),
full_apply_limit: 1,
combined_apply_limit: 1,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("first commit starts"),
Some(block::Height(1))
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block2.clone().into(),
})
.await
.expect("queued block can be submitted before shutdown");
let _ = shutdown_tx.send(());
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut driver)
.await
.is_err(),
"shutdown must wait for the already-started apply to finish"
);
release_first.notify_waiters();
driver.await.expect("driver task exits after apply drains");
assert_eq!(
commit_count.load(Ordering::SeqCst),
1,
"shutdown must drop queued bodies instead of starting new commits"
);
reactor_task.abort();
}
#[tokio::test(start_paused = true)]
async fn accounting_failure_stops_an_idle_or_unstarted_block_driver() {
for before_start in [false, true] {
let (_actions, action_rx) = mpsc::channel(1);
let (block_sync, controls) = zakura_network::zakura::BlockSyncHandle::detached();
if before_start {
controls.fail_accounting();
}
let (driver, _shutdown) = DriverParams::default().spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
panicking_read_state("accounting failure"),
panicking_verifier("accounting failure"),
);
if !before_start {
tokio::time::sleep(Duration::from_millis(1)).await;
assert!(!driver.is_finished(), "the healthy idle driver stays alive");
controls.fail_accounting();
}
tokio::time::sleep(Duration::from_millis(1)).await;
assert!(
driver.is_finished(),
"the idle driver must observe accounting closure"
);
tokio::time::timeout(Duration::from_secs(1), driver)
.await
.expect("accounting closure wakes the critical driver")
.expect("the driver drains without panic");
}
}
#[tokio::test(start_paused = true)]
async fn accounting_failure_drains_the_block_driver_before_node_shutdown() {
assert_failed_block_driver_drains_before_node_shutdown(true).await;
}
#[tokio::test(start_paused = true)]
async fn accounting_failure_abandons_an_apply_that_never_finishes() {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let held = Arc::downgrade(&block1);
let hash = block1.hash();
let (action_tx, action_rx) = mpsc::channel(8);
let (block_sync, mut controls) = zakura_network::zakura::BlockSyncHandle::detached();
let never = Arc::new(tokio::sync::Notify::new());
let (verifier, mut commits) =
height_reporting_verifier(Arc::new(AtomicUsize::new(0)), Some(never));
let handoff = super::zakura::SyncCoordinator::new();
let (driver, _shutdown) = DriverParams {
handoff: handoff.clone(),
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
fixed_tip_read_state(block::Height(1), hash),
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.into(),
})
.await
.unwrap();
assert_eq!(commits.recv().await, Some(block::Height(1)));
controls.fail_accounting();
tokio::time::sleep(ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT - Duration::from_secs(1)).await;
assert!(!driver.is_finished(), "the drain waits out its bound");
tokio::time::timeout(Duration::from_secs(2), driver)
.await
.expect("the drain bound ends the drain")
.expect("the driver drains without panic");
assert!(held.upgrade().is_none(), "abandoning releases the body");
assert!(matches!(
controls.try_next_control(),
Some(BlockSyncEvent::BlockApplyFinished { token: 1, .. })
));
assert!(
!matches!(
handoff.apply_phase(),
zakura_node_services::sync_lifecycle::ApplyPhase::Failed { .. }
),
"the abandoned operation gets a terminal result"
);
}
#[tokio::test(start_paused = true)]
async fn closed_action_channel_drains_the_block_driver_before_node_shutdown() {
assert_failed_block_driver_drains_before_node_shutdown(false).await;
}
async fn assert_failed_block_driver_drains_before_node_shutdown(accounting_failure: bool) {
let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES);
let held_first = Arc::downgrade(&block1);
let held_second = Arc::downgrade(&block2);
let (action_tx, action_rx) = mpsc::channel(8);
let (block_sync, controls) = zakura_network::zakura::BlockSyncHandle::detached();
let release_first = Arc::new(tokio::sync::Notify::new());
let commit_count = Arc::new(AtomicUsize::new(0));
let (verifier, mut commits) =
height_reporting_verifier(commit_count.clone(), Some(release_first.clone()));
let read_state = fixed_tip_read_state(block::Height(1), block1.hash());
let (driver, _shutdown) = DriverParams {
max_checkpoint_height: block::Height(0),
full_apply_limit: 1,
combined_apply_limit: 1,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
let shutdown = CancellationToken::new();
let (fatal_tx, mut fatal_rx) = mpsc::unbounded_channel();
let supervised = tokio::spawn(supervise_block_sync_driver(
async move {
driver.await.expect("the real driver exits without panic");
},
fatal_tx,
shutdown.clone(),
));
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block1.into(),
})
.await
.unwrap();
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commits.recv())
.await
.unwrap(),
Some(block::Height(1))
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block2.into(),
})
.await
.unwrap();
if accounting_failure {
controls.fail_accounting();
} else {
drop(action_tx);
}
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
fatal_rx.try_recv().is_err(),
"accepted work must finish before the driver exits"
);
assert!(
held_first.upgrade().is_some(),
"the accepted verifier retains its body"
);
release_first.notify_waiters();
tokio::time::timeout(Duration::from_secs(1), supervised)
.await
.unwrap()
.unwrap();
assert_eq!(
commit_count.load(Ordering::SeqCst),
1,
"closure cannot start a queued apply"
);
assert!(held_first.upgrade().is_none());
assert!(held_second.upgrade().is_none());
assert_eq!(
fatal_rx.try_recv(),
Ok(()),
"the critical driver reports its exit"
);
assert!(StartCmd::handle_block_sync_driver_exit(&shutdown).is_err());
assert!(
shutdown.is_cancelled(),
"the node starts shutdown after an unexpected driver exit"
);
}
#[tokio::test(start_paused = true)]
async fn accounting_failure_during_dequeue_prevents_verifier_start() {
use std::{
future::Future,
task::{Context, Wake, Waker},
};
struct OnWake(Box<dyn Fn() + Send + Sync>);
impl Wake for OnWake {
fn wake(self: Arc<Self>) {
(self.0)();
}
fn wake_by_ref(self: &Arc<Self>) {
(self.0)();
}
}
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let read_state = fixed_tip_read_state(block::Height(1), block.hash());
let (block_sync, controls) = zakura_network::zakura::BlockSyncHandle::detached();
let (actions, action_rx) = mpsc::channel(1);
actions
.try_send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block.into(),
})
.unwrap();
let closure_count = Arc::new(AtomicUsize::new(0));
let counted = closure_count.clone();
let waker = Waker::from(Arc::new(OnWake(Box::new(move || {
controls.fail_accounting();
counted.fetch_add(1, Ordering::SeqCst);
}))));
let mut reservation = Box::pin(actions.reserve());
assert!(reservation
.as_mut()
.poll(&mut Context::from_waker(&waker))
.is_pending());
let commits = Arc::new(AtomicUsize::new(0));
let (driver, _shutdown) = DriverParams {
max_checkpoint_height: block::Height(0),
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
counting_verifier(commits.clone(), None),
);
tokio::time::timeout(Duration::from_secs(1), driver)
.await
.unwrap()
.unwrap();
assert!(
closure_count.load(Ordering::SeqCst) > 0,
"dequeue triggered the planted accounting failure"
);
assert_eq!(
commits.load(Ordering::SeqCst),
0,
"dequeue closure must precede verifier authorization"
);
}
#[test]
fn block_sync_driver_releases_yielded_submit_blocks_without_queueing() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_height = block.coinbase_height().expect("test block has height");
let block_hash = block.hash();
let owner = test_block_work_owner();
let source = test_block_source();
let Some((height, hash, result, event)) =
abandoned_block_apply_finished_event(owner, source, 99, &block)
else {
panic!("test block has a coinbase height");
};
assert_eq!(height, block_height);
assert_eq!(hash, block_hash);
assert_eq!(result, BlockApplyResult::Unavailable);
assert!(matches!(
event,
BlockSyncEvent::BlockApplyFinished {
owner: event_owner,
source: event_source,
token: 99,
height,
hash,
outcome: event_outcome,
} if event_owner == owner
&& event_source == source
&& height == block_height
&& hash == block_hash
&& matches!(
event_outcome.verification(),
zakura_header_chain::BodyVerificationOutcome::Retryable(
zakura_header_chain::TransientBodyFailure {
kind: zakura_header_chain::TransientBodyFailureKind::Canceled,
..
}
)
)
));
}
#[tokio::test(flavor = "current_thread", start_paused = true)]
async fn block_commits_wait_past_driver_timeout_for_every_apply_class() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let verifier = service_fn(|request: zakura_consensus::Request| async move {
match request {
zakura_consensus::Request::Commit(_) => {
std::future::pending::<Result<block::Hash, zakura_consensus::BoxError>>().await
}
request => panic!("unexpected consensus request: {request:?}"),
}
});
for class in [BlockApplyClass::Full, BlockApplyClass::Checkpoint] {
let waited = tokio::time::timeout(
ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT * 4,
commit_block_sync_body(
verifier,
test_block_work_owner(),
test_block_source(),
block.clone().into(),
class,
),
)
.await;
assert!(
waited.is_err(),
"{class:?} commit must keep waiting past the driver timeout, got {waited:?}"
);
}
}
#[tokio::test]
async fn unmatched_checkpoint_commit_success_does_not_refresh_block_sync_frontiers() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let (tip_tx, tip_rx) = tokio::sync::watch::channel((block::Height(10), block::Hash([10; 32])));
let _tip_tx = tip_tx;
let startup = block_sync_startup_with_snapshot(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: block::Hash([0; 32]),
},
(block::Height(10), block::Hash([10; 32])),
tip_rx,
zakura_network::zakura::ZakuraBlockSyncConfig::default(),
);
let (block_sync, mut reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let startup_action = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv())
.await
.expect("reactor emits startup action")
.expect("reactor action channel remains open");
assert!(
matches!(
startup_action,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
limit: 10,
best_header_tip: block::Height(10),
..
}
),
"test setup should start with an initial body query, got {startup_action:?}"
);
let verifier = service_fn(|request: zakura_consensus::Request| async move {
match request {
zakura_consensus::Request::Commit(block) => {
Ok::<_, zakura_consensus::BoxError>(block.hash())
}
request => panic!("unexpected consensus request: {request:?}"),
}
});
let read_state = service_fn(move |request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::FinalizedTip => {
Ok::<_, zakura_state::BoxError>(zakura_state::ReadResponse::FinalizedTip(Some((
block::Height(2),
block::Hash([2; 32]),
))))
}
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block::Height(1),
block_hash,
)))),
request => panic!("unexpected read request: {request:?}"),
}
});
apply_block_sync_body(
verifier,
zakura_chain::chain_tip::NoChainTip,
None,
read_state,
block_sync.clone(),
test_block_work_owner(),
test_block_source(),
1,
block.into(),
BlockApplyClass::Checkpoint,
zakura_network::zakura::ZakuraTrace::noop(),
None,
)
.await;
assert!(
tokio::time::timeout(Duration::from_millis(50), reactor_actions.recv())
.await
.is_err(),
"a synthetic commit completion without a matching reactor apply must not refresh frontiers"
);
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_apply_emits_commit_state_trace_rows() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let block_height = block.coinbase_height().expect("test block has height");
let mut capture =
TraceCapture::for_test("block_sync_apply_emits_commit_state_trace_rows").unwrap();
let trace = zakura_network::zakura::ZakuraTrace::new(capture.tracer(), "01");
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let verifier = service_fn(|request: zakura_consensus::Request| async move {
match request {
zakura_consensus::Request::Commit(block) => {
Ok::<_, zakura_consensus::BoxError>(block.hash())
}
request => panic!("unexpected consensus request: {request:?}"),
}
});
let read_state = service_fn(move |request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::FinalizedTip => {
Ok::<_, zakura_state::BoxError>(zakura_state::ReadResponse::FinalizedTip(None))
}
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block_height,
block_hash,
)))),
request => panic!("unexpected read request: {request:?}"),
}
});
apply_block_sync_body(
verifier,
zakura_chain::chain_tip::NoChainTip,
None,
read_state,
block_sync,
test_block_work_owner(),
test_block_source(),
77,
block.into(),
BlockApplyClass::Full,
trace,
None,
)
.await;
capture.flush().await;
let reader = capture.reader().unwrap();
let commit_state = reader.table(COMMIT_STATE_TABLE.table());
let hash_label = format!("{block_hash}");
let common = [
(cs_trace::APPLY_TOKEN, TraceValue::U64(77)),
(cs_trace::HEIGHT, TraceValue::U64(u64::from(block_height.0))),
(cs_trace::HASH, TraceValue::Str(&hash_label)),
];
commit_state.assert_row(cs_trace::COMMIT_START, &common);
commit_state.assert_row(
cs_trace::COMMIT_FINISH,
&[
(cs_trace::APPLY_TOKEN, TraceValue::U64(77)),
(cs_trace::RESULT, TraceValue::Str("committed")),
],
);
commit_state.assert_row(
cs_trace::REACTOR_EVENT_SENT,
&[
(cs_trace::APPLY_TOKEN, TraceValue::U64(77)),
(cs_trace::RESULT, TraceValue::Str("committed")),
],
);
let _ = capture.finish().await.unwrap();
reactor_task.abort();
}
#[tokio::test(flavor = "current_thread", start_paused = true)]
async fn block_sync_pending_full_apply_emits_stalled_trace_without_finishing() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let block_height = block.coinbase_height().expect("test block has height");
let mut capture = TraceCapture::for_test(
"block_sync_pending_full_apply_emits_stalled_trace_without_finishing",
)
.unwrap();
let trace = zakura_network::zakura::ZakuraTrace::new(capture.tracer(), "01");
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let verifier = service_fn(|request: zakura_consensus::Request| async move {
match request {
zakura_consensus::Request::Commit(_block) => {
future::pending::<Result<block::Hash, zakura_consensus::BoxError>>().await
}
request => panic!("unexpected consensus request: {request:?}"),
}
});
let read_state = service_fn(move |request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::FinalizedTip => {
Ok::<_, zakura_state::BoxError>(zakura_state::ReadResponse::FinalizedTip(None))
}
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block_height,
block_hash,
)))),
request => panic!("unexpected read request: {request:?}"),
}
});
let apply_task = tokio::spawn(apply_block_sync_body(
verifier,
zakura_chain::chain_tip::NoChainTip,
None,
read_state,
block_sync,
test_block_work_owner(),
test_block_source(),
88,
block.into(),
BlockApplyClass::Full,
trace,
None,
));
tokio::task::yield_now().await;
tokio::time::advance(ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT).await;
tokio::task::yield_now().await;
capture.flush().await;
let reader = capture.reader().unwrap();
let commit_state = reader.table(COMMIT_STATE_TABLE.table());
commit_state.assert_row(
cs_trace::COMMIT_START,
&[(cs_trace::APPLY_TOKEN, TraceValue::U64(88))],
);
commit_state.assert_row(
cs_trace::COMMIT_STALLED,
&[(cs_trace::APPLY_TOKEN, TraceValue::U64(88))],
);
assert_eq!(
commit_state.count(cs_trace::COMMIT_FINISH),
0,
"pending full verifier must not produce a finish row before it resolves"
);
apply_task.abort();
let _ = capture.finish().await.unwrap();
reactor_task.abort();
}
#[tokio::test(flavor = "current_thread", start_paused = true)]
async fn buffered_native_commit_blocks_legacy_fallback_past_every_diagnostic_interval() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let block_height = block.coinbase_height().expect("test block has height");
let (release_commit_tx, release_commit_rx) = tokio::sync::watch::channel(false);
let verifier = service_fn(move |request: zakura_consensus::Request| {
let mut release_commit_rx = release_commit_rx.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
while !*release_commit_rx.borrow() {
release_commit_rx
.changed()
.await
.expect("test commit release sender stays open");
}
Ok::<_, zakura_consensus::BoxError>(block.hash())
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
let read_state = service_fn(move |request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::FinalizedTip => {
Ok::<_, zakura_state::BoxError>(zakura_state::ReadResponse::FinalizedTip(None))
}
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block_height,
block_hash,
)))),
request => panic!("unexpected read request: {request:?}"),
}
});
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let handoff = super::zakura::SyncCoordinator::new();
let permit = handoff
.begin_apply()
.expect("native apply owns a permit before fallback");
let apply = tokio::spawn(async move {
let _permit = permit;
apply_block_sync_body(
verifier,
zakura_chain::chain_tip::NoChainTip,
None,
read_state,
block_sync,
test_block_work_owner(),
test_block_source(),
89,
block.into(),
BlockApplyClass::Full,
zakura_network::zakura::ZakuraTrace::noop(),
None,
)
.await
});
let legacy_started = Arc::new(AtomicBool::new(false));
let fallback_handoff = handoff.clone();
let fallback_started = legacy_started.clone();
let fallback = tokio::spawn(async move {
let lease = fallback_handoff
.acquire_legacy_fallback(Duration::from_secs(60))
.await
.expect("one fallback request owns the drain");
fallback_started.store(true, Ordering::SeqCst);
lease
});
tokio::task::yield_now().await;
for _ in 0..3 {
tokio::time::advance(Duration::from_secs(60)).await;
tokio::task::yield_now().await;
assert!(
!legacy_started.load(Ordering::SeqCst),
"legacy apply authorization must stay blocked while the verifier commit is alive"
);
assert!(!fallback.is_finished());
}
release_commit_tx
.send(true)
.expect("the buffered commit receiver remains live");
apply.await.expect("the native buffered commit finishes");
let lease = fallback
.await
.expect("the fallback acquisition task finishes after the native commit");
assert!(legacy_started.load(Ordering::SeqCst));
assert!(handoff.begin_apply().is_none());
drop(lease);
assert!(
handoff.begin_apply().is_some(),
"native ownership resumes after the legacy fallback lease ends"
);
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_pending_checkpoint_apply_does_not_block_control_plane_actions() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let (release_commit_tx, release_commit_rx) = tokio::sync::watch::channel(false);
let verifier = service_fn(move |request: zakura_consensus::Request| {
let mut release_commit_rx = release_commit_rx.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
while !*release_commit_rx.borrow() {
release_commit_rx
.changed()
.await
.expect("test commit release sender stays open");
}
Ok::<_, zakura_consensus::BoxError>(block.hash())
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
let (query_seen_tx, query_seen_rx) = oneshot::channel();
let query_seen_tx = Arc::new(Mutex::new(Some(query_seen_tx)));
let read_state = service_fn(move |request: zakura_state::ReadRequest| {
let query_seen_tx = query_seen_tx.clone();
async move {
match request {
zakura_state::ReadRequest::MissingBlockBodyMetadata { from, limit } => {
assert_eq!(from, block::Height(1));
assert_eq!(limit, 1);
if let Some(query_seen_tx) = query_seen_tx
.lock()
.expect("query seen sender mutex is not poisoned")
.take()
{
let _ = query_seen_tx.send(());
}
Ok::<_, zakura_state::BoxError>(
zakura_state::ReadResponse::MissingBlockBodyMetadata(
zakura_state::BlockSyncBodyMetadata {
anchor: zakura_header_chain::Frontier::new(
block::Height(0),
block::Hash([0; 32]),
),
blocks: Vec::new(),
},
),
)
}
zakura_state::ReadRequest::FinalizedTip => Ok(
zakura_state::ReadResponse::FinalizedTip(Some((block::Height(1), block_hash))),
),
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block::Height(1),
block_hash,
)))),
request => panic!("unexpected read request: {request:?}"),
}
}
});
let (driver, shutdown_tx) = DriverParams::default().spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block.into(),
})
.await
.expect("driver action channel stays open");
action_tx
.send(needed_blocks_query(
1,
block::Height(1),
1,
block::Height(1),
))
.await
.expect("driver action channel stays open");
tokio::time::timeout(Duration::from_secs(1), query_seen_rx)
.await
.expect("driver processes unrelated query while checkpoint apply is pending")
.expect("read service reports query");
release_commit_tx
.send(true)
.expect("test commit release receiver stays open");
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_checkpoint_apply_limit_allows_two_checkpoint_gaps() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let block_hash = block.hash();
let two_checkpoint_gaps = zakura_consensus::MAX_CHECKPOINT_HEIGHT_GAP.saturating_mul(2);
let (action_tx, action_rx) = mpsc::channel(two_checkpoint_gaps + 8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let commit_count = Arc::new(AtomicUsize::new(0));
let (release_commits_tx, release_commits_rx) = tokio::sync::watch::channel(false);
let verifier_count = commit_count.clone();
let verifier = service_fn(move |request: zakura_consensus::Request| {
let verifier_count = verifier_count.clone();
let mut release_commits_rx = release_commits_rx.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
verifier_count.fetch_add(1, Ordering::SeqCst);
while !*release_commits_rx.borrow() {
release_commits_rx
.changed()
.await
.expect("test commit release sender stays open");
}
Ok::<_, zakura_consensus::BoxError>(block.hash())
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
let read_state = service_fn(move |request: zakura_state::ReadRequest| async move {
match request {
zakura_state::ReadRequest::FinalizedTip => Ok::<_, zakura_state::BoxError>(
zakura_state::ReadResponse::FinalizedTip(Some((block::Height(1), block_hash))),
),
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block::Height(1),
block_hash,
)))),
request => {
panic!("unexpected read request while checkpoint applies are pending: {request:?}")
}
}
});
let (driver, shutdown_tx) = DriverParams {
checkpoint_apply_limit: two_checkpoint_gaps,
combined_apply_limit: zakura_consensus::MAX_CHECKPOINT_HEIGHT_GAP,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
for token in 0..=two_checkpoint_gaps {
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: u64::try_from(token).expect("test token fits in u64"),
block: block.clone().into(),
})
.await
.expect("driver action channel stays open");
}
tokio::time::timeout(Duration::from_secs(1), async {
while commit_count.load(Ordering::SeqCst) < two_checkpoint_gaps {
tokio::task::yield_now().await;
}
})
.await
.expect("driver starts exactly two checkpoint gaps of applies");
tokio::time::sleep(Duration::from_millis(50)).await;
assert_eq!(
commit_count.load(Ordering::SeqCst),
two_checkpoint_gaps,
"driver must not submit a third checkpoint range before earlier ranges complete"
);
release_commits_tx
.send(true)
.expect("test commit release receivers stay open");
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_treats_duplicate_commit_as_idempotent_and_keeps_draining() {
let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let attempts = Arc::new(AtomicUsize::new(0));
let verifier_attempts = attempts.clone();
let (commit_tx, mut commit_rx) = mpsc::channel(8);
let verifier = service_fn(move |request: zakura_consensus::Request| {
let attempts = verifier_attempts.clone();
let commit_tx = commit_tx.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
let hash = block.hash();
if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
return Err(zakura_consensus::RouterError::Block {
source: Box::new(zakura_consensus::VerifyBlockError::Block {
source: zakura_consensus::BlockError::AlreadyInChain(
hash,
zakura_state::KnownBlock::BestChain,
),
}),
});
}
commit_tx
.send(hash)
.await
.expect("test commit receiver stays open");
Ok(hash)
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
let read_requests = Arc::new(Mutex::new(Vec::new()));
let read_requests_for_service = read_requests.clone();
let block_hash = block.hash();
let read_state = service_fn(move |request: zakura_state::ReadRequest| {
let read_requests = read_requests_for_service.clone();
async move {
read_requests
.lock()
.expect("test read request log is not poisoned")
.push(request.clone());
match request {
zakura_state::ReadRequest::FinalizedTip => {
Ok(zakura_state::ReadResponse::FinalizedTip(None))
}
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block::Height(1),
block_hash,
)))),
request => panic!("unexpected read request: {request:?}"),
}
}
});
let (driver, shutdown_tx) = DriverParams::default().spawn(
action_rx,
block_sync,
zakura_chain::chain_tip::NoChainTip,
read_state,
verifier,
);
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block.clone().into(),
})
.await
.expect("driver action channel stays open");
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 2,
block: block.clone().into(),
})
.await
.expect("driver action channel stays open");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), commit_rx.recv())
.await
.expect("second commit arrives after duplicate"),
Some(block.hash())
);
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert!(
read_requests
.lock()
.expect("test read request log is not poisoned")
.is_empty(),
"duplicate commit completion must not publish a frontier or re-read state"
);
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn legacy_fallback_completes_transferred_checkpoint_range() {
const CHECKPOINT_HEIGHT: u32 = 10;
const WITHHELD: u32 = 5;
let chain: Vec<(block::Height, Arc<block::Block>)> = (0..=CHECKPOINT_HEIGHT)
.map(|height| {
let bytes: &[u8] = zakura_test::vectors::CONTINUOUS_MAINNET_BLOCKS
.get(&height)
.copied()
.expect("a contiguous mainnet block vector exists for heights 0..=10");
let block = mainnet_block(bytes);
assert_eq!(
block.coinbase_height(),
Some(block::Height(height)),
"mainnet block vector height matches its coinbase height",
);
(block::Height(height), block)
})
.collect();
let genesis_hash = chain[0].1.hash();
let checkpoint_hash = chain[CHECKPOINT_HEIGHT as usize].1.hash();
let network = zakura_chain::parameters::Network::Mainnet;
let zakura_state::InitializedState {
state: write_state,
read_state,
latest_chain_tip,
..
} = zakura_state::init_test_services(&network).await;
let mut checkpoint_verifier = zakura_consensus::CheckpointVerifier::from_list(
[
(block::Height(0), genesis_hash),
(block::Height(CHECKPOINT_HEIGHT), checkpoint_hash),
],
&network,
None,
write_state,
)
.expect("a checkpoint list with genesis and one mid-chain checkpoint is valid");
let submitted_count = Arc::new(AtomicUsize::new(0));
let verifier_submitted_count = submitted_count.clone();
let checkpoint_verifier = service_fn(move |block| {
verifier_submitted_count.fetch_add(1, Ordering::SeqCst);
checkpoint_verifier.call(block)
});
let checkpoint_verifier = tower::buffer::Buffer::new(BoxService::new(checkpoint_verifier), 16);
let legacy_verifier = checkpoint_verifier.clone();
let verifier = service_fn(move |request: zakura_consensus::Request| {
let checkpoint_verifier = checkpoint_verifier.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
let commit: futures::future::BoxFuture<
'static,
Result<block::Hash, tower::BoxError>,
> = Box::pin(checkpoint_verifier.oneshot(block));
commit.await
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
let (action_tx, action_rx) = mpsc::channel(64);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let handoff = super::zakura::SyncCoordinator::new();
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(CHECKPOINT_HEIGHT),
handoff: handoff.clone(),
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
latest_chain_tip,
read_state.clone(),
verifier,
);
let finalized_tip = || {
let read_state = read_state.clone();
async move {
match read_state
.oneshot(zakura_state::ReadRequest::FinalizedTip)
.await
.expect("finalized tip read succeeds")
{
zakura_state::ReadResponse::FinalizedTip(tip) => tip.map(|(height, _hash)| height),
response => panic!("unexpected FinalizedTip response: {response:?}"),
}
}
};
for (height, block) in &chain {
if height.0 == WITHHELD {
continue;
}
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: u64::from(height.0),
block: block.clone().into(),
})
.await
.expect("driver action channel stays open");
}
let expected_submissions = chain.len().saturating_sub(1);
tokio::time::timeout(Duration::from_secs(1), async {
while submitted_count.load(Ordering::SeqCst) != expected_submissions {
tokio::task::yield_now().await;
}
})
.await
.expect("the driver must submit every available checkpoint body before fallback starts");
assert_ne!(
finalized_tip().await,
Some(block::Height(CHECKPOINT_HEIGHT)),
"checkpoint range must not commit while a mid-range body is missing",
);
let fallback = tokio::spawn(async move {
handoff
.acquire_legacy_fallback(Duration::from_secs(1))
.await
});
let _fallback_lease = tokio::time::timeout(Duration::from_secs(1), fallback)
.await
.expect("fallback must not wait for an incomplete checkpoint range")
.expect("fallback acquisition task must finish")
.expect("fallback must acquire the transferred checkpoint range");
let (_withheld_height, withheld_block) = chain
.iter()
.find(|(height, _)| height.0 == WITHHELD)
.expect("withheld block is part of the test chain");
legacy_verifier
.oneshot(withheld_block.clone().into())
.await
.expect("legacy verifier submission completes the checkpoint range");
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if finalized_tip().await == Some(block::Height(CHECKPOINT_HEIGHT)) {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("legacy fallback must let the checkpoint range commit to the tip");
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test(flavor = "multi_thread")]
async fn legacy_fallback_transfers_full_commit_waiting_for_parent() {
let network = zakura_chain::parameters::Network::Mainnet;
let zakura_state::InitializedState {
state: write_state,
read_state,
latest_chain_tip: latest_tip,
..
} = zakura_state::init_test_services(&network).await;
let barrier_state = write_state.clone();
let submitted_count = Arc::new(AtomicUsize::new(0));
let resolved = Arc::new(AtomicBool::new(false));
let verifier_submitted_count = submitted_count.clone();
let verifier_resolved = resolved.clone();
let verifier = service_fn(move |request: zakura_consensus::Request| {
let mut write_state = write_state.clone();
let submitted_count = verifier_submitted_count.clone();
let resolved = verifier_resolved.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
let commit: futures::future::BoxFuture<
'static,
Result<zakura_state::Response, zakura_state::BoxError>,
> = Box::pin(write_state.ready().await?.call(
zakura_state::Request::CommitSemanticallyVerifiedBlock(block.into()),
));
submitted_count.fetch_add(1, Ordering::SeqCst);
let response = commit.await;
resolved.store(true, Ordering::SeqCst);
match response? {
zakura_state::Response::Committed(hash) => {
Ok::<_, zakura_consensus::BoxError>(hash)
}
response => panic!("unexpected commit response: {response:?}"),
}
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
let (action_tx, action_rx) = mpsc::channel(8);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let handoff = super::zakura::SyncCoordinator::new();
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: block::Height(0),
handoff: handoff.clone(),
..DriverParams::default()
}
.spawn(action_rx, block_sync, latest_tip, read_state, verifier);
let block = mainnet_block(&zakura_test::vectors::BLOCK_MAINNET_1687107_BYTES);
let block_hash = block.hash();
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: 1,
block: block.into(),
})
.await
.expect("driver action channel stays open");
tokio::time::timeout(Duration::from_secs(5), async {
while submitted_count.load(Ordering::SeqCst) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("the driver submits the full-verification body before fallback starts");
barrier_state
.oneshot(zakura_state::Request::KnownBlock(block_hash))
.await
.expect("known-block request succeeds");
assert!(
!resolved.load(Ordering::SeqCst),
"the commit must still be waiting for its parent when fallback starts"
);
let fallback = tokio::spawn(async move {
handoff
.acquire_legacy_fallback(Duration::from_secs(1))
.await
});
let _fallback_lease = tokio::time::timeout(Duration::from_secs(5), fallback)
.await
.expect("fallback must not wait for a commit whose parent only fallback can supply")
.expect("fallback acquisition task must finish")
.expect("fallback must acquire after transferring the waiting commit");
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_driver_finalizes_across_two_checkpoint_boundaries() {
const FIRST_CHECKPOINT_HEIGHT: u32 = 5;
const SECOND_CHECKPOINT_HEIGHT: u32 = 10;
let chain: Vec<(block::Height, Arc<block::Block>)> = (0..=SECOND_CHECKPOINT_HEIGHT)
.map(|height| {
let bytes: &[u8] = zakura_test::vectors::CONTINUOUS_MAINNET_BLOCKS
.get(&height)
.copied()
.expect("a contiguous mainnet block vector exists for heights 0..=10");
let block = mainnet_block(bytes);
assert_eq!(
block.coinbase_height(),
Some(block::Height(height)),
"mainnet block vector height matches its coinbase height",
);
(block::Height(height), block)
})
.collect();
let genesis_hash = chain[0].1.hash();
let first_checkpoint_hash = chain[FIRST_CHECKPOINT_HEIGHT as usize].1.hash();
let second_checkpoint_height = block::Height(SECOND_CHECKPOINT_HEIGHT);
let second_checkpoint_hash = chain[SECOND_CHECKPOINT_HEIGHT as usize].1.hash();
let network = zakura_chain::parameters::Network::Mainnet;
let zakura_state::InitializedState {
state: write_state,
read_state,
latest_chain_tip: latest_tip,
..
} = zakura_state::init_test_services(&network).await;
let checkpoint_verifier = zakura_consensus::CheckpointVerifier::from_list(
[
(block::Height(0), genesis_hash),
(
block::Height(FIRST_CHECKPOINT_HEIGHT),
first_checkpoint_hash,
),
(second_checkpoint_height, second_checkpoint_hash),
],
&network,
None,
write_state,
)
.expect("a checkpoint list with two low checkpoint boundaries is valid");
let checkpoint_verifier = tower::buffer::Buffer::new(BoxService::new(checkpoint_verifier), 32);
let verifier = service_fn(move |request: zakura_consensus::Request| {
let checkpoint_verifier = checkpoint_verifier.clone();
async move {
match request {
zakura_consensus::Request::Commit(block) => {
let commit: futures::future::BoxFuture<
'static,
Result<block::Hash, tower::BoxError>,
> = Box::pin(checkpoint_verifier.oneshot(block));
commit.await
}
request => panic!("unexpected consensus request: {request:?}"),
}
}
});
let (action_tx, action_rx) = mpsc::channel(64);
let startup = block_sync_startup_for_test();
let (block_sync, _reactor_actions, reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let (driver, shutdown_tx) = DriverParams {
max_checkpoint_height: second_checkpoint_height,
..DriverParams::default()
}
.spawn(
action_rx,
block_sync,
latest_tip,
read_state.clone(),
verifier,
);
for (height, block) in &chain {
action_tx
.send(BlockSyncAction::SubmitBlock {
owner: test_block_work_owner(),
source: test_block_source(),
token: u64::from(height.0),
block: block.clone().into(),
})
.await
.expect("driver action channel stays open");
}
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let finalized_tip = match read_state
.clone()
.oneshot(zakura_state::ReadRequest::FinalizedTip)
.await
.expect("finalized tip read succeeds")
{
zakura_state::ReadResponse::FinalizedTip(tip) => tip,
response => panic!("unexpected FinalizedTip response: {response:?}"),
};
if finalized_tip == Some((second_checkpoint_height, second_checkpoint_hash)) {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("driver must finalize through both checkpoint boundaries");
let _ = shutdown_tx.send(());
driver.await.expect("driver task exits cleanly");
reactor_task.abort();
}
#[tokio::test]
async fn block_sync_restart_reloads_checkpoint_frontier_after_missed_live_update() {
const CHECKPOINT_HEIGHT: u32 = 10;
let chain: Vec<(block::Height, Arc<block::Block>)> = (0..=CHECKPOINT_HEIGHT)
.map(|height| {
let bytes: &[u8] = zakura_test::vectors::CONTINUOUS_MAINNET_BLOCKS
.get(&height)
.copied()
.expect("a contiguous mainnet block vector exists for heights 0..=10");
let block = mainnet_block(bytes);
assert_eq!(
block.coinbase_height(),
Some(block::Height(height)),
"mainnet block vector height matches its coinbase height",
);
(block::Height(height), block)
})
.collect();
let genesis_hash = chain[0].1.hash();
let checkpoint_height = block::Height(CHECKPOINT_HEIGHT);
let checkpoint_hash = chain[CHECKPOINT_HEIGHT as usize].1.hash();
let best_header_tip = (block::Height(20), block::Hash([20; 32]));
let network = zakura_chain::parameters::Network::Mainnet;
let zakura_state::InitializedState {
state: write_state,
read_state,
..
} = zakura_state::init_test_services(&network).await;
let (_tip_tx, tip_rx) = tokio::sync::watch::channel(best_header_tip);
let startup = block_sync_startup_with_snapshot(
BlockSyncFrontiers {
finalized_height: block::Height(0),
verified_block_tip: block::Height(0),
verified_block_hash: genesis_hash,
},
best_header_tip,
tip_rx,
zakura_network::zakura::ZakuraBlockSyncConfig::default(),
);
let (stale_block_sync, mut stale_actions, stale_reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(startup).expect("valid block-sync config");
let startup_action = tokio::time::timeout(Duration::from_secs(1), stale_actions.recv())
.await
.expect("stale reactor emits startup action")
.expect("stale reactor action channel remains open");
assert!(
matches!(
startup_action,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(1),
limit: 20,
best_header_tip: block::Height(20),
..
}
),
"stale reactor should start querying from genesis, got {startup_action:?}"
);
let checkpoint_verifier = zakura_consensus::CheckpointVerifier::from_list(
[
(block::Height(0), genesis_hash),
(checkpoint_height, checkpoint_hash),
],
&network,
None,
write_state,
)
.expect("a checkpoint list with genesis and one mid-chain checkpoint is valid");
let checkpoint_verifier = tower::buffer::Buffer::new(BoxService::new(checkpoint_verifier), 16);
let mut commits = FuturesUnordered::new();
for (_height, block) in chain {
let checkpoint_verifier = checkpoint_verifier.clone();
commits.push(async move { checkpoint_verifier.oneshot(block.into()).await });
}
while let Some(result) = commits.next().await {
result.expect("checkpoint verifier commits the contiguous range");
}
let finalized_tip = || {
let read_state = read_state.clone();
async move {
match read_state
.oneshot(zakura_state::ReadRequest::FinalizedTip)
.await
.expect("finalized tip read succeeds")
{
zakura_state::ReadResponse::FinalizedTip(tip) => tip,
response => panic!("unexpected FinalizedTip response: {response:?}"),
}
}
};
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if finalized_tip()
.await
.is_some_and(|(height, _hash)| height == checkpoint_height)
{
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("checkpoint range must reach durable finalized state");
assert_eq!(
stale_block_sync.local_status().servable_high,
block::Height(0),
"without a live frontier event, the old reactor remains stale"
);
stale_reactor_task.abort();
let restart_read_state = {
let read_state = read_state.clone();
service_fn(move |request: zakura_state::ReadRequest| {
let read_state = read_state.clone();
async move {
match request {
zakura_state::ReadRequest::FinalizedTip => read_state.oneshot(request).await,
zakura_state::ReadRequest::Tip => Ok(zakura_state::ReadResponse::Tip(Some((
block::Height(0),
genesis_hash,
)))),
request => panic!("unexpected restart read request: {request:?}"),
}
}
})
};
let restart_frontiers =
query_block_sync_frontiers(restart_read_state, zakura_chain::chain_tip::NoChainTip)
.await
.expect("restart reads block-sync frontiers from durable state");
assert_eq!(restart_frontiers.finalized_height, checkpoint_height);
assert_eq!(restart_frontiers.verified_block_tip, checkpoint_height);
assert_eq!(restart_frontiers.verified_block_hash, checkpoint_hash);
let (_restart_tip_tx, restart_tip_rx) = tokio::sync::watch::channel(best_header_tip);
let restart_startup = block_sync_startup_with_snapshot(
restart_frontiers,
best_header_tip,
restart_tip_rx,
zakura_network::zakura::ZakuraBlockSyncConfig::default(),
);
let (_fresh_block_sync, mut fresh_actions, fresh_reactor_task) =
zakura_network::zakura::spawn_block_sync_reactor(restart_startup)
.expect("valid block-sync config");
let restart_action = tokio::time::timeout(Duration::from_secs(1), fresh_actions.recv())
.await
.expect("fresh reactor emits startup action")
.expect("fresh reactor action channel remains open");
assert!(
matches!(
restart_action,
BlockSyncAction::QueryNeededBlocks {
from: block::Height(11),
limit: 10,
best_header_tip: block::Height(20),
..
}
),
"fresh reactor should query from the durable checkpoint frontier, got {restart_action:?}"
);
fresh_reactor_task.abort();
}