commonware_macros::stability_scope!(ALPHA {
mod deferred;
pub use deferred::Deferred;
mod inline;
pub use inline::Inline;
mod validation;
});
mod variant;
pub use variant::Standard;
#[cfg(test)]
mod tests {
use super::{Deferred, Inline, Standard};
pub(super) use crate::marshal::mocks::block::ready;
use crate::{
Application as ConsensusApplication, Automaton, CertifiableAutomaton, Handoff, Heightable,
Relay, Reporter, Viewable,
marshal::{
Identifier, Update,
ancestry::{Ancestry, BlockProvider},
application::{
gates::{GateOutcome, Gates},
relay,
},
config::{Config, Start},
core::{
Actor, CommitmentFallback, Cutoff, DigestFallback, Mailbox, Processed, cache,
durability::Durable as _,
},
mocks::{
application::Application,
block::DigestCounter,
harness::{
self, B, BLOCKS_PER_EPOCH, Ctx, D, DeferredHarness, EmptyProvider,
InlineHarness, LINK, NAMESPACE, NUM_VALIDATORS, PAGE_CACHE_SIZE, PAGE_SIZE,
QUORUM, S, StandardHarness, TEST_QUOTA, TestHarness, UNRELIABLE_LINK, V,
ValidatorHandle, default_leader, make_raw_block, setup_network_links,
setup_network_with_participants,
},
store::{Op, Recording},
strategy::{Gate, GatedStrategy},
verifying::MockVerifyingApp,
},
resolver::handler,
},
simplex::{
self, Plan,
config::{ForwardPolicy, SkipBudget, SkipPolicy},
elector::{Config as _, Elector as _, RoundRobin, RoundRobinElector},
scheme::bls12381_threshold::vrf as bls12381_threshold_vrf,
types::{
Activity, Certificate, Finalization, Notarization, Notarize, Nullification,
Nullify, Proposal, Vote,
},
},
types::{Epoch, EpochInfo, Epocher, FixedEpocher, Height, Round, View, ViewDelta},
};
use bytes::{BufMut, Bytes};
use commonware_actor::{Feedback, mailbox};
use commonware_broadcast::{Broadcaster as _, buffered};
use commonware_codec::{Buf, DecodeExt as _, Encode, FixedSize, Read, Write, varint::UInt};
use commonware_cryptography::{
Digested, Digestible, Hasher as _,
certificate::{ConstantProvider, Provider, Scoped, Verifier as _, mocks::Fixture},
ed25519::PublicKey,
sha256::Sha256,
};
use commonware_macros::{select, test_group, test_traced};
use commonware_p2p::{Manager as _, Receiver, Recipients, Sender as _};
use commonware_parallel::Sequential;
use commonware_resolver::{
Consumer, Delivery, Fetch, Outcome, Resolution, Resolver, TargetedResolver,
};
use commonware_runtime::{
Clock, Handle, Metrics, Quota, Runner, Spawner, Storage as _, Supervisor as _,
buffer::paged::CacheRef, deterministic, utils::reschedule,
};
use commonware_storage::{
archive::{Archive as _, Identifier as ArchiveID, immutable, prunable},
metadata::{self, Metadata},
translator::EightCap,
};
use commonware_utils::{
Acknowledgement as _, NZDuration, NZU16, NZU64, NZUsize,
acknowledgement::Exact,
channel::{fallible::OneshotExt, mpsc, oneshot, oneshot::error::TryRecvError},
non_empty,
ordered::{Quorum as _, Set},
probability,
sequence::U64,
sync::Mutex,
vec::NonEmptyVec,
};
use std::{
collections::BTreeMap,
num::{NonZeroU32, NonZeroU64, NonZeroUsize},
sync::{
Arc, Weak,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
#[test]
fn mailbox_provides_application_blocks() {
fn assert_provider<P: BlockProvider<Block = B>>() {}
assert_provider::<Mailbox<S, Standard<B>>>();
}
#[test_traced("WARN")]
fn test_standard_local_blocks_compute_digest_once_and_replay_once() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let validator = participants[0].clone();
let setup = StandardHarness::setup_validator_with(
context.child("validator"),
&mut oracle,
validator.clone(),
ConstantProvider::new(schemes[0].clone()),
NZUsize!(3),
Application::manual_ack(),
)
.await;
let marshal = setup.mailbox.clone();
assert_eq!(setup.application.acknowledged().await, Height::zero());
while marshal.get_processed().await.map(Processed::height) != Some(Height::zero()) {
reschedule().await;
}
let mut counter = DigestCounter::default();
let mut digests = Vec::new();
let mut parent = StandardHarness::genesis_block(NUM_VALIDATORS as u16).digest();
for (built, height) in (1..=3).enumerate() {
let built = built + 1;
let round = Round::new(Epoch::zero(), View::new(height));
let block_context = Ctx {
round,
leader: validator.clone(),
parent: (View::new(height - 1), parent),
};
let block = B::new::<Sha256>(
block_context.clone(),
parent,
Height::new(height),
0x1DE_0000 + height,
);
counter.track(&block);
let app: MockVerifyingApp<B, S> =
MockVerifyingApp::new().with_propose_result(block.clone());
let mut wrapper = Wrapper::new(
kind,
context.child("proposal").with_attribute("height", height),
app,
marshal.clone(),
);
let digest = wrapper
.propose(block_context)
.await
.await
.expect("proposal must build a block");
assert_eq!(counter.count(), built, "{kind:?} proposal hashing");
assert!(
wrapper
.broadcast(digest, Plan::Propose { round })
.accepted()
);
assert!(wrapper.certify(round, digest).await.await.unwrap());
let cached = marshal.get_block(&digest).await.unwrap();
assert_eq!(cached.inner(), &block);
assert_eq!(counter.count(), built, "{kind:?} seal and relay hashing");
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::new(height - 1), digest),
&schemes,
QUORUM,
);
assert!(
wrapper
.report(Activity::Finalization(finalization))
.accepted()
);
wait_until(&context, Duration::from_secs(1), "block dispatched", || {
setup
.application
.blocks()
.contains_key(&Height::new(height))
})
.await;
assert_eq!(counter.count(), built, "{kind:?} live dispatch hashing");
parent = digest;
digests.push(digest);
}
let mut cursor = marshal.get_block(&parent).await.unwrap();
for expected in (0..3).rev() {
cursor = marshal.subscribe_parent(&cursor).await.unwrap();
assert_eq!(cursor.height(), Height::new(expected));
}
assert_eq!(counter.count(), 3, "{kind:?} ancestry hashing");
assert_eq!(setup.application.pending_ack_heights().len(), 3);
drop(marshal);
setup.crash().await;
let restart = StandardHarness::setup_validator_with(
context.child("restart"),
&mut oracle,
validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(3),
Application::manual_ack(),
)
.await;
assert_eq!(restart.height, Some(Height::zero()));
assert_eq!(counter.count(), 4, "{kind:?} successor restoration");
wait_until(&context, Duration::from_secs(1), "blocks replayed", || {
restart.application.pending_ack_heights().len() == 3
})
.await;
assert_eq!(counter.count(), 7, "{kind:?} one restoration per replay");
for digest in digests {
for _ in 0..2 {
assert_eq!(
restart.mailbox.get_block(&digest).await.unwrap().digest(),
digest
);
}
}
assert_eq!(counter.count(), 7, "{kind:?} keyed archive reads");
restart.crash().await;
});
}
}
#[derive(Clone)]
struct VerifierProvider {
scheme: Arc<S>,
}
impl VerifierProvider {
fn new(scheme: S) -> Self {
Self {
scheme: Arc::new(scheme),
}
}
}
impl Provider for VerifierProvider {
type Scope = Epoch;
type Scheme = S;
fn scoped(&self, _: Epoch) -> Option<Scoped<S>> {
Some(Scoped::verifier(self.scheme.clone()))
}
}
#[derive(Clone, Default)]
struct RetiringProvider {
scopes: BTreeMap<Epoch, (Arc<S>, Arc<AtomicUsize>)>,
}
impl RetiringProvider {
fn with(mut self, epoch: Epoch, scheme: S, remaining: usize) -> Self {
self.scopes.insert(
epoch,
(Arc::new(scheme), Arc::new(AtomicUsize::new(remaining))),
);
self
}
fn retired(&self, epoch: Epoch) -> bool {
self.scopes[&epoch].1.load(Ordering::Acquire) == 0
}
}
impl Provider for RetiringProvider {
type Scope = Epoch;
type Scheme = S;
fn scoped(&self, epoch: Epoch) -> Option<Scoped<S>> {
let (scheme, remaining) = self.scopes.get(&epoch)?;
remaining
.try_update(Ordering::AcqRel, Ordering::Acquire, |n| n.checked_sub(1))
.ok()?;
Some(Scoped::scheme(Arc::clone(scheme)))
}
}
#[test_traced("WARN")]
fn test_standard_block_provider_parent_fetches_by_commitment() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let buffer = RecordingBuffer::default();
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"standard-provider-parent-commitment",
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(buffer),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let parent = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let child = make_raw_block(parent.digest(), Height::new(2), 200);
let subscription = mailbox.subscribe_parent(&ready(child.clone()));
context.sleep(Duration::from_millis(100)).await;
assert_eq!(
buffer.commitment_subscription_count(),
1,
"parent walkback should use the standard parent commitment"
);
drop(subscription);
});
}
fn assert_finalize_deterministic<H: TestHarness>(
seed: u64,
link: commonware_p2p::simulated::Link,
quorum_sees_finalization: bool,
) {
let r1 = harness::finalize::<H>(seed, link.clone(), quorum_sees_finalization);
let r2 = harness::finalize::<H>(seed, link, quorum_sees_finalization);
assert_eq!(r1, r2);
}
fn assert_hailstorm_deterministic<H: TestHarness>(seed: u64) {
let r1 = harness::hailstorm::<H>(seed, 4, 4, 1, LINK);
let r2 = harness::hailstorm::<H>(seed, 4, 4, 1, LINK);
assert_eq!(r1, r2);
}
fn assert_hailstorm_multi_deterministic<H: TestHarness>(seed: u64) {
let r1 = harness::hailstorm::<H>(seed, 4, 4, 2, LINK);
let r2 = harness::hailstorm::<H>(seed, 4, 4, 2, LINK);
assert_eq!(r1, r2);
}
#[test_group("slow")]
#[test_traced("WARN")]
fn test_standard_finalize_good_links() {
for seed in 0..5 {
assert_finalize_deterministic::<InlineHarness>(seed, LINK, false);
assert_finalize_deterministic::<DeferredHarness>(seed, LINK, false);
}
}
#[test_group("slow")]
#[test_traced("WARN")]
fn test_standard_finalize_bad_links() {
for seed in 0..5 {
assert_finalize_deterministic::<InlineHarness>(seed, UNRELIABLE_LINK, false);
assert_finalize_deterministic::<DeferredHarness>(seed, UNRELIABLE_LINK, false);
}
}
#[test_group("slow")]
#[test_traced("WARN")]
fn test_standard_finalize_good_links_quorum_sees_finalization() {
for seed in 0..5 {
assert_finalize_deterministic::<InlineHarness>(seed, LINK, true);
assert_finalize_deterministic::<DeferredHarness>(seed, LINK, true);
}
}
#[test_group("slow")]
#[test_traced("WARN")]
fn test_standard_finalize_bad_links_quorum_sees_finalization() {
for seed in 0..5 {
assert_finalize_deterministic::<InlineHarness>(seed, UNRELIABLE_LINK, true);
assert_finalize_deterministic::<DeferredHarness>(seed, UNRELIABLE_LINK, true);
}
}
#[test_group("slow")]
#[test_traced("WARN")]
fn test_standard_hailstorm_restarts() {
for seed in 0..2 {
assert_hailstorm_deterministic::<InlineHarness>(seed);
assert_hailstorm_deterministic::<DeferredHarness>(seed);
}
}
#[test_group("slow")]
#[test_traced("WARN")]
fn test_standard_hailstorm_multi_restarts() {
for seed in 0..2 {
assert_hailstorm_multi_deterministic::<InlineHarness>(seed);
assert_hailstorm_multi_deterministic::<DeferredHarness>(seed);
}
}
#[test_group("slow")]
#[test_traced("WARN")]
fn test_standard_hailstorm_restarts_across_epochs() {
let _ = harness::hailstorm::<InlineHarness>(13, 5, 5, 2, LINK);
let _ = harness::hailstorm::<DeferredHarness>(13, 5, 5, 2, LINK);
}
#[test_traced("WARN")]
fn test_standard_ack_pipeline_backlog() {
harness::ack_pipeline_backlog::<InlineHarness>();
harness::ack_pipeline_backlog::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_ack_pipeline_backlog_persists_on_restart() {
harness::ack_pipeline_backlog_persists_on_restart::<InlineHarness>();
harness::ack_pipeline_backlog_persists_on_restart::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_genesis_emitted_once() {
harness::genesis_emitted_once::<InlineHarness>();
harness::genesis_emitted_once::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_proposed_success_implies_recoverable_after_restart() {
harness::proposed_success_implies_recoverable_after_restart::<InlineHarness>(0..16);
harness::proposed_success_implies_recoverable_after_restart::<DeferredHarness>(0..16);
}
#[test_traced("WARN")]
fn test_standard_verified_success_implies_recoverable_after_restart() {
harness::verified_success_implies_recoverable_after_restart::<InlineHarness>(0..16);
harness::verified_success_implies_recoverable_after_restart::<DeferredHarness>(0..16);
}
#[test_traced("WARN")]
fn test_standard_certify_persists_equivocated_block() {
harness::certify_persists_equivocated_block::<InlineHarness>();
harness::certify_persists_equivocated_block::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_verified_after_restart_reverify_same_round_implies_recoverable() {
harness::verified_after_restart_reverify_same_round_implies_recoverable::<InlineHarness>();
harness::verified_after_restart_reverify_same_round_implies_recoverable::<DeferredHarness>(
);
}
#[test_traced("WARN")]
fn test_standard_certify_after_restart_reverify_same_round_implies_recoverable() {
harness::certify_after_restart_reverify_same_round_implies_recoverable::<InlineHarness>();
harness::certify_after_restart_reverify_same_round_implies_recoverable::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_certified_success_implies_recoverable_after_restart() {
harness::certified_success_implies_recoverable_after_restart::<InlineHarness>(0..16);
harness::certified_success_implies_recoverable_after_restart::<DeferredHarness>(0..16);
}
#[test_traced("WARN")]
fn test_standard_certify_at_later_view_survives_earlier_view_pruning() {
harness::certify_at_later_view_survives_earlier_view_pruning::<InlineHarness>();
harness::certify_at_later_view_survives_earlier_view_pruning::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_delivery_visibility_implies_recoverable_after_restart() {
harness::delivery_visibility_implies_recoverable_after_restart::<InlineHarness>(0..16);
harness::delivery_visibility_implies_recoverable_after_restart::<DeferredHarness>(0..16);
}
#[test_traced("WARN")]
fn test_standard_sync_height_floor() {
harness::sync_height_floor::<InlineHarness>();
harness::sync_height_floor::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_reject_stale_block_delivery_after_floor_update() {
harness::reject_stale_block_delivery_after_floor_update::<InlineHarness>();
harness::reject_stale_block_delivery_after_floor_update::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_commitment_fetch_height_hint_mismatch_wakes_subscriber() {
harness::commitment_fetch_height_hint_mismatch_wakes_subscriber::<InlineHarness>();
harness::commitment_fetch_height_hint_mismatch_wakes_subscriber::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_prune_finalized_archives() {
harness::prune_finalized_archives::<InlineHarness>();
harness::prune_finalized_archives::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_floor_retains_processed_predecessor() {
harness::floor_retains_processed_predecessor::<InlineHarness>();
harness::floor_retains_processed_predecessor::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_subscribe_basic_block_delivery() {
harness::subscribe_basic_block_delivery::<InlineHarness>();
harness::subscribe_basic_block_delivery::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_subscribe_multiple_subscriptions() {
harness::subscribe_multiple_subscriptions::<InlineHarness>();
harness::subscribe_multiple_subscriptions::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_subscribe_canceled_subscriptions() {
harness::subscribe_canceled_subscriptions::<InlineHarness>();
harness::subscribe_canceled_subscriptions::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_subscribe_blocks_from_different_sources() {
harness::subscribe_blocks_from_different_sources::<InlineHarness>();
harness::subscribe_blocks_from_different_sources::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_get_info_basic_queries_present_and_missing() {
harness::get_info_basic_queries_present_and_missing::<InlineHarness>();
harness::get_info_basic_queries_present_and_missing::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_get_info_latest_progression_multiple_finalizations() {
harness::get_info_latest_progression_multiple_finalizations::<InlineHarness>();
harness::get_info_latest_progression_multiple_finalizations::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_get_block_by_height_and_latest() {
harness::get_block_by_height_and_latest::<InlineHarness>();
harness::get_block_by_height_and_latest::<DeferredHarness>();
}
async fn seed_inconsistent_restart_state(
context: deterministic::Context,
partition_prefix: &str,
blocks: &[B],
finalizations: &[(Height, Finalization<S, D>)],
) {
let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE);
let replay_buffer = NonZeroUsize::new(1024).unwrap();
let write_buffer = NonZeroUsize::new(1024).unwrap();
let items_per_section = NonZeroU64::new(10).unwrap();
let mut finalizations_by_height = immutable::Archive::init(
context.child("seed_finalizations_by_height"),
immutable::Config {
metadata_partition: format!("{partition_prefix}-finalizations-by-height-metadata"),
freezer_table_partition: format!(
"{partition_prefix}-finalizations-by-height-freezer-table"
),
freezer_table_initial_size: 64,
freezer_table_resize_frequency: 10,
freezer_table_resize_chunk_size: 10,
freezer_key_partition: format!(
"{partition_prefix}-finalizations-by-height-freezer-key"
),
freezer_key_page_cache: page_cache.clone(),
freezer_value_partition: format!(
"{partition_prefix}-finalizations-by-height-freezer-value"
),
freezer_value_target_size: 1024,
freezer_value_compression: None,
ordinal_partition: format!("{partition_prefix}-finalizations-by-height-ordinal"),
items_per_section,
codec_config: S::certificate_codec_config_unbounded(),
replay_buffer,
freezer_key_write_buffer: write_buffer,
freezer_value_write_buffer: write_buffer,
ordinal_write_buffer: write_buffer,
},
)
.await
.expect("failed to initialize finalizations archive for seeded restart state");
let mut finalized_blocks = immutable::Archive::init(
context.child("seed_finalized_blocks"),
immutable::Config {
metadata_partition: format!("{partition_prefix}-finalized_blocks-metadata"),
freezer_table_partition: format!(
"{partition_prefix}-finalized_blocks-freezer-table"
),
freezer_table_initial_size: 64,
freezer_table_resize_frequency: 10,
freezer_table_resize_chunk_size: 10,
freezer_key_partition: format!("{partition_prefix}-finalized_blocks-freezer-key"),
freezer_key_page_cache: page_cache,
freezer_value_partition: format!(
"{partition_prefix}-finalized_blocks-freezer-value"
),
freezer_value_target_size: 1024,
freezer_value_compression: None,
ordinal_partition: format!("{partition_prefix}-finalized_blocks-ordinal"),
items_per_section,
codec_config: (),
replay_buffer,
freezer_key_write_buffer: write_buffer,
freezer_value_write_buffer: write_buffer,
ordinal_write_buffer: write_buffer,
},
)
.await
.expect("failed to initialize finalized blocks archive for seeded restart state");
for block in blocks {
finalized_blocks = finalized_blocks
.put(block.height().get(), block.digest(), block)
.await
.expect("failed to seed finalized block");
}
finalized_blocks
.sync()
.await
.expect("failed to sync seeded finalized blocks");
for (height, finalization) in finalizations {
finalizations_by_height = finalizations_by_height
.put(height.get(), finalization.proposal.payload, finalization)
.await
.expect("failed to seed finalization");
}
finalizations_by_height
.sync()
.await
.expect("failed to sync seeded finalizations");
}
async fn seed_processed_height(
context: deterministic::Context,
partition_prefix: &str,
height: Height,
round: Option<Round>,
) {
let mut metadata: Metadata<deterministic::Context, U64, UInt<u64>> = Metadata::init(
context.child("seed_application_metadata"),
metadata::Config {
partition: format!("{partition_prefix}-application-metadata"),
codec_config: (),
},
)
.await
.expect("failed to initialize application metadata for seeded restart state");
metadata.clear();
metadata.put(U64::new(0xFF), UInt(height.get()));
if let Some(round) = round {
metadata.put(U64::new(0xFE), UInt(round.epoch().get()));
metadata.put(U64::new(0xFD), UInt(round.view().get()));
}
metadata
.sync()
.await
.expect("failed to sync seeded application metadata");
}
async fn seed_cache_block(
context: deterministic::Context,
partition_prefix: &str,
epoch: Epoch,
view: View,
block: &B,
) {
let cache = cache::Manager::<_, Standard<B>, S>::init(
context.child("seed_cache"),
cache::Config {
partition_prefix: format!("{partition_prefix}-cache"),
prunable_items_per_section: NZU64!(10),
replay_buffer: NZUsize!(1024),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
key_page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
},
(),
)
.await;
let (_cache, synced) = cache
.put_notarized(
Round::new(epoch, view),
block.digest(),
&ready(block.clone()),
)
.await;
synced.await.expect("seeded notarized block must sync");
}
#[test_traced("WARN")]
fn test_standard_restart_repairs_trailing_missing_finalized_block() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(3),
participants.clone(),
1024 * 1024,
)
.await;
setup_network_links(&mut oracle, &participants, LINK).await;
let recovering_validator = participants[0].clone();
let peer_validator = participants[1].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let block_one = make_raw_block(genesis.digest(), Height::new(1), 100);
let block_two = make_raw_block(block_one.digest(), Height::new(2), 200);
let finalization_two = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(2)),
View::new(1),
block_two.digest(),
),
&schemes,
3,
);
let mut peer_mailbox = StandardHarness::setup_validator(
context.child("peer_validator"),
&mut oracle,
peer_validator.clone(),
ConstantProvider::new(schemes[1].clone()),
)
.await
.mailbox;
assert!(
peer_mailbox
.verified(
Round::new(Epoch::zero(), View::new(1)),
ready(block_one.clone())
)
.await
);
assert!(
peer_mailbox
.verified(
Round::new(Epoch::zero(), View::new(2)),
ready(block_two.clone())
)
.await
);
StandardHarness::report_finalization(&mut peer_mailbox, finalization_two.clone()).await;
context.sleep(Duration::from_millis(200)).await;
let partition_prefix = format!("validator-{recovering_validator}");
seed_inconsistent_restart_state(
context.child("storage"),
&partition_prefix,
&[block_one],
&[(Height::new(2), finalization_two)],
)
.await;
let recovering = StandardHarness::setup_validator_with(
context.child("recovering_validator"),
&mut oracle,
recovering_validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
crate::marshal::mocks::application::Application::manual_ack(),
)
.await;
assert_eq!(recovering.application.acknowledged().await, Height::zero());
for expected_height in 1..=2 {
let h = recovering.application.acknowledged().await;
assert_eq!(h, Height::new(expected_height));
}
});
}
#[test_traced("WARN")]
fn test_standard_restart_repairs_internal_missing_finalized_block() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(3),
participants.clone(),
1024 * 1024,
)
.await;
setup_network_links(&mut oracle, &participants, LINK).await;
let recovering_validator = participants[0].clone();
let peer_validator = participants[1].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let block_one = make_raw_block(genesis.digest(), Height::new(1), 100);
let block_two = make_raw_block(block_one.digest(), Height::new(2), 200);
let block_three = make_raw_block(block_two.digest(), Height::new(3), 300);
let finalization_two = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(2)),
View::new(1),
block_two.digest(),
),
&schemes,
3,
);
let finalization_three = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(3)),
View::new(2),
block_three.digest(),
),
&schemes,
3,
);
let mut peer_mailbox = StandardHarness::setup_validator(
context.child("peer_validator"),
&mut oracle,
peer_validator.clone(),
ConstantProvider::new(schemes[1].clone()),
)
.await
.mailbox;
assert!(
peer_mailbox
.verified(
Round::new(Epoch::zero(), View::new(1)),
ready(block_one.clone())
)
.await
);
assert!(
peer_mailbox
.verified(
Round::new(Epoch::zero(), View::new(2)),
ready(block_two.clone())
)
.await
);
assert!(
peer_mailbox
.verified(
Round::new(Epoch::zero(), View::new(3)),
ready(block_three.clone())
)
.await
);
StandardHarness::report_finalization(&mut peer_mailbox, finalization_two.clone()).await;
StandardHarness::report_finalization(&mut peer_mailbox, finalization_three.clone())
.await;
context.sleep(Duration::from_millis(200)).await;
let partition_prefix = format!("validator-{recovering_validator}");
seed_inconsistent_restart_state(
context.child("storage"),
&partition_prefix,
&[block_one, block_three.clone()],
&[
(Height::new(2), finalization_two),
(Height::new(3), finalization_three),
],
)
.await;
let recovering = StandardHarness::setup_validator_with(
context.child("recovering_validator"),
&mut oracle,
recovering_validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
crate::marshal::mocks::application::Application::manual_ack(),
)
.await;
assert_eq!(recovering.application.acknowledged().await, Height::zero());
for expected_height in 1..=3 {
let h = recovering.application.acknowledged().await;
assert_eq!(h, Height::new(expected_height));
}
});
}
#[test_traced("WARN")]
fn test_standard_restart_surfaces_block_without_finalization() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(3),
participants.clone(),
1024 * 1024,
)
.await;
setup_network_links(&mut oracle, &participants, LINK).await;
let recovering_validator = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let block_one = make_raw_block(genesis.digest(), Height::new(1), 100);
let block_two = make_raw_block(block_one.digest(), Height::new(2), 200);
let finalization_one = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(1)),
View::zero(),
block_one.digest(),
),
&schemes,
3,
);
let partition_prefix = format!("validator-{recovering_validator}");
seed_inconsistent_restart_state(
context.child("storage"),
&partition_prefix,
&[block_one.clone(), block_two.clone()],
&[(Height::new(1), finalization_one)],
)
.await;
let recovering = StandardHarness::setup_validator_with(
context.child("recovering_validator"),
&mut oracle,
recovering_validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
crate::marshal::mocks::application::Application::manual_ack(),
)
.await;
assert_eq!(
recovering.mailbox.get_info(Identifier::Latest).await,
Some((Height::new(1), block_one.digest())),
"latest tip should be derived from the highest stored finalization"
);
assert_eq!(
recovering
.mailbox
.get_block(Height::new(2))
.await
.map(|block| block.inner_shared()),
Some(block_two.clone().into()),
"block without a finalization row should still be queryable by height"
);
assert_eq!(
recovering.mailbox.get_info(Height::new(2)).await,
Some((Height::new(2), block_two.digest())),
"block info should fall back to the finalized-block archive"
);
assert_eq!(recovering.application.acknowledged().await, Height::zero());
for expected_height in 1..=2 {
let h = recovering.application.acknowledged().await;
assert_eq!(h, Height::new(expected_height));
}
});
}
#[test_traced("WARN")]
fn test_standard_stores_reported_finalization_for_processed_block() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(3),
participants.clone(),
1024 * 1024,
)
.await;
setup_network_links(&mut oracle, &participants, LINK).await;
let recovering_validator = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let block_one = make_raw_block(genesis.digest(), Height::new(1), 100);
let block_two = make_raw_block(block_one.digest(), Height::new(2), 200);
let finalization_one = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(1)),
View::zero(),
block_one.digest(),
),
&schemes,
3,
);
let finalization_two = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(2)),
View::new(1),
block_two.digest(),
),
&schemes,
3,
);
let partition_prefix = format!("validator-{recovering_validator}");
seed_inconsistent_restart_state(
context.child("storage"),
&partition_prefix,
&[block_one.clone(), block_two.clone()],
&[(Height::new(1), finalization_one)],
)
.await;
let mut recovering = StandardHarness::setup_validator_with(
context.child("recovering_validator"),
&mut oracle,
recovering_validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
crate::marshal::mocks::application::Application::manual_ack(),
)
.await;
assert_eq!(recovering.application.acknowledged().await, Height::zero());
for expected_height in 1..=2 {
let h = recovering.application.acknowledged().await;
assert_eq!(h, Height::new(expected_height));
}
while recovering.mailbox.get_processed().await != Some(Processed::Block(Height::new(2)))
{
context.sleep(Duration::from_millis(10)).await;
}
assert!(
recovering
.mailbox
.get_finalization(Height::new(2))
.await
.is_none()
);
StandardHarness::report_finalization(&mut recovering.mailbox, finalization_two.clone())
.await;
let stored = recovering
.mailbox
.get_finalization(Height::new(2))
.await
.expect("finalization reported for a stored block must be kept");
assert_eq!(stored.proposal, finalization_two.proposal);
assert_eq!(
recovering.mailbox.get_info(Identifier::Latest).await,
Some((Height::new(2), block_two.digest())),
"latest finalized height must follow the kept finalization"
);
assert_eq!(
recovering.application.tip(),
Some((Height::new(2), block_two.digest())),
"application tip must follow the kept finalization"
);
});
}
#[test_traced("WARN")]
fn test_standard_keeps_late_finalization_for_repaired_block_once() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let one = make_raw_block(genesis.digest(), Height::new(1), 100);
let two = make_raw_block(one.digest(), Height::new(2), 200);
let three = make_raw_block(two.digest(), Height::new(3), 300);
let finalize = |block: &B| {
let view = block.height().get();
StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(view)),
View::new(view - 1),
block.digest(),
),
&schemes,
QUORUM,
)
};
let buffer = RecordingBuffer::default();
for block in [&one, &two, &three] {
buffer.insert(block.clone());
}
let application = Application::<B>::manual_ack();
let (mut mailbox, _buffer, _resolver, _handle) = start_standard_actor(
context.child("validator"),
"late-finalization-repaired",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(buffer),
Start::Genesis(genesis),
)
.await;
assert_eq!(application.acknowledged().await, Height::zero());
StandardHarness::report_finalization(&mut mailbox, finalize(&three)).await;
for height in 1..=3 {
assert_eq!(application.acknowledged().await, Height::new(height));
}
while mailbox.get_processed().await != Some(Processed::Block(Height::new(3))) {
reschedule().await;
}
assert!(mailbox.get_finalization(Height::new(2)).await.is_none());
let finalization = finalize(&two);
StandardHarness::report_finalization(&mut mailbox, finalization.clone()).await;
let kept = mailbox
.get_finalization(Height::new(2))
.await
.expect("finalization for a repaired block must be kept");
assert_eq!(kept.proposal, finalization.proposal);
let sync_metrics = |metrics: String| -> Vec<String> {
metrics
.lines()
.filter(|line| {
!line.starts_with('#')
&& line.contains("_syncs")
&& (line.contains("finalized_blocks")
|| line.contains("finalizations_by_height"))
})
.map(str::to_owned)
.collect()
};
let before = sync_metrics(context.encode());
assert!(
!before.is_empty(),
"sync metrics must cover the finalized archives"
);
for _ in 0..3 {
StandardHarness::report_finalization(&mut mailbox, finalization.clone()).await;
assert!(mailbox.get_finalization(Height::new(2)).await.is_some());
}
assert_eq!(
sync_metrics(context.encode()),
before,
"duplicate reports must not sync the finalized archives"
);
assert_eq!(application.tip(), Some((Height::new(3), three.digest())));
});
}
#[test_traced("WARN")]
fn test_standard_drops_finalization_for_block_below_floor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let one = make_raw_block(genesis.digest(), Height::new(1), 100);
let two = make_raw_block(one.digest(), Height::new(2), 200);
let finalize = |block: &B| {
let view = block.height().get();
StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(view)),
View::new(view - 1),
block.digest(),
),
&schemes,
QUORUM,
)
};
let buffer = RecordingBuffer::default();
buffer.insert(one.clone());
buffer.insert(two.clone());
let application = Application::<B>::manual_ack();
let (mut mailbox, _buffer, _resolver, _handle) = start_standard_actor(
context.child("validator"),
"finalization-below-floor",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(buffer),
Start::Genesis(genesis),
)
.await;
assert_eq!(application.acknowledged().await, Height::zero());
mailbox.set_floor(finalize(&two));
while mailbox.get_processed().await != Some(Processed::Absent(Height::new(1))) {
reschedule().await;
}
assert!(mailbox.get_block(Height::new(1)).await.is_none());
StandardHarness::report_finalization(&mut mailbox, finalize(&one)).await;
assert!(
mailbox.get_finalization(Height::new(1)).await.is_none(),
"a block the archive lacks must not acquire a finalization"
);
});
}
#[test_traced("WARN")]
fn test_standard_kept_finalization_survives_prune_and_restart() {
const PREFIX: &str = "kept-finalization-prune-restart";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
let (schemes, checkpoint) = runner.start_and_recover(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut parent = StandardHarness::genesis_block(NUM_VALIDATORS as u16).digest();
let blocks: Vec<B> = (1..=20)
.map(|height| {
let block = make_raw_block(parent, Height::new(height), height * 100);
parent = block.digest();
block
})
.collect();
let finalize = |height: u64| {
StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(height)),
View::new(height - 1),
blocks[height as usize - 1].digest(),
),
&schemes,
QUORUM,
)
};
let (certificates, mut finalized) =
prunable_finalized_stores::<Arc<B>>(&context, PREFIX).await;
for block in &blocks {
finalized = finalized
.put(
block.height().get(),
block.digest(),
&Arc::new(block.clone()),
)
.await
.expect("seeded block must be stored");
}
let finalized = finalized.sync().await.expect("seeded blocks must sync");
let certificates = certificates
.put(10, blocks[9].digest(), &finalize(10))
.await
.expect("seeded finalization must be stored")
.sync()
.await
.expect("seeded finalization must sync");
seed_processed_height(context.child("metadata"), PREFIX, Height::new(20), None).await;
let config = Config {
epocher: FixedEpocher::new(NZU64!(100)),
..test_config(
PREFIX,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
)
};
let (actor, mut mailbox, floor) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("actor"),
certificates,
finalized,
config,
)
.await;
assert_eq!(
floor.cutoff(),
Cutoff::Round(Round::new(Epoch::zero(), View::new(10)))
);
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let _handle =
actor.start_unbuffered(Application::<B>::manual_ack(), (resolver_rx, resolver));
StandardHarness::report_finalization(&mut mailbox, finalize(20)).await;
assert!(mailbox.get_finalization(Height::new(20)).await.is_some());
mailbox.prune(Height::new(20));
assert!(mailbox.get_finalization(Height::new(10)).await.is_none());
schemes
});
deterministic::Runner::from(checkpoint).start(|context| async move {
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, PREFIX).await;
assert!(
certificates
.has(ArchiveID::Index(20))
.await
.expect("kept finalization lookup must succeed"),
"the kept finalization must survive the unclean shutdown"
);
let config = Config {
epocher: FixedEpocher::new(NZU64!(100)),
..test_config(
PREFIX,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
)
};
let (_actor, _mailbox, floor) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("actor"),
certificates,
finalized,
config,
)
.await;
assert_eq!(
floor.cutoff(),
Cutoff::Round(Round::new(Epoch::zero(), View::new(20))),
"restart must recover the round of the kept finalization"
);
});
}
#[test_traced("WARN")]
fn test_standard_late_finalization_in_batch_keeps_dispatch() {
const PREFIX: &str = "late-finalization-batch";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
schemes,
participants,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let one = make_raw_block(genesis.digest(), Height::new(1), 100);
let two = make_raw_block(one.digest(), Height::new(2), 200);
let finalize = |block: &B| {
let view = block.height().get();
StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(view)),
View::new(view - 1),
block.digest(),
),
&schemes,
QUORUM,
)
};
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, PREFIX).await;
let finalized = finalized
.put(1, one.digest(), &Arc::new(one.clone()))
.await
.expect("seeded block must be stored")
.sync()
.await
.expect("seeded block must sync");
let (actor, mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("actor"),
certificates,
finalized,
test_config(
PREFIX,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let _handle =
actor.start_unbuffered(application.clone(), (resolver_rx, resolver.clone()));
assert_eq!(application.acknowledged().await, Height::zero());
while application.pending_ack_heights() != vec![Height::new(1)] {
reschedule().await;
}
mailbox.hint_finalized(Height::new(1), NonEmptyVec::new(participants[1].clone()));
mailbox.hint_finalized(Height::new(2), NonEmptyVec::new(participants[1].clone()));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::zero()))
);
assert_eq!(resolver.targeted().len(), 2);
let mut responses = Vec::new();
for block in [&two, &one] {
let height = block.height();
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height
}),
tracing::Span::none(),
)),
},
value: (finalize(block), block.clone()).encode(),
response,
})
.accepted()
);
responses.push(response_rx);
}
assert_eq!(application.acknowledge_next(), Some(Height::new(1)));
for response in responses {
assert_eq!(
response.await.expect("marshal must answer each delivery"),
Outcome::Resolved(Resolution::Complete)
);
}
assert!(mailbox.get_finalization(Height::new(1)).await.is_some());
assert!(mailbox.get_finalization(Height::new(2)).await.is_some());
select! {
height = application.acknowledged() => assert_eq!(height, Height::new(2)),
_ = context.sleep(Duration::from_secs(1)) => {
panic!("block two must be dispatched after the batch")
}
}
});
}
#[test_traced("WARN")]
fn test_standard_restart_repairs_multiple_trailing_missing_finalized_blocks() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(3),
participants.clone(),
1024 * 1024,
)
.await;
setup_network_links(&mut oracle, &participants, LINK).await;
let recovering_validator = participants[0].clone();
let peer_validator = participants[1].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let block_one = make_raw_block(genesis.digest(), Height::new(1), 100);
let block_two = make_raw_block(block_one.digest(), Height::new(2), 200);
let block_three = make_raw_block(block_two.digest(), Height::new(3), 300);
let block_four = make_raw_block(block_three.digest(), Height::new(4), 400);
let block_five = make_raw_block(block_four.digest(), Height::new(5), 500);
let mut finalizations = Vec::new();
let blocks = [
&block_one,
&block_two,
&block_three,
&block_four,
&block_five,
];
for (i, block) in blocks.iter().enumerate() {
let view = View::new(block.height().get());
let parent_view = if i == 0 {
View::zero()
} else {
View::new(blocks[i - 1].height().get())
};
finalizations.push(StandardHarness::make_finalization(
Proposal::new(Round::new(Epoch::zero(), view), parent_view, block.digest()),
&schemes,
3,
));
}
let mut peer_mailbox = StandardHarness::setup_validator(
context.child("peer_validator"),
&mut oracle,
peer_validator.clone(),
ConstantProvider::new(schemes[1].clone()),
)
.await
.mailbox;
for (i, block) in blocks.iter().enumerate() {
assert!(
peer_mailbox
.verified(
Round::new(Epoch::zero(), View::new(block.height().get())),
ready((*block).clone()),
)
.await
);
StandardHarness::report_finalization(&mut peer_mailbox, finalizations[i].clone())
.await;
}
context.sleep(Duration::from_millis(200)).await;
let partition_prefix = format!("validator-{recovering_validator}");
seed_inconsistent_restart_state(
context.child("storage"),
&partition_prefix,
&[block_one],
&finalizations
.iter()
.enumerate()
.map(|(i, f)| (Height::new(i as u64 + 1), f.clone()))
.collect::<Vec<_>>(),
)
.await;
let recovering = StandardHarness::setup_validator_with(
context.child("recovering_validator"),
&mut oracle,
recovering_validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
crate::marshal::mocks::application::Application::manual_ack(),
)
.await;
assert_eq!(recovering.application.acknowledged().await, Height::zero());
for expected_height in 1..=5 {
let h = recovering.application.acknowledged().await;
assert_eq!(h, Height::new(expected_height));
}
});
}
#[test_traced("WARN")]
fn test_standard_restart_repairs_large_pending_tip_by_commitment() {
let runner = deterministic::Runner::timed(Duration::from_secs(120));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(3),
participants.clone(),
1024 * 1024,
)
.await;
setup_network_links(&mut oracle, &participants, LINK).await;
let recovering_validator = participants[0].clone();
let peer_validator = participants[1].clone();
let pending_tip = 18;
let mut blocks = Vec::new();
let mut parent = Sha256::hash(&[b""]);
for height in 1..=pending_tip {
let block = make_raw_block(parent, Height::new(height), height * 100);
parent = block.digest();
blocks.push(block);
}
let tip_block = blocks.last().expect("tip block exists");
let tip_finalization = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(pending_tip)),
View::new(pending_tip - 1),
tip_block.digest(),
),
&schemes,
QUORUM,
);
let peer_mailbox = StandardHarness::setup_validator(
context.child("peer_validator"),
&mut oracle,
peer_validator.clone(),
ConstantProvider::new(schemes[1].clone()),
)
.await
.mailbox;
for block in blocks.iter() {
assert!(
peer_mailbox
.verified(
Round::new(Epoch::zero(), View::new(block.height().get())),
ready(block.clone()),
)
.await
);
}
context.sleep(Duration::from_millis(200)).await;
let partition_prefix = format!("validator-{recovering_validator}");
seed_inconsistent_restart_state(
context.child("storage"),
&partition_prefix,
&[blocks[0].clone()],
&[(Height::new(pending_tip), tip_finalization)],
)
.await;
let recovering = StandardHarness::setup_validator_with(
context.child("recovering_validator"),
&mut oracle,
recovering_validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
crate::marshal::mocks::application::Application::manual_ack(),
)
.await;
for _ in 0..100 {
if recovering.application.tip().map(|(height, _)| height)
== Some(Height::new(pending_tip))
{
break;
}
context.sleep(Duration::from_millis(10)).await;
}
assert_eq!(
recovering.application.tip().map(|(height, _)| height),
Some(Height::new(pending_tip)),
"restart should surface the pending finalized tip before all blocks are repaired"
);
assert_eq!(recovering.application.acknowledged().await, Height::zero());
for expected_height in 1..=pending_tip {
let h = recovering.application.acknowledged().await;
assert_eq!(h, Height::new(expected_height));
}
for height in [2, 10, pending_tip] {
let block = recovering
.mailbox
.get_block(Height::new(height))
.await
.unwrap_or_else(|| panic!("block {height} should be recoverable"));
assert_eq!(block.digest(), blocks[(height - 1) as usize].digest());
}
});
}
#[test_traced("WARN")]
fn test_standard_restart_no_trailing_finalizations_is_noop() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(3),
participants.clone(),
1024 * 1024,
)
.await;
setup_network_links(&mut oracle, &participants, LINK).await;
let recovering_validator = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let block_one = make_raw_block(genesis.digest(), Height::new(1), 100);
let block_two = make_raw_block(block_one.digest(), Height::new(2), 200);
let finalization_one = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(1)),
View::zero(),
block_one.digest(),
),
&schemes,
3,
);
let finalization_two = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(2)),
View::new(1),
block_two.digest(),
),
&schemes,
3,
);
let partition_prefix = format!("validator-{recovering_validator}");
seed_inconsistent_restart_state(
context.child("storage"),
&partition_prefix,
&[block_one.clone(), block_two.clone()],
&[
(Height::new(1), finalization_one),
(Height::new(2), finalization_two),
],
)
.await;
let recovering = StandardHarness::setup_validator_with(
context.child("recovering_validator"),
&mut oracle,
recovering_validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
crate::marshal::mocks::application::Application::manual_ack(),
)
.await;
assert_eq!(recovering.application.acknowledged().await, Height::zero());
for expected_height in 1..=2 {
let h = recovering.application.acknowledged().await;
assert_eq!(h, Height::new(expected_height));
}
});
}
#[test_traced("WARN")]
fn test_standard_restart_repairs_trailing_block_from_local_cache() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(3),
participants.clone(),
1024 * 1024,
)
.await;
let recovering_validator = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let block_one = make_raw_block(genesis.digest(), Height::new(1), 100);
let block_two = make_raw_block(block_one.digest(), Height::new(2), 200);
let finalization_two = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(2)),
View::new(1),
block_two.digest(),
),
&schemes,
3,
);
let partition_prefix = format!("validator-{recovering_validator}");
seed_cache_block(
context.child("storage"),
&partition_prefix,
Epoch::zero(),
View::new(2),
&block_two,
)
.await;
seed_inconsistent_restart_state(
context.child("storage"),
&partition_prefix,
&[block_one],
&[(Height::new(2), finalization_two)],
)
.await;
let recovering = StandardHarness::setup_validator_with(
context.child("recovering_validator"),
&mut oracle,
recovering_validator,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
crate::marshal::mocks::application::Application::manual_ack(),
)
.await;
assert_eq!(recovering.application.acknowledged().await, Height::zero());
for expected_height in 1..=2 {
let h = recovering.application.acknowledged().await;
assert_eq!(h, Height::new(expected_height));
}
});
}
#[test_traced("WARN")]
fn test_cache_load_persisted_epochs_finds_blocks() {
let executor = deterministic::Runner::timed(Duration::from_secs(10));
executor.start(|context| async move {
let prefix = "test-cache";
let make_cfg = || cache::Config {
partition_prefix: prefix.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NonZeroUsize::new(1024).unwrap(),
key_write_buffer: NonZeroUsize::new(1024).unwrap(),
value_write_buffer: NonZeroUsize::new(1024).unwrap(),
key_page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
};
let block = ready(make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100));
let digest = block.digest();
let round = Round::new(Epoch::zero(), View::new(1));
{
let mgr = cache::Manager::<_, Standard<B>, S>::init(
context.child("write"),
make_cfg(),
(),
)
.await;
let (_mgr, handle) = mgr.put_notarized(round, digest, &block).await;
handle.await.expect("failed to sync block");
}
let mut mgr =
cache::Manager::<_, Standard<B>, S>::init(context.child("read"), make_cfg(), ())
.await;
assert_eq!(
mgr.find_block_matching(digest, |_| true).await,
None,
"cache should not find block before loading persisted epochs"
);
mgr = mgr.load_persisted_epochs().await;
assert_eq!(
mgr.find_block_matching(digest, |_| true)
.await
.map(|block| block.inner_shared()),
Some(block.inner_shared()),
"cache should find block after loading persisted epochs"
);
});
}
fn opened_cache_epochs(context: &deterministic::Context, root: &str) -> Vec<u64> {
let needle = format!("{root}_cache_finalizations_items_tracked{{");
let mut epochs: Vec<u64> = context
.encode()
.lines()
.filter(|line| line.starts_with(&needle))
.map(|line| {
let start = line
.find("epoch=\"")
.expect("gauge must have an epoch label")
+ "epoch=\"".len();
let end = start
+ line[start..]
.find('"')
.expect("epoch label must have a closing quote");
line[start..end]
.parse()
.expect("epoch label must be an integer")
})
.collect();
epochs.sort_unstable();
epochs
}
#[test_traced("WARN")]
fn test_cache_restart_after_far_epoch_write_reopens_only_written_epochs() {
let executor = deterministic::Runner::timed(Duration::from_secs(60));
executor.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let prefix = "test-cache-far-epoch";
let make_cfg = || cache::Config {
partition_prefix: prefix.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NonZeroUsize::new(1024).unwrap(),
key_write_buffer: NonZeroUsize::new(1024).unwrap(),
value_write_buffer: NonZeroUsize::new(1024).unwrap(),
key_page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
};
let block = make_raw_block(Sha256::hash(&[b"far"]), Height::new(1290), 100);
let digest = block.digest();
let round = Round::new(Epoch::new(64), View::new(1290));
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::new(1289), digest),
&schemes,
QUORUM,
);
{
let mgr = cache::Manager::<_, Standard<B>, S>::init(
context.child("write"),
make_cfg(),
(),
)
.await;
let _mgr = mgr.put_finalization(round, digest, &finalization).await;
}
let mgr =
cache::Manager::<_, Standard<B>, S>::init(context.child("read"), make_cfg(), ())
.await;
let mgr = mgr.load_persisted_epochs().await;
assert!(
mgr.get_finalization_for(digest).await.is_some(),
"cached finalization must survive restart"
);
assert_eq!(
opened_cache_epochs(&context, "read"),
vec![64],
"reopen must open only the written epoch"
);
});
}
#[test_traced("WARN")]
fn test_standard_start_floor_restart_before_anchor_reopens_only_floor_epoch() {
let partition_prefix = "start-floor-far-epoch-restart";
let runner = deterministic::Runner::timed(Duration::from_secs(60));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::new(64), View::new(1290));
let floor_block =
make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(1290), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(1289), floor_block.digest()),
&schemes,
QUORUM,
);
let (mailbox, _buffer, resolver, handle) = start_standard_actor(
context.child("first"),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Floor(floor_finalization.clone()),
)
.await;
wait_until(
&context,
Duration::from_secs(5),
"floor fetch issued",
|| !resolver.fetches().is_empty(),
)
.await;
assert_eq!(opened_cache_epochs(&context, "first_actor_cache"), vec![64]);
handle.abort();
let _ = handle.await;
drop(mailbox);
let (_mailbox, _buffer, resolver, _handle) = start_standard_actor(
context.child("restarted"),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Floor(floor_finalization),
)
.await;
wait_until(
&context,
Duration::from_secs(5),
"floor fetch reissued",
|| !resolver.fetches().is_empty(),
)
.await;
assert_eq!(
opened_cache_epochs(&context, "restarted_actor_cache"),
vec![64],
"restart must open only the floor's epoch"
);
});
}
#[test_traced("WARN")]
fn test_cache_write_to_skipped_epoch_initializes_on_demand() {
let executor = deterministic::Runner::timed(Duration::from_secs(60));
executor.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let prefix = "test-cache-skipped-epoch";
let make_cfg = || cache::Config {
partition_prefix: prefix.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NonZeroUsize::new(1024).unwrap(),
key_write_buffer: NonZeroUsize::new(1024).unwrap(),
value_write_buffer: NonZeroUsize::new(1024).unwrap(),
key_page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
};
let far = make_raw_block(Sha256::hash(&[b"far"]), Height::new(1290), 100);
let far_round = Round::new(Epoch::new(64), View::new(1290));
let far_finalization = StandardHarness::make_finalization(
Proposal::new(far_round, View::new(1289), far.digest()),
&schemes,
QUORUM,
);
{
let mgr = cache::Manager::<_, Standard<B>, S>::init(
context.child("write"),
make_cfg(),
(),
)
.await;
let _mgr = mgr
.put_finalization(far_round, far.digest(), &far_finalization)
.await;
}
let mgr =
cache::Manager::<_, Standard<B>, S>::init(context.child("reopen"), make_cfg(), ())
.await
.load_persisted_epochs()
.await;
assert_eq!(opened_cache_epochs(&context, "reopen"), vec![64]);
let old = make_raw_block(Sha256::hash(&[b"old"]), Height::new(210), 100);
let old_round = Round::new(Epoch::new(10), View::new(210));
let notarization = StandardHarness::make_notarization(
Proposal::new(old_round, View::new(209), old.digest()),
&schemes,
QUORUM,
);
let (mgr, synced) = mgr
.put_notarization(old_round, old.digest(), ¬arization)
.await;
synced.await.expect("notarization must sync");
assert!(mgr.get_notarization(old_round).await.is_some());
assert_eq!(opened_cache_epochs(&context, "reopen"), vec![10, 64]);
drop(mgr);
let mgr =
cache::Manager::<_, Standard<B>, S>::init(context.child("again"), make_cfg(), ())
.await
.load_persisted_epochs()
.await;
assert_eq!(opened_cache_epochs(&context, "again"), vec![10, 64]);
assert!(mgr.get_notarization(old_round).await.is_some());
assert!(mgr.get_finalization_for(far.digest()).await.is_some());
});
}
#[test_traced("WARN")]
fn test_cache_restart_reopens_epoch_left_by_interrupted_destroy() {
for (survivor, emptied) in [("metadata", true), ("key", false), ("value", false)] {
let executor = deterministic::Runner::timed(Duration::from_secs(60));
executor.start(|mut context| async move {
let Fixture { schemes, .. } = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let prefix = "test-cache-interrupted-destroy";
let make_cfg = || cache::Config {
partition_prefix: prefix.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NonZeroUsize::new(1024).unwrap(),
key_write_buffer: NonZeroUsize::new(1024).unwrap(),
value_write_buffer: NonZeroUsize::new(1024).unwrap(),
key_page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
};
let epoch = Epoch::new(64);
let cfg = make_cfg();
let partitions: Vec<String> = cache::ARCHIVES
.into_iter()
.flat_map(|name| {
cache::Manager::<deterministic::Context, Standard<B>, S>::partitions(
&cfg, epoch, name,
)
})
.collect();
let remaining = |context: deterministic::Context| {
let partitions = partitions.clone();
async move {
let mut remaining = Vec::new();
for partition in partitions {
if context.scan(&partition).await.is_ok() {
remaining.push(partition);
}
}
remaining
}
};
let block = make_raw_block(Sha256::hash(&[b"far"]), Height::new(1290), 100);
let digest = block.digest();
let round = Round::new(epoch, View::new(1290));
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::new(1289), digest),
&schemes,
QUORUM,
);
{
let mgr = cache::Manager::<_, Standard<B>, S>::init(
context.child("write"),
make_cfg(),
(),
)
.await;
let _mgr = mgr.put_finalization(round, digest, &finalization).await;
}
let kept = format!("{prefix}-cache-64-finalizations-{survivor}");
for partition in partitions.iter().filter(|partition| **partition != kept) {
match context.remove(partition, None).await {
Ok(()) | Err(commonware_runtime::Error::PartitionMissing(_)) => {}
Err(err) => panic!("partition must be removable: {err}"),
}
}
if emptied {
for blob in context.scan(&kept).await.expect("kept partition must scan") {
context
.remove(&kept, Some(&blob))
.await
.expect("blob must be removable");
}
assert!(
context
.scan(&kept)
.await
.expect("empty partition must scan")
.is_empty()
);
}
assert_eq!(remaining(context.child("remnant")).await, vec![kept]);
let mgr = cache::Manager::<_, Standard<B>, S>::init(
context.child("reopen"),
make_cfg(),
(),
)
.await
.load_persisted_epochs()
.await;
assert_eq!(opened_cache_epochs(&context, "reopen"), vec![64]);
let _mgr = mgr
.prune_by_view(Round::new(Epoch::new(65), View::zero()))
.await;
assert!(remaining(context.child("retired")).await.is_empty());
});
}
}
#[test_traced("WARN")]
fn test_cache_start_sync_notarizations_covers_prior_write() {
let executor = deterministic::Runner::timed(Duration::from_secs(10));
executor.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let prefix = "test-cache-notarizations";
let make_cfg = || cache::Config {
partition_prefix: prefix.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NonZeroUsize::new(1024).unwrap(),
key_write_buffer: NonZeroUsize::new(1024).unwrap(),
value_write_buffer: NonZeroUsize::new(1024).unwrap(),
key_page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
};
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let digest = block.digest();
let round = Round::new(Epoch::zero(), View::new(1));
let notarization = StandardHarness::make_notarization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
{
let mgr = cache::Manager::<_, Standard<B>, S>::init(
context.child("write"),
make_cfg(),
(),
)
.await;
let (mgr, handle) = mgr.put_notarization(round, digest, ¬arization).await;
drop(handle);
let (_mgr, sync) = mgr.start_sync_notarizations(round).await;
sync.await.expect("failed to sync notarizations");
}
let mgr =
cache::Manager::<_, Standard<B>, S>::init(context.child("read"), make_cfg(), ())
.await;
let mgr = mgr.load_persisted_epochs().await;
assert!(
mgr.get_notarization(round).await.is_some(),
"notarization covered by the barrier must be durable"
);
});
}
#[test_traced("WARN")]
fn test_standard_get_block_by_commitment_from_sources_and_missing() {
harness::get_block_by_commitment_from_sources_and_missing::<InlineHarness>();
harness::get_block_by_commitment_from_sources_and_missing::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_get_finalization_by_height() {
harness::get_finalization_by_height::<InlineHarness>();
harness::get_finalization_by_height::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_hint_finalized_triggers_fetch() {
harness::hint_finalized_triggers_fetch::<InlineHarness>();
harness::hint_finalized_triggers_fetch::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_ancestry_stream() {
harness::ancestry_stream::<InlineHarness>();
harness::ancestry_stream::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_finalize_same_height_different_views() {
harness::finalize_same_height_different_views::<InlineHarness>();
harness::finalize_same_height_different_views::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_init_processed_height() {
harness::init_processed_height::<InlineHarness>();
harness::init_processed_height::<DeferredHarness>();
}
#[test_traced("INFO")]
fn test_standard_broadcast_caches_block() {
harness::broadcast_caches_block::<InlineHarness>();
harness::broadcast_caches_block::<DeferredHarness>();
}
#[test_traced("INFO")]
fn test_standard_rejects_block_delivery_below_floor() {
harness::reject_stale_block_delivery_after_floor_update::<InlineHarness>();
harness::reject_stale_block_delivery_after_floor_update::<DeferredHarness>();
}
#[test_traced("WARN")]
fn test_standard_buffered_block_installs_floor_anchor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
participants[0].clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let app = setup.application;
let mailbox = setup.mailbox;
let buffer = setup.extra;
const ANCHOR_HEIGHT: u64 = 5;
let mut parent = Sha256::hash(&[b""]);
let mut anchor = None;
for i in 1..=ANCHOR_HEIGHT {
let block = make_raw_block(parent, Height::new(i), i);
parent = block.digest();
anchor = Some(block);
}
let anchor = anchor.unwrap();
let subscription = mailbox.subscribe_by_digest(anchor.digest(), DigestFallback::Wait);
let round = Round::new(Epoch::zero(), View::new(ANCHOR_HEIGHT));
let finalization = StandardHarness::make_finalization(
Proposal {
round,
parent: View::new(ANCHOR_HEIGHT - 1),
payload: anchor.digest(),
},
&schemes,
QUORUM,
);
mailbox.set_floor(finalization);
assert!(
mailbox
.get_block(Identifier::Height(Height::new(ANCHOR_HEIGHT)))
.await
.is_none()
);
let _ = buffer.broadcast(Recipients::All, ready(anchor.clone()));
let received = subscription.await.unwrap();
assert_eq!(received.digest(), anchor.digest());
while !app.blocks().contains_key(&Height::new(ANCHOR_HEIGHT)) {
context.sleep(Duration::from_millis(50)).await;
}
let (tip_height, tip_digest) = app.tip().unwrap();
assert_eq!(tip_height, Height::new(ANCHOR_HEIGHT));
assert_eq!(tip_digest, anchor.digest());
let stored = mailbox
.get_block(Identifier::Height(Height::new(ANCHOR_HEIGHT)))
.await
.unwrap();
assert_eq!(stored.digest(), anchor.digest());
})
}
fn buffered_anchor_installs_floor(subscribe: bool) {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
participants[0].clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let app = setup.application;
let mailbox = setup.mailbox;
let buffer = setup.extra;
let height = Height::new(5);
let anchor = make_raw_block(Sha256::hash(&[b"floor-parent"]), height, 500);
let finalization = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(5)),
View::new(4),
anchor.digest(),
),
&schemes,
QUORUM,
);
mailbox.set_floor(finalization);
assert!(
mailbox
.get_block(Identifier::Height(height))
.await
.is_none()
);
let _ = buffer.broadcast(Recipients::All, ready(anchor.clone()));
if subscribe {
let subscription =
mailbox.subscribe_by_commitment(anchor.digest(), CommitmentFallback::Wait);
let (_, block) = mailbox.get_anchor().await.unwrap();
assert_eq!(block.digest(), anchor.digest());
assert_eq!(subscription.await.unwrap().digest(), anchor.digest());
}
while !app.blocks().contains_key(&height) {
reschedule().await;
}
assert_eq!(app.tip(), Some((height, anchor.digest())));
})
}
#[test_traced("WARN")]
fn test_standard_unsubscribed_buffered_anchor_installs_floor() {
buffered_anchor_installs_floor(false);
}
#[test_traced("WARN")]
fn test_standard_late_subscription_observes_buffered_anchor_floor() {
buffered_anchor_installs_floor(true);
}
#[test_traced("WARN")]
fn test_standard_start_floor_buffered_anchor_installs_floor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let anchor = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let finalization = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(5)),
View::new(4),
anchor.digest(),
),
&schemes,
QUORUM,
);
let (application, started_rx) = HoldingBlockReporter::new_after(Height::zero());
let buffer = RecordingBuffer::default();
let (_mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"start-floor-buffered-anchor",
ConstantProvider::new(schemes[0].clone()),
application,
Some(buffer.clone()),
Start::Floor(finalization),
)
.await;
while buffer.commitment_subscription_count() != 1 {
reschedule().await;
}
assert!(buffer.deliver_commitment_subscription(0, anchor));
assert_eq!(started_rx.await.unwrap(), Height::new(5));
});
}
#[test_traced("WARN")]
fn test_standard_closed_floor_waiter_preserves_subscriptions() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let anchor = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let commitment = anchor.digest();
let (application, started_rx) = HoldingBlockReporter::new_after(Height::zero());
let buffer = RecordingBuffer::default();
let (mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"closed-floor-waiter",
ConstantProvider::new(schemes[0].clone()),
application,
Some(buffer.clone()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
mailbox.set_floor(StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(5)),
View::new(4),
commitment,
),
&schemes,
QUORUM,
));
while buffer.commitment_subscription_count() != 1 {
reschedule().await;
}
let subscription =
mailbox.subscribe_by_commitment(commitment, CommitmentFallback::Wait);
while buffer.commitment_subscription_count() != 2 {
reschedule().await;
}
buffer.close_commitment_subscription(0);
assert!(
mailbox
.get_block(Identifier::Height(Height::new(5)))
.await
.is_none()
);
assert!(!buffer.commitment_subscription_closed(1));
assert!(buffer.deliver_commitment_subscription(1, anchor.clone()));
assert_eq!(subscription.await.unwrap().digest(), anchor.digest());
assert_eq!(started_rx.await.unwrap(), Height::new(5));
});
}
#[test_traced("WARN")]
fn test_standard_floor_preserves_registered_subscriptions() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
participants[0].clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let mailbox = setup.mailbox;
let buffer = setup.extra;
const ANCHOR_HEIGHT: u64 = 5;
let mut parent = Sha256::hash(&[b""]);
let mut anchor = None;
for i in 1..=ANCHOR_HEIGHT {
let block = make_raw_block(parent, Height::new(i), i);
parent = block.digest();
anchor = Some(block);
}
let anchor = anchor.unwrap();
let anchor_round = Round::new(Epoch::zero(), View::new(ANCHOR_HEIGHT));
let missing = make_raw_block(Sha256::hash(&[b"missing-parent"]), Height::new(2), 999);
let missing_digest = missing.digest();
let child = make_raw_block(missing_digest, Height::new(3), 1_000);
let mut by_round = mailbox.subscribe_by_digest(
missing_digest,
DigestFallback::FetchByRound {
round: Round::new(Epoch::zero(), View::new(2)),
},
);
let mut by_height = mailbox.subscribe_by_commitment(
missing_digest,
CommitmentFallback::FetchByCommitment {
height: Height::new(2),
},
);
let mut wait = mailbox.subscribe_by_digest(missing_digest, DigestFallback::Wait);
let mut parent = Box::pin(mailbox.subscribe_parent(&ready(child.clone())));
let anchor_wait = mailbox.subscribe_by_digest(
anchor.digest(),
DigestFallback::FetchByRound {
round: anchor_round,
},
);
let finalization = StandardHarness::make_finalization(
Proposal {
round: anchor_round,
parent: View::new(ANCHOR_HEIGHT - 1),
payload: anchor.digest(),
},
&schemes,
QUORUM,
);
mailbox.set_floor(finalization);
let _ = buffer.broadcast(Recipients::All, ready(anchor.clone()));
let received = anchor_wait.await.unwrap();
assert_eq!(received.digest(), anchor.digest());
let anchored = Some(Processed::Block(Height::new(ANCHOR_HEIGHT)));
while mailbox.get_processed().await != anchored {
context.sleep(Duration::from_millis(50)).await;
}
assert!(matches!(by_round.try_recv(), Err(TryRecvError::Empty)));
assert!(matches!(by_height.try_recv(), Err(TryRecvError::Empty)));
assert!(matches!(wait.try_recv(), Err(TryRecvError::Empty)));
select! {
result = &mut parent => {
panic!("parent subscription resolved at processed floor: {result:?}");
},
_ = context.sleep(Duration::from_millis(100)) => {},
};
let mut late_by_round = mailbox.subscribe_by_digest(
missing_digest,
DigestFallback::FetchByRound {
round: Round::new(Epoch::zero(), View::new(3)),
},
);
let mut late_parent = Box::pin(mailbox.subscribe_parent(&ready(child.clone())));
let _ = mailbox.get_processed().await;
assert!(matches!(late_by_round.try_recv(), Err(TryRecvError::Empty)));
select! {
result = &mut late_parent => {
panic!("late parent subscription resolved before availability: {result:?}");
},
_ = context.sleep(Duration::from_millis(100)) => {},
};
let _ = buffer.broadcast(Recipients::All, ready(missing.clone()));
assert_eq!(by_round.await.unwrap().digest(), missing_digest);
assert_eq!(by_height.await.unwrap().digest(), missing_digest);
assert_eq!(wait.await.unwrap().digest(), missing_digest);
assert_eq!(parent.await.unwrap().digest(), missing_digest);
assert_eq!(late_by_round.await.unwrap().digest(), missing_digest);
assert_eq!(late_parent.await.unwrap().digest(), missing_digest);
})
}
#[test_traced("WARN")]
fn test_standard_resolver_floor_anchor_install_wakes_subscriber() {
let runner = deterministic::Runner::timed(Duration::from_secs(60));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let victim = participants[0].clone();
let server = participants[1].clone();
let peers = vec![victim.clone(), server.clone()];
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
peers.clone(),
1024 * 1024,
)
.await;
let victim_setup = StandardHarness::setup_validator(
context.child("victim"),
&mut oracle,
victim.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let server_setup = StandardHarness::setup_validator(
context.child("server"),
&mut oracle,
server.clone(),
ConstantProvider::new(schemes[1].clone()),
)
.await;
let app = victim_setup.application;
let mailbox = victim_setup.mailbox;
let mut server_handle: ValidatorHandle<StandardHarness> = ValidatorHandle {
mailbox: server_setup.mailbox,
extra: server_setup.extra,
};
const ANCHOR_HEIGHT: u64 = 5;
let mut parent = Sha256::hash(&[b""]);
let mut anchor = None;
for i in 1..=ANCHOR_HEIGHT {
let block = make_raw_block(parent, Height::new(i), i);
parent = block.digest();
anchor = Some(block);
}
let anchor = anchor.unwrap();
let round = Round::new(Epoch::zero(), View::new(ANCHOR_HEIGHT));
StandardHarness::propose(&mut server_handle, round, &ready(anchor.clone())).await;
let subscription = mailbox.subscribe_by_digest(anchor.digest(), DigestFallback::Wait);
let finalization = StandardHarness::make_finalization(
Proposal {
round,
parent: View::new(ANCHOR_HEIGHT - 1),
payload: anchor.digest(),
},
&schemes,
QUORUM,
);
mailbox.set_floor(finalization);
assert!(
mailbox
.get_block(Identifier::Height(Height::new(ANCHOR_HEIGHT)))
.await
.is_none()
);
setup_network_links(&mut oracle, &peers, LINK).await;
let received = select! {
result = subscription => {
result.expect("subscription should receive the floor anchor")
},
_ = context.sleep(Duration::from_secs(10)) => {
panic!("subscriber not woken by installed anchor delivery");
},
};
assert_eq!(received.digest(), anchor.digest());
while !app.blocks().contains_key(&Height::new(ANCHOR_HEIGHT)) {
context.sleep(Duration::from_millis(50)).await;
}
let (tip_height, tip_digest) = app.tip().unwrap();
assert_eq!(tip_height, Height::new(ANCHOR_HEIGHT));
assert_eq!(tip_digest, anchor.digest());
})
}
#[test_traced("WARN")]
fn test_standard_below_floor_anchor_delivery_wakes_subscriber() {
let runner = deterministic::Runner::timed(Duration::from_secs(60));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let victim = participants[0].clone();
let server = participants[1].clone();
let peers = vec![victim.clone(), server.clone()];
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
peers.clone(),
1024 * 1024,
)
.await;
let victim_setup = StandardHarness::setup_validator(
context.child("victim"),
&mut oracle,
victim.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let server_setup = StandardHarness::setup_validator(
context.child("server"),
&mut oracle,
server.clone(),
ConstantProvider::new(schemes[1].clone()),
)
.await;
let app = victim_setup.application;
let mut mailbox = victim_setup.mailbox;
let mut server_handle: ValidatorHandle<StandardHarness> = ValidatorHandle {
mailbox: server_setup.mailbox,
extra: server_setup.extra,
};
let mut parent = Sha256::hash(&[b""]);
let mut canonical = Vec::new();
for i in 1..=3u64 {
let block = make_raw_block(parent, Height::new(i), i);
parent = block.digest();
canonical.push(block);
}
for block in &canonical {
let height = block.height();
let round = Round::new(Epoch::zero(), View::new(height.get()));
assert!(mailbox.verified(round, ready(block.clone())).await);
let finalization = StandardHarness::make_finalization(
Proposal {
round,
parent: View::new(height.get() - 1),
payload: block.digest(),
},
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
}
while !app.blocks().contains_key(&Height::new(3)) {
context.sleep(Duration::from_millis(50)).await;
}
let fork = make_raw_block(canonical[0].digest(), Height::new(2), 999);
let fork_round = Round::new(Epoch::zero(), View::new(4));
StandardHarness::propose(&mut server_handle, fork_round, &ready(fork.clone())).await;
let subscription = mailbox.subscribe_by_digest(fork.digest(), DigestFallback::Wait);
let fork_finalization = StandardHarness::make_finalization(
Proposal {
round: fork_round,
parent: View::new(1),
payload: fork.digest(),
},
&schemes,
QUORUM,
);
mailbox.set_floor(fork_finalization);
assert!(mailbox.get_block(&fork.digest()).await.is_none());
setup_network_links(&mut oracle, &peers, LINK).await;
let received = select! {
result = subscription => {
result.expect("subscription should receive the floor anchor")
},
_ = context.sleep(Duration::from_secs(10)) => {
panic!("subscriber not woken by below-floor anchor delivery");
},
};
assert_eq!(received.digest(), fork.digest());
let next = make_raw_block(canonical[2].digest(), Height::new(4), 4);
let next_round = Round::new(Epoch::zero(), View::new(5));
assert!(mailbox.verified(next_round, ready(next.clone())).await);
let next_finalization = StandardHarness::make_finalization(
Proposal {
round: next_round,
parent: View::new(3),
payload: next.digest(),
},
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, next_finalization).await;
while !app.blocks().contains_key(&Height::new(4)) {
context.sleep(Duration::from_millis(50)).await;
}
})
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum WrapperKind {
Inline,
Deferred,
}
fn wrapper_kinds() -> [WrapperKind; 2] {
[WrapperKind::Inline, WrapperKind::Deferred]
}
type Runtime = deterministic::Context;
type App = MockVerifyingApp<B, S>;
type InlineWrapper = Inline<Runtime, S, App, B, FixedEpocher>;
type DeferredWrapper = Deferred<Runtime, S, App, B, FixedEpocher>;
#[derive(Clone)]
enum Wrapper {
Inline(InlineWrapper),
Deferred(DeferredWrapper),
}
impl Wrapper {
fn new(
kind: WrapperKind,
context: Runtime,
app: App,
marshal: Mailbox<S, Standard<B>>,
) -> Self {
match kind {
WrapperKind::Inline => Self::Inline(Inline::new(
context,
app,
marshal,
FixedEpocher::new(BLOCKS_PER_EPOCH),
)),
WrapperKind::Deferred => Self::Deferred(Deferred::new(
context,
app,
marshal,
FixedEpocher::new(BLOCKS_PER_EPOCH),
)),
}
}
fn kind(&self) -> WrapperKind {
match self {
Self::Inline(_) => WrapperKind::Inline,
Self::Deferred(_) => WrapperKind::Deferred,
}
}
const fn gates(&self) -> &Gates<D, Digested<B>> {
match self {
Self::Inline(inline) => inline.gates(),
Self::Deferred(deferred) => deferred.gates(),
}
}
async fn propose(&mut self, context: Ctx) -> oneshot::Receiver<D> {
match self {
Self::Inline(inline) => inline.propose(context).await,
Self::Deferred(deferred) => deferred.propose(context).await,
}
}
async fn verify(&mut self, context: Ctx, digest: D) -> oneshot::Receiver<bool> {
match self {
Self::Inline(inline) => inline.verify(context, digest).await,
Self::Deferred(deferred) => deferred.verify(context, digest).await,
}
}
async fn certify(&mut self, round: Round, digest: D) -> oneshot::Receiver<bool> {
match self {
Self::Inline(inline) => inline.certify(round, digest).await,
Self::Deferred(deferred) => deferred.certify(round, digest).await,
}
}
}
impl Automaton for Wrapper {
type Context = Ctx;
type Digest = D;
async fn propose(&mut self, context: Self::Context) -> oneshot::Receiver<Self::Digest> {
Self::propose(self, context).await
}
async fn verify(
&mut self,
context: Self::Context,
digest: Self::Digest,
) -> oneshot::Receiver<bool> {
Self::verify(self, context, digest).await
}
}
impl CertifiableAutomaton for Wrapper {
async fn prepare(
&mut self,
context: Self::Context,
) -> oneshot::Receiver<Handoff<Self::Digest>> {
match self {
Self::Inline(inline) => inline.prepare(context).await,
Self::Deferred(deferred) => deferred.prepare(context).await,
}
}
async fn certify(&mut self, round: Round, digest: Self::Digest) -> oneshot::Receiver<bool> {
Self::certify(self, round, digest).await
}
}
impl Relay for Wrapper {
type Digest = D;
type PublicKey = PublicKey;
type Plan = Plan<PublicKey>;
fn broadcast(&mut self, digest: Self::Digest, plan: Self::Plan) -> Feedback {
match self {
Self::Inline(inline) => inline.broadcast(digest, plan),
Self::Deferred(deferred) => deferred.broadcast(digest, plan),
}
}
}
impl Reporter for Wrapper {
type Activity = Activity<S, D>;
fn report(&mut self, activity: Self::Activity) -> Feedback {
match self {
Self::Inline(inline) => inline.report(activity),
Self::Deferred(deferred) => deferred.report(activity),
}
}
}
fn pending_conflicting_verify_does_not_poison_certification(kind: WrapperKind) {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let setup = StandardHarness::setup_validator(
context.child("validator"),
&mut oracle,
participants[0].clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let buffer = setup.extra;
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let certified_round = Round::new(Epoch::zero(), View::new(1));
let certified_block = B::new::<Sha256>(
Ctx {
round: certified_round,
leader: participants[0].clone(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
Height::new(1),
100,
);
let certified_digest = certified_block.digest();
assert!(marshal.verified(certified_round, ready(certified_block)).await);
let skipped_round = Round::new(Epoch::zero(), View::new(2));
let skipped_block = B::new::<Sha256>(
Ctx {
round: skipped_round,
leader: participants[1].clone(),
parent: (View::new(1), certified_digest),
},
certified_digest,
Height::new(2),
200,
);
let skipped_digest = skipped_block.digest();
assert!(marshal.verified(skipped_round, ready(skipped_block)).await);
let round = Round::new(Epoch::zero(), View::new(3));
let block = B::new::<Sha256>(
Ctx {
round,
leader: participants[1].clone(),
parent: (View::new(2), skipped_digest),
},
skipped_digest,
Height::new(3),
300,
);
let digest = block.digest();
let conflicting_context = Ctx {
round,
leader: participants[1].clone(),
parent: (View::new(1), certified_digest),
};
let mut wrapper = Wrapper::new(
kind,
context.child("wrapper"),
MockVerifyingApp::new(),
marshal,
);
let verify_rx = wrapper.verify(conflicting_context, digest).await;
let certify_rx = wrapper.certify(round, digest).await;
assert!(
buffer
.broadcast(Recipients::Some(vec![]), ready(block))
.accepted(),
"candidate broadcast should be accepted"
);
select! {
result = verify_rx => {
assert!(
!result.expect("verify result missing"),
"{kind:?}: the conflicting proposal must be rejected"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("{kind:?}: conflicting verification did not resolve");
},
}
select! {
result = certify_rx => {
assert!(
result.expect("certify result missing"),
"{kind:?}: pending certification must not adopt the conflicting verification verdict"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("{kind:?}: pending certification did not resolve");
},
}
});
}
#[test_traced("WARN")]
fn test_inline_pending_conflicting_verify_does_not_poison_certification() {
pending_conflicting_verify_does_not_poison_certification(WrapperKind::Inline);
}
#[test_traced("WARN")]
fn test_deferred_pending_conflicting_verify_does_not_poison_certification() {
pending_conflicting_verify_does_not_poison_certification(WrapperKind::Deferred);
}
fn scripted_byzantine_parent_equivocation(kind: WrapperKind) {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let byzantine = participants[3].clone();
let victim = participants[1].clone();
let observer = participants[0].clone();
let elector: RoundRobinElector<S> =
RoundRobin::<Sha256>::default().build(schemes[1].participants());
assert_eq!(
schemes[1]
.participants()
.key(elector.elect(Round::new(Epoch::zero(), View::new(3)), ())),
Some(&byzantine),
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let setup = StandardHarness::setup_validator(
context.child("marshal"),
&mut oracle,
victim.clone(),
ConstantProvider::new(schemes[1].clone()),
)
.await;
let mut marshal = setup.mailbox;
let buffer = setup.extra;
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let floor_round = Round::new(Epoch::zero(), View::new(1));
let floor_block = B::new::<Sha256>(
Ctx {
round: floor_round,
leader: byzantine.clone(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
Height::new(1),
100,
);
let floor_digest = floor_block.digest();
assert!(marshal.verified(floor_round, ready(floor_block)).await);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::zero(), floor_digest),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut marshal, floor_finalization.clone()).await;
let skipped_round = Round::new(Epoch::zero(), View::new(2));
let skipped_block = B::new::<Sha256>(
Ctx {
round: skipped_round,
leader: participants[3].clone(),
parent: (View::new(1), floor_digest),
},
floor_digest,
Height::new(2),
200,
);
let skipped_digest = skipped_block.digest();
assert!(marshal.verified(skipped_round, ready(skipped_block)).await);
let round = Round::new(Epoch::zero(), View::new(3));
let embedded_context = Ctx {
round,
leader: byzantine.clone(),
parent: (View::new(2), skipped_digest),
};
let block = B::new::<Sha256>(
embedded_context,
skipped_digest,
Height::new(3),
300,
);
let digest = block.digest();
assert!(
buffer
.broadcast(Recipients::Some(vec![]), ready(block))
.accepted(),
"candidate broadcast should be accepted"
);
let victim_control = oracle.control(victim.clone());
let vote_network = victim_control.register(3, TEST_QUOTA).await.unwrap();
let certificate_network = victim_control.register(4, TEST_QUOTA).await.unwrap();
let resolver_network = victim_control.register(5, TEST_QUOTA).await.unwrap();
let byzantine_control = oracle.control(byzantine.clone());
let (mut byzantine_vote_sender, _byzantine_vote_receiver) =
byzantine_control.register(3, TEST_QUOTA).await.unwrap();
let (mut byzantine_certificate_sender, _byzantine_certificate_receiver) =
byzantine_control.register(4, TEST_QUOTA).await.unwrap();
let _byzantine_resolver = byzantine_control.register(5, TEST_QUOTA).await.unwrap();
let (_observer_vote_sender, mut observer_vote_receiver) = oracle
.control(observer)
.register(3, TEST_QUOTA)
.await
.unwrap();
setup_network_links(&mut oracle, &participants, LINK).await;
let wrapper = Wrapper::new(
kind,
context.child("wrapper"),
MockVerifyingApp::new(),
marshal.clone(),
);
let engine = simplex::Engine::new(
context.child("simplex"),
simplex::config::Config {
scheme: schemes[1].clone(),
elector: RoundRobin::<Sha256>::default(),
blocker: oracle.control(victim.clone()),
automaton: wrapper.clone(),
relay: wrapper.clone(),
reporter: wrapper,
strategy: Sequential,
partition: format!("scripted-equivocation-{kind:?}"),
mailbox_size: NZUsize!(128),
epoch: Epoch::zero(),
floor: simplex::config::Floor::Finalized(floor_finalization),
replay_buffer: NZUsize!(1024),
write_buffer: NZUsize!(1024),
page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
leader_timeout: Duration::from_secs(2),
certification_timeout: Duration::from_secs(4),
timeout_retry: Duration::from_secs(3),
view_retention: ViewDelta::new(10),
skip: SkipPolicy::Enabled {
timeout: Duration::from_secs(6),
budget: SkipBudget::Participants,
},
fetch_timeout: NZDuration!(Duration::from_secs(1)),
fetch_retry_delay: Duration::from_millis(100),
forward: ForwardPolicy::Disabled,
track_historical_votes: false,
},
);
let _engine = engine.start(vote_network, certificate_network, resolver_network);
let nullifies: Vec<_> = [0usize, 2, 3]
.into_iter()
.map(|index| Nullify::sign::<D>(&schemes[index], skipped_round).unwrap())
.collect();
let nullification =
Nullification::from_nullifies(&schemes[0], non_empty![@&nullifies], &Sequential).unwrap();
byzantine_certificate_sender.send(
Recipients::One(victim.clone()),
Certificate::<S, D>::Nullification(nullification).encode(),
true,
);
context.sleep(Duration::from_millis(250)).await;
let bad_proposal = Proposal::new(round, View::new(1), digest);
let good_proposal = Proposal::new(round, View::new(2), digest);
assert_eq!(bad_proposal.payload, good_proposal.payload);
assert_ne!(bad_proposal, good_proposal);
let bad_vote = Notarize::sign(&schemes[3], bad_proposal).unwrap();
byzantine_vote_sender.send(
Recipients::One(victim.clone()),
Vote::<S, D>::Notarize(bad_vote).encode(),
true,
);
select! {
_ = context.sleep(Duration::from_secs(5)) => {
panic!("{kind:?}: victim did not reject the conflicting proposal");
},
result = async {
loop {
let (_, message) = observer_vote_receiver.recv().await.unwrap();
let vote = Vote::<S, D>::decode(message).unwrap();
if matches!(vote, Vote::Nullify(ref nullify) if nullify.round == round) {
break;
}
}
} => result,
}
context.sleep(Duration::from_millis(250)).await;
let good_votes: Vec<_> = [0usize, 2, 3]
.into_iter()
.map(|index| Notarize::sign(&schemes[index], good_proposal.clone()).unwrap())
.collect();
let notarization =
Notarization::from_notarizes(&schemes[0], non_empty![@&good_votes], &Sequential).unwrap();
byzantine_certificate_sender.send(
Recipients::One(victim),
Certificate::<S, D>::Notarization(notarization).encode(),
true,
);
let next_round = Round::new(Epoch::zero(), View::new(4));
select! {
_ = context.sleep(Duration::from_secs(5)) => {
panic!("{kind:?}: victim did not advance after certification");
},
result = async {
loop {
let (_, message) = observer_vote_receiver.recv().await.unwrap();
let vote = Vote::<S, D>::decode(message).unwrap();
if matches!(vote, Vote::Nullify(ref nullify) if nullify.round == next_round) {
break;
}
}
} => result,
}
});
}
#[test_traced("WARN")]
fn test_inline_scripted_byzantine_parent_equivocation() {
scripted_byzantine_parent_equivocation(WrapperKind::Inline);
}
#[test_traced("WARN")]
fn test_deferred_scripted_byzantine_parent_equivocation() {
scripted_byzantine_parent_equivocation(WrapperKind::Deferred);
}
fn certify_below_finalized_round(kind: WrapperKind, fork: bool) -> bool {
let runner = deterministic::Runner::timed(Duration::from_secs(60));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let reporter = setup.application;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let mut wrapper = Wrapper::new(
kind,
context.child("wrapper"),
MockVerifyingApp::new(),
marshal.clone(),
);
let parent_round = Round::new(Epoch::zero(), View::new(1));
let parent = B::new::<Sha256>(
Ctx {
round: parent_round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
Height::new(1),
100,
);
let parent_digest = parent.digest();
assert!(marshal.verified(parent_round, ready(parent)).await);
let (target_view, target_timestamp) = if fork { (6, 350) } else { (5, 200) };
let target_round = Round::new(Epoch::zero(), View::new(target_view));
let target_ctx = Ctx {
round: target_round,
leader: me.clone(),
parent: (View::new(1), parent_digest),
};
let target = B::new::<Sha256>(
target_ctx.clone(),
parent_digest,
Height::new(2),
target_timestamp,
);
let target_digest = target.digest();
assert!(marshal.verified(target_round, ready(target.clone())).await);
let a_round = Round::new(Epoch::zero(), View::new(5));
let (a_digest, a_block) = if fork {
let a = B::new::<Sha256>(
Ctx {
round: a_round,
leader: me.clone(),
parent: (View::new(1), parent_digest),
},
parent_digest,
Height::new(2),
200,
);
(a.digest(), a)
} else {
(target_digest, target.clone())
};
assert!(marshal.verified(a_round, ready(a_block)).await);
let c_round = Round::new(Epoch::zero(), View::new(10));
let c = B::new::<Sha256>(
Ctx {
round: c_round,
leader: me.clone(),
parent: (View::new(5), a_digest),
},
a_digest,
Height::new(3),
300,
);
let c_digest = c.digest();
assert!(marshal.verified(c_round, ready(c)).await);
assert!(
wrapper
.verify(target_ctx, target_digest)
.await
.await
.unwrap()
);
let finalization = StandardHarness::make_finalization(
Proposal::new(c_round, View::new(5), c_digest),
&schemes,
QUORUM,
);
wrapper.report(Activity::Finalization(finalization));
assert!(
wrapper.gates().take(target_round, target_digest).is_none(),
"{kind:?}: the finalization report must prune the gate at or below its round"
);
while reporter.tip() != Some((Height::new(3), c_digest)) {
reschedule().await;
}
wrapper
.certify(target_round, target_digest)
.await
.await
.expect("certify result")
})
}
#[test_traced("INFO")]
fn test_inline_certify_ancestor_below_finalized_round_after_prune() {
assert!(
certify_below_finalized_round(WrapperKind::Inline, false),
"certify of an ancestor below the finalized round must still resolve true"
);
}
#[test_traced("INFO")]
fn test_deferred_certify_ancestor_below_finalized_round_after_prune() {
assert!(
certify_below_finalized_round(WrapperKind::Deferred, false),
"certify of an ancestor below the finalized round must still resolve true"
);
}
#[test_traced("INFO")]
fn test_inline_certify_fork_below_finalized_round_after_prune() {
assert!(certify_below_finalized_round(WrapperKind::Inline, true));
}
#[test_traced("INFO")]
fn test_deferred_certify_fork_below_finalized_round_after_prune() {
assert!(certify_below_finalized_round(WrapperKind::Deferred, true));
}
fn epoch_zero_entries_remain(kind: WrapperKind, notarized: Round) -> bool {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let setup = StandardHarness::setup_validator(
context.child("validator"),
&mut oracle,
participants[0].clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let mut wrapper = Wrapper::new(
kind,
context.child("wrapper"),
MockVerifyingApp::new(),
setup.mailbox,
);
let earlier_round = Round::new(Epoch::zero(), View::new(5));
let earlier = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let earlier_digest = earlier.digest();
stage_block(&context, wrapper.gates(), earlier_round, earlier).await;
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(2), 200);
let digest = block.digest();
stage_block(&context, wrapper.gates(), notarized, block).await;
let notarization = StandardHarness::make_notarization(
Proposal::new(notarized, View::zero(), digest),
&schemes,
QUORUM,
);
wrapper.report(Activity::Notarization(notarization));
let gates = wrapper.gates();
assert!(
gates.take(notarized, digest).is_some(),
"{kind:?}: the notarized round must keep its gate"
);
assert!(
gates.take_staged(notarized, digest).is_some(),
"{kind:?}: the notarized round must keep its proposal"
);
let gate = gates.take(earlier_round, earlier_digest).is_some();
let staged = gates.take_staged(earlier_round, earlier_digest).is_some();
assert_eq!(
gate, staged,
"{kind:?}: a report must keep or release a round's gate and proposal together"
);
gate
})
}
#[test_traced("WARN")]
fn test_inline_later_epoch_activity_releases_earlier_epoch() {
let notarized = Round::new(Epoch::new(1), View::new(1));
assert!(!epoch_zero_entries_remain(WrapperKind::Inline, notarized));
}
#[test_traced("WARN")]
fn test_deferred_later_epoch_activity_releases_earlier_epoch() {
let notarized = Round::new(Epoch::new(1), View::new(1));
assert!(!epoch_zero_entries_remain(WrapperKind::Deferred, notarized));
}
#[test_traced("WARN")]
fn test_inline_same_epoch_activity_keeps_gates() {
let notarized = Round::new(Epoch::zero(), View::new(7));
assert!(epoch_zero_entries_remain(WrapperKind::Inline, notarized));
}
#[test_traced("WARN")]
fn test_deferred_same_epoch_activity_keeps_gates() {
let notarized = Round::new(Epoch::zero(), View::new(7));
assert!(epoch_zero_entries_remain(WrapperKind::Deferred, notarized));
}
#[test_traced("WARN")]
fn test_standard_certify_first_block_fetches_genesis_parent() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper =
Wrapper::new(kind, context.child("wrapper"), mock_app, marshal.clone());
let round = Round::new(Epoch::zero(), View::new(1));
let block_context = Ctx {
round,
leader: me,
parent: (View::zero(), genesis.digest()),
};
let block =
B::new::<Sha256>(block_context.clone(), genesis.digest(), Height::new(1), 100);
let digest = block.digest();
assert!(marshal.verified(round, ready(block)).await);
context.sleep(Duration::from_millis(10)).await;
let verify_result = wrapper
.verify(block_context, digest)
.await
.await
.expect("verify result missing");
assert!(
verify_result,
"{kind:?}: height-1 block should verify with genesis as parent"
);
let certify_result = wrapper
.certify(round, digest)
.await
.await
.expect("certify result missing");
assert!(
certify_result,
"{kind:?}: height-1 block should certify with genesis as parent"
);
});
}
}
#[test_traced("WARN")]
fn test_standard_switch_wrapper_at_epoch_boundary() {
const INVALID: u64 = u64::MAX;
for (old, new) in [
(WrapperKind::Inline, WrapperKind::Deferred),
(WrapperKind::Deferred, WrapperKind::Inline),
] {
let runner = deterministic::Runner::timed(Duration::from_secs(60));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let setup = StandardHarness::setup_validator(
context.child("validator"),
&mut oracle,
participants[0].clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let app = setup.application;
let mut marshal = setup.mailbox;
let epocher = FixedEpocher::new(BLOCKS_PER_EPOCH);
let tip = epocher.last(Epoch::new(1)).unwrap();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let mut parent = (View::zero(), genesis.digest());
let mut chain = vec![genesis.digest()];
for (epoch, kind) in [(Epoch::zero(), old), (Epoch::new(1), new)] {
let mut wrapper = Wrapper::new(
kind,
context.child("wrapper").with_attribute("epoch", epoch.get()),
MockVerifyingApp::new().with_reject(|block: &B| block.timestamp == INVALID),
marshal.clone(),
);
let first = epocher.first(epoch).unwrap().max(Height::new(1));
let last = epocher.last(epoch).unwrap();
let invalid_height = if epoch.is_zero() { last } else { first };
let mut rejected = 0;
let mut view = View::zero();
parent.0 = View::zero();
for height in (first.get()..=last.get()).map(Height::new) {
view = view.next();
if height == invalid_height {
let round = Round::new(epoch, view);
let block_context = Ctx {
round,
leader: participants[1].clone(),
parent,
};
let block =
B::new::<Sha256>(block_context.clone(), parent.1, height, INVALID);
let digest = block.digest();
assert!(marshal.verified(round, ready(block)).await);
assert_eq!(
wrapper
.verify(block_context, digest)
.await
.await
.expect("verify result missing"),
kind == WrapperKind::Deferred,
"{kind:?}: unexpected verification verdict for Byzantine block at {round}"
);
if kind == WrapperKind::Deferred {
let notarization = StandardHarness::make_notarization(
Proposal {
round,
parent: parent.0,
payload: digest,
},
&schemes,
QUORUM,
);
StandardHarness::report_notarization(&mut marshal, notarization)
.await;
assert!(
!wrapper
.certify(round, digest)
.await
.await
.expect("certify result missing"),
"{kind:?}: Byzantine block at {round} should not certify"
);
}
rejected += 1;
view = view.next();
}
let round = Round::new(epoch, view);
let block_context = Ctx {
round,
leader: participants[0].clone(),
parent,
};
let block =
B::new::<Sha256>(block_context.clone(), parent.1, height, height.get());
let digest = block.digest();
assert!(marshal.verified(round, ready(block)).await);
assert!(
wrapper
.verify(block_context, digest)
.await
.await
.expect("verify result missing"),
"{kind:?}: block at {round} should verify"
);
assert!(
wrapper
.certify(round, digest)
.await
.await
.expect("certify result missing"),
"{kind:?}: block at {round} should certify"
);
let finalization = StandardHarness::make_finalization(
Proposal {
round,
parent: parent.0,
payload: digest,
},
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut marshal, finalization).await;
parent = (view, digest);
chain.push(digest);
}
assert_eq!(
rejected, 1,
"{kind:?}: epoch {epoch} should reject one Byzantine proposal"
);
}
while !app.blocks().contains_key(&tip) {
context.sleep(Duration::from_millis(50)).await;
}
let delivered: Vec<_> = app.blocks().values().map(|block| block.digest()).collect();
assert_eq!(delivered, chain, "{old:?} -> {new:?}");
});
}
}
#[test_traced("WARN")]
fn test_standard_verify_missing_candidate_waits_without_fetching() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let me = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let (marshal, buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
&format!("missing-candidate-{kind:?}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper =
Wrapper::new(kind, context.child("wrapper"), mock_app, marshal.clone());
let round = Round::new(Epoch::zero(), View::new(1));
let consensus_context = Ctx {
round,
leader: me,
parent: (View::zero(), genesis.digest()),
};
let missing = Sha256::hash(&[b"missing candidate"]);
let mut verify = wrapper.verify(consensus_context, missing).await;
context.sleep(Duration::from_millis(50)).await;
assert!(
buffer.subscription_count() > 0,
"{kind:?}: unavailable candidate verification must register a local wait"
);
assert!(
resolver.fetches().is_empty(),
"{kind:?}: unavailable candidate verification must not fetch from peers"
);
assert!(
resolver.targeted_is_empty(),
"{kind:?}: unavailable candidate verification must not issue targeted fetches"
);
assert!(
matches!(
verify.try_recv(),
Err(commonware_utils::channel::oneshot::error::TryRecvError::Empty)
),
"{kind:?}: unavailable candidate verification must remain pending"
);
drop(verify);
context.sleep(Duration::from_millis(10)).await;
assert!(
resolver.fetches().is_empty(),
"{kind:?}: canceling a missing candidate wait must not fetch from peers"
);
});
}
}
#[test_traced("WARN")]
fn test_standard_certify_missing_candidate_fetches_by_round() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let me = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let (marshal, buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
&format!("missing-certify-candidate-{kind:?}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper =
Wrapper::new(kind, context.child("wrapper"), mock_app, marshal.clone());
let round = Round::new(Epoch::zero(), View::new(1));
let block_context = Ctx {
round,
leader: me,
parent: (View::zero(), genesis.digest()),
};
let block = B::new::<Sha256>(block_context, genesis.digest(), Height::new(1), 100);
let digest = block.digest();
let proposal = Proposal::new(round, View::zero(), digest);
let notarization = StandardHarness::make_notarization(proposal, &schemes, QUORUM);
resolver.respond_to_next_fetch((notarization, block).encode());
let certify = wrapper.certify(round, digest).await;
let result = certify.await.expect("certify result missing");
assert!(
result,
"{kind:?}: fetched notarized candidate should certify"
);
assert!(
resolver.wait_for_delivery_response().await,
"{kind:?}: notarized delivery should validate"
);
assert!(
resolver.fetches().iter().any(|fetch| matches!(
(&fetch.key, &fetch.subscriber),
(
handler::Key::Notarized { round: request_round },
handler::Annotation::Notarization { round: subscriber_round },
) if *request_round == round && *subscriber_round == round
)),
"{kind:?}: certify should fetch notarized block by round"
);
assert!(
buffer.subscription_count() > 0,
"{kind:?}: unavailable candidate certification must register a local wait"
);
assert!(
resolver.targeted_is_empty(),
"{kind:?}: certification must not issue targeted fetches"
);
});
}
}
struct CertificationCase {
wrapper: Wrapper,
marshal: Mailbox<S, Standard<B>>,
buffer: RecordingBuffer,
resolver: RecordingResolver,
schemes: Vec<S>,
block_context: Ctx,
block: B,
}
async fn certification_case(
context: &mut Runtime,
kind: WrapperKind,
partition_prefix: String,
) -> CertificationCase {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let (marshal, buffer, resolver, _) = start_standard_actor(
context.child("validator"),
&partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let wrapper = Wrapper::new(
kind,
context.child("wrapper"),
MockVerifyingApp::new(),
marshal.clone(),
);
let round = Round::new(Epoch::zero(), View::new(1));
let block_context = Ctx {
round,
leader: me,
parent: (View::zero(), genesis.digest()),
};
let block = B::new::<Sha256>(block_context.clone(), genesis.digest(), Height::new(1), 100);
CertificationCase {
wrapper,
marshal,
buffer,
resolver,
schemes,
block_context,
block,
}
}
#[test_traced("WARN")]
fn test_standard_certify_bumps_notarized_fetch_for_pending_verify() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let mut case = certification_case(
&mut context,
kind,
format!("certify-bumps-fetch-{kind:?}"),
)
.await;
let round = case.block_context.round;
let digest = case.block.digest();
let block_context = case.block_context.clone();
let verify_rx = case.wrapper.verify(block_context, digest).await;
let proposal = Proposal::new(round, View::zero(), digest);
let notarization =
StandardHarness::make_notarization(proposal, &case.schemes, QUORUM);
case.resolver
.respond_to_next_fetch((notarization, case.block.clone()).encode());
let certify_rx = case.wrapper.certify(round, digest).await;
select! {
result = verify_rx => {
assert!(
result.expect("verify resolves"),
"{kind:?}: verify should accept the fetched block"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!(
"{kind:?}: verify must resolve after certification bumps a notarized fetch"
);
},
}
select! {
result = certify_rx => {
assert!(
result.expect("certify resolves"),
"{kind:?}: certify should succeed via the shared gate"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("{kind:?}: certify must resolve via the bumped notarized fetch");
},
}
assert!(
case.resolver.fetches().iter().any(|fetch| matches!(
(&fetch.key, &fetch.subscriber),
(
handler::Key::Notarized { round: request_round },
handler::Annotation::Notarization { round: subscriber_round },
) if *request_round == round && *subscriber_round == round
)),
"{kind:?}: certify must bump a notarized round fetch when verify is in progress"
);
});
}
}
#[test_traced("WARN")]
fn test_standard_certify_retains_transient_buffer_hit_through_eviction() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let mut case = certification_case(
&mut context,
kind,
format!("certify-transient-buffer-{kind:?}"),
)
.await;
let round = case.block_context.round;
let digest = case.block.digest();
case.buffer.insert_transient(case.block.clone());
let block_context = case.block_context.clone();
let verify_rx = case.wrapper.verify(block_context, digest).await;
let certify_rx = case.wrapper.certify(round, digest).await;
case.marshal.get_processed().await;
assert!(
!case.buffer.contains(digest),
"{kind:?}: the buffered block must be evicted before verification completes"
);
select! {
result = verify_rx => {
assert!(
result.expect("verify resolves"),
"{kind:?}: verify should retain and accept the transient block"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!(
"{kind:?}: verify must survive transient buffer eviction"
);
},
}
select! {
result = certify_rx => {
assert!(
result.expect("certify resolves"),
"{kind:?}: certify should succeed via the shared verification gate"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("{kind:?}: certify must survive transient buffer eviction");
},
}
});
}
}
#[test_traced("WARN")]
fn test_standard_deferred_certify_falls_back_after_canceled_verify() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let (marshal, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"deferred-certify-canceled-verify",
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper = Wrapper::new(
WrapperKind::Deferred,
context.child("wrapper"),
mock_app,
marshal.clone(),
);
let round = Round::new(Epoch::zero(), View::new(1));
let block_context = Ctx {
round,
leader: me,
parent: (View::zero(), genesis.digest()),
};
let block =
B::new::<Sha256>(block_context.clone(), genesis.digest(), Height::new(1), 100);
let digest = block.digest();
let verify_rx = wrapper.verify(block_context, digest).await;
drop(verify_rx);
context.sleep(Duration::from_millis(10)).await;
let proposal = Proposal::new(round, View::zero(), digest);
let notarization = StandardHarness::make_notarization(proposal, &schemes, QUORUM);
resolver.respond_to_next_fetch((notarization, block).encode());
let certify_rx = wrapper.certify(round, digest).await;
select! {
result = certify_rx => {
assert!(
result.expect("certify result missing"),
"certify should recover after canceled optimistic verify"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("certify should recover after canceled optimistic verify");
},
}
assert!(
resolver.wait_for_delivery_response().await,
"notarized delivery should validate"
);
assert!(
resolver.fetches().iter().any(|fetch| matches!(
(&fetch.key, &fetch.subscriber),
(
handler::Key::Notarized { round: request_round },
handler::Annotation::Notarization { round: subscriber_round },
) if *request_round == round && *subscriber_round == round
)),
"certify must recover by fetching the notarized round"
);
});
}
#[test_traced("WARN")]
fn test_standard_verify_height_lie_parent_fetch_is_round_bound() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let me = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let (marshal, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
&format!("height-lie-{kind:?}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper =
Wrapper::new(kind, context.child("wrapper"), mock_app, marshal.clone());
let parent_round = Round::new(Epoch::zero(), View::new(1));
let parent_context = Ctx {
round: parent_round,
leader: me.clone(),
parent: (View::zero(), genesis.digest()),
};
let parent =
B::new::<Sha256>(parent_context, genesis.digest(), Height::new(1), 100);
let parent_digest = parent.digest();
let child_round = Round::new(Epoch::zero(), View::new(2));
let child_context = Ctx {
round: child_round,
leader: me,
parent: (View::new(1), parent_digest),
};
let child =
B::new::<Sha256>(child_context.clone(), parent_digest, Height::new(3), 200);
let child_digest = child.digest();
assert!(marshal.verified(child_round, ready(child)).await);
let verify = wrapper.verify(child_context, child_digest).await;
wait_until(
&context,
Duration::from_secs(5),
"round-bound parent fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Notarized { round } if round == parent_round
) && matches!(
fetch.subscriber,
handler::Annotation::Notarization { round }
if round == parent_round
)
})
},
)
.await;
let fetches = resolver.fetches();
assert!(
fetches.iter().all(|fetch| {
!matches!(fetch.key, handler::Key::Block(_))
&& !matches!(
fetch.subscriber,
handler::Annotation::Certified { height }
| handler::Annotation::Untrusted { height }
if height == Height::new(2)
)
}),
"{kind:?}: malicious child height must not drive parent fetches"
);
assert!(marshal.verified(parent_round, ready(parent)).await);
let verify_result = verify.await.expect("verify result missing");
if kind == WrapperKind::Inline {
assert!(
!verify_result,
"inline verify should reject non-contiguous ancestry"
);
} else {
assert!(
verify_result,
"deferred verify should optimistically pass pre-checks"
);
let certify = wrapper.certify(child_round, child_digest).await;
assert!(
!certify.await.expect("certify result missing"),
"deferred certify should reject non-contiguous ancestry"
);
}
});
}
}
#[test_traced("WARN")]
fn test_standard_verify_parent_fetch_invalid_first_retries() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let victim = participants[0].clone();
let malicious = participants[1].clone();
let honest = participants[2].clone();
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(2),
[victim.clone(), malicious.clone()],
1024 * 1024,
)
.await;
setup_network_links(
&mut oracle,
&[victim.clone(), malicious.clone()],
LINK,
)
.await;
let victim_setup = StandardHarness::setup_validator(
context.child("victim").with_attribute("index", 0),
&mut oracle,
victim.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let victim_mailbox = victim_setup.mailbox;
let honest_setup = StandardHarness::setup_validator(
context.child("honest").with_attribute("index", 2),
&mut oracle,
honest.clone(),
ConstantProvider::new(schemes[2].clone()),
)
.await;
let mut honest_mailbox = honest_setup.mailbox;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let parent_round = Round::new(Epoch::zero(), View::new(1));
let parent_context = Ctx {
round: parent_round,
leader: victim.clone(),
parent: (View::zero(), genesis.digest()),
};
let parent =
B::new::<Sha256>(parent_context, genesis.digest(), Height::new(1), 100);
let parent_digest = parent.digest();
let parent_proposal = Proposal::new(parent_round, View::zero(), parent_digest);
let parent_notarization =
StandardHarness::make_notarization(parent_proposal, &schemes, QUORUM);
assert!(honest_mailbox.verified(parent_round, ready(parent.clone())).await);
StandardHarness::report_notarization(&mut honest_mailbox, parent_notarization)
.await;
assert!(honest_mailbox.get_block(&parent_digest).await.is_some());
let malicious_backfill = oracle
.control(malicious.clone())
.register(1, Quota::per_second(NonZeroU32::MAX))
.await
.unwrap();
let (malicious_engine, _malicious_mailbox) = commonware_resolver::p2p::Engine::new(
context.child("malicious_resolver"),
commonware_resolver::p2p::Config {
peer_provider: oracle.manager(),
blocker: oracle.control(malicious.clone()),
consumer: NoopConsumer,
producer: StaticProducer::new(
handler::Key::Notarized {
round: parent_round,
},
Bytes::from_static(b"not a valid notarization"),
),
mailbox_size: NZUsize!(100),
me: Some(malicious.clone()),
fetch_timeout: NZDuration!(Duration::from_secs(2)),
fetch_retry_delay: Duration::from_millis(100),
priority_requests: false,
priority_responses: false,
},
);
malicious_engine.start(malicious_backfill);
let child_round = Round::new(Epoch::zero(), View::new(2));
let child_context = Ctx {
round: child_round,
leader: victim.clone(),
parent: (View::new(1), parent_digest),
};
let child =
B::new::<Sha256>(child_context.clone(), parent_digest, Height::new(2), 200);
let child_digest = child.digest();
assert!(victim_mailbox.verified(child_round, ready(child)).await);
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper = Wrapper::new(
kind,
context.child("wrapper"),
mock_app,
victim_mailbox.clone(),
);
let verify = wrapper.verify(child_context, child_digest).await;
let verify_or_certify = if kind == WrapperKind::Deferred {
let optimistic = verify.await.expect("verify result missing");
assert!(optimistic, "deferred verify should optimistically succeed");
wrapper.certify(child_round, child_digest).await
} else {
verify
};
let start = context.current();
loop {
let blocked = oracle.blocked().await.unwrap();
if blocked
.iter()
.any(|(blocker, blocked)| blocker == &victim && blocked == &malicious)
{
break;
}
if context.current().duration_since(start).unwrap_or_default()
> Duration::from_secs(5)
{
panic!("{kind:?}: malicious peer was not blocked");
}
context.sleep(Duration::from_millis(10)).await;
}
oracle
.add_link(victim.clone(), honest.clone(), LINK)
.await
.unwrap();
oracle
.add_link(honest.clone(), victim.clone(), LINK)
.await
.unwrap();
let mut manager = oracle.manager();
manager.track(1, Set::from_iter_dedup([honest.clone()]));
select! {
result = verify_or_certify => {
assert!(
result.expect("verification result missing"),
"{kind:?}: verification should retry against the honest peer and complete"
);
},
_ = context.sleep(Duration::from_secs(10)) => {
panic!("{kind:?}: verification did not complete after honest retry");
},
}
let blocked = oracle.blocked().await.unwrap();
assert!(
blocked
.iter()
.any(|(blocker, blocked)| blocker == &victim && blocked == &malicious),
"{kind:?}: malicious peer should remain blocked"
);
});
}
}
#[test_traced("WARN")]
fn test_propose_paths() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper = Wrapper::new(
kind,
context.child("wrapper_under_test"),
mock_app,
marshal.clone(),
);
let non_boundary_context = Ctx {
round: Round::new(Epoch::zero(), View::new(1)),
leader: me.clone(),
parent: (View::zero(), genesis.digest()),
};
let proposal_rx = wrapper.propose(non_boundary_context.clone()).await;
assert!(
proposal_rx.await.is_err(),
"{kind:?}: proposal should be dropped when application returns no block"
);
assert!(
context
.encode()
.contains("wrapper_under_test_build_duration_count 0"),
"{kind:?}: failed application builds should not be timed"
);
let failing_app = MockVerifyingApp::new().with_handoff(Handoff::Stage(()));
let mut failing =
Wrapper::new(kind, context.child("failed"), failing_app, marshal.clone());
let handoff_rx = failing.prepare(non_boundary_context.clone()).await;
assert_eq!(
handoff_rx.await.expect("handoff decision missing"),
Handoff::Wait,
"{kind:?}: a failed handoff build must answer Wait"
);
let (gated_app, started, dropped) = MockVerifyingApp::new()
.with_handoff(Handoff::Vote(()))
.with_proposal_gate();
let mut gated =
Wrapper::new(kind, context.child("cancelled"), gated_app, marshal.clone());
let response = gated.prepare(non_boundary_context.clone()).await;
started.await.expect("handoff build should start");
drop(response);
assert!(dropped.await.is_err(), "handoff build should be cancelled");
let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
let boundary_round = Round::new(Epoch::zero(), View::new(boundary_height.get()));
let boundary_block = B::new::<Sha256>(
Ctx {
round: boundary_round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
boundary_height,
1900,
);
let boundary_digest = boundary_block.digest();
assert!(
marshal
.clone()
.verified(boundary_round, ready(boundary_block.clone()))
.await
);
context.sleep(Duration::from_millis(10)).await;
let reproposal_round =
Round::new(Epoch::zero(), View::new(boundary_height.get() + 1));
let reproposal_context = Ctx {
round: reproposal_round,
leader: me,
parent: (View::new(boundary_height.get()), boundary_digest),
};
let reproposal_rx = wrapper.propose(reproposal_context).await;
assert_eq!(
reproposal_rx.await.expect("reproposal result missing"),
boundary_digest,
"{kind:?}: epoch-boundary proposal should re-propose parent digest"
);
let certify_rx = wrapper.certify(reproposal_round, boundary_digest).await;
assert!(
certify_rx.await.expect("certify result missing"),
"{kind:?}: certify must succeed for the re-proposed boundary block"
);
assert!(
marshal.get_verified(reproposal_round).await.is_some(),
"{kind:?}: re-proposed boundary block must be stored at the re-proposal round"
);
let pipeline_round =
Round::new(Epoch::zero(), View::new(boundary_height.get() + 2));
let pipeline_context = Ctx {
round: pipeline_round,
leader: default_leader(),
parent: (View::new(boundary_height.get()), boundary_digest),
};
let unwanted = B::new::<Sha256>(
pipeline_context.clone(),
boundary_digest,
boundary_height.next(),
2000,
);
let pipeline_app = MockVerifyingApp::new()
.with_handoff(Handoff::Vote(()))
.with_propose_result(unwanted);
let mut pipeline = Wrapper::new(
kind,
context.child("pipeline"),
pipeline_app,
marshal.clone(),
);
let pipeline_rx = pipeline.prepare(pipeline_context.clone()).await;
assert_eq!(
pipeline_rx.await.expect("pipeline result missing"),
Handoff::Stage(boundary_digest),
"{kind:?}: a re-proposed boundary block is staged regardless of the decision"
);
let certify_rx = pipeline.certify(pipeline_round, boundary_digest).await;
assert!(
certify_rx.await.expect("pipeline certify result missing"),
"{kind:?}: pipelined boundary re-proposal must certify"
);
});
}
}
#[test_traced("WARN")]
fn test_propose_rebuilds_after_restart() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let round = Round::new(Epoch::zero(), View::new(2));
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let stale = B::new::<Sha256>(
Ctx {
round,
leader: me.clone(),
parent: (View::new(1), Sha256::hash(&[b"replaced parent"])),
},
genesis.digest(),
Height::new(1),
100,
);
let stale_digest = stale.digest();
let (ack, persisted) = oneshot::channel();
let _ = setup
.mailbox
.proposed(round, ready(stale), Recipients::All, ack);
let sync = persisted.await.expect("stale block sync handle missing");
assert!(sync.durable(round, "stale block").await);
setup.crash().await;
let setup = StandardHarness::setup_validator(
context.child("restarted").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox.clone();
assert_eq!(
marshal
.get_verified(round)
.await
.map(|block| block.digest()),
Some(stale_digest),
"{kind:?}: the restarted marshal must restore the stale block"
);
let ctx = Ctx {
round,
leader: me.clone(),
parent: (View::zero(), genesis.digest()),
};
let fresh = B::new::<Sha256>(ctx.clone(), genesis.digest(), Height::new(1), 200);
let fresh_digest = fresh.digest();
let app: MockVerifyingApp<B, S> =
MockVerifyingApp::new().with_propose_result(fresh);
let mut wrapper =
Wrapper::new(kind, context.child("wrapper"), app, marshal.clone());
let digest = wrapper
.propose(ctx)
.await
.await
.expect("propose must return a digest");
assert_eq!(
digest, fresh_digest,
"{kind:?}: a restarted leader must build a fresh block"
);
let _ = wrapper.broadcast(digest, Plan::Propose { round });
assert!(
wrapper
.certify(round, digest)
.await
.await
.expect("certify result missing"),
"{kind:?}: the fresh block must certify through the relay handshake"
);
assert_eq!(
marshal
.get_verified(round)
.await
.map(|block| block.digest()),
Some(stale_digest),
"{kind:?}: the stale block stays stored first at the round"
);
setup.crash().await;
let setup = StandardHarness::setup_validator(
context.child("recovered").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
for (label, digest) in [("stale", stale_digest), ("fresh", fresh_digest)] {
assert!(
setup.mailbox.get_block(&digest).await.is_some(),
"{kind:?}: the {label} block must be recoverable from storage"
);
}
let app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut certifier =
Wrapper::new(kind, context.child("certifier"), app, setup.mailbox.clone());
assert!(
certifier
.certify(round, fresh_digest)
.await
.await
.expect("certify result missing"),
"{kind:?}: the fresh block must certify from storage after a restart"
);
});
}
}
fn spawned_task(metrics: &str, label: &str) -> bool {
metrics
.lines()
.any(|line| line.starts_with("runtime_tasks_spawned_total{") && line.contains(label))
}
#[test_traced("WARN")]
fn test_prepare_paths() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let handoff_context = Ctx {
round: Round::new(Epoch::zero(), View::new(1)),
leader: me.clone(),
parent: (View::zero(), genesis.digest()),
};
let child = B::new::<Sha256>(
handoff_context.clone(),
genesis.digest(),
Height::new(1),
100,
);
let mut declining = Wrapper::new(
kind,
context.child("declining"),
MockVerifyingApp::<B, S>::new(),
marshal.clone(),
);
let response = declining.prepare(handoff_context.clone()).await;
assert_eq!(
response.await.expect("prepare decision missing"),
Handoff::Wait,
"{kind:?}: the default prepare declines"
);
assert!(
context
.encode()
.contains("declining_parent_fetch_duration_count 1"),
"{kind:?}: a declining prepare still fetches the parent"
);
let rebuilt_round = Round::new(Epoch::zero(), View::new(2));
let stored = B::new::<Sha256>(
Ctx {
round: rebuilt_round,
leader: me.clone(),
parent: (View::new(1), Sha256::hash(&[b"replaced parent"])),
},
genesis.digest(),
Height::new(1),
101,
);
assert!(marshal.clone().verified(rebuilt_round, ready(stored)).await);
let rebuilt_context = Ctx {
round: rebuilt_round,
leader: me.clone(),
parent: (View::zero(), genesis.digest()),
};
let rebuilt = B::new::<Sha256>(
rebuilt_context.clone(),
genesis.digest(),
Height::new(1),
102,
);
let rebuilding_app = MockVerifyingApp::new()
.with_handoff(Handoff::Vote(()))
.with_propose_result(rebuilt.clone());
let mut rebuilding = Wrapper::new(
kind,
context.child("rebuilding"),
rebuilding_app,
marshal.clone(),
);
let response = rebuilding.prepare(rebuilt_context).await;
assert_eq!(
response.await.expect("prepare decision missing"),
Handoff::Vote(rebuilt.digest()),
"{kind:?}: a stored block must not replace the application's build"
);
for (label, decision) in [
("staging", Handoff::Stage(())),
("voting", Handoff::Vote(())),
] {
let building_app = MockVerifyingApp::new()
.with_handoff(decision)
.with_propose_result(child.clone());
let mut building =
Wrapper::new(kind, context.child(label), building_app, marshal.clone());
let response = building.prepare(handoff_context.clone()).await;
assert_eq!(
response.await.expect("prepare decision missing"),
decision.map(|()| child.digest()),
"{kind:?}: a prepared block carries the application's {label} decision"
);
let metrics = context.encode();
assert!(
metrics.contains(&format!("{label}_parent_fetch_duration_count 1")),
"{kind:?}: a prepare fetches the parent once"
);
assert!(
metrics.contains(&format!("{label}_build_duration_count 1")),
"{kind:?}: a prepared build is timed"
);
assert!(
spawned_task(&metrics, &format!("{label}_prepare")),
"{kind:?}: a prepare is driven by a task"
);
}
});
}
}
#[test_traced("WARN")]
fn test_deferred_handoff_rejects_unvalidated_parent_link() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let tip_round = Round::new(Epoch::zero(), View::new(1));
let tip_context = Ctx {
round: tip_round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
};
let forged = B::new::<Sha256>(
tip_context.clone(),
Sha256::hash(&[b"forged parent link"]),
Height::new(1),
100,
);
let linked = B::new::<Sha256>(tip_context, genesis.digest(), Height::new(1), 101);
for (label, tip, built) in [("forged", &forged, false), ("linked", &linked, true)] {
assert!(
marshal
.clone()
.verified(tip_round, ready(tip.clone()))
.await
);
let handoff_context = Ctx {
round: Round::new(Epoch::zero(), View::new(2)),
leader: me.clone(),
parent: (View::new(1), tip.digest()),
};
let child =
B::new::<Sha256>(handoff_context.clone(), tip.digest(), Height::new(2), 200);
let app = MockVerifyingApp::new()
.with_handoff(Handoff::Stage(()))
.with_propose_result(child.clone());
let mut deferred = Wrapper::new(
WrapperKind::Deferred,
context.child(label),
app,
marshal.clone(),
);
let response = deferred.prepare(handoff_context).await.await;
let expected = if built {
Handoff::Stage(child.digest())
} else {
Handoff::Wait
};
assert_eq!(
response.expect("handoff decision missing"),
expected,
"{label}: a handoff build must require a parent linked to its context"
);
}
});
}
#[test_traced("WARN")]
fn test_verify_reproposal_validation() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper =
Wrapper::new(kind, context.child("wrapper"), mock_app, marshal.clone());
let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
let boundary_round = Round::new(Epoch::zero(), View::new(boundary_height.get()));
let boundary_block = B::new::<Sha256>(
Ctx {
round: boundary_round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
boundary_height,
1900,
);
let boundary_digest = boundary_block.digest();
assert!(
marshal
.clone()
.verified(boundary_round, ready(boundary_block))
.await
);
context.sleep(Duration::from_millis(10)).await;
let valid_reproposal_context = Ctx {
round: Round::new(Epoch::zero(), View::new(boundary_height.get() + 1)),
leader: me.clone(),
parent: (View::new(boundary_height.get()), boundary_digest),
};
assert!(
wrapper
.verify(valid_reproposal_context, boundary_digest)
.await
.await
.expect("verify result missing"),
"{kind:?}: boundary re-proposal should be accepted"
);
let non_boundary_height = Height::new(10);
let non_boundary_round =
Round::new(Epoch::zero(), View::new(non_boundary_height.get()));
let non_boundary_block = B::new::<Sha256>(
Ctx {
round: non_boundary_round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
non_boundary_height,
1000,
);
let non_boundary_digest = non_boundary_block.digest();
assert!(
marshal
.clone()
.verified(non_boundary_round, ready(non_boundary_block))
.await
);
context.sleep(Duration::from_millis(10)).await;
let invalid_reproposal_context = Ctx {
round: Round::new(Epoch::zero(), View::new(15)),
leader: me.clone(),
parent: (View::new(non_boundary_height.get()), non_boundary_digest),
};
assert!(
!wrapper
.verify(invalid_reproposal_context, non_boundary_digest)
.await
.await
.expect("verify result missing"),
"{kind:?}: non-boundary re-proposal should be rejected"
);
let cross_epoch_context = Ctx {
round: Round::new(Epoch::new(1), View::new(boundary_height.get() + 1)),
leader: me,
parent: (View::new(boundary_height.get()), boundary_digest),
};
assert!(
!wrapper
.verify(cross_epoch_context, boundary_digest)
.await
.await
.expect("verify result missing"),
"{kind:?}: cross-epoch re-proposal should be rejected"
);
if wrapper.kind() == WrapperKind::Deferred {
let certify_only_round = Round::new(Epoch::zero(), View::new(21));
let certify_result = wrapper
.certify(certify_only_round, boundary_digest)
.await
.await;
assert!(
certify_result.expect("certify result missing"),
"deferred certify-only path for re-proposal should succeed"
);
}
});
}
}
#[test_traced("WARN")]
fn test_verify_rejects_invalid_ancestry() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
let mut wrapper =
Wrapper::new(kind, context.child("wrapper"), mock_app, marshal.clone());
let malformed_round = Round::new(Epoch::zero(), View::new(2));
let malformed_context = Ctx {
round: malformed_round,
leader: me.clone(),
parent: (View::zero(), genesis.digest()),
};
let malformed_block = B::new::<Sha256>(
malformed_context.clone(),
genesis.digest(),
Height::new(2),
200,
);
let malformed_digest = malformed_block.digest();
assert!(
marshal
.clone()
.verified(malformed_round, ready(malformed_block))
.await
);
context.sleep(Duration::from_millis(10)).await;
let malformed_verify = wrapper
.verify(malformed_context.clone(), malformed_digest)
.await
.await
.expect("verify result missing");
if kind == WrapperKind::Inline {
assert!(
!malformed_verify,
"inline verify should reject non-contiguous ancestry"
);
} else {
assert!(
malformed_verify,
"deferred verify should optimistically pass pre-checks"
);
let certify = wrapper.certify(malformed_round, malformed_digest).await;
assert!(
!certify.await.expect("certify result missing"),
"deferred certify should reject non-contiguous ancestry"
);
}
let parent_round = Round::new(Epoch::zero(), View::new(1));
let parent_context = Ctx {
round: parent_round,
leader: me.clone(),
parent: (View::zero(), genesis.digest()),
};
let parent =
B::new::<Sha256>(parent_context, genesis.digest(), Height::new(1), 300);
let parent_digest = parent.digest();
assert!(marshal.verified(parent_round, ready(parent)).await);
let mismatch_round = Round::new(Epoch::zero(), View::new(3));
let mismatched_context = Ctx {
round: mismatch_round,
leader: me,
parent: (View::new(1), parent_digest),
};
let mismatched_block = B::new::<Sha256>(
mismatched_context.clone(),
genesis.digest(),
Height::new(2),
400,
);
let mismatched_digest = mismatched_block.digest();
assert!(
marshal
.clone()
.verified(mismatch_round, ready(mismatched_block))
.await
);
context.sleep(Duration::from_millis(10)).await;
let mismatch_verify = wrapper
.verify(mismatched_context, mismatched_digest)
.await
.await
.expect("verify result missing");
if kind == WrapperKind::Inline {
assert!(
!mismatch_verify,
"inline verify should reject mismatched parent digest"
);
} else {
assert!(
mismatch_verify,
"deferred verify should optimistically pass pre-checks"
);
let certify = wrapper.certify(mismatch_round, mismatched_digest).await;
assert!(
!certify.await.expect("certify result missing"),
"deferred certify should reject mismatched parent digest"
);
}
});
}
}
#[test_traced("WARN")]
fn test_application_verify_failure() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let mock_app: MockVerifyingApp<B, S> =
MockVerifyingApp::with_verify_result(false);
let mut wrapper = Wrapper::new(kind, context.child("wrapper"), mock_app, marshal.clone());
let parent_round = Round::new(Epoch::zero(), View::new(1));
let parent_context = Ctx {
round: parent_round,
leader: me.clone(),
parent: (View::zero(), genesis.digest()),
};
let parent = B::new::<Sha256>(parent_context, genesis.digest(), Height::new(1), 100);
let parent_digest = parent.digest();
assert!(marshal.verified(parent_round, ready(parent)).await);
let round = Round::new(Epoch::zero(), View::new(2));
let verify_context = Ctx {
round,
leader: me,
parent: (View::new(1), parent_digest),
};
let block = B::new::<Sha256>(verify_context.clone(), parent_digest, Height::new(2), 200);
let digest = block.digest();
assert!(marshal.verified(round, ready(block)).await);
context.sleep(Duration::from_millis(10)).await;
let verify_result = wrapper
.verify(verify_context, digest)
.await
.await
.expect("verify result missing");
if kind == WrapperKind::Inline {
assert!(
!verify_result,
"inline verify should return application-level failure"
);
let certify = wrapper.certify(round, digest).await;
assert!(
certify.await.expect("certify result missing"),
"inline certify should trust the notarization"
);
} else {
assert!(
verify_result,
"deferred verify should pass pre-checks and schedule deferred verification"
);
let certify = wrapper.certify(round, digest).await;
assert!(
!certify.await.expect("certify result missing"),
"deferred certify should propagate deferred application verification failure"
);
}
});
}
}
#[test_traced("WARN")]
#[should_panic(expected = "failed to sync verified")]
fn test_mailbox_verified_sync_failure_panics() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let application = Application::<B>::manual_ack();
let setup = StandardHarness::setup_validator_with(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
application.clone(),
)
.await;
let marshal = setup.mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
context.sleep(Duration::from_millis(10)).await;
context.storage_fault_config().write().sync_rate = Some(probability!(1.0));
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let round = Round::new(Epoch::zero(), View::new(1));
let block = B::new::<Sha256>(
Ctx {
round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
Height::new(1),
100,
);
let _ = marshal.verified(round, ready(block)).await;
});
}
#[test_traced("WARN")]
#[should_panic(expected = "failed to sync certified")]
fn test_mailbox_certified_sync_failure_panics() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let application = Application::<B>::manual_ack();
let setup = StandardHarness::setup_validator_with(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
application.clone(),
)
.await;
let marshal = setup.mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
context.sleep(Duration::from_millis(10)).await;
context.storage_fault_config().write().sync_rate = Some(probability!(1.0));
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let round = Round::new(Epoch::zero(), View::new(1));
let block = B::new::<Sha256>(
Ctx {
round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
Height::new(1),
100,
);
let _ = marshal.certified(round, ready(block)).await;
});
}
#[test_traced("WARN")]
#[should_panic(expected = "failed to sync notarization")]
fn test_notarization_sync_failure_panics() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let application = Application::<B>::manual_ack();
let setup = StandardHarness::setup_validator_with(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
application.clone(),
)
.await;
let mut mailbox = setup.mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
context.sleep(Duration::from_millis(10)).await;
context.storage_fault_config().write().sync_rate = Some(probability!(1.0));
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let round = Round::new(Epoch::zero(), View::new(1));
let block = B::new::<Sha256>(
Ctx {
round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
Height::new(1),
100,
);
let notarization = StandardHarness::make_notarization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_notarization(&mut mailbox, notarization).await;
context.sleep(Duration::from_secs(5)).await;
});
}
#[test_traced("WARN")]
#[should_panic(expected = "failed to get block")]
fn test_get_info_by_digest_read_failure_panics() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let application = Application::<B>::manual_ack();
let setup = StandardHarness::setup_validator_with(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
application.clone(),
)
.await;
let marshal = setup.mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
context.sleep(Duration::from_millis(10)).await;
context.storage_fault_config().write().read_rate = Some(probability!(1.0));
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let _ = marshal.get_info(&genesis.digest()).await;
});
}
#[test_traced("WARN")]
fn test_certified_covered_by_verified_write_recoverable_after_restart() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let me = participants[0].clone();
let setup = StandardHarness::setup_validator(
context.child("validator").with_attribute("index", 0),
&mut oracle,
me.clone(),
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal = setup.mailbox;
let actor_handle = setup.actor_handle;
let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
let round = Round::new(Epoch::zero(), View::new(1));
let block = B::new::<Sha256>(
Ctx {
round,
leader: default_leader(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
Height::new(1),
100,
);
let digest = block.digest();
assert!(marshal.verified(round, ready(block.clone())).await);
assert!(marshal.certified(round, ready(block)).await);
actor_handle.abort();
let _ = actor_handle.await;
drop(marshal);
let setup2 = StandardHarness::setup_validator(
context
.child("validator_restart")
.with_attribute("index", 0),
&mut oracle,
me,
ConstantProvider::new(schemes[0].clone()),
)
.await;
let marshal2 = setup2.mailbox;
assert!(
marshal2.get_block(&digest).await.is_some(),
"certified block covered by a verified write must survive restart"
);
});
}
type BufferSend = (Round, Digested<B>, Recipients<PublicKey>);
#[derive(Clone, Default)]
struct RecordingBuffer {
blocks: Arc<Mutex<Vec<B>>>,
last_handed: Arc<Mutex<Option<Weak<B>>>>,
evict_on_next_hit: Arc<Mutex<bool>>,
digest_subscriptions: Arc<Mutex<Vec<oneshot::Sender<Digested<B>>>>>,
commitment_subscriptions: Arc<Mutex<Vec<oneshot::Sender<Digested<B>>>>>,
sends: Arc<Mutex<Vec<BufferSend>>>,
}
impl RecordingBuffer {
fn insert(&self, block: B) {
self.blocks.lock().push(block);
}
fn insert_transient(&self, block: B) {
self.insert(block);
*self.evict_on_next_hit.lock() = true;
}
fn contains(&self, digest: D) -> bool {
self.blocks
.lock()
.iter()
.any(|block| block.digest() == digest)
}
fn find(&self, digest: D) -> Option<Digested<B>> {
let mut blocks = self.blocks.lock();
let index = blocks.iter().position(|block| block.digest() == digest)?;
let block = if std::mem::take(&mut *self.evict_on_next_hit.lock()) {
blocks.remove(index)
} else {
blocks[index].clone()
};
let block = ready(block);
*self.last_handed.lock() = Some(Arc::downgrade(&block.inner_shared()));
Some(block)
}
fn last_handed(&self) -> Option<Weak<B>> {
self.last_handed.lock().clone()
}
fn sends(&self) -> Vec<BufferSend> {
self.sends.lock().clone()
}
fn subscription_count(&self) -> usize {
self.digest_subscriptions.lock().len() + self.commitment_subscriptions.lock().len()
}
fn commitment_subscription_count(&self) -> usize {
self.commitment_subscriptions.lock().len()
}
fn commitment_subscription_closed(&self, index: usize) -> bool {
self.commitment_subscriptions.lock()[index].is_closed()
}
fn close_commitment_subscription(&self, index: usize) {
self.commitment_subscriptions.lock()[index] = oneshot::channel().0;
}
fn deliver_commitment_subscription(&self, index: usize, block: B) -> bool {
let sender = std::mem::replace(
&mut self.commitment_subscriptions.lock()[index],
oneshot::channel().0,
);
sender.send(ready(block)).is_ok()
}
}
impl crate::marshal::core::Buffer<Standard<B>> for RecordingBuffer {
type PublicKey = PublicKey;
async fn find_by_digest(&self, digest: D) -> Option<Digested<B>> {
self.find(digest)
}
async fn find_by_commitment(&self, commitment: D) -> Option<Digested<B>> {
self.find(commitment)
}
fn subscribe_by_digest(&self, _digest: D) -> Option<oneshot::Receiver<Digested<B>>> {
let (sender, receiver) = oneshot::channel();
self.digest_subscriptions.lock().push(sender);
Some(receiver)
}
fn subscribe_by_commitment(
&self,
_commitment: D,
) -> Option<oneshot::Receiver<Digested<B>>> {
let (sender, receiver) = oneshot::channel();
self.commitment_subscriptions.lock().push(sender);
Some(receiver)
}
fn send(&self, round: Round, block: Digested<B>, recipients: Recipients<PublicKey>) {
self.sends.lock().push((round, block, recipients));
}
}
type TargetedFetch = (handler::Key<D>, NonEmptyVec<PublicKey>);
type FetchRecord = Fetch<handler::Key<D>, handler::Annotation>;
#[derive(Clone, Default)]
struct RecordingResolver {
fetches: Arc<Mutex<Vec<FetchRecord>>>,
active_fetches: Arc<Mutex<Vec<FetchRecord>>>,
targeted: Arc<Mutex<Vec<TargetedFetch>>>,
retains: Arc<Mutex<usize>>,
auto_delivery: Arc<Mutex<Option<Bytes>>>,
delivery_responses: Arc<Mutex<Vec<oneshot::Receiver<Outcome>>>>,
sender: Option<mailbox::Sender<handler::Message<D>>>,
}
impl RecordingResolver {
fn holding(metrics: impl Metrics) -> (handler::Receiver<D>, Self) {
let (sender, receiver) = mailbox::new(metrics, NZUsize!(100));
(
handler::Receiver::new(receiver),
Self {
fetches: Arc::new(Mutex::new(Vec::new())),
active_fetches: Arc::new(Mutex::new(Vec::new())),
targeted: Arc::new(Mutex::new(Vec::new())),
retains: Arc::new(Mutex::new(0)),
auto_delivery: Arc::new(Mutex::new(None)),
delivery_responses: Arc::new(Mutex::new(Vec::new())),
sender: Some(sender),
},
)
}
fn record_fetch(&self, fetch: FetchRecord) {
self.fetches.lock().push(fetch.clone());
self.active_fetches.lock().push(fetch.clone());
let Some(value) = self.auto_delivery.lock().take() else {
return;
};
let Some(sender) = &self.sender else {
return;
};
let (response, response_rx) = oneshot::channel();
self.delivery_responses.lock().push(response_rx);
let _ = sender.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: fetch.key,
subscribers: NonEmptyVec::new((fetch.subscriber, tracing::Span::none())),
},
value,
response,
});
}
fn respond_to_next_fetch(&self, value: Bytes) {
let replaced = self.auto_delivery.lock().replace(value);
assert!(
replaced.is_none(),
"recording resolver already has an automatic delivery"
);
}
async fn wait_for_delivery_response(&self) -> bool {
let response = self
.delivery_responses
.lock()
.pop()
.expect("delivery response missing");
response.await.expect("delivery response sender dropped")
== Outcome::Resolved(Resolution::Complete)
}
fn fetches(&self) -> Vec<FetchRecord> {
self.fetches.lock().clone()
}
fn active_fetches(&self) -> Vec<FetchRecord> {
self.active_fetches.lock().clone()
}
fn targeted(&self) -> Vec<TargetedFetch> {
self.targeted.lock().clone()
}
fn targeted_is_empty(&self) -> bool {
self.targeted.lock().is_empty()
}
fn retain_count(&self) -> usize {
*self.retains.lock()
}
fn enqueue(&self, message: handler::Message<D>) -> Feedback {
self.sender
.as_ref()
.expect("recording resolver sender missing")
.enqueue(message)
}
}
impl Resolver for RecordingResolver {
type Key = handler::Key<D>;
type Subscriber = handler::Annotation;
fn fetch<F>(&mut self, fetch: F) -> Feedback
where
F: Into<Fetch<Self::Key, Self::Subscriber>> + Send,
{
self.record_fetch(fetch.into());
Feedback::Ok
}
fn fetch_all<F>(&mut self, fetches: Vec<F>) -> Feedback
where
F: Into<Fetch<Self::Key, Self::Subscriber>> + Send,
{
for fetch in fetches {
self.record_fetch(fetch.into());
}
Feedback::Ok
}
fn retain(
&mut self,
predicate: impl Fn(&Self::Key, &Self::Subscriber) -> bool + Send + 'static,
) -> Feedback {
self.active_fetches
.lock()
.retain(|fetch| predicate(&fetch.key, &fetch.subscriber));
*self.retains.lock() += 1;
Feedback::Ok
}
}
impl TargetedResolver for RecordingResolver {
type PublicKey = PublicKey;
fn fetch_targeted(
&mut self,
fetch: impl Into<Fetch<Self::Key, Self::Subscriber>> + Send,
targets: NonEmptyVec<Self::PublicKey>,
) -> Feedback {
self.targeted.lock().push((fetch.into().key, targets));
Feedback::Ok
}
fn fetch_all_targeted<F>(
&mut self,
fetches: Vec<(F, NonEmptyVec<Self::PublicKey>)>,
) -> Feedback
where
F: Into<Fetch<Self::Key, Self::Subscriber>> + Send,
{
let mut targeted = self.targeted.lock();
for (fetch, targets) in fetches {
targeted.push((fetch.into().key, targets));
}
Feedback::Ok
}
}
#[derive(Clone)]
struct NoopConsumer;
impl Consumer for NoopConsumer {
type Key = handler::Key<D>;
type Value = Bytes;
type Subscriber = handler::Annotation;
type Outcome = bool;
fn deliver(
&mut self,
_delivery: Delivery<Self::Key, Self::Subscriber>,
_value: Self::Value,
) -> oneshot::Receiver<bool> {
let (sender, receiver) = oneshot::channel();
sender.send_lossy(false);
receiver
}
}
#[derive(Clone)]
struct StaticProducer {
key: handler::Key<D>,
value: Bytes,
}
impl StaticProducer {
fn new(key: handler::Key<D>, value: Bytes) -> Self {
Self { key, value }
}
}
impl commonware_resolver::p2p::Producer for StaticProducer {
type Key = handler::Key<D>;
fn produce(&mut self, key: Self::Key) -> oneshot::Receiver<Bytes> {
let (sender, receiver) = oneshot::channel();
if key == self.key {
sender.send_lossy(self.value.clone());
}
receiver
}
}
async fn wait_until<F: FnMut() -> bool>(
context: &deterministic::Context,
deadline: Duration,
label: &str,
mut cond: F,
) {
let start = context.current();
while !cond() {
if context.current().duration_since(start).unwrap_or_default() > deadline {
panic!("{label} did not hold within {deadline:?}");
}
context.sleep(Duration::from_millis(10)).await;
}
}
#[derive(Clone)]
struct HoldingBlockReporter {
started: Arc<Mutex<Option<oneshot::Sender<Height>>>>,
pending: Arc<Mutex<Vec<Exact>>>,
min_signal_height: Height,
}
impl HoldingBlockReporter {
fn new() -> (Self, oneshot::Receiver<Height>) {
Self::new_from(Height::zero())
}
fn new_after(height: Height) -> (Self, oneshot::Receiver<Height>) {
Self::new_from(height.next())
}
fn new_from(min_signal_height: Height) -> (Self, oneshot::Receiver<Height>) {
let (started_tx, started_rx) = oneshot::channel();
(
Self {
started: Arc::new(Mutex::new(Some(started_tx))),
pending: Arc::new(Mutex::new(Vec::new())),
min_signal_height,
},
started_rx,
)
}
}
impl Reporter for HoldingBlockReporter {
type Activity = Update<B>;
fn report(&mut self, activity: Self::Activity) -> Feedback {
match activity {
Update::Block(block, ack) => {
if block.height() < self.min_signal_height {
ack.acknowledge();
return Feedback::Ok;
}
if let Some(started) = self.started.lock().take() {
started.send_lossy(block.height());
}
self.pending.lock().push(ack);
}
Update::Tip(_, _, _) => {}
}
Feedback::Ok
}
}
#[derive(Clone)]
struct ApplicationReporter {
updates: mpsc::UnboundedSender<Update<B>>,
}
impl Reporter for ApplicationReporter {
type Activity = Update<B>;
fn report(&mut self, activity: Self::Activity) -> Feedback {
if self.updates.send(activity).is_ok() {
Feedback::Ok
} else {
Feedback::Closed
}
}
}
async fn start_standard_actor<R, Buf, P>(
context: deterministic::Context,
partition_prefix: &str,
provider: P,
application: R,
buffer: Option<Buf>,
start: Start<S, D, Digested<B>>,
) -> (
Mailbox<S, Standard<B>>,
Option<Buf>,
RecordingResolver,
commonware_runtime::Handle<()>,
)
where
R: Reporter<Activity = Update<B>>,
P: Provider<Scope = Epoch, Scheme = S>,
Buf: crate::marshal::core::Buffer<Standard<B>, PublicKey = PublicKey>,
{
let config = Config {
provider,
epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
start,
mailbox_size: NZUsize!(100),
view_retention: ViewDelta::new(10),
max_repair: NZUsize!(10),
max_pending_acks: NZUsize!(1),
block_codec_config: (),
partition_prefix: partition_prefix.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NZUsize!(1024),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
strategy: Sequential,
};
let finalizations_by_height = immutable::Archive::init(
context.child("finalizations_by_height"),
immutable::Config {
metadata_partition: format!("{partition_prefix}-finalizations-by-height-metadata"),
freezer_table_partition: format!(
"{partition_prefix}-finalizations-by-height-freezer-table"
),
freezer_table_initial_size: 64,
freezer_table_resize_frequency: 10,
freezer_table_resize_chunk_size: 10,
freezer_key_partition: format!(
"{partition_prefix}-finalizations-by-height-freezer-key"
),
freezer_key_page_cache: config.page_cache.clone(),
freezer_value_partition: format!(
"{partition_prefix}-finalizations-by-height-freezer-value"
),
freezer_value_target_size: 1024,
freezer_value_compression: None,
ordinal_partition: format!("{partition_prefix}-finalizations-by-height-ordinal"),
items_per_section: NZU64!(10),
codec_config: S::certificate_codec_config_unbounded(),
replay_buffer: config.replay_buffer,
freezer_key_write_buffer: config.key_write_buffer,
freezer_value_write_buffer: config.value_write_buffer,
ordinal_write_buffer: config.key_write_buffer,
},
)
.await
.expect("failed to initialize finalizations by height archive");
let finalized_blocks = immutable::Archive::init(
context.child("finalized_blocks"),
immutable::Config {
metadata_partition: format!("{partition_prefix}-finalized_blocks-metadata"),
freezer_table_partition: format!(
"{partition_prefix}-finalized_blocks-freezer-table"
),
freezer_table_initial_size: 64,
freezer_table_resize_frequency: 10,
freezer_table_resize_chunk_size: 10,
freezer_key_partition: format!("{partition_prefix}-finalized_blocks-freezer-key"),
freezer_key_page_cache: config.page_cache.clone(),
freezer_value_partition: format!(
"{partition_prefix}-finalized_blocks-freezer-value"
),
freezer_value_target_size: 1024,
freezer_value_compression: None,
ordinal_partition: format!("{partition_prefix}-finalized_blocks-ordinal"),
items_per_section: NZU64!(10),
codec_config: config.block_codec_config,
replay_buffer: config.replay_buffer,
freezer_key_write_buffer: config.key_write_buffer,
freezer_value_write_buffer: config.value_write_buffer,
ordinal_write_buffer: config.key_write_buffer,
},
)
.await
.expect("failed to initialize finalized blocks archive");
let (actor, mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
config,
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("mailbox"));
let actor_handle = if let Some(buffer) = buffer.clone() {
actor.start(application, buffer, (resolver_rx, resolver.clone()))
} else {
actor.start_unbuffered(application, (resolver_rx, resolver.clone()))
};
(mailbox, buffer, resolver, actor_handle)
}
#[test_traced("WARN")]
fn test_standard_application_shutdown_redelivers_pending_ack_after_restart() {
const PARTITION_PREFIX: &str = "application-shutdown-with-pending-ack";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
let first_run = |mut context: deterministic::Context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let provider = ConstantProvider::new(schemes[0].clone());
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let (updates, mut application_mailbox) = mpsc::unbounded_channel::<Update<B>>();
let (delivered, delivery) = oneshot::channel();
let application_handle = context.child("application").spawn(move |_| async move {
let mut delivered = Some(delivered);
while let Some(update) = application_mailbox.recv().await {
let Update::Block(block, acknowledgement) = update else {
continue;
};
if block.height() == Height::zero() {
acknowledgement.acknowledge();
continue;
}
delivered
.take()
.expect("normal block should only be delivered once")
.send_lossy(block.height());
std::future::pending::<()>().await;
acknowledgement.acknowledge();
}
});
let (mut mailbox, _buffer, _resolver, marshal_handle) = start_standard_actor(
context.child("validator"),
PARTITION_PREFIX,
provider.clone(),
ApplicationReporter { updates },
Some(RecordingBuffer::default()),
Start::Genesis(genesis.clone()),
)
.await;
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(genesis.digest(), Height::new(1), 100);
let block_digest = block.digest();
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(round, ready(block)).await);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
assert_eq!(
delivery.await.expect("normal block should be delivered"),
Height::new(1)
);
application_handle.abort();
let _ = application_handle.await;
marshal_handle
.await
.expect("application shutdown should stop marshal cleanly");
(provider, genesis, block_digest)
};
let ((provider, genesis, block_digest), checkpoint) = runner.start_and_recover(first_run);
deterministic::Runner::from(checkpoint).start(move |context| async move {
let restart_application = Application::<B>::manual_ack();
let (_mailbox, _buffer, _resolver, _marshal_handle) = start_standard_actor(
context.child("validator").with_attribute("restart", 0),
PARTITION_PREFIX,
provider,
restart_application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(genesis),
)
.await;
select! {
height = restart_application.acknowledged() => {
assert_eq!(height, Height::new(1));
assert_eq!(
restart_application
.blocks()
.get(&height)
.expect("redelivered block must be recorded")
.digest(),
block_digest,
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("unacknowledged block was not redelivered after restart");
},
}
});
}
#[test_traced("WARN")]
fn test_standard_actor_starts_without_buffer() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let (mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"standard-no-buffer",
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
None::<RecordingBuffer>,
Start::Genesis(genesis.clone()),
)
.await;
let stored_genesis = mailbox
.get_block(Identifier::Height(Height::zero()))
.await
.expect("genesis should be available without a buffer");
assert_eq!(stored_genesis.digest(), genesis.digest());
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(genesis.digest(), Height::new(1), 100);
let digest = block.digest();
assert!(mailbox.verified(round, ready(block)).await);
mailbox.forward(
round,
digest,
Recipients::Some(vec![participants[1].clone()]),
);
let verified = mailbox
.get_verified(round)
.await
.expect("verified block should remain available without a buffer");
assert_eq!(verified.digest(), digest);
});
}
#[test_traced("WARN")]
#[should_panic(expected = "floor finalization must verify")]
fn test_standard_start_floor_rejects_invalid_finalization() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let Fixture {
schemes: wrong_schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
let (application, _started_rx) = HoldingBlockReporter::new();
let (_mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"start-floor-invalid",
ConstantProvider::new(wrong_schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Floor(floor_finalization),
)
.await;
context.sleep(Duration::from_secs(1)).await;
});
}
#[test_traced("WARN")]
#[should_panic(expected = "floor finalization must verify")]
fn test_standard_set_floor_rejects_invalid_finalization() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let Fixture {
schemes: wrong_schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
let (mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"set-floor-invalid",
ConstantProvider::new(wrong_schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
mailbox.set_floor(floor_finalization);
context.sleep(Duration::from_secs(1)).await;
});
}
#[test_traced("WARN")]
fn test_standard_start_floor_fetches_async_and_serves_requests() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
let (application, mut started_rx) = HoldingBlockReporter::new_after(Height::zero());
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"start-floor-async",
ConstantProvider::new(schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Floor(floor_finalization),
)
.await;
wait_until(
&context,
Duration::from_secs(5),
"floor block fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == floor_block.digest()
)
})
},
)
.await;
let served = make_raw_block(Sha256::hash(&[b"served-parent"]), Height::new(1), 100);
let served_round = Round::new(Epoch::zero(), View::new(1));
assert!(mailbox.verified(served_round, ready(served.clone())).await);
let (response, response_rx) = oneshot::channel();
resolver.enqueue(handler::Message::Produce {
key: handler::Key::Block(served.digest()),
response,
});
assert_eq!(response_rx.await.unwrap(), served.encode());
let next = make_raw_block(floor_block.digest(), Height::new(6), 600);
let next_round = Round::new(Epoch::zero(), View::new(6));
assert!(mailbox.verified(next_round, ready(next.clone())).await);
let next_finalization = StandardHarness::make_finalization(
Proposal::new(next_round, View::new(5), next.digest()),
&schemes,
QUORUM,
);
let mut mailbox_for_report = mailbox.clone();
StandardHarness::report_finalization(&mut mailbox_for_report, next_finalization).await;
context.sleep(Duration::from_millis(100)).await;
assert!(matches!(started_rx.try_recv(), Err(TryRecvError::Empty)));
assert!(mailbox.verified(floor_round, ready(floor_block)).await);
assert_eq!(started_rx.await.unwrap(), Height::new(5));
});
}
#[test_traced("WARN")]
fn test_standard_start_floor_applies_local_anchor_without_fetch() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block =
make_raw_block(Sha256::hash(&[b"local-floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
let buffer = RecordingBuffer::default();
buffer.insert(floor_block.clone());
let (application, started_rx) = HoldingBlockReporter::new();
let (mailbox, _buffer, resolver, actor_handle) = start_standard_actor(
context.child("validator"),
"start-floor-local-anchor",
ConstantProvider::new(schemes[0].clone()),
application,
Some(buffer),
Start::Floor(floor_finalization.clone()),
)
.await;
let mut mailbox = mailbox;
wait_until(
&context,
Duration::from_secs(5),
"local floor anchor application",
|| resolver.retain_count() >= 2,
)
.await;
assert!(
resolver.fetches().is_empty(),
"local startup floor anchor must not be fetched"
);
assert_eq!(
mailbox.get_block(Height::new(5)).await.unwrap().digest(),
floor_block.digest()
);
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(4)))
);
assert!(mailbox.get_block(Height::new(4)).await.is_none());
assert_eq!(
mailbox
.get_anchor()
.await
.map(|(processed, block)| (processed, block.digest())),
Some((Processed::Absent(Height::new(4)), floor_block.digest()))
);
let next = make_raw_block(floor_block.digest(), Height::new(6), 600);
let next_round = Round::new(Epoch::zero(), View::new(6));
assert!(mailbox.verified(next_round, ready(next.clone())).await);
let next_finalization = StandardHarness::make_finalization(
Proposal::new(next_round, View::new(5), next.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, next_finalization).await;
assert_eq!(started_rx.await.unwrap(), Height::new(5));
actor_handle.abort();
let _ = actor_handle.await;
let (application, _started_rx) = HoldingBlockReporter::new();
let (mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("restart"),
"start-floor-local-anchor",
ConstantProvider::new(schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Floor(floor_finalization),
)
.await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(4)))
);
assert_eq!(
mailbox
.get_anchor()
.await
.map(|(processed, block)| (processed, block.digest())),
Some((Processed::Absent(Height::new(4)), floor_block.digest()))
);
});
}
#[test_traced("WARN")]
fn test_standard_start_floor_height_one_without_metadata_delivers_anchor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let partition_prefix = "start-floor-height-one-without-metadata";
let floor_round = Round::new(Epoch::zero(), View::new(1));
let floor_block =
make_raw_block(Sha256::hash(&[b"genesis-parent"]), Height::new(1), 100);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::zero(), floor_block.digest()),
&schemes,
QUORUM,
);
let blocks = [floor_block.clone()];
let finalizations = [(Height::new(1), floor_finalization.clone())];
seed_inconsistent_restart_state(
context.child("seed_restart_state"),
partition_prefix,
&blocks,
&finalizations,
)
.await;
let (application, started_rx) = HoldingBlockReporter::new_after(Height::zero());
let (_mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Floor(floor_finalization),
)
.await;
select! {
height = started_rx => {
assert_eq!(
height.expect("floor anchor delivery signal missing"),
Height::new(1),
"height-one startup floor must be delivered without prior metadata"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("height-one startup floor was not delivered");
},
}
assert!(
resolver.fetches().is_empty(),
"stored startup floor anchor must not be fetched"
);
});
}
#[test_traced("WARN")]
fn test_standard_set_floor_holds_dispatch_until_anchor_arrives() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
let (application, mut started_rx) = HoldingBlockReporter::new_after(Height::zero());
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"set-floor-holds-dispatch",
ConstantProvider::new(schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
mailbox.set_floor(floor_finalization);
wait_until(
&context,
Duration::from_secs(5),
"floor block fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == floor_block.digest()
)
})
},
)
.await;
let next = make_raw_block(floor_block.digest(), Height::new(6), 600);
let next_round = Round::new(Epoch::zero(), View::new(6));
assert!(mailbox.verified(next_round, ready(next.clone())).await);
let next_finalization = StandardHarness::make_finalization(
Proposal::new(next_round, View::new(5), next.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, next_finalization).await;
context.sleep(Duration::from_millis(100)).await;
assert!(matches!(started_rx.try_recv(), Err(TryRecvError::Empty)));
let floor_fetch = resolver
.fetches()
.into_iter()
.find(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == floor_block.digest()
)
})
.expect("floor fetch missing");
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: floor_fetch.key,
subscribers: NonEmptyVec::new((
floor_fetch.subscriber,
tracing::Span::none()
)),
},
value: floor_block.encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"floor block delivery should validate"
);
assert_eq!(started_rx.await.unwrap(), Height::new(5));
});
}
#[test_traced("WARN")]
fn test_standard_floor_jump_ignores_stale_application_ack() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"floor-jump-ignores-stale-ack",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
let block1_round = Round::new(Epoch::zero(), View::new(1));
let block1 = make_raw_block(Sha256::hash(&[b"block1-parent"]), Height::new(1), 100);
let block1_finalization = StandardHarness::make_finalization(
Proposal::new(block1_round, View::zero(), block1.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block1_round, ready(block1.clone())).await);
StandardHarness::report_finalization(&mut mailbox, block1_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"first block dispatch",
|| application.pending_ack_heights() == vec![Height::new(1)],
)
.await;
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(floor_finalization);
wait_until(
&context,
Duration::from_secs(5),
"floor block fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == floor_block.digest()
)
})
},
)
.await;
assert!(
mailbox
.verified(floor_round, ready(floor_block.clone()))
.await
);
assert_eq!(
mailbox
.get_block(Height::new(5))
.await
.expect("floor block missing")
.digest(),
floor_block.digest()
);
let fetches_after_floor = resolver.fetches().len();
assert_eq!(application.acknowledge_next(), Some(Height::new(1)));
context.sleep(Duration::from_millis(100)).await;
let _subscription = mailbox.subscribe_by_commitment(
Sha256::hash(&[b"below-floor-after-stale-ack"]),
CommitmentFallback::FetchByCommitment {
height: Height::new(2),
},
);
let _ = mailbox.get_block(Height::new(5)).await;
assert_eq!(
resolver.fetches().len(),
fetches_after_floor,
"stale pre-floor ack must not lower the processed height floor"
);
});
}
#[test_traced("WARN")]
fn test_standard_local_floor_jump_ignores_stale_application_ack() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let application = Application::<B>::manual_ack();
let buffer = RecordingBuffer::default();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"local-floor-jump-ignores-stale-ack",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(buffer.clone()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
let block1_round = Round::new(Epoch::zero(), View::new(1));
let block1 = make_raw_block(Sha256::hash(&[b"block1-parent"]), Height::new(1), 100);
let block1_finalization = StandardHarness::make_finalization(
Proposal::new(block1_round, View::zero(), block1.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block1_round, ready(block1)).await);
StandardHarness::report_finalization(&mut mailbox, block1_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"first block dispatch",
|| application.pending_ack_heights() == vec![Height::new(1)],
)
.await;
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block =
make_raw_block(Sha256::hash(&[b"local-floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
buffer.insert(floor_block.clone());
mailbox.set_floor(floor_finalization);
assert_eq!(
mailbox
.get_block(Height::new(5))
.await
.expect("floor block missing")
.digest(),
floor_block.digest()
);
assert!(
resolver.fetches().is_empty(),
"local floor anchor must not be fetched"
);
let fetches_after_floor = resolver.fetches().len();
assert_eq!(application.acknowledge_next(), Some(Height::new(1)));
context.sleep(Duration::from_millis(100)).await;
let _subscription = mailbox.subscribe_by_commitment(
Sha256::hash(&[b"below-local-floor-after-stale-ack"]),
CommitmentFallback::FetchByCommitment {
height: Height::new(2),
},
);
let _ = mailbox.get_block(Height::new(5)).await;
assert_eq!(
resolver.fetches().len(),
fetches_after_floor,
"stale pre-floor ack must not lower a locally installed floor"
);
});
}
#[test_traced("WARN")]
fn test_standard_set_floor_repairs_gap_after_anchor_arrives() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"set-floor-repairs-gap-after-anchor",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
mailbox.set_floor(floor_finalization);
wait_until(
&context,
Duration::from_secs(5),
"floor block fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == floor_block.digest()
)
})
},
)
.await;
let missing = make_raw_block(floor_block.digest(), Height::new(6), 600);
let later = make_raw_block(missing.digest(), Height::new(7), 700);
let later_round = Round::new(Epoch::zero(), View::new(7));
assert!(mailbox.verified(later_round, ready(later.clone())).await);
let later_finalization = StandardHarness::make_finalization(
Proposal::new(later_round, View::new(6), later.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, later_finalization).await;
assert!(mailbox.get_finalization(Height::new(7)).await.is_some());
assert!(
resolver.fetches().iter().all(|fetch| {
!matches!(
(&fetch.key, &fetch.subscriber),
(
handler::Key::Block(commitment),
handler::Annotation::Finalized(handler::Finalized::ByHeight { height })
) if *commitment == missing.digest() && *height == Height::new(6)
)
}),
"gap repair must wait until the floor anchor is resolved"
);
assert!(mailbox.verified(floor_round, ready(floor_block)).await);
wait_until(
&context,
Duration::from_secs(5),
"missing post-floor parent fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
(&fetch.key, &fetch.subscriber),
(
handler::Key::Block(commitment),
handler::Annotation::Finalized(
handler::Finalized::ByHeight { height }
)
) if *commitment == missing.digest() && *height == Height::new(6)
)
})
},
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_local_floor_anchor_resumes_gap_repair() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
let next_round = Round::new(Epoch::zero(), View::new(6));
let next_block = make_raw_block(floor_block.digest(), Height::new(6), 600);
let next_finalization = StandardHarness::make_finalization(
Proposal::new(next_round, View::new(5), next_block.digest()),
&schemes,
QUORUM,
);
let partition_prefix = "local-floor-anchor-resumes-gap-repair";
seed_inconsistent_restart_state(
context.child("storage"),
partition_prefix,
&[],
&[(Height::new(6), next_finalization)],
)
.await;
let (application, _started_rx) = HoldingBlockReporter::new();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Floor(floor_finalization),
)
.await;
wait_until(
&context,
Duration::from_secs(5),
"floor block fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == floor_block.digest()
)
})
},
)
.await;
assert!(mailbox.verified(floor_round, ready(floor_block)).await);
wait_until(
&context,
Duration::from_secs(5),
"gap repair after local floor anchor",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == next_block.digest()
)
})
},
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_newer_pending_floor_supersedes_older_anchor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let (application, mut started_rx) = HoldingBlockReporter::new_after(Height::zero());
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"set-floor-supersedes-pending",
ConstantProvider::new(schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
let old_floor_round = Round::new(Epoch::zero(), View::new(5));
let old_floor_block =
make_raw_block(Sha256::hash(&[b"old-floor-parent"]), Height::new(5), 500);
let old_floor_finalization = StandardHarness::make_finalization(
Proposal::new(old_floor_round, View::new(4), old_floor_block.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(old_floor_finalization);
wait_until(
&context,
Duration::from_secs(5),
"old floor block fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == old_floor_block.digest()
)
})
},
)
.await;
let new_floor_round = Round::new(Epoch::zero(), View::new(7));
let new_floor_block =
make_raw_block(Sha256::hash(&[b"new-floor-parent"]), Height::new(7), 700);
let new_floor_finalization = StandardHarness::make_finalization(
Proposal::new(new_floor_round, View::new(6), new_floor_block.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(new_floor_finalization);
wait_until(
&context,
Duration::from_secs(5),
"new floor block fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == new_floor_block.digest()
)
})
},
)
.await;
let old_next_round = Round::new(Epoch::zero(), View::new(6));
let old_next = make_raw_block(old_floor_block.digest(), Height::new(6), 600);
assert!(
mailbox
.verified(old_next_round, ready(old_next.clone()))
.await
);
let old_next_finalization = StandardHarness::make_finalization(
Proposal::new(old_next_round, View::new(5), old_next.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, old_next_finalization).await;
assert!(mailbox.get_finalization(Height::new(6)).await.is_some());
let new_next_round = Round::new(Epoch::zero(), View::new(8));
let new_next = make_raw_block(new_floor_block.digest(), Height::new(8), 800);
assert!(
mailbox
.verified(new_next_round, ready(new_next.clone()))
.await
);
let new_next_finalization = StandardHarness::make_finalization(
Proposal::new(new_next_round, View::new(7), new_next.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, new_next_finalization).await;
assert!(mailbox.get_finalization(Height::new(8)).await.is_some());
let old_floor_fetch = resolver
.fetches()
.into_iter()
.find(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == old_floor_block.digest()
)
})
.expect("old floor fetch missing");
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: old_floor_fetch.key,
subscribers: NonEmptyVec::new((
old_floor_fetch.subscriber,
tracing::Span::none()
)),
},
value: old_floor_block.encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx
.await
.expect("old floor delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"old floor block delivery should validate"
);
context.sleep(Duration::from_millis(100)).await;
assert!(matches!(started_rx.try_recv(), Err(TryRecvError::Empty)));
let new_floor_fetch = resolver
.fetches()
.into_iter()
.find(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == new_floor_block.digest()
)
})
.expect("new floor fetch missing");
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: new_floor_fetch.key,
subscribers: NonEmptyVec::new((
new_floor_fetch.subscriber,
tracing::Span::none()
)),
},
value: new_floor_block.encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx
.await
.expect("new floor delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"new floor block delivery should validate"
);
assert_eq!(started_rx.await.unwrap(), Height::new(7));
});
}
#[test_traced("WARN")]
fn test_standard_set_floor_applies_buffered_anchor_on_notarization() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_proposal = Proposal::new(floor_round, View::new(4), floor_block.digest());
let floor_finalization =
StandardHarness::make_finalization(floor_proposal.clone(), &schemes, QUORUM);
let (application, mut started_rx) = HoldingBlockReporter::new_after(Height::zero());
let buffer = RecordingBuffer::default();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"set-floor-buffered-anchor-notarization",
ConstantProvider::new(schemes[0].clone()),
application,
Some(buffer.clone()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
mailbox.set_floor(floor_finalization);
wait_until(
&context,
Duration::from_secs(5),
"floor block fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == floor_block.digest()
)
})
},
)
.await;
let next = make_raw_block(floor_block.digest(), Height::new(6), 600);
let next_round = Round::new(Epoch::zero(), View::new(6));
assert!(mailbox.verified(next_round, ready(next.clone())).await);
let next_finalization = StandardHarness::make_finalization(
Proposal::new(next_round, View::new(5), next.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, next_finalization).await;
context.sleep(Duration::from_millis(100)).await;
assert!(matches!(started_rx.try_recv(), Err(TryRecvError::Empty)));
buffer.insert(floor_block);
let floor_notarization =
StandardHarness::make_notarization(floor_proposal, &schemes, QUORUM);
StandardHarness::report_notarization(&mut mailbox, floor_notarization).await;
assert_eq!(started_rx.await.unwrap(), Height::new(5));
});
}
#[test_traced("WARN")]
fn test_standard_pending_floor_releases_buffer_waiter() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let old = make_raw_block(Sha256::hash(&[b"old-parent"]), Height::new(5), 500);
let new = make_raw_block(Sha256::hash(&[b"new-parent"]), Height::new(6), 600);
let old_proposal = Proposal::new(
Round::new(Epoch::zero(), View::new(5)),
View::new(4),
old.digest(),
);
let new_proposal = Proposal::new(
Round::new(Epoch::zero(), View::new(6)),
View::new(5),
new.digest(),
);
let (application, started_rx) = HoldingBlockReporter::new_after(Height::zero());
let buffer = RecordingBuffer::default();
let (mut mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"pending-floor-releases-waiter",
ConstantProvider::new(schemes[0].clone()),
application,
Some(buffer.clone()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
mailbox.set_floor(StandardHarness::make_finalization(
old_proposal,
&schemes,
QUORUM,
));
while buffer.commitment_subscription_count() != 1 {
reschedule().await;
}
mailbox.set_floor(StandardHarness::make_finalization(
new_proposal.clone(),
&schemes,
QUORUM,
));
while !buffer.commitment_subscription_closed(0) {
reschedule().await;
}
assert_eq!(buffer.commitment_subscription_count(), 2);
assert!(!buffer.commitment_subscription_closed(1));
buffer.insert(new);
StandardHarness::report_notarization(
&mut mailbox,
StandardHarness::make_notarization(new_proposal, &schemes, QUORUM),
)
.await;
assert_eq!(started_rx.await.unwrap(), Height::new(6));
while !buffer.commitment_subscription_closed(1) {
reschedule().await;
}
let missing = make_raw_block(Sha256::hash(&[b"missing-parent"]), Height::new(7), 700);
mailbox.set_floor(StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(7)),
View::new(6),
missing.digest(),
),
&schemes,
QUORUM,
));
while buffer.commitment_subscription_count() != 3 {
reschedule().await;
}
let local = make_raw_block(Sha256::hash(&[b"local-parent"]), Height::new(8), 800);
let finalization = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(8)),
View::new(7),
local.digest(),
),
&schemes,
QUORUM,
);
buffer.insert(local);
mailbox.set_floor(finalization);
while !buffer.commitment_subscription_closed(2) {
reschedule().await;
}
assert_eq!(buffer.commitment_subscription_count(), 3);
});
}
#[test_traced("WARN")]
fn test_standard_pending_floor_drops_in_flight_ack_before_anchor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"pending-floor-drops-in-flight-ack",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
let block1_round = Round::new(Epoch::zero(), View::new(1));
let block1 = make_raw_block(Sha256::hash(&[b"block1-parent"]), Height::new(1), 100);
let block1_finalization = StandardHarness::make_finalization(
Proposal::new(block1_round, View::zero(), block1.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block1_round, ready(block1.clone())).await);
StandardHarness::report_finalization(&mut mailbox, block1_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"first block dispatch",
|| application.pending_ack_heights() == vec![Height::new(1)],
)
.await;
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"stale-floor"]), Height::zero(), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(floor_finalization);
wait_until(
&context,
Duration::from_secs(5),
"stale floor fetch",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == floor_block.digest()
)
})
},
)
.await;
let block2_round = Round::new(Epoch::zero(), View::new(2));
let block2 = make_raw_block(block1.digest(), Height::new(2), 200);
let block2_finalization = StandardHarness::make_finalization(
Proposal::new(block2_round, View::new(1), block2.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block2_round, ready(block2)).await);
StandardHarness::report_finalization(&mut mailbox, block2_finalization).await;
assert!(mailbox.get_finalization(Height::new(2)).await.is_some());
context.sleep(Duration::from_millis(100)).await;
assert_eq!(application.pending_ack_heights(), vec![Height::new(1)]);
assert_eq!(application.acknowledge_next(), Some(Height::new(1)));
context.sleep(Duration::from_millis(100)).await;
assert!(mailbox.verified(floor_round, ready(floor_block)).await);
select! {
height = application.acknowledged() => {
assert_eq!(
height,
Height::new(1),
"floor install must ignore acks released while the anchor is pending"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("pending floor anchor did not restart dispatch at its successor");
},
}
assert_eq!(application.acknowledged().await, Height::new(2));
});
}
#[test_traced("WARN")]
fn test_standard_local_floor_below_in_flight_keeps_pending_ack() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let application = Application::<B>::manual_ack();
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"local-floor-below-in-flight-keeps-pending-ack",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(genesis.clone()),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
let block1_round = Round::new(Epoch::zero(), View::new(1));
let block1 = make_raw_block(genesis.digest(), Height::new(1), 100);
let block1_finalization = StandardHarness::make_finalization(
Proposal::new(block1_round, View::zero(), block1.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block1_round, ready(block1)).await);
StandardHarness::report_finalization(&mut mailbox, block1_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"first block dispatch",
|| application.pending_ack_heights() == vec![Height::new(1)],
)
.await;
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), genesis.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(floor_finalization);
assert!(mailbox.get_block(Height::zero()).await.is_some());
assert_eq!(
application.pending_ack_heights(),
vec![Height::new(1)],
"local floor below the in-flight block must not clear or duplicate its ack"
);
let retain_before_ack = resolver.retain_count();
assert_eq!(application.acknowledge_next(), Some(Height::new(1)));
wait_until(
&context,
Duration::from_secs(5),
"first ack processed",
|| resolver.retain_count() > retain_before_ack,
)
.await;
assert!(application.pending_ack_heights().is_empty());
});
}
#[test_traced("WARN")]
fn test_standard_set_floor_below_processed_height_preserves_in_flight_ack() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let application = Application::<B>::manual_ack();
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"set-floor-below-processed-height-preserves-in-flight-ack",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(genesis.clone()),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
let block1_round = Round::new(Epoch::zero(), View::new(1));
let block1 = make_raw_block(genesis.digest(), Height::new(1), 100);
let block1_finalization = StandardHarness::make_finalization(
Proposal::new(block1_round, View::zero(), block1.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block1_round, ready(block1.clone())).await);
StandardHarness::report_finalization(&mut mailbox, block1_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"first block dispatch",
|| application.pending_ack_heights() == vec![Height::new(1)],
)
.await;
let retain_before_block1_ack = resolver.retain_count();
assert_eq!(application.acknowledge_next(), Some(Height::new(1)));
wait_until(
&context,
Duration::from_secs(5),
"first ack processed",
|| resolver.retain_count() > retain_before_block1_ack,
)
.await;
let block2_round = Round::new(Epoch::zero(), View::new(2));
let block2 = make_raw_block(block1.digest(), Height::new(2), 200);
let block2_finalization = StandardHarness::make_finalization(
Proposal::new(block2_round, View::new(1), block2.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block2_round, ready(block2)).await);
StandardHarness::report_finalization(&mut mailbox, block2_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"second block dispatch",
|| application.pending_ack_heights() == vec![Height::new(2)],
)
.await;
let stale_floor_round = Round::new(Epoch::zero(), View::new(5));
let missing = Sha256::hash(&[b"missing-before-stale-lower-floor"]);
let _subscription = mailbox.subscribe_by_commitment(
missing,
CommitmentFallback::FetchByRound {
round: stale_floor_round,
},
);
wait_until(
&context,
Duration::from_secs(5),
"round-bound fetch before stale lower floor",
|| {
resolver.active_fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Notarized { round } if round == stale_floor_round
)
})
},
)
.await;
let stale_floor_finalization = StandardHarness::make_finalization(
Proposal::new(stale_floor_round, View::new(4), genesis.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(stale_floor_finalization);
assert!(mailbox.get_block(Height::zero()).await.is_some());
assert_eq!(
application.pending_ack_heights(),
vec![Height::new(2)],
"stale lower floor must not clear or duplicate in-flight acks"
);
wait_until(
&context,
Duration::from_secs(5),
"stale lower floor round-bound prune",
|| {
resolver.active_fetches().iter().all(|fetch| {
!matches!(
fetch.key,
handler::Key::Notarized { round } if round == stale_floor_round
)
})
},
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_floor_with_stored_predecessor_reports_block() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let application = Application::<B>::manual_ack();
let (mut mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"floor-stored-predecessor",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert_eq!(application.acknowledged().await, Height::zero());
let block1_round = Round::new(Epoch::zero(), View::new(1));
let block1 = make_raw_block(Sha256::hash(&[b"block1-parent"]), Height::new(1), 100);
let block1_finalization = StandardHarness::make_finalization(
Proposal::new(block1_round, View::zero(), block1.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block1_round, ready(block1.clone())).await);
StandardHarness::report_finalization(&mut mailbox, block1_finalization).await;
assert_eq!(application.acknowledged().await, Height::new(1));
while mailbox.get_processed().await != Some(Processed::Block(Height::new(1))) {
context.sleep(Duration::from_millis(10)).await;
}
let block2_round = Round::new(Epoch::zero(), View::new(2));
let block2 = make_raw_block(block1.digest(), Height::new(2), 200);
let block2_finalization = StandardHarness::make_finalization(
Proposal::new(block2_round, View::new(1), block2.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block2_round, ready(block2)).await);
mailbox.set_floor(block2_finalization);
while application.pending_ack_heights() != vec![Height::new(2)] {
context.sleep(Duration::from_millis(10)).await;
}
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(1)))
);
assert_eq!(
mailbox
.get_anchor()
.await
.map(|(processed, block)| (processed, block.digest())),
Some((Processed::Block(Height::new(1)), block1.digest()))
);
});
}
#[test_traced("WARN")]
fn test_standard_same_height_floor_anchor_keeps_pending_ack() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"same-height-floor-keeps-pending-ack",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
let block1_round = Round::new(Epoch::zero(), View::new(1));
let block1 = make_raw_block(Sha256::hash(&[b"block1-parent"]), Height::new(1), 100);
let block1_finalization = StandardHarness::make_finalization(
Proposal::new(block1_round, View::zero(), block1.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block1_round, ready(block1.clone())).await);
StandardHarness::report_finalization(&mut mailbox, block1_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"first block dispatch",
|| application.pending_ack_heights() == vec![Height::new(1)],
)
.await;
let retain_before_ack = resolver.retain_count();
assert_eq!(application.acknowledge_next(), Some(Height::new(1)));
wait_until(
&context,
Duration::from_secs(5),
"first ack processed",
|| resolver.retain_count() > retain_before_ack,
)
.await;
let block2_round = Round::new(Epoch::zero(), View::new(2));
let block2 = make_raw_block(block1.digest(), Height::new(2), 200);
let block2_finalization = StandardHarness::make_finalization(
Proposal::new(block2_round, View::new(1), block2.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block2_round, ready(block2)).await);
StandardHarness::report_finalization(&mut mailbox, block2_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"second block dispatch",
|| application.pending_ack_heights() == vec![Height::new(2)],
)
.await;
let same_height_round = Round::new(Epoch::zero(), View::new(5));
let same_height_finalization = StandardHarness::make_finalization(
Proposal::new(same_height_round, View::zero(), block1.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(same_height_finalization);
assert!(mailbox.get_block(Height::new(1)).await.is_some());
assert_eq!(
application.pending_ack_heights(),
vec![Height::new(2)],
"same-height floor anchor must not clear or duplicate in-flight acks"
);
});
}
#[test_traced("WARN")]
fn test_standard_stale_floor_anchor_advances_round_floor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"stale-floor-anchor-round-floor",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
let block_round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b"processed-parent"]), Height::new(1), 100);
let finalization = StandardHarness::make_finalization(
Proposal::new(block_round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
assert!(mailbox.verified(block_round, ready(block.clone())).await);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
assert_eq!(application.acknowledged().await, Height::new(1));
let floor_round = Round::new(Epoch::zero(), View::new(5));
let missing = Sha256::hash(&[b"missing-before-stale-floor"]);
let _subscription = mailbox.subscribe_by_commitment(
missing,
CommitmentFallback::FetchByRound { round: floor_round },
);
wait_until(
&context,
Duration::from_secs(5),
"round-bound fetch before stale floor",
|| {
resolver.active_fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Notarized { round } if round == floor_round
)
})
},
)
.await;
let stale_floor = StandardHarness::make_finalization(
Proposal::new(floor_round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(stale_floor);
wait_until(
&context,
Duration::from_secs(5),
"stale floor round-bound prune",
|| {
resolver.active_fetches().iter().all(|fetch| {
!matches!(
fetch.key,
handler::Key::Notarized { round } if round == floor_round
)
})
},
)
.await;
let fetches_before = resolver.fetches().len();
mailbox.hint_notarized(floor_round, Sha256::hash(&[b"missing-after-stale-floor"]));
let barrier = make_raw_block(block.digest(), Height::new(2), 200);
assert!(
mailbox
.verified(Round::new(Epoch::zero(), View::new(2)), ready(barrier))
.await
);
assert_eq!(
resolver.fetches().len(),
fetches_before,
"stale floor anchor must apply its round floor to future fetches"
);
});
}
#[test_traced("WARN")]
fn test_standard_floor_anchor_uses_parent_digest_as_commitment() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"standard-floor-parent-digest-commitment",
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let parent_round = Round::new(Epoch::zero(), View::new(1));
let parent_context = Ctx {
round: parent_round,
leader: me.clone(),
parent: (
View::zero(),
StandardHarness::genesis_parent_commitment(NUM_VALIDATORS as u16),
),
};
let parent =
B::new::<Sha256>(parent_context, Sha256::hash(&[b""]), Height::new(1), 100);
let floor_round = Round::new(Epoch::zero(), View::new(2));
let bad_context = Ctx {
round: floor_round,
leader: me,
parent: (
View::new(1),
StandardHarness::genesis_parent_commitment(NUM_VALIDATORS as u16),
),
};
let floor_block = B::new::<Sha256>(bad_context, parent.digest(), Height::new(2), 200);
assert_ne!(floor_block.parent, floor_block.context.parent.1);
let finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(1), floor_block.digest()),
&schemes,
QUORUM,
);
resolver.respond_to_next_fetch(floor_block.encode());
mailbox.set_floor(finalization);
wait_until(
&context,
Duration::from_secs(5),
"floor block fetch",
|| !resolver.fetches().is_empty(),
)
.await;
assert!(
resolver.wait_for_delivery_response().await,
"floor block delivery should be accepted at the resolver boundary"
);
assert!(
mailbox.get_block(Height::new(2)).await.is_some(),
"standard floor anchor should be archived using its parent digest"
);
assert!(
mailbox.get_finalization(Height::new(2)).await.is_some(),
"standard floor finalization should be archived by height"
);
});
}
#[test_traced("WARN")]
fn test_standard_notarized_delivery_wakes_fetch_by_round_subscriber() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let proposal = Proposal::new(round, View::zero(), block.digest());
let notarization = StandardHarness::make_notarization(proposal, &schemes, QUORUM);
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"notarized-delivery-wakes-subscriber",
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let subscription = mailbox.subscribe_by_commitment(
notarization.proposal.payload,
CommitmentFallback::FetchByRound { round },
);
wait_until(
&context,
Duration::from_secs(5),
"fetch-by-round request",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
(&fetch.key, &fetch.subscriber),
(
handler::Key::Notarized { round: request_round },
handler::Annotation::Notarization { round: subscriber_round },
) if *request_round == round && *subscriber_round == round
)
})
},
)
.await;
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Notarized { round },
subscribers: NonEmptyVec::new((
handler::Annotation::Notarization { round },
tracing::Span::none(),
)),
},
value: (notarization, block.clone()).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"notarized delivery should validate"
);
select! {
result = subscription => {
let delivered = result.expect("block subscription should resolve");
assert_eq!(delivered.digest(), block.digest());
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("notarized delivery did not wake block subscriber");
},
}
});
}
#[test_traced("WARN")]
fn test_standard_notarized_delivery_rejects_wrong_round() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let requested_round = Round::new(Epoch::new(1), View::new(1));
let delivered_round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let proposal = Proposal::new(delivered_round, View::zero(), block.digest());
let notarization = StandardHarness::make_notarization(proposal, &schemes, QUORUM);
let (_mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"notarized-delivery-wrong-round",
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Notarized {
round: requested_round,
},
subscribers: NonEmptyVec::new((
handler::Annotation::Notarization {
round: requested_round,
},
tracing::Span::none(),
)),
},
value: (notarization, block).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Invalid,
"wrong-round notarized delivery must not satisfy the request"
);
});
}
#[test_traced("WARN")]
fn test_standard_notarized_delivery_ignored_without_full_scheme() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let round = Round::new(Epoch::zero(), View::new(1));
let (_mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"notarized-delivery-stale-certificate-only",
VerifierProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Notarized { round },
subscribers: NonEmptyVec::new((
handler::Annotation::Notarization { round },
tracing::Span::none(),
)),
},
value: Bytes::from_static(b"unverifiable"),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Ignored,
"a delivery without a full scheme must be ignored, not credited or blamed"
);
});
}
#[test_traced("WARN")]
fn test_standard_finalized_delivery_verifies_with_verify_only_scope() {
const PARTITION_PREFIX: &str = "finalized-delivery-verify-only";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
let (fixture, checkpoint) = runner.start_and_recover(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let height = Height::new(1);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), height, 100);
let proposal = Proposal::new(round, View::zero(), block.digest());
let finalization = StandardHarness::make_finalization(proposal, &schemes, QUORUM);
let verifier = schemes[0].clone();
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, actor_handle) = start_standard_actor(
context.child("validator"),
PARTITION_PREFIX,
VerifierProvider::new(verifier.clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height
}),
tracing::Span::none(),
)),
},
value: (finalization.clone(), block.clone()).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"finalization verified through a verify-only scope should be accepted"
);
assert_eq!(application.acknowledged().await, Height::zero());
assert_eq!(application.acknowledged().await, height);
assert_eq!(
application.blocks().get(&height).unwrap().digest(),
block.digest()
);
actor_handle.abort();
drop(mailbox);
(verifier, block, finalization)
});
deterministic::Runner::from(checkpoint).start(|context| async move {
let (verifier, block, finalization) = fixture;
let (mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("recovered"),
PARTITION_PREFIX,
VerifierProvider::new(verifier),
Application::<B>::default(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let recovered_block = mailbox
.get_block(Height::new(1))
.await
.expect("delivered finalized block must be durable");
assert_eq!(recovered_block.digest(), block.digest());
let recovered_finalization = mailbox
.get_finalization(Height::new(1))
.await
.expect("delivered finalization must be durable");
assert_eq!(recovered_finalization.proposal, finalization.proposal);
});
}
#[test_traced("WARN")]
fn test_standard_finalized_delivery_rejects_different_block_bytes() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let height = Height::new(1);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(genesis.digest(), height, 100);
let other = make_raw_block(genesis.digest(), height, 200);
let digest = block.digest();
let other_digest = other.digest();
assert_eq!(block.height(), other.height());
assert_ne!(digest, other_digest);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), digest),
&schemes,
QUORUM,
);
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"finalized-delivery-different-bytes",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(genesis),
)
.await;
assert_eq!(application.acknowledged().await, Height::zero());
let mut subscriber = mailbox.subscribe_by_digest(digest, DigestFallback::Wait);
let mut other_subscriber =
mailbox.subscribe_by_digest(other_digest, DigestFallback::Wait);
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height,
}),
tracing::Span::none(),
)),
},
value: (finalization.clone(), other).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Invalid
);
assert!(mailbox.get_block(height).await.is_none());
assert!(mailbox.get_block(&digest).await.is_none());
assert!(mailbox.get_block(&other_digest).await.is_none());
assert!(mailbox.get_finalization(height).await.is_none());
assert!(!application.blocks().contains_key(&height));
assert!(application.tip().is_none());
assert!(matches!(subscriber.try_recv(), Err(TryRecvError::Empty)));
assert!(matches!(
other_subscriber.try_recv(),
Err(TryRecvError::Empty)
));
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height,
}),
tracing::Span::none(),
)),
},
value: (finalization, block.clone()).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete)
);
assert_eq!(subscriber.await.unwrap().inner(), &block);
assert_eq!(application.acknowledged().await, height);
assert!(matches!(
other_subscriber.try_recv(),
Err(TryRecvError::Empty)
));
});
}
#[test_traced("WARN")]
fn test_standard_resolver_delivery_waits_for_identity_job() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let prefix = "gated-finalized-identity";
let strategy = GatedStrategy::new(context.child("strategy"));
let (finalizations, blocks) = prunable_finalized_stores(&context, prefix).await;
let blocks = Recording::new(blocks, |block: &Arc<B>| Arc::as_ptr(block) as usize);
let operations = blocks.ops();
let (actor, mut mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("actor"),
finalizations,
blocks,
Config {
provider: ConstantProvider::new(schemes[0].clone()),
epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
start: Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
mailbox_size: NZUsize!(100),
view_retention: ViewDelta::new(10),
max_repair: NZUsize!(10),
max_pending_acks: NZUsize!(1),
block_codec_config: (),
partition_prefix: prefix.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NZUsize!(1024),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
strategy: strategy.clone(),
},
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let actor_handle =
actor.start_unbuffered(application.clone(), (resolver_rx, resolver.clone()));
assert_eq!(application.acknowledged().await, Height::zero());
while mailbox.get_processed().await.map(Processed::height) != Some(Height::zero()) {
reschedule().await;
}
let height = Height::new(1);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(
StandardHarness::genesis_block(NUM_VALIDATORS as u16).digest(),
height,
0x1DE_1000,
);
let digest = block.digest();
let mut counter = DigestCounter::default();
counter.track(&block);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), digest),
&schemes,
QUORUM,
);
let mut subscriber = mailbox.subscribe_by_digest(digest, DigestFallback::Wait);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
let _ = mailbox.get_processed().await;
let Gate { started, release } = strategy.arm();
let (response, mut response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Block(digest),
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height,
}),
tracing::Span::none(),
)),
},
value: block.encode(),
response,
})
.accepted()
);
started.await.expect("identity job must start");
context.sleep(Duration::from_millis(10)).await;
assert_eq!(counter.count(), 0);
assert!(matches!(response_rx.try_recv(), Err(TryRecvError::Empty)));
assert!(matches!(subscriber.try_recv(), Err(TryRecvError::Empty)));
assert!(!application.blocks().contains_key(&height));
assert!(
!operations
.lock()
.iter()
.any(|op| matches!(op, Op::Put(h, _) if *h == height))
);
release.send(()).expect("identity job remains alive");
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete)
);
assert_eq!(subscriber.await.unwrap().inner(), &block);
assert_eq!(application.acknowledged().await, height);
assert_eq!(counter.count(), 2);
assert!(
operations
.lock()
.iter()
.any(|op| matches!(op, Op::Put(h, _) if *h == height))
);
actor_handle.abort();
let _ = actor_handle.await;
});
}
#[test_traced("WARN")]
fn test_standard_finalized_delivery_rejects_epoch_mismatch() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let height = Height::new(1);
let round = Round::new(Epoch::new(1), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), height, 100);
let proposal = Proposal::new(round, View::zero(), block.digest());
let finalization = StandardHarness::make_finalization(proposal, &schemes, QUORUM);
let (_mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"finalized-delivery-epoch-mismatch",
VerifierProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height
}),
tracing::Span::none(),
)),
},
value: (finalization, block).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Invalid,
"finalization whose epoch mismatches the height's epoch must blame the peer"
);
});
}
#[test_traced("WARN")]
fn test_standard_finalized_delivery_verifies_after_scope_retires() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let height = Height::new(1);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), height, 100);
let proposal = Proposal::new(round, View::zero(), block.digest());
let finalization = StandardHarness::make_finalization(proposal, &schemes, QUORUM);
let application = Application::<B>::manual_ack();
let provider = RetiringProvider::default().with(Epoch::zero(), schemes[0].clone(), 1);
let (_mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"finalized-delivery-scope-retires",
provider.clone(),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert!(
!provider.retired(Epoch::zero()),
"no lookup may consume the scope before admission"
);
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height
}),
tracing::Span::none(),
)),
},
value: (finalization, block.clone()).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"finalization admitted under a live scope must not blame the peer"
);
assert!(
provider.retired(Epoch::zero()),
"admission must have consumed the scope"
);
assert_eq!(application.acknowledged().await, Height::zero());
assert_eq!(application.acknowledged().await, height);
assert_eq!(
application.blocks().get(&height).unwrap().digest(),
block.digest()
);
});
}
#[test_traced("WARN")]
fn test_standard_notarized_delivery_verifies_after_scope_retires() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let proposal = Proposal::new(round, View::zero(), block.digest());
let notarization = StandardHarness::make_notarization(proposal, &schemes, QUORUM);
let provider = RetiringProvider::default().with(Epoch::zero(), schemes[0].clone(), 1);
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"notarized-delivery-scope-retires",
provider.clone(),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert!(
!provider.retired(Epoch::zero()),
"no lookup may consume the scope before admission"
);
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Notarized { round },
subscribers: NonEmptyVec::new((
handler::Annotation::Notarization { round },
tracing::Span::none(),
)),
},
value: (notarization, block.clone()).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"notarization admitted under a live scope must not blame the peer"
);
assert!(
provider.retired(Epoch::zero()),
"admission must have consumed the scope"
);
assert_eq!(
mailbox
.get_block(&block.digest())
.await
.map(|cached| cached.digest()),
Some(block.digest()),
"notarized block verified under the admission scope must be cached"
);
});
}
#[test_traced("WARN")]
fn test_standard_finalized_delivery_rejects_foreign_certificate_after_scope_retires() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let Fixture {
schemes: foreign, ..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let height = Height::new(1);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), height, 100);
let proposal = Proposal::new(round, View::zero(), block.digest());
let finalization = StandardHarness::make_finalization(proposal, &foreign, QUORUM);
let provider = RetiringProvider::default().with(Epoch::zero(), schemes[0].clone(), 1);
let (_mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"finalized-delivery-foreign-certificate-scope-retires",
provider.clone(),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert!(
!provider.retired(Epoch::zero()),
"no lookup may consume the scope before admission"
);
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height
}),
tracing::Span::none(),
)),
},
value: (finalization, block).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Invalid,
"certificate from a foreign committee must be rejected"
);
assert!(
provider.retired(Epoch::zero()),
"admission must have consumed the scope"
);
});
}
#[test_traced("WARN")]
fn test_standard_finalized_batch_verifies_each_epoch_under_admission_scope() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes: first, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let Fixture {
schemes: second, ..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let early_height = Height::new(1);
let early_block = make_raw_block(Sha256::hash(&[b""]), early_height, 100);
let early_finalization = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(1)),
View::zero(),
early_block.digest(),
),
&first,
QUORUM,
);
let late_height = Height::new(BLOCKS_PER_EPOCH.get() + 1);
let late_block = make_raw_block(Sha256::hash(&[b"late"]), late_height, 2100);
let late_finalization = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::new(1), View::new(late_height.get())),
View::zero(),
late_block.digest(),
),
&second,
QUORUM,
);
let application = Application::<B>::manual_ack();
let provider = RetiringProvider::default()
.with(Epoch::zero(), first[0].clone(), 1)
.with(Epoch::new(1), second[0].clone(), usize::MAX);
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"finalized-batch-mixed-scopes",
provider.clone(),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert!(
!provider.retired(Epoch::zero()),
"no lookup may consume the scope before admission"
);
let mut responses = Vec::new();
for (height, finalization, block) in [
(early_height, early_finalization, early_block.clone()),
(late_height, late_finalization, late_block.clone()),
] {
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height
}),
tracing::Span::none(),
)),
},
value: (finalization, block).encode(),
response,
})
.accepted()
);
responses.push(response_rx);
}
for response_rx in responses {
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"every delivery admitted under a live scope must be accepted"
);
}
assert!(
provider.retired(Epoch::zero()),
"admission must have consumed the retired scope"
);
assert_eq!(application.acknowledged().await, Height::zero());
assert_eq!(application.acknowledged().await, early_height);
assert_eq!(
mailbox
.get_finalization(late_height)
.await
.map(|finalization| finalization.proposal.payload),
Some(late_block.digest()),
"finalization verified under the live epoch's scope must be stored"
);
});
}
#[test_traced("WARN")]
fn test_standard_processed_round_skips_fetch_and_preserves_subscription() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let proposal = Proposal::new(round, View::zero(), block.digest());
let finalization = StandardHarness::make_finalization(proposal, &schemes, QUORUM);
let application = Application::<B>::manual_ack();
let (mailbox, buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"fetch-notarized-processed-round",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
assert!(
mailbox.verified(round, ready(block.clone())).await,
"verified block should persist before finalization"
);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
let retain_floor = resolver.retain_count() + 2;
assert_eq!(
application.acknowledged().await,
Height::new(1),
"application should receive the finalized block"
);
wait_until(
&context,
Duration::from_secs(5),
"processed-round pruning",
|| resolver.retain_count() >= retain_floor,
)
.await;
let fetches_before = resolver.fetches().len();
mailbox.hint_notarized(round, Sha256::hash(&[b"missing-at-processed-round"]));
let missing = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 101);
let missing_digest = missing.digest();
let mut subscription = mailbox.subscribe_by_commitment(
missing_digest,
CommitmentFallback::FetchByRound { round },
);
let barrier = make_raw_block(block.digest(), Height::new(2), 200);
assert!(
mailbox
.verified(Round::new(Epoch::zero(), View::new(2)), ready(barrier))
.await,
"barrier verification should be processed"
);
assert_eq!(
resolver.fetches().len(),
fetches_before,
"hint_notarized must not enqueue the already-pruned processed round"
);
assert!(matches!(subscription.try_recv(), Err(TryRecvError::Empty)));
buffer
.commitment_subscriptions
.lock()
.pop()
.expect("commitment subscription should be registered")
.send_lossy(ready(missing));
assert_eq!(subscription.await.unwrap().digest(), missing_digest);
});
}
#[test_traced("WARN")]
fn test_standard_finalization_rejects_processed_round_block_fetch() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"finalization-processed-round-fetch",
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
assert!(mailbox.verified(round, ready(block.clone())).await);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
let retain_floor = resolver.retain_count() + 2;
assert_eq!(application.acknowledged().await, Height::new(1));
wait_until(
&context,
Duration::from_secs(5),
"processed-round pruning",
|| resolver.retain_count() >= retain_floor,
)
.await;
let fetches_before = resolver.fetches().len();
let stale_finalization = StandardHarness::make_finalization(
Proposal::new(
round,
View::zero(),
Sha256::hash(&[b"missing-finalized-at-processed-round"]),
),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, stale_finalization).await;
let barrier = make_raw_block(block.digest(), Height::new(2), 200);
assert!(
mailbox
.verified(Round::new(Epoch::zero(), View::new(2)), ready(barrier))
.await
);
assert_eq!(
resolver.fetches().len(),
fetches_before,
"stale finalization must not enqueue a round-bound block fetch"
);
});
}
#[test_traced("WARN")]
#[should_panic(expected = "stored genesis block does not match configured anchor")]
fn test_standard_restart_rejects_mismatched_genesis_anchor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let partition_prefix = "restart-keeps-existing-genesis";
let original_genesis =
make_raw_block(Sha256::hash(&[b"original-genesis"]), Height::zero(), 0);
let replacement_genesis =
make_raw_block(Sha256::hash(&[b"replacement-genesis"]), Height::zero(), 1);
assert_ne!(original_genesis.digest(), replacement_genesis.digest());
let (mailbox, buffer, resolver, actor_handle) = start_standard_actor(
context.child("validator"),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(ready(original_genesis.clone())),
)
.await;
assert_eq!(
mailbox.get_block(Height::zero()).await.unwrap().digest(),
original_genesis.digest()
);
actor_handle.abort();
let _ = actor_handle.await;
drop(mailbox);
drop(buffer);
drop(resolver);
let (_mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator_restart"),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(ready(replacement_genesis.clone())),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_processed_round_restored_after_restart() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let partition_prefix = format!("processed-round-restart-{me}");
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
let application = Application::<B>::manual_ack();
let (mailbox, _buffer, resolver, actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&partition_prefix,
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut mailbox = mailbox;
assert_eq!(application.acknowledged().await, Height::zero());
assert!(mailbox.verified(round, ready(block.clone())).await);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
let retain_floor = resolver.retain_count() + 2;
assert_eq!(application.acknowledged().await, Height::new(1));
wait_until(
&context,
Duration::from_secs(5),
"processed-round pruning",
|| resolver.retain_count() >= retain_floor,
)
.await;
assert_eq!(
mailbox.get_info(Identifier::Latest).await,
Some((Height::new(1), block.digest()))
);
actor_handle.abort();
let _ = actor_handle.await;
drop(mailbox);
let (mailbox, buffer, resolver, _actor_handle) = start_standard_actor(
context
.child("validator_restart")
.with_attribute("index", 0),
&partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let fetches_before = resolver.fetches().len();
mailbox.hint_notarized(round, Sha256::hash(&[b"missing-after-restart"]));
let missing = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 101);
let missing_digest = missing.digest();
let mut subscription = mailbox.subscribe_by_commitment(
missing_digest,
CommitmentFallback::FetchByRound { round },
);
let barrier = make_raw_block(block.digest(), Height::new(2), 200);
assert!(
mailbox
.verified(Round::new(Epoch::zero(), View::new(2)), ready(barrier))
.await
);
assert_eq!(
resolver.fetches().len(),
fetches_before,
"restart must restore the processed round floor"
);
assert!(matches!(subscription.try_recv(), Err(TryRecvError::Empty)));
buffer
.commitment_subscriptions
.lock()
.pop()
.expect("commitment subscription should be registered")
.send_lossy(ready(missing));
assert_eq!(subscription.await.unwrap().digest(), missing_digest);
});
}
#[test_traced("WARN")]
fn test_standard_round_floor_does_not_restore_missing_next_block() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let partition_prefix = "round-floor-missing-next-block";
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let proposal = Proposal::new(round, View::zero(), block.digest());
let notarization =
StandardHarness::make_notarization(proposal.clone(), &schemes, QUORUM);
let finalization = StandardHarness::make_finalization(proposal, &schemes, QUORUM);
seed_processed_height(
context.child("metadata"),
partition_prefix,
Height::zero(),
None,
)
.await;
seed_inconsistent_restart_state(
context.child("storage"),
partition_prefix,
&[],
&[(Height::new(1), finalization)],
)
.await;
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let mut subscription = mailbox.subscribe_by_commitment(
block.digest(),
CommitmentFallback::FetchByRound { round },
);
context.sleep(Duration::from_millis(100)).await;
assert!(
matches!(subscription.try_recv(), Err(TryRecvError::Empty)),
"missing next block must not advance the restored round floor"
);
wait_until(
&context,
Duration::from_secs(5),
"fetch-by-round after missing-next restart",
|| {
resolver.fetches().iter().any(|fetch| {
matches!(
(&fetch.key, &fetch.subscriber),
(
handler::Key::Notarized { round: request_round },
handler::Annotation::Notarization { round: subscriber_round },
) if *request_round == round && *subscriber_round == round
)
})
},
)
.await;
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Notarized { round },
subscribers: NonEmptyVec::new((
handler::Annotation::Notarization { round },
tracing::Span::none(),
)),
},
value: (notarization, block.clone()).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("delivery response missing"),
Outcome::Resolved(Resolution::Complete),
"notarized delivery should validate"
);
select! {
result = subscription => {
let delivered = result.expect("block subscription should resolve");
assert_eq!(delivered.digest(), block.digest());
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("round-floor subscription did not resolve");
},
}
});
}
#[test_traced("WARN")]
fn test_standard_round_floor_restores_unacknowledged_anchor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let partition_prefix = "round-floor-before-anchor-ack";
let floor_round = Round::new(Epoch::zero(), View::new(5));
let floor_block = make_raw_block(Sha256::hash(&[b"floor-parent"]), Height::new(5), 500);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(4), floor_block.digest()),
&schemes,
QUORUM,
);
let (application, started) = HoldingBlockReporter::new_after(Height::zero());
let (mailbox, buffer, resolver, actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert!(
mailbox
.verified(floor_round, ready(floor_block.clone()))
.await
);
mailbox.set_floor(floor_finalization);
select! {
height = started => {
assert_eq!(
height.expect("floor anchor delivery signal missing"),
Height::new(5),
"floor anchor should be delivered before the simulated crash"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("floor anchor was not delivered before restart");
},
}
actor_handle.abort();
let _ = actor_handle.await;
drop(mailbox);
drop(buffer);
drop(resolver);
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context
.child("validator_restart")
.with_attribute("index", 0),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let fetches_before = resolver.fetches().len();
let mut subscription = mailbox.subscribe_by_commitment(
Sha256::hash(&[b"missing-before-anchor-ack"]),
CommitmentFallback::FetchByRound { round: floor_round },
);
let barrier = make_raw_block(floor_block.digest(), Height::new(6), 600);
assert!(
mailbox
.verified(Round::new(Epoch::zero(), View::new(6)), ready(barrier))
.await,
"barrier verification should be processed"
);
assert_eq!(
resolver.fetches().len(),
fetches_before,
"durable floor round must suppress round-bound fetches after restart"
);
assert!(
matches!(subscription.try_recv(), Err(TryRecvError::Empty)),
"durable floor round must preserve local subscriptions after restart"
);
});
}
#[test_traced("WARN")]
fn test_standard_set_floor_prunes_round_bound_fetches() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
"set-floor-round-prune",
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let missing = Sha256::hash(&[b"missing-before-set-floor"]);
let _subscription = mailbox
.subscribe_by_commitment(missing, CommitmentFallback::FetchByRound { round });
wait_until(
&context,
Duration::from_secs(5),
"round-bound fetch",
|| {
resolver.active_fetches().iter().any(|fetch| {
matches!(fetch.key, handler::Key::Notarized { round: r } if r == round)
})
},
)
.await;
mailbox.set_floor(finalization.clone());
assert!(mailbox.verified(round, ready(block.clone())).await);
wait_until(
&context,
Duration::from_secs(5),
"round-bound prune",
|| {
resolver.active_fetches().iter().all(|fetch| {
!matches!(fetch.key, handler::Key::Notarized { round: r } if r == round)
})
},
)
.await;
assert!(
resolver.active_fetches().iter().all(|fetch| {
!matches!(fetch.key, handler::Key::Notarized { round: r } if r == round)
}),
"processed finalization after set_floor must prune existing round-bound fetches"
);
let fetches_before = resolver.fetches().len();
mailbox.hint_notarized(round, Sha256::hash(&[b"missing-after-set-floor"]));
let barrier = make_raw_block(block.digest(), Height::new(2), 200);
assert!(
mailbox
.verified(Round::new(Epoch::zero(), View::new(2)), ready(barrier))
.await
);
assert_eq!(
resolver.fetches().len(),
fetches_before,
"set_floor must apply the round floor to future fetches"
);
});
}
#[test_traced("WARN")]
fn test_standard_round_floor_advance_releases_superseded_pending_floor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let partition_prefix = "round-floor-releases-pending-floor";
let anchor_round = Round::new(Epoch::zero(), View::new(2));
let anchor = make_raw_block(Sha256::hash(&[b"anchor-parent"]), Height::new(2), 200);
let anchor_finalization = StandardHarness::make_finalization(
Proposal::new(
anchor_round,
View::new(1),
anchor.digest(),
),
&schemes,
QUORUM,
);
let processed_round = Round::new(Epoch::zero(), View::new(3));
let processed = make_raw_block(anchor.digest(), Height::new(3), 300);
let processed_finalization = StandardHarness::make_finalization(
Proposal::new(
processed_round,
View::new(2),
processed.digest(),
),
&schemes,
QUORUM,
);
let next = make_raw_block(processed.digest(), Height::new(4), 400);
seed_processed_height(
context.child("metadata"),
partition_prefix,
Height::new(3),
None,
)
.await;
seed_inconsistent_restart_state(
context.child("storage"),
partition_prefix,
&[processed.clone(), next.clone()],
&[],
)
.await;
let application = Application::<B>::manual_ack();
let buffer = RecordingBuffer::default();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator"),
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
application.clone(),
Some(buffer.clone()),
Start::Floor(anchor_finalization),
)
.await;
let mut mailbox = mailbox;
let is_anchor_fetch = |fetch: &FetchRecord| {
matches!(
(&fetch.key, &fetch.subscriber),
(
handler::Key::Block(commitment),
handler::Annotation::Finalized(handler::Finalized::ByRound { round }),
) if *commitment == anchor.digest()
&& *round == anchor_round
)
};
wait_until(
&context,
Duration::from_secs(5),
"floor anchor fetch",
|| resolver.active_fetches().iter().any(is_anchor_fetch),
)
.await;
context.sleep(Duration::from_millis(100)).await;
assert!(
application.pending_ack_heights().is_empty(),
"pending floor must hold dispatch until the anchor resolves"
);
assert!(!buffer.commitment_subscription_closed(0));
let retains_before = resolver.retain_count();
StandardHarness::report_finalization(&mut mailbox, processed_finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"round floor advance",
|| resolver.retain_count() > retains_before,
)
.await;
assert_eq!(buffer.commitment_subscription_count(), 1);
while !buffer.commitment_subscription_closed(0) {
reschedule().await;
}
if let Some(fetch) = resolver
.active_fetches()
.into_iter()
.find(is_anchor_fetch)
{
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: fetch.key,
subscribers: NonEmptyVec::new((
fetch.subscriber,
tracing::Span::none()
)),
},
value: anchor.encode(),
response,
})
.accepted()
);
assert_eq!(response_rx.await.expect("delivery response missing"), Outcome::Resolved(Resolution::Complete));
}
select! {
height = application.acknowledged() => {
assert_eq!(height, Height::new(4));
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!(
"superseded pending floor stranded dispatch: anchor fetch pruned without releasing the floor"
);
},
}
});
}
#[test_traced("WARN")]
fn test_standard_unverifiable_delivery_is_ignored() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|context| async move {
let me = default_leader();
let oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
[me.clone()],
1024 * 1024,
)
.await;
let control = oracle.control(me.clone());
let network_channel = control
.register(0, Quota::per_second(NonZeroU32::MAX))
.await
.unwrap();
let page_cache = CacheRef::new(NZU16!(1024), NZUsize!(10));
let partition_prefix = "stale-finalized-test".to_string();
let config = Config {
provider: EmptyProvider,
epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
start: Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
mailbox_size: NZUsize!(100),
view_retention: ViewDelta::new(10),
max_repair: NZUsize!(10),
max_pending_acks: NZUsize!(1),
block_codec_config: (),
partition_prefix: partition_prefix.clone(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NZUsize!(1024),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
page_cache: page_cache.clone(),
strategy: Sequential,
};
let finalizations_by_height = prunable::Archive::init(
context.child("finalizations_by_height"),
prunable::Config {
translator: EightCap,
metadata_partition: format!("{partition_prefix}-fbh-metadata"),
key_partition: format!("{partition_prefix}-fbh-key"),
key_page_cache: page_cache.clone(),
value_partition: format!("{partition_prefix}-fbh-value"),
compression: None,
codec_config: S::certificate_codec_config_unbounded(),
items_per_section: NZU64!(10),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
replay_buffer: NZUsize!(1024),
},
)
.await
.expect("failed to initialize finalizations archive");
let finalized_blocks = prunable::Archive::init(
context.child("finalized_blocks"),
prunable::Config {
translator: EightCap,
metadata_partition: format!("{partition_prefix}-fb-metadata"),
key_partition: format!("{partition_prefix}-fb-key"),
key_page_cache: page_cache,
value_partition: format!("{partition_prefix}-fb-value"),
compression: None,
codec_config: (),
items_per_section: NZU64!(10),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
replay_buffer: NZUsize!(1024),
},
)
.await
.expect("failed to initialize finalized blocks archive");
let broadcast_config = buffered::Config {
public_key: me.clone(),
mailbox_size: NZUsize!(100),
deque_size: 10,
priority: false,
codec_config: (),
peer_provider: oracle.manager(),
strategy: Sequential,
};
let (broadcast_engine, buffer) =
buffered::Engine::new(context.child("broadcast"), broadcast_config);
broadcast_engine.start(network_channel);
let (resolver_tx, resolver_rx) = mailbox::new(context.child("mailbox"), NZUsize!(100));
let (actor, _mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
config,
)
.await;
actor.start(
Application::<B>::default(),
buffer,
(
handler::Receiver::new(resolver_rx),
RecordingResolver::default(),
),
);
let (response, response_rx) = oneshot::channel();
assert!(
resolver_tx
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized {
height: Height::new(5),
},
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height: Height::new(5),
}),
tracing::Span::none(),
)),
},
value: Bytes::from_static(b"unverifiable"),
response,
})
.accepted()
);
assert_eq!(response_rx.await.unwrap(), Outcome::Ignored);
let (response, response_rx) = oneshot::channel();
assert!(
resolver_tx
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Notarized {
round: Round::new(Epoch::zero(), View::new(1)),
},
subscribers: NonEmptyVec::new((
handler::Annotation::Notarization {
round: Round::new(Epoch::zero(), View::new(1)),
},
tracing::Span::none(),
)),
},
value: Bytes::from_static(b"unverifiable"),
response,
})
.accepted()
);
assert_eq!(response_rx.await.unwrap(), Outcome::Ignored);
});
}
#[test_traced("WARN")]
fn test_standard_dispatches_finalized_blocks_after_sync() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let partition_prefix = format!("validator-{me}");
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
let (application, started) = HoldingBlockReporter::new_after(Height::zero());
let (mut mailbox, _buffer, _resolver, actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&partition_prefix,
ConstantProvider::new(schemes[0].clone()),
application,
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert!(mailbox.verified(round, ready(block.clone())).await);
StandardHarness::report_finalization(&mut mailbox, finalization.clone()).await;
select! {
height = started => {
assert_eq!(
height.expect("delivery signal missing"),
Height::new(1),
"application should observe the first finalized block"
);
},
_ = context.sleep(Duration::from_secs(5)) => {
panic!("application should observe block delivery promptly");
},
}
actor_handle.abort();
let _ = actor_handle.await;
drop(mailbox);
let (mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context
.child("validator_restart")
.with_attribute("index", 0),
&partition_prefix,
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let recovered = mailbox
.get_block(Height::new(1))
.await
.expect("finalized block must be durable before delivery");
assert_eq!(
recovered.digest(),
block.digest(),
"restart should recover the delivered finalized block by height"
);
assert_eq!(
mailbox
.get_finalization(Height::new(1))
.await
.expect("finalization must be durable before delivery")
.round(),
round,
"restart should recover the delivered finalization by height"
);
});
}
struct PacedStore<T> {
inner: T,
context: deterministic::Context,
pace: Duration,
}
impl<T: crate::marshal::store::Blocks> crate::marshal::store::Blocks for PacedStore<T> {
type Block = T::Block;
type Error = T::Error;
async fn put(
mut self,
digest: <Self::Block as Digestible>::Digest,
block: &Self::Block,
) -> Result<Self, Self::Error> {
self.inner = self.inner.put(digest, block).await?;
Ok(self)
}
async fn sync(mut self) -> Result<Self, Self::Error> {
self.context.sleep(self.pace).await;
self.inner = self.inner.sync().await?;
Ok(self)
}
async fn start_sync(
mut self,
) -> Result<(Self, commonware_runtime::Handle<()>), Self::Error> {
let handle;
(self.inner, handle) = self.inner.start_sync().await?;
let sleep = self.context.sleep(self.pace);
Ok((
self,
commonware_runtime::Handle::from_future(async move {
sleep.await;
handle.await
}),
))
}
async fn get(
&self,
id: commonware_storage::archive::Identifier<'_, <Self::Block as Digestible>::Digest>,
) -> Result<Option<Self::Block>, Self::Error> {
self.inner.get(id).await
}
async fn prune(mut self, min: Height) -> Result<Self, Self::Error> {
self.inner = self.inner.prune(min).await?;
Ok(self)
}
fn missing_items(&self, start: Height, max: usize) -> Vec<Height> {
self.inner.missing_items(start, max)
}
fn next_gap(&self, value: Height) -> (Option<Height>, Option<Height>) {
self.inner.next_gap(value)
}
fn last_index(&self) -> Option<Height> {
self.inner.last_index()
}
}
impl<T: crate::marshal::store::Certificates> crate::marshal::store::Certificates for PacedStore<T> {
type BlockDigest = T::BlockDigest;
type Commitment = T::Commitment;
type Scheme = T::Scheme;
type Error = T::Error;
async fn put(
mut self,
height: Height,
digest: Self::BlockDigest,
finalization: &Finalization<Self::Scheme, Self::Commitment>,
) -> Result<Self, Self::Error> {
self.inner = self.inner.put(height, digest, finalization).await?;
Ok(self)
}
async fn sync(mut self) -> Result<Self, Self::Error> {
self.context.sleep(self.pace).await;
self.inner = self.inner.sync().await?;
Ok(self)
}
async fn start_sync(
mut self,
) -> Result<(Self, commonware_runtime::Handle<()>), Self::Error> {
let handle;
(self.inner, handle) = self.inner.start_sync().await?;
let sleep = self.context.sleep(self.pace);
Ok((
self,
commonware_runtime::Handle::from_future(async move {
sleep.await;
handle.await
}),
))
}
async fn get(
&self,
id: commonware_storage::archive::Identifier<'_, Self::BlockDigest>,
) -> Result<Option<Finalization<Self::Scheme, Self::Commitment>>, Self::Error> {
self.inner.get(id).await
}
async fn has(&self, height: Height) -> Result<bool, Self::Error> {
self.inner.has(height).await
}
async fn prune(mut self, min: Height) -> Result<Self, Self::Error> {
self.inner = self.inner.prune(min).await?;
Ok(self)
}
fn last_index(&self) -> Option<Height> {
self.inner.last_index()
}
fn ranges_from(&self, from: Height) -> impl Iterator<Item = (Height, Height)> {
self.inner.ranges_from(from)
}
}
type Finalizations = prunable::Archive<EightCap, deterministic::Context, D, Finalization<S, D>>;
type FinalizedBlocks<T = Arc<B>> = prunable::Archive<EightCap, deterministic::Context, D, T>;
async fn prunable_finalized_stores<T: crate::Block<Digest = D> + Read<Cfg = ()>>(
context: &deterministic::Context,
partition_prefix: &str,
) -> (Finalizations, FinalizedBlocks<T>) {
let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE);
let finalizations_by_height = prunable::Archive::init(
context.child("finalizations_by_height"),
prunable::Config {
translator: EightCap,
metadata_partition: format!("{partition_prefix}-fbh-metadata"),
key_partition: format!("{partition_prefix}-fbh-key"),
key_page_cache: page_cache.clone(),
value_partition: format!("{partition_prefix}-fbh-value"),
compression: None,
codec_config: S::certificate_codec_config_unbounded(),
items_per_section: NZU64!(10),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
replay_buffer: NZUsize!(1024),
},
)
.await
.expect("failed to initialize finalizations archive");
let finalized_blocks = prunable::Archive::init(
context.child("finalized_blocks"),
prunable::Config {
translator: EightCap,
metadata_partition: format!("{partition_prefix}-fb-metadata"),
key_partition: format!("{partition_prefix}-fb-key"),
key_page_cache: page_cache,
value_partition: format!("{partition_prefix}-fb-value"),
compression: None,
codec_config: (),
items_per_section: NZU64!(10),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
replay_buffer: NZUsize!(1024),
},
)
.await
.expect("failed to initialize finalized blocks archive");
(finalizations_by_height, finalized_blocks)
}
async fn paced_finalized_stores(
context: &deterministic::Context,
partition_prefix: &str,
pace: Duration,
) -> (PacedStore<Finalizations>, PacedStore<FinalizedBlocks>) {
let (finalizations_by_height, finalized_blocks) =
prunable_finalized_stores(context, partition_prefix).await;
(
PacedStore {
inner: finalizations_by_height,
context: context.child("finalizations_pacer"),
pace,
},
PacedStore {
inner: finalized_blocks,
context: context.child("blocks_pacer"),
pace,
},
)
}
fn test_config<P: Provider<Scope = Epoch, Scheme = S>>(
partition_prefix: &str,
provider: P,
max_pending_acks: NonZeroUsize,
) -> Config<P, FixedEpocher, Sequential, B, Digested<B>, D> {
Config {
provider,
epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
start: Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
mailbox_size: NZUsize!(100),
view_retention: ViewDelta::new(10),
max_repair: NZUsize!(10),
max_pending_acks,
block_codec_config: (),
partition_prefix: partition_prefix.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NZUsize!(1024),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
strategy: Sequential,
}
}
#[derive(Debug, Default)]
struct CloneCounter(Arc<AtomicUsize>);
impl Clone for CloneCounter {
fn clone(&self) -> Self {
self.0.fetch_add(1, Ordering::Relaxed);
Self(Arc::clone(&self.0))
}
}
impl FixedSize for CloneCounter {
const SIZE: usize = 0;
}
impl Write for CloneCounter {
fn write(&self, _: &mut impl BufMut) {}
}
impl Read for CloneCounter {
type Cfg = ();
fn read_cfg(_: &mut impl Buf, _: &()) -> Result<Self, commonware_codec::Error> {
Ok(Self::default())
}
}
#[test_traced("WARN")]
fn test_standard_cache_writes_do_not_clone_block_payloads() {
type CountedBlock = crate::marshal::mocks::block::Block<D, CloneCounter>;
const PREFIX: &str = "shared-cache";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let genesis = CountedBlock::new::<Sha256>(
CloneCounter::default(),
Sha256::hash(&[b""]),
Height::zero(),
0,
);
let parent = genesis.digest();
let (finalizations, blocks) = prunable_finalized_stores(&context, PREFIX).await;
let (actor, mailbox, _) = Actor::<_, Standard<CountedBlock>, _, _, _, _, _>::init(
context.child("actor"),
finalizations,
blocks,
Config {
provider: ConstantProvider::new(schemes[0].clone()),
epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
start: Start::Genesis(ready(genesis)),
mailbox_size: NZUsize!(100),
view_retention: ViewDelta::new(10),
max_repair: NZUsize!(10),
max_pending_acks: NZUsize!(1),
block_codec_config: (),
partition_prefix: PREFIX.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NZUsize!(1024),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
strategy: Sequential,
},
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let _actor_handle = actor.start_unbuffered(
Application::<CountedBlock>::default(),
(resolver_rx, resolver),
);
for (view, certified) in [(1, false), (2, true)] {
let round = Round::new(Epoch::zero(), View::new(view));
let block = Arc::new(CountedBlock::new::<Sha256>(
CloneCounter::default(),
parent,
Height::new(1),
view,
));
for _ in 0..2 {
let durable = if certified {
mailbox
.certified(
round,
Digested::precomputed(Arc::clone(&block), block.digest()),
)
.await
} else {
mailbox
.verified(
round,
Digested::precomputed(Arc::clone(&block), block.digest()),
)
.await
};
assert!(durable);
assert_eq!(
block.context.0.load(Ordering::Relaxed),
0,
"cache persistence must not clone the block payload (certified={certified})"
);
assert_eq!(
mailbox.get_block(&block.digest()).await.unwrap().digest(),
block.digest()
);
}
}
});
}
#[derive(Clone, Debug, Default)]
struct DecodeCounter;
static DECODES: AtomicUsize = AtomicUsize::new(0);
impl FixedSize for DecodeCounter {
const SIZE: usize = 0;
}
impl Write for DecodeCounter {
fn write(&self, _: &mut impl BufMut) {}
}
impl Read for DecodeCounter {
type Cfg = ();
fn read_cfg(_: &mut impl Buf, _: &()) -> Result<Self, commonware_codec::Error> {
DECODES.fetch_add(1, Ordering::Relaxed);
Ok(Self)
}
}
#[test_traced("WARN")]
fn test_standard_notarized_delivery_decodes_after_verification() {
type CountedBlock = crate::marshal::mocks::block::Block<D, DecodeCounter>;
const PREFIX: &str = "notarized-decode";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let Fixture {
schemes: forged, ..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, b"forged", NUM_VALIDATORS);
let genesis =
CountedBlock::new::<Sha256>(DecodeCounter, Sha256::hash(&[b""]), Height::zero(), 0);
let parent = genesis.digest();
let (finalizations, blocks) = prunable_finalized_stores(&context, PREFIX).await;
let (actor, mailbox, _) = Actor::<_, Standard<CountedBlock>, _, _, _, _, _>::init(
context.child("actor"),
finalizations,
blocks,
Config {
provider: ConstantProvider::new(schemes[0].clone()),
epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
start: Start::Genesis(ready(genesis)),
mailbox_size: NZUsize!(100),
view_retention: ViewDelta::new(10),
max_repair: NZUsize!(10),
max_pending_acks: NZUsize!(1),
block_codec_config: (),
partition_prefix: PREFIX.to_string(),
prunable_items_per_section: NZU64!(10),
replay_buffer: NZUsize!(1024),
key_write_buffer: NZUsize!(1024),
value_write_buffer: NZUsize!(1024),
page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
strategy: Sequential,
},
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let _actor_handle = actor.start_unbuffered(
Application::<CountedBlock>::default(),
(resolver_rx, resolver.clone()),
);
let round = Round::new(Epoch::zero(), View::new(1));
let block = CountedBlock::new::<Sha256>(DecodeCounter, parent, Height::new(1), 1);
let proposal = Proposal::new(round, View::zero(), block.digest());
let deliver = |notarization: Notarization<S, D>| {
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Notarized { round },
subscribers: NonEmptyVec::new((
handler::Annotation::Notarization { round },
tracing::Span::none(),
)),
},
value: (notarization, block.clone()).encode(),
response,
})
.accepted()
);
response_rx
};
mailbox.get_processed().await;
let before = DECODES.load(Ordering::Relaxed);
let forged = StandardHarness::make_notarization(proposal.clone(), &forged, QUORUM);
assert_eq!(deliver(forged).await.unwrap(), Outcome::Invalid);
assert_eq!(DECODES.load(Ordering::Relaxed), before);
let notarization = StandardHarness::make_notarization(proposal, &schemes, QUORUM);
assert_eq!(
deliver(notarization).await.unwrap(),
Outcome::Resolved(Resolution::Complete)
);
assert!(DECODES.load(Ordering::Relaxed) > before);
});
}
#[test_traced("WARN")]
fn test_standard_finalization_sync_does_not_block_mailbox() {
const PACE: Duration = Duration::from_millis(100);
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let config = test_config(
"paced-finalized-sync",
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
);
let (finalizations_by_height, finalized_blocks) =
paced_finalized_stores(&context, "paced-finalized-sync", PACE).await;
let (actor, mut mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
config,
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::default();
let _actor_handle =
actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
wait_until(
&context,
Duration::from_secs(1),
"genesis dispatched",
|| application.blocks().contains_key(&Height::zero()),
)
.await;
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(genesis.digest(), Height::new(1), 100);
assert!(mailbox.verified(round, ready(block.clone())).await);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
let finalized_at = context.current();
StandardHarness::report_finalization(&mut mailbox, finalization).await;
context.sleep(Duration::from_millis(1)).await;
let requested_at = context.current();
let verified = mailbox.get_verified(round).await;
let elapsed = context
.current()
.duration_since(requested_at)
.expect("time went backwards");
assert_eq!(
verified.expect("verified block missing").digest(),
block.digest()
);
tracing::info!(?elapsed, "get_verified answered");
assert!(
elapsed < Duration::from_millis(5),
"get_verified queued behind the finalized-archive sync: took {elapsed:?}"
);
assert!(
!application.blocks().contains_key(&Height::new(1)),
"block dispatched before the finalized archives were durable"
);
context.sleep(Duration::from_millis(90)).await;
assert!(
!application.blocks().contains_key(&Height::new(1)),
"block dispatched before the finalized archives were durable"
);
wait_until(
&context,
Duration::from_millis(50),
"finalized block dispatched",
|| application.blocks().contains_key(&Height::new(1)),
)
.await;
let dispatched = context
.current()
.duration_since(finalized_at)
.expect("time went backwards");
tracing::info!(?dispatched, "finalized block dispatched");
assert!(
dispatched >= PACE,
"block dispatched before the paced sync completed: {dispatched:?}"
);
});
}
#[test_traced("WARN")]
fn test_standard_stale_floor_anchor_holds_dispatch_until_durable() {
const PACE: Duration = Duration::from_millis(100);
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let config = test_config(
"stale-floor-anchor",
ConstantProvider::new(schemes[0].clone()),
NZUsize!(4),
);
let (finalizations_by_height, finalized_blocks) =
paced_finalized_stores(&context, "stale-floor-anchor", PACE).await;
let (actor, mut mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
config,
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::default();
let _actor_handle =
actor.start_unbuffered(application.clone(), (resolver_rx, resolver.clone()));
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(genesis.digest(), Height::new(1), 100);
assert!(mailbox.verified(round, ready(block.clone())).await);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
wait_until(
&context,
Duration::from_secs(5),
"block 1 processed",
|| parse_processed_height(&context.encode()) == Some(1),
)
.await;
let fork = make_raw_block(genesis.digest(), Height::new(1), 999);
let fork_finalization = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(5)),
View::new(4),
fork.digest(),
),
&schemes,
QUORUM,
);
mailbox.set_floor(fork_finalization);
wait_until(&context, Duration::from_secs(5), "anchor fetch", || {
resolver.fetches().iter().any(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == fork.digest()
)
})
})
.await;
let anchor_fetch = resolver
.fetches()
.into_iter()
.find(|fetch| {
matches!(
fetch.key,
handler::Key::Block(commitment)
if commitment == fork.digest()
)
})
.expect("anchor fetch missing");
let next = make_raw_block(block.digest(), Height::new(2), 200);
let (next_response, next_response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Block(next.digest()),
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height: Height::new(2),
}),
tracing::Span::none(),
)),
},
value: next.encode(),
response: next_response,
})
.accepted()
);
let above = make_raw_block(next.digest(), Height::new(3), 300);
let (above_response, above_response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Block(above.digest()),
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height: Height::new(3),
}),
tracing::Span::none(),
)),
},
value: above.encode(),
response: above_response,
})
.accepted()
);
let (anchor_response, anchor_response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: anchor_fetch.key,
subscribers: NonEmptyVec::new((
anchor_fetch.subscriber,
tracing::Span::none()
)),
},
value: fork.encode(),
response: anchor_response,
})
.accepted()
);
let delivered_at = context.current();
assert_eq!(
next_response_rx.await.expect("repair response missing"),
Outcome::Resolved(Resolution::Complete),
"repair block delivery should validate"
);
assert_eq!(
above_response_rx.await.expect("repair response missing"),
Outcome::Resolved(Resolution::Complete),
"repair block delivery should validate"
);
assert_eq!(
anchor_response_rx.await.expect("anchor response missing"),
Outcome::Resolved(Resolution::Complete),
"anchor delivery should validate"
);
context.sleep(Duration::from_millis(1)).await;
assert!(
!application.blocks().contains_key(&Height::new(2)),
"block dispatched before the finalized archives were durable"
);
context.sleep(Duration::from_millis(90)).await;
assert!(
!application.blocks().contains_key(&Height::new(2)),
"block dispatched before the finalized archives were durable"
);
wait_until(
&context,
Duration::from_millis(50),
"repair blocks dispatched",
|| application.blocks().contains_key(&Height::new(3)),
)
.await;
assert!(application.blocks().contains_key(&Height::new(2)));
let dispatched = context
.current()
.duration_since(delivered_at)
.expect("time went backwards");
tracing::info!(?dispatched, "repair blocks dispatched");
assert!(
dispatched >= PACE,
"block dispatched before the paced sync completed: {dispatched:?}"
);
});
}
#[test_traced("WARN")]
fn test_standard_overlapping_finalized_syncs_release_per_batch() {
const PACE: Duration = Duration::from_millis(100);
const STAGGER: Duration = Duration::from_millis(50);
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let config = test_config(
"overlapping-finalized-syncs",
ConstantProvider::new(schemes[0].clone()),
NZUsize!(4),
);
let (finalizations_by_height, finalized_blocks) =
paced_finalized_stores(&context, "overlapping-finalized-syncs", PACE).await;
let (actor, mut mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
config,
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::default();
let _actor_handle =
actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
wait_until(
&context,
Duration::from_secs(5),
"genesis processed",
|| application.blocks().contains_key(&Height::zero()),
)
.await;
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let first_round = Round::new(Epoch::zero(), View::new(1));
let first = make_raw_block(genesis.digest(), Height::new(1), 100);
assert!(mailbox.verified(first_round, ready(first.clone())).await);
let second_round = Round::new(Epoch::zero(), View::new(2));
let second = make_raw_block(first.digest(), Height::new(2), 200);
assert!(mailbox.verified(second_round, ready(second.clone())).await);
let started_at = context.current();
let finalization = StandardHarness::make_finalization(
Proposal::new(first_round, View::zero(), first.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
context.sleep(STAGGER).await;
let finalization = StandardHarness::make_finalization(
Proposal::new(second_round, View::new(1), second.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
wait_until(
&context,
Duration::from_millis(150),
"first block dispatched",
|| application.blocks().contains_key(&Height::new(1)),
)
.await;
let first_dispatched = context
.current()
.duration_since(started_at)
.expect("time went backwards");
tracing::info!(?first_dispatched, "first block dispatched");
assert!(
first_dispatched >= PACE,
"block dispatched before its sync completed: {first_dispatched:?}"
);
assert!(
first_dispatched < STAGGER + PACE,
"first block waited for the second sync: {first_dispatched:?}"
);
assert!(
!application.blocks().contains_key(&Height::new(2)),
"block dispatched before its sync completed"
);
context.sleep(Duration::from_millis(20)).await;
assert!(
!application.blocks().contains_key(&Height::new(2)),
"block dispatched before its sync completed"
);
wait_until(
&context,
Duration::from_millis(50),
"second block dispatched",
|| application.blocks().contains_key(&Height::new(2)),
)
.await;
let second_dispatched = context
.current()
.duration_since(started_at)
.expect("time went backwards");
tracing::info!(?second_dispatched, "second block dispatched");
assert!(
second_dispatched >= STAGGER + PACE,
"block dispatched before its sync completed: {second_dispatched:?}"
);
});
}
#[test_traced("WARN")]
fn test_standard_dispatch_delivers_staged_block_without_archive_read() {
const PARTITION_PREFIX: &str = "staged-dispatch";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let (finalizations_by_height, finalized_blocks) =
prunable_finalized_stores(&context, PARTITION_PREFIX).await;
let finalized_blocks =
Recording::new(finalized_blocks, |block: &Arc<B>| Arc::as_ptr(block).addr());
let ops = finalized_blocks.ops();
let (actor, mut mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
test_config(
PARTITION_PREFIX,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::default();
let buffer = RecordingBuffer::default();
let _actor_handle =
actor.start(application.clone(), buffer.clone(), (resolver_rx, resolver));
while !application.blocks().contains_key(&Height::zero()) {
reschedule().await;
}
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(genesis.digest(), Height::new(1), 100);
buffer.insert(block.clone());
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
while !application.blocks().contains_key(&Height::new(1)) {
reschedule().await;
}
let delivered = application
.blocks()
.get(&Height::new(1))
.cloned()
.expect("finalized block dispatched");
assert_eq!(delivered.digest(), block.digest());
assert!(
buffer
.last_handed()
.and_then(|handed| handed.upgrade())
.is_some_and(|handed| Arc::ptr_eq(&handed, &delivered)),
"dispatch must deliver the block object the buffer handed to marshal"
);
let ops = ops.lock();
let written = ops
.iter()
.position(|op| matches!(op, Op::Put(height, _) if *height == Height::new(1)))
.expect("finalized block written");
assert_eq!(
ops[written],
Op::Put(Height::new(1), Arc::as_ptr(&delivered).addr()),
"storage must share the block payload dispatched to the application"
);
assert!(
!ops[written..].contains(&Op::Get(Some(Height::new(1)))),
"dispatch must not read the finalized block back from the archive: {ops:?}"
);
});
}
fn staged_dispatch_deduplicates_finalizations(preexisting: bool) {
const PARTITION_PREFIX: &str = "staged-dedup";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let block = make_raw_block(genesis.digest(), Height::new(1), 100);
let (finalizations_by_height, mut finalized_blocks) =
prunable_finalized_stores(&context, PARTITION_PREFIX).await;
if preexisting {
finalized_blocks = finalized_blocks
.put_sync(
block.height().get(),
block.digest(),
&Arc::new(block.clone()),
)
.await
.unwrap();
}
let (actor, mut mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
test_config(
PARTITION_PREFIX,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let buffer = RecordingBuffer::default();
let _actor_handle =
actor.start(application.clone(), buffer.clone(), (resolver_rx, resolver));
while application.pending_ack_heights() != vec![Height::zero()] {
reschedule().await;
}
buffer.insert(block.clone());
let finalization = StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(1)),
View::zero(),
block.digest(),
),
&schemes,
QUORUM,
);
let mut handed = Vec::new();
for _ in 0..2 {
StandardHarness::report_finalization(&mut mailbox, finalization.clone()).await;
let archived = mailbox.get_block(Height::new(1)).await.unwrap();
assert_eq!(archived.digest(), block.digest());
handed.push(
buffer
.last_handed()
.expect("buffer must have served the finalized block"),
);
}
assert!(
handed[0].upgrade().is_some(),
"the first buffered object must remain staged even if already archived"
);
assert!(
handed[1].upgrade().is_none(),
"a duplicate finalization must not replace the staged object"
);
assert_eq!(application.acknowledged().await, Height::zero());
while !application.blocks().contains_key(&Height::new(1)) {
reschedule().await;
}
let delivered = application.blocks()[&Height::new(1)].clone();
assert!(
handed[0]
.upgrade()
.is_some_and(|block| Arc::ptr_eq(&block, &delivered)),
"dispatch must reuse the first staged object"
);
let archived = mailbox.get_block(Height::new(1)).await.unwrap();
assert_eq!(
delivered.digest(),
archived.digest(),
"dispatch must deliver the block retained by the archive"
);
});
}
#[test_traced("WARN")]
fn test_standard_staged_dispatch_deduplicates_finalizations() {
staged_dispatch_deduplicates_finalizations(false);
}
#[test_traced("WARN")]
fn test_standard_staged_dispatch_deduplicates_preexisting_block() {
staged_dispatch_deduplicates_finalizations(true);
}
fn staged_blocks_bounded(skip_to_floor: bool) {
const PARTITION_PREFIX: &str = "staged-bounded";
const ANCHOR_HEIGHT: u64 = 5;
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let (finalizations_by_height, finalized_blocks) =
prunable_finalized_stores(&context, PARTITION_PREFIX).await;
let finalized_blocks =
Recording::new(finalized_blocks, |block: &Arc<B>| Arc::as_ptr(block).addr());
let ops = finalized_blocks.ops();
let (actor, mut mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
test_config(
PARTITION_PREFIX,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let buffer = RecordingBuffer::default();
let _actor_handle =
actor.start(application.clone(), buffer.clone(), (resolver_rx, resolver));
while application.pending_ack_heights() != vec![Height::zero()] {
reschedule().await;
}
let mut parent = StandardHarness::genesis_block(NUM_VALIDATORS as u16).digest();
let mut chain = Vec::new();
for i in 1..=ANCHOR_HEIGHT {
let block = make_raw_block(parent, Height::new(i), i);
parent = block.digest();
chain.push(block);
}
let mut handed = Vec::new();
for block in &chain[..3] {
let height = block.height();
let round = Round::new(Epoch::zero(), View::new(height.get()));
buffer.insert(block.clone());
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::new(height.get() - 1), block.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
assert!(mailbox.get_block(height).await.is_some());
handed.push(
buffer
.last_handed()
.expect("buffer must have served the finalized block"),
);
}
assert!(
handed[..2].iter().all(|block| block.upgrade().is_some()),
"staging must retain two blocks without evicting either on overflow"
);
assert!(
handed[2].upgrade().is_none(),
"staging must stop at twice the pending-ack capacity"
);
assert_eq!(application.pending_ack_heights(), vec![Height::zero()]);
if skip_to_floor {
let anchor = chain.last().unwrap();
let anchor_round = Round::new(Epoch::zero(), View::new(ANCHOR_HEIGHT));
buffer.insert(anchor.clone());
let finalization = StandardHarness::make_finalization(
Proposal::new(anchor_round, View::new(ANCHOR_HEIGHT - 1), anchor.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(finalization);
while !application
.blocks()
.contains_key(&Height::new(ANCHOR_HEIGHT))
{
reschedule().await;
}
assert!(
handed.iter().all(|block| block.upgrade().is_none()),
"floor transition must release skipped staged blocks"
);
for block in &chain[..3] {
assert!(!application.blocks().contains_key(&block.height()));
}
} else {
ops.lock().clear();
for height in 0..=3 {
assert_eq!(application.acknowledged().await, Height::new(height));
}
let ops = ops.lock();
for height in 1..=3 {
let reads = ops
.iter()
.filter(|op| **op == Op::Get(Some(Height::new(height))))
.count();
assert_eq!(reads, usize::from(height == 3), "archive reads at {height}");
}
}
});
}
#[test_traced("WARN")]
fn test_standard_staged_blocks_bounded_drain() {
staged_blocks_bounded(false);
}
#[test_traced("WARN")]
fn test_standard_staged_blocks_bounded_floor() {
staged_blocks_bounded(true);
}
#[test_traced("WARN")]
fn test_standard_does_not_restage_dispatched_block() {
const PARTITION_PREFIX: &str = "restaged-ack";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let (finalizations_by_height, finalized_blocks) =
prunable_finalized_stores(&context, PARTITION_PREFIX).await;
let (actor, mut mailbox, _) = Actor::init(
context.child("actor"),
finalizations_by_height,
finalized_blocks,
test_config(
PARTITION_PREFIX,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let buffer = RecordingBuffer::default();
let _actor_handle =
actor.start(application.clone(), buffer.clone(), (resolver_rx, resolver));
assert_eq!(application.acknowledged().await, Height::zero());
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(genesis.digest(), Height::new(1), 100);
buffer.insert(block.clone());
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, finalization.clone()).await;
while application.pending_ack_heights() != vec![Height::new(1)] {
reschedule().await;
}
StandardHarness::report_finalization(&mut mailbox, finalization).await;
assert!(
mailbox.get_block(Height::new(1)).await.is_some(),
"finalized block must be stored"
);
let repeated = buffer
.last_handed()
.expect("buffer must have served the finalized block again");
assert_eq!(application.pending_ack_heights(), vec![Height::new(1)]);
assert!(
repeated.upgrade().is_none(),
"an already-dispatched block must not consume staging capacity"
);
});
}
fn parse_processed_height(metrics: &str) -> Option<u64> {
for line in metrics.lines() {
let line = line.trim();
if line.starts_with('#') {
continue;
}
let Some(idx) = line.find("processed_height") else {
continue;
};
let mut rest = &line[idx + "processed_height".len()..];
if let Some(labeled) = rest.strip_prefix('{') {
let Some(labels_end) = labeled.find('}') else {
continue;
};
rest = &labeled[labels_end + 1..];
}
if rest.chars().next().is_some_and(char::is_whitespace) {
let value = rest.split_whitespace().next()?;
return value.parse().ok();
}
}
None
}
#[test_traced("WARN")]
fn test_standard_update_block_processed_height_invariant() {
const MAX_PENDING_ACKS: u64 = 4;
const NUM_BLOCKS: u64 = 12;
let runner = deterministic::Runner::timed(Duration::from_secs(60));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let validator = participants[0].clone();
let application = Application::<B>::manual_ack();
let setup = StandardHarness::setup_validator_with(
context.child("validator").with_attribute("index", 0),
&mut oracle,
validator,
ConstantProvider::new(schemes[0].clone()),
NonZeroUsize::new(MAX_PENDING_ACKS as usize).unwrap(),
application,
)
.await;
let application = setup.application;
let mut handle = ValidatorHandle {
mailbox: setup.mailbox,
extra: setup.extra,
};
let mut handles = vec![handle.clone()];
assert_eq!(application.acknowledged().await, Height::zero());
let epocher = FixedEpocher::new(BLOCKS_PER_EPOCH);
let mut parent = Sha256::hash(&[b""]);
let mut parent_commitment =
StandardHarness::genesis_parent_commitment(NUM_VALIDATORS as u16);
for i in 1..=NUM_BLOCKS {
let block = StandardHarness::make_test_block(
parent,
parent_commitment,
Height::new(i),
i,
NUM_VALIDATORS as u16,
);
let commitment = block.digest();
parent = StandardHarness::digest(&block);
parent_commitment = commitment;
let round = Round::new(
epocher
.containing(StandardHarness::height(&block))
.unwrap()
.epoch(),
View::new(i),
);
StandardHarness::verify(&mut handle, round, &block, &mut handles).await;
let proposal = Proposal {
round,
parent: View::new(i.saturating_sub(1)),
payload: commitment,
};
let finalization = StandardHarness::make_finalization(proposal, &schemes, QUORUM);
StandardHarness::report_finalization(&mut handle.mailbox, finalization).await;
}
let check_invariant = |label: &str| {
let Some(highest) = application.blocks().keys().max().copied() else {
return;
};
let processed = parse_processed_height(&context.encode())
.expect("processed_height gauge missing");
let gap = highest.get().saturating_sub(processed);
assert!(
gap <= MAX_PENDING_ACKS,
"{label}: highest={} processed={} gap={} > max_pending_acks={}",
highest.get(),
processed,
gap,
MAX_PENDING_ACKS,
);
};
while (application.blocks().len() as u64) < MAX_PENDING_ACKS {
context.sleep(Duration::from_millis(10)).await;
}
check_invariant("pipeline full");
loop {
let acked = application.acknowledged().await;
check_invariant(&format!("after ack {acked}"));
if acked.get() == NUM_BLOCKS {
break;
}
}
});
}
#[test_traced("WARN")]
fn test_standard_forward_unknown_block_is_noop() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let unknown = Sha256::hash(&[b"unknown-block"]);
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("forward-unknown-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
mailbox.forward(
round,
unknown,
Recipients::Some(vec![participants[1].clone()]),
);
context.sleep(Duration::from_millis(50)).await;
assert!(
buffer.sends().is_empty(),
"forward for an unknown block must not dispatch"
);
});
}
#[test_traced("WARN")]
fn test_standard_proposed_broadcasts_then_persists() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let digest = block.digest();
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("proposed-cache-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let targets = vec![participants[1].clone()];
let (ack, persist) = oneshot::channel();
mailbox.proposed(
round,
ready(block.clone()),
Recipients::Some(targets.clone()),
ack,
);
wait_until(&context, Duration::from_secs(5), "proposed send", || {
!buffer.sends.lock().is_empty()
})
.await;
let sends = buffer.sends();
assert_eq!(sends.len(), 1, "proposal must dispatch exactly once");
assert_eq!(sends[0].0, round);
assert_eq!(sends[0].1.digest(), digest);
let sync = persist.await.expect("proposed sync handle missing");
assert!(sync.durable(round, "proposed").await);
mailbox.forward(round, digest, Recipients::Some(targets));
wait_until(&context, Duration::from_secs(5), "forward send", || {
buffer.sends.lock().len() >= 2
})
.await;
let sends = buffer.sends();
assert_eq!(sends.len(), 2);
assert_eq!(sends[1].1.digest(), digest);
});
}
async fn stage_block(
context: &deterministic::Context,
gates: &Gates<D, Digested<B>>,
round: Round,
block: B,
) -> Handle<()> {
let digest = block.digest();
let (tx, rx) = oneshot::channel();
let stager = context.child("stager").spawn({
let gates = gates.clone();
move |_| async move {
gates
.stage(
round,
digest,
ready(block),
|digest| {
tx.send_lossy(digest);
},
"test",
)
.await;
}
});
assert_eq!(
rx.await.expect("staging must publish the block digest"),
digest
);
stager
}
#[test_traced("WARN")]
fn test_standard_propose_relay_after_claim_sends_once() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let digest = block.digest();
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("relay-claim-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let gates = Gates::new();
stage_block(&context, &gates, round, block).await;
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Prepare { round });
assert!(matches!(feedback, Feedback::Ok));
assert!(
mailbox.get_verified(round).await.is_none(),
"an early relay must not store the candidate"
);
assert_eq!(buffer.sends().len(), 1, "early relay must send once");
let gate = gates
.claim(round, digest, |block, ack| {
mailbox.verified_deferred(round, block, ack)
})
.expect("gate registered");
assert_eq!(
gate.await.expect("gate resolved"),
GateOutcome::Ready(true),
"certify must persist the staged block and resolve the gate durably",
);
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Propose { round });
assert!(matches!(feedback, Feedback::Ok));
assert!(
mailbox.get_verified(round).await.is_some(),
"certify must store the candidate"
);
assert_eq!(
buffer.sends().len(),
1,
"the lock-in must not send a claimed candidate again"
);
});
}
#[test_traced("WARN")]
fn test_standard_restage_after_prune_sends_once() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let digest = block.digest();
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("relay-restage-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let gates = Gates::new();
let stager = stage_block(&context, &gates, round, block.clone()).await;
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Prepare { round });
assert!(matches!(feedback, Feedback::Ok));
assert!(
mailbox.get_verified(round).await.is_none(),
"an early relay must not store the candidate"
);
assert_eq!(buffer.sends().len(), 1, "early relay must send once");
gates.retain_after(Round::new(Epoch::zero(), View::new(2)));
stager.await.expect("pruning must end the staging");
stage_block(&context, &gates, round, block).await;
let gate = gates.take(round, digest).expect("gate registered");
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Prepare { round });
assert!(matches!(feedback, Feedback::Ok));
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Propose { round });
assert!(matches!(feedback, Feedback::Ok));
assert_eq!(
gate.await.expect("gate resolved"),
GateOutcome::Ready(true),
"lock-in handshake must resolve the gate durably",
);
assert!(
mailbox.get_verified(round).await.is_some(),
"lock-in must store the candidate"
);
assert_eq!(
buffer.sends().len(),
1,
"a candidate staged again for a decided round must not be sent again"
);
});
}
#[test_traced("WARN")]
fn test_standard_propose_relay_sends_staged_block() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let digest = block.digest();
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("relay-hit-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let gates = Gates::new();
stage_block(&context, &gates, round, block).await;
let gate = gates.take(round, digest).expect("gate registered");
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Propose { round });
assert!(matches!(feedback, Feedback::Ok));
wait_until(&context, Duration::from_secs(5), "staged send", || {
!buffer.sends.lock().is_empty()
})
.await;
let sends = buffer.sends();
assert_eq!(sends.len(), 1, "staged proposal must dispatch exactly once");
assert_eq!(sends[0].0, round);
assert_eq!(sends[0].1.digest(), digest);
assert!(matches!(sends[0].2, Recipients::All));
assert!(
gates.take_staged(round, digest).is_none(),
"relay must consume the staged proposal"
);
assert_eq!(
gate.await.expect("gate resolved"),
GateOutcome::Ready(true),
"relay handshake must resolve the gate durably",
);
});
}
#[test_traced("WARN")]
fn test_standard_prepare_relay_sends_once_and_propose_stores() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let digest = block.digest();
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("relay-prepare-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let gates = Gates::new();
stage_block(&context, &gates, round, block).await;
let gate = gates.take(round, digest).expect("gate registered");
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Prepare { round });
assert!(matches!(feedback, Feedback::Ok));
while buffer.sends().is_empty() {
reschedule().await;
}
assert_eq!(buffer.sends().len(), 1, "early relay must send once");
assert!(
mailbox.get_verified(round).await.is_none(),
"an early relay must not store the candidate"
);
let mut gate = gate;
assert!(
gate.try_recv().is_err(),
"an early relay must not resolve the certification gate"
);
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Prepare { round });
assert!(matches!(feedback, Feedback::Ok));
let feedback = relay::broadcast(&gates, &mailbox, digest, Plan::Propose { round });
assert!(matches!(feedback, Feedback::Ok));
assert!(
gates.take_staged(round, digest).is_none(),
"lock-in must consume the staged candidate"
);
assert_eq!(
gate.await.expect("gate resolved"),
GateOutcome::Ready(true),
"lock-in handshake must resolve the gate durably",
);
assert!(
mailbox.get_verified(round).await.is_some(),
"lock-in must store the candidate"
);
let sends = buffer.sends();
assert_eq!(
sends.len(),
1,
"neither a repeated prepare relay nor the lock-in may send a relayed candidate again"
);
assert_eq!(sends[0].0, round);
assert_eq!(sends[0].1.digest(), digest);
assert!(matches!(sends[0].2, Recipients::All));
});
}
#[test_traced("WARN")]
fn test_standard_boundary_restage_sends_once() {
for kind in wrapper_kinds() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(
&mut context,
NAMESPACE,
NUM_VALIDATORS,
);
let outgoing = participants[0].clone();
let incoming = participants[1].clone();
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let (mut marshal, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 1),
"boundary-restage",
ConstantProvider::new(schemes[1].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(genesis.clone()),
)
.await;
let buffer = buffer.expect("buffer was provided");
let mut wrapper = Wrapper::new(
kind,
context.child("wrapper"),
MockVerifyingApp::new().with_handoff(Handoff::Stage(())),
marshal.clone(),
);
let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
let boundary_round = Round::new(Epoch::zero(), View::new(boundary_height.get()));
let mut boundary = genesis.inner().clone();
for height in 1..=boundary_height.get() {
let parent = boundary.digest();
let round = Round::new(Epoch::zero(), View::new(height));
boundary = B::new::<Sha256>(
Ctx {
round,
leader: outgoing.clone(),
parent: (View::new(height - 1), parent),
},
parent,
Height::new(height),
height * 100,
);
assert!(marshal.verified(round, ready(boundary.clone())).await);
}
let digest = boundary.digest();
let notarization = StandardHarness::make_notarization(
Proposal::new(boundary_round, View::new(boundary_height.get() - 1), digest),
&schemes,
QUORUM,
);
StandardHarness::report_notarization(&mut marshal, notarization).await;
assert!(
wrapper.certify(boundary_round, digest).await.await.unwrap(),
"{kind:?}: boundary must certify"
);
assert!(
marshal.get_info(boundary_height).await.is_none(),
"the boundary is not finalized"
);
let outgoing_round = Round::new(Epoch::zero(), boundary_round.view().next());
let outgoing_context = Ctx {
round: outgoing_round,
leader: outgoing,
parent: (boundary_round.view(), digest),
};
assert!(
wrapper
.verify(outgoing_context, digest)
.await
.await
.unwrap(),
"{kind:?}: the outgoing boundary re-proposal must verify"
);
let round = Round::new(Epoch::zero(), outgoing_round.view().next());
let held_context = Ctx {
round,
leader: incoming,
parent: (outgoing_round.view(), digest),
};
assert_eq!(
wrapper.prepare(held_context.clone()).await.await.unwrap(),
Handoff::Stage(digest),
"{kind:?}: prepare must reuse the boundary payload"
);
assert!(matches!(
wrapper.broadcast(digest, Plan::Prepare { round }),
Feedback::Ok
));
while buffer.sends().is_empty() {
reschedule().await;
}
assert_eq!(buffer.sends().len(), 1);
assert!(
marshal.get_verified(round).await.is_none(),
"the held candidate must not be stored"
);
let replacement_context = Ctx {
parent: (boundary_round.view(), digest),
..held_context.clone()
};
assert_ne!(held_context.parent, replacement_context.parent);
assert_eq!(
wrapper.propose(replacement_context).await.await.unwrap(),
digest,
"{kind:?}: the replacement must reuse the same payload"
);
assert!(
marshal.get_verified(round).await.is_none(),
"restaging must not store the replacement before lock-in"
);
assert!(matches!(
wrapper.broadcast(digest, Plan::Propose { round }),
Feedback::Ok
));
let notarization = StandardHarness::make_notarization(
Proposal::new(round, boundary_round.view(), digest),
&schemes,
QUORUM,
);
StandardHarness::report_notarization(&mut marshal, notarization).await;
assert!(
wrapper.certify(round, digest).await.await.unwrap(),
"{kind:?}: the replacement durability gate must complete"
);
assert_eq!(
marshal.get_verified(round).await.unwrap().digest(),
digest,
"{kind:?}: the replacement must be stored at the incoming round"
);
let sends = buffer.sends();
assert_eq!(
sends.len(),
1,
"{kind:?}: replacing the parent must not resend the same boundary payload"
);
assert_eq!(sends[0].0, round);
assert_eq!(sends[0].1.digest(), digest);
assert!(matches!(sends[0].2, Recipients::All));
});
}
}
#[test_traced("WARN")]
fn test_standard_proposed_conflicting_blocks_both_ack() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block_a = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let block_b = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 200);
assert_ne!(block_a.digest(), block_b.digest());
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("proposed-conflict-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
let (ack_a, persist_a) = oneshot::channel();
mailbox.proposed(round, ready(block_a.clone()), Recipients::All, ack_a);
let (ack_b, persist_b) = oneshot::channel();
mailbox.proposed(round, ready(block_b.clone()), Recipients::All, ack_b);
let sync_a = persist_a.await.expect("first proposed sync handle missing");
assert!(sync_a.durable(round, "proposed").await);
let sync_b = persist_b
.await
.expect("second proposed sync handle missing");
assert!(sync_b.durable(round, "proposed").await);
let sends = buffer.sends();
assert_eq!(sends.len(), 2, "both conflicting proposals must dispatch");
assert_eq!(sends[0].1.digest(), block_a.digest());
assert_eq!(sends[1].1.digest(), block_b.digest());
});
}
#[test_traced("WARN")]
fn test_standard_forward_cached_block_sends_to_peers() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let digest = block.digest();
let (mailbox, buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("forward-cached-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let buffer = buffer.expect("buffer was provided");
assert!(mailbox.verified(round, ready(block.clone())).await);
let targets = vec![participants[1].clone(), participants[2].clone()];
mailbox.forward(round, digest, Recipients::Some(targets.clone()));
wait_until(&context, Duration::from_secs(5), "buffer.send", || {
!buffer.sends.lock().is_empty()
})
.await;
let sends = buffer.sends();
assert_eq!(sends.len(), 1);
let (sent_round, sent_block, sent_recipients) = &sends[0];
assert_eq!(*sent_round, round);
assert_eq!(sent_block.digest(), digest);
match sent_recipients {
Recipients::Some(peers) => assert_eq!(peers, &targets),
other => panic!("expected Recipients::Some, got {other:?}"),
}
});
}
#[test_traced("WARN")]
fn test_standard_hint_finalized_below_floor_is_noop() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("hint-below-floor-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let floor_anchor = StandardHarness::make_test_block(
Sha256::hash(&[b"floor-parent"]),
StandardHarness::genesis_parent_commitment(NUM_VALIDATORS as u16),
Height::new(10),
10,
NUM_VALIDATORS as u16,
);
let floor_round = Round::new(Epoch::zero(), View::new(10));
let finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::new(9), floor_anchor.digest()),
&schemes,
QUORUM,
);
mailbox.set_floor(finalization);
assert!(mailbox.verified(floor_round, floor_anchor).await);
context.sleep(Duration::from_millis(50)).await;
mailbox.hint_finalized(Height::new(5), NonEmptyVec::new(participants[1].clone()));
context.sleep(Duration::from_millis(50)).await;
assert!(
resolver.targeted_is_empty(),
"hint at or below floor must not fetch"
);
});
}
#[test_traced("WARN")]
fn test_standard_hint_finalized_skips_when_already_finalized() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
let (mut mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("hint-already-final-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert!(mailbox.verified(round, ready(block.clone())).await);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
while mailbox.get_finalization(Height::new(1)).await.is_none() {
context.sleep(Duration::from_millis(10)).await;
}
mailbox.hint_finalized(Height::new(1), NonEmptyVec::new(participants[1].clone()));
context.sleep(Duration::from_millis(50)).await;
assert!(
resolver.targeted_is_empty(),
"hint for a locally-finalized height must not fetch"
);
});
}
#[test_traced("WARN")]
fn test_standard_hint_finalized_emits_targeted_fetch() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let (mailbox, _buffer, resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("hint-targets-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
let target = participants[1].clone();
mailbox.hint_finalized(Height::new(7), NonEmptyVec::new(target.clone()));
wait_until(&context, Duration::from_secs(5), "fetch_targeted", || {
!resolver.targeted.lock().is_empty()
})
.await;
let targeted = resolver.targeted();
assert_eq!(targeted.len(), 1);
let (request, targets) = &targeted[0];
assert_eq!(
request,
&handler::Key::Finalized {
height: Height::new(7)
}
);
assert_eq!(&targets[..], &[target]);
});
}
#[test_traced("WARN")]
fn test_standard_prune_above_floor_is_rejected() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let me = participants[0].clone();
let round = Round::new(Epoch::zero(), View::new(1));
let block = make_raw_block(Sha256::hash(&[b""]), Height::new(1), 100);
let finalization = StandardHarness::make_finalization(
Proposal::new(round, View::zero(), block.digest()),
&schemes,
QUORUM,
);
let (mut mailbox, _buffer, _resolver, _actor_handle) = start_standard_actor(
context.child("validator").with_attribute("index", 0),
&format!("prune-above-floor-{me}"),
ConstantProvider::new(schemes[0].clone()),
Application::<B>::manual_ack(),
Some(RecordingBuffer::default()),
Start::Genesis(StandardHarness::genesis_block(NUM_VALIDATORS as u16)),
)
.await;
assert!(mailbox.verified(round, ready(block.clone())).await);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
while mailbox.get_finalization(Height::new(1)).await.is_none() {
context.sleep(Duration::from_millis(10)).await;
}
mailbox.prune(Height::new(100));
context.sleep(Duration::from_millis(50)).await;
assert!(mailbox.get_block(Height::new(1)).await.is_some());
assert!(mailbox.get_finalization(Height::new(1)).await.is_some());
});
}
#[derive(Clone, Copy, Debug)]
enum Pipeline {
Certification,
Build,
Abandon,
}
#[derive(Clone)]
struct PipelineApp {
verify_started: Arc<Mutex<Option<oneshot::Sender<()>>>>,
verify_release: Arc<Mutex<Option<oneshot::Receiver<()>>>>,
build_started: Arc<Mutex<Option<oneshot::Sender<Ctx>>>>,
build_release: Arc<Mutex<Option<oneshot::Receiver<()>>>>,
build_completed: Arc<Mutex<Option<oneshot::Sender<()>>>>,
prepares: Arc<AtomicUsize>,
block: B,
}
impl ConsensusApplication<Runtime> for PipelineApp {
type Block = B;
type Context = Ctx;
type SigningScheme = S;
type Input = ();
async fn prepare(
&mut self,
context: (Runtime, Ctx),
ancestry: impl Ancestry<B>,
input: (),
) -> Handoff<Digested<B>> {
self.prepares.fetch_add(1, Ordering::SeqCst);
self.propose(context, ancestry, input)
.await
.map_or(Handoff::Wait, Handoff::Stage)
}
async fn propose(
&mut self,
(_, context): (Runtime, Ctx),
_: impl Ancestry<B>,
_: (),
) -> Option<Digested<B>> {
self.build_started
.lock()
.take()
.expect("the retained handoff must not be replaced by an ordinary build")
.send_lossy(context);
let release = self.build_release.lock().take().unwrap();
release.await.unwrap();
self.build_completed.lock().take().unwrap().send_lossy(());
Some(ready(self.block.clone()))
}
async fn verify(&mut self, _: (Runtime, Ctx), _: impl Ancestry<B>) -> bool {
let started = self.verify_started.lock().take();
if let Some(started) = started {
let release = self.verify_release.lock().take().unwrap();
started.send_lossy(());
release.await.unwrap();
}
true
}
}
async fn next_vote(
receiver: &mut impl Receiver<PublicKey = PublicKey>,
victim: &PublicKey,
) -> Vote<S, D> {
let (sender, message) = receiver.recv().await.unwrap();
assert_eq!(&sender, victim);
Vote::decode(message).unwrap()
}
fn pipeline_handoff(outcome: Pipeline) {
deterministic::Runner::timed(Duration::from_secs(30)).start(|mut context| async move {
let Fixture {
participants,
schemes,
..
} = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let victim = participants[3].clone();
let peer = participants[2].clone();
let mut oracle = setup_network_with_participants(
context.child("network"),
NZUsize!(1),
participants.clone(),
1024 * 1024,
)
.await;
let reporter = Application::<B>::default();
let setup = StandardHarness::setup_validator_with(
context.child("marshal"),
&mut oracle,
victim.clone(),
ConstantProvider::new(schemes[3].clone()),
NZUsize!(1),
reporter.clone(),
)
.await;
let mut marshal = setup.mailbox;
let genesis = StandardHarness::genesis_block(NUM_VALIDATORS as u16);
let floor_round = Round::new(Epoch::zero(), View::new(1));
let floor_block = B::new::<Sha256>(
Ctx {
round: floor_round,
leader: participants[1].clone(),
parent: (View::zero(), genesis.digest()),
},
genesis.digest(),
Height::new(1),
100,
);
let floor_digest = floor_block.digest();
assert!(marshal.verified(floor_round, ready(floor_block)).await);
let floor_finalization = StandardHarness::make_finalization(
Proposal::new(floor_round, View::zero(), floor_digest),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut marshal, floor_finalization.clone()).await;
let parent_round = Round::new(Epoch::zero(), View::new(2));
let parent_block = B::new::<Sha256>(
Ctx {
round: parent_round,
leader: peer.clone(),
parent: (View::new(1), floor_digest),
},
floor_digest,
Height::new(2),
200,
);
let parent_digest = parent_block.digest();
assert!(
setup
.extra
.broadcast(Recipients::Some(vec![]), ready(parent_block))
.accepted()
);
let round = Round::new(Epoch::zero(), View::new(3));
let expected_context = Ctx {
round,
leader: victim.clone(),
parent: (View::new(2), parent_digest),
};
let block =
B::new::<Sha256>(expected_context.clone(), parent_digest, Height::new(3), 300);
let digest = block.digest();
let (verify_tx, verify_rx) = oneshot::channel();
let (verify_release_tx, verify_release_rx) = oneshot::channel();
let (build_tx, build_rx) = oneshot::channel();
let (build_release_tx, build_release_rx) = oneshot::channel();
let (completed_tx, completed_rx) = oneshot::channel();
let prepares = Arc::new(AtomicUsize::new(0));
let app = PipelineApp {
verify_started: Arc::new(Mutex::new(Some(verify_tx))),
verify_release: Arc::new(Mutex::new(Some(verify_release_rx))),
build_started: Arc::new(Mutex::new(Some(build_tx))),
build_release: Arc::new(Mutex::new(Some(build_release_rx))),
build_completed: Arc::new(Mutex::new(Some(completed_tx))),
prepares: prepares.clone(),
block,
};
let control = oracle.control(victim.clone());
let vote_network = control.register(3, TEST_QUOTA).await.unwrap();
let certificate_network = control.register(4, TEST_QUOTA).await.unwrap();
let resolver_network = control.register(5, TEST_QUOTA).await.unwrap();
let control = oracle.control(peer);
let (mut vote_sender, mut vote_receiver) =
control.register(3, TEST_QUOTA).await.unwrap();
let (mut certificate_sender, _certificate_receiver) =
control.register(4, TEST_QUOTA).await.unwrap();
let _resolver = control.register(5, TEST_QUOTA).await.unwrap();
setup_network_links(&mut oracle, &participants, LINK).await;
let wrapper = Deferred::new(
context.child("wrapper"),
app,
marshal.clone(),
FixedEpocher::new(BLOCKS_PER_EPOCH),
);
let engine = simplex::Engine::new(
context.child("simplex"),
simplex::config::Config {
scheme: schemes[3].clone(),
elector: RoundRobin::<Sha256>::default(),
blocker: oracle.control(victim.clone()),
automaton: wrapper.clone(),
relay: wrapper.clone(),
reporter: wrapper.clone(),
strategy: Sequential,
partition: format!("pipeline-{outcome:?}"),
mailbox_size: NZUsize!(128),
epoch: Epoch::zero(),
floor: simplex::config::Floor::Finalized(floor_finalization),
replay_buffer: NZUsize!(1024),
write_buffer: NZUsize!(1024),
page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
leader_timeout: Duration::from_secs(10),
certification_timeout: Duration::from_secs(15),
timeout_retry: Duration::from_secs(3),
view_retention: ViewDelta::new(10),
skip: SkipPolicy::Enabled {
timeout: Duration::from_secs(20),
budget: SkipBudget::Participants,
},
fetch_timeout: NZDuration!(Duration::from_secs(1)),
fetch_retry_delay: Duration::from_millis(100),
forward: ForwardPolicy::Disabled,
track_historical_votes: false,
},
);
let engine = engine.start(vote_network, certificate_network, resolver_network);
let parent = Proposal::new(parent_round, View::new(1), parent_digest);
vote_sender.send(
Recipients::One(victim.clone()),
Vote::<S, D>::Notarize(Notarize::sign(&schemes[2], parent.clone()).unwrap())
.encode(),
true,
);
verify_rx.await.unwrap();
let notarization = StandardHarness::make_notarization(parent.clone(), &schemes, QUORUM);
certificate_sender.send(
Recipients::One(victim.clone()),
Certificate::<S, D>::Notarization(notarization).encode(),
true,
);
let built_context = build_rx.await.expect("handoff build should start");
assert_eq!(built_context, expected_context);
match outcome {
Pipeline::Abandon => {
build_release_tx.send_lossy(());
completed_rx.await.unwrap();
let relayed = setup.extra.subscribe(digest).await.unwrap();
assert_eq!(relayed.digest(), digest);
drop(relayed);
assert!(marshal.get_verified(round).await.is_none());
assert_eq!(
commonware_runtime::telemetry::metrics::count_running_tasks(
&context,
"wrapper_prepare",
),
1,
);
let nullifies: Vec<_> = schemes[..QUORUM as usize]
.iter()
.map(|scheme| Nullify::sign::<D>(scheme, round).unwrap())
.collect();
let nullification = Nullification::from_nullifies(
&schemes[0],
non_empty![@&nullifies],
&Sequential,
)
.unwrap();
certificate_sender.send(
Recipients::One(victim.clone()),
Certificate::<S, D>::Nullification(nullification).encode(),
true,
);
while !context.encode().lines().any(|line| {
line.contains("handoff_abandoned_total{")
&& line.contains("reason=\"ViewExit\"")
&& line
.rsplit(' ')
.next()
.and_then(|v| v.parse::<u64>().ok())
.is_some_and(|count| count > 0)
}) {
reschedule().await;
}
let replacement_round = Round::new(Epoch::zero(), View::new(4));
let replacement = B::new::<Sha256>(
Ctx {
round: replacement_round,
leader: participants[0].clone(),
parent: (View::new(2), parent_digest),
},
parent_digest,
Height::new(3),
400,
);
let replacement_digest = replacement.digest();
assert!(
setup
.extra
.broadcast(Recipients::Some(vec![]), ready(replacement))
.accepted()
);
let finalization = StandardHarness::make_finalization(
Proposal::new(replacement_round, View::new(2), replacement_digest),
&schemes,
QUORUM,
);
certificate_sender.send(
Recipients::One(victim.clone()),
Certificate::<S, D>::Finalization(finalization).encode(),
true,
);
while reporter.tip() != Some((Height::new(3), replacement_digest)) {
reschedule().await;
}
assert!(marshal.get_verified(round).await.is_none());
assert_eq!(prepares.load(Ordering::SeqCst), 1);
verify_release_tx.send_lossy(());
engine.abort();
let _ = engine.await;
while commonware_runtime::telemetry::metrics::count_running_tasks(
&context,
"wrapper_prepare",
) != 0
{
reschedule().await;
}
return;
}
Pipeline::Certification => {
verify_release_tx.send_lossy(());
loop {
if let Vote::Finalize(vote) = next_vote(&mut vote_receiver, &victim).await
&& vote.proposal == parent
{
break;
}
}
build_release_tx.send_lossy(());
completed_rx.await.unwrap();
}
Pipeline::Build => {
build_release_tx.send_lossy(());
completed_rx.await.unwrap();
let quiet_until = context.current() + 3 * LINK.latency;
loop {
select! {
vote = next_vote(&mut vote_receiver, &victim) => {
assert_ne!(
vote.view(),
round.view(),
"staged handoff must not vote before parent certification"
);
},
_ = context.sleep_until(quiet_until) => break,
}
}
assert!(
marshal.get_verified(round).await.is_none(),
"the completed candidate must remain staged until its parent certifies"
);
verify_release_tx.send_lossy(());
}
}
assert_eq!(
prepares.load(Ordering::SeqCst),
1,
"one handoff request reaches the application once"
);
let proposal = loop {
if let Vote::Notarize(vote) = next_vote(&mut vote_receiver, &victim).await
&& vote.proposal.round == round
{
break vote.proposal;
}
};
assert_eq!(proposal, Proposal::new(round, View::new(2), digest));
assert!(marshal.get_verified(round).await.is_some());
let notarization =
StandardHarness::make_notarization(proposal.clone(), &schemes, QUORUM);
certificate_sender.send(
Recipients::One(victim.clone()),
Certificate::<S, D>::Notarization(notarization).encode(),
true,
);
loop {
if let Vote::Finalize(vote) = next_vote(&mut vote_receiver, &victim).await
&& vote.proposal.round == round
{
assert_eq!(vote.proposal, proposal);
break;
}
}
});
}
#[test_traced("WARN")]
fn test_pipelined_handoff_retains_completed_build_until_parent_certification() {
pipeline_handoff(Pipeline::Build);
}
#[test_traced("WARN")]
fn test_pipelined_handoff_retains_build_across_parent_certification() {
pipeline_handoff(Pipeline::Certification);
}
#[test_traced("WARN")]
fn test_pipelined_handoff_releases_abandoned_build_after_finalization() {
pipeline_handoff(Pipeline::Abandon);
}
fn configured_floor_config<P: Provider<Scope = Epoch, Scheme = S>>(
partition_prefix: &str,
provider: P,
floor: Finalization<S, D>,
) -> Config<P, FixedEpocher, Sequential, B, Digested<B>, D> {
Config {
epocher: FixedEpocher::new(NZU64!(100)),
start: Start::Floor(floor),
..test_config(partition_prefix, provider, NZUsize!(1))
}
}
#[test_traced("WARN")]
fn test_standard_prune_keeps_round_floor_evidence_for_configured_floor_restart() {
let partition_prefix = "prune-configured-floor-restart";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
let mut parent = StandardHarness::genesis_block(NUM_VALIDATORS as u16).digest();
let blocks: Vec<B> = (1..=30)
.map(|height| {
let block = make_raw_block(parent, Height::new(height), height * 100);
parent = block.digest();
block
})
.collect();
let finalize = |height: u64| {
StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(height)),
View::new(height - 1),
blocks[height as usize - 1].digest(),
),
&schemes,
QUORUM,
)
};
let floor = finalize(10);
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mut mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("first"),
certificates,
finalized,
configured_floor_config(
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
floor.clone(),
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let buffer = RecordingBuffer::default();
for block in &blocks[9..] {
buffer.insert(block.clone());
}
let application = Application::<B>::manual_ack();
let handle = actor.start(application.clone(), buffer, (resolver_rx, resolver));
assert_eq!(application.acknowledged().await, Height::new(10));
StandardHarness::report_finalization(&mut mailbox, finalize(30)).await;
for height in 11..=20 {
assert_eq!(application.acknowledged().await, Height::new(height));
}
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
mailbox.prune(Height::new(20));
assert!(mailbox.get_block(Height::new(10)).await.is_none());
handle.abort();
let _ = handle.await;
seed_processed_height(
context.child("metadata"),
partition_prefix,
Height::new(20),
None,
)
.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, _mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
configured_floor_config(partition_prefix, EmptyProvider, floor),
)
.await;
let (resolver_rx, resolver) =
RecordingResolver::holding(context.child("restarted_resolver"));
let application = Application::<B>::manual_ack();
let _handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
assert_eq!(application.acknowledged().await, Height::new(21));
});
}
const REPROPOSAL_EPOCH_LENGTH: NonZeroU64 = NZU64!(11);
fn reproposal_floor_config<P: Provider<Scope = Epoch, Scheme = S>>(
partition_prefix: &str,
provider: P,
floor: Finalization<S, D>,
) -> Config<P, FixedEpocher, Sequential, B, Digested<B>, D> {
Config {
epocher: FixedEpocher::new(REPROPOSAL_EPOCH_LENGTH),
..configured_floor_config(partition_prefix, provider, floor)
}
}
struct ReproposalRun {
handle: Handle<()>,
mailbox: Mailbox<S, Standard<B>>,
resolver: RecordingResolver,
application: Application<B>,
schemes: Vec<S>,
blocks: Vec<B>,
}
impl ReproposalRun {
fn scheme(&self) -> S {
self.schemes[0].clone()
}
fn block(&self, height: u64) -> B {
self.blocks[height as usize - 1].clone()
}
fn finalize(&self, height: u64, view: u64) -> Finalization<S, D> {
finalize_in_epoch(&self.schemes, &self.blocks, height, view)
}
}
fn finalize_in_epoch(
schemes: &[S],
blocks: &[B],
height: u64,
view: u64,
) -> Finalization<S, D> {
let epoch = FixedEpocher::new(REPROPOSAL_EPOCH_LENGTH)
.containing(Height::new(height))
.expect("test heights must be covered by the epocher")
.epoch();
StandardHarness::make_finalization(
Proposal::new(
Round::new(epoch, View::new(view)),
View::new(view - 1),
blocks[height as usize - 1].digest(),
),
schemes,
QUORUM,
)
}
fn with_epocher<P: Provider<Scope = Epoch, Scheme = S>, ES: Epocher>(
config: Config<P, FixedEpocher, Sequential, B, Digested<B>, D>,
epocher: ES,
) -> Config<P, ES, Sequential, B, Digested<B>, D> {
Config {
provider: config.provider,
epocher,
start: config.start,
mailbox_size: config.mailbox_size,
view_retention: config.view_retention,
max_repair: config.max_repair,
max_pending_acks: config.max_pending_acks,
block_codec_config: config.block_codec_config,
partition_prefix: config.partition_prefix,
prunable_items_per_section: config.prunable_items_per_section,
replay_buffer: config.replay_buffer,
key_write_buffer: config.key_write_buffer,
value_write_buffer: config.value_write_buffer,
page_cache: config.page_cache,
strategy: config.strategy,
}
}
#[derive(Clone)]
struct RetiredThrough<E> {
epocher: E,
retired: Height,
}
impl<E: Epocher> Epocher for RetiredThrough<E> {
fn containing(&self, height: Height) -> Option<EpochInfo> {
if height <= self.retired {
return None;
}
self.epocher.containing(height)
}
fn first(&self, epoch: Epoch) -> Option<Height> {
self.epocher.first(epoch)
}
fn last(&self, epoch: Epoch) -> Option<Height> {
self.epocher.last(epoch)
}
}
struct BoundaryBlock {
schemes: Vec<S>,
block: B,
}
impl BoundaryBlock {
fn new(context: &mut deterministic::Context) -> Self {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(context, NAMESPACE, NUM_VALIDATORS);
let block = make_raw_block(Sha256::hash(&[b"parent"]), Height::new(10), 1_000);
Self { schemes, block }
}
fn scheme(&self) -> S {
self.schemes[0].clone()
}
fn finalize(&self, view: u64) -> Finalization<S, D> {
StandardHarness::make_finalization(
Proposal::new(
Round::new(Epoch::zero(), View::new(view)),
View::new(view - 1),
self.block.digest(),
),
&self.schemes,
QUORUM,
)
}
}
async fn start_reproposal_processed_through_20(
context: &mut deterministic::Context,
partition_prefix: &str,
) -> ReproposalRun {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(context, NAMESPACE, NUM_VALIDATORS);
let mut parent = StandardHarness::genesis_block(NUM_VALIDATORS as u16).digest();
let blocks: Vec<B> = (1..=30)
.map(|height| {
let block = make_raw_block(parent, Height::new(height), height * 100);
parent = block.digest();
block
})
.collect();
let finalize = |height: u64, view: u64| finalize_in_epoch(&schemes, &blocks, height, view);
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(context, partition_prefix).await;
let (actor, mut mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("first"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
finalize(10, 10),
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let buffer = RecordingBuffer::default();
for block in &blocks[9..] {
buffer.insert(block.clone());
}
let application = Application::<B>::manual_ack();
let handle = actor.start(application.clone(), buffer, (resolver_rx, resolver.clone()));
assert_eq!(application.acknowledged().await, Height::new(10));
StandardHarness::report_finalization(&mut mailbox, finalize(30, 30)).await;
for height in 11..=20 {
assert_eq!(application.acknowledged().await, Height::new(height));
}
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
ReproposalRun {
handle,
mailbox,
resolver,
application,
schemes,
blocks,
}
}
async fn restart_with_retired_floor_resumes_at(
context: &deterministic::Context,
partition_prefix: &str,
floor: Finalization<S, D>,
next: Height,
) {
let round = floor.round();
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(context, partition_prefix).await;
let (actor, _mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, EmptyProvider, floor),
)
.await;
assert!(
restored.cutoff().covers(round),
"restart must restore a round floor covering the finalization: {:?} misses {round:?}",
restored.cutoff()
);
let (resolver_rx, resolver) =
RecordingResolver::holding(context.child("restarted_resolver"));
let application = Application::<B>::manual_ack();
let _handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
assert_eq!(application.acknowledged().await, next);
}
#[test_traced("WARN")]
fn test_standard_reproposal_floor_below_processed_is_covered_after_prune_restart() {
let partition_prefix = "reproposal-floor-prune-restart";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let scheme = run.scheme();
let reproposal = run.finalize(10, 40);
run.handle.abort();
let _ = run.handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("reproposal"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(scheme),
reproposal.clone(),
),
)
.await;
let (resolver_rx, resolver) =
RecordingResolver::holding(context.child("reproposal_resolver"));
let application = Application::<B>::manual_ack();
let handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
mailbox.prune(Height::new(20));
assert!(mailbox.get_block(Height::new(10)).await.is_none());
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
handle.abort();
let _ = handle.await;
restart_with_retired_floor_resumes_at(
&context,
partition_prefix,
reproposal,
Height::new(21),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_reproposal_reported_below_processed_survives_restart_before_ack() {
let partition_prefix = "reproposal-reported-restart-before-ack";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let reproposal = run.finalize(10, 40);
let ReproposalRun {
handle,
mut mailbox,
application,
..
} = run;
StandardHarness::report_finalization(&mut mailbox, reproposal.clone()).await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert_eq!(application.pending_ack_heights(), vec![Height::new(21)]);
handle.abort();
let _ = handle.await;
restart_with_retired_floor_resumes_at(
&context,
partition_prefix,
reproposal,
Height::new(21),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_reproposal_delivered_below_processed_survives_restart_before_ack() {
let partition_prefix = "reproposal-delivered-restart-before-ack";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let boundary = run.block(10);
let reproposal = run.finalize(10, 40);
let ReproposalRun {
handle,
mailbox,
resolver,
application,
..
} = run;
let height = Height::new(10);
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height
}),
tracing::Span::none(),
)),
},
value: (reproposal.clone(), boundary).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("marshal must answer the delivery"),
Outcome::Resolved(Resolution::Complete)
);
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert_eq!(application.pending_ack_heights(), vec![Height::new(21)]);
handle.abort();
let _ = handle.await;
restart_with_retired_floor_resumes_at(
&context,
partition_prefix,
reproposal,
Height::new(21),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_late_finalization_block_delivered_below_processed_survives_restart_before_ack()
{
let partition_prefix = "late-finalization-block-delivered-restart-before-ack";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let scheme = run.scheme();
let first = run.finalize(10, 10);
let repaired = run.block(11);
let late = run.finalize(11, 11);
run.handle.abort();
let _ = run.handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mut mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("pruned"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, ConstantProvider::new(scheme), first),
)
.await;
assert!(
restored
.cutoff()
.covers(Round::new(Epoch::zero(), View::new(10)))
);
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let handle =
actor.start_unbuffered(application.clone(), (resolver_rx, resolver.clone()));
mailbox.prune(Height::new(20));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert!(mailbox.get_block(Height::new(11)).await.is_none());
assert!(mailbox.get_finalization(Height::new(11)).await.is_none());
assert!(mailbox.get_finalization(Height::new(10)).await.is_some());
assert_eq!(application.pending_ack_heights(), vec![Height::new(21)]);
StandardHarness::report_finalization(&mut mailbox, late.clone()).await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
let round = late.round();
let digest = repaired.digest();
assert!(resolver.fetches().iter().any(|fetch| {
fetch.key == handler::Key::Block(digest)
&& fetch.subscriber
== handler::Annotation::Finalized(handler::Finalized::ByRound { round })
}));
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Block(digest),
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByRound {
round
}),
tracing::Span::none(),
)),
},
value: repaired.encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("marshal must answer the delivery"),
Outcome::Resolved(Resolution::Complete)
);
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert!(mailbox.get_finalization(Height::new(11)).await.is_none());
assert_eq!(application.pending_ack_heights(), vec![Height::new(21)]);
handle.abort();
let _ = handle.await;
restart_with_retired_floor_resumes_at(
&context,
partition_prefix,
late,
Height::new(21),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_reproposal_delivered_as_pending_floor_anchor_survives_restart_before_ack() {
let partition_prefix = "reproposal-delivered-pending-anchor-restart-before-ack";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let boundary = BoundaryBlock::new(&mut context);
let pending = boundary.finalize(30);
let reproposal = boundary.finalize(40);
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("pending"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(boundary.scheme()),
pending.clone(),
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let handle =
actor.start_unbuffered(application.clone(), (resolver_rx, resolver.clone()));
assert_eq!(mailbox.get_processed().await, None);
let digest = boundary.block.digest();
assert!(resolver.fetches().iter().any(|fetch| {
fetch.key == handler::Key::Block(digest)
&& fetch.subscriber
== handler::Annotation::Finalized(handler::Finalized::ByRound {
round: pending.round(),
})
}));
assert!(application.pending_ack_heights().is_empty());
let height = Height::new(10);
let (response, response_rx) = oneshot::channel();
assert!(
resolver
.enqueue(handler::Message::Deliver {
delivery: Delivery {
key: handler::Key::Finalized { height },
subscribers: NonEmptyVec::new((
handler::Annotation::Finalized(handler::Finalized::ByHeight {
height
}),
tracing::Span::none(),
)),
},
value: (reproposal.clone(), boundary.block.clone()).encode(),
response,
})
.accepted()
);
assert_eq!(
response_rx.await.expect("marshal must answer the delivery"),
Outcome::Resolved(Resolution::Complete)
);
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(9)))
);
assert_eq!(application.pending_ack_heights(), vec![Height::new(10)]);
handle.abort();
let _ = handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, _mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, EmptyProvider, reproposal.clone()),
)
.await;
assert!(restored.cutoff().covers(reproposal.round()));
let (resolver_rx, resolver) =
RecordingResolver::holding(context.child("restarted_resolver"));
let application = Application::<B>::manual_ack();
let _handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
assert_eq!(application.acknowledged().await, Height::new(10));
});
}
#[test_traced("WARN")]
fn test_standard_reproposal_reported_for_pruned_block_is_covered_without_fetch() {
let partition_prefix = "reproposal-reported-pruned-covered";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let scheme = run.scheme();
let boundary = run.block(10);
let first = run.finalize(10, 10);
let reproposal = run.finalize(10, 40);
run.handle.abort();
let _ = run.handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mut mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("pruned"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, ConstantProvider::new(scheme), first),
)
.await;
assert!(restored.cutoff().covers(reproposal.round()));
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let handle =
actor.start_unbuffered(application.clone(), (resolver_rx, resolver.clone()));
mailbox.prune(Height::new(20));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert!(mailbox.get_block(Height::new(10)).await.is_none());
StandardHarness::report_finalization(&mut mailbox, reproposal.clone()).await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
let digest = boundary.digest();
assert!(
resolver
.fetches()
.iter()
.all(|fetch| fetch.key != handler::Key::Block(digest))
);
assert_eq!(application.pending_ack_heights(), vec![Height::new(21)]);
handle.abort();
let _ = handle.await;
restart_with_retired_floor_resumes_at(
&context,
partition_prefix,
reproposal,
Height::new(21),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_set_floor_for_installed_anchor_is_covered() {
let partition_prefix = "set-floor-installed-anchor-covered";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let boundary = BoundaryBlock::new(&mut context);
let first = boundary.finalize(10);
let reproposal = boundary.finalize(40);
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("first"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(boundary.scheme()),
first,
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let buffer = RecordingBuffer::default();
buffer.insert(boundary.block.clone());
let application = Application::<B>::manual_ack();
let _handle = actor.start(application.clone(), buffer, (resolver_rx, resolver));
wait_until(&context, Duration::from_secs(5), "floor dispatch", || {
application.pending_ack_heights() == [Height::new(10)]
})
.await;
mailbox.set_floor(reproposal);
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(9)))
);
assert_eq!(application.pending_ack_heights(), vec![Height::new(10)]);
});
}
async fn restored_closed_epoch_floor_is_written_back(
context: &mut deterministic::Context,
partition_prefix: &str,
round: Option<Round>,
) {
let run = start_reproposal_processed_through_20(context, partition_prefix).await;
let reproposal = run.finalize(10, 40);
run.handle.abort();
let _ = run.handle.await;
seed_processed_height(
context.child("metadata"),
partition_prefix,
Height::new(20),
round,
)
.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(context, partition_prefix).await;
let (actor, mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("derived"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, EmptyProvider, reproposal.clone()),
)
.await;
assert!(restored.cutoff().covers(reproposal.round()));
drop((actor, mailbox));
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(context, partition_prefix).await;
let (_actor, _mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("persisted"),
certificates,
finalized,
with_epocher(
reproposal_floor_config(partition_prefix, EmptyProvider, reproposal.clone()),
RetiredThrough {
epocher: FixedEpocher::new(REPROPOSAL_EPOCH_LENGTH),
retired: Height::new(20),
},
),
)
.await;
assert!(
restored.cutoff().covers(reproposal.round()),
"the epoch cutoff derived on the first start must be persisted: {}",
restored.cutoff()
);
}
#[test_traced("WARN")]
fn test_standard_restored_closed_epoch_floor_is_written_back() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
restored_closed_epoch_floor_is_written_back(
&mut context,
"closed-epoch-floor-written-back",
None,
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_restored_closed_epoch_floor_is_written_back_over_round_floor() {
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
restored_closed_epoch_floor_is_written_back(
&mut context,
"closed-epoch-floor-written-back-over-round-floor",
Some(Round::new(Epoch::zero(), View::new(10))),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_closed_epoch_floor_restores_from_unacknowledged_successor() {
let partition_prefix = "closed-epoch-floor-unacknowledged-successor";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let boundary_late = run.finalize(21, 25);
let boundary_reproposal = run.finalize(21, 50);
let ReproposalRun {
handle,
mut mailbox,
application,
..
} = run;
StandardHarness::report_finalization(&mut mailbox, boundary_late).await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert_eq!(application.pending_ack_heights(), vec![Height::new(21)]);
handle.abort();
let _ = handle.await;
seed_processed_height(
context.child("metadata"),
partition_prefix,
Height::new(20),
None,
)
.await;
restart_with_retired_floor_resumes_at(
&context,
partition_prefix,
boundary_reproposal,
Height::new(21),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_certificate_only_successor_closes_no_epoch() {
let partition_prefix = "certificate-only-successor-closes-no-epoch";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let boundary = BoundaryBlock::new(&mut context);
let first = boundary.finalize(10);
seed_processed_height(
context.child("metadata"),
partition_prefix,
Height::new(9),
None,
)
.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let certificates = certificates
.put(10, boundary.block.digest(), &first)
.await
.expect("seeded finalization must be stored")
.sync()
.await
.expect("seeded finalization must sync");
let (actor, mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(boundary.scheme()),
first.clone(),
),
)
.await;
assert!(!restored.cutoff().covers(first.round()));
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let _handle = actor.start_unbuffered(application, (resolver_rx, resolver.clone()));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(9)))
);
assert!(mailbox.get_finalization(Height::new(10)).await.is_some());
assert!(mailbox.get_block(Height::new(10)).await.is_none());
let digest = boundary.block.digest();
assert!(resolver.fetches().iter().any(|fetch| {
fetch.key == handler::Key::Block(digest)
&& fetch.subscriber
== handler::Annotation::Finalized(handler::Finalized::ByRound {
round: first.round(),
})
}));
});
}
#[test_traced("WARN")]
fn test_standard_closed_epoch_floor_restores_from_installed_anchor() {
let partition_prefix = "closed-epoch-floor-installed-anchor";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let boundary = BoundaryBlock::new(&mut context);
let first = boundary.finalize(10);
let reproposal = boundary.finalize(40);
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("first"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(boundary.scheme()),
first,
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let buffer = RecordingBuffer::default();
buffer.insert(boundary.block.clone());
let application = Application::<B>::manual_ack();
let handle = actor.start(application.clone(), buffer, (resolver_rx, resolver));
wait_until(&context, Duration::from_secs(5), "floor dispatch", || {
application.pending_ack_heights() == [Height::new(10)]
})
.await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(9)))
);
handle.abort();
let _ = handle.await;
seed_processed_height(
context.child("metadata"),
partition_prefix,
Height::new(9),
None,
)
.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, _mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, EmptyProvider, reproposal.clone()),
)
.await;
assert!(restored.cutoff().covers(reproposal.round()));
let (resolver_rx, resolver) =
RecordingResolver::holding(context.child("restarted_resolver"));
let application = Application::<B>::manual_ack();
let _handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
assert_eq!(application.acknowledged().await, Height::new(10));
});
}
#[test_traced("WARN")]
fn test_standard_cache_prunes_behind_covered_reproposals() {
let partition_prefix = "cache-prunes-behind-covered-reproposals";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let scheme = run.scheme();
let first = run.finalize(10, 10);
let reproposals: Vec<_> = [40, 60, 80].map(|view| run.finalize(10, view)).into();
run.handle.abort();
let _ = run.handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mut mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, ConstantProvider::new(scheme), first),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let handle = actor.start_unbuffered(application, (resolver_rx, resolver));
let pruned = |context: &deterministic::Context| -> u64 {
context
.encode()
.lines()
.filter(|line| {
line.contains("cache_finalizations_indices_pruned_total")
&& line.contains("epoch=\"0\"")
})
.filter_map(|line| line.split_whitespace().last()?.parse::<u64>().ok())
.sum()
};
let before = pruned(&context);
for reproposal in reproposals {
StandardHarness::report_finalization(&mut mailbox, reproposal).await;
}
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert_eq!(
pruned(&context),
before + 2,
"the cache must prune exactly the views behind the previously witnessed round"
);
handle.abort();
let _ = handle.await;
});
}
#[test_traced("WARN")]
fn test_standard_restored_witnessed_round_prunes_cache_after_restart() {
let partition_prefix = "restored-witnessed-round-prunes-cache";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let scheme = run.scheme();
let first = run.finalize(10, 10);
let early = run.finalize(11, 11);
let boundary = run.finalize(21, 50);
let reproposal = run.finalize(21, 70);
let ReproposalRun {
handle,
mut mailbox,
application,
..
} = run;
StandardHarness::report_finalization(&mut mailbox, early).await;
StandardHarness::report_finalization(&mut mailbox, boundary).await;
assert_eq!(application.acknowledge_next(), Some(Height::new(21)));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(21)))
);
handle.abort();
let _ = handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mut mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, ConstantProvider::new(scheme), first),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let handle = actor.start_unbuffered(application, (resolver_rx, resolver));
let pruned = |context: &deterministic::Context| -> u64 {
context
.encode()
.lines()
.filter(|line| {
line.contains("cache_finalizations_indices_pruned_total")
&& line.contains("epoch=\"1\"")
})
.filter_map(|line| line.split_whitespace().last()?.parse::<u64>().ok())
.sum()
};
let before = pruned(&context);
StandardHarness::report_finalization(&mut mailbox, reproposal).await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(21)))
);
assert_eq!(
pruned(&context),
before + 1,
"the cache must prune the views behind the restored witnessed round"
);
handle.abort();
let _ = handle.await;
});
}
async fn epoch_zero_retention(
context: &mut deterministic::Context,
partition_prefix: &str,
epoch_length: NonZeroU64,
blocks: u64,
) -> Option<Round> {
let Fixture { schemes, .. } =
bls12381_threshold_vrf::fixture::<V, _>(context, NAMESPACE, NUM_VALIDATORS);
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(context, partition_prefix).await;
let (actor, mut mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("actor"),
certificates,
finalized,
Config {
epocher: FixedEpocher::new(epoch_length),
..test_config(
partition_prefix,
ConstantProvider::new(schemes[0].clone()),
NZUsize!(1),
)
},
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let application = Application::<B>::manual_ack();
let _handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
assert_eq!(application.acknowledged().await, Height::zero());
let epocher = FixedEpocher::new(epoch_length);
let boundary = epocher.last(Epoch::zero()).expect("epoch zero is covered");
let boundary_round = Round::new(Epoch::zero(), View::new(boundary.get()));
let mut parent = StandardHarness::genesis_block(NUM_VALIDATORS as u16).digest();
let mut last_kept = None;
let mut gone = false;
for height in 1..=blocks {
let block = make_raw_block(parent, Height::new(height), height * 100);
parent = block.digest();
let info = epocher
.containing(Height::new(height))
.expect("height is covered");
let offset = height - info.first().get();
let view = View::new(if info.epoch().is_zero() {
offset
} else {
offset + 1
});
let round = Round::new(info.epoch(), view);
assert!(mailbox.verified(round, ready(block.clone())).await);
let finalization = StandardHarness::make_finalization(
Proposal::new(
round,
view.previous().expect("view is above zero"),
block.digest(),
),
&schemes,
QUORUM,
);
StandardHarness::report_finalization(&mut mailbox, finalization).await;
assert_eq!(application.acknowledged().await, Height::new(height));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(height)))
);
if height < boundary.get() {
continue;
}
if mailbox.get_verified(boundary_round).await.is_some() {
assert!(
!gone,
"epoch-zero cache must not reappear after it was destroyed, at {round}"
);
last_kept = Some(round);
} else {
gone = true;
}
}
last_kept
}
#[test_traced("WARN")]
fn test_standard_cache_keeps_previous_epoch_through_retention_window() {
let runner = deterministic::Runner::timed(Duration::from_secs(60));
runner.start(|mut context| async move {
let kept = epoch_zero_retention(
&mut context,
"cache-keeps-previous-epoch",
BLOCKS_PER_EPOCH,
31,
)
.await;
assert_eq!(kept, Some(Round::new(Epoch::new(1), View::new(10))));
});
}
#[test_traced("WARN")]
fn test_standard_cache_retention_counts_short_epochs() {
let runner = deterministic::Runner::timed(Duration::from_secs(60));
runner.start(|mut context| async move {
let kept =
epoch_zero_retention(&mut context, "cache-retention-short-epochs", NZU64!(3), 30)
.await;
assert_eq!(kept, Some(Round::new(Epoch::new(8), View::new(3))));
});
}
#[test_traced("WARN")]
fn test_standard_reproposal_reported_above_processed_before_ack_survives_restart() {
let partition_prefix = "reproposal-reported-above-processed-before-ack";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let boundary_late = run.finalize(21, 25);
let boundary_reproposal = run.finalize(21, 50);
let ReproposalRun {
handle,
mut mailbox,
resolver,
application,
..
} = run;
StandardHarness::report_finalization(&mut mailbox, boundary_late).await;
StandardHarness::report_finalization(&mut mailbox, boundary_reproposal.clone()).await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert_eq!(application.pending_ack_heights(), vec![Height::new(21)]);
assert_eq!(application.acknowledge_next(), Some(Height::new(21)));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(21)))
);
let covered = Round::new(Epoch::new(1), View::new(60));
mailbox.hint_notarized(covered, Sha256::hash(&[b"covered"]));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(21)))
);
assert!(
resolver
.fetches()
.iter()
.all(|fetch| fetch.key != handler::Key::Notarized { round: covered })
);
handle.abort();
let _ = handle.await;
restart_with_retired_floor_resumes_at(
&context,
partition_prefix,
boundary_reproposal,
Height::new(22),
)
.await;
});
}
#[test_traced("WARN")]
fn test_standard_reproposal_reported_for_installed_anchor_before_ack_survives_restart() {
let partition_prefix = "reproposal-reported-installed-anchor-before-ack";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let boundary = BoundaryBlock::new(&mut context);
let first = boundary.finalize(10);
let reproposal = boundary.finalize(40);
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mut mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("first"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(boundary.scheme()),
first,
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let buffer = RecordingBuffer::default();
buffer.insert(boundary.block.clone());
let application = Application::<B>::manual_ack();
let handle = actor.start(application.clone(), buffer, (resolver_rx, resolver.clone()));
wait_until(&context, Duration::from_secs(5), "floor dispatch", || {
application.pending_ack_heights() == [Height::new(10)]
})
.await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(9)))
);
StandardHarness::report_finalization(&mut mailbox, reproposal.clone()).await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(9)))
);
assert_eq!(application.acknowledge_next(), Some(Height::new(10)));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(10)))
);
let covered = Round::new(Epoch::zero(), View::new(60));
mailbox.hint_notarized(covered, Sha256::hash(&[b"covered"]));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(10)))
);
assert!(
resolver
.fetches()
.iter()
.all(|fetch| fetch.key != handler::Key::Notarized { round: covered })
);
handle.abort();
let _ = handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, EmptyProvider, reproposal.clone()),
)
.await;
assert!(
restored.cutoff().covers(reproposal.round()),
"restart must restore a round floor covering the re-proposal: {:?}",
restored.cutoff()
);
let (resolver_rx, resolver) =
RecordingResolver::holding(context.child("restarted_resolver"));
let application = Application::<B>::manual_ack();
let _handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(10)))
);
});
}
#[test_traced("WARN")]
fn test_standard_reproposal_floor_above_processed_is_covered_on_restart() {
let partition_prefix = "reproposal-floor-above-processed-restart";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let boundary = BoundaryBlock::new(&mut context);
let first = boundary.finalize(10);
let reproposal = boundary.finalize(40);
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("first"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(boundary.scheme()),
first,
),
)
.await;
let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
let buffer = RecordingBuffer::default();
buffer.insert(boundary.block.clone());
let application = Application::<B>::manual_ack();
let handle = actor.start(application.clone(), buffer, (resolver_rx, resolver));
wait_until(&context, Duration::from_secs(5), "floor dispatch", || {
application.pending_ack_heights() == [Height::new(10)]
})
.await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(9)))
);
handle.abort();
let _ = handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, mailbox, restored) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("reproposal"),
certificates,
finalized,
reproposal_floor_config(
partition_prefix,
ConstantProvider::new(boundary.scheme()),
reproposal.clone(),
),
)
.await;
assert!(restored.cutoff().covers(reproposal.round()));
let (resolver_rx, resolver) =
RecordingResolver::holding(context.child("reproposal_resolver"));
let application = Application::<B>::manual_ack();
let handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
wait_until(&context, Duration::from_secs(5), "floor redispatch", || {
application.pending_ack_heights() == [Height::new(10)]
})
.await;
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Absent(Height::new(9)))
);
handle.abort();
let _ = handle.await;
let (certificates, finalized) =
prunable_finalized_stores::<Arc<B>>(&context, partition_prefix).await;
let (actor, _mailbox, _) = Actor::<_, Standard<B>, _, _, _, _, _>::init(
context.child("restarted"),
certificates,
finalized,
reproposal_floor_config(partition_prefix, EmptyProvider, reproposal),
)
.await;
let (resolver_rx, resolver) =
RecordingResolver::holding(context.child("restarted_resolver"));
let application = Application::<B>::manual_ack();
let _handle = actor.start_unbuffered(application.clone(), (resolver_rx, resolver));
assert_eq!(application.acknowledged().await, Height::new(10));
});
}
#[test_traced("WARN")]
fn test_standard_ack_of_certificate_less_boundary_block_closes_its_epoch() {
let partition_prefix = "ack-certificate-less-boundary-block";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let ReproposalRun {
handle,
mailbox,
resolver,
application,
..
} = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
assert_eq!(application.acknowledge_next(), Some(Height::new(21)));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(21)))
);
assert!(mailbox.get_finalization(Height::new(21)).await.is_none());
let covered = Round::new(Epoch::new(1), View::new(60));
mailbox.hint_notarized(covered, Sha256::hash(&[b"covered"]));
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(21)))
);
assert!(
resolver
.fetches()
.iter()
.all(|fetch| fetch.key != handler::Key::Notarized { round: covered }),
"acknowledging the certificate-less boundary block must close its epoch"
);
handle.abort();
let _ = handle.await;
});
}
#[test_traced("WARN")]
fn test_standard_floor_applied_below_processed_in_open_epoch_survives_restart_before_ack() {
let partition_prefix = "floor-below-processed-open-epoch";
let runner = deterministic::Runner::timed(Duration::from_secs(30));
runner.start(|mut context| async move {
let run = start_reproposal_processed_through_20(&mut context, partition_prefix).await;
let floor = run.finalize(15, 15);
let ReproposalRun {
handle,
mailbox,
application,
..
} = run;
mailbox.set_floor(floor.clone());
assert_eq!(
mailbox.get_processed().await,
Some(Processed::Block(Height::new(20)))
);
assert_eq!(application.pending_ack_heights(), vec![Height::new(21)]);
handle.abort();
let _ = handle.await;
restart_with_retired_floor_resumes_at(
&context,
partition_prefix,
floor,
Height::new(21),
)
.await;
});
}
}