Skip to main content

commonware_consensus/marshal/coding/
mod.rs

1//! Ordered delivery of erasure-coded blocks.
2//!
3//! # Overview
4//!
5//! The coding marshal couples the consensus pipeline with erasure-coded block broadcast.
6//! Blocks are produced by an application, encoded into [`types::Shard`]s, fanned out to peers, and
7//! later reconstructed when a notarization or finalization proves that the data is needed.
8//! Compared to [`super::standard`], this variant makes more efficient usage of the network's bandwidth
9//! by spreading the load of block dissemination across all participants.
10//!
11//! # Components
12//!
13//! - [`crate::marshal::core::Actor`]: The unified marshal actor that orders finalized blocks,
14//!   handles acknowledgements from the application, and requests repairs when gaps are detected.
15//!   Used with [`Coding`] as the variant type parameter.
16//! - [`crate::marshal::core::Mailbox`]: Accepts requests from other local subsystems and forwards
17//!   them to the actor. Used with [`Coding`] as the variant type parameter.
18//! - [`shards::Engine`]: Broadcasts shards, verifies locally held fragments, and reconstructs
19//!   entire [`types::CodedBlock`]s on demand.
20//! - [`crate::marshal::resolver`]: Issues outbound fetches to remote peers when marshal is missing
21//!   a block, notarization, or finalization referenced by consensus.
22//! - [`types`]: Defines commitments, distribution shards, and helper builders used across the
23//!   module.
24//! - [`Marshaled`]: Wraps an [`crate::Application`] implementation so it automatically enforces
25//!   epoch boundaries and performs erasure encoding before a proposal leaves the application.
26//!
27//! # Data Flow
28//!
29//! 1. The application produces a block through [`Marshaled`], which encodes the payload and
30//!    obtains a [`crate::types::coding::Commitment`] describing the shard layout.
31//! 2. The block is broadcast via [`shards::Engine`]; each participant receives exactly one shard
32//!    and reshares it to everyone else once it verifies the fragment.
33//! 3. The actor ingests notarizations/finalizations from `simplex`, pulls reconstructed blocks
34//!    from the shard engine or backfills them through [`crate::marshal::resolver`], and durably
35//!    persists the ordered data.
36//! 4. The actor reports finalized blocks to the node's [`crate::Reporter`] at-least-once and
37//!    drives repair loops whenever notarizations reference yet-to-be-delivered payloads.
38//!
39//! # Storage and Repair
40//!
41//! Notarized data and certificates live in prunable archives managed internally, while finalized
42//! blocks are migrated into immutable archives. Any gaps are filled by asking peers for specific
43//! commitments through the resolver pipeline. The shard engine keeps only ephemeral, in-memory
44//! caches; once a block is finalized it is evicted from the reconstruction map, reducing memory
45//! pressure.
46//!
47//! # When to Use
48//!
49//! Choose this module when the consensus deployment wants erasure-coded dissemination with the
50//! same ordering guarantees provided by [`super::standard`]. The API is a breaking change from
51//! the standard marshal: applications must adapt to the coding-specific variant type and buffer
52//! implementation required by this module.
53
54pub mod shards;
55pub mod types;
56pub(crate) mod validation;
57
58mod variant;
59pub use variant::Coding;
60
61mod marshaled;
62pub use marshaled::{Marshaled, MarshaledConfig};
63
64#[cfg(test)]
65mod tests {
66    use crate::{
67        Automaton, Block, CertifiableAutomaton, CertifiableBlock, Relay,
68        marshal::{
69            ancestry::BlockProvider,
70            coding::{
71                Coding, Marshaled, MarshaledConfig, shards,
72                types::{CodedBlock, coding_config_for_participants, hash_context},
73            },
74            config::{Config, Start},
75            core,
76            mocks::{
77                application::Application,
78                harness::{
79                    self, BLOCKS_PER_EPOCH, CodingB, CodingCtx, CodingHarness, D, EmptyProvider, K,
80                    LINK, NAMESPACE, NUM_VALIDATORS, QUORUM, S, TEST_QUOTA, TestHarness,
81                    UNRELIABLE_LINK, V, default_leader, genesis_commitment, make_coding_block,
82                    setup_network_links, setup_network_with_participants,
83                },
84                verifying::{GatedVerifyingApp, MockVerifyingApp},
85            },
86            resolver::handler,
87        },
88        simplex::{
89            Plan, scheme::bls12381_threshold::vrf as bls12381_threshold_vrf, types::Proposal,
90        },
91        types::{Epoch, Epocher, FixedEpocher, Height, Round, View, ViewDelta, coding::Commitment},
92    };
93    use bytes::Bytes;
94    use commonware_actor::{Feedback, mailbox};
95    use commonware_codec::{Encode, FixedSize};
96    use commonware_coding::{CodecConfig, Config as CodingConfig, ReedSolomon};
97    use commonware_cryptography::{
98        Committable, Digestible, Hasher,
99        certificate::{ConstantProvider, Verifier as _, mocks::Fixture},
100        sha256::Sha256,
101    };
102    use commonware_macros::{select, test_group, test_traced};
103    use commonware_p2p::{Recipients, Sender as _};
104    use commonware_parallel::Sequential;
105    use commonware_resolver::{Delivery, Fetch, Resolver, TargetedResolver};
106    use commonware_runtime::{
107        Clock, Metrics, Runner, Supervisor as _, buffer::paged::CacheRef, deterministic,
108    };
109    use commonware_storage::archive::immutable;
110    use commonware_utils::{
111        NZU16, NZU64, NZUsize, channel::oneshot, sync::Mutex, vec::NonEmptyVec,
112    };
113    use std::{sync::Arc, time::Duration};
114
115    type TestCodingVariant = Coding<CodingB, ReedSolomon<Sha256>, Sha256, K>;
116    type TestCodedBlock = CodedBlock<CodingB, ReedSolomon<Sha256>, Sha256>;
117    type TestCommitment = Commitment<CodingB, ReedSolomon<Sha256>, Sha256>;
118    type CodingSendRecord = (Round, Arc<TestCodedBlock>, Recipients<K>);
119
120    // Smallest valid coding config used to build trusted genesis commitments.
121    const GENESIS_CODING_CONFIG: CodingConfig = CodingConfig {
122        minimum_shards: NZU16!(1),
123        extra_shards: NZU16!(1),
124    };
125
126    #[test]
127    fn mailbox_provides_application_blocks() {
128        fn assert_provider<P: BlockProvider<Block = CodingB>>() {}
129        assert_provider::<core::Mailbox<S, TestCodingVariant>>();
130    }
131
132    /// A coding buffer that records subscriptions and never resolves them.
133    #[derive(Clone, Default)]
134    struct RecordingCodingBuffer {
135        digest_subscriptions: Arc<Mutex<Vec<oneshot::Sender<Arc<TestCodedBlock>>>>>,
136        commitment_subscriptions: Arc<Mutex<Vec<oneshot::Sender<Arc<TestCodedBlock>>>>>,
137        sends: Arc<Mutex<Vec<CodingSendRecord>>>,
138    }
139
140    impl RecordingCodingBuffer {
141        fn subscription_count(&self) -> usize {
142            self.digest_subscriptions.lock().len() + self.commitment_subscriptions.lock().len()
143        }
144
145        fn commitment_subscription_count(&self) -> usize {
146            self.commitment_subscriptions.lock().len()
147        }
148    }
149
150    impl core::Buffer<TestCodingVariant> for RecordingCodingBuffer {
151        type PublicKey = K;
152
153        async fn find_by_digest(&self, _digest: D) -> Option<Arc<TestCodedBlock>> {
154            None
155        }
156
157        async fn find_by_commitment(
158            &self,
159            _commitment: TestCommitment,
160        ) -> Option<Arc<TestCodedBlock>> {
161            None
162        }
163
164        fn subscribe_by_digest(
165            &self,
166            _digest: D,
167        ) -> Option<oneshot::Receiver<Arc<TestCodedBlock>>> {
168            let (sender, receiver) = oneshot::channel();
169            self.digest_subscriptions.lock().push(sender);
170            Some(receiver)
171        }
172
173        fn subscribe_by_commitment(
174            &self,
175            _commitment: TestCommitment,
176        ) -> Option<oneshot::Receiver<Arc<TestCodedBlock>>> {
177            let (sender, receiver) = oneshot::channel();
178            self.commitment_subscriptions.lock().push(sender);
179            Some(receiver)
180        }
181
182        fn retire(&self, _update: core::Retirement<TestCommitment>) {}
183
184        fn send(&self, round: Round, block: Arc<TestCodedBlock>, recipients: Recipients<K>) {
185            self.sends.lock().push((round, block, recipients));
186        }
187    }
188
189    type CodingFetchRecord = Fetch<handler::Key<TestCommitment>, handler::Annotation>;
190    type CodingTargetedFetch = (handler::Key<TestCommitment>, NonEmptyVec<K>);
191
192    /// A resolver that records each fetch invocation; other methods are no-ops.
193    #[derive(Clone, Default)]
194    struct RecordingResolver {
195        fetches: Arc<Mutex<Vec<CodingFetchRecord>>>,
196        targeted: Arc<Mutex<Vec<CodingTargetedFetch>>>,
197        auto_delivery: Arc<Mutex<Option<Bytes>>>,
198        delivery_responses: Arc<Mutex<Vec<oneshot::Receiver<bool>>>>,
199        sender: Option<mailbox::Sender<handler::Message<TestCommitment>>>,
200    }
201
202    impl RecordingResolver {
203        fn holding(metrics: impl Metrics) -> (handler::Receiver<TestCommitment>, Self) {
204            let (sender, receiver) = mailbox::new(metrics, NZUsize!(100));
205            (
206                handler::Receiver::new(receiver),
207                Self {
208                    fetches: Arc::new(Mutex::new(Vec::new())),
209                    targeted: Arc::new(Mutex::new(Vec::new())),
210                    auto_delivery: Arc::new(Mutex::new(None)),
211                    delivery_responses: Arc::new(Mutex::new(Vec::new())),
212                    sender: Some(sender),
213                },
214            )
215        }
216
217        fn record_fetch(&self, fetch: CodingFetchRecord) {
218            self.fetches.lock().push(fetch.clone());
219            let Some(value) = self.auto_delivery.lock().take() else {
220                return;
221            };
222            let Some(sender) = &self.sender else {
223                return;
224            };
225            let (response, response_rx) = oneshot::channel();
226            self.delivery_responses.lock().push(response_rx);
227            let _ = sender.enqueue(handler::Message::Deliver {
228                delivery: Delivery {
229                    key: fetch.key,
230                    subscribers: NonEmptyVec::new((fetch.subscriber, tracing::Span::none())),
231                },
232                value,
233                response,
234            });
235        }
236
237        fn respond_to_next_fetch(&self, value: Bytes) {
238            let replaced = self.auto_delivery.lock().replace(value);
239            assert!(
240                replaced.is_none(),
241                "recording resolver already has an automatic delivery"
242            );
243        }
244
245        async fn wait_for_delivery_response(&self) -> bool {
246            let response = self
247                .delivery_responses
248                .lock()
249                .pop()
250                .expect("delivery response missing");
251            response.await.expect("delivery response sender dropped")
252        }
253
254        fn fetches(&self) -> Vec<CodingFetchRecord> {
255            self.fetches.lock().clone()
256        }
257
258        fn targeted(&self) -> Vec<CodingTargetedFetch> {
259            self.targeted.lock().clone()
260        }
261    }
262
263    impl Resolver for RecordingResolver {
264        type Key = handler::Key<TestCommitment>;
265        type Subscriber = handler::Annotation;
266
267        fn fetch<F>(&mut self, fetch: F) -> Feedback
268        where
269            F: Into<Fetch<Self::Key, Self::Subscriber>> + Send,
270        {
271            self.record_fetch(fetch.into());
272            Feedback::Ok
273        }
274
275        fn fetch_all<F>(&mut self, fetches: Vec<F>) -> Feedback
276        where
277            F: Into<Fetch<Self::Key, Self::Subscriber>> + Send,
278        {
279            for fetch in fetches {
280                self.record_fetch(fetch.into());
281            }
282            Feedback::Ok
283        }
284
285        fn retain(
286            &mut self,
287            _predicate: impl Fn(&Self::Key, &Self::Subscriber) -> bool + Send + 'static,
288        ) -> Feedback {
289            Feedback::Ok
290        }
291    }
292
293    impl TargetedResolver for RecordingResolver {
294        type PublicKey = K;
295
296        fn fetch_targeted(
297            &mut self,
298            fetch: impl Into<Fetch<Self::Key, Self::Subscriber>> + Send,
299            targets: NonEmptyVec<Self::PublicKey>,
300        ) -> Feedback {
301            self.targeted.lock().push((fetch.into().key, targets));
302            Feedback::Ok
303        }
304
305        fn fetch_all_targeted<F>(
306            &mut self,
307            fetches: Vec<(F, NonEmptyVec<Self::PublicKey>)>,
308        ) -> Feedback
309        where
310            F: Into<Fetch<Self::Key, Self::Subscriber>> + Send,
311        {
312            let mut targeted = self.targeted.lock();
313            for (fetch, targets) in fetches {
314                targeted.push((fetch.into().key, targets));
315            }
316            Feedback::Ok
317        }
318    }
319
320    async fn start_coding_actor_with_recording(
321        context: deterministic::Context,
322        partition_prefix: &str,
323        provider: ConstantProvider<S, Epoch>,
324        buffer: RecordingCodingBuffer,
325    ) -> (
326        core::Mailbox<S, TestCodingVariant>,
327        RecordingResolver,
328        commonware_runtime::Handle<()>,
329    ) {
330        let config = Config {
331            provider,
332            epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
333            start: Start::Genesis(CodingHarness::genesis_block(NUM_VALIDATORS as u16)),
334            mailbox_size: NZUsize!(100),
335            view_retention: ViewDelta::new(10),
336            max_repair: NZUsize!(10),
337            max_pending_acks: NZUsize!(1),
338            block_codec_config: (),
339            partition_prefix: partition_prefix.to_string(),
340            prunable_items_per_section: NZU64!(10),
341            replay_buffer: NZUsize!(1024),
342            key_write_buffer: NZUsize!(1024),
343            value_write_buffer: NZUsize!(1024),
344            page_cache: CacheRef::from_pooler(
345                &context,
346                harness::PAGE_SIZE,
347                harness::PAGE_CACHE_SIZE,
348            ),
349            strategy: Sequential,
350        };
351
352        let finalizations_by_height = immutable::Archive::init(
353            context.child("finalizations_by_height"),
354            immutable::Config {
355                metadata_partition: format!("{partition_prefix}-finalizations-by-height-metadata"),
356                freezer_table_partition: format!(
357                    "{partition_prefix}-finalizations-by-height-freezer-table"
358                ),
359                freezer_table_initial_size: 64,
360                freezer_table_resize_frequency: 10,
361                freezer_table_resize_chunk_size: 10,
362                freezer_key_partition: format!(
363                    "{partition_prefix}-finalizations-by-height-freezer-key"
364                ),
365                freezer_key_page_cache: config.page_cache.clone(),
366                freezer_value_partition: format!(
367                    "{partition_prefix}-finalizations-by-height-freezer-value"
368                ),
369                freezer_value_target_size: 1024,
370                freezer_value_compression: None,
371                ordinal_partition: format!("{partition_prefix}-finalizations-by-height-ordinal"),
372                items_per_section: NZU64!(10),
373                codec_config: S::certificate_codec_config_unbounded(),
374                replay_buffer: config.replay_buffer,
375                freezer_key_write_buffer: config.key_write_buffer,
376                freezer_value_write_buffer: config.value_write_buffer,
377                ordinal_write_buffer: config.key_write_buffer,
378            },
379        )
380        .await
381        .expect("failed to initialize finalizations by height archive");
382
383        let finalized_blocks = immutable::Archive::init(
384            context.child("finalized_blocks"),
385            immutable::Config {
386                metadata_partition: format!("{partition_prefix}-finalized_blocks-metadata"),
387                freezer_table_partition: format!(
388                    "{partition_prefix}-finalized_blocks-freezer-table"
389                ),
390                freezer_table_initial_size: 64,
391                freezer_table_resize_frequency: 10,
392                freezer_table_resize_chunk_size: 10,
393                freezer_key_partition: format!("{partition_prefix}-finalized_blocks-freezer-key"),
394                freezer_key_page_cache: config.page_cache.clone(),
395                freezer_value_partition: format!(
396                    "{partition_prefix}-finalized_blocks-freezer-value"
397                ),
398                freezer_value_target_size: 1024,
399                freezer_value_compression: None,
400                ordinal_partition: format!("{partition_prefix}-finalized_blocks-ordinal"),
401                items_per_section: NZU64!(10),
402                codec_config: config.block_codec_config,
403                replay_buffer: config.replay_buffer,
404                freezer_key_write_buffer: config.key_write_buffer,
405                freezer_value_write_buffer: config.value_write_buffer,
406                ordinal_write_buffer: config.key_write_buffer,
407            },
408        )
409        .await
410        .expect("failed to initialize finalized blocks archive");
411
412        let (actor, mailbox, _) = core::Actor::init(
413            context.child("actor"),
414            finalizations_by_height,
415            finalized_blocks,
416            config,
417        )
418        .await;
419        let (resolver_rx, resolver) = RecordingResolver::holding(context.child("resolver"));
420        let actor_handle = actor.start(
421            Application::<CodingB>::default(),
422            buffer,
423            (resolver_rx, resolver.clone()),
424        );
425        (mailbox, resolver, actor_handle)
426    }
427
428    async fn start_shard_mailbox(
429        context: deterministic::Context,
430        participants: Vec<K>,
431        provider: ConstantProvider<S, Epoch>,
432    ) -> shards::Mailbox<CodingB, ReedSolomon<Sha256>, Sha256, K> {
433        let me = participants[0].clone();
434        let oracle =
435            setup_network_with_participants(context.child("network"), NZUsize!(1), participants)
436                .await;
437        let control = oracle.control(me.clone());
438        let shard_config: shards::Config<_, _, _, _, _, Sha256, _, _> = shards::Config {
439            scheme_provider: provider,
440            blocker: control.clone(),
441            shard_codec_cfg: CodecConfig {
442                maximum_shard_size: 1024 * 1024,
443            },
444            block_codec_cfg: (),
445            strategy: Sequential,
446            mailbox_size: NZUsize!(10),
447            peer_buffer_size: NZUsize!(64),
448            background_channel_capacity: NZUsize!(1024),
449            peer_provider: oracle.manager(),
450        };
451        let (shard_engine, shard_mailbox) =
452            shards::Engine::new(context.child("shards"), shard_config);
453        let network = control.register(0, TEST_QUOTA).await.unwrap();
454        shard_engine.start(network);
455        shard_mailbox
456    }
457
458    fn genesis_block() -> CodingB {
459        let genesis_ctx = CodingCtx {
460            round: Round::zero(),
461            leader: default_leader(),
462            parent: (View::zero(), genesis_commitment()),
463        };
464        make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0)
465    }
466
467    fn genesis_coding_commitment(block: &CodingB) -> TestCommitment {
468        TestCommitment::from((
469            block.digest(),
470            block.digest(),
471            hash_context::<Sha256, _>(&block.context()),
472            GENESIS_CODING_CONFIG,
473        ))
474    }
475
476    fn missing_candidate(me: K) -> (CodingCtx, TestCodedBlock) {
477        let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
478        let genesis = genesis_block();
479        let genesis_parent_commitment = genesis_coding_commitment(&genesis);
480        let round = Round::new(Epoch::zero(), View::new(1));
481        let candidate_ctx = CodingCtx {
482            round,
483            leader: me,
484            parent: (View::zero(), genesis_parent_commitment),
485        };
486        let candidate =
487            make_coding_block(candidate_ctx.clone(), genesis.digest(), Height::new(1), 100);
488        let coded_candidate: TestCodedBlock =
489            CodedBlock::new(candidate, coding_config, &Sequential);
490        (candidate_ctx, coded_candidate)
491    }
492
493    #[test_traced("WARN")]
494    fn test_coding_batched_acks_retire_each_exact_commitment() {
495        let runner = deterministic::Runner::timed(Duration::from_secs(30));
496        runner.start(|mut context| async move {
497            let Fixture {
498                participants,
499                schemes,
500                ..
501            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
502            let mut oracle = setup_network_with_participants(
503                context.child("network"),
504                NZUsize!(1),
505                participants.clone(),
506            )
507            .await;
508            let mut setup = CodingHarness::setup_validator_with(
509                context.child("validator"),
510                &mut oracle,
511                participants[0].clone(),
512                ConstantProvider::new(schemes[0].clone()),
513                NZUsize!(2),
514                Application::manual_ack(),
515            )
516            .await;
517            assert_eq!(setup.application.acknowledged().await, Height::zero());
518
519            let mut parent = Sha256::hash(&[b""]);
520            let mut parent_commitment =
521                CodingHarness::genesis_parent_commitment(NUM_VALIDATORS as u16);
522            let mut commitments = Vec::new();
523            for height in 1..=2 {
524                let round = Round::new(Epoch::zero(), View::new(height));
525                let block = CodingHarness::make_test_block(
526                    parent,
527                    parent_commitment,
528                    Height::new(height),
529                    height,
530                    NUM_VALIDATORS as u16,
531                );
532                let commitment = block.commitment();
533                parent = block.digest();
534                parent_commitment = commitment;
535                commitments.push(commitment);
536
537                setup.extra.proposed(
538                    Round::new(Epoch::zero(), View::new(height + 10)),
539                    block.clone(),
540                );
541                assert!(setup.extra.get(commitment).await.is_some());
542                assert!(setup.mailbox.verified(round, block).await);
543                CodingHarness::report_finalization(
544                    &mut setup.mailbox,
545                    CodingHarness::make_finalization(
546                        Proposal {
547                            round,
548                            parent: View::new(height - 1),
549                            payload: commitment,
550                        },
551                        &schemes,
552                        QUORUM,
553                    ),
554                )
555                .await;
556            }
557
558            while setup.application.pending_ack_heights() != vec![Height::new(1), Height::new(2)] {
559                context.sleep(Duration::from_millis(10)).await;
560            }
561            assert_eq!(setup.application.acknowledge_next(), Some(Height::new(1)));
562            assert_eq!(setup.application.acknowledge_next(), Some(Height::new(2)));
563
564            while setup.extra.get(commitments[1]).await.is_some() {
565                context.sleep(Duration::from_millis(10)).await;
566            }
567            assert!(setup.extra.get(commitments[0]).await.is_none());
568        });
569    }
570
571    #[test_traced("WARN")]
572    fn test_coding_floor_retires_only_superseded_ack_commitments() {
573        let runner = deterministic::Runner::timed(Duration::from_secs(30));
574        runner.start(|mut context| async move {
575            let Fixture {
576                participants,
577                schemes,
578                ..
579            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
580            let mut oracle = setup_network_with_participants(
581                context.child("network"),
582                NZUsize!(1),
583                participants.clone(),
584            )
585            .await;
586            let mut setup = CodingHarness::setup_validator_with(
587                context.child("validator"),
588                &mut oracle,
589                participants[0].clone(),
590                ConstantProvider::new(schemes[0].clone()),
591                NZUsize!(3),
592                Application::manual_ack(),
593            )
594            .await;
595            assert_eq!(setup.application.acknowledged().await, Height::zero());
596
597            let mut parent = Sha256::hash(&[b""]);
598            let mut parent_commitment =
599                CodingHarness::genesis_parent_commitment(NUM_VALIDATORS as u16);
600            let mut commitments = Vec::new();
601            let mut floor = None;
602            for height in 1..=3 {
603                let round = Round::new(Epoch::zero(), View::new(height));
604                let block = CodingHarness::make_test_block(
605                    parent,
606                    parent_commitment,
607                    Height::new(height),
608                    height * 100,
609                    NUM_VALIDATORS as u16,
610                );
611                let commitment = block.commitment();
612                parent = block.digest();
613                parent_commitment = commitment;
614                commitments.push(commitment);
615
616                // Every cache observation is newer than the floor, so only an exact
617                // commitment retirement can remove it.
618                setup.extra.proposed(
619                    Round::new(Epoch::zero(), View::new(height + 10)),
620                    block.clone(),
621                );
622                assert!(setup.mailbox.verified(round, block).await);
623                let finalization = CodingHarness::make_finalization(
624                    Proposal {
625                        round,
626                        parent: View::new(height - 1),
627                        payload: commitment,
628                    },
629                    &schemes,
630                    QUORUM,
631                );
632                if height == 2 {
633                    floor = Some(finalization.clone());
634                }
635                CodingHarness::report_finalization(&mut setup.mailbox, finalization).await;
636            }
637
638            while setup.application.pending_ack_heights()
639                != vec![Height::new(1), Height::new(2), Height::new(3)]
640            {
641                context.sleep(Duration::from_millis(10)).await;
642            }
643
644            setup
645                .mailbox
646                .set_floor(floor.expect("height 2 floor missing"));
647
648            // The height-2 floor makes height 1 durable application progress. Heights 2 and 3
649            // are re-dispatched, so their coding-buffer commitments remain live.
650            while setup.mailbox.get_processed_height().await != Some(Height::new(1)) {
651                context.sleep(Duration::from_millis(10)).await;
652            }
653            assert!(setup.extra.get(commitments[0]).await.is_none());
654            assert!(setup.extra.get(commitments[1]).await.is_some());
655            assert!(setup.extra.get(commitments[2]).await.is_some());
656        });
657    }
658
659    #[test_traced("WARN")]
660    fn test_coding_notarized_delivery_rejects_dishonest_payload_config() {
661        let runner = deterministic::Runner::timed(Duration::from_secs(30));
662        runner.start(|mut context| async move {
663            let Fixture {
664                participants,
665                schemes,
666                ..
667            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
668            let provider = ConstantProvider::new(schemes[0].clone());
669            let honest_config = coding_config_for_participants(NUM_VALIDATORS as u16);
670            let dishonest_config = coding_config_for_participants((NUM_VALIDATORS + 3) as u16);
671            assert_ne!(honest_config, dishonest_config);
672
673            let buffer = RecordingCodingBuffer::default();
674            let (marshal, resolver, _actor_handle) = start_coding_actor_with_recording(
675                context.child("actor_stack"),
676                "coding-dishonest-payload-config",
677                provider,
678                buffer,
679            )
680            .await;
681            let resolver_tx = resolver
682                .sender
683                .clone()
684                .expect("recording resolver should keep its sender");
685
686            let genesis = genesis_block();
687            let round = Round::new(Epoch::zero(), View::new(1));
688            let height = Height::new(1);
689            let candidate_ctx = CodingCtx {
690                round,
691                leader: participants[0].clone(),
692                parent: (View::zero(), genesis_coding_commitment(&genesis)),
693            };
694            let candidate = make_coding_block(candidate_ctx, genesis.digest(), height, 100);
695            let dishonest_block: TestCodedBlock =
696                CodedBlock::new(candidate.clone(), dishonest_config, &Sequential);
697            let proposal = Proposal {
698                round,
699                parent: View::zero(),
700                payload: dishonest_block.commitment(),
701            };
702            let notarization = CodingHarness::make_notarization(proposal, &schemes, QUORUM);
703
704            let (response, response_rx) = oneshot::channel();
705            assert!(
706                resolver_tx
707                    .enqueue(handler::Message::Deliver {
708                        delivery: Delivery {
709                            key: handler::Key::Notarized { round },
710                            subscribers: NonEmptyVec::new((
711                                handler::Annotation::Notarization { round },
712                                tracing::Span::none(),
713                            )),
714                        },
715                        value: (notarization, dishonest_block).encode(),
716                        response,
717                    })
718                    .accepted()
719            );
720            assert!(
721                !response_rx.await.unwrap(),
722                "notarized delivery should reject a dishonest coding config"
723            );
724
725            context.sleep(Duration::from_millis(100)).await;
726            assert!(
727                marshal.get_block(height).await.is_none(),
728                "dishonest deliveries must not store a finalized block"
729            );
730            assert!(
731                marshal.get_finalization(height).await.is_none(),
732                "dishonest deliveries must not archive a finalization"
733            );
734        });
735    }
736
737    #[test_traced("WARN")]
738    fn test_coding_block_provider_parent_fetches_by_commitment() {
739        let runner = deterministic::Runner::timed(Duration::from_secs(30));
740        runner.start(|mut context| async move {
741            let Fixture {
742                participants,
743                schemes,
744                ..
745            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
746            let provider = ConstantProvider::new(schemes[0].clone());
747            let buffer = RecordingCodingBuffer::default();
748            let (marshal, _resolver, _actor_handle) = start_coding_actor_with_recording(
749                context.child("actor_stack"),
750                "coding-provider-parent-commitment",
751                provider,
752                buffer.clone(),
753            )
754            .await;
755
756            let (parent_ctx, parent) = missing_candidate(participants[0].clone());
757            let child_ctx = CodingCtx {
758                round: Round::new(Epoch::zero(), View::new(2)),
759                leader: participants[0].clone(),
760                parent: (parent_ctx.round.view(), parent.commitment()),
761            };
762            let child = make_coding_block(child_ctx, parent.digest(), Height::new(2), 200);
763            let subscription = marshal.subscribe_parent(&child);
764
765            context.sleep(Duration::from_millis(100)).await;
766            assert_eq!(
767                buffer.commitment_subscription_count(),
768                1,
769                "parent walkback should use the coding parent commitment"
770            );
771            drop(subscription);
772        });
773    }
774
775    #[test_traced("WARN")]
776    fn test_coding_verify_missing_candidate_waits_without_fetching() {
777        let runner = deterministic::Runner::timed(Duration::from_secs(30));
778        runner.start(|mut context| async move {
779            let Fixture {
780                participants,
781                schemes,
782                ..
783            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
784            let provider = ConstantProvider::new(schemes[0].clone());
785            let me = participants[0].clone();
786            let buffer = RecordingCodingBuffer::default();
787            let (marshal, resolver, _actor_handle) = start_coding_actor_with_recording(
788                context.child("actor_stack"),
789                "coding-verify-missing-candidate",
790                provider.clone(),
791                buffer.clone(),
792            )
793            .await;
794            let shards =
795                start_shard_mailbox(context.child("shard_stack"), participants, provider.clone())
796                    .await;
797            let (candidate_ctx, candidate) = missing_candidate(me);
798            let commitment = candidate.commitment();
799
800            let cfg = MarshaledConfig {
801                application: MockVerifyingApp::<CodingB, S>::new(),
802                marshal,
803                shards,
804                scheme_provider: provider,
805                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
806                strategy: Sequential,
807            };
808            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
809
810            let verify_rx = marshaled.verify(candidate_ctx, commitment).await;
811            context.sleep(Duration::from_millis(100)).await;
812
813            assert!(
814                buffer.subscription_count() > 0,
815                "missing candidate should register a local buffer wait"
816            );
817            assert!(
818                resolver.fetches().is_empty(),
819                "missing candidate verify must not fetch from peers"
820            );
821            assert!(
822                resolver.targeted().is_empty(),
823                "missing candidate verify must not issue targeted fetches"
824            );
825            drop(verify_rx);
826        });
827    }
828
829    /// Crash-recovery shape: after an unclean shutdown, Simplex may recover a
830    /// notarized commitment while marshal has no local certification gate task and no
831    /// durable block. If enough shards or a peer response can provide the block,
832    /// certification should fetch it by notarized round and persist it instead
833    /// of treating the missing local copy as a hard failure.
834    #[test_traced("WARN")]
835    fn test_coding_certify_missing_candidate_fetches_by_round() {
836        let runner = deterministic::Runner::timed(Duration::from_secs(30));
837        runner.start(|mut context| async move {
838            let Fixture {
839                participants,
840                schemes,
841                ..
842            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
843            let provider = ConstantProvider::new(schemes[0].clone());
844            let me = participants[0].clone();
845            let buffer = RecordingCodingBuffer::default();
846            let (marshal, resolver, _actor_handle) = start_coding_actor_with_recording(
847                context.child("actor_stack"),
848                "coding-certify-missing-candidate",
849                provider.clone(),
850                buffer.clone(),
851            )
852            .await;
853            let shards =
854                start_shard_mailbox(context.child("shard_stack"), participants, provider.clone())
855                    .await;
856
857            let cfg = MarshaledConfig {
858                application: MockVerifyingApp::<CodingB, S>::new(),
859                marshal,
860                shards,
861                scheme_provider: provider,
862                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
863                strategy: Sequential,
864            };
865            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
866
867            let (candidate_ctx, candidate) = missing_candidate(me);
868            let commitment = candidate.commitment();
869            let round = candidate_ctx.round;
870            let proposal = Proposal::new(round, View::zero(), commitment);
871            let notarization = CodingHarness::make_notarization(proposal, &schemes, QUORUM);
872            resolver.respond_to_next_fetch((notarization, candidate).encode());
873            let certify_rx = marshaled.certify(round, commitment).await;
874
875            let result = certify_rx.await.expect("certify result missing");
876            assert!(result, "fetched notarized candidate should certify");
877            assert!(
878                resolver.wait_for_delivery_response().await,
879                "notarized delivery should validate"
880            );
881            assert!(
882                resolver.fetches().iter().any(|fetch| matches!(
883                    (&fetch.key, &fetch.subscriber),
884                    (
885                        handler::Key::Notarized { round: request_round },
886                        handler::Annotation::Notarization { round: subscriber_round },
887                    ) if *request_round == round && *subscriber_round == round
888                )),
889                "certify should fetch notarized block by round"
890            );
891
892            assert!(
893                buffer.subscription_count() > 0,
894                "missing candidate should register a local buffer wait"
895            );
896            assert!(
897                resolver.targeted().is_empty(),
898                "missing candidate certify must not issue targeted fetches"
899            );
900        });
901    }
902
903    #[test_traced("WARN")]
904    fn test_coding_certify_pending_verify_fetches_by_round() {
905        let runner = deterministic::Runner::timed(Duration::from_secs(30));
906        runner.start(|mut context| async move {
907            let Fixture {
908                participants,
909                schemes,
910                ..
911            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
912            let provider = ConstantProvider::new(schemes[0].clone());
913            let me = participants[0].clone();
914            let buffer = RecordingCodingBuffer::default();
915            let (marshal, resolver, _actor_handle) = start_coding_actor_with_recording(
916                context.child("actor_stack"),
917                "coding-certify-pending-verify",
918                provider.clone(),
919                buffer,
920            )
921            .await;
922            let shards =
923                start_shard_mailbox(context.child("shard_stack"), participants, provider.clone())
924                    .await;
925
926            let cfg = MarshaledConfig {
927                application: MockVerifyingApp::<CodingB, S>::new(),
928                marshal,
929                shards,
930                scheme_provider: provider,
931                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
932                strategy: Sequential,
933            };
934            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
935
936            let (candidate_ctx, candidate) = missing_candidate(me);
937            let commitment = candidate.commitment();
938            let round = candidate_ctx.round;
939            let _verify_rx = marshaled.verify(candidate_ctx, commitment).await;
940
941            let proposal = Proposal::new(round, View::zero(), commitment);
942            let notarization = CodingHarness::make_notarization(proposal, &schemes, QUORUM);
943            resolver.respond_to_next_fetch((notarization, candidate).encode());
944            let certify_rx = marshaled.certify(round, commitment).await;
945
946            let result = certify_rx.await.expect("certify result missing");
947            assert!(
948                result,
949                "pending verify should complete after certification recovery"
950            );
951            assert!(
952                resolver.wait_for_delivery_response().await,
953                "notarized delivery should validate"
954            );
955            assert!(
956                resolver.fetches().iter().any(|fetch| matches!(
957                    (&fetch.key, &fetch.subscriber),
958                    (
959                        handler::Key::Notarized { round: request_round },
960                        handler::Annotation::Notarization { round: subscriber_round },
961                    ) if *request_round == round && *subscriber_round == round
962                )),
963                "certify should recover a pending verify by notarized round"
964            );
965            assert!(
966                resolver.targeted().is_empty(),
967                "certify recovery must not issue targeted fetches"
968            );
969        });
970    }
971
972    #[test_group("slow")]
973    #[test_traced("WARN")]
974    fn test_coding_finalize_good_links() {
975        for seed in 0..5 {
976            let r1 = harness::finalize::<CodingHarness>(seed, LINK, false);
977            let r2 = harness::finalize::<CodingHarness>(seed, LINK, false);
978            assert_eq!(r1, r2);
979        }
980    }
981
982    #[test_group("slow")]
983    #[test_traced("WARN")]
984    fn test_coding_finalize_bad_links() {
985        for seed in 0..5 {
986            let r1 = harness::finalize::<CodingHarness>(seed, UNRELIABLE_LINK, false);
987            let r2 = harness::finalize::<CodingHarness>(seed, UNRELIABLE_LINK, false);
988            assert_eq!(r1, r2);
989        }
990    }
991
992    #[test_group("slow")]
993    #[test_traced("WARN")]
994    fn test_coding_finalize_good_links_quorum_sees_finalization() {
995        for seed in 0..5 {
996            let r1 = harness::finalize::<CodingHarness>(seed, LINK, true);
997            let r2 = harness::finalize::<CodingHarness>(seed, LINK, true);
998            assert_eq!(r1, r2);
999        }
1000    }
1001
1002    #[test_group("slow")]
1003    #[test_traced("WARN")]
1004    fn test_coding_finalize_bad_links_quorum_sees_finalization() {
1005        for seed in 0..5 {
1006            let r1 = harness::finalize::<CodingHarness>(seed, UNRELIABLE_LINK, true);
1007            let r2 = harness::finalize::<CodingHarness>(seed, UNRELIABLE_LINK, true);
1008            assert_eq!(r1, r2);
1009        }
1010    }
1011
1012    #[test_group("slow")]
1013    #[test_traced("WARN")]
1014    fn test_coding_hailstorm_restarts() {
1015        for seed in 0..2 {
1016            let r1 = harness::hailstorm::<CodingHarness>(seed, 4, 4, 1, LINK);
1017            let r2 = harness::hailstorm::<CodingHarness>(seed, 4, 4, 1, LINK);
1018            assert_eq!(r1, r2);
1019        }
1020    }
1021
1022    #[test_group("slow")]
1023    #[test_traced("WARN")]
1024    fn test_coding_hailstorm_multi_restarts() {
1025        for seed in 0..2 {
1026            let r1 = harness::hailstorm::<CodingHarness>(seed, 4, 4, 2, LINK);
1027            let r2 = harness::hailstorm::<CodingHarness>(seed, 4, 4, 2, LINK);
1028            assert_eq!(r1, r2);
1029        }
1030    }
1031
1032    #[test_traced("WARN")]
1033    fn test_coding_ack_pipeline_backlog() {
1034        harness::ack_pipeline_backlog::<CodingHarness>();
1035    }
1036
1037    #[test_traced("WARN")]
1038    fn test_coding_ack_pipeline_backlog_persists_on_restart() {
1039        harness::ack_pipeline_backlog_persists_on_restart::<CodingHarness>();
1040    }
1041
1042    #[test_traced("WARN")]
1043    fn test_coding_genesis_emitted_once() {
1044        harness::genesis_emitted_once::<CodingHarness>();
1045    }
1046
1047    #[test_traced("WARN")]
1048    fn test_coding_proposed_success_implies_recoverable_after_restart() {
1049        harness::proposed_success_implies_recoverable_after_restart::<CodingHarness>(0..16);
1050    }
1051
1052    #[test_traced("WARN")]
1053    fn test_coding_verified_success_implies_recoverable_after_restart() {
1054        harness::verified_success_implies_recoverable_after_restart::<CodingHarness>(0..16);
1055    }
1056
1057    #[test_traced("WARN")]
1058    fn test_coding_certified_success_implies_recoverable_after_restart() {
1059        harness::certified_success_implies_recoverable_after_restart::<CodingHarness>(0..16);
1060    }
1061
1062    #[test_traced("WARN")]
1063    fn test_coding_delivery_visibility_implies_recoverable_after_restart() {
1064        harness::delivery_visibility_implies_recoverable_after_restart::<CodingHarness>(0..16);
1065    }
1066
1067    #[test_traced("WARN")]
1068    fn test_coding_sync_height_floor() {
1069        harness::sync_height_floor::<CodingHarness>();
1070    }
1071
1072    #[test_traced("WARN")]
1073    fn test_coding_prune_finalized_archives() {
1074        harness::prune_finalized_archives::<CodingHarness>();
1075    }
1076
1077    #[test_traced("WARN")]
1078    fn test_coding_rejects_block_delivery_below_floor() {
1079        harness::reject_stale_block_delivery_after_floor_update::<CodingHarness>();
1080    }
1081
1082    #[test_traced("WARN")]
1083    fn test_coding_commitment_fetch_height_hint_mismatch_wakes_subscriber() {
1084        harness::commitment_fetch_height_hint_mismatch_wakes_subscriber::<CodingHarness>();
1085    }
1086
1087    #[test_traced("WARN")]
1088    fn test_coding_subscribe_basic_block_delivery() {
1089        harness::subscribe_basic_block_delivery::<CodingHarness>();
1090    }
1091
1092    #[test_traced("WARN")]
1093    fn test_coding_subscribe_multiple_subscriptions() {
1094        harness::subscribe_multiple_subscriptions::<CodingHarness>();
1095    }
1096
1097    #[test_traced("WARN")]
1098    fn test_coding_subscribe_canceled_subscriptions() {
1099        harness::subscribe_canceled_subscriptions::<CodingHarness>();
1100    }
1101
1102    #[test_traced("WARN")]
1103    fn test_coding_subscribe_blocks_from_different_sources() {
1104        harness::subscribe_blocks_from_different_sources::<CodingHarness>();
1105    }
1106
1107    #[test_traced("WARN")]
1108    fn test_coding_get_info_basic_queries_present_and_missing() {
1109        harness::get_info_basic_queries_present_and_missing::<CodingHarness>();
1110    }
1111
1112    #[test_traced("WARN")]
1113    fn test_coding_get_info_latest_progression_multiple_finalizations() {
1114        harness::get_info_latest_progression_multiple_finalizations::<CodingHarness>();
1115    }
1116
1117    #[test_traced("WARN")]
1118    fn test_coding_get_block_by_height_and_latest() {
1119        harness::get_block_by_height_and_latest::<CodingHarness>();
1120    }
1121
1122    #[test_traced("WARN")]
1123    fn test_coding_get_block_by_commitment_from_sources_and_missing() {
1124        harness::get_block_by_commitment_from_sources_and_missing::<CodingHarness>();
1125    }
1126
1127    #[test_traced("WARN")]
1128    fn test_coding_get_finalization_by_height() {
1129        harness::get_finalization_by_height::<CodingHarness>();
1130    }
1131
1132    #[test_traced("WARN")]
1133    fn test_coding_hint_finalized_triggers_fetch() {
1134        harness::hint_finalized_triggers_fetch::<CodingHarness>();
1135    }
1136
1137    #[test_traced("WARN")]
1138    fn test_coding_ancestry_stream() {
1139        harness::ancestry_stream::<CodingHarness>();
1140    }
1141
1142    #[test_traced("WARN")]
1143    fn test_coding_finalize_same_height_different_views() {
1144        harness::finalize_same_height_different_views::<CodingHarness>();
1145    }
1146
1147    #[test_traced("WARN")]
1148    fn test_coding_certify_persists_equivocated_block() {
1149        harness::certify_persists_equivocated_block::<CodingHarness>();
1150    }
1151
1152    #[test_traced("WARN")]
1153    fn test_coding_verified_after_restart_reverify_same_round_implies_recoverable() {
1154        harness::verified_after_restart_reverify_same_round_implies_recoverable::<CodingHarness>();
1155    }
1156
1157    #[test_traced("WARN")]
1158    fn test_coding_certify_after_restart_reverify_same_round_implies_recoverable() {
1159        harness::certify_after_restart_reverify_same_round_implies_recoverable::<CodingHarness>();
1160    }
1161
1162    #[test_traced("WARN")]
1163    fn test_coding_certify_at_later_view_survives_earlier_view_pruning() {
1164        harness::certify_at_later_view_survives_earlier_view_pruning::<CodingHarness>();
1165    }
1166
1167    #[test_traced("WARN")]
1168    fn test_coding_certify_first_block_fetches_genesis_parent() {
1169        let runner = deterministic::Runner::timed(Duration::from_secs(60));
1170        runner.start(|mut context| async move {
1171            let Fixture {
1172                participants,
1173                schemes,
1174                ..
1175            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1176            let mut oracle = setup_network_with_participants(
1177                context.child("network"),
1178                NZUsize!(1),
1179                participants.clone(),
1180            )
1181            .await;
1182
1183            let me = participants[0].clone();
1184            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
1185
1186            let setup = CodingHarness::setup_validator(
1187                context.child("validator").with_attribute("index", 0),
1188                &mut oracle,
1189                me.clone(),
1190                ConstantProvider::new(schemes[0].clone()),
1191            )
1192            .await;
1193            let marshal = setup.mailbox;
1194            let shards = setup.extra;
1195
1196            let genesis_ctx = CodingCtx {
1197                round: Round::zero(),
1198                leader: default_leader(),
1199                parent: (View::zero(), genesis_commitment()),
1200            };
1201            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
1202            let genesis_parent_commitment = genesis_coding_commitment(&genesis);
1203
1204            let round = Round::new(Epoch::zero(), View::new(1));
1205            let block_ctx = CodingCtx {
1206                round,
1207                leader: me.clone(),
1208                parent: (View::zero(), genesis_parent_commitment),
1209            };
1210            let block = make_coding_block(block_ctx.clone(), genesis.digest(), Height::new(1), 100);
1211            let coded_block = CodedBlock::new(block, coding_config, &Sequential);
1212            let commitment = coded_block.commitment();
1213            shards.proposed(round, coded_block);
1214
1215            context.sleep(Duration::from_millis(10)).await;
1216
1217            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
1218            let cfg = MarshaledConfig {
1219                application: mock_app,
1220                marshal,
1221                shards,
1222                scheme_provider: ConstantProvider::new(schemes[0].clone()),
1223                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
1224                strategy: Sequential,
1225            };
1226            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
1227
1228            let shard_validity = marshaled
1229                .verify(block_ctx, commitment)
1230                .await
1231                .await
1232                .expect("verify result missing");
1233            assert!(shard_validity, "shard validity should pass");
1234
1235            let certify_result = marshaled
1236                .certify(round, commitment)
1237                .await
1238                .await
1239                .expect("certify result missing");
1240            assert!(
1241                certify_result,
1242                "height-1 block should certify with genesis as parent"
1243            );
1244        });
1245    }
1246
1247    /// Finalizing a descendant must not height-prune the shard-engine buffer before
1248    /// `try_repair_gaps` has consumed buffer-only ancestors.
1249    ///
1250    /// Places parent (height 1) and descendant (height 2) in the shard engine's
1251    /// reconstructed-block cache via `proposed()`, then reports a finalization
1252    /// for the descendant only.
1253    #[test_traced("WARN")]
1254    fn test_coding_store_finalization_does_not_prune_buffer_before_repair() {
1255        let runner = deterministic::Runner::timed(Duration::from_secs(60));
1256        runner.start(|mut context| async move {
1257            let Fixture {
1258                participants,
1259                schemes,
1260                ..
1261            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1262            let mut oracle = setup_network_with_participants(
1263                context.child("network"),
1264                NZUsize!(1),
1265                participants.clone(),
1266            )
1267            .await;
1268
1269            let setup = CodingHarness::setup_validator(
1270                context.child("validator").with_attribute("index", 0),
1271                &mut oracle,
1272                participants[0].clone(),
1273                ConstantProvider::new(schemes[0].clone()),
1274            )
1275            .await;
1276            let mut handle = harness::ValidatorHandle::<CodingHarness> {
1277                mailbox: setup.mailbox,
1278                extra: setup.extra,
1279            };
1280
1281            // Build a 2-block chain: parent at height 1, descendant at height 2.
1282            let parent_block = CodingHarness::make_test_block(
1283                Sha256::hash(&[b""]),
1284                CodingHarness::genesis_parent_commitment(NUM_VALIDATORS as u16),
1285                Height::new(1),
1286                1,
1287                NUM_VALIDATORS as u16,
1288            );
1289            let parent_digest = CodingHarness::digest(&parent_block);
1290            let parent_commitment = CodingHarness::commitment(&parent_block);
1291
1292            let descendant_block = CodingHarness::make_test_block(
1293                parent_digest,
1294                parent_commitment,
1295                Height::new(2),
1296                2,
1297                NUM_VALIDATORS as u16,
1298            );
1299            let descendant_commitment = CodingHarness::commitment(&descendant_block);
1300
1301            // Seed the shard engine's reconstructed-block cache with both blocks.
1302            CodingHarness::propose(
1303                &mut handle,
1304                Round::new(Epoch::new(0), View::new(1)),
1305                &parent_block,
1306            )
1307            .await;
1308            CodingHarness::propose(
1309                &mut handle,
1310                Round::new(Epoch::new(0), View::new(2)),
1311                &descendant_block,
1312            )
1313            .await;
1314
1315            // Report finalization for the descendant only. The parent has no
1316            // finalization certificate: it must be archived by walking the
1317            // parent link from the descendant and sourcing the block from the
1318            // shard-engine buffer.
1319            let descendant_proposal = Proposal {
1320                round: Round::new(Epoch::new(0), View::new(2)),
1321                parent: View::new(1),
1322                payload: descendant_commitment,
1323            };
1324            let descendant_finalization =
1325                CodingHarness::make_finalization(descendant_proposal, &schemes, QUORUM);
1326            CodingHarness::report_finalization(&mut handle.mailbox, descendant_finalization).await;
1327
1328            // Wait until the descendant is archived: that proves finalization processing
1329            // has completed, at which point the parent must already have been repaired
1330            // from the shard buffer.
1331            while handle.mailbox.get_block(Height::new(2)).await.is_none() {
1332                context.sleep(Duration::from_millis(10)).await;
1333            }
1334
1335            let parent = handle.mailbox.get_block(Height::new(1)).await;
1336            assert!(
1337                parent.is_some(),
1338                "parent must be archived from shard buffer before height-prune evicts it"
1339            );
1340        });
1341    }
1342
1343    #[test_traced("WARN")]
1344    fn test_coding_init_processed_height() {
1345        harness::init_processed_height::<CodingHarness>();
1346    }
1347
1348    #[test_traced("INFO")]
1349    fn test_coding_broadcast_caches_block() {
1350        harness::broadcast_caches_block::<CodingHarness>();
1351    }
1352
1353    /// Test that certifying a lower-view block after a higher-view block succeeds.
1354    ///
1355    /// This is a critical test for crash recovery scenarios where a validator may need
1356    /// to certify blocks in non-sequential view order.
1357    #[test_traced("INFO")]
1358    fn test_certify_lower_view_after_higher_view() {
1359        let runner = deterministic::Runner::timed(Duration::from_secs(60));
1360        runner.start(|mut context| async move {
1361            let Fixture {
1362                participants,
1363                schemes,
1364                ..
1365            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1366            let mut oracle = setup_network_with_participants(
1367                context.child("network"),
1368                NZUsize!(1),
1369                participants.clone(),
1370            )
1371            .await;
1372
1373            let me = participants[0].clone();
1374            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
1375
1376            let setup = CodingHarness::setup_validator(
1377                context.child("validator").with_attribute("index", 0),
1378                &mut oracle,
1379                me.clone(),
1380                ConstantProvider::new(schemes[0].clone()),
1381            )
1382            .await;
1383            let marshal = setup.mailbox;
1384            let shards = setup.extra;
1385
1386            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
1387
1388            let cfg = MarshaledConfig {
1389                application: mock_app,
1390                marshal: marshal.clone(),
1391                shards: shards.clone(),
1392                scheme_provider: ConstantProvider::new(schemes[0].clone()),
1393                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
1394                strategy: Sequential,
1395            };
1396            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
1397
1398            let genesis_ctx = CodingCtx {
1399                round: Round::zero(),
1400                leader: default_leader(),
1401                parent: (View::zero(), genesis_commitment()),
1402            };
1403            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
1404
1405            // Create parent block at height 1
1406            let parent_ctx = CodingCtx {
1407                round: Round::new(Epoch::new(0), View::new(1)),
1408                leader: default_leader(),
1409                parent: (View::zero(), genesis_commitment()),
1410            };
1411            let parent = make_coding_block(parent_ctx, genesis.digest(), Height::new(1), 100);
1412            let parent_digest = parent.digest();
1413            let coded_parent = CodedBlock::new(parent.clone(), coding_config, &Sequential);
1414            let parent_commitment = coded_parent.commitment();
1415            shards.proposed(Round::new(Epoch::new(0), View::new(1)), coded_parent);
1416
1417            // Block A at view 5 (height 2) - create with context matching what verify will receive
1418            let round_a = Round::new(Epoch::new(0), View::new(5));
1419            let context_a = CodingCtx {
1420                round: round_a,
1421                leader: me.clone(),
1422                parent: (View::new(1), parent_commitment),
1423            };
1424            let block_a = make_coding_block(context_a.clone(), parent_digest, Height::new(2), 200);
1425            let coded_block_a = CodedBlock::new(block_a.clone(), coding_config, &Sequential);
1426            let commitment_a = coded_block_a.commitment();
1427            shards.proposed(round_a, coded_block_a);
1428
1429            // Block B at view 10 (height 2, different block same height - could happen with
1430            // different proposers or re-proposals)
1431            let round_b = Round::new(Epoch::new(0), View::new(10));
1432            let context_b = CodingCtx {
1433                round: round_b,
1434                leader: me.clone(),
1435                parent: (View::new(1), parent_commitment),
1436            };
1437            let block_b = make_coding_block(context_b.clone(), parent_digest, Height::new(2), 300);
1438            let coded_block_b = CodedBlock::new(block_b.clone(), coding_config, &Sequential);
1439            let commitment_b = coded_block_b.commitment();
1440            shards.proposed(round_b, coded_block_b);
1441
1442            context.sleep(Duration::from_millis(10)).await;
1443
1444            // Step 1: Verify block A at view 5
1445            let _ = marshaled.verify(context_a, commitment_a).await.await;
1446
1447            // Step 2: Verify block B at view 10
1448            let _ = marshaled.verify(context_b, commitment_b).await.await;
1449
1450            // Step 3: Certify block B at view 10 FIRST
1451            let certify_b = marshaled.certify(round_b, commitment_b).await;
1452            assert!(
1453                certify_b.await.unwrap(),
1454                "Block B certification should succeed"
1455            );
1456
1457            // Step 4: Certify block A at view 5 - should succeed
1458            let certify_a = marshaled.certify(round_a, commitment_a).await;
1459
1460            // Use select with timeout to detect never-resolving receiver
1461            select! {
1462                result = certify_a => {
1463                    assert!(result.unwrap(), "Block A certification should succeed");
1464                },
1465                _ = context.sleep(Duration::from_secs(5)) => {
1466                    panic!("Block A certification timed out");
1467                },
1468            }
1469        })
1470    }
1471
1472    /// Regression test for re-proposal validation in optimistic_verify.
1473    ///
1474    /// Verifies that:
1475    /// 1. Valid re-proposals at epoch boundaries are accepted
1476    /// 2. Invalid re-proposals (not at epoch boundary) are rejected
1477    ///
1478    /// A re-proposal occurs when the parent digest equals the block being verified,
1479    /// meaning the same block is being proposed again in a new view.
1480    #[test_traced("INFO")]
1481    fn test_marshaled_reproposal_validation() {
1482        let runner = deterministic::Runner::timed(Duration::from_secs(60));
1483        runner.start(|mut context| async move {
1484            let Fixture {
1485                participants,
1486                schemes,
1487                ..
1488            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1489            let mut oracle = setup_network_with_participants(
1490                context.child("network"),
1491                NZUsize!(1),
1492                participants.clone(),
1493            )
1494            .await;
1495
1496            let me = participants[0].clone();
1497            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
1498
1499            let setup = CodingHarness::setup_validator(
1500                context.child("validator").with_attribute("index", 0),
1501                &mut oracle,
1502                me.clone(),
1503                ConstantProvider::new(schemes[0].clone()),
1504            )
1505            .await;
1506            setup_network_links(&mut oracle, &participants[..2], LINK).await;
1507            let reproposer_control = oracle.control(participants[1].clone());
1508            let (mut reproposer_sender, _reproposer_receiver) =
1509                reproposer_control.register(2, TEST_QUOTA).await.unwrap();
1510            let marshal = setup.mailbox;
1511            let shards = setup.extra;
1512
1513            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
1514            let cfg = MarshaledConfig {
1515                application: mock_app,
1516                marshal: marshal.clone(),
1517                shards: shards.clone(),
1518                scheme_provider: ConstantProvider::new(schemes[0].clone()),
1519                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
1520                strategy: Sequential,
1521            };
1522            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
1523
1524            let genesis_ctx = CodingCtx {
1525                round: Round::zero(),
1526                leader: default_leader(),
1527                parent: (View::zero(), genesis_commitment()),
1528            };
1529            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
1530
1531            // Build a chain up to just below the epoch boundary (height 19 is the last
1532            // block in epoch 0 with BLOCKS_PER_EPOCH=20, since epoch 0 covers heights
1533            // 0-19), so the boundary block below chains onto height 18.
1534            let mut parent = genesis.digest();
1535            let mut last_view = View::zero();
1536            let mut last_commitment = genesis_commitment();
1537
1538            // Capture a genuinely-chained non-boundary block (height 10) so Test 2
1539            // can re-propose one that validates cleanly under its own embedded
1540            // context. Without this, a certify rejection there could come from an
1541            // ancestry error rather than the re-proposal boundary gate under test.
1542            let mut non_boundary = None;
1543            for i in 1..BLOCKS_PER_EPOCH.get() - 1 {
1544                let round = Round::new(Epoch::new(0), View::new(i));
1545                let ctx = CodingCtx {
1546                    round,
1547                    leader: me.clone(),
1548                    parent: (last_view, last_commitment),
1549                };
1550                let block = make_coding_block(ctx.clone(), parent, Height::new(i), i * 100);
1551                let coded_block = CodedBlock::new(block.clone(), coding_config, &Sequential);
1552                last_commitment = coded_block.commitment();
1553                if i == 10 {
1554                    non_boundary = Some((View::new(i), last_commitment));
1555                }
1556                shards.proposed(round, coded_block);
1557                parent = block.digest();
1558                last_view = View::new(i);
1559            }
1560            let (non_boundary_view, non_boundary_commitment) =
1561                non_boundary.expect("chain includes a non-boundary block");
1562
1563            // Create the epoch boundary block (height 19, last block in epoch 0)
1564            let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
1565            let boundary_round = Round::new(Epoch::new(0), View::new(boundary_height.get()));
1566            let boundary_context = CodingCtx {
1567                round: boundary_round,
1568                leader: me.clone(),
1569                parent: (last_view, last_commitment),
1570            };
1571            let boundary_block = make_coding_block(
1572                boundary_context.clone(),
1573                parent,
1574                boundary_height,
1575                boundary_height.get() * 100,
1576            );
1577            let coded_boundary =
1578                CodedBlock::new(boundary_block.clone(), coding_config, &Sequential);
1579            let boundary_commitment = coded_boundary.commitment();
1580            shards.discovered(
1581                boundary_commitment,
1582                boundary_context.leader.clone(),
1583                boundary_round,
1584            );
1585            shards.proposed(boundary_round, coded_boundary.clone());
1586
1587            context.sleep(Duration::from_millis(10)).await;
1588
1589            // Test 1: Valid re-proposal at epoch boundary should be accepted
1590            // Re-proposal context: parent digest equals the block being verified
1591            // Re-proposals happen within the same epoch when the parent is the last block
1592            //
1593            // In the coding marshal, verify() returns shard validity while deferred_verify
1594            // runs in the background. We call verify() to register the certification gate task,
1595            // then certify() returns the deferred_verify result.
1596            let reproposal_round = Round::new(Epoch::new(0), View::new(20));
1597            let reproposal_context = CodingCtx {
1598                round: reproposal_round,
1599                leader: participants[1].clone(),
1600                parent: (View::new(boundary_height.get()), boundary_commitment), // Parent IS the boundary block
1601            };
1602
1603            // Call verify to kick off deferred verification.
1604            // We must await the verify result to ensure the certification gate task is
1605            // registered before calling certify.
1606            let shard_validity = marshaled
1607                .verify(reproposal_context.clone(), boundary_commitment)
1608                .await
1609                .await;
1610            assert!(
1611                shard_validity.unwrap(),
1612                "Re-proposal verify should return true for shard validity"
1613            );
1614
1615            let assigned = shards.subscribe_assigned_shard_verified(boundary_commitment);
1616            let assigned_shard = coded_boundary
1617                .shard(0)
1618                .expect("missing assigned shard")
1619                .encode();
1620            reproposer_sender.send(Recipients::One(me.clone()), assigned_shard, true);
1621            select! {
1622                result = assigned => result.expect("assigned shard sender dropped"),
1623                _ = context.sleep(Duration::from_secs(5)) => {
1624                    panic!("assigned shard from reproposer was not accepted");
1625                },
1626            }
1627
1628            // Use certify to get the actual deferred_verify result
1629            let certify_result = marshaled
1630                .certify(reproposal_round, boundary_commitment)
1631                .await
1632                .await;
1633            assert!(
1634                certify_result.unwrap(),
1635                "Valid re-proposal at epoch boundary should be accepted"
1636            );
1637
1638            // The same boundary commitment may be re-proposed again after the
1639            // first re-proposal certifies. Its parent view advances while its
1640            // parent payload remains the boundary commitment. Re-proposal
1641            // validation is intrinsic to the committed block and must return
1642            // the same result under both headers.
1643            let repeated_reproposal_round =
1644                Round::new(Epoch::new(0), View::new(boundary_height.get() + 2));
1645            let repeated_reproposal_context = CodingCtx {
1646                round: repeated_reproposal_round,
1647                leader: me.clone(),
1648                parent: (reproposal_round.view(), boundary_commitment),
1649            };
1650            let repeated_verify = marshaled
1651                .verify(repeated_reproposal_context, boundary_commitment)
1652                .await
1653                .await;
1654            assert!(
1655                repeated_verify.unwrap(),
1656                "Repeated re-proposal should remain valid as the parent view advances"
1657            );
1658            let repeated_certify = marshaled
1659                .certify(repeated_reproposal_round, boundary_commitment)
1660                .await
1661                .await;
1662            assert!(
1663                repeated_certify.unwrap(),
1664                "Repeated re-proposal certification should remain valid"
1665            );
1666
1667            // Test 2: Invalid re-proposal (not at epoch boundary) should be
1668            // rejected. Re-propose the genuinely-chained height-10 block at view
1669            // 15. Because that block validates cleanly under its embedded context,
1670            // a certify rejection can only come from the re-proposal boundary gate,
1671            // not from an ancestry error.
1672            let invalid_reproposal_round = Round::new(Epoch::new(0), View::new(15));
1673            let invalid_reproposal_context = CodingCtx {
1674                round: invalid_reproposal_round,
1675                leader: me.clone(),
1676                parent: (non_boundary_view, non_boundary_commitment),
1677            };
1678
1679            // Call verify to kick off deferred verification.
1680            // We must await the verify result to ensure the certification gate task is
1681            // registered before calling certify.
1682            let shard_validity = marshaled
1683                .verify(invalid_reproposal_context, non_boundary_commitment)
1684                .await
1685                .await;
1686            assert!(
1687                !shard_validity.unwrap(),
1688                "Invalid re-proposal verify should return false"
1689            );
1690
1691            // Use certify to get the actual deferred_verify result
1692            let certify_result = marshaled
1693                .certify(invalid_reproposal_round, non_boundary_commitment)
1694                .await
1695                .await;
1696            assert!(
1697                !certify_result.unwrap(),
1698                "Invalid re-proposal (not at epoch boundary) should be rejected"
1699            );
1700
1701            // Test 3: Re-proposal with mismatched epoch should be rejected
1702            // This is a regression test - re-proposals must be in the same epoch as the block.
1703            let cross_epoch_reproposal_round = Round::new(Epoch::new(1), View::new(20));
1704            let cross_epoch_reproposal_context = CodingCtx {
1705                round: cross_epoch_reproposal_round,
1706                leader: me.clone(),
1707                parent: (View::new(boundary_height.get()), boundary_commitment),
1708            };
1709
1710            // Call verify to kick off deferred verification.
1711            // We must await the verify result to ensure the certification gate task is
1712            // registered before calling certify.
1713            let shard_validity = marshaled
1714                .verify(cross_epoch_reproposal_context.clone(), boundary_commitment)
1715                .await
1716                .await;
1717            assert!(
1718                !shard_validity.unwrap(),
1719                "Cross-epoch re-proposal verify should return false"
1720            );
1721
1722            // Use certify to get the actual deferred_verify result
1723            let certify_result = marshaled
1724                .certify(cross_epoch_reproposal_round, boundary_commitment)
1725                .await
1726                .await;
1727            assert!(
1728                !certify_result.unwrap(),
1729                "Re-proposal with mismatched epoch should be rejected"
1730            );
1731
1732            // Note: Tests for certify-only paths (crash recovery scenarios) are not included here
1733            // because they require multiple validators to reconstruct blocks from shards. In a
1734            // single-validator test setup, block reconstruction fails due to insufficient shards.
1735            // These paths are tested in integration tests with multiple validators.
1736        })
1737    }
1738
1739    /// An invalid re-proposal must not open shard reconstruction for its payload.
1740    ///
1741    /// A re-proposal's consensus round is not bound to its payload because the context
1742    /// digest check is skipped: the block carries the context of its original proposal.
1743    /// The shard engine resolves the participant set from the announced round's epoch, so
1744    /// announcing before the boundary check lets a leader bind a commitment to a round it
1745    /// was never proposed in and misclassify shards from honest peers.
1746    #[test_traced("INFO")]
1747    fn test_invalid_reproposal_does_not_open_reconstruction() {
1748        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1749        runner.start(|mut context| async move {
1750            let Fixture {
1751                participants,
1752                schemes,
1753                ..
1754            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1755            let mut oracle = setup_network_with_participants(
1756                context.child("network"),
1757                NZUsize!(1),
1758                participants.clone(),
1759            )
1760            .await;
1761
1762            let me = participants[0].clone();
1763            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
1764
1765            let setup = CodingHarness::setup_validator(
1766                context.child("validator").with_attribute("index", 0),
1767                &mut oracle,
1768                me.clone(),
1769                ConstantProvider::new(schemes[0].clone()),
1770            )
1771            .await;
1772            setup_network_links(&mut oracle, &participants[..2], LINK).await;
1773            let reproposer_control = oracle.control(participants[1].clone());
1774            let (mut reproposer_sender, _reproposer_receiver) =
1775                reproposer_control.register(2, TEST_QUOTA).await.unwrap();
1776            let marshal = setup.mailbox;
1777            let shards = setup.extra;
1778
1779            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
1780            let cfg = MarshaledConfig {
1781                application: mock_app,
1782                marshal: marshal.clone(),
1783                shards: shards.clone(),
1784                scheme_provider: ConstantProvider::new(schemes[0].clone()),
1785                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
1786                strategy: Sequential,
1787            };
1788            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
1789
1790            // Store a mid-epoch block in core marshal without opening shard reconstruction.
1791            let original_round = Round::new(Epoch::new(0), View::new(5));
1792            let original_context = CodingCtx {
1793                round: original_round,
1794                leader: participants[1].clone(),
1795                parent: (View::new(4), genesis_commitment()),
1796            };
1797            let block = make_coding_block(
1798                original_context,
1799                Sha256::hash(&[b"parent"]),
1800                Height::new(5),
1801                500,
1802            );
1803            let coded_block: TestCodedBlock = CodedBlock::new(block, coding_config, &Sequential);
1804            let commitment = coded_block.commitment();
1805            assert!(marshal.verified(original_round, coded_block.clone()).await);
1806            assert!(shards.get(commitment).await.is_none());
1807
1808            // Re-proposing the old block as epoch 1's genesis parent is invalid.
1809            // The payload is the parent, so `verify` takes the re-proposal path.
1810            let reproposal_round = Round::new(Epoch::new(1), View::new(1));
1811            let reproposal_context = CodingCtx {
1812                round: reproposal_round,
1813                leader: participants[1].clone(),
1814                parent: (View::zero(), commitment),
1815            };
1816            let assigned = shards.subscribe_assigned_shard_verified(commitment);
1817            let verdict = marshaled.verify(reproposal_context, commitment).await.await;
1818            assert!(!verdict.expect("re-proposal verdict missing"));
1819
1820            // The re-proposer delivers this node's assigned shard. Without reconstruction
1821            // state it stays buffered, so assigned shard verification cannot complete.
1822            let assigned_shard = coded_block
1823                .shard(0)
1824                .expect("missing assigned shard")
1825                .encode();
1826            reproposer_sender.send(Recipients::One(me.clone()), assigned_shard, true);
1827
1828            select! {
1829                _ = assigned => {
1830                    panic!("invalid re-proposal opened reconstruction for its payload");
1831                },
1832                _ = context.sleep(Duration::from_secs(1)) => {},
1833            }
1834        })
1835    }
1836
1837    /// Exact commitment retirement (durable application progress) can occur on
1838    /// either side of re-proposal verification. The re-proposal must remain live
1839    /// through core marshal's verified cache, while a post-retirement `discovered`
1840    /// announcement must recreate shard state needed before notarization.
1841    #[test_traced("WARN")]
1842    fn test_coding_reproposal_recreates_shard_state_after_retirement() {
1843        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1844        runner.start(|mut context| async move {
1845            let Fixture {
1846                participants,
1847                schemes,
1848                ..
1849            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1850            let mut oracle = setup_network_with_participants(
1851                context.child("network"),
1852                NZUsize!(1),
1853                participants.clone(),
1854            )
1855            .await;
1856
1857            let me = participants[0].clone();
1858            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
1859
1860            let setup = CodingHarness::setup_validator(
1861                context.child("validator").with_attribute("index", 0),
1862                &mut oracle,
1863                me.clone(),
1864                ConstantProvider::new(schemes[0].clone()),
1865            )
1866            .await;
1867            setup_network_links(&mut oracle, &participants[..2], LINK).await;
1868            let reproposer_control = oracle.control(participants[1].clone());
1869            let (mut reproposer_sender, _reproposer_receiver) =
1870                reproposer_control.register(2, TEST_QUOTA).await.unwrap();
1871            let marshal = setup.mailbox;
1872            let shards = setup.extra;
1873
1874            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
1875            let cfg = MarshaledConfig {
1876                application: mock_app,
1877                marshal: marshal.clone(),
1878                shards: shards.clone(),
1879                scheme_provider: ConstantProvider::new(schemes[0].clone()),
1880                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
1881                strategy: Sequential,
1882            };
1883            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
1884
1885            // Build the epoch boundary block, store it in core marshal (the durable
1886            // backstop), and cache it in the shard engine.
1887            let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
1888            let boundary_round = Round::new(Epoch::new(0), View::new(boundary_height.get()));
1889            let boundary_context = CodingCtx {
1890                round: boundary_round,
1891                leader: me.clone(),
1892                parent: (View::new(boundary_height.get() - 1), genesis_commitment()),
1893            };
1894            let boundary_block = make_coding_block(
1895                boundary_context.clone(),
1896                Sha256::hash(&[b"parent"]),
1897                boundary_height,
1898                1900,
1899            );
1900            let coded_boundary: TestCodedBlock =
1901                CodedBlock::new(boundary_block, coding_config, &Sequential);
1902            let boundary_commitment = coded_boundary.commitment();
1903            assert!(
1904                marshal
1905                    .verified(boundary_round, coded_boundary.clone())
1906                    .await
1907            );
1908            shards.discovered(boundary_commitment, me.clone(), boundary_round);
1909            shards.proposed(boundary_round, coded_boundary.clone());
1910            context.sleep(Duration::from_millis(10)).await;
1911            assert!(shards.get(boundary_commitment).await.is_some());
1912
1913            // Exact retirement with a round floor below the commitment's observed
1914            // round: only the exact commitment list can retire it.
1915            shards.retire(core::Retirement {
1916                round_floor: Round::new(Epoch::zero(), View::new(1)),
1917                exact_retirements: vec![boundary_commitment],
1918            });
1919            context.sleep(Duration::from_millis(10)).await;
1920            assert!(shards.get(boundary_commitment).await.is_none());
1921
1922            // Re-propose the boundary block in the same epoch. Verification must
1923            // fetch the block from the core backstop and re-announce discovery.
1924            let reproposal_round = Round::new(Epoch::new(0), View::new(20));
1925            let reproposal_context = CodingCtx {
1926                round: reproposal_round,
1927                leader: participants[1].clone(),
1928                parent: (View::new(boundary_height.get()), boundary_commitment),
1929            };
1930            let verdict = marshaled
1931                .verify(reproposal_context, boundary_commitment)
1932                .await
1933                .await;
1934            assert!(
1935                verdict.expect("re-proposal verdict missing"),
1936                "re-proposal should verify from the core backstop after exact retirement"
1937            );
1938
1939            // The recreated reconstruction state must accept this node's assigned
1940            // shard from the re-proposer (not the original leader).
1941            let assigned = shards.subscribe_assigned_shard_verified(boundary_commitment);
1942            let assigned_shard = coded_boundary
1943                .shard(0)
1944                .expect("missing assigned shard")
1945                .encode();
1946            reproposer_sender.send(Recipients::One(me.clone()), assigned_shard, true);
1947            select! {
1948                result = assigned => result.expect("assigned shard sender dropped"),
1949                _ = context.sleep(Duration::from_secs(5)) => {
1950                    panic!("assigned shard was not accepted after state recreation");
1951                },
1952            }
1953
1954            // Retiring again after verification removes the newly recreated shard
1955            // state, but not the block persisted in core marshal. Certification and
1956            // caller-owned core subscriptions must remain live through that cache.
1957            shards.retire(core::Retirement {
1958                round_floor: Round::new(Epoch::zero(), View::new(1)),
1959                exact_retirements: vec![boundary_commitment],
1960            });
1961            while shards.get(boundary_commitment).await.is_some() {
1962                context.sleep(Duration::from_millis(10)).await;
1963            }
1964
1965            let block = marshal
1966                .subscribe_by_commitment(boundary_commitment, core::CommitmentFallback::Wait)
1967                .await
1968                .expect("core block subscription closed after shard retirement");
1969            assert_eq!(block.commitment(), boundary_commitment);
1970
1971            let certify = marshaled
1972                .certify(reproposal_round, boundary_commitment)
1973                .await
1974                .await;
1975            assert!(
1976                certify.expect("certify result missing"),
1977                "re-proposal should certify after exact retirement"
1978            );
1979        })
1980    }
1981
1982    /// A participant that never held the payload cannot wait for shards it
1983    /// cannot yet classify. Re-proposal verification must acquire the block by
1984    /// fetching at the parent's certified round and return the boundary verdict.
1985    #[test_traced("WARN")]
1986    fn test_coding_reproposal_verify_fetches_block_by_parent_round() {
1987        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1988        runner.start(|mut context| async move {
1989            let Fixture {
1990                participants,
1991                schemes,
1992                ..
1993            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1994            let mut oracle = setup_network_with_participants(
1995                context.child("network"),
1996                NZUsize!(1),
1997                participants[..2].iter().cloned(),
1998            )
1999            .await;
2000
2001            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2002
2003            let v0_setup = CodingHarness::setup_validator(
2004                context.child("validator").with_attribute("index", 0),
2005                &mut oracle,
2006                participants[0].clone(),
2007                ConstantProvider::new(schemes[0].clone()),
2008            )
2009            .await;
2010            let v1_setup = CodingHarness::setup_validator(
2011                context.child("validator").with_attribute("index", 1),
2012                &mut oracle,
2013                participants[1].clone(),
2014                ConstantProvider::new(schemes[1].clone()),
2015            )
2016            .await;
2017            setup_network_links(&mut oracle, &participants[..2], LINK).await;
2018
2019            let mut v0_mailbox = v0_setup.mailbox;
2020            let v1_marshal = v1_setup.mailbox;
2021            let v1_shards = v1_setup.extra;
2022
2023            // The boundary block of epoch 0, originally proposed at view 5.
2024            let original_round = Round::new(Epoch::new(0), View::new(5));
2025            let original_context = CodingCtx {
2026                round: original_round,
2027                leader: participants[0].clone(),
2028                parent: (View::new(4), genesis_commitment()),
2029            };
2030            let block = make_coding_block(
2031                original_context,
2032                Sha256::hash(&[b"parent"]),
2033                Height::new(BLOCKS_PER_EPOCH.get() - 1),
2034                1900,
2035            );
2036            let coded_block: TestCodedBlock = CodedBlock::new(block, coding_config, &Sequential);
2037            let commitment = coded_block.commitment();
2038
2039            // Serving a fetch by round requires the block and the round's
2040            // notarization certificate.
2041            assert!(
2042                v0_mailbox
2043                    .verified(original_round, coded_block.clone())
2044                    .await
2045            );
2046            CodingHarness::report_notarization(
2047                &mut v0_mailbox,
2048                CodingHarness::make_notarization(
2049                    Proposal {
2050                        round: original_round,
2051                        parent: View::new(4),
2052                        payload: commitment,
2053                    },
2054                    &schemes,
2055                    QUORUM,
2056                ),
2057            )
2058            .await;
2059
2060            // The re-proposal names its own parent, so verification targets the
2061            // parent round. Validator 1 never held the payload.
2062            assert!(v1_shards.get(commitment).await.is_none());
2063            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
2064            let cfg = MarshaledConfig {
2065                application: mock_app,
2066                marshal: v1_marshal.clone(),
2067                shards: v1_shards.clone(),
2068                scheme_provider: ConstantProvider::new(schemes[1].clone()),
2069                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
2070                strategy: Sequential,
2071            };
2072            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
2073            let reproposal_context = CodingCtx {
2074                round: Round::new(Epoch::new(0), View::new(7)),
2075                leader: participants[1].clone(),
2076                parent: (View::new(5), commitment),
2077            };
2078            let verdict = marshaled.verify(reproposal_context, commitment).await.await;
2079            assert!(
2080                verdict.expect("re-proposal verdict missing"),
2081                "re-proposal should verify after fetching the block by parent round"
2082            );
2083        })
2084    }
2085
2086    #[test_traced("WARN")]
2087    fn test_marshaled_rejects_mismatched_context_digest() {
2088        let runner = deterministic::Runner::timed(Duration::from_secs(30));
2089        runner.start(|mut context| async move {
2090            let Fixture {
2091                participants,
2092                schemes,
2093                ..
2094            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2095            let mut oracle = setup_network_with_participants(
2096                context.child("network"),
2097                NZUsize!(1),
2098                participants.clone(),
2099            )
2100            .await;
2101
2102            let me = participants[0].clone();
2103            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2104
2105            let setup = CodingHarness::setup_validator(
2106                context.child("validator").with_attribute("index", 0),
2107                &mut oracle,
2108                me.clone(),
2109                ConstantProvider::new(schemes[0].clone()),
2110            )
2111            .await;
2112            let marshal = setup.mailbox;
2113            let shards = setup.extra;
2114
2115            let genesis_ctx = CodingCtx {
2116                round: Round::zero(),
2117                leader: default_leader(),
2118                parent: (View::zero(), genesis_commitment()),
2119            };
2120            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
2121
2122            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
2123            let cfg = MarshaledConfig {
2124                application: mock_app,
2125                marshal: marshal.clone(),
2126                shards: shards.clone(),
2127                scheme_provider: ConstantProvider::new(schemes[0].clone()),
2128                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
2129                strategy: Sequential,
2130            };
2131            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
2132
2133            // Create parent block at height 1 so the commitment is well-formed.
2134            let parent_ctx = CodingCtx {
2135                round: Round::new(Epoch::zero(), View::new(1)),
2136                leader: default_leader(),
2137                parent: (View::zero(), genesis_commitment()),
2138            };
2139            let parent = make_coding_block(parent_ctx, genesis.digest(), Height::new(1), 100);
2140            let coded_parent = CodedBlock::new(parent.clone(), coding_config, &Sequential);
2141            let parent_commitment = coded_parent.commitment();
2142            shards.proposed(Round::new(Epoch::zero(), View::new(1)), coded_parent);
2143
2144            // Build a block with context A (commitment hash uses this context).
2145            let round_a = Round::new(Epoch::zero(), View::new(2));
2146            let context_a = CodingCtx {
2147                round: round_a,
2148                leader: me.clone(),
2149                parent: (View::new(1), parent_commitment),
2150            };
2151            let block_a = make_coding_block(context_a, parent.digest(), Height::new(2), 200);
2152            let coded_block_a: CodedBlock<_, ReedSolomon<Sha256>, Sha256> =
2153                CodedBlock::new(block_a, coding_config, &Sequential);
2154            let commitment_a = coded_block_a.commitment();
2155
2156            // Verify using a different consensus context B (hash mismatch).
2157            let round_b = Round::new(Epoch::zero(), View::new(3));
2158            let context_b = CodingCtx {
2159                round: round_b,
2160                leader: participants[1].clone(),
2161                parent: (View::new(1), parent_commitment),
2162            };
2163
2164            let verify_rx = marshaled.verify(context_b, commitment_a).await;
2165            select! {
2166                result = verify_rx => {
2167                    assert!(
2168                        !result.unwrap(),
2169                        "mismatched context digest should be rejected"
2170                    );
2171                },
2172                _ = context.sleep(Duration::from_secs(5)) => {
2173                    panic!("verify should reject mismatched context digest promptly");
2174                },
2175            }
2176        })
2177    }
2178
2179    #[test_traced("WARN")]
2180    fn test_reproposal_certify_recovers_after_verify_receiver_drop() {
2181        let runner = deterministic::Runner::timed(Duration::from_secs(30));
2182        runner.start(|mut context| async move {
2183            let Fixture {
2184                participants,
2185                schemes,
2186                ..
2187            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2188            let mut oracle = setup_network_with_participants(
2189                context.child("network"),
2190                NZUsize!(1),
2191                participants.clone(),
2192            )
2193            .await;
2194
2195            let me = participants[0].clone();
2196            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2197
2198            let setup = CodingHarness::setup_validator(
2199                context.child("validator").with_attribute("index", 0),
2200                &mut oracle,
2201                me.clone(),
2202                ConstantProvider::new(schemes[0].clone()),
2203            )
2204            .await;
2205            let marshal = setup.mailbox;
2206            let shards = setup.extra;
2207
2208            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
2209            let cfg = MarshaledConfig {
2210                application: mock_app,
2211                marshal: marshal.clone(),
2212                shards: shards.clone(),
2213                scheme_provider: ConstantProvider::new(schemes[0].clone()),
2214                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
2215                strategy: Sequential,
2216            };
2217            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
2218
2219            let genesis_ctx = CodingCtx {
2220                round: Round::zero(),
2221                leader: default_leader(),
2222                parent: (View::zero(), genesis_commitment()),
2223            };
2224            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
2225
2226            // Build a valid boundary re-proposal, but keep it unavailable until
2227            // after the optimistic verify receiver has been dropped.
2228            let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
2229            let boundary_round = Round::new(Epoch::zero(), View::new(boundary_height.get()));
2230            let boundary_context = CodingCtx {
2231                round: boundary_round,
2232                leader: me.clone(),
2233                parent: (View::zero(), genesis_commitment()),
2234            };
2235            let boundary_block = make_coding_block(
2236                boundary_context,
2237                genesis.digest(),
2238                boundary_height,
2239                boundary_height.get() * 100,
2240            );
2241            let coded_boundary = CodedBlock::new(boundary_block, coding_config, &Sequential);
2242            let boundary_commitment = coded_boundary.commitment();
2243            let reproposal_round = Round::new(Epoch::zero(), View::new(boundary_height.get() + 1));
2244            let reproposal_context = CodingCtx {
2245                round: reproposal_round,
2246                leader: me,
2247                parent: (View::new(boundary_height.get()), boundary_commitment),
2248            };
2249
2250            // Start verify, then drop the receiver before the block is available.
2251            let verify_rx = marshaled
2252                .verify(reproposal_context, boundary_commitment)
2253                .await;
2254            drop(verify_rx);
2255            context.sleep(Duration::from_millis(10)).await;
2256
2257            shards.proposed(boundary_round, coded_boundary);
2258            context.sleep(Duration::from_millis(10)).await;
2259
2260            // Certify should not return the stale closed certification gate task; it
2261            // should recover through the embedded-context certification path.
2262            let certify_rx = marshaled
2263                .certify(reproposal_round, boundary_commitment)
2264                .await;
2265            select! {
2266                result = certify_rx => {
2267                    assert!(
2268                        result.expect("certify result missing"),
2269                        "certify should recover after verify receiver drop"
2270                    );
2271                },
2272                _ = context.sleep(Duration::from_secs(5)) => {
2273                    panic!("certify should recover after verify receiver drop");
2274                },
2275            }
2276        })
2277    }
2278
2279    #[test_traced("WARN")]
2280    fn test_reproposal_missing_block_does_not_synthesize_false() {
2281        let runner = deterministic::Runner::timed(Duration::from_secs(30));
2282        runner.start(|mut context| async move {
2283            let Fixture {
2284                participants,
2285                schemes,
2286                ..
2287            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2288            let mut oracle = setup_network_with_participants(
2289                context.child("network"),
2290                NZUsize!(1),
2291                participants.clone(),
2292            )
2293            .await;
2294
2295            let me = participants[0].clone();
2296            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2297
2298            let setup = CodingHarness::setup_validator(
2299                context.child("validator").with_attribute("index", 0),
2300                &mut oracle,
2301                me.clone(),
2302                ConstantProvider::new(schemes[0].clone()),
2303            )
2304            .await;
2305            let marshal = setup.mailbox;
2306            let shards = setup.extra;
2307
2308            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
2309            let cfg = MarshaledConfig {
2310                application: mock_app,
2311                marshal: marshal.clone(),
2312                shards: shards.clone(),
2313                scheme_provider: ConstantProvider::new(schemes[0].clone()),
2314                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
2315                strategy: Sequential,
2316            };
2317            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
2318
2319            // Re-proposal payload with valid coding config, but no block available.
2320            let missing_payload = TestCommitment::from((
2321                Sha256::hash(&[b"missing_block"]),
2322                Sha256::hash(&[b"missing_root"]),
2323                Sha256::hash(&[b"missing_context"]),
2324                coding_config,
2325            ));
2326            let round = Round::new(Epoch::zero(), View::new(1));
2327            let reproposal_context = CodingCtx {
2328                round,
2329                leader: me,
2330                parent: (View::zero(), missing_payload),
2331            };
2332
2333            // Verify must not synthesize `false` when the block cannot be fetched.
2334            let verify_rx = marshaled.verify(reproposal_context, missing_payload).await;
2335
2336            // Ensure the certification gate task has registered its subscription, then
2337            // force cancellation by pruning the missing commitment.
2338            context.sleep(Duration::from_millis(100)).await;
2339            shards.retire(core::Retirement {
2340                round_floor: round,
2341                exact_retirements: vec![missing_payload],
2342            });
2343
2344            select! {
2345                result = verify_rx => {
2346                    assert!(
2347                        result.is_err(),
2348                        "verify should resolve without explicit false when re-proposal block is unavailable"
2349                    );
2350                },
2351                _ = context.sleep(Duration::from_secs(5)) => {
2352                    panic!("verify should resolve promptly when re-proposal block is unavailable");
2353                },
2354            }
2355
2356            // Certify should not surface the closed certification gate task as the final result.
2357            // With no block available, it remains pending on the recovery path until the
2358            // certifier's caller times out or data arrives.
2359            let mut certify_rx = marshaled.certify(round, missing_payload).await;
2360            context.sleep(Duration::from_millis(100)).await;
2361            assert!(
2362                matches!(
2363                    certify_rx.try_recv(),
2364                    Err(commonware_utils::channel::oneshot::error::TryRecvError::Empty)
2365                ),
2366                "certify should remain pending without explicit false or stale cancellation"
2367            );
2368            drop(certify_rx);
2369        })
2370    }
2371
2372    #[test_traced("WARN")]
2373    fn test_core_subscription_closes_when_coding_buffer_prunes_missing_commitment() {
2374        let runner = deterministic::Runner::timed(Duration::from_secs(30));
2375        runner.start(|mut context| async move {
2376            let Fixture {
2377                participants,
2378                schemes,
2379                ..
2380            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2381            let mut oracle = setup_network_with_participants(
2382                context.child("network"),
2383                NZUsize!(1),
2384                participants.clone(),
2385            )
2386            .await;
2387
2388            let setup = CodingHarness::setup_validator(
2389                context.child("validator").with_attribute("index", 0),
2390                &mut oracle,
2391                participants[0].clone(),
2392                ConstantProvider::new(schemes[0].clone()),
2393            )
2394            .await;
2395            let marshal = setup.mailbox;
2396            let shards = setup.extra;
2397
2398            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2399            let missing_commitment = TestCommitment::from((
2400                Sha256::hash(&[b"missing_block"]),
2401                Sha256::hash(&[b"missing_root"]),
2402                Sha256::hash(&[b"missing_context"]),
2403                coding_config,
2404            ));
2405            let round = Round::new(Epoch::zero(), View::new(1));
2406
2407            // Subscribe through the core actor. This internally subscribes to the
2408            // coding shard buffer and registers local waiters.
2409            let block_rx = marshal.subscribe_by_commitment(
2410                missing_commitment,
2411                core::CommitmentFallback::FetchByRound { round },
2412            );
2413
2414            // Allow core actor to register the underlying buffer subscription.
2415            context.sleep(Duration::from_millis(100)).await;
2416
2417            // Prune the missing commitment in the shard engine, which should cancel
2418            // the underlying buffer subscription.
2419            shards.retire(core::Retirement {
2420                round_floor: round,
2421                exact_retirements: vec![missing_commitment],
2422            });
2423
2424            // The core actor must surface cancellation by closing the subscription,
2425            // not by panicking or leaving the waiter parked indefinitely.
2426            select! {
2427                result = block_rx => {
2428                    assert!(
2429                        result.is_err(),
2430                        "core subscription should close when coding buffer drops subscription"
2431                    );
2432                },
2433                _ = context.sleep(Duration::from_secs(5)) => {
2434                    panic!("core subscription should resolve promptly after coding prune");
2435                },
2436            }
2437        })
2438    }
2439
2440    #[test_traced("WARN")]
2441    fn test_coding_floor_preserves_registered_commitment_subscription() {
2442        let runner = deterministic::Runner::timed(Duration::from_secs(30));
2443        runner.start(|mut context| async move {
2444            let Fixture {
2445                participants,
2446                schemes,
2447                ..
2448            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2449            let mut oracle = setup_network_with_participants(
2450                context.child("network"),
2451                NZUsize!(1),
2452                participants.clone(),
2453            )
2454            .await;
2455
2456            let setup = CodingHarness::setup_validator(
2457                context.child("validator").with_attribute("index", 0),
2458                &mut oracle,
2459                participants[0].clone(),
2460                ConstantProvider::new(schemes[0].clone()),
2461            )
2462            .await;
2463            let marshal = setup.mailbox;
2464            let shards = setup.extra;
2465
2466            let missing_round = Round::new(Epoch::zero(), View::new(1));
2467            let missing = CodingHarness::make_test_block(
2468                Sha256::hash(&[b""]),
2469                CodingHarness::genesis_parent_commitment(NUM_VALIDATORS as u16),
2470                Height::new(1),
2471                100,
2472                NUM_VALIDATORS as u16,
2473            );
2474            let missing_commitment = missing.commitment();
2475            shards.discovered(missing_commitment, participants[1].clone(), missing_round);
2476
2477            let subscription =
2478                marshal.subscribe_by_commitment(missing_commitment, core::CommitmentFallback::Wait);
2479            context.sleep(Duration::from_millis(100)).await;
2480
2481            let floor_round = Round::new(Epoch::zero(), View::new(2));
2482            let floor = CodingHarness::make_test_block(
2483                missing.digest(),
2484                missing_commitment,
2485                Height::new(2),
2486                200,
2487                NUM_VALIDATORS as u16,
2488            );
2489            let floor_commitment = floor.commitment();
2490            shards.proposed(floor_round, floor);
2491            assert!(shards.get(floor_commitment).await.is_some());
2492            marshal.set_floor(CodingHarness::make_finalization(
2493                Proposal {
2494                    round: floor_round,
2495                    parent: View::new(1),
2496                    payload: floor_commitment,
2497                },
2498                &schemes,
2499                QUORUM,
2500            ));
2501
2502            while marshal.get_processed_height().await != Some(Height::new(2)) {
2503                context.sleep(Duration::from_millis(10)).await;
2504            }
2505
2506            shards.proposed(Round::new(Epoch::zero(), View::new(3)), missing);
2507            select! {
2508                result = subscription => {
2509                    let block = result.expect("floor update closed commitment subscription");
2510                    assert_eq!(block.commitment(), missing_commitment);
2511                },
2512                _ = context.sleep(Duration::from_secs(5)) => {
2513                    panic!("commitment subscription did not resolve after local ingress");
2514                },
2515            }
2516        })
2517    }
2518
2519    #[test_traced("WARN")]
2520    fn test_marshaled_rejects_unsupported_epoch() {
2521        #[derive(Clone)]
2522        struct LimitedEpocher {
2523            inner: FixedEpocher,
2524            max_epoch: u64,
2525        }
2526
2527        impl Epocher for LimitedEpocher {
2528            fn containing(&self, height: Height) -> Option<crate::types::EpochInfo> {
2529                let bounds = self.inner.containing(height)?;
2530                if bounds.epoch().get() > self.max_epoch {
2531                    None
2532                } else {
2533                    Some(bounds)
2534                }
2535            }
2536
2537            fn first(&self, epoch: Epoch) -> Option<Height> {
2538                if epoch.get() > self.max_epoch {
2539                    None
2540                } else {
2541                    self.inner.first(epoch)
2542                }
2543            }
2544
2545            fn last(&self, epoch: Epoch) -> Option<Height> {
2546                if epoch.get() > self.max_epoch {
2547                    None
2548                } else {
2549                    self.inner.last(epoch)
2550                }
2551            }
2552        }
2553
2554        let runner = deterministic::Runner::timed(Duration::from_secs(60));
2555        runner.start(|mut context| async move {
2556            let Fixture {
2557                participants,
2558                schemes,
2559                ..
2560            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2561            let mut oracle = setup_network_with_participants(
2562                context.child("network"),
2563                NZUsize!(1),
2564                participants.clone(),
2565            )
2566            .await;
2567
2568            let me = participants[0].clone();
2569            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2570
2571            let setup = CodingHarness::setup_validator(
2572                context.child("validator").with_attribute("index", 0),
2573                &mut oracle,
2574                me.clone(),
2575                ConstantProvider::new(schemes[0].clone()),
2576            )
2577            .await;
2578            let marshal = setup.mailbox;
2579            let shards = setup.extra;
2580
2581            let genesis_ctx = CodingCtx {
2582                round: Round::zero(),
2583                leader: default_leader(),
2584                parent: (View::zero(), genesis_commitment()),
2585            };
2586            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
2587
2588            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
2589            let limited_epocher = LimitedEpocher {
2590                inner: FixedEpocher::new(BLOCKS_PER_EPOCH),
2591                max_epoch: 0,
2592            };
2593            let cfg = MarshaledConfig {
2594                application: mock_app,
2595                marshal: marshal.clone(),
2596                shards: shards.clone(),
2597                scheme_provider: ConstantProvider::new(schemes[0].clone()),
2598                epocher: limited_epocher,
2599                strategy: Sequential,
2600            };
2601            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
2602
2603            // Create a parent block at height 19 (last block in epoch 0, which is supported)
2604            let parent_ctx = CodingCtx {
2605                round: Round::new(Epoch::zero(), View::new(19)),
2606                leader: default_leader(),
2607                parent: (View::zero(), genesis_commitment()),
2608            };
2609            let parent = make_coding_block(parent_ctx, genesis.digest(), Height::new(19), 1000);
2610            let parent_digest = parent.digest();
2611            let coded_parent = CodedBlock::new(parent.clone(), coding_config, &Sequential);
2612            let parent_commitment = coded_parent.commitment();
2613            shards.proposed(Round::new(Epoch::zero(), View::new(19)), coded_parent);
2614
2615            // Create a block at height 20 (first block in epoch 1, which is NOT supported)
2616            let block_ctx = CodingCtx {
2617                round: Round::new(Epoch::new(1), View::new(20)),
2618                leader: default_leader(),
2619                parent: (View::new(19), parent_commitment),
2620            };
2621            let block = make_coding_block(block_ctx, parent_digest, Height::new(20), 2000);
2622            let coded_block = CodedBlock::new(block.clone(), coding_config, &Sequential);
2623            let block_commitment = coded_block.commitment();
2624            shards.proposed(Round::new(Epoch::new(1), View::new(20)), coded_block);
2625
2626            context.sleep(Duration::from_millis(10)).await;
2627
2628            // In the coding marshal, verify() returns shard validity while deferred_verify
2629            // runs in the background. We need to use certify() to get the deferred_verify result.
2630            let unsupported_round = Round::new(Epoch::new(1), View::new(20));
2631            let unsupported_context = CodingCtx {
2632                round: unsupported_round,
2633                leader: me.clone(),
2634                parent: (View::new(19), parent_commitment),
2635            };
2636
2637            // Call verify to kick off deferred verification
2638            let _shard_validity = marshaled
2639                .verify(unsupported_context, block_commitment)
2640                .await;
2641
2642            // Use certify to get the actual deferred_verify result
2643            let certify_result = marshaled
2644                .certify(unsupported_round, block_commitment)
2645                .await
2646                .await;
2647
2648            assert!(
2649                !certify_result.unwrap(),
2650                "Block in unsupported epoch should be rejected"
2651            );
2652        })
2653    }
2654
2655    #[test_traced("WARN")]
2656    fn test_marshaled_rejects_invalid_ancestry() {
2657        let runner = deterministic::Runner::timed(Duration::from_secs(60));
2658        runner.start(|mut context| async move {
2659            let Fixture {
2660                participants,
2661                schemes,
2662                ..
2663            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2664            let mut oracle = setup_network_with_participants(
2665                context.child("network"),
2666                NZUsize!(1),
2667                participants.clone(),
2668            )
2669            .await;
2670
2671            let me = participants[0].clone();
2672            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2673
2674            let setup = CodingHarness::setup_validator(
2675                context.child("validator").with_attribute("index", 0),
2676                &mut oracle,
2677                me.clone(),
2678                ConstantProvider::new(schemes[0].clone()),
2679            )
2680            .await;
2681            let marshal = setup.mailbox;
2682            let shards = setup.extra;
2683
2684            // Create genesis block
2685            let genesis_ctx = CodingCtx {
2686                round: Round::zero(),
2687                leader: default_leader(),
2688                parent: (View::zero(), genesis_commitment()),
2689            };
2690            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
2691
2692            // Wrap with Marshaled verifier
2693            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
2694            let cfg = MarshaledConfig {
2695                application: mock_app,
2696                marshal: marshal.clone(),
2697                shards: shards.clone(),
2698                scheme_provider: ConstantProvider::new(schemes[0].clone()),
2699                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
2700                strategy: Sequential,
2701            };
2702            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
2703
2704            // Test case 1: Non-contiguous height
2705            //
2706            // We need both blocks in the same epoch.
2707            // With BLOCKS_PER_EPOCH=20: epoch 0 is heights 0-19, epoch 1 is heights 20-39
2708            //
2709            // Store honest parent at height 21 (epoch 1)
2710            let honest_parent_ctx = CodingCtx {
2711                round: Round::new(Epoch::new(1), View::new(21)),
2712                leader: default_leader(),
2713                parent: (View::zero(), genesis_commitment()),
2714            };
2715            let honest_parent = make_coding_block(
2716                honest_parent_ctx,
2717                genesis.digest(),
2718                Height::new(BLOCKS_PER_EPOCH.get() + 1),
2719                1000,
2720            );
2721            let parent_digest = honest_parent.digest();
2722            let coded_parent = CodedBlock::new(honest_parent.clone(), coding_config, &Sequential);
2723            let parent_commitment = coded_parent.commitment();
2724            shards.proposed(Round::new(Epoch::new(1), View::new(21)), coded_parent);
2725
2726            // Byzantine proposer broadcasts malicious block at height 35
2727            // The block has the correct context (matching what consensus will provide)
2728            // but contains invalid content (non-contiguous height: 21 -> 35 instead of 21 -> 22)
2729            let byzantine_round = Round::new(Epoch::new(1), View::new(35));
2730            let byzantine_context = CodingCtx {
2731                round: byzantine_round,
2732                leader: me.clone(),
2733                parent: (View::new(21), parent_commitment), // Consensus says parent is at height 21
2734            };
2735            let malicious_block = make_coding_block(
2736                byzantine_context.clone(),
2737                parent_digest,
2738                Height::new(BLOCKS_PER_EPOCH.get() + 15), // Byzantine: non-contiguous height
2739                2000,
2740            );
2741            let coded_malicious =
2742                CodedBlock::new(malicious_block.clone(), coding_config, &Sequential);
2743            let malicious_commitment = coded_malicious.commitment();
2744            shards.proposed(byzantine_round, coded_malicious);
2745
2746            // Small delay to ensure broadcast is processed
2747            context.sleep(Duration::from_millis(10)).await;
2748
2749            // Marshaled.verify() kicks off deferred verification in the background.
2750            // The Marshaled verifier will:
2751            // 1. Fetch honest_parent (height 21) from marshal based on context.parent
2752            // 2. Fetch malicious_block (height 35) from marshal based on digest
2753            // 3. Validate height is contiguous (fail)
2754            // 4. Return false
2755            let _shard_validity = marshaled
2756                .verify(byzantine_context, malicious_commitment)
2757                .await;
2758
2759            // Use certify to get the actual deferred_verify result
2760            let certify_result = marshaled
2761                .certify(byzantine_round, malicious_commitment)
2762                .await
2763                .await;
2764
2765            assert!(
2766                !certify_result.unwrap(),
2767                "Byzantine block with non-contiguous heights should be rejected"
2768            );
2769
2770            // Test case 2: Mismatched parent digest
2771            //
2772            // Create another malicious block with correct context and height
2773            // but referencing the wrong parent digest (genesis instead of honest_parent)
2774            let byzantine_round2 = Round::new(Epoch::new(1), View::new(22));
2775            let byzantine_context2 = CodingCtx {
2776                round: byzantine_round2,
2777                leader: me.clone(),
2778                parent: (View::new(21), parent_commitment), // Consensus says parent is at height 21
2779            };
2780            let malicious_block2 = make_coding_block(
2781                byzantine_context2.clone(),
2782                genesis.digest(), // Byzantine: wrong parent digest
2783                Height::new(BLOCKS_PER_EPOCH.get() + 2),
2784                3000,
2785            );
2786            let coded_malicious2 =
2787                CodedBlock::new(malicious_block2.clone(), coding_config, &Sequential);
2788            let malicious_commitment2 = coded_malicious2.commitment();
2789            shards.proposed(byzantine_round2, coded_malicious2);
2790
2791            // Small delay to ensure broadcast is processed
2792            context.sleep(Duration::from_millis(10)).await;
2793
2794            // Marshaled.verify() kicks off deferred verification in the background.
2795            // The Marshaled verifier will:
2796            // 1. Fetch honest_parent (height 21) from marshal based on context.parent
2797            // 2. Fetch malicious_block (height 22) from marshal based on digest
2798            // 3. Validate height is contiguous
2799            // 4. Validate parent commitment matches (fail)
2800            // 5. Return false
2801            let _shard_validity = marshaled
2802                .verify(byzantine_context2, malicious_commitment2)
2803                .await;
2804
2805            // Use certify to get the actual deferred_verify result
2806            let certify_result = marshaled
2807                .certify(byzantine_round2, malicious_commitment2)
2808                .await
2809                .await;
2810
2811            assert!(
2812                !certify_result.unwrap(),
2813                "Byzantine block with mismatched parent commitment should be rejected"
2814            );
2815        })
2816    }
2817
2818    #[test_traced("WARN")]
2819    fn test_certify_without_prior_verify_crash_recovery() {
2820        // After a crash, consensus may call certify() without a prior verify().
2821        // The certify path should:
2822        //   1. Find no in-progress certification gate task
2823        //   2. Subscribe to the block from the shard engine
2824        //   3. Use the block's embedded context for deferred_verify
2825        //   4. Return Ok(true) for a valid block
2826        let runner = deterministic::Runner::timed(Duration::from_secs(30));
2827        runner.start(|mut context| async move {
2828            let Fixture {
2829                participants,
2830                schemes,
2831                ..
2832            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2833            let mut oracle = setup_network_with_participants(
2834                context.child("network"),
2835                NZUsize!(1),
2836                participants.clone(),
2837            )
2838            .await;
2839
2840            let me = participants[0].clone();
2841            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2842
2843            let setup = CodingHarness::setup_validator(
2844                context.child("validator").with_attribute("index", 0),
2845                &mut oracle,
2846                me.clone(),
2847                ConstantProvider::new(schemes[0].clone()),
2848            )
2849            .await;
2850            let marshal = setup.mailbox;
2851            let shards = setup.extra;
2852
2853            let genesis_ctx = CodingCtx {
2854                round: Round::zero(),
2855                leader: default_leader(),
2856                parent: (View::zero(), genesis_commitment()),
2857            };
2858            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
2859
2860            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
2861            let cfg = MarshaledConfig {
2862                application: mock_app,
2863                marshal: marshal.clone(),
2864                shards: shards.clone(),
2865                scheme_provider: ConstantProvider::new(schemes[0].clone()),
2866                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
2867                strategy: Sequential,
2868            };
2869            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
2870
2871            // Create parent at height 1.
2872            let parent_round = Round::new(Epoch::zero(), View::new(1));
2873            let parent_ctx = CodingCtx {
2874                round: parent_round,
2875                leader: default_leader(),
2876                parent: (View::zero(), genesis_commitment()),
2877            };
2878            let parent = make_coding_block(parent_ctx, genesis.digest(), Height::new(1), 100);
2879            let coded_parent = CodedBlock::new(parent.clone(), coding_config, &Sequential);
2880            let parent_commitment = coded_parent.commitment();
2881            shards.proposed(parent_round, coded_parent);
2882
2883            // Create child at height 2.
2884            let child_round = Round::new(Epoch::zero(), View::new(2));
2885            let child_ctx = CodingCtx {
2886                round: child_round,
2887                leader: me.clone(),
2888                parent: (View::new(1), parent_commitment),
2889            };
2890            let child = make_coding_block(child_ctx, parent.digest(), Height::new(2), 200);
2891            let coded_child = CodedBlock::new(child, coding_config, &Sequential);
2892            let child_commitment = coded_child.commitment();
2893            shards.proposed(child_round, coded_child);
2894
2895            context.sleep(Duration::from_millis(10)).await;
2896
2897            // Call certify directly without any prior verify (simulating crash recovery).
2898            let certify_rx = marshaled.certify(child_round, child_commitment).await;
2899            select! {
2900                result = certify_rx => {
2901                    assert!(
2902                        result.unwrap(),
2903                        "certify without prior verify should succeed for valid block"
2904                    );
2905                },
2906                _ = context.sleep(Duration::from_secs(5)) => {
2907                    panic!("certify should complete within timeout");
2908                },
2909            }
2910        })
2911    }
2912
2913    /// Without a registered gate, certify runs the application against the
2914    /// block's embedded context. A live rejection there must reach consensus.
2915    #[test_traced("WARN")]
2916    fn test_certify_without_prior_verify_honors_application_rejection() {
2917        let runner = deterministic::Runner::timed(Duration::from_secs(30));
2918        runner.start(|mut context| async move {
2919            let Fixture {
2920                participants,
2921                schemes,
2922                ..
2923            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
2924            let mut oracle = setup_network_with_participants(
2925                context.child("network"),
2926                NZUsize!(1),
2927                participants.clone(),
2928            )
2929            .await;
2930
2931            let me = participants[0].clone();
2932            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
2933
2934            let setup = CodingHarness::setup_validator(
2935                context.child("validator").with_attribute("index", 0),
2936                &mut oracle,
2937                me.clone(),
2938                ConstantProvider::new(schemes[0].clone()),
2939            )
2940            .await;
2941            let marshal = setup.mailbox;
2942            let shards = setup.extra;
2943
2944            let genesis_ctx = CodingCtx {
2945                round: Round::zero(),
2946                leader: default_leader(),
2947                parent: (View::zero(), genesis_commitment()),
2948            };
2949            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
2950
2951            let mock_app: MockVerifyingApp<CodingB, S> =
2952                MockVerifyingApp::with_verify_result(false);
2953            let cfg = MarshaledConfig {
2954                application: mock_app,
2955                marshal: marshal.clone(),
2956                shards: shards.clone(),
2957                scheme_provider: ConstantProvider::new(schemes[0].clone()),
2958                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
2959                strategy: Sequential,
2960            };
2961            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
2962
2963            // Create parent at height 1.
2964            let parent_round = Round::new(Epoch::zero(), View::new(1));
2965            let parent_ctx = CodingCtx {
2966                round: parent_round,
2967                leader: default_leader(),
2968                parent: (View::zero(), genesis_commitment()),
2969            };
2970            let parent = make_coding_block(parent_ctx, genesis.digest(), Height::new(1), 100);
2971            let coded_parent = CodedBlock::new(parent.clone(), coding_config, &Sequential);
2972            let parent_commitment = coded_parent.commitment();
2973            shards.proposed(parent_round, coded_parent);
2974
2975            // Create child at height 2.
2976            let child_round = Round::new(Epoch::zero(), View::new(2));
2977            let child_ctx = CodingCtx {
2978                round: child_round,
2979                leader: me.clone(),
2980                parent: (View::new(1), parent_commitment),
2981            };
2982            let child = make_coding_block(child_ctx, parent.digest(), Height::new(2), 200);
2983            let coded_child = CodedBlock::new(child, coding_config, &Sequential);
2984            let child_commitment = coded_child.commitment();
2985            shards.proposed(child_round, coded_child);
2986
2987            context.sleep(Duration::from_millis(10)).await;
2988
2989            // No prior verify, so no gate exists and certify falls through to
2990            // the embedded-context path.
2991            let certify_rx = marshaled.certify(child_round, child_commitment).await;
2992            select! {
2993                result = certify_rx => {
2994                    assert!(
2995                        !result.expect("certify result missing"),
2996                        "certify must propagate the application rejection"
2997                    );
2998                },
2999                _ = context.sleep(Duration::from_secs(5)) => {
3000                    panic!("certify should resolve promptly");
3001                },
3002            }
3003        })
3004    }
3005
3006    /// A Byzantine leader can deliver assigned shards to only f+1 honest
3007    /// validators and form a notarization with its own vote, leaving no peer
3008    /// able to serve a full-block fetch. A validator that never saw the
3009    /// proposal then holds only buffered sender-indexed gossip shards, and
3010    /// draining them requires the reconstruction interest that `certify`
3011    /// registers with the shard engine (`shards.notarized`).
3012    #[test_traced("WARN")]
3013    fn test_certify_reconstructs_from_buffered_gossip_shards() {
3014        let runner = deterministic::Runner::timed(Duration::from_secs(30));
3015        runner.start(|mut context| async move {
3016            let Fixture {
3017                participants,
3018                schemes,
3019                ..
3020            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3021            let mut oracle = setup_network_with_participants(
3022                context.child("network"),
3023                NZUsize!(1),
3024                participants.clone(),
3025            )
3026            .await;
3027
3028            let me = participants[0].clone();
3029            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3030
3031            let setup = CodingHarness::setup_validator(
3032                context.child("validator").with_attribute("index", 0),
3033                &mut oracle,
3034                me.clone(),
3035                ConstantProvider::new(schemes[0].clone()),
3036            )
3037            .await;
3038            let marshal = setup.mailbox;
3039            let shards = setup.extra;
3040
3041            // Register the peers' shard-channel senders and link the network.
3042            // No peer runs a marshal actor, so round-bound block fetches go
3043            // unanswered.
3044            let mut peer_senders = Vec::new();
3045            for peer in participants.iter().skip(1) {
3046                let (sender, _receiver) = oracle
3047                    .control(peer.clone())
3048                    .register(2, TEST_QUOTA)
3049                    .await
3050                    .unwrap();
3051                peer_senders.push(sender);
3052            }
3053            setup_network_links(&mut oracle, &participants, LINK).await;
3054
3055            let genesis_ctx = CodingCtx {
3056                round: Round::zero(),
3057                leader: default_leader(),
3058                parent: (View::zero(), genesis_commitment()),
3059            };
3060            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
3061
3062            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
3063            let cfg = MarshaledConfig {
3064                application: mock_app,
3065                marshal: marshal.clone(),
3066                shards: shards.clone(),
3067                scheme_provider: ConstantProvider::new(schemes[0].clone()),
3068                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
3069                strategy: Sequential,
3070            };
3071            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
3072
3073            // The parent is available locally. The notarized candidate is not:
3074            // this validator never saw its proposal.
3075            let parent_round = Round::new(Epoch::zero(), View::new(1));
3076            let parent_ctx = CodingCtx {
3077                round: parent_round,
3078                leader: default_leader(),
3079                parent: (View::zero(), genesis_commitment()),
3080            };
3081            let parent = make_coding_block(parent_ctx, genesis.digest(), Height::new(1), 100);
3082            let coded_parent = CodedBlock::new(parent.clone(), coding_config, &Sequential);
3083            let parent_commitment = coded_parent.commitment();
3084            shards.proposed(parent_round, coded_parent);
3085
3086            let child_round = Round::new(Epoch::zero(), View::new(2));
3087            let child_ctx = CodingCtx {
3088                round: child_round,
3089                leader: participants[1].clone(),
3090                parent: (View::new(1), parent_commitment),
3091            };
3092            let child = make_coding_block(child_ctx, parent.digest(), Height::new(2), 200);
3093            let coded_child: TestCodedBlock = CodedBlock::new(child, coding_config, &Sequential);
3094            let child_commitment = coded_child.commitment();
3095
3096            // Deliver each peer's sender-indexed gossip shard. Without leader
3097            // discovery or reconstruction interest, the shard engine only
3098            // buffers them.
3099            for (i, sender) in peer_senders.iter_mut().enumerate() {
3100                let index = u16::try_from(i + 1).expect("peer index fits in u16");
3101                let shard = coded_child.shard(index).expect("missing shard").encode();
3102                sender.send(Recipients::One(me.clone()), shard, true);
3103            }
3104            context.sleep(Duration::from_millis(250)).await;
3105
3106            // Certify must register reconstruction interest with the shard
3107            // engine, drain the buffered shards, and verify the reconstructed
3108            // block through its embedded context.
3109            let certify_rx = marshaled.certify(child_round, child_commitment).await;
3110            select! {
3111                result = certify_rx => {
3112                    assert!(
3113                        result.expect("certify result missing"),
3114                        "certify must reconstruct the candidate from buffered gossip shards"
3115                    );
3116                },
3117                _ = context.sleep(Duration::from_secs(5)) => {
3118                    panic!("certify did not reconstruct from buffered gossip shards");
3119                },
3120            }
3121        })
3122    }
3123
3124    /// Regression test: a Byzantine leader must not be able to crash honest nodes
3125    /// by proposing a `Commitment` with invalid `CodingConfig` bytes (e.g.
3126    /// zero-valued `NonZeroU16` fields). The fix validates the embedded config
3127    /// during deserialization so malformed commitments are rejected at the codec
3128    /// level before reaching `verify()`.
3129    #[test_traced("WARN")]
3130    fn test_malformed_commitment_config_rejected_at_deserialization() {
3131        use commonware_codec::{Encode, ReadExt};
3132
3133        // Construct a Commitment with all-zero bytes (invalid CodingConfig:
3134        // minimum_shards=0, extra_shards=0). Serialize it and attempt to
3135        // deserialize -- this must fail.
3136        let malformed_bytes = [0u8; <TestCommitment as FixedSize>::SIZE];
3137        let result = TestCommitment::read(&mut &malformed_bytes[..]);
3138        assert!(
3139            result.is_err(),
3140            "deserialization of Commitment with zeroed CodingConfig must fail"
3141        );
3142
3143        // A validly-constructed Commitment must still round-trip.
3144        let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3145        let valid = TestCommitment::from((
3146            Sha256::hash(&[b"block"]),
3147            Sha256::hash(&[b"root"]),
3148            Sha256::hash(&[b"context"]),
3149            coding_config,
3150        ));
3151        let encoded = valid.encode();
3152        let decoded =
3153            TestCommitment::read(&mut &encoded[..]).expect("valid Commitment must deserialize");
3154        assert_eq!(valid, decoded);
3155    }
3156
3157    #[test_traced("WARN")]
3158    fn test_certify_propagates_application_verify_failure() {
3159        let runner = deterministic::Runner::timed(Duration::from_secs(30));
3160        runner.start(|mut context| async move {
3161            // 1) Set up a single validator marshal stack.
3162            let Fixture {
3163                participants,
3164                schemes,
3165                ..
3166            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3167            let mut oracle = setup_network_with_participants(
3168                context.child("network"),
3169                NZUsize!(1),
3170                participants.clone(),
3171            )
3172            .await;
3173
3174            let me = participants[0].clone();
3175            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3176
3177            let setup = CodingHarness::setup_validator(
3178                context.child("validator").with_attribute("index", 0),
3179                &mut oracle,
3180                me.clone(),
3181                ConstantProvider::new(schemes[0].clone()),
3182            )
3183            .await;
3184            let marshal = setup.mailbox;
3185            let shards = setup.extra;
3186
3187            let genesis_ctx = CodingCtx {
3188                round: Round::zero(),
3189                leader: default_leader(),
3190                parent: (View::zero(), genesis_commitment()),
3191            };
3192            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
3193            // 2) Force application verification to fail in deferred verification.
3194            let mock_app: MockVerifyingApp<CodingB, S> =
3195                MockVerifyingApp::with_verify_result(false);
3196
3197            let cfg = MarshaledConfig {
3198                application: mock_app,
3199                marshal: marshal.clone(),
3200                shards: shards.clone(),
3201                scheme_provider: ConstantProvider::new(schemes[0].clone()),
3202                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
3203                strategy: Sequential,
3204            };
3205            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
3206
3207            let parent_round = Round::new(Epoch::zero(), View::new(1));
3208            let parent_context = CodingCtx {
3209                round: parent_round,
3210                leader: me.clone(),
3211                parent: (View::zero(), genesis_commitment()),
3212            };
3213            let parent = make_coding_block(parent_context, genesis.digest(), Height::new(1), 100);
3214            let coded_parent = CodedBlock::new(parent.clone(), coding_config, &Sequential);
3215            let parent_commitment = coded_parent.commitment();
3216            shards.proposed(parent_round, coded_parent);
3217
3218            // 3) Publish a valid child so optimistic verify can succeed.
3219            let round = Round::new(Epoch::zero(), View::new(2));
3220            let verify_context = CodingCtx {
3221                round,
3222                leader: me,
3223                parent: (View::new(1), parent_commitment),
3224            };
3225            let block =
3226                make_coding_block(verify_context.clone(), parent.digest(), Height::new(2), 200);
3227            let coded_block = CodedBlock::new(block, coding_config, &Sequential);
3228            let commitment = coded_block.commitment();
3229            shards.proposed(round, coded_block);
3230
3231            context.sleep(Duration::from_millis(10)).await;
3232
3233            let optimistic = marshaled.verify(verify_context, commitment).await;
3234            assert!(
3235                optimistic.await.expect("verify result missing"),
3236                "optimistic verify should pass pre-checks and schedule deferred verification"
3237            );
3238
3239            // 4) Certify must observe the deferred application failure and return false.
3240            let certify = marshaled.certify(round, commitment).await;
3241            assert!(
3242                !certify.await.expect("certify result missing"),
3243                "certify should propagate deferred application verify failure"
3244            );
3245        })
3246    }
3247
3248    /// Coding analog of the standard equivocation tests. The commitment binds
3249    /// the proposal context, so a proposal that names a different parent for
3250    /// the same commitment fails `validate_proposal` before any certification
3251    /// gate is registered. Certification of the notarized commitment then
3252    /// recovers through the block's embedded context.
3253    #[test_traced("WARN")]
3254    fn test_certify_not_poisoned_by_equivocating_parent_verify() {
3255        let runner = deterministic::Runner::timed(Duration::from_secs(30));
3256        runner.start(|mut context| async move {
3257            let Fixture {
3258                participants,
3259                schemes,
3260                ..
3261            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3262            let mut oracle = setup_network_with_participants(
3263                context.child("network"),
3264                NZUsize!(1),
3265                participants.clone(),
3266            )
3267            .await;
3268
3269            let me = participants[0].clone();
3270            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3271
3272            let setup = CodingHarness::setup_validator(
3273                context.child("validator").with_attribute("index", 0),
3274                &mut oracle,
3275                me.clone(),
3276                ConstantProvider::new(schemes[0].clone()),
3277            )
3278            .await;
3279            let marshal = setup.mailbox;
3280            let shards = setup.extra;
3281
3282            let genesis_ctx = CodingCtx {
3283                round: Round::zero(),
3284                leader: default_leader(),
3285                parent: (View::zero(), genesis_commitment()),
3286            };
3287            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
3288
3289            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
3290            let cfg = MarshaledConfig {
3291                application: mock_app,
3292                marshal: marshal.clone(),
3293                shards: shards.clone(),
3294                scheme_provider: ConstantProvider::new(schemes[0].clone()),
3295                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
3296                strategy: Sequential,
3297            };
3298            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
3299
3300            // The view-1 block: the parent this validator last certified.
3301            let certified_round = Round::new(Epoch::zero(), View::new(1));
3302            let certified_ctx = CodingCtx {
3303                round: certified_round,
3304                leader: default_leader(),
3305                parent: (View::zero(), genesis_commitment()),
3306            };
3307            let certified = make_coding_block(certified_ctx, genesis.digest(), Height::new(1), 100);
3308            let certified_digest = certified.digest();
3309            let coded_certified = CodedBlock::new(certified, coding_config, &Sequential);
3310            let certified_commitment = coded_certified.commitment();
3311            shards.proposed(certified_round, coded_certified);
3312
3313            // The view-2 block: notarized by the network but only nullified
3314            // here, so this validator never certified it.
3315            let notarized_round = Round::new(Epoch::zero(), View::new(2));
3316            let notarized_ctx = CodingCtx {
3317                round: notarized_round,
3318                leader: me.clone(),
3319                parent: (View::new(1), certified_commitment),
3320            };
3321            let notarized = make_coding_block(notarized_ctx, certified_digest, Height::new(2), 200);
3322            let notarized_digest = notarized.digest();
3323            let coded_notarized = CodedBlock::new(notarized, coding_config, &Sequential);
3324            let notarized_commitment = coded_notarized.commitment();
3325            shards.proposed(notarized_round, coded_notarized);
3326
3327            // The view-3 block builds on the view-2 block.
3328            let round = Round::new(Epoch::zero(), View::new(3));
3329            let block_ctx = CodingCtx {
3330                round,
3331                leader: me.clone(),
3332                parent: (View::new(2), notarized_commitment),
3333            };
3334            let block = make_coding_block(block_ctx, notarized_digest, Height::new(3), 300);
3335            let coded_block = CodedBlock::new(block, coding_config, &Sequential);
3336            let commitment = coded_block.commitment();
3337            shards.proposed(round, coded_block);
3338
3339            context.sleep(Duration::from_millis(10)).await;
3340
3341            // The equivocating proposal names the certified view-1 block as
3342            // parent, which this validator's parent selection accepts
3343            // (view 1 certified, view 2 nullified). The commitment's context
3344            // digest exposes the mismatch, so verify rejects it without
3345            // registering a certification gate.
3346            let equivocating_ctx = CodingCtx {
3347                round,
3348                leader: me.clone(),
3349                parent: (View::new(1), certified_commitment),
3350            };
3351            let verify_rx = marshaled.verify(equivocating_ctx, commitment).await;
3352            select! {
3353                result = verify_rx => {
3354                    assert!(
3355                        !result.expect("verify result missing"),
3356                        "the equivocating proposal must not be notarized"
3357                    );
3358                },
3359                _ = context.sleep(Duration::from_secs(5)) => {
3360                    panic!("verify should reject the equivocating proposal promptly");
3361                },
3362            }
3363
3364            // The honest notarization for the same `(round, commitment)`
3365            // arrives. Certification recovers through the embedded context.
3366            let certify_rx = marshaled.certify(round, commitment).await;
3367            select! {
3368                result = certify_rx => {
3369                    assert!(
3370                        result.expect("certify result missing"),
3371                        "certify of the notarized commitment must succeed via the embedded context"
3372                    );
3373                },
3374                _ = context.sleep(Duration::from_secs(5)) => {
3375                    panic!("certify should resolve promptly");
3376                },
3377            }
3378        })
3379    }
3380
3381    #[test_traced("WARN")]
3382    fn test_backfill_block_mismatched_commitment() {
3383        // Regression: when backfilling by Key::Block(commitment), a peer may return
3384        // a coded block with matching inner digest but a different coding commitment.
3385        // If a finalization for this digest is already cached, marshal must reject
3386        // the block unless V::commitment(block) matches the finalization payload.
3387        let runner = deterministic::Runner::timed(Duration::from_secs(30));
3388        runner.start(|mut context| async move {
3389            let Fixture {
3390                participants,
3391                schemes,
3392                ..
3393            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3394            let mut oracle = setup_network_with_participants(
3395                context.child("network"),
3396                NZUsize!(1),
3397                participants[..2].iter().cloned(),
3398            )
3399            .await;
3400
3401            let coding_config_a = coding_config_for_participants(NUM_VALIDATORS as u16);
3402            // Same total shards (4) but different min/extra split produces a different
3403            // coding root and config bytes, yielding a different commitment.
3404            let coding_config_b = commonware_coding::Config {
3405                minimum_shards: coding_config_a.minimum_shards.checked_add(1).unwrap(),
3406                extra_shards: NZU16!(coding_config_a.extra_shards.get() - 1),
3407            };
3408
3409            let v0_setup = CodingHarness::setup_validator(
3410                context.child("validator").with_attribute("index", 0),
3411                &mut oracle,
3412                participants[0].clone(),
3413                ConstantProvider::new(schemes[0].clone()),
3414            )
3415            .await;
3416            let v1_setup = CodingHarness::setup_validator(
3417                context.child("validator").with_attribute("index", 1),
3418                &mut oracle,
3419                participants[1].clone(),
3420                ConstantProvider::new(schemes[1].clone()),
3421            )
3422            .await;
3423
3424            setup_network_links(&mut oracle, &participants[..2], LINK).await;
3425
3426            let mut v0_mailbox = v0_setup.mailbox;
3427            let v1_mailbox = v1_setup.mailbox;
3428
3429            let genesis_ctx = CodingCtx {
3430                round: Round::zero(),
3431                leader: default_leader(),
3432                parent: (View::zero(), genesis_commitment()),
3433            };
3434            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
3435
3436            let round1 = Round::new(Epoch::zero(), View::new(1));
3437            let block1_ctx = CodingCtx {
3438                round: round1,
3439                leader: participants[0].clone(),
3440                parent: (View::zero(), genesis_commitment()),
3441            };
3442            let block1 = make_coding_block(block1_ctx, genesis.digest(), Height::new(1), 100);
3443
3444            let coded_block_a: CodedBlock<_, ReedSolomon<Sha256>, Sha256> =
3445                CodedBlock::new(block1.clone(), coding_config_a, &Sequential);
3446            let commitment_a = coded_block_a.commitment();
3447
3448            let coded_block_b: CodedBlock<_, ReedSolomon<Sha256>, Sha256> =
3449                CodedBlock::new(block1.clone(), coding_config_b, &Sequential);
3450            let commitment_b = coded_block_b.commitment();
3451
3452            assert_eq!(coded_block_a.digest(), coded_block_b.digest());
3453            assert_ne!(commitment_a, commitment_b);
3454
3455            // Validator 1 proposes coded_block_b (same inner block, different coding).
3456            // This stores it in v1's shard engine and actor cache.
3457            assert!(v1_mailbox.verified(round1, coded_block_b.clone()).await);
3458            context.sleep(Duration::from_millis(100)).await;
3459
3460            // Create finalization referencing commitment_a (the "correct" commitment).
3461            let proposal: Proposal<TestCommitment> = Proposal {
3462                round: round1,
3463                parent: View::zero(),
3464                payload: commitment_a,
3465            };
3466            let finalization = CodingHarness::make_finalization(proposal.clone(), &schemes, QUORUM);
3467
3468            // Report finalization to v0. v0 doesn't have the block:
3469            //   - it fetches Key::Block(commitment)
3470            //   - v1 responds with coded_block_b (same digest, wrong commitment)
3471            //   - deliver path must reject because the response commitment does not
3472            //     match the request key
3473            CodingHarness::report_finalization(&mut v0_mailbox, finalization).await;
3474
3475            // Wait for the fetch cycle to complete.
3476            context.sleep(Duration::from_secs(5)).await;
3477
3478            // The mismatched block must not be stored.
3479            let stored = v0_mailbox.get_block(Height::new(1)).await;
3480            assert!(
3481                stored.is_none(),
3482                "v0 should reject backfilled block with mismatched commitment"
3483            );
3484
3485            // Without the block, finalization should not be persisted by height yet.
3486            let stored_finalization = v0_mailbox.get_finalization(Height::new(1)).await;
3487            assert!(
3488                stored_finalization.is_none(),
3489                "finalization should not be archived until matching block is available"
3490            );
3491        })
3492    }
3493
3494    #[test_traced("WARN")]
3495    #[should_panic(expected = "floor block parent commitment mismatch")]
3496    fn test_coding_floor_anchor_panics_on_parent_commitment_mismatch() {
3497        let runner = deterministic::Runner::timed(Duration::from_secs(30));
3498        runner.start(|mut context| async move {
3499            let Fixture {
3500                participants,
3501                schemes,
3502                ..
3503            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3504            let (mailbox, resolver, _actor_handle) = start_coding_actor_with_recording(
3505                context.child("validator"),
3506                "floor-parent-commitment-mismatch",
3507                ConstantProvider::new(schemes[0].clone()),
3508                RecordingCodingBuffer::default(),
3509            )
3510            .await;
3511
3512            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3513            let parent_round = Round::new(Epoch::zero(), View::new(1));
3514            let parent_context = CodingCtx {
3515                round: parent_round,
3516                leader: participants[0].clone(),
3517                parent: (View::zero(), genesis_commitment()),
3518            };
3519            let parent =
3520                make_coding_block(parent_context, Sha256::hash(&[b""]), Height::new(1), 100);
3521
3522            let floor_round = Round::new(Epoch::zero(), View::new(2));
3523            let bad_context = CodingCtx {
3524                round: floor_round,
3525                leader: participants[0].clone(),
3526                parent: (View::new(1), genesis_commitment()),
3527            };
3528            let floor_block = make_coding_block(bad_context, parent.digest(), Height::new(2), 200);
3529            let coded_floor = CodedBlock::new(floor_block, coding_config, &Sequential);
3530            assert_ne!(coded_floor.parent(), coded_floor.context().parent.1.block());
3531
3532            let finalization = CodingHarness::make_finalization(
3533                Proposal::new(
3534                    floor_round,
3535                    View::new(1),
3536                    CodingHarness::commitment(&coded_floor),
3537                ),
3538                &schemes,
3539                QUORUM,
3540            );
3541            resolver.respond_to_next_fetch(coded_floor.encode());
3542            mailbox.set_floor(finalization);
3543            context.sleep(Duration::from_secs(5)).await;
3544        })
3545    }
3546
3547    /// When the scheme provider has no entry for the current epoch,
3548    /// `Marshaled::propose` and `Marshaled::verify` must return a dropped
3549    /// receiver (the consensus engine treats `RecvError` as "abstain").
3550    #[test_traced("WARN")]
3551    fn test_marshaled_missing_scheme_skips_propose_and_verify() {
3552        let runner = deterministic::Runner::timed(Duration::from_secs(30));
3553        runner.start(|mut context| async move {
3554            let Fixture {
3555                participants,
3556                schemes,
3557                ..
3558            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3559            let mut oracle = setup_network_with_participants(
3560                context.child("network"),
3561                NZUsize!(1),
3562                participants.clone(),
3563            )
3564            .await;
3565
3566            let me = participants[0].clone();
3567
3568            let setup = CodingHarness::setup_validator(
3569                context.child("validator").with_attribute("index", 0),
3570                &mut oracle,
3571                me.clone(),
3572                ConstantProvider::new(schemes[0].clone()),
3573            )
3574            .await;
3575
3576            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
3577
3578            let cfg = MarshaledConfig {
3579                application: mock_app,
3580                marshal: setup.mailbox,
3581                shards: setup.extra,
3582                scheme_provider: EmptyProvider,
3583                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
3584                strategy: Sequential,
3585            };
3586            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
3587
3588            let ctx = CodingCtx {
3589                round: Round::new(Epoch::zero(), View::new(1)),
3590                leader: me.clone(),
3591                parent: (View::zero(), genesis_commitment()),
3592            };
3593
3594            // propose with a missing scheme returns a dropped sender
3595            let rx = marshaled.propose(ctx.clone()).await;
3596            assert!(rx.await.is_err());
3597
3598            // verify with a missing scheme returns a dropped sender
3599            let rx = marshaled.verify(ctx, genesis_commitment()).await;
3600            assert!(rx.await.is_err());
3601        });
3602    }
3603
3604    /// Regression: a validator must not vote finalize on a block that is not
3605    /// durably persisted. If `certify` resolves true, the block is on disk for
3606    /// this validator. We assert this by aborting the marshal actor the
3607    /// instant `certify` returns true, then restarting from the same partition.
3608    #[test_traced("WARN")]
3609    fn test_marshaled_certify_persists_block_before_resolving() {
3610        for seed in 0u64..16 {
3611            certify_persists_block_before_resolving_at(seed);
3612        }
3613    }
3614
3615    fn certify_persists_block_before_resolving_at(seed: u64) {
3616        let runner = deterministic::Runner::new(
3617            deterministic::Config::new()
3618                .with_seed(seed)
3619                .with_timeout(Some(Duration::from_secs(60))),
3620        );
3621        runner.start(|mut context| async move {
3622            let Fixture {
3623                participants,
3624                schemes,
3625                ..
3626            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3627            let mut oracle = setup_network_with_participants(
3628                context.child("network"),
3629                NZUsize!(1),
3630                participants.clone(),
3631            )
3632            .await;
3633
3634            let me = participants[0].clone();
3635            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3636
3637            let setup = CodingHarness::setup_validator(
3638                context.child("validator").with_attribute("index", 0),
3639                &mut oracle,
3640                me.clone(),
3641                ConstantProvider::new(schemes[0].clone()),
3642            )
3643            .await;
3644            let marshal = setup.mailbox;
3645            let shards = setup.extra;
3646            let marshal_actor_handle = setup.actor_handle;
3647
3648            let genesis_ctx = CodingCtx {
3649                round: Round::zero(),
3650                leader: default_leader(),
3651                parent: (View::zero(), genesis_commitment()),
3652            };
3653            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
3654
3655            // Push parent (height 1) and child (height 2) into the shards
3656            // engine. These are reconstructable but NOT durably persisted.
3657            let parent_round = Round::new(Epoch::zero(), View::new(1));
3658            let parent_ctx = CodingCtx {
3659                round: parent_round,
3660                leader: default_leader(),
3661                parent: (View::zero(), genesis_commitment()),
3662            };
3663            let parent = make_coding_block(parent_ctx, genesis.digest(), Height::new(1), 100);
3664            let coded_parent = CodedBlock::new(parent.clone(), coding_config, &Sequential);
3665            let parent_commitment = coded_parent.commitment();
3666            shards.proposed(parent_round, coded_parent);
3667
3668            let child_round = Round::new(Epoch::zero(), View::new(2));
3669            let child_ctx = CodingCtx {
3670                round: child_round,
3671                leader: me.clone(),
3672                parent: (View::new(1), parent_commitment),
3673            };
3674            let child = make_coding_block(child_ctx.clone(), parent.digest(), Height::new(2), 200);
3675            let coded_child = CodedBlock::new(child.clone(), coding_config, &Sequential);
3676            let child_commitment = coded_child.commitment();
3677            let child_digest = coded_child.digest();
3678            shards.proposed(child_round, coded_child);
3679
3680            context.sleep(Duration::from_millis(10)).await;
3681
3682            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
3683            let cfg = MarshaledConfig {
3684                application: mock_app,
3685                marshal: marshal.clone(),
3686                shards: shards.clone(),
3687                scheme_provider: ConstantProvider::new(schemes[0].clone()),
3688                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
3689                strategy: Sequential,
3690            };
3691            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
3692
3693            // Optimistic verify - returns shard validity (true).
3694            let shard_validity = marshaled
3695                .verify(child_ctx, child_commitment)
3696                .await
3697                .await
3698                .expect("verify result missing");
3699            assert!(shard_validity, "shard validity should pass");
3700
3701            // Certify - this is the safety gate before finalize voting.
3702            let certify_result = marshaled
3703                .certify(child_round, child_commitment)
3704                .await
3705                .await
3706                .expect("certify result missing");
3707            assert!(certify_result, "certify should succeed");
3708
3709            // Abort marshal immediately after certify returns to prove the
3710            // block is already persisted at that point.
3711            marshal_actor_handle.abort();
3712            drop(marshaled);
3713            drop(marshal);
3714            drop(shards);
3715
3716            // Restart from the same partition. The block must be durably
3717            // persisted - otherwise the validator would have voted finalize
3718            // for a block it cannot serve from local storage.
3719            let setup2 = CodingHarness::setup_validator(
3720                context
3721                    .child("validator_restart")
3722                    .with_attribute("index", 0),
3723                &mut oracle,
3724                me.clone(),
3725                ConstantProvider::new(schemes[0].clone()),
3726            )
3727            .await;
3728            let marshal2 = setup2.mailbox;
3729
3730            let post_restart = marshal2.get_block(&child_digest).await;
3731            assert!(
3732                post_restart.is_some(),
3733                "certify resolved true, so block must be durably persisted"
3734            );
3735        });
3736    }
3737
3738    /// Regression: a leader must be able to recover its own block across an unclean restart.
3739    /// `propose` registers a certification gate, so the leader establishes durability by
3740    /// certifying its own proposal. After certify, the block must survive restart even if
3741    /// `Relay::broadcast` never runs. This is the >= f+1 guarantee for the leader's own block.
3742    #[test_traced("WARN")]
3743    fn test_marshaled_proposed_block_persists_across_restart() {
3744        let runner = deterministic::Runner::timed(Duration::from_secs(60));
3745        runner.start(|mut context| async move {
3746            let Fixture {
3747                participants,
3748                schemes,
3749                ..
3750            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3751            let mut oracle = setup_network_with_participants(
3752                context.child("network"),
3753                NZUsize!(1),
3754                participants.clone(),
3755            )
3756            .await;
3757
3758            let me = participants[0].clone();
3759            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3760
3761            let setup = CodingHarness::setup_validator(
3762                context.child("validator").with_attribute("index", 0),
3763                &mut oracle,
3764                me.clone(),
3765                ConstantProvider::new(schemes[0].clone()),
3766            )
3767            .await;
3768            let marshal = setup.mailbox;
3769            let shards = setup.extra;
3770            let marshal_actor_handle = setup.actor_handle;
3771
3772            let genesis_ctx = CodingCtx {
3773                round: Round::zero(),
3774                leader: default_leader(),
3775                parent: (View::zero(), genesis_commitment()),
3776            };
3777            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
3778            let genesis_parent_commitment = genesis_coding_commitment(&genesis);
3779
3780            // Build the block we want propose() to return. Its embedded context
3781            // uses the proper genesis commitment so the parent lookup matches
3782            // the cached genesis.
3783            let propose_round = Round::new(Epoch::zero(), View::new(1));
3784            let propose_context = CodingCtx {
3785                round: propose_round,
3786                leader: me.clone(),
3787                parent: (View::zero(), genesis_parent_commitment),
3788            };
3789            let block_to_propose = make_coding_block(
3790                propose_context.clone(),
3791                genesis.digest(),
3792                Height::new(1),
3793                100,
3794            );
3795            let block_digest = block_to_propose.digest();
3796            let expected_commitment = CodedBlock::<_, ReedSolomon<Sha256>, Sha256>::new(
3797                block_to_propose.clone(),
3798                coding_config,
3799                &Sequential,
3800            )
3801            .commitment();
3802
3803            let mock_app: MockVerifyingApp<CodingB, S> =
3804                MockVerifyingApp::new().with_propose_result(block_to_propose);
3805            let cfg = MarshaledConfig {
3806                application: mock_app,
3807                marshal: marshal.clone(),
3808                shards: shards.clone(),
3809                scheme_provider: ConstantProvider::new(schemes[0].clone()),
3810                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
3811                strategy: Sequential,
3812            };
3813            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
3814
3815            // Drive the leader-side propose path. `propose` stages the block
3816            // and returns the commitment. Durability is established by the
3817            // certify flush below.
3818            let commitment = marshaled
3819                .propose(propose_context)
3820                .await
3821                .await
3822                .expect("propose should produce a commitment");
3823            assert_eq!(commitment, expected_commitment);
3824
3825            // The leader certifies its own proposal, which awaits the deferred propose
3826            // sync handle and establishes durability before the finalize vote.
3827            assert!(
3828                marshaled
3829                    .certify(propose_round, commitment)
3830                    .await
3831                    .await
3832                    .expect("certify result missing"),
3833                "certify must succeed for the leader's own proposal"
3834            );
3835
3836            // Abort marshal after certify; the leader's own block must be durable.
3837            marshal_actor_handle.abort();
3838            drop(marshaled);
3839            drop(marshal);
3840            drop(shards);
3841
3842            let setup2 = CodingHarness::setup_validator(
3843                context
3844                    .child("validator_restart")
3845                    .with_attribute("index", 0),
3846                &mut oracle,
3847                me.clone(),
3848                ConstantProvider::new(schemes[0].clone()),
3849            )
3850            .await;
3851            let marshal2 = setup2.mailbox;
3852
3853            // The proposer must recover its own block after restart. Without
3854            // the broadcast-path persistence fix, the block lived only in the
3855            // shards engine's in-memory cache and is now gone.
3856            let post_restart = marshal2.get_block(&block_digest).await;
3857            assert!(
3858                post_restart.is_some(),
3859                "proposer should recover its own block after restart"
3860            );
3861        });
3862    }
3863
3864    /// A propose relay with a staged proposal must send it through the shard
3865    /// engine and complete the durability handshake. The freshly built block
3866    /// is nowhere persisted at broadcast time, so the forward fallback has
3867    /// nothing to serve: only the staged-hit path can seed the shard engine.
3868    #[test_traced("WARN")]
3869    fn test_marshaled_propose_relay_sends_staged_block() {
3870        let runner = deterministic::Runner::timed(Duration::from_secs(60));
3871        runner.start(|mut context| async move {
3872            let Fixture {
3873                participants,
3874                schemes,
3875                ..
3876            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3877            let mut oracle = setup_network_with_participants(
3878                context.child("network"),
3879                NZUsize!(1),
3880                participants.clone(),
3881            )
3882            .await;
3883
3884            let me = participants[0].clone();
3885            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3886
3887            let setup = CodingHarness::setup_validator(
3888                context.child("validator").with_attribute("index", 0),
3889                &mut oracle,
3890                me.clone(),
3891                ConstantProvider::new(schemes[0].clone()),
3892            )
3893            .await;
3894            let marshal = setup.mailbox;
3895            let shards = setup.extra;
3896
3897            let genesis_ctx = CodingCtx {
3898                round: Round::zero(),
3899                leader: default_leader(),
3900                parent: (View::zero(), genesis_commitment()),
3901            };
3902            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
3903            let genesis_parent_commitment = genesis_coding_commitment(&genesis);
3904
3905            let propose_round = Round::new(Epoch::zero(), View::new(1));
3906            let propose_context = CodingCtx {
3907                round: propose_round,
3908                leader: me.clone(),
3909                parent: (View::zero(), genesis_parent_commitment),
3910            };
3911            let block_to_propose = make_coding_block(
3912                propose_context.clone(),
3913                genesis.digest(),
3914                Height::new(1),
3915                100,
3916            );
3917            let block_digest = block_to_propose.digest();
3918            let expected_commitment = CodedBlock::<_, ReedSolomon<Sha256>, Sha256>::new(
3919                block_to_propose.clone(),
3920                coding_config,
3921                &Sequential,
3922            )
3923            .commitment();
3924
3925            let mock_app: MockVerifyingApp<CodingB, S> =
3926                MockVerifyingApp::new().with_propose_result(block_to_propose);
3927            let cfg = MarshaledConfig {
3928                application: mock_app,
3929                marshal: marshal.clone(),
3930                shards: shards.clone(),
3931                scheme_provider: ConstantProvider::new(schemes[0].clone()),
3932                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
3933                strategy: Sequential,
3934            };
3935            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
3936
3937            let commitment = marshaled
3938                .propose(propose_context)
3939                .await
3940                .await
3941                .expect("propose should produce a commitment");
3942            assert_eq!(commitment, expected_commitment);
3943
3944            // The relay must take the staged proposal and broadcast its
3945            // shards, seeding the shard engine's local cache.
3946            let subscription = shards.subscribe(commitment);
3947            let _ = marshaled.broadcast(
3948                commitment,
3949                Plan::Propose {
3950                    round: propose_round,
3951                },
3952            );
3953            let cached = subscription
3954                .await
3955                .expect("shard engine must cache the relayed proposal");
3956            assert_eq!(cached.digest(), block_digest);
3957
3958            // The relayed proposal is persisted through the staged ack, so
3959            // certification resolves durably without a flush.
3960            assert!(
3961                marshaled
3962                    .certify(propose_round, commitment)
3963                    .await
3964                    .await
3965                    .expect("certify result missing"),
3966                "certify must succeed for the relayed proposal"
3967            );
3968        });
3969    }
3970
3971    /// Regression: if marshal already holds a verified block for a round
3972    /// (say, persisted by a pre-crash propose whose notarize vote never
3973    /// reached the journal), a restarted leader's `propose` must return
3974    /// that block's commitment instead of rebuilding. The pre-crash
3975    /// commitment may already have been broadcast, so proposing a rebuilt
3976    /// block for the same round would equivocate. The recovered proposal
3977    /// must also be staged for the relay, so the broadcast re-sends its
3978    /// shards and certification resolves through the deduplicated
3979    /// re-persist.
3980    #[test_traced("WARN")]
3981    fn test_propose_reuses_verified_block_on_restart() {
3982        let runner = deterministic::Runner::timed(Duration::from_secs(60));
3983        runner.start(|mut context| async move {
3984            let Fixture {
3985                participants,
3986                schemes,
3987                ..
3988            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
3989            let mut oracle = setup_network_with_participants(
3990                context.child("network"),
3991                NZUsize!(1),
3992                participants.clone(),
3993            )
3994            .await;
3995
3996            let me = participants[0].clone();
3997            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
3998
3999            let setup = CodingHarness::setup_validator(
4000                context.child("validator").with_attribute("index", 0),
4001                &mut oracle,
4002                me.clone(),
4003                ConstantProvider::new(schemes[0].clone()),
4004            )
4005            .await;
4006            let marshal = setup.mailbox;
4007            let shards = setup.extra;
4008
4009            let genesis_ctx = CodingCtx {
4010                round: Round::zero(),
4011                leader: default_leader(),
4012                parent: (View::zero(), genesis_commitment()),
4013            };
4014            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
4015            let genesis_parent_commitment = genesis_coding_commitment(&genesis);
4016
4017            let round = Round::new(Epoch::zero(), View::new(1));
4018            let ctx = CodingCtx {
4019                round,
4020                leader: me.clone(),
4021                parent: (View::zero(), genesis_parent_commitment),
4022            };
4023
4024            // Seed block A in marshal's verified cache for `round`.
4025            let block_a = make_coding_block(ctx.clone(), genesis.digest(), Height::new(1), 100);
4026            let coded_a: CodedBlock<_, ReedSolomon<Sha256>, Sha256> =
4027                CodedBlock::new(block_a.clone(), coding_config, &Sequential);
4028            let commitment_a = coded_a.commitment();
4029            assert!(marshal.verified(round, coded_a).await);
4030
4031            // The app cannot build (`propose` returns None) and its
4032            // verification never completes, so the assertions below hold
4033            // only if the stored block is reused as-is and certification
4034            // resolves through the durability gate registered by the
4035            // recovery staging.
4036            let (mock_app, verify_started, _release_verify): (GatedVerifyingApp<CodingB, S>, _, _) =
4037                GatedVerifyingApp::new();
4038            let cfg = MarshaledConfig {
4039                application: mock_app,
4040                marshal: marshal.clone(),
4041                shards: shards.clone(),
4042                scheme_provider: ConstantProvider::new(schemes[0].clone()),
4043                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
4044                strategy: Sequential,
4045            };
4046            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
4047
4048            let commitment = marshaled
4049                .propose(ctx)
4050                .await
4051                .await
4052                .expect("propose must return a commitment");
4053            assert_eq!(
4054                commitment, commitment_a,
4055                "propose must reuse the block marshal already persisted for this round"
4056            );
4057
4058            // The relay broadcast must find the recovered proposal staged and
4059            // re-persist it (a dedup no-op whose handle covers the pre-crash
4060            // write), resolving the certification gate registered by the
4061            // recovery path.
4062            let _ = marshaled.broadcast(commitment, Plan::Propose { round });
4063            let certify_rx = marshaled.certify(round, commitment).await;
4064            select! {
4065                result = certify_rx => {
4066                    assert!(
4067                        result.expect("certify result missing"),
4068                        "recovered proposal must certify through the relay handshake"
4069                    );
4070                },
4071                _ = verify_started => {
4072                    panic!("certifying a recovered proposal must not run app verification");
4073                },
4074            }
4075        });
4076    }
4077
4078    /// Regression: a boundary re-proposal stores the parent block itself at
4079    /// the re-proposal round, under the parent's original embedded context.
4080    /// A leader that crashes after that relay broadcast must recognize the
4081    /// cached parent as the re-proposal on restart and propose it again,
4082    /// rather than skipping the round because its embedded context names an
4083    /// older round.
4084    #[test_traced("WARN")]
4085    fn test_propose_reuses_reproposed_boundary_block_on_restart() {
4086        let runner = deterministic::Runner::timed(Duration::from_secs(60));
4087        runner.start(|mut context| async move {
4088            let Fixture {
4089                participants,
4090                schemes,
4091                ..
4092            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
4093            let mut oracle = setup_network_with_participants(
4094                context.child("network"),
4095                NZUsize!(1),
4096                participants.clone(),
4097            )
4098            .await;
4099
4100            let me = participants[0].clone();
4101            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
4102
4103            let setup = CodingHarness::setup_validator(
4104                context.child("validator").with_attribute("index", 0),
4105                &mut oracle,
4106                me.clone(),
4107                ConstantProvider::new(schemes[0].clone()),
4108            )
4109            .await;
4110            let marshal = setup.mailbox;
4111            let shards = setup.extra;
4112
4113            let genesis_ctx = CodingCtx {
4114                round: Round::zero(),
4115                leader: default_leader(),
4116                parent: (View::zero(), genesis_commitment()),
4117            };
4118            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
4119            let genesis_parent_commitment = genesis_coding_commitment(&genesis);
4120
4121            // Seed the boundary block at the re-proposal round, where the
4122            // pre-crash relay broadcast of the re-proposal persisted it.
4123            let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
4124            let boundary_round = Round::new(Epoch::zero(), View::new(boundary_height.get()));
4125            let boundary_ctx = CodingCtx {
4126                round: boundary_round,
4127                leader: default_leader(),
4128                parent: (View::zero(), genesis_parent_commitment),
4129            };
4130            let boundary_block =
4131                make_coding_block(boundary_ctx, genesis.digest(), boundary_height, 1900);
4132            let coded_boundary: CodedBlock<_, ReedSolomon<Sha256>, Sha256> =
4133                CodedBlock::new(boundary_block, coding_config, &Sequential);
4134            let boundary_commitment = coded_boundary.commitment();
4135            let round = Round::new(Epoch::zero(), View::new(boundary_height.get() + 1));
4136            assert!(marshal.verified(round, coded_boundary).await);
4137
4138            let ctx = CodingCtx {
4139                round,
4140                leader: me.clone(),
4141                parent: (View::new(boundary_height.get()), boundary_commitment),
4142            };
4143
4144            // The app cannot build and its verification never completes, so
4145            // the assertions below hold only if the cached parent is
4146            // re-proposed as-is.
4147            let (mock_app, verify_started, _release_verify): (GatedVerifyingApp<CodingB, S>, _, _) =
4148                GatedVerifyingApp::new();
4149            let cfg = MarshaledConfig {
4150                application: mock_app,
4151                marshal: marshal.clone(),
4152                shards: shards.clone(),
4153                scheme_provider: ConstantProvider::new(schemes[0].clone()),
4154                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
4155                strategy: Sequential,
4156            };
4157            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
4158
4159            let commitment = marshaled
4160                .propose(ctx)
4161                .await
4162                .await
4163                .expect("propose must return a commitment");
4164            assert_eq!(
4165                commitment, boundary_commitment,
4166                "propose must re-propose the boundary block marshal already persisted for this round"
4167            );
4168
4169            let _ = marshaled.broadcast(commitment, Plan::Propose { round });
4170            let certify_rx = marshaled.certify(round, commitment).await;
4171            select! {
4172                result = certify_rx => {
4173                    assert!(
4174                        result.expect("certify result missing"),
4175                        "re-proposed boundary block must certify through the relay handshake"
4176                    );
4177                },
4178                _ = verify_started => {
4179                    panic!("certifying a re-proposed boundary block must not run app verification");
4180                },
4181            }
4182        });
4183    }
4184
4185    /// Regression: if a pre-crash leader persisted a verified block for a
4186    /// round but the simplex `Notarize` never reached the journal, replay
4187    /// can recover a `consensus_context` whose parent differs from the one
4188    /// the cached block was built against. The restarted leader must then
4189    /// drop the receiver so the voter nullifies the view via
4190    /// `MissingProposal`, rather than broadcasting the stale cached block
4191    /// under a header that peers will reject.
4192    #[test_traced("WARN")]
4193    fn test_propose_skips_when_verified_block_context_changed() {
4194        let runner = deterministic::Runner::timed(Duration::from_secs(60));
4195        runner.start(|mut context| async move {
4196            let Fixture {
4197                participants,
4198                schemes,
4199                ..
4200            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
4201            let mut oracle = setup_network_with_participants(
4202                context.child("network"),
4203                NZUsize!(1),
4204                participants.clone(),
4205            )
4206            .await;
4207
4208            let me = participants[0].clone();
4209            let coding_config = coding_config_for_participants(NUM_VALIDATORS as u16);
4210
4211            let setup = CodingHarness::setup_validator(
4212                context.child("validator").with_attribute("index", 0),
4213                &mut oracle,
4214                me.clone(),
4215                ConstantProvider::new(schemes[0].clone()),
4216            )
4217            .await;
4218            let marshal = setup.mailbox;
4219            let shards = setup.extra;
4220
4221            let genesis_ctx = CodingCtx {
4222                round: Round::zero(),
4223                leader: default_leader(),
4224                parent: (View::zero(), genesis_commitment()),
4225            };
4226            let genesis = make_coding_block(genesis_ctx, Sha256::hash(&[b""]), Height::zero(), 0);
4227            let genesis_parent_commitment = genesis_coding_commitment(&genesis);
4228
4229            // Stash a stale block built against genesis as its parent at round V=2.
4230            let round = Round::new(Epoch::zero(), View::new(2));
4231            let stale_ctx = CodingCtx {
4232                round,
4233                leader: me.clone(),
4234                parent: (View::zero(), genesis_parent_commitment),
4235            };
4236            let stale_block = make_coding_block(stale_ctx, genesis.digest(), Height::new(1), 100);
4237            let stale_coded: CodedBlock<_, ReedSolomon<Sha256>, Sha256> =
4238                CodedBlock::new(stale_block, coding_config, &Sequential);
4239            assert!(marshal.verified(round, stale_coded).await);
4240
4241            // Simulate a replay where parent selection now points to a
4242            // different parent commitment than the cached block was built for.
4243            let new_parent_commitment = TestCommitment::from((
4244                Sha256::hash(&[b"different-parent-block"]),
4245                Sha256::hash(&[b"different-parent-inner"]),
4246                Sha256::hash(&[b"different-parent-ctx"]),
4247                coding_config,
4248            ));
4249            let new_ctx = CodingCtx {
4250                round,
4251                leader: me.clone(),
4252                parent: (View::new(1), new_parent_commitment),
4253            };
4254
4255            let mock_app: MockVerifyingApp<CodingB, S> = MockVerifyingApp::new();
4256            let cfg = MarshaledConfig {
4257                application: mock_app,
4258                marshal: marshal.clone(),
4259                shards: shards.clone(),
4260                scheme_provider: ConstantProvider::new(schemes[0].clone()),
4261                epocher: FixedEpocher::new(BLOCKS_PER_EPOCH),
4262                strategy: Sequential,
4263            };
4264            let mut marshaled = Marshaled::new(context.child("marshaled"), cfg);
4265
4266            let commitment_rx = marshaled.propose(new_ctx).await;
4267            assert!(
4268                commitment_rx.await.is_err(),
4269                "propose must drop the receiver when the cached block's context no longer matches"
4270            );
4271        });
4272    }
4273}