Skip to main content

commonware_consensus/marshal/standard/
deferred.rs

1//! Wrapper for consensus applications that handles epochs and block dissemination.
2//!
3//! # Overview
4//!
5//! [`Deferred`] is an adapter that wraps any [`Application`] implementation to handle
6//! epoch transitions automatically. It intercepts consensus operations (propose, verify) and
7//! ensures blocks are only produced within valid epoch boundaries.
8//!
9//! # Epoch Boundaries
10//!
11//! When the parent is the last block in an epoch (as determined by the [`Epocher`]), this wrapper
12//! re-proposes that boundary block instead of building a new block. This avoids producing blocks
13//! that would be pruned by the epoch transition.
14//!
15//! # Deferred Verification
16//!
17//! Before casting a notarize vote, [`Deferred`] waits for the block to become available and
18//! then verifies that the block's embedded context matches the consensus context. However, it does not
19//! wait for the application to finish verifying the block contents before voting. This enables verification
20//! to run while we wait for a quorum of votes to form a certificate (hiding verification latency behind network
21//! latency). Once a certificate is formed, we wait on the verification result in [`CertifiableAutomaton::certify`]
22//! before voting to finalize (ensuring no invalid blocks are admitted to the canonical chain).
23//!
24//! # Usage
25//!
26//! Wrap your [`Application`] implementation with [`Deferred::new`] and provide it to your
27//! consensus engine for the [`Automaton`] and [`Relay`]. The wrapper handles all epoch logic transparently.
28//!
29//! ```rust,ignore
30//! let application = Deferred::new(
31//!     context,
32//!     my_application,
33//!     marshal_mailbox,
34//!     epocher,
35//! );
36//! ```
37//!
38//! # Implementation Notes
39//!
40//! - Genesis blocks are handled specially: epoch 0 returns the application's genesis block,
41//!   while subsequent epochs use the last block of the previous epoch as genesis
42//! - Blocks are automatically verified to be within the current epoch
43//!
44//! # Notarization and Data Availability
45//!
46//! In rare crash cases, it is possible for a notarization certificate to exist without a block being
47//! available to the honest parties if [`CertifiableAutomaton::certify`] fails after a notarization is
48//! formed.
49//!
50//! For this reason, it should not be expected that every notarized payload will be certifiable due
51//! to the lack of an available block. However, if even one honest and online party has the block,
52//! they will attempt to forward it to others via marshal's resolver.
53//!
54//! ```text
55//!                                      ┌───────────────────────────────────────────────────┐
56//!                                      ▼                                                   │
57//! ┌─────────────────────┐   ┌─────────────────────┐   ┌─────────────────────┐   ┌─────────────────────┐
58//! │          B1         │◀──│          B2         │◀──│          B3         │XXX│          B4         │
59//! └─────────────────────┘   └─────────────────────┘   └──────────┬──────────┘   └─────────────────────┘
60//!                                                                │
61//!                                                          Failed Certify
62//! ```
63//!
64//! # Future Work
65//!
66//! - To further reduce view latency, a participant could optimistically vote for a block prior to
67//!   observing its availability during [`Automaton::verify`]. However, this would require updating
68//!   other components (like [`crate::marshal`]) to handle backfill where notarization does not imply
69//!   a block is fetchable (without modification, a malicious leader that withholds blocks during propose
70//!   could get an honest node to exhaust their network rate limit fetching things that don't exist rather
71//!   than blocks they need AND can fetch).
72
73use crate::{
74    marshal::{
75        application::{
76            gates::{self, Gates},
77            validation::{is_inferred_reproposal_at_certify, Stage},
78        },
79        core::{CommitmentFallback, DigestFallback, Mailbox},
80        standard::{
81            relay,
82            validation::{
83                await_and_validate_parent, precheck_epoch_and_reproposal, run_app_verify, Decision,
84                ParentCheck,
85            },
86            Standard,
87        },
88        Update,
89    },
90    simplex::{types::Context, Plan},
91    types::{Epocher, Round},
92    Application, Automaton, CertifiableAutomaton, CertifiableBlock, Epochable, Relay, Reporter,
93};
94use commonware_actor::Feedback;
95use commonware_cryptography::{certificate::Scheme, Digestible};
96use commonware_macros::select;
97use commonware_runtime::{
98    telemetry::{
99        metrics::{
100            histogram::{Buckets, Timed},
101            MetricsExt as _,
102        },
103        traces::TracedExt as _,
104    },
105    Clock, Metrics, Spawner,
106};
107use commonware_utils::{
108    channel::{fallible::OneshotExt, oneshot},
109    sync::TracedAsyncMutex,
110};
111use rand_core::Rng;
112use std::sync::Arc;
113use tracing::{debug, info_span, Instrument as _};
114
115/// An [`Application`] adapter that handles epoch transitions and validates block ancestry.
116///
117/// This wrapper intercepts consensus operations to enforce epoch boundaries and validate
118/// block ancestry. It prevents blocks from being produced outside their valid epoch,
119/// handles the special case of re-proposing boundary blocks at epoch boundaries,
120/// and ensures all blocks have valid parent linkage and contiguous heights.
121///
122/// # Ancestry Validation
123///
124/// Applications wrapped by [`Deferred`] can rely on the following ancestry checks being
125/// performed automatically during verification:
126/// - Parent digest matches the consensus context's expected parent
127/// - Block height is exactly one greater than the parent's height
128///
129/// Verifying only the immediate parent is sufficient since the parent itself must have
130/// been notarized by consensus, which guarantees it was verified and accepted by a quorum.
131/// This means the entire ancestry chain back to genesis is transitively validated.
132///
133/// Applications do not need to re-implement these checks in their own verification logic.
134///
135/// # Context Recovery
136///
137/// With deferred verification, validators wait for data availability (DA) and verify the context
138/// before voting. If a validator crashes after voting but before certification, they lose their in-memory
139/// certification gate task. When recovering, validators extract context from a [`CertifiableBlock`].
140///
141/// _This embedded context is trustworthy because the notarizing quorum (which contains at least f+1 honest
142/// validators) verified that the block's context matched the consensus context before voting._
143pub struct Deferred<E, S, A, B, ES>
144where
145    E: Rng + Spawner + Metrics + Clock,
146    S: Scheme,
147    A: Application<E>,
148    B: CertifiableBlock,
149    ES: Epocher,
150{
151    context: Arc<TracedAsyncMutex<E>>,
152    application: A,
153    marshal: Mailbox<S, Standard<B>>,
154    epocher: ES,
155    gates: Gates<<B as Digestible>::Digest, B>,
156
157    build_duration: Timed,
158    proposal_parent_fetch_duration: Timed,
159    ancestor_fetch_duration: Timed,
160}
161
162impl<E, S, A, B, ES> Clone for Deferred<E, S, A, B, ES>
163where
164    E: Rng + Spawner + Metrics + Clock,
165    S: Scheme,
166    A: Application<E>,
167    B: CertifiableBlock,
168    ES: Epocher,
169{
170    fn clone(&self) -> Self {
171        Self {
172            context: self.context.clone(),
173            application: self.application.clone(),
174            marshal: self.marshal.clone(),
175            epocher: self.epocher.clone(),
176            gates: self.gates.clone(),
177            build_duration: self.build_duration.clone(),
178            proposal_parent_fetch_duration: self.proposal_parent_fetch_duration.clone(),
179            ancestor_fetch_duration: self.ancestor_fetch_duration.clone(),
180        }
181    }
182}
183
184impl<E, S, A, B, ES> Deferred<E, S, A, B, ES>
185where
186    E: Rng + Spawner + Metrics + Clock,
187    S: Scheme,
188    A: Application<E, Block = B, SigningScheme = S, Context = Context<B::Digest, S::PublicKey>>,
189    B: CertifiableBlock<Context = <A as Application<E>>::Context>,
190    ES: Epocher,
191{
192    /// Creates a new [`Deferred`] wrapper.
193    pub fn new(context: E, application: A, marshal: Mailbox<S, Standard<B>>, epocher: ES) -> Self {
194        let build_histogram = context.histogram(
195            "build_duration",
196            "Histogram of time taken for the application to build a new block, in seconds",
197            Buckets::LOCAL,
198        );
199        let build_duration = Timed::new(build_histogram);
200        let parent_fetch_histogram = context.histogram(
201            "parent_fetch_duration",
202            "Histogram of time taken to fetch a parent block in propose, in seconds",
203            Buckets::LOCAL,
204        );
205        let proposal_parent_fetch_duration = Timed::new(parent_fetch_histogram);
206        let ancestor_fetch_histogram = context.histogram(
207            "ancestor_fetch_duration",
208            "Histogram of time taken to fetch a block via the ancestry stream, in seconds",
209            Buckets::LOCAL,
210        );
211        let ancestor_fetch_duration = Timed::new(ancestor_fetch_histogram);
212
213        Self {
214            context: Arc::new(TracedAsyncMutex::new("marshal.context", context)),
215            application,
216            marshal,
217            epocher,
218            gates: Gates::new(),
219
220            build_duration,
221            proposal_parent_fetch_duration,
222            ancestor_fetch_duration,
223        }
224    }
225
226    /// Verifies a proposed block's application-level validity.
227    ///
228    /// This method validates that:
229    /// 1. The block's parent digest matches the expected parent
230    /// 2. The block's height is exactly one greater than the parent's height
231    /// 3. The underlying application's verification logic passes
232    ///
233    /// The `parent_request` must be a subscription to the parent named by `context.parent`,
234    /// started by the caller so the parent fetch can overlap work that precedes this call.
235    ///
236    /// Verification is spawned in a background task and returns a receiver that will contain
237    /// the verification result. Valid blocks are reported to the marshal as verified.
238    #[inline]
239    async fn deferred_verify(
240        &mut self,
241        context: <Self as Automaton>::Context,
242        block: Arc<B>,
243        parent_request: oneshot::Receiver<Arc<B>>,
244        stage: Stage,
245    ) -> oneshot::Receiver<bool> {
246        let marshal = self.marshal.clone();
247        let mut application = self.application.clone();
248        let (mut tx, rx) = oneshot::channel();
249        let ancestor_fetch_duration = self.ancestor_fetch_duration.clone();
250        let runtime_context = self
251            .context
252            .lock()
253            .await
254            .child("deferred_verify")
255            .with_attribute("round", context.round);
256        let span = info_span!(
257            "marshal.deferred.verify.deferred",
258            round = %context.round
259        );
260        runtime_context.spawn(move |runtime_context| {
261            async move {
262                let round = context.round;
263
264                // Start the candidate store immediately: it depends on neither the
265                // parent fetch (which may hit the network) nor the verdict below.
266                // Storing before validation is intentional: these caches provide
267                // candidate availability/recovery, not a validity decision. This
268                // task gates the finalize vote by resolving true only after both
269                // app verification succeeds and the store is durable.
270                let store = stage.store(&marshal, round, Arc::clone(&block));
271                let verify = async {
272                    // Validate the parent we already started fetching.
273                    let parent = match await_and_validate_parent(
274                        context.parent.1,
275                        block.as_ref(),
276                        parent_request,
277                        &mut tx,
278                    )
279                    .await
280                    {
281                        Some(ParentCheck::Valid(parent)) => parent,
282                        Some(ParentCheck::Invalid) => return Some(false),
283                        None => return None,
284                    };
285                    run_app_verify(
286                        runtime_context,
287                        context,
288                        Arc::clone(&block),
289                        parent,
290                        &mut application,
291                        &marshal,
292                        &mut tx,
293                        ancestor_fetch_duration,
294                    )
295                    .await
296                };
297                let (verdict, durable) = futures::join!(verify, store);
298
299                // Publish only when the block is both valid and durable. App-invalid
300                // candidates may already be in the cache from the concurrent store above,
301                // so the gate verdict is the authority for consensus progress.
302                if let Some(application_valid) = gates::resolve(verdict, durable) {
303                    tx.send_lossy(application_valid);
304                }
305            }
306            .instrument(span)
307        });
308
309        rx
310    }
311
312    async fn certify_from_embedded_context(
313        &mut self,
314        round: Round,
315        digest: B::Digest,
316    ) -> oneshot::Receiver<bool> {
317        // No in-progress task means we never verified this proposal locally. We can use the
318        // block's embedded context to help complete finalization when Byzantine validators
319        // withhold their finalize votes. If a Byzantine proposer embedded a malicious context,
320        // the f+1 honest validators from the notarizing quorum will verify against the proper
321        // context and reject the mismatch, preventing a 2f+1 finalization quorum.
322        //
323        // We must fetch here rather than only wait for local broadcast delivery. A Byzantine
324        // leader can send a proposal to just f+1 honest validators, collect enough honest
325        // notarize votes to form a notarization, and leave the remaining honest validators
326        // without the block. Those validators need the notarized round to recover the block
327        // and certify; otherwise they can remain stuck if the Byzantine validators stop
328        // participating in the next view.
329        //
330        // Subscribe to the block and verify using its embedded context once available.
331        debug!(
332            ?round,
333            ?digest,
334            "subscribing to block for certification using embedded context"
335        );
336        let block_rx = self
337            .marshal
338            .subscribe_by_digest(digest, DigestFallback::FetchByRound { round });
339        let mut marshaled = self.clone();
340        let epocher = self.epocher.clone();
341        let (mut tx, rx) = oneshot::channel();
342        let context = self
343            .context
344            .lock()
345            .await
346            .child("certify")
347            .with_attribute("round", round);
348        context.spawn(move |_| {
349            async move {
350                let block = select! {
351                    _ = tx.closed() => {
352                        debug!(
353                            reason = "consensus dropped receiver",
354                            "skipping certification"
355                        );
356                        return;
357                    },
358                    result = block_rx => match result {
359                        Ok(block) => block,
360                        Err(_) => {
361                            debug!(
362                                ?digest,
363                                reason = "failed to fetch block for certification",
364                                "skipping certification"
365                            );
366                            return;
367                        }
368                    },
369                };
370
371                // Re-proposal detection for certify path: we don't have the consensus context,
372                // only the block's embedded context from original proposal. Infer re-proposal from:
373                // 1. Block is at epoch boundary (only boundary blocks can be re-proposed)
374                // 2. Certification round's view > embedded context's view (re-proposals retain their
375                //    original embedded context, so a later view indicates the block was re-proposed)
376                // 3. Same epoch (re-proposals don't cross epoch boundaries)
377                let embedded_context = block.context();
378                let is_reproposal = is_inferred_reproposal_at_certify(
379                    &epocher,
380                    block.height(),
381                    embedded_context.round,
382                    round,
383                );
384                if is_reproposal {
385                    // Certifier holds a notarization for this block, so route
386                    // the write to the notarized cache. `certified` is
387                    // idempotent, so crash-recovery double-invocation is safe.
388                    if !marshaled.marshal.certified(round, block).await {
389                        return;
390                    }
391                    tx.send_lossy(true);
392                    return;
393                }
394
395                // Start the parent fetch for the deferred verification below,
396                // which expects a caller-started subscription. Certify does not
397                // carry the consensus context, so the parent round comes from
398                // the block's embedded context. That context is trustworthy
399                // because the digest is notarized and the notarizing quorum's
400                // f+1 honest validators verified it against the consensus
401                // context before voting.
402                let (parent_view, parent_commitment) = embedded_context.parent;
403                let parent_request = marshaled.marshal.subscribe_by_commitment(
404                    parent_commitment,
405                    CommitmentFallback::FetchByRound {
406                        round: Round::new(embedded_context.epoch(), parent_view),
407                    },
408                );
409
410                let verify_rx = marshaled
411                    .deferred_verify(embedded_context, block, parent_request, Stage::Certified)
412                    .await;
413                if let Ok(result) = verify_rx.await {
414                    tx.send_lossy(result);
415                }
416            }
417            .instrument(info_span!(
418                "marshal.deferred.certify.embedded",
419                round = %round,
420                digest = %digest
421            ))
422        });
423        rx
424    }
425
426    #[allow(clippy::async_yields_async)]
427    async fn certify_from_existing_task(
428        &mut self,
429        round: Round,
430        digest: B::Digest,
431        task: oneshot::Receiver<bool>,
432    ) -> oneshot::Receiver<bool> {
433        // `verify()` waits only on local broadcast delivery, so nudge a
434        // round-bound notarized fetch that can unblock the existing waiter
435        // if local broadcast never arrives. For the standard variant, the
436        // digest is also the variant commitment.
437        self.marshal.hint_notarized(round, digest);
438
439        // A completed gate is a live local verdict. After an unclean restart the
440        // in-memory task is gone, so recover via the embedded-context fetch path.
441        let mut marshaled = self.clone();
442        let (tx, rx) = oneshot::channel();
443        let context = self
444            .context
445            .lock()
446            .await
447            .child("certify_existing")
448            .with_attribute("round", round);
449        context.spawn(move |_| {
450            gates::drive(tx, task, round, digest, move || async move {
451                marshaled.certify_from_embedded_context(round, digest).await
452            })
453            .instrument(info_span!(
454                "marshal.deferred.certify.existing",
455                round = %round,
456                digest = %digest
457            ))
458        });
459        rx
460    }
461}
462
463impl<E, S, A, B, ES> Automaton for Deferred<E, S, A, B, ES>
464where
465    E: Rng + Spawner + Metrics + Clock,
466    S: Scheme,
467    A: Application<E, Block = B, SigningScheme = S, Context = Context<B::Digest, S::PublicKey>>,
468    B: CertifiableBlock<Context = <A as Application<E>>::Context>,
469    ES: Epocher,
470{
471    type Digest = B::Digest;
472    type Context = Context<Self::Digest, S::PublicKey>;
473
474    /// Proposes a new block or re-proposes the epoch boundary block.
475    ///
476    /// This method builds a new block from the underlying application unless the parent block
477    /// is the last block in the current epoch. When at an epoch boundary, it re-proposes the
478    /// boundary block to avoid creating blocks that would be invalidated by the epoch transition.
479    ///
480    /// The proposal operation is spawned in a background task and returns a receiver that will
481    /// contain the proposed block's digest when ready. The block is staged before the digest is
482    /// delivered and handed to marshal when consensus requests the relay broadcast, which
483    /// persists it after the send. The resulting sync handle is awaited only at certification so
484    /// it overlaps consensus voting. The digest does not imply durability on
485    /// its own; [`CertifiableAutomaton::certify`] awaits the registered certification gate before
486    /// the finalize vote.
487    #[allow(clippy::async_yields_async)]
488    #[tracing::instrument(name = "marshal.deferred.propose", level = "info", skip_all, fields(round = %consensus_context.round))]
489    async fn propose(
490        &mut self,
491        consensus_context: Context<Self::Digest, S::PublicKey>,
492    ) -> oneshot::Receiver<Self::Digest> {
493        let marshal = self.marshal.clone();
494        let mut application = self.application.clone();
495        let epocher = self.epocher.clone();
496        let gates = self.gates.clone();
497
498        // Metrics
499        let build_duration = self.build_duration.clone();
500        let proposal_parent_fetch_duration = self.proposal_parent_fetch_duration.clone();
501        let ancestor_fetch_duration = self.ancestor_fetch_duration.clone();
502
503        let (mut tx, rx) = oneshot::channel();
504        let context = self
505            .context
506            .lock()
507            .await
508            .child("propose")
509            .with_attribute("round", consensus_context.round);
510        let span = info_span!(
511            "marshal.deferred.propose.task",
512            round = %consensus_context.round
513        );
514        context.spawn(move |runtime_context| {
515            async move {
516                // On leader recovery, marshal may already hold a verified block
517                // for this round (persisted by a pre-crash propose that reached
518                // its relay broadcast while the notarize vote never reached the
519                // journal).
520                //
521                // The pre-crash digest may already have been broadcast, so
522                // building a fresh block would equivocate. The stored block is
523                // the only proposal we can broadcast for this round.
524                //
525                // The recovered block is safe to reuse only if its embedded
526                // context matches the context simplex just recovered. Otherwise the
527                // cached block was built against a different parent and cannot be
528                // broadcast under the current header, so drop the receiver
529                // and let the voter nullify the view via timeout.
530                if let Some(block) = marshal.get_verified(consensus_context.round).await {
531                    let block_context = block.context();
532                    if block_context != consensus_context {
533                        debug!(
534                            round = ?consensus_context.round,
535                            ?consensus_context,
536                            ?block_context,
537                            "skipping proposal: cached verified block context no longer matches"
538                        );
539                        return;
540                    }
541                    // Stage the recovered block so the relay broadcast re-sends
542                    // it through the same handshake as a fresh proposal. The
543                    // relay-time persist deduplicates against the pre-crash
544                    // write, with the handle covering the original.
545                    let digest = block.digest();
546                    debug!(
547                        round = ?consensus_context.round,
548                        ?digest,
549                        "reusing verified block from marshal on leader recovery"
550                    );
551                    gates
552                        .stage(
553                            consensus_context.round,
554                            digest,
555                            Arc::new(block),
556                            tx,
557                            "recovered block",
558                        )
559                        .await;
560                    return;
561                }
562
563                // The parent for any consensus context is in the same epoch: the
564                // boundary block of the previous epoch is the genesis block of the
565                // current epoch.
566                //
567                // Proposal context carries the certified parent view/commitment but
568                // not the parent height. The parent may be certified above the
569                // finalized tip, so this must stay round-bound until the block is
570                // returned.
571                let (parent_view, parent_commitment) = consensus_context.parent;
572                let parent_request = marshal.subscribe_by_commitment(
573                    parent_commitment,
574                    CommitmentFallback::FetchByRound {
575                        round: Round::new(consensus_context.epoch(), parent_view),
576                    },
577                );
578
579                let parent_timer = proposal_parent_fetch_duration.timer(&runtime_context);
580                let parent = select! {
581                    _ = tx.closed() => {
582                        debug!(reason = "consensus dropped receiver", "skipping proposal");
583                        return;
584                    },
585                    result = parent_request => match result {
586                        Ok(parent) => parent,
587                        Err(_) => {
588                            debug!(
589                                ?parent_commitment,
590                                reason = "failed to fetch parent block",
591                                "skipping proposal"
592                            );
593                            return;
594                        }
595                    },
596                };
597                parent_timer.observe(&runtime_context);
598
599                // Special case: If the parent block is the last block in the epoch,
600                // re-propose it as to not produce any blocks that will be cut out
601                // by the epoch transition.
602                let last_in_epoch = epocher
603                    .last(consensus_context.epoch())
604                    .expect("current epoch should exist");
605                if parent.height() == last_in_epoch {
606                    let digest = parent.digest();
607                    gates
608                        .stage(
609                            consensus_context.round,
610                            digest,
611                            parent,
612                            tx,
613                            "re-proposed boundary block",
614                        )
615                        .await;
616                    return;
617                }
618
619                let ancestor_stream = marshal.ancestor_stream(
620                    Arc::new(runtime_context.child("ancestor_stream")),
621                    [parent],
622                    ancestor_fetch_duration,
623                );
624                let build_request = application
625                    .propose(
626                        (
627                            runtime_context.child("app_propose"),
628                            consensus_context.clone(),
629                        ),
630                        ancestor_stream,
631                    )
632                    .instrument(info_span!(
633                        "marshal.deferred.application.propose",
634                        round = %consensus_context.round,
635                        parent_view = parent_view.traced(),
636                        parent = %parent_commitment
637                    ));
638
639                let build_timer = build_duration.timer(&runtime_context);
640                let built_block = select! {
641                    _ = tx.closed() => {
642                        debug!(reason = "consensus dropped receiver", "skipping proposal");
643                        return;
644                    },
645                    result = build_request => match result {
646                        Some(block) => block,
647                        None => {
648                            debug!(
649                                ?parent_commitment,
650                                reason = "block building failed",
651                                "skipping proposal"
652                            );
653                            return;
654                        }
655                    },
656                };
657                build_timer.observe(&runtime_context);
658
659                let digest = built_block.digest();
660                gates
661                    .stage(
662                        consensus_context.round,
663                        digest,
664                        Arc::new(built_block),
665                        tx,
666                        "proposed block",
667                    )
668                    .await;
669            }
670            .instrument(span)
671        });
672        rx
673    }
674
675    #[allow(clippy::async_yields_async)]
676    #[tracing::instrument(name = "marshal.deferred.verify", level = "info", skip_all, fields(round = %context.round, digest = %digest))]
677    async fn verify(
678        &mut self,
679        context: Context<Self::Digest, S::PublicKey>,
680        digest: Self::Digest,
681    ) -> oneshot::Receiver<bool> {
682        let marshal = self.marshal.clone();
683        let mut marshaled = self.clone();
684        let round = context.round;
685
686        // Register the certification gate task synchronously so `certify` finds a pending
687        // entry even while the optimistic block subscription is still waiting locally.
688        // This lets `certify` take the task and bump a round-bound notarized fetch
689        // via `hint_notarized`.
690        let (task_tx, task_rx) = oneshot::channel();
691        self.gates.insert(round, digest, task_rx);
692
693        let (mut tx, rx) = oneshot::channel();
694        let runtime_context = self
695            .context
696            .lock()
697            .await
698            .child("optimistic_verify")
699            .with_attribute("round", round);
700        runtime_context.spawn(move |_| {
701            async move {
702                // Start the parent fetch immediately: its commitment and certified
703                // round are known from the consensus context, so it can proceed in
704                // parallel with broadcast delivery of the candidate block.
705                // Reproposals (digest == context.parent.1) skip parent validation
706                // entirely, so they must not fetch: the "parent" is the candidate
707                // itself, and candidate acquisition is deliberately local-only.
708                let parent_request = (digest != context.parent.1).then(|| {
709                    let (parent_view, parent_commitment) = context.parent;
710                    marshal.subscribe_by_commitment(
711                        parent_commitment,
712                        CommitmentFallback::FetchByRound {
713                            round: Round::new(context.epoch(), parent_view),
714                        },
715                    )
716                });
717
718                let block_request = marshal.subscribe_by_digest(digest, DigestFallback::Wait);
719                let block = select! {
720                    _ = tx.closed() => {
721                        debug!(
722                            reason = "consensus dropped receiver",
723                            "skipping optimistic verification"
724                        );
725                        return;
726                    },
727                    result = block_request => match result {
728                        Ok(block) => block,
729                        Err(_) => {
730                            debug!(
731                                ?digest,
732                                reason = "failed to fetch block for optimistic verification",
733                                "skipping optimistic verification"
734                            );
735                            return;
736                        }
737                    },
738                };
739
740                // Shared pre-checks enforce:
741                // - Block epoch membership.
742                // - Re-proposal detection via `digest == context.parent.1`.
743                //
744                // Re-proposals return early and skip normal parent/height checks
745                // because they were already verified when originally proposed and
746                // parent-child checks would fail by construction when parent == block.
747                let Some(decision) = precheck_epoch_and_reproposal(
748                    &marshaled.epocher,
749                    &marshal,
750                    &context,
751                    digest,
752                    block,
753                )
754                .await
755                else {
756                    return;
757                };
758                let block = match decision {
759                    Decision::Complete(valid) => {
760                        // `Complete` means either immediate rejection or successful
761                        // re-proposal handling with no further ancestry validation.
762                        task_tx.send_lossy(valid);
763                        tx.send_lossy(valid);
764                        return;
765                    }
766                    Decision::Continue(block) => block,
767                };
768
769                // `Continue` implies a non-reproposal, so the parent subscription
770                // was started above.
771                let parent_request =
772                    parent_request.expect("non-reproposal has a parent subscription");
773
774                // Before casting a notarize vote, ensure the block's embedded context matches
775                // the consensus context.
776                //
777                // This is a critical step - the notarize quorum is guaranteed to have at least
778                // f+1 honest validators who will verify against this context, preventing a Byzantine
779                // proposer from embedding a malicious context. The other f honest validators who did
780                // not vote will later use the block-embedded context to help finalize if Byzantine
781                // validators withhold their finalize votes.
782                if block.context() != context {
783                    debug!(
784                        ?context,
785                        block_context = ?block.context(),
786                        "block-embedded context does not match consensus context during optimistic verification"
787                    );
788                    task_tx.send_lossy(false);
789                    tx.send_lossy(false);
790                    return;
791                }
792
793                // Optimistic verify returns immediately; the deferred_verify task
794                // runs in the background and forwards its final verdict to
795                // `task_tx` so `certify` observes the same result via the
796                // synchronously-registered `task_rx`.
797                //
798                // The awaits below are deliberately not guarded on `tx.closed()`.
799                // Once the optimistic verdict is delivered, the gate is the only
800                // remaining consumer, and certification can still want it after
801                // the view exits (nullification does not cancel certification
802                // work), so deferred verification must run to completion into
803                // the gate.
804                let deferred_rx = marshaled
805                    .deferred_verify(context, block, parent_request, Stage::Verified)
806                    .await;
807                tx.send_lossy(true);
808                if let Ok(result) = deferred_rx.await {
809                    task_tx.send_lossy(result);
810                }
811            }
812            .instrument(info_span!(
813                "marshal.deferred.verify.optimistic",
814                round = %round,
815                digest = %digest
816            ))
817        });
818        rx
819    }
820}
821
822impl<E, S, A, B, ES> CertifiableAutomaton for Deferred<E, S, A, B, ES>
823where
824    E: Rng + Spawner + Metrics + Clock,
825    S: Scheme,
826    A: Application<E, Block = B, SigningScheme = S, Context = Context<B::Digest, S::PublicKey>>,
827    B: CertifiableBlock<Context = <A as Application<E>>::Context>,
828    ES: Epocher,
829{
830    #[allow(clippy::async_yields_async)]
831    #[tracing::instrument(name = "marshal.deferred.certify", level = "info", skip_all, fields(round = %round, digest = %digest))]
832    async fn certify(&mut self, round: Round, digest: Self::Digest) -> oneshot::Receiver<bool> {
833        self.gates.flush_unrelayed(&self.marshal, round, digest);
834
835        // Attempt to retrieve the existing certification gate task for this round/digest.
836        let task = self.gates.take(round, digest);
837        if let Some(task) = task {
838            return self.certify_from_existing_task(round, digest, task).await;
839        }
840
841        self.certify_from_embedded_context(round, digest).await
842    }
843}
844
845impl<E, S, A, B, ES> Relay for Deferred<E, S, A, B, ES>
846where
847    E: Rng + Spawner + Metrics + Clock,
848    S: Scheme,
849    A: Application<E, Block = B, Context = Context<B::Digest, S::PublicKey>>,
850    B: CertifiableBlock<Context = <A as Application<E>>::Context>,
851    ES: Epocher,
852{
853    type Digest = B::Digest;
854    type PublicKey = S::PublicKey;
855    type Plan = Plan<S::PublicKey>;
856
857    fn broadcast(&mut self, commitment: Self::Digest, plan: Plan<S::PublicKey>) -> Feedback {
858        relay::broadcast(&self.gates, &self.marshal, commitment, plan)
859    }
860}
861
862impl<E, S, A, B, ES> Reporter for Deferred<E, S, A, B, ES>
863where
864    E: Rng + Spawner + Metrics + Clock,
865    S: Scheme,
866    A: Application<E, Block = B, Context = Context<B::Digest, S::PublicKey>>
867        + Reporter<Activity = Update<B>>,
868    B: CertifiableBlock<Context = <A as Application<E>>::Context>,
869    ES: Epocher,
870{
871    type Activity = A::Activity;
872
873    /// Relays a report to the underlying [`Application`] and cleans up old certification gate tasks.
874    fn report(&mut self, update: Self::Activity) -> Feedback {
875        // Clean up certification gate tasks for rounds <= the finalized round.
876        if let Update::Tip(round, _, _) = &update {
877            self.gates.retain_after(round);
878        }
879        self.application.report(update)
880    }
881}
882
883#[cfg(test)]
884mod tests {
885    use super::Deferred;
886    use crate::{
887        marshal::mocks::{
888            harness::{
889                default_leader, make_raw_block, setup_network_with_participants, Ctx,
890                StandardHarness, TestHarness, B, BLOCKS_PER_EPOCH, NAMESPACE, NUM_VALIDATORS, S, V,
891            },
892            verifying::{GatedVerifyingApp, MockVerifyingApp},
893        },
894        simplex::{scheme::bls12381_threshold::vrf as bls12381_threshold_vrf, Plan},
895        types::{Epoch, Epocher, FixedEpocher, Height, Round, View},
896        Automaton, CertifiableAutomaton, Relay,
897    };
898    use commonware_broadcast::Broadcaster;
899    use commonware_cryptography::{
900        certificate::{mocks::Fixture, ConstantProvider},
901        sha256::Sha256,
902        Digestible, Hasher as _,
903    };
904    use commonware_macros::{select, test_traced};
905    use commonware_runtime::{deterministic, Clock, Runner, Supervisor as _};
906    use commonware_utils::{channel::fallible::OneshotExt, NZUsize};
907    use std::time::Duration;
908
909    #[test_traced("INFO")]
910    fn test_certify_lower_view_after_higher_view() {
911        let runner = deterministic::Runner::timed(Duration::from_secs(60));
912        runner.start(|mut context| async move {
913            let Fixture {
914                participants,
915                schemes,
916                ..
917            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
918            let mut oracle = setup_network_with_participants(
919                context.child("network"),
920                NZUsize!(1),
921                participants.clone(),
922            )
923            .await;
924
925            let me = participants[0].clone();
926
927            let setup = StandardHarness::setup_validator(
928                context.child("validator").with_attribute("index", 0),
929                &mut oracle,
930                me.clone(),
931                ConstantProvider::new(schemes[0].clone()),
932            )
933            .await;
934            let marshal = setup.mailbox;
935
936            let genesis = make_raw_block(Sha256::hash(b""), Height::zero(), 0);
937            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
938
939            let mut marshaled = Deferred::new(
940                context.child("deferred"),
941                mock_app,
942                marshal.clone(),
943                FixedEpocher::new(BLOCKS_PER_EPOCH),
944            );
945
946            // Create parent block at height 1
947            let parent = make_raw_block(genesis.digest(), Height::new(1), 100);
948            let parent_digest = parent.digest();
949
950            assert!(
951                marshal
952                    .verified(Round::new(Epoch::new(0), View::new(1)), parent.clone())
953                    .await
954            );
955
956            // Block A at view 5 (height 2)
957            let round_a = Round::new(Epoch::new(0), View::new(5));
958            let context_a = Ctx {
959                round: round_a,
960                leader: me.clone(),
961                parent: (View::new(1), parent_digest),
962            };
963            let block_a = B::new::<Sha256>(context_a.clone(), parent_digest, Height::new(2), 200);
964            let commitment_a = StandardHarness::commitment(&block_a);
965            assert!(marshal.verified(round_a, block_a.clone()).await);
966
967            // Block B at view 10 (height 2, different block same height)
968            let round_b = Round::new(Epoch::new(0), View::new(10));
969            let context_b = Ctx {
970                round: round_b,
971                leader: me.clone(),
972                parent: (View::new(1), parent_digest),
973            };
974            let block_b = B::new::<Sha256>(context_b.clone(), parent_digest, Height::new(2), 300);
975            let commitment_b = StandardHarness::commitment(&block_b);
976            assert!(marshal.verified(round_b, block_b.clone()).await);
977
978            context.sleep(Duration::from_millis(10)).await;
979
980            // Step 1: Verify block A at view 5
981            let _ = marshaled.verify(context_a, commitment_a).await.await;
982
983            // Step 2: Verify block B at view 10
984            let _ = marshaled.verify(context_b, commitment_b).await.await;
985
986            // Step 3: Certify block B at view 10 FIRST
987            let certify_b = marshaled.certify(round_b, commitment_b).await;
988            assert!(
989                certify_b.await.unwrap(),
990                "Block B certification should succeed"
991            );
992
993            // Step 4: Certify block A at view 5 - should succeed
994            let certify_a = marshaled.certify(round_a, commitment_a).await;
995
996            select! {
997                result = certify_a => {
998                    assert!(result.unwrap(), "Block A certification should succeed");
999                },
1000                _ = context.sleep(Duration::from_secs(5)) => {
1001                    panic!("Block A certification timed out");
1002                },
1003            }
1004        })
1005    }
1006
1007    #[test_traced("WARN")]
1008    fn test_marshaled_rejects_unsupported_epoch() {
1009        #[derive(Clone)]
1010        struct LimitedEpocher {
1011            inner: FixedEpocher,
1012            max_epoch: u64,
1013        }
1014
1015        impl Epocher for LimitedEpocher {
1016            fn containing(&self, height: Height) -> Option<crate::types::EpochInfo> {
1017                let bounds = self.inner.containing(height)?;
1018                if bounds.epoch().get() > self.max_epoch {
1019                    None
1020                } else {
1021                    Some(bounds)
1022                }
1023            }
1024
1025            fn first(&self, epoch: Epoch) -> Option<Height> {
1026                if epoch.get() > self.max_epoch {
1027                    None
1028                } else {
1029                    self.inner.first(epoch)
1030                }
1031            }
1032
1033            fn last(&self, epoch: Epoch) -> Option<Height> {
1034                if epoch.get() > self.max_epoch {
1035                    None
1036                } else {
1037                    self.inner.last(epoch)
1038                }
1039            }
1040        }
1041
1042        let runner = deterministic::Runner::timed(Duration::from_secs(60));
1043        runner.start(|mut context| async move {
1044            let Fixture {
1045                participants,
1046                schemes,
1047                ..
1048            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1049            let mut oracle = setup_network_with_participants(
1050                context.child("network"),
1051                NZUsize!(1),
1052                participants.clone(),
1053            )
1054            .await;
1055
1056            let me = participants[0].clone();
1057
1058            let setup = StandardHarness::setup_validator(
1059                context.child("validator").with_attribute("index", 0),
1060                &mut oracle,
1061                me.clone(),
1062                ConstantProvider::new(schemes[0].clone()),
1063            )
1064            .await;
1065            let marshal = setup.mailbox;
1066
1067            let genesis = make_raw_block(Sha256::hash(b""), Height::zero(), 0);
1068            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1069            let limited_epocher = LimitedEpocher {
1070                inner: FixedEpocher::new(BLOCKS_PER_EPOCH),
1071                max_epoch: 0,
1072            };
1073
1074            let mut marshaled = Deferred::new(
1075                context.child("deferred"),
1076                mock_app,
1077                marshal.clone(),
1078                limited_epocher,
1079            );
1080
1081            // Create a parent block at height 19 (last block in epoch 0, which is supported)
1082            let parent_ctx = Ctx {
1083                round: Round::new(Epoch::zero(), View::new(19)),
1084                leader: default_leader(),
1085                parent: (View::zero(), genesis.digest()),
1086            };
1087            let parent =
1088                B::new::<Sha256>(parent_ctx.clone(), genesis.digest(), Height::new(19), 1000);
1089            let parent_digest = parent.digest();
1090
1091            assert!(
1092                marshal
1093                    .clone()
1094                    .verified(Round::new(Epoch::zero(), View::new(19)), parent.clone())
1095                    .await
1096            );
1097
1098            // Create a block at height 20 (first block in epoch 1, which is NOT supported)
1099            let unsupported_round = Round::new(Epoch::new(1), View::new(20));
1100            let unsupported_context = Ctx {
1101                round: unsupported_round,
1102                leader: me.clone(),
1103                parent: (View::new(19), parent_digest),
1104            };
1105            let block = B::new::<Sha256>(
1106                unsupported_context.clone(),
1107                parent_digest,
1108                Height::new(20),
1109                2000,
1110            );
1111            let block_commitment = StandardHarness::commitment(&block);
1112
1113            assert!(
1114                marshal
1115                    .clone()
1116                    .verified(unsupported_round, block.clone())
1117                    .await
1118            );
1119
1120            context.sleep(Duration::from_millis(10)).await;
1121
1122            // Call verify and wait for the result (verify returns optimistic result,
1123            // but also spawns deferred verification)
1124            let verify_result = marshaled
1125                .verify(unsupported_context, block_commitment)
1126                .await;
1127
1128            // Wait for optimistic verify to complete so the certification gate task is registered
1129            let optimistic_result = verify_result.await;
1130
1131            // The optimistic verify should return false because the block is in an unsupported epoch
1132            assert!(
1133                !optimistic_result.unwrap(),
1134                "Optimistic verify should reject block in unsupported epoch"
1135            );
1136        })
1137    }
1138
1139    /// Test that marshaled rejects blocks when consensus context doesn't match block's embedded context.
1140    ///
1141    /// This tests that when verify() is called with a context that doesn't match what's embedded
1142    /// in the block, the verification should fail. A Byzantine proposer could broadcast a block
1143    /// with one embedded context but consensus could call verify() with a different context.
1144    #[test_traced("WARN")]
1145    fn test_marshaled_rejects_mismatched_context() {
1146        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1147        runner.start(|mut context| async move {
1148            let Fixture {
1149                participants,
1150                schemes,
1151                ..
1152            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1153            let mut oracle = setup_network_with_participants(
1154                context.child("network"),
1155                NZUsize!(1),
1156                participants.clone(),
1157            )
1158            .await;
1159
1160            let me = participants[0].clone();
1161
1162            let setup = StandardHarness::setup_validator(
1163                context.child("validator").with_attribute("index", 0),
1164                &mut oracle,
1165                me.clone(),
1166                ConstantProvider::new(schemes[0].clone()),
1167            )
1168            .await;
1169            let marshal = setup.mailbox;
1170
1171            let genesis = make_raw_block(Sha256::hash(b""), Height::zero(), 0);
1172            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1173
1174            let mut marshaled = Deferred::new(
1175                context.child("deferred"),
1176                mock_app,
1177                marshal.clone(),
1178                FixedEpocher::new(BLOCKS_PER_EPOCH),
1179            );
1180
1181            // Create parent block at height 1 so the commitment is well-formed.
1182            let parent_ctx = Ctx {
1183                round: Round::new(Epoch::zero(), View::new(1)),
1184                leader: default_leader(),
1185                parent: (View::zero(), genesis.digest()),
1186            };
1187            let parent = B::new::<Sha256>(parent_ctx, genesis.digest(), Height::new(1), 100);
1188            let parent_commitment = StandardHarness::commitment(&parent);
1189
1190            assert!(
1191                marshal
1192                    .clone()
1193                    .verified(Round::new(Epoch::zero(), View::new(1)), parent.clone())
1194                    .await
1195            );
1196
1197            // Build a block with context A (embedded in the block).
1198            let round_a = Round::new(Epoch::zero(), View::new(2));
1199            let context_a = Ctx {
1200                round: round_a,
1201                leader: me.clone(),
1202                parent: (View::new(1), parent_commitment),
1203            };
1204            let block_a = B::new::<Sha256>(context_a, parent.digest(), Height::new(2), 200);
1205            let commitment_a = StandardHarness::commitment(&block_a);
1206            assert!(marshal.verified(round_a, block_a).await);
1207
1208            context.sleep(Duration::from_millis(10)).await;
1209
1210            // Verify using a different consensus context B (hash mismatch).
1211            let round_b = Round::new(Epoch::zero(), View::new(3));
1212            let context_b = Ctx {
1213                round: round_b,
1214                leader: participants[1].clone(),
1215                parent: (View::new(1), parent_commitment),
1216            };
1217
1218            let verify_rx = marshaled.verify(context_b, commitment_a).await;
1219            select! {
1220                result = verify_rx => {
1221                    assert!(
1222                        !result.unwrap(),
1223                        "mismatched context hash should be rejected"
1224                    );
1225                },
1226                _ = context.sleep(Duration::from_secs(5)) => {
1227                    panic!("verify should reject mismatched context hash promptly");
1228                },
1229            }
1230        })
1231    }
1232
1233    /// Dropping the optimistic verify receiver before the block is available can close the
1234    /// synchronously-registered certification gate task. `certify` must recover through the
1235    /// embedded-context path instead of returning the closed task to consensus.
1236    #[test_traced("WARN")]
1237    fn test_deferred_certify_recovers_after_verify_receiver_drop() {
1238        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1239        runner.start(|mut context| async move {
1240            let Fixture {
1241                participants,
1242                schemes,
1243                ..
1244            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1245            let mut oracle = setup_network_with_participants(
1246                context.child("network"),
1247                NZUsize!(1),
1248                participants.clone(),
1249            )
1250            .await;
1251
1252            let me = participants[0].clone();
1253            let setup = StandardHarness::setup_validator(
1254                context.child("validator").with_attribute("index", 0),
1255                &mut oracle,
1256                me.clone(),
1257                ConstantProvider::new(schemes[0].clone()),
1258            )
1259            .await;
1260            let marshal = setup.mailbox;
1261
1262            let genesis = make_raw_block(Sha256::hash(b""), Height::zero(), 0);
1263            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1264            let mut marshaled = Deferred::new(
1265                context.child("deferred"),
1266                mock_app,
1267                marshal.clone(),
1268                FixedEpocher::new(BLOCKS_PER_EPOCH),
1269            );
1270
1271            let round = Round::new(Epoch::zero(), View::new(1));
1272            let block_context = Ctx {
1273                round,
1274                leader: me,
1275                parent: (View::zero(), genesis.digest()),
1276            };
1277            let block =
1278                B::new::<Sha256>(block_context.clone(), genesis.digest(), Height::new(1), 100);
1279            let digest = block.digest();
1280
1281            let verify_rx = marshaled.verify(block_context, digest).await;
1282            drop(verify_rx);
1283
1284            // Give the optimistic task a chance to observe the dropped receiver while its
1285            // block subscription is still pending.
1286            context.sleep(Duration::from_millis(10)).await;
1287
1288            assert!(marshal.verified(round, block).await);
1289            let certify_rx = marshaled.certify(round, digest).await;
1290            select! {
1291                result = certify_rx => {
1292                    assert!(
1293                        result.expect("certify result missing"),
1294                        "certify should recover after verify receiver drop"
1295                    );
1296                },
1297                _ = context.sleep(Duration::from_secs(5)) => {
1298                    panic!("certify should recover promptly after verify drop");
1299                },
1300            }
1301        });
1302    }
1303
1304    /// The store request runs concurrently with `app.verify`, not after it: while
1305    /// gated application verification is still blocked, the block has already
1306    /// reached marshal and is locally queryable even though the sync handle may
1307    /// still be pending. Releasing verification then lets certification await
1308    /// the registered certification gate. Separate restart tests cover durable
1309    /// recovery after certification.
1310    #[test_traced("WARN")]
1311    fn test_deferred_store_overlaps_app_verify() {
1312        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1313        runner.start(|mut context| async move {
1314            let Fixture {
1315                participants,
1316                schemes,
1317                ..
1318            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1319            let mut oracle = setup_network_with_participants(
1320                context.child("network"),
1321                NZUsize!(1),
1322                participants.clone(),
1323            )
1324            .await;
1325
1326            let me = participants[0].clone();
1327
1328            let setup = StandardHarness::setup_validator(
1329                context.child("validator").with_attribute("index", 0),
1330                &mut oracle,
1331                me.clone(),
1332                ConstantProvider::new(schemes[0].clone()),
1333            )
1334            .await;
1335            let marshal = setup.mailbox;
1336            let buffer = setup.extra;
1337
1338            let genesis = make_raw_block(Sha256::hash(b""), Height::zero(), 0);
1339            let (mock_app, verify_started, release_verify): (GatedVerifyingApp<B, S>, _, _) =
1340                GatedVerifyingApp::new();
1341            let mut marshaled = Deferred::new(
1342                context.child("deferred"),
1343                mock_app,
1344                marshal.clone(),
1345                FixedEpocher::new(BLOCKS_PER_EPOCH),
1346            );
1347
1348            // Seed parent and child via the buffer (in-memory only) so
1349            // `deferred_verify` can fetch them without going through the
1350            // persisted marshal path.
1351            let parent = make_raw_block(genesis.digest(), Height::new(1), 100);
1352            let parent_digest = parent.digest();
1353
1354            let child_round = Round::new(Epoch::zero(), View::new(2));
1355            let child_ctx = Ctx {
1356                round: child_round,
1357                leader: me,
1358                parent: (View::new(1), parent_digest),
1359            };
1360            let child = B::new::<Sha256>(child_ctx.clone(), parent_digest, Height::new(2), 200);
1361            let child_digest = child.digest();
1362
1363            assert!(
1364                buffer
1365                    .broadcast(commonware_p2p::Recipients::Some(vec![]), parent)
1366                    .accepted(),
1367                "buffer broadcast for parent should be accepted"
1368            );
1369            assert!(
1370                buffer
1371                    .broadcast(commonware_p2p::Recipients::Some(vec![]), child)
1372                    .accepted(),
1373                "buffer broadcast for child should be accepted"
1374            );
1375
1376            // Kick off the optimistic verify, which spawns `deferred_verify`. Its gated
1377            // `app.verify` blocks until we release it.
1378            let optimistic_rx = marshaled.verify(child_ctx, child_digest).await;
1379            assert!(
1380                optimistic_rx
1381                    .await
1382                    .expect("optimistic verify should resolve"),
1383                "optimistic verify should accept the available block"
1384            );
1385
1386            // Application verification is now blocked. The store request runs concurrently
1387            // with it, so the block is locally queryable even though verification has not
1388            // returned and the sync handle may still be pending.
1389            verify_started
1390                .await
1391                .expect("verify should reach the gated application");
1392            assert!(
1393                marshal.get_block(&child_digest).await.is_some(),
1394                "the store request runs concurrently with app.verify, so the block is locally queryable while verification is still gated"
1395            );
1396
1397            // Releasing verification lets certification succeed (valid and durable).
1398            release_verify.send_lossy(());
1399            let certify_rx = marshaled.certify(child_round, child_digest).await;
1400            select! {
1401                result = certify_rx => {
1402                    assert!(
1403                        result.expect("certify result missing"),
1404                        "certify should succeed once verification passes"
1405                    );
1406                },
1407                _ = context.sleep(Duration::from_secs(5)) => {
1408                    panic!("certify should resolve after verification is released");
1409                },
1410            }
1411        });
1412    }
1413
1414    /// Regression: when marshal holds a verified block for a round from a
1415    /// pre-crash propose, a restarted leader's `propose` must return that
1416    /// block's digest instead of asking the application to build afresh.
1417    /// The recovered proposal must also be staged for the relay, so the
1418    /// broadcast re-sends it and certification resolves through the
1419    /// deduplicated re-persist. The inline variant skips the view instead
1420    /// (see `inline::tests::test_propose_skips_when_verified_block_exists_on_restart`).
1421    #[test_traced("WARN")]
1422    fn test_propose_reuses_verified_block_on_restart() {
1423        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1424        runner.start(|mut context| async move {
1425            let Fixture {
1426                participants,
1427                schemes,
1428                ..
1429            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1430            let mut oracle = setup_network_with_participants(
1431                context.child("network"),
1432                NZUsize!(1),
1433                participants.clone(),
1434            )
1435            .await;
1436
1437            let me = participants[0].clone();
1438            let setup = StandardHarness::setup_validator(
1439                context.child("validator").with_attribute("index", 0),
1440                &mut oracle,
1441                me.clone(),
1442                ConstantProvider::new(schemes[0].clone()),
1443            )
1444            .await;
1445            let marshal = setup.mailbox;
1446
1447            let genesis = make_raw_block(Sha256::hash(b""), Height::zero(), 0);
1448            let round = Round::new(Epoch::zero(), View::new(1));
1449            let ctx = Ctx {
1450                round,
1451                leader: me.clone(),
1452                parent: (View::zero(), genesis.digest()),
1453            };
1454            let block_a = B::new::<Sha256>(ctx.clone(), genesis.digest(), Height::new(1), 100);
1455            let digest_a = block_a.digest();
1456            assert!(marshal.verified(round, block_a.clone()).await);
1457
1458            // The app cannot build (`propose` returns None) and its
1459            // verification never completes, so the assertions below hold
1460            // only if the stored block is reused as-is and certification
1461            // resolves through the durability gate registered by the
1462            // recovery staging.
1463            let (mock_app, verify_started, _release_verify): (GatedVerifyingApp<B, S>, _, _) =
1464                GatedVerifyingApp::new();
1465            let mut marshaled = Deferred::new(
1466                context.child("deferred"),
1467                mock_app,
1468                marshal.clone(),
1469                FixedEpocher::new(BLOCKS_PER_EPOCH),
1470            );
1471
1472            let digest_rx = marshaled.propose(ctx).await;
1473            let digest = digest_rx.await.expect("propose must return a digest");
1474            assert_eq!(
1475                digest, digest_a,
1476                "propose must reuse the block marshal already persisted for this round"
1477            );
1478
1479            // The relay broadcast must find the recovered proposal staged and
1480            // re-persist it (a dedup no-op whose handle covers the pre-crash
1481            // write), resolving the certification gate registered by the
1482            // recovery path.
1483            let _ = marshaled.broadcast(digest, Plan::Propose { round });
1484            let certify_rx = marshaled.certify(round, digest).await;
1485            select! {
1486                result = certify_rx => {
1487                    assert!(
1488                        result.expect("certify result missing"),
1489                        "recovered proposal must certify through the relay handshake"
1490                    );
1491                },
1492                _ = verify_started => {
1493                    panic!("certifying a recovered proposal must not run app verification");
1494                },
1495            }
1496        });
1497    }
1498
1499    /// Regression: if a pre-crash leader persisted a verified block for a
1500    /// round but the simplex `Notarize` never reached the journal, replay
1501    /// can recover a `consensus_context` whose parent differs from the one
1502    /// the cached block was built against (e.g. a late certification of an
1503    /// older view changes the parent selected by `State::find_parent`).
1504    /// In that case the restarted leader must not broadcast the stale
1505    /// cached block; it must drop the receiver so the voter nullifies the
1506    /// view via `MissingProposal`.
1507    #[test_traced("WARN")]
1508    fn test_propose_skips_when_verified_block_context_changed() {
1509        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1510        runner.start(|mut context| async move {
1511            let Fixture {
1512                participants,
1513                schemes,
1514                ..
1515            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1516            let mut oracle = setup_network_with_participants(
1517                context.child("network"),
1518                NZUsize!(1),
1519                participants.clone(),
1520            )
1521            .await;
1522
1523            let me = participants[0].clone();
1524            let setup = StandardHarness::setup_validator(
1525                context.child("validator").with_attribute("index", 0),
1526                &mut oracle,
1527                me.clone(),
1528                ConstantProvider::new(schemes[0].clone()),
1529            )
1530            .await;
1531            let marshal = setup.mailbox;
1532
1533            let genesis = make_raw_block(Sha256::hash(b""), Height::zero(), 0);
1534
1535            // Stash a stale block built against genesis as its parent at round V=2.
1536            let round = Round::new(Epoch::zero(), View::new(2));
1537            let stale_ctx = Ctx {
1538                round,
1539                leader: me.clone(),
1540                parent: (View::zero(), genesis.digest()),
1541            };
1542            let stale_block = B::new::<Sha256>(stale_ctx, genesis.digest(), Height::new(1), 100);
1543            assert!(marshal.verified(round, stale_block).await);
1544
1545            // Simulate a replay where parent selection now points to a
1546            // different parent view than the cached block was built for.
1547            let new_parent_digest = Sha256::hash(b"late-certified-parent");
1548            let new_ctx = Ctx {
1549                round,
1550                leader: me.clone(),
1551                parent: (View::new(1), new_parent_digest),
1552            };
1553
1554            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1555            let mut marshaled = Deferred::new(
1556                context.child("deferred"),
1557                mock_app,
1558                marshal.clone(),
1559                FixedEpocher::new(BLOCKS_PER_EPOCH),
1560            );
1561
1562            let digest_rx = marshaled.propose(new_ctx).await;
1563            assert!(
1564                digest_rx.await.is_err(),
1565                "propose must drop the receiver when the cached block's context no longer matches"
1566            );
1567        });
1568    }
1569
1570    /// Regression: in deferred mode `propose` registers a certification gate that
1571    /// `certify` awaits. After the leader certifies its own proposal, the block must be
1572    /// durably recoverable across an unclean restart. This is the >= f+1 guarantee
1573    /// for the leader's own block.
1574    #[test_traced("WARN")]
1575    fn test_deferred_propose_then_certify_persists_block() {
1576        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1577        runner.start(|mut context| async move {
1578            let Fixture {
1579                participants,
1580                schemes,
1581                ..
1582            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1583            let mut oracle = setup_network_with_participants(
1584                context.child("network"),
1585                NZUsize!(1),
1586                participants.clone(),
1587            )
1588            .await;
1589
1590            let me = participants[0].clone();
1591            let setup = StandardHarness::setup_validator(
1592                context.child("validator").with_attribute("index", 0),
1593                &mut oracle,
1594                me.clone(),
1595                ConstantProvider::new(schemes[0].clone()),
1596            )
1597            .await;
1598            let marshal = setup.mailbox;
1599            let actor_handle = setup.actor_handle;
1600
1601            let genesis = make_raw_block(Sha256::hash(b""), Height::zero(), 0);
1602
1603            // Seed the parent at its round so `propose` can fetch it locally.
1604            let parent_round = Round::new(Epoch::zero(), View::new(1));
1605            let parent_ctx = Ctx {
1606                round: parent_round,
1607                leader: default_leader(),
1608                parent: (View::zero(), genesis.digest()),
1609            };
1610            let parent = B::new::<Sha256>(parent_ctx, genesis.digest(), Height::new(1), 100);
1611            let parent_digest = parent.digest();
1612            assert!(marshal.verified(parent_round, parent).await);
1613
1614            // The leader builds the child via `app.propose`.
1615            let round = Round::new(Epoch::zero(), View::new(2));
1616            let ctx = Ctx {
1617                round,
1618                leader: me.clone(),
1619                parent: (View::new(1), parent_digest),
1620            };
1621            let child = B::new::<Sha256>(ctx.clone(), parent_digest, Height::new(2), 200);
1622            let child_digest = child.digest();
1623            let mock_app: MockVerifyingApp<B, S> =
1624                MockVerifyingApp::new().with_propose_result(child);
1625            let mut marshaled = Deferred::new(
1626                context.child("deferred"),
1627                mock_app,
1628                marshal.clone(),
1629                FixedEpocher::new(BLOCKS_PER_EPOCH),
1630            );
1631
1632            let digest = marshaled
1633                .propose(ctx)
1634                .await
1635                .await
1636                .expect("propose must return a digest");
1637            assert_eq!(
1638                digest, child_digest,
1639                "propose must return the built block's digest"
1640            );
1641
1642            // The leader certifies its own proposal; this awaits the deferred propose sync handle.
1643            assert!(
1644                marshaled
1645                    .certify(round, child_digest)
1646                    .await
1647                    .await
1648                    .expect("certify result missing"),
1649                "certify must succeed for the leader's own proposal"
1650            );
1651
1652            // After certify, the block must be durable across an unclean restart.
1653            actor_handle.abort();
1654            drop(marshaled);
1655            drop(marshal);
1656
1657            let setup2 = StandardHarness::setup_validator(
1658                context
1659                    .child("validator_restart")
1660                    .with_attribute("index", 0),
1661                &mut oracle,
1662                me,
1663                ConstantProvider::new(schemes[0].clone()),
1664            )
1665            .await;
1666            let marshal2 = setup2.mailbox;
1667
1668            assert!(
1669                marshal2.get_block(&child_digest).await.is_some(),
1670                "certify resolved true for the leader's own proposal so the block must be durable"
1671            );
1672        });
1673    }
1674}