Skip to main content

commonware_consensus/marshal/standard/
inline.rs

1//! Wrapper for standard marshal with inline verification.
2//!
3//! # Overview
4//!
5//! [`Inline`] adapts any [`Application`] to the marshal/consensus interfaces
6//! while keeping block validation in the [`Automaton::verify`] path. Unlike
7//! [`super::Deferred`], it does not defer application verification to certification.
8//! Instead, it only reports `true` from `verify` after parent/height checks and
9//! application verification complete.
10//!
11//! # Epoch Boundaries
12//!
13//! As with [`super::Deferred`], when the parent is the last block of the epoch,
14//! [`Inline`] re-proposes that boundary block instead of building a new block.
15//! This prevents proposing blocks that would be excluded by epoch transition.
16//!
17//! # Verification Model
18//!
19//! Inline mode intentionally avoids relying on embedded block context. This allows
20//! usage with block types that implement [`crate::Block`] but not
21//! [`crate::CertifiableBlock`].
22//!
23//! Because verification is completed inline, `certify` must only wait for data
24//! availability in marshal. No additional deferred verification state needs to
25//! be awaited at certify time.
26//!
27//! # Usage
28//!
29//! ```rust,ignore
30//! let application = Inline::new(
31//!     context,
32//!     my_application,
33//!     marshal_mailbox,
34//!     epocher,
35//! );
36//! ```
37//!
38//! # When to Use
39//!
40//! Prefer this wrapper when:
41//! - Your application block type is not certifiable.
42//! - You prefer simpler verification semantics over deferred verification latency hiding.
43//! - You are willing to perform full application verification before casting a notarize vote.
44
45use crate::{
46    Application, Automaton, Block, CertifiableAutomaton, Epochable, Relay, Reporter,
47    marshal::{
48        Update,
49        application::gates::{GateOutcome, Gates},
50        core::{CommitmentFallback, DigestFallback, Mailbox},
51        standard::{
52            Standard, relay,
53            validation::{
54                Decision, ParentCheck, await_and_validate_parent, precheck_epoch_and_reproposal,
55                run_app_verify,
56            },
57        },
58    },
59    simplex::{Plan, types::Context},
60    types::{Epocher, Round},
61};
62use commonware_actor::Feedback;
63use commonware_cryptography::certificate::Scheme;
64use commonware_macros::select;
65use commonware_runtime::{
66    Clock, Metrics, Spawner,
67    telemetry::{
68        metrics::{
69            MetricsExt as _,
70            histogram::{Buckets, Timed},
71        },
72        traces::TracedExt as _,
73    },
74};
75use commonware_utils::{
76    channel::{fallible::OneshotExt, oneshot},
77    sync::TracedAsyncMutex,
78};
79use rand_core::Rng;
80use std::sync::Arc;
81use tracing::{Instrument as _, debug, info_span};
82
83/// Waits for a marshal block subscription while allowing consensus to cancel the work.
84async fn await_block_subscription<T, D>(
85    tx: &mut oneshot::Sender<bool>,
86    block_rx: oneshot::Receiver<T>,
87    digest: &D,
88    stage: &'static str,
89) -> Option<T>
90where
91    D: std::fmt::Debug,
92{
93    select! {
94        _ = tx.closed() => {
95            debug!(
96                stage,
97                reason = "consensus dropped receiver",
98                "skipping block wait"
99            );
100            None
101        },
102        result = block_rx => {
103            if result.is_err() {
104                debug!(
105                    stage,
106                    ?digest,
107                    reason = "failed to fetch block",
108                    "skipping block wait"
109                );
110            }
111            result.ok()
112        },
113    }
114}
115
116/// Standard marshal wrapper that verifies blocks inline in `verify`.
117///
118/// # Ancestry Validation
119///
120/// [`Inline`] always validates immediate ancestry before invoking application
121/// verification:
122/// - Parent digest matches consensus context's expected parent
123/// - Child height is exactly parent height plus one
124///
125/// This is sufficient because the parent was either notarized by consensus or verified locally
126/// before this participant voted for it, so its own linkage was already checked.
127///
128/// # Certifiability
129///
130/// This wrapper requires only [`crate::Block`] for `B`, not
131/// [`crate::CertifiableBlock`]. It is designed for applications that cannot
132/// recover consensus context directly from block payloads.
133pub struct Inline<E, S, A, B, ES>
134where
135    E: Rng + Spawner + Metrics + Clock,
136    S: Scheme,
137    A: Application<E>,
138    B: Block + Clone,
139    ES: Epocher,
140{
141    context: Arc<TracedAsyncMutex<E>>,
142    application: A,
143    marshal: Mailbox<S, Standard<B>>,
144    epocher: ES,
145    gates: Gates<B::Digest, B>,
146
147    build_duration: Timed,
148    proposal_parent_fetch_duration: Timed,
149    ancestor_fetch_duration: Timed,
150}
151
152impl<E, S, A, B, ES> Clone for Inline<E, S, A, B, ES>
153where
154    E: Rng + Spawner + Metrics + Clock,
155    S: Scheme,
156    A: Application<E>,
157    B: Block + Clone,
158    ES: Epocher,
159{
160    fn clone(&self) -> Self {
161        Self {
162            context: self.context.clone(),
163            application: self.application.clone(),
164            marshal: self.marshal.clone(),
165            epocher: self.epocher.clone(),
166            gates: self.gates.clone(),
167            build_duration: self.build_duration.clone(),
168            proposal_parent_fetch_duration: self.proposal_parent_fetch_duration.clone(),
169            ancestor_fetch_duration: self.ancestor_fetch_duration.clone(),
170        }
171    }
172}
173
174impl<E, S, A, B, ES> Inline<E, S, A, B, ES>
175where
176    E: Rng + Spawner + Metrics + Clock,
177    S: Scheme,
178    A: Application<
179            E,
180            Block = B,
181            SigningScheme = S,
182            Context = Context<B::Digest, S::PublicKey>,
183            Input = (),
184        >,
185    B: Block + Clone,
186    ES: Epocher,
187{
188    /// Creates a new inline-verification wrapper.
189    ///
190    /// Registers a `build_duration` histogram for proposal latency.
191    pub fn new(context: E, application: A, marshal: Mailbox<S, Standard<B>>, epocher: ES) -> Self {
192        let build_histogram = context.histogram(
193            "build_duration",
194            "Histogram of time taken for the application to build a new block, in seconds",
195            Buckets::LOCAL,
196        );
197        let build_duration = Timed::new(build_histogram);
198        let parent_fetch_histogram = context.histogram(
199            "parent_fetch_duration",
200            "Histogram of time taken to fetch a parent block in propose, in seconds",
201            Buckets::LOCAL,
202        );
203        let proposal_parent_fetch_duration = Timed::new(parent_fetch_histogram);
204        let ancestor_fetch_histogram = context.histogram(
205            "ancestor_fetch_duration",
206            "Histogram of time taken to fetch a block via the ancestry stream, in seconds",
207            Buckets::LOCAL,
208        );
209        let ancestor_fetch_duration = Timed::new(ancestor_fetch_histogram);
210
211        Self {
212            context: Arc::new(TracedAsyncMutex::new("marshal.context", context)),
213            application,
214            marshal,
215            epocher,
216            gates: Gates::new(),
217            build_duration,
218            proposal_parent_fetch_duration,
219            ancestor_fetch_duration,
220        }
221    }
222}
223
224impl<E, S, A, B, ES> Automaton for Inline<E, S, A, B, ES>
225where
226    E: Rng + Spawner + Metrics + Clock,
227    S: Scheme,
228    A: Application<
229            E,
230            Block = B,
231            SigningScheme = S,
232            Context = Context<B::Digest, S::PublicKey>,
233            Input = (),
234        >,
235    B: Block + Clone,
236    ES: Epocher,
237{
238    type Digest = B::Digest;
239    type Context = Context<Self::Digest, S::PublicKey>;
240
241    /// Proposes a new block or re-proposes an epoch boundary block.
242    ///
243    /// Proposal runs in a spawned task and returns a receiver for the resulting digest. The
244    /// block is staged before the digest is delivered and handed to marshal when consensus
245    /// requests the relay broadcast, which persists it after the send. The resulting sync
246    /// handle is awaited only at certification so it overlaps consensus voting. The digest does
247    /// not imply durability on its own. [`CertifiableAutomaton::certify`] awaits the registered
248    /// certification gate before the finalize vote.
249    #[allow(clippy::async_yields_async)]
250    #[tracing::instrument(name = "marshal.inline.propose", level = "info", skip_all, fields(round = %consensus_context.round))]
251    async fn propose(
252        &mut self,
253        consensus_context: Context<Self::Digest, S::PublicKey>,
254    ) -> oneshot::Receiver<Self::Digest> {
255        let marshal = self.marshal.clone();
256        let mut application = self.application.clone();
257        let epocher = self.epocher.clone();
258        let gates = self.gates.clone();
259        let build_duration = self.build_duration.clone();
260        let proposal_parent_fetch_duration = self.proposal_parent_fetch_duration.clone();
261        let ancestor_fetch_duration = self.ancestor_fetch_duration.clone();
262
263        let (mut tx, rx) = oneshot::channel();
264        let context = self
265            .context
266            .lock()
267            .await
268            .child("propose")
269            .with_attribute("round", consensus_context.round);
270        let span = info_span!(
271            "marshal.inline.propose.task",
272            round = %consensus_context.round
273        );
274        context.spawn(move |runtime_context| {
275            async move {
276                // On leader recovery, marshal may already hold a verified block
277                // for this round (persisted by a pre-crash propose that reached
278                // its relay broadcast while the notarize vote never reached the
279                // journal).
280                //
281                // The parent context recovered by simplex may differ from the one
282                // the cached block was built against, so the stored block is not
283                // safe to reuse, and proposing a fresh block for a round whose
284                // digest may already have been broadcast would equivocate.
285                //
286                // Skip this view and let the voter nullify it via timeout.
287                if marshal
288                    .get_verified(consensus_context.round)
289                    .await
290                    .is_some()
291                {
292                    debug!(
293                        round = ?consensus_context.round,
294                        "skipping proposal: verified block already exists for round on restart"
295                    );
296                    return;
297                }
298
299                // The parent for any consensus context is in the same epoch: the
300                // boundary block of the previous epoch is the genesis block of the
301                // current epoch.
302                //
303                // Proposal context carries the certified parent view/commitment but
304                // not the parent height. The parent may be certified above the
305                // finalized tip, so this must stay round-bound until the block is
306                // returned.
307                let (parent_view, parent_commitment) = consensus_context.parent;
308                let parent_request = marshal.subscribe_by_commitment(
309                    parent_commitment,
310                    CommitmentFallback::FetchByRound {
311                        round: Round::new(consensus_context.epoch(), parent_view),
312                    },
313                );
314
315                let parent_timer = proposal_parent_fetch_duration.timer(&runtime_context);
316                let parent = select! {
317                    _ = tx.closed() => {
318                        debug!(reason = "consensus dropped receiver", "skipping proposal");
319                        return;
320                    },
321                    result = parent_request => match result {
322                        Ok(parent) => parent,
323                        Err(_) => {
324                            debug!(
325                                ?parent_commitment,
326                                reason = "failed to fetch parent block",
327                                "skipping proposal"
328                            );
329                            return;
330                        }
331                    },
332                };
333                parent_timer.observe(&runtime_context);
334
335                // At epoch boundary, re-propose the parent block.
336                let last_in_epoch = epocher
337                    .last(consensus_context.epoch())
338                    .expect("current epoch should exist");
339                if parent.height() == last_in_epoch {
340                    let digest = parent.digest();
341                    gates
342                        .stage(
343                            consensus_context.round,
344                            digest,
345                            parent,
346                            tx,
347                            "re-proposed boundary block",
348                        )
349                        .await;
350                    return;
351                }
352
353                let ancestor_stream = marshal.ancestor_stream(
354                    Arc::new(runtime_context.child("ancestor_stream")),
355                    [parent],
356                    ancestor_fetch_duration,
357                );
358                let build_request = application
359                    .propose(
360                        (
361                            runtime_context.child("app_propose"),
362                            consensus_context.clone(),
363                        ),
364                        ancestor_stream,
365                        (),
366                    )
367                    .instrument(info_span!(
368                        "marshal.inline.application.propose",
369                        round = %consensus_context.round,
370                        parent_view = parent_view.traced(),
371                        parent = %parent_commitment
372                    ));
373
374                let build_timer = build_duration.timer(&runtime_context);
375                let built_block = select! {
376                    _ = tx.closed() => {
377                        debug!(reason = "consensus dropped receiver", "skipping proposal");
378                        return;
379                    },
380                    result = build_request => match result {
381                        Some(block) => block,
382                        None => {
383                            debug!(
384                                ?parent_commitment,
385                                reason = "block building failed",
386                                "skipping proposal"
387                            );
388                            return;
389                        }
390                    },
391                };
392                build_timer.observe(&runtime_context);
393
394                let digest = built_block.digest();
395                gates
396                    .stage(
397                        consensus_context.round,
398                        digest,
399                        Arc::new(built_block),
400                        tx,
401                        "proposed block",
402                    )
403                    .await;
404            }
405            .instrument(span)
406        });
407        rx
408    }
409
410    /// Performs complete verification inline.
411    ///
412    /// This method:
413    /// 1. Waits for the block by digest
414    /// 2. Enforces epoch/re-proposal rules
415    /// 3. Fetches and validates the parent relationship
416    /// 4. Runs application verification over ancestry
417    ///
418    /// The notarize vote is cast as soon as application verification completes. The block's
419    /// durable sync is deferred (it runs concurrently with consensus voting) and its
420    /// completion is registered in `gates` for [`Self::certify`] to await before
421    /// the finalize vote.
422    #[allow(clippy::async_yields_async)]
423    #[tracing::instrument(name = "marshal.inline.verify", level = "info", skip_all, fields(round = %context.round, digest = %digest))]
424    async fn verify(
425        &mut self,
426        context: Context<Self::Digest, S::PublicKey>,
427        digest: Self::Digest,
428    ) -> oneshot::Receiver<bool> {
429        let round = context.round;
430
431        // Verification needs the full block but waits only for local delivery. Certification starts
432        // recovery only when the block is not buffered. If a buffered block is evicted before
433        // verification registers its wait, verification is left with neither the block nor an
434        // active fetch. Register the wait before publishing the gate so it receives the buffered
435        // block or is waiting when recovery delivers it.
436        let block_request = self
437            .marshal
438            .subscribe_by_digest(digest, DigestFallback::Wait);
439        let (durable_tx, durable_rx) = oneshot::channel();
440        self.gates.insert(round, digest, durable_rx);
441
442        let marshal = self.marshal.clone();
443        let mut application = self.application.clone();
444        let epocher = self.epocher.clone();
445        let ancestor_fetch_duration = self.ancestor_fetch_duration.clone();
446
447        let (mut tx, rx) = oneshot::channel();
448        let runtime_context = self
449            .context
450            .lock()
451            .await
452            .child("inline_verify")
453            .with_attribute("round", round);
454        let span = info_span!(
455            "marshal.inline.verify.task",
456            round = %round,
457            digest = %digest
458        );
459        runtime_context.spawn(move |runtime_context| {
460            async move {
461                // Start the parent fetch immediately: its commitment and certified
462                // round are known from the consensus context, so it can proceed in
463                // parallel with broadcast delivery of the candidate block.
464                // Reproposals (digest == context.parent.1) skip parent validation
465                // entirely, so they must not fetch: the "parent" is the candidate
466                // itself, and candidate acquisition is deliberately local-only.
467                let parent_request = (digest != context.parent.1).then(|| {
468                    let (parent_view, parent_commitment) = context.parent;
469                    marshal.subscribe_by_commitment(
470                        parent_commitment,
471                        CommitmentFallback::FetchByRound {
472                            round: Round::new(context.epoch(), parent_view),
473                        },
474                    )
475                });
476
477                let Some(block) =
478                    await_block_subscription(&mut tx, block_request, &digest, "verification").await
479                else {
480                    return;
481                };
482
483                // Shared pre-checks:
484                // - Blocks are invalid if they are not in the expected epoch and are
485                //   not a valid boundary re-proposal.
486                // - Re-proposals are detected when `digest == context.parent.1`.
487                // - Re-proposals skip normal parent/height checks because:
488                //   1) consensus settles their validity when certifying the view that
489                //      first carried the block
490                //   2) parent-child checks would fail by construction when parent == block
491                let Some(decision) =
492                    precheck_epoch_and_reproposal(&epocher, &marshal, &context, digest, block)
493                        .await
494                else {
495                    return;
496                };
497                let block = match decision {
498                    Decision::Complete(valid) => {
499                        // Re-proposal: a valid precheck already persisted the block
500                        // (durable), an invalid one is an epoch rejection. The re-proposal
501                        // reading is an artifact of this header's declared parent: a
502                        // conflicting header naming the block's real parent lets honest
503                        // validators verify the same block as a normal proposal and
504                        // notarize `(round, digest)`, so certification must recover
505                        // rather than adopt the rejection.
506                        tx.send_lossy(valid);
507                        durable_tx.send_lossy(if valid {
508                            GateOutcome::Ready(true)
509                        } else {
510                            GateOutcome::Recover
511                        });
512                        return;
513                    }
514                    Decision::Continue(block) => block,
515                };
516
517                // `Continue` implies a non-reproposal, so the parent subscription
518                // was started above.
519                let parent_request =
520                    parent_request.expect("non-reproposal has a parent subscription");
521
522                // Start the candidate store immediately: it depends on neither the
523                // parent fetch (which may hit the network) nor the verdict below.
524                // Storing before validation is intentional: these caches provide
525                // candidate availability/recovery, not a validity decision. The
526                // notarize vote follows the app verdict, while certify independently
527                // awaits the registered durability gate.
528                //
529                // The verify future below aborts when consensus drops its receiver
530                // (the view exited via nullification or finalization), even though
531                // certification can still fire for a nullified view. That is
532                // deliberate: inline certification does not need the local app
533                // verdict (a notarization implies f+1 honest validators already
534                // verified), and the store still completes through the join.
535                let store = async {
536                    if marshal.verified(round, Arc::clone(&block)).await {
537                        durable_tx.send_lossy(GateOutcome::Ready(true));
538                    }
539                };
540                let verify_then_vote = async {
541                    // Non-reproposal path: validate the parent we already started
542                    // fetching.
543                    let parent = match await_and_validate_parent(
544                        context.parent.1,
545                        block.as_ref(),
546                        parent_request,
547                        &mut tx,
548                    )
549                    .await
550                    {
551                        Some(ParentCheck::Valid(parent)) => parent,
552                        Some(ParentCheck::Invalid) => {
553                            tx.send_lossy(false);
554                            return Some(false);
555                        }
556                        None => return None,
557                    };
558                    let valid = run_app_verify(
559                        runtime_context,
560                        context,
561                        Arc::clone(&block),
562                        parent,
563                        &mut application,
564                        &marshal,
565                        &mut tx,
566                        ancestor_fetch_duration,
567                    )
568                    .await;
569                    if let Some(valid) = valid {
570                        tx.send_lossy(valid);
571                    }
572                    valid
573                };
574                futures::join!(verify_then_vote, store);
575            }
576            .instrument(span)
577        });
578        rx
579    }
580}
581
582/// Inline certification consumes a registered certification gate when present, and
583/// falls back to a round-bound fetch/persist path when the gate is missing (after
584/// an unclean restart) or cannot speak for the notarized proposal.
585impl<E, S, A, B, ES> CertifiableAutomaton for Inline<E, S, A, B, ES>
586where
587    E: Rng + Spawner + Metrics + Clock,
588    S: Scheme,
589    A: Application<
590            E,
591            Block = B,
592            SigningScheme = S,
593            Context = Context<B::Digest, S::PublicKey>,
594            Input = (),
595        >,
596    B: Block + Clone,
597    ES: Epocher,
598{
599    #[allow(clippy::async_yields_async)]
600    #[tracing::instrument(name = "marshal.inline.certify", level = "info", skip_all, fields(round = %round, digest = %digest))]
601    async fn certify(&mut self, round: Round, digest: Self::Digest) -> oneshot::Receiver<bool> {
602        self.gates.flush_unrelayed(&self.marshal, round, digest);
603
604        // `propose`/`verify` register an in-flight certification gate whose result resolves
605        // once the block's sync handle completes. Awaiting it here is the durability barrier
606        // for the finalize vote, and it lets the sync overlap consensus voting
607        // instead of freezing certify with a fresh fsync.
608        let task = self.gates.take(round, digest);
609
610        // `verify()` waits only on local broadcast delivery, so nudge a
611        // round-bound notarized fetch that can unblock the existing waiter
612        // if local broadcast never arrives. For the standard variant, the
613        // digest is also the variant commitment.
614        if task.is_some() {
615            self.marshal.hint_notarized(round, digest);
616        }
617        let marshal = self.marshal.clone();
618        let (mut tx, rx) = oneshot::channel();
619        let context = self
620            .context
621            .lock()
622            .await
623            .child("inline_certify")
624            .with_attribute("round", round);
625        context.spawn(move |_| {
626            async move {
627                // A ready gate proves the notarized block is durable. A gate that
628                // cannot speak for the notarized proposal falls through to recovery.
629                if let Some(task) = task {
630                    let result = select! {
631                        _ = tx.closed() => {
632                            debug!(
633                                reason = "consensus dropped receiver",
634                                "skipping certification"
635                            );
636                            return;
637                        },
638                        result = task => result,
639                    };
640                    match result {
641                        Ok(GateOutcome::Ready(verdict)) => {
642                            tx.send_lossy(verdict);
643                            return;
644                        }
645                        Ok(GateOutcome::Recover) | Err(_) => {}
646                    }
647                }
648
649                // No local certification gate task (for example after an unclean restart):
650                // fetch the notarized block and persist it. A Byzantine leader can form a
651                // notarization after sending the proposal to only f+1 honest validators, so
652                // the validators left without the block must fetch it here to certify and
653                // avoid getting stuck.
654                let block_rx =
655                    marshal.subscribe_by_digest(digest, DigestFallback::FetchByRound { round });
656                let Some(block) =
657                    await_block_subscription(&mut tx, block_rx, &digest, "certification").await
658                else {
659                    return;
660                };
661                if !marshal.certified(round, block).await {
662                    return;
663                }
664                tx.send_lossy(true);
665            }
666            .instrument(info_span!(
667                "marshal.inline.certify.task",
668                round = %round,
669                digest = %digest
670            ))
671        });
672
673        rx
674    }
675}
676
677impl<E, S, A, B, ES> Relay for Inline<E, S, A, B, ES>
678where
679    E: Rng + Spawner + Metrics + Clock,
680    S: Scheme,
681    A: Application<E, Block = B, Context = Context<B::Digest, S::PublicKey>>,
682    B: Block + Clone,
683    ES: Epocher,
684{
685    type Digest = B::Digest;
686    type PublicKey = S::PublicKey;
687    type Plan = Plan<S::PublicKey>;
688
689    fn broadcast(&mut self, commitment: Self::Digest, plan: Plan<S::PublicKey>) -> Feedback {
690        relay::broadcast(&self.gates, &self.marshal, commitment, plan)
691    }
692}
693
694impl<E, S, A, B, ES> Reporter for Inline<E, S, A, B, ES>
695where
696    E: Rng + Spawner + Metrics + Clock,
697    S: Scheme,
698    A: Application<E, Block = B, Context = Context<B::Digest, S::PublicKey>>
699        + Reporter<Activity = Update<B>>,
700    B: Block + Clone,
701    ES: Epocher,
702{
703    type Activity = A::Activity;
704
705    /// Forwards consensus activity to the wrapped application reporter.
706    fn report(&mut self, update: Self::Activity) -> Feedback {
707        if let Update::Tip(tip_round, _, _) = &update {
708            self.gates.retain_after(tip_round);
709        }
710        self.application.report(update)
711    }
712}
713
714#[cfg(test)]
715mod tests {
716    use super::Inline;
717    use crate::{
718        Application, Automaton, Block, CertifiableAutomaton, Relay,
719        marshal::mocks::{
720            harness::{
721                B, BLOCKS_PER_EPOCH, Ctx, NAMESPACE, NUM_VALIDATORS, S, StandardHarness,
722                TestHarness, V, default_leader, make_raw_block, setup_network_with_participants,
723            },
724            verifying::{GatedVerifyingApp, MockVerifyingApp},
725        },
726        simplex::{scheme::bls12381_threshold::vrf as bls12381_threshold_vrf, types::Context},
727        types::{Epoch, FixedEpocher, Height, Round, View},
728    };
729    use commonware_broadcast::Broadcaster;
730    use commonware_cryptography::{
731        Digestible, Hasher as _,
732        certificate::{ConstantProvider, Scheme, mocks::Fixture},
733        sha256::Sha256,
734    };
735    use commonware_macros::{select, test_traced};
736    use commonware_runtime::{Clock, Metrics, Runner, Spawner, Supervisor as _, deterministic};
737    use commonware_utils::{NZUsize, channel::fallible::OneshotExt};
738    use rand::Rng;
739    use std::time::Duration;
740
741    // Compile-time assertion only: inline standard wrapper must not require `CertifiableBlock`.
742    #[allow(dead_code)]
743    fn assert_non_certifiable_block_supported<E, S, A, B, ES>()
744    where
745        E: Rng + Spawner + Metrics + Clock,
746        S: Scheme,
747        A: Application<
748                E,
749                Block = B,
750                SigningScheme = S,
751                Context = Context<B::Digest, S::PublicKey>,
752                Input = (),
753            >,
754        B: Block + Clone,
755        ES: crate::types::Epocher,
756    {
757        fn assert_automaton<T: Automaton>() {}
758        fn assert_certifiable<T: CertifiableAutomaton>() {}
759        fn assert_relay<T: Relay>() {}
760
761        assert_automaton::<Inline<E, S, A, B, ES>>();
762        assert_certifiable::<Inline<E, S, A, B, ES>>();
763        assert_relay::<Inline<E, S, A, B, ES>>();
764    }
765
766    #[test_traced("INFO")]
767    fn test_certify_returns_immediately_after_verify_fetches_block() {
768        let runner = deterministic::Runner::timed(Duration::from_secs(30));
769        runner.start(|mut context| async move {
770            let Fixture {
771                participants,
772                schemes,
773                ..
774            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
775            let mut oracle = setup_network_with_participants(
776                context.child("network"),
777                NZUsize!(1),
778                participants.clone(),
779            )
780            .await;
781
782            let me = participants[0].clone();
783            let setup = StandardHarness::setup_validator(
784                context.child("validator").with_attribute("index", 0),
785                &mut oracle,
786                me.clone(),
787                ConstantProvider::new(schemes[0].clone()),
788            )
789            .await;
790            let marshal = setup.mailbox;
791
792            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
793            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
794            let mut inline = Inline::new(
795                context.child("inline"),
796                mock_app,
797                marshal.clone(),
798                FixedEpocher::new(BLOCKS_PER_EPOCH),
799            );
800
801            // Seed the parent and child blocks in marshal so verify can fetch locally.
802            let parent_round = Round::new(Epoch::zero(), View::new(1));
803            let parent_ctx = Ctx {
804                round: parent_round,
805                leader: default_leader(),
806                parent: (View::zero(), genesis.digest()),
807            };
808            let parent = B::new::<Sha256>(parent_ctx, genesis.digest(), Height::new(1), 100);
809            let parent_digest = parent.digest();
810            assert!(marshal.verified(parent_round, parent).await);
811
812            let round = Round::new(Epoch::zero(), View::new(2));
813            let verify_context = Ctx {
814                round,
815                leader: me,
816                parent: (View::new(1), parent_digest),
817            };
818            let block =
819                B::new::<Sha256>(verify_context.clone(), parent_digest, Height::new(2), 200);
820            let digest = block.digest();
821            assert!(marshal.verified(round, block).await);
822
823            // Complete verify first so the block is already available locally.
824            let verify_rx = inline.verify(verify_context, digest).await;
825            assert!(
826                verify_rx.await.unwrap(),
827                "verify should complete successfully before certify"
828            );
829
830            // Certify should return immediately instead of waiting on marshal.
831            let certify_rx = inline.certify(round, digest).await;
832
833            select! {
834                result = certify_rx => {
835                    assert!(
836                        result.unwrap(),
837                        "certify should return immediately once verify has fetched the block"
838                    );
839                },
840                _ = context.sleep(Duration::from_secs(5)) => {
841                    panic!("certify should not hang after local verify completed");
842                },
843            }
844        });
845    }
846
847    #[test_traced("INFO")]
848    fn test_certify_succeeds_without_verify_task() {
849        let runner = deterministic::Runner::timed(Duration::from_secs(30));
850        runner.start(|mut context| async move {
851            let Fixture {
852                participants,
853                schemes,
854                ..
855            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
856            let mut oracle = setup_network_with_participants(
857                context.child("network"),
858                NZUsize!(1),
859                participants.clone(),
860            )
861            .await;
862
863            let me = participants[0].clone();
864            let setup = StandardHarness::setup_validator(
865                context.child("validator").with_attribute("index", 0),
866                &mut oracle,
867                me.clone(),
868                ConstantProvider::new(schemes[0].clone()),
869            )
870            .await;
871            let marshal = setup.mailbox;
872
873            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
874            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
875            let mut inline = Inline::new(
876                context.child("inline"),
877                mock_app,
878                marshal.clone(),
879                FixedEpocher::new(BLOCKS_PER_EPOCH),
880            );
881
882            // Seed the parent and child blocks in marshal without starting a verify task.
883            let parent_round = Round::new(Epoch::zero(), View::new(1));
884            let parent_ctx = Ctx {
885                round: parent_round,
886                leader: default_leader(),
887                parent: (View::zero(), genesis.digest()),
888            };
889            let parent = B::new::<Sha256>(parent_ctx, genesis.digest(), Height::new(1), 100);
890            let parent_digest = parent.digest();
891            assert!(marshal.verified(parent_round, parent).await);
892
893            let round = Round::new(Epoch::zero(), View::new(2));
894            let verify_context = Ctx {
895                round,
896                leader: me,
897                parent: (View::new(1), parent_digest),
898            };
899            let block =
900                B::new::<Sha256>(verify_context.clone(), parent_digest, Height::new(2), 200);
901            let digest = block.digest();
902            assert!(marshal.verified(round, block).await);
903
904            // Certify should still resolve by waiting on marshal block availability directly.
905            let certify_rx = inline.certify(round, digest).await;
906
907            select! {
908                result = certify_rx => {
909                    assert!(
910                        result.unwrap(),
911                        "certify should resolve once block availability is known"
912                    );
913                },
914                _ = context.sleep(Duration::from_secs(5)) => {
915                    panic!("certify should not hang when block is already available in marshal");
916                },
917            }
918        });
919    }
920
921    #[test_traced("INFO")]
922    fn test_certify_reproposal_uses_available_blocks_after_verify() {
923        let runner = deterministic::Runner::timed(Duration::from_secs(30));
924        runner.start(|mut context| async move {
925            let Fixture {
926                participants,
927                schemes,
928                ..
929            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
930            let mut oracle =
931                setup_network_with_participants(context.child("network"), NZUsize!(1), participants.clone())
932                    .await;
933
934            let me = participants[0].clone();
935            let setup = StandardHarness::setup_validator(
936                context.child("validator").with_attribute("index", 0),
937                &mut oracle,
938                me.clone(),
939                ConstantProvider::new(schemes[0].clone()),
940            )
941            .await;
942            let marshal = setup.mailbox;
943            let marshal_actor_handle = setup.actor_handle;
944
945            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
946            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
947            let mut inline = Inline::new(context.child("inline"),
948                mock_app,
949                marshal.clone(),
950                FixedEpocher::new(BLOCKS_PER_EPOCH),
951            );
952
953            let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
954            let boundary_round = Round::new(Epoch::zero(), View::new(boundary_height.get()));
955            let boundary_block = B::new::<Sha256>(
956                Ctx {
957                    round: boundary_round,
958                    leader: default_leader(),
959                    parent: (View::zero(), genesis.digest()),
960                },
961                genesis.digest(),
962                boundary_height,
963                1900,
964            );
965            let boundary_digest = boundary_block.digest();
966            assert!(
967                marshal.verified(boundary_round, boundary_block).await
968            );
969
970            let reproposal_round = Round::new(Epoch::zero(), View::new(boundary_height.get() + 1));
971            let reproposal_context = Ctx {
972                round: reproposal_round,
973                leader: me,
974                parent: (View::new(boundary_height.get()), boundary_digest),
975            };
976
977            let verify_rx = inline.verify(reproposal_context, boundary_digest).await;
978            assert!(
979                verify_rx.await.unwrap(),
980                "verify should accept a valid boundary re-proposal"
981            );
982
983            marshal_actor_handle.abort();
984            drop(marshal);
985            context.sleep(Duration::from_millis(1)).await;
986
987            let certify_rx = inline.certify(reproposal_round, boundary_digest).await;
988            select! {
989                result = certify_rx => {
990                    assert!(
991                        result.unwrap(),
992                        "certify should use the available_blocks fast path for verified re-proposals"
993                    );
994                },
995                _ = context.sleep(Duration::from_secs(5)) => {
996                    panic!("certify should not depend on marshal after verify cached a re-proposal");
997                },
998            }
999        });
1000    }
1001
1002    /// A header can only read as a re-proposal under its declared parent, so an
1003    /// invalid-precheck rejection (non-boundary block) is scoped to that header,
1004    /// not to the notarized `(round, digest)`. Certification must route through
1005    /// recovery instead of adopting the rejection as its verdict.
1006    #[test_traced("WARN")]
1007    fn test_certify_recovers_from_invalid_reproposal_precheck() {
1008        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1009        runner.start(|mut context| async move {
1010            let Fixture {
1011                participants,
1012                schemes,
1013                ..
1014            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1015            let mut oracle = setup_network_with_participants(
1016                context.child("network"),
1017                NZUsize!(1),
1018                participants.clone(),
1019            )
1020            .await;
1021
1022            let me = participants[0].clone();
1023            let setup = StandardHarness::setup_validator(
1024                context.child("validator").with_attribute("index", 0),
1025                &mut oracle,
1026                me.clone(),
1027                ConstantProvider::new(schemes[0].clone()),
1028            )
1029            .await;
1030            let marshal = setup.mailbox;
1031
1032            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1033            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1034            let mut inline = Inline::new(
1035                context.child("inline"),
1036                mock_app,
1037                marshal.clone(),
1038                FixedEpocher::new(BLOCKS_PER_EPOCH),
1039            );
1040
1041            // A non-boundary block notarized at view 2 and available locally.
1042            let block_round = Round::new(Epoch::zero(), View::new(2));
1043            let block = B::new::<Sha256>(
1044                Ctx {
1045                    round: block_round,
1046                    leader: default_leader(),
1047                    parent: (View::zero(), genesis.digest()),
1048                },
1049                genesis.digest(),
1050                Height::new(2),
1051                200,
1052            );
1053            let digest = block.digest();
1054            assert!(marshal.verified(block_round, block).await);
1055
1056            // The view-3 header names the block as its own parent, so the
1057            // precheck reads it as a re-proposal and rejects it (not at the
1058            // epoch boundary).
1059            let round = Round::new(Epoch::zero(), View::new(3));
1060            let reproposal_context = Ctx {
1061                round,
1062                leader: me,
1063                parent: (View::new(2), digest),
1064            };
1065            let verify_rx = inline.verify(reproposal_context, digest).await;
1066            assert!(
1067                !verify_rx.await.expect("verify result missing"),
1068                "a non-boundary re-proposal must be rejected"
1069            );
1070
1071            // The header-scoped rejection must not become the certification
1072            // verdict for the notarized digest.
1073            let certify_rx = inline.certify(round, digest).await;
1074            select! {
1075                result = certify_rx => {
1076                    assert!(
1077                        result.expect("certify result missing"),
1078                        "certify must recover instead of adopting the precheck rejection"
1079                    );
1080                },
1081                _ = context.sleep(Duration::from_secs(5)) => {
1082                    panic!("certify should resolve promptly");
1083                },
1084            }
1085        });
1086    }
1087
1088    /// Regression: `certify` resolving true drives the finalize vote in inline
1089    /// mode, so it must imply the block is durably persisted even when the
1090    /// certify path subscribed before `verify()` finished.
1091    #[test_traced("WARN")]
1092    fn test_inline_certify_persists_block_before_resolving() {
1093        for seed in 0u64..16 {
1094            inline_certify_persists_block_before_resolving_at(seed);
1095        }
1096    }
1097
1098    fn inline_certify_persists_block_before_resolving_at(seed: u64) {
1099        let runner = deterministic::Runner::new(
1100            deterministic::Config::new()
1101                .with_seed(seed)
1102                .with_timeout(Some(Duration::from_secs(60))),
1103        );
1104        runner.start(|mut context| async move {
1105            let Fixture {
1106                participants,
1107                schemes,
1108                ..
1109            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1110            let mut oracle = setup_network_with_participants(
1111                context.child("network"),
1112                NZUsize!(1),
1113                participants.clone(),
1114            )
1115            .await;
1116
1117            let me = participants[0].clone();
1118
1119            let setup = StandardHarness::setup_validator(
1120                context.child("validator").with_attribute("index", 0),
1121                &mut oracle,
1122                me.clone(),
1123                ConstantProvider::new(schemes[0].clone()),
1124            )
1125            .await;
1126            let marshal = setup.mailbox;
1127            let buffer = setup.extra;
1128            let actor_handle = setup.actor_handle;
1129
1130            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1131            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1132            let mut inline = Inline::new(
1133                context.child("inline"),
1134                mock_app,
1135                marshal.clone(),
1136                FixedEpocher::new(BLOCKS_PER_EPOCH),
1137            );
1138
1139            let parent = make_raw_block(genesis.digest(), Height::new(1), 100);
1140            let parent_digest = parent.digest();
1141
1142            let child_round = Round::new(Epoch::zero(), View::new(2));
1143            let child_ctx = Ctx {
1144                round: child_round,
1145                leader: me.clone(),
1146                parent: (View::new(1), parent_digest),
1147            };
1148            let child = B::new::<Sha256>(child_ctx.clone(), parent_digest, Height::new(2), 200);
1149            let child_digest = child.digest();
1150
1151            assert!(
1152                buffer
1153                    .broadcast(commonware_p2p::Recipients::Some(vec![]), parent.clone())
1154                    .accepted(),
1155                "buffer broadcast for parent should be accepted"
1156            );
1157            assert!(
1158                buffer
1159                    .broadcast(commonware_p2p::Recipients::Some(vec![]), child.clone())
1160                    .accepted(),
1161                "buffer broadcast for child should be accepted"
1162            );
1163
1164            let verify_rx = inline.verify(child_ctx, child_digest).await;
1165            let certify_result = inline
1166                .certify(child_round, child_digest)
1167                .await
1168                .await
1169                .expect("certify result missing");
1170            assert!(certify_result, "certify should succeed");
1171
1172            actor_handle.abort();
1173            drop(verify_rx);
1174            drop(inline);
1175            drop(marshal);
1176            drop(buffer);
1177
1178            let setup2 = StandardHarness::setup_validator(
1179                context
1180                    .child("validator_restart")
1181                    .with_attribute("index", 0),
1182                &mut oracle,
1183                me.clone(),
1184                ConstantProvider::new(schemes[0].clone()),
1185            )
1186            .await;
1187            let marshal2 = setup2.mailbox;
1188
1189            let post_restart = marshal2.get_block(&child_digest).await;
1190            assert!(
1191                post_restart.is_some(),
1192                "certify resolved true so block must be durably persisted (seed={seed})"
1193            );
1194        });
1195    }
1196
1197    /// Regression: in inline mode `propose` registers a certification gate for the
1198    /// built block that `certify` awaits. After the leader certifies its own proposal,
1199    /// the block must be durably recoverable. This is the >=f+1 guarantee: the leader
1200    /// certifies its own block through marshal so it awaits durability before the
1201    /// finalize vote.
1202    #[test_traced("WARN")]
1203    fn test_inline_propose_then_certify_persists_block() {
1204        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1205        runner.start(|mut context| async move {
1206            let Fixture {
1207                participants,
1208                schemes,
1209                ..
1210            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1211            let mut oracle = setup_network_with_participants(
1212                context.child("network"),
1213                NZUsize!(1),
1214                participants.clone(),
1215            )
1216            .await;
1217
1218            let me = participants[0].clone();
1219            let setup = StandardHarness::setup_validator(
1220                context.child("validator").with_attribute("index", 0),
1221                &mut oracle,
1222                me.clone(),
1223                ConstantProvider::new(schemes[0].clone()),
1224            )
1225            .await;
1226            let marshal = setup.mailbox;
1227            let actor_handle = setup.actor_handle;
1228
1229            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1230
1231            // Seed the parent at its round so `propose` can fetch it locally.
1232            let parent_round = Round::new(Epoch::zero(), View::new(1));
1233            let parent_ctx = Ctx {
1234                round: parent_round,
1235                leader: default_leader(),
1236                parent: (View::zero(), genesis.digest()),
1237            };
1238            let parent = B::new::<Sha256>(parent_ctx, genesis.digest(), Height::new(1), 100);
1239            let parent_digest = parent.digest();
1240            assert!(marshal.verified(parent_round, parent).await);
1241
1242            // The leader builds the child via `app.propose`.
1243            let round = Round::new(Epoch::zero(), View::new(2));
1244            let ctx = Ctx {
1245                round,
1246                leader: me.clone(),
1247                parent: (View::new(1), parent_digest),
1248            };
1249            let child = B::new::<Sha256>(ctx.clone(), parent_digest, Height::new(2), 200);
1250            let child_digest = child.digest();
1251            let mock_app: MockVerifyingApp<B, S> =
1252                MockVerifyingApp::new().with_propose_result(child);
1253            let mut inline = Inline::new(
1254                context.child("inline"),
1255                mock_app,
1256                marshal.clone(),
1257                FixedEpocher::new(BLOCKS_PER_EPOCH),
1258            );
1259
1260            let digest = inline
1261                .propose(ctx)
1262                .await
1263                .await
1264                .expect("propose must return a digest");
1265            assert_eq!(
1266                digest, child_digest,
1267                "propose must return the built block's digest"
1268            );
1269
1270            // The leader certifies its own proposal, which awaits the deferred sync handle.
1271            assert!(
1272                inline
1273                    .certify(round, child_digest)
1274                    .await
1275                    .await
1276                    .expect("certify result missing"),
1277                "certify must succeed for the leader's own proposal"
1278            );
1279
1280            // After certify, the block must be durable across an unclean restart.
1281            actor_handle.abort();
1282            drop(inline);
1283            drop(marshal);
1284
1285            let setup2 = StandardHarness::setup_validator(
1286                context
1287                    .child("validator_restart")
1288                    .with_attribute("index", 0),
1289                &mut oracle,
1290                me,
1291                ConstantProvider::new(schemes[0].clone()),
1292            )
1293            .await;
1294            let marshal2 = setup2.mailbox;
1295
1296            assert!(
1297                marshal2.get_block(&child_digest).await.is_some(),
1298                "certify resolved true for the leader's own proposal so the block must be durable"
1299            );
1300        });
1301    }
1302
1303    /// Dropping the verify receiver before the block is available closes the
1304    /// synchronously-registered certification gate. `certify` must recover through
1305    /// the fetch/certified path instead of returning the closed gate to consensus.
1306    #[test_traced("WARN")]
1307    fn test_inline_certify_recovers_after_verify_receiver_drop() {
1308        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1309        runner.start(|mut context| async move {
1310            let Fixture {
1311                participants,
1312                schemes,
1313                ..
1314            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1315            let mut oracle = setup_network_with_participants(
1316                context.child("network"),
1317                NZUsize!(1),
1318                participants.clone(),
1319            )
1320            .await;
1321
1322            let me = participants[0].clone();
1323            let setup = StandardHarness::setup_validator(
1324                context.child("validator").with_attribute("index", 0),
1325                &mut oracle,
1326                me.clone(),
1327                ConstantProvider::new(schemes[0].clone()),
1328            )
1329            .await;
1330            let marshal = setup.mailbox;
1331
1332            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1333            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1334            let mut inline = Inline::new(
1335                context.child("inline"),
1336                mock_app,
1337                marshal.clone(),
1338                FixedEpocher::new(BLOCKS_PER_EPOCH),
1339            );
1340
1341            let round = Round::new(Epoch::zero(), View::new(1));
1342            let block_context = Ctx {
1343                round,
1344                leader: me,
1345                parent: (View::zero(), genesis.digest()),
1346            };
1347            let block =
1348                B::new::<Sha256>(block_context.clone(), genesis.digest(), Height::new(1), 100);
1349            let digest = block.digest();
1350
1351            let verify_rx = inline.verify(block_context, digest).await;
1352            drop(verify_rx);
1353
1354            // Give the verify task a chance to observe the dropped receiver while its
1355            // block subscription is still pending.
1356            context.sleep(Duration::from_millis(10)).await;
1357
1358            assert!(marshal.verified(round, block).await);
1359            let certify_rx = inline.certify(round, digest).await;
1360            select! {
1361                result = certify_rx => {
1362                    assert!(
1363                        result.expect("certify result missing"),
1364                        "certify should recover after verify receiver drop"
1365                    );
1366                },
1367                _ = context.sleep(Duration::from_secs(5)) => {
1368                    panic!("certify should recover promptly after verify drop");
1369                },
1370            }
1371        });
1372    }
1373
1374    /// The store request runs concurrently with `app.verify`, not after the
1375    /// notarize vote: while gated application verification is still blocked, the
1376    /// block has already reached marshal and is locally queryable even though the
1377    /// sync handle may still be pending. Releasing verification then lets the
1378    /// notarize vote resolve and certification await the registered certification
1379    /// gate. Separate restart tests cover durable recovery after certification.
1380    #[test_traced("WARN")]
1381    fn test_inline_store_overlaps_app_verify() {
1382        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1383        runner.start(|mut context| async move {
1384            let Fixture {
1385                participants,
1386                schemes,
1387                ..
1388            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1389            let mut oracle = setup_network_with_participants(
1390                context.child("network"),
1391                NZUsize!(1),
1392                participants.clone(),
1393            )
1394            .await;
1395
1396            let me = participants[0].clone();
1397
1398            let setup = StandardHarness::setup_validator(
1399                context.child("validator").with_attribute("index", 0),
1400                &mut oracle,
1401                me.clone(),
1402                ConstantProvider::new(schemes[0].clone()),
1403            )
1404            .await;
1405            let marshal = setup.mailbox;
1406            let buffer = setup.extra;
1407
1408            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1409            let (mock_app, verify_started, release_verify): (GatedVerifyingApp<B, S>, _, _) =
1410                GatedVerifyingApp::new();
1411            let mut inline = Inline::new(
1412                context.child("inline"),
1413                mock_app,
1414                marshal.clone(),
1415                FixedEpocher::new(BLOCKS_PER_EPOCH),
1416            );
1417
1418            let parent = make_raw_block(genesis.digest(), Height::new(1), 100);
1419            let parent_digest = parent.digest();
1420
1421            let child_round = Round::new(Epoch::zero(), View::new(2));
1422            let child_ctx = Ctx {
1423                round: child_round,
1424                leader: me.clone(),
1425                parent: (View::new(1), parent_digest),
1426            };
1427            let child = B::new::<Sha256>(child_ctx.clone(), parent_digest, Height::new(2), 200);
1428            let child_digest = child.digest();
1429
1430            assert!(
1431                buffer
1432                    .broadcast(commonware_p2p::Recipients::Some(vec![]), parent)
1433                    .accepted(),
1434                "buffer broadcast for parent should be accepted"
1435            );
1436            assert!(
1437                buffer
1438                    .broadcast(commonware_p2p::Recipients::Some(vec![]), child)
1439                    .accepted(),
1440                "buffer broadcast for child should be accepted"
1441            );
1442
1443            let verify_rx = inline.verify(child_ctx, child_digest).await;
1444
1445            // Application verification is now blocked. The store request runs concurrently
1446            // with it, so the block is locally queryable even though the notarize vote has
1447            // not been cast and the sync handle may still be pending.
1448            verify_started
1449                .await
1450                .expect("verify should reach the gated application");
1451            assert!(
1452                marshal.get_block(&child_digest).await.is_some(),
1453                "the store request runs concurrently with app.verify, so the block is locally queryable while verification is still gated"
1454            );
1455
1456            // Releasing verification resolves the notarize vote and lets certification
1457            // succeed (valid and durable).
1458            release_verify.send_lossy(());
1459            assert!(
1460                verify_rx.await.expect("verify result missing"),
1461                "inline verify should pass once verification is released"
1462            );
1463            let certify_rx = inline.certify(child_round, child_digest).await;
1464            select! {
1465                result = certify_rx => {
1466                    assert!(
1467                        result.expect("certify result missing"),
1468                        "certify should succeed once verification passes"
1469                    );
1470                },
1471                _ = context.sleep(Duration::from_secs(5)) => {
1472                    panic!("certify should resolve after verification is released");
1473                },
1474            }
1475        });
1476    }
1477
1478    /// Regression: if marshal persisted a verified block for a round before
1479    /// a crash (via a prior `propose` call) but the simplex notarize artifact
1480    /// never reached the journal, the restarted leader must skip proposing
1481    /// for that round. The cached block was built against a parent context
1482    /// that replay may have changed, so reusing it can broadcast a proposal
1483    /// whose payload no longer matches the recovered header. Building a
1484    /// fresh block would also be unsafe because the pre-crash digest may
1485    /// already have been broadcast, so a second proposal for the round would
1486    /// equivocate. Dropping the receiver lets the voter nullify the view via
1487    /// `MissingProposal`.
1488    #[test_traced("WARN")]
1489    fn test_propose_skips_when_verified_block_exists_on_restart() {
1490        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1491        runner.start(|mut context| async move {
1492            let Fixture {
1493                participants,
1494                schemes,
1495                ..
1496            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1497            let mut oracle = setup_network_with_participants(
1498                context.child("network"),
1499                NZUsize!(1),
1500                participants.clone(),
1501            )
1502            .await;
1503
1504            let me = participants[0].clone();
1505            let round = Round::new(Epoch::zero(), View::new(1));
1506            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1507            let ctx = Ctx {
1508                round,
1509                leader: me.clone(),
1510                parent: (View::zero(), genesis.digest()),
1511            };
1512
1513            // Pre-crash: seed `verified_blocks[V=1]` through the live mailbox,
1514            // mirroring an aborted pre-crash `Inline::propose` that persisted
1515            // its verified block before the voter could journal a notarize.
1516            let pre_setup = StandardHarness::setup_validator(
1517                context.child("validator").with_attribute("index", 0),
1518                &mut oracle,
1519                me.clone(),
1520                ConstantProvider::new(schemes[0].clone()),
1521            )
1522            .await;
1523            let pre_marshal = pre_setup.mailbox;
1524            let pre_actor = pre_setup.actor_handle;
1525            let pre_extra = pre_setup.extra;
1526            let pre_application = pre_setup.application;
1527
1528            let stale_block = B::new::<Sha256>(ctx.clone(), genesis.digest(), Height::new(1), 100);
1529            assert!(pre_marshal.verified(round, stale_block).await);
1530
1531            // Simulate a crash: abort the actor and drop every handle so the
1532            // storage partition is fully released before reopening.
1533            pre_actor.abort();
1534            drop(pre_marshal);
1535            drop(pre_extra);
1536            drop(pre_application);
1537
1538            // Post-crash: reopen the same partition. The verified block must
1539            // be recovered from storage during archive restore so that
1540            // `Message::GetVerified` on the new mailbox observes it.
1541            let post_setup = StandardHarness::setup_validator(
1542                context
1543                    .child("validator_restart")
1544                    .with_attribute("index", 0),
1545                &mut oracle,
1546                me.clone(),
1547                ConstantProvider::new(schemes[0].clone()),
1548            )
1549            .await;
1550            let post_marshal = post_setup.mailbox;
1551
1552            let fresh_block = B::new::<Sha256>(ctx.clone(), genesis.digest(), Height::new(1), 200);
1553            let mock_app: MockVerifyingApp<B, S> =
1554                MockVerifyingApp::new().with_propose_result(fresh_block);
1555            let mut inline = Inline::new(
1556                context.child("inline"),
1557                mock_app,
1558                post_marshal.clone(),
1559                FixedEpocher::new(BLOCKS_PER_EPOCH),
1560            );
1561
1562            let digest_rx = inline.propose(ctx).await;
1563            assert!(
1564                digest_rx.await.is_err(),
1565                "propose must drop the receiver so the voter nullifies the round via timeout"
1566            );
1567        });
1568    }
1569
1570    /// Inline analog of the deferred equivocation test. A proposal that names
1571    /// the certified view-1 parent for a block built on the view-2 block
1572    /// fails structural parent validation and is rejected for voting. The
1573    /// candidate is nevertheless stored durably, and certification of the
1574    /// honest notarization for the same `(round, digest)` must use that
1575    /// durability result rather than the context-scoped verification verdict.
1576    #[test_traced("WARN")]
1577    fn test_certify_not_poisoned_by_equivocating_parent_verify() {
1578        let runner = deterministic::Runner::timed(Duration::from_secs(30));
1579        runner.start(|mut context| async move {
1580            let Fixture {
1581                participants,
1582                schemes,
1583                ..
1584            } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1585            let mut oracle = setup_network_with_participants(
1586                context.child("network"),
1587                NZUsize!(1),
1588                participants.clone(),
1589            )
1590            .await;
1591
1592            let me = participants[0].clone();
1593            let setup = StandardHarness::setup_validator(
1594                context.child("validator").with_attribute("index", 0),
1595                &mut oracle,
1596                me,
1597                ConstantProvider::new(schemes[0].clone()),
1598            )
1599            .await;
1600            let marshal = setup.mailbox;
1601            let buffer = setup.extra;
1602
1603            let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1604            let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1605            let mut inline = Inline::new(
1606                context.child("inline"),
1607                mock_app,
1608                marshal.clone(),
1609                FixedEpocher::new(BLOCKS_PER_EPOCH),
1610            );
1611
1612            let leader = participants[1].clone();
1613
1614            // The view-1 block: the parent this validator last certified.
1615            let certified_round = Round::new(Epoch::zero(), View::new(1));
1616            let certified_ctx = Ctx {
1617                round: certified_round,
1618                leader: default_leader(),
1619                parent: (View::zero(), genesis.digest()),
1620            };
1621            let certified = B::new::<Sha256>(certified_ctx, genesis.digest(), Height::new(1), 100);
1622            let certified_digest = certified.digest();
1623            assert!(marshal.verified(certified_round, certified).await);
1624
1625            // The view-2 block: notarized by the network but only nullified
1626            // here, so this validator never certified it.
1627            let notarized_round = Round::new(Epoch::zero(), View::new(2));
1628            let notarized_ctx = Ctx {
1629                round: notarized_round,
1630                leader: leader.clone(),
1631                parent: (View::new(1), certified_digest),
1632            };
1633            let notarized = B::new::<Sha256>(notarized_ctx, certified_digest, Height::new(2), 200);
1634            let notarized_digest = notarized.digest();
1635            assert!(marshal.verified(notarized_round, notarized).await);
1636
1637            // The view-3 block builds on the view-2 block. The leader's
1638            // broadcast delivered it into the local buffer without a local
1639            // verification.
1640            let round = Round::new(Epoch::zero(), View::new(3));
1641            let block_ctx = Ctx {
1642                round,
1643                leader: leader.clone(),
1644                parent: (View::new(2), notarized_digest),
1645            };
1646            let block = B::new::<Sha256>(block_ctx, notarized_digest, Height::new(3), 300);
1647            let digest = block.digest();
1648            assert!(
1649                buffer
1650                    .broadcast(commonware_p2p::Recipients::Some(vec![]), block)
1651                    .accepted(),
1652                "buffer broadcast for the candidate should be accepted"
1653            );
1654            context.sleep(Duration::from_millis(10)).await;
1655
1656            // The equivocating proposal names the certified view-1 block as
1657            // parent, which this validator's parent selection accepts
1658            // (view 1 certified, view 2 nullified). Refusing to notarize it
1659            // is correct.
1660            let equivocating_ctx = Ctx {
1661                round,
1662                leader,
1663                parent: (View::new(1), certified_digest),
1664            };
1665            let verify_rx = inline.verify(equivocating_ctx, digest).await;
1666            assert!(
1667                !verify_rx.await.expect("verify result missing"),
1668                "the equivocating proposal must not be notarized"
1669            );
1670
1671            // The honest notarization for the same `(round, digest)` arrives.
1672            let certify_rx = inline.certify(round, digest).await;
1673            select! {
1674                result = certify_rx => {
1675                    assert!(
1676                        result.expect("certify result missing"),
1677                        "certify of the notarized digest must not adopt the verdict computed under the equivocating context"
1678                    );
1679                },
1680                _ = context.sleep(Duration::from_secs(5)) => {
1681                    panic!("certify should resolve promptly");
1682                },
1683            }
1684        });
1685    }
1686}