1use crate::{
46 Application, Automaton, Block, CertifiableAutomaton, Epochable, Relay, Reporter,
47 marshal::{
48 Update,
49 application::gates::{GateOutcome, Gates},
50 core::{CommitmentFallback, DigestFallback, Mailbox},
51 standard::{
52 Standard, relay,
53 validation::{
54 Decision, ParentCheck, await_and_validate_parent, precheck_epoch_and_reproposal,
55 run_app_verify,
56 },
57 },
58 },
59 simplex::{Plan, types::Context},
60 types::{Epocher, Round},
61};
62use commonware_actor::Feedback;
63use commonware_cryptography::certificate::Scheme;
64use commonware_macros::select;
65use commonware_runtime::{
66 Clock, Metrics, Spawner,
67 telemetry::{
68 metrics::{
69 MetricsExt as _,
70 histogram::{Buckets, Timed},
71 },
72 traces::TracedExt as _,
73 },
74};
75use commonware_utils::{
76 channel::{fallible::OneshotExt, oneshot},
77 sync::TracedAsyncMutex,
78};
79use rand_core::Rng;
80use std::sync::Arc;
81use tracing::{Instrument as _, debug, info_span};
82
83async fn await_block_subscription<T, D>(
85 tx: &mut oneshot::Sender<bool>,
86 block_rx: oneshot::Receiver<T>,
87 digest: &D,
88 stage: &'static str,
89) -> Option<T>
90where
91 D: std::fmt::Debug,
92{
93 select! {
94 _ = tx.closed() => {
95 debug!(
96 stage,
97 reason = "consensus dropped receiver",
98 "skipping block wait"
99 );
100 None
101 },
102 result = block_rx => {
103 if result.is_err() {
104 debug!(
105 stage,
106 ?digest,
107 reason = "failed to fetch block",
108 "skipping block wait"
109 );
110 }
111 result.ok()
112 },
113 }
114}
115
116pub struct Inline<E, S, A, B, ES>
134where
135 E: Rng + Spawner + Metrics + Clock,
136 S: Scheme,
137 A: Application<E>,
138 B: Block + Clone,
139 ES: Epocher,
140{
141 context: Arc<TracedAsyncMutex<E>>,
142 application: A,
143 marshal: Mailbox<S, Standard<B>>,
144 epocher: ES,
145 gates: Gates<B::Digest, B>,
146
147 build_duration: Timed,
148 proposal_parent_fetch_duration: Timed,
149 ancestor_fetch_duration: Timed,
150}
151
152impl<E, S, A, B, ES> Clone for Inline<E, S, A, B, ES>
153where
154 E: Rng + Spawner + Metrics + Clock,
155 S: Scheme,
156 A: Application<E>,
157 B: Block + Clone,
158 ES: Epocher,
159{
160 fn clone(&self) -> Self {
161 Self {
162 context: self.context.clone(),
163 application: self.application.clone(),
164 marshal: self.marshal.clone(),
165 epocher: self.epocher.clone(),
166 gates: self.gates.clone(),
167 build_duration: self.build_duration.clone(),
168 proposal_parent_fetch_duration: self.proposal_parent_fetch_duration.clone(),
169 ancestor_fetch_duration: self.ancestor_fetch_duration.clone(),
170 }
171 }
172}
173
174impl<E, S, A, B, ES> Inline<E, S, A, B, ES>
175where
176 E: Rng + Spawner + Metrics + Clock,
177 S: Scheme,
178 A: Application<
179 E,
180 Block = B,
181 SigningScheme = S,
182 Context = Context<B::Digest, S::PublicKey>,
183 Input = (),
184 >,
185 B: Block + Clone,
186 ES: Epocher,
187{
188 pub fn new(context: E, application: A, marshal: Mailbox<S, Standard<B>>, epocher: ES) -> Self {
192 let build_histogram = context.histogram(
193 "build_duration",
194 "Histogram of time taken for the application to build a new block, in seconds",
195 Buckets::LOCAL,
196 );
197 let build_duration = Timed::new(build_histogram);
198 let parent_fetch_histogram = context.histogram(
199 "parent_fetch_duration",
200 "Histogram of time taken to fetch a parent block in propose, in seconds",
201 Buckets::LOCAL,
202 );
203 let proposal_parent_fetch_duration = Timed::new(parent_fetch_histogram);
204 let ancestor_fetch_histogram = context.histogram(
205 "ancestor_fetch_duration",
206 "Histogram of time taken to fetch a block via the ancestry stream, in seconds",
207 Buckets::LOCAL,
208 );
209 let ancestor_fetch_duration = Timed::new(ancestor_fetch_histogram);
210
211 Self {
212 context: Arc::new(TracedAsyncMutex::new("marshal.context", context)),
213 application,
214 marshal,
215 epocher,
216 gates: Gates::new(),
217 build_duration,
218 proposal_parent_fetch_duration,
219 ancestor_fetch_duration,
220 }
221 }
222}
223
224impl<E, S, A, B, ES> Automaton for Inline<E, S, A, B, ES>
225where
226 E: Rng + Spawner + Metrics + Clock,
227 S: Scheme,
228 A: Application<
229 E,
230 Block = B,
231 SigningScheme = S,
232 Context = Context<B::Digest, S::PublicKey>,
233 Input = (),
234 >,
235 B: Block + Clone,
236 ES: Epocher,
237{
238 type Digest = B::Digest;
239 type Context = Context<Self::Digest, S::PublicKey>;
240
241 #[allow(clippy::async_yields_async)]
250 #[tracing::instrument(name = "marshal.inline.propose", level = "info", skip_all, fields(round = %consensus_context.round))]
251 async fn propose(
252 &mut self,
253 consensus_context: Context<Self::Digest, S::PublicKey>,
254 ) -> oneshot::Receiver<Self::Digest> {
255 let marshal = self.marshal.clone();
256 let mut application = self.application.clone();
257 let epocher = self.epocher.clone();
258 let gates = self.gates.clone();
259 let build_duration = self.build_duration.clone();
260 let proposal_parent_fetch_duration = self.proposal_parent_fetch_duration.clone();
261 let ancestor_fetch_duration = self.ancestor_fetch_duration.clone();
262
263 let (mut tx, rx) = oneshot::channel();
264 let context = self
265 .context
266 .lock()
267 .await
268 .child("propose")
269 .with_attribute("round", consensus_context.round);
270 let span = info_span!(
271 "marshal.inline.propose.task",
272 round = %consensus_context.round
273 );
274 context.spawn(move |runtime_context| {
275 async move {
276 if marshal
288 .get_verified(consensus_context.round)
289 .await
290 .is_some()
291 {
292 debug!(
293 round = ?consensus_context.round,
294 "skipping proposal: verified block already exists for round on restart"
295 );
296 return;
297 }
298
299 let (parent_view, parent_commitment) = consensus_context.parent;
308 let parent_request = marshal.subscribe_by_commitment(
309 parent_commitment,
310 CommitmentFallback::FetchByRound {
311 round: Round::new(consensus_context.epoch(), parent_view),
312 },
313 );
314
315 let parent_timer = proposal_parent_fetch_duration.timer(&runtime_context);
316 let parent = select! {
317 _ = tx.closed() => {
318 debug!(reason = "consensus dropped receiver", "skipping proposal");
319 return;
320 },
321 result = parent_request => match result {
322 Ok(parent) => parent,
323 Err(_) => {
324 debug!(
325 ?parent_commitment,
326 reason = "failed to fetch parent block",
327 "skipping proposal"
328 );
329 return;
330 }
331 },
332 };
333 parent_timer.observe(&runtime_context);
334
335 let last_in_epoch = epocher
337 .last(consensus_context.epoch())
338 .expect("current epoch should exist");
339 if parent.height() == last_in_epoch {
340 let digest = parent.digest();
341 gates
342 .stage(
343 consensus_context.round,
344 digest,
345 parent,
346 tx,
347 "re-proposed boundary block",
348 )
349 .await;
350 return;
351 }
352
353 let ancestor_stream = marshal.ancestor_stream(
354 Arc::new(runtime_context.child("ancestor_stream")),
355 [parent],
356 ancestor_fetch_duration,
357 );
358 let build_request = application
359 .propose(
360 (
361 runtime_context.child("app_propose"),
362 consensus_context.clone(),
363 ),
364 ancestor_stream,
365 (),
366 )
367 .instrument(info_span!(
368 "marshal.inline.application.propose",
369 round = %consensus_context.round,
370 parent_view = parent_view.traced(),
371 parent = %parent_commitment
372 ));
373
374 let build_timer = build_duration.timer(&runtime_context);
375 let built_block = select! {
376 _ = tx.closed() => {
377 debug!(reason = "consensus dropped receiver", "skipping proposal");
378 return;
379 },
380 result = build_request => match result {
381 Some(block) => block,
382 None => {
383 debug!(
384 ?parent_commitment,
385 reason = "block building failed",
386 "skipping proposal"
387 );
388 return;
389 }
390 },
391 };
392 build_timer.observe(&runtime_context);
393
394 let digest = built_block.digest();
395 gates
396 .stage(
397 consensus_context.round,
398 digest,
399 Arc::new(built_block),
400 tx,
401 "proposed block",
402 )
403 .await;
404 }
405 .instrument(span)
406 });
407 rx
408 }
409
410 #[allow(clippy::async_yields_async)]
423 #[tracing::instrument(name = "marshal.inline.verify", level = "info", skip_all, fields(round = %context.round, digest = %digest))]
424 async fn verify(
425 &mut self,
426 context: Context<Self::Digest, S::PublicKey>,
427 digest: Self::Digest,
428 ) -> oneshot::Receiver<bool> {
429 let round = context.round;
430
431 let block_request = self
437 .marshal
438 .subscribe_by_digest(digest, DigestFallback::Wait);
439 let (durable_tx, durable_rx) = oneshot::channel();
440 self.gates.insert(round, digest, durable_rx);
441
442 let marshal = self.marshal.clone();
443 let mut application = self.application.clone();
444 let epocher = self.epocher.clone();
445 let ancestor_fetch_duration = self.ancestor_fetch_duration.clone();
446
447 let (mut tx, rx) = oneshot::channel();
448 let runtime_context = self
449 .context
450 .lock()
451 .await
452 .child("inline_verify")
453 .with_attribute("round", round);
454 let span = info_span!(
455 "marshal.inline.verify.task",
456 round = %round,
457 digest = %digest
458 );
459 runtime_context.spawn(move |runtime_context| {
460 async move {
461 let parent_request = (digest != context.parent.1).then(|| {
468 let (parent_view, parent_commitment) = context.parent;
469 marshal.subscribe_by_commitment(
470 parent_commitment,
471 CommitmentFallback::FetchByRound {
472 round: Round::new(context.epoch(), parent_view),
473 },
474 )
475 });
476
477 let Some(block) =
478 await_block_subscription(&mut tx, block_request, &digest, "verification").await
479 else {
480 return;
481 };
482
483 let Some(decision) =
492 precheck_epoch_and_reproposal(&epocher, &marshal, &context, digest, block)
493 .await
494 else {
495 return;
496 };
497 let block = match decision {
498 Decision::Complete(valid) => {
499 tx.send_lossy(valid);
507 durable_tx.send_lossy(if valid {
508 GateOutcome::Ready(true)
509 } else {
510 GateOutcome::Recover
511 });
512 return;
513 }
514 Decision::Continue(block) => block,
515 };
516
517 let parent_request =
520 parent_request.expect("non-reproposal has a parent subscription");
521
522 let store = async {
536 if marshal.verified(round, Arc::clone(&block)).await {
537 durable_tx.send_lossy(GateOutcome::Ready(true));
538 }
539 };
540 let verify_then_vote = async {
541 let parent = match await_and_validate_parent(
544 context.parent.1,
545 block.as_ref(),
546 parent_request,
547 &mut tx,
548 )
549 .await
550 {
551 Some(ParentCheck::Valid(parent)) => parent,
552 Some(ParentCheck::Invalid) => {
553 tx.send_lossy(false);
554 return Some(false);
555 }
556 None => return None,
557 };
558 let valid = run_app_verify(
559 runtime_context,
560 context,
561 Arc::clone(&block),
562 parent,
563 &mut application,
564 &marshal,
565 &mut tx,
566 ancestor_fetch_duration,
567 )
568 .await;
569 if let Some(valid) = valid {
570 tx.send_lossy(valid);
571 }
572 valid
573 };
574 futures::join!(verify_then_vote, store);
575 }
576 .instrument(span)
577 });
578 rx
579 }
580}
581
582impl<E, S, A, B, ES> CertifiableAutomaton for Inline<E, S, A, B, ES>
586where
587 E: Rng + Spawner + Metrics + Clock,
588 S: Scheme,
589 A: Application<
590 E,
591 Block = B,
592 SigningScheme = S,
593 Context = Context<B::Digest, S::PublicKey>,
594 Input = (),
595 >,
596 B: Block + Clone,
597 ES: Epocher,
598{
599 #[allow(clippy::async_yields_async)]
600 #[tracing::instrument(name = "marshal.inline.certify", level = "info", skip_all, fields(round = %round, digest = %digest))]
601 async fn certify(&mut self, round: Round, digest: Self::Digest) -> oneshot::Receiver<bool> {
602 self.gates.flush_unrelayed(&self.marshal, round, digest);
603
604 let task = self.gates.take(round, digest);
609
610 if task.is_some() {
615 self.marshal.hint_notarized(round, digest);
616 }
617 let marshal = self.marshal.clone();
618 let (mut tx, rx) = oneshot::channel();
619 let context = self
620 .context
621 .lock()
622 .await
623 .child("inline_certify")
624 .with_attribute("round", round);
625 context.spawn(move |_| {
626 async move {
627 if let Some(task) = task {
630 let result = select! {
631 _ = tx.closed() => {
632 debug!(
633 reason = "consensus dropped receiver",
634 "skipping certification"
635 );
636 return;
637 },
638 result = task => result,
639 };
640 match result {
641 Ok(GateOutcome::Ready(verdict)) => {
642 tx.send_lossy(verdict);
643 return;
644 }
645 Ok(GateOutcome::Recover) | Err(_) => {}
646 }
647 }
648
649 let block_rx =
655 marshal.subscribe_by_digest(digest, DigestFallback::FetchByRound { round });
656 let Some(block) =
657 await_block_subscription(&mut tx, block_rx, &digest, "certification").await
658 else {
659 return;
660 };
661 if !marshal.certified(round, block).await {
662 return;
663 }
664 tx.send_lossy(true);
665 }
666 .instrument(info_span!(
667 "marshal.inline.certify.task",
668 round = %round,
669 digest = %digest
670 ))
671 });
672
673 rx
674 }
675}
676
677impl<E, S, A, B, ES> Relay for Inline<E, S, A, B, ES>
678where
679 E: Rng + Spawner + Metrics + Clock,
680 S: Scheme,
681 A: Application<E, Block = B, Context = Context<B::Digest, S::PublicKey>>,
682 B: Block + Clone,
683 ES: Epocher,
684{
685 type Digest = B::Digest;
686 type PublicKey = S::PublicKey;
687 type Plan = Plan<S::PublicKey>;
688
689 fn broadcast(&mut self, commitment: Self::Digest, plan: Plan<S::PublicKey>) -> Feedback {
690 relay::broadcast(&self.gates, &self.marshal, commitment, plan)
691 }
692}
693
694impl<E, S, A, B, ES> Reporter for Inline<E, S, A, B, ES>
695where
696 E: Rng + Spawner + Metrics + Clock,
697 S: Scheme,
698 A: Application<E, Block = B, Context = Context<B::Digest, S::PublicKey>>
699 + Reporter<Activity = Update<B>>,
700 B: Block + Clone,
701 ES: Epocher,
702{
703 type Activity = A::Activity;
704
705 fn report(&mut self, update: Self::Activity) -> Feedback {
707 if let Update::Tip(tip_round, _, _) = &update {
708 self.gates.retain_after(tip_round);
709 }
710 self.application.report(update)
711 }
712}
713
714#[cfg(test)]
715mod tests {
716 use super::Inline;
717 use crate::{
718 Application, Automaton, Block, CertifiableAutomaton, Relay,
719 marshal::mocks::{
720 harness::{
721 B, BLOCKS_PER_EPOCH, Ctx, NAMESPACE, NUM_VALIDATORS, S, StandardHarness,
722 TestHarness, V, default_leader, make_raw_block, setup_network_with_participants,
723 },
724 verifying::{GatedVerifyingApp, MockVerifyingApp},
725 },
726 simplex::{scheme::bls12381_threshold::vrf as bls12381_threshold_vrf, types::Context},
727 types::{Epoch, FixedEpocher, Height, Round, View},
728 };
729 use commonware_broadcast::Broadcaster;
730 use commonware_cryptography::{
731 Digestible, Hasher as _,
732 certificate::{ConstantProvider, Scheme, mocks::Fixture},
733 sha256::Sha256,
734 };
735 use commonware_macros::{select, test_traced};
736 use commonware_runtime::{Clock, Metrics, Runner, Spawner, Supervisor as _, deterministic};
737 use commonware_utils::{NZUsize, channel::fallible::OneshotExt};
738 use rand::Rng;
739 use std::time::Duration;
740
741 #[allow(dead_code)]
743 fn assert_non_certifiable_block_supported<E, S, A, B, ES>()
744 where
745 E: Rng + Spawner + Metrics + Clock,
746 S: Scheme,
747 A: Application<
748 E,
749 Block = B,
750 SigningScheme = S,
751 Context = Context<B::Digest, S::PublicKey>,
752 Input = (),
753 >,
754 B: Block + Clone,
755 ES: crate::types::Epocher,
756 {
757 fn assert_automaton<T: Automaton>() {}
758 fn assert_certifiable<T: CertifiableAutomaton>() {}
759 fn assert_relay<T: Relay>() {}
760
761 assert_automaton::<Inline<E, S, A, B, ES>>();
762 assert_certifiable::<Inline<E, S, A, B, ES>>();
763 assert_relay::<Inline<E, S, A, B, ES>>();
764 }
765
766 #[test_traced("INFO")]
767 fn test_certify_returns_immediately_after_verify_fetches_block() {
768 let runner = deterministic::Runner::timed(Duration::from_secs(30));
769 runner.start(|mut context| async move {
770 let Fixture {
771 participants,
772 schemes,
773 ..
774 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
775 let mut oracle = setup_network_with_participants(
776 context.child("network"),
777 NZUsize!(1),
778 participants.clone(),
779 )
780 .await;
781
782 let me = participants[0].clone();
783 let setup = StandardHarness::setup_validator(
784 context.child("validator").with_attribute("index", 0),
785 &mut oracle,
786 me.clone(),
787 ConstantProvider::new(schemes[0].clone()),
788 )
789 .await;
790 let marshal = setup.mailbox;
791
792 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
793 let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
794 let mut inline = Inline::new(
795 context.child("inline"),
796 mock_app,
797 marshal.clone(),
798 FixedEpocher::new(BLOCKS_PER_EPOCH),
799 );
800
801 let parent_round = Round::new(Epoch::zero(), View::new(1));
803 let parent_ctx = Ctx {
804 round: parent_round,
805 leader: default_leader(),
806 parent: (View::zero(), genesis.digest()),
807 };
808 let parent = B::new::<Sha256>(parent_ctx, genesis.digest(), Height::new(1), 100);
809 let parent_digest = parent.digest();
810 assert!(marshal.verified(parent_round, parent).await);
811
812 let round = Round::new(Epoch::zero(), View::new(2));
813 let verify_context = Ctx {
814 round,
815 leader: me,
816 parent: (View::new(1), parent_digest),
817 };
818 let block =
819 B::new::<Sha256>(verify_context.clone(), parent_digest, Height::new(2), 200);
820 let digest = block.digest();
821 assert!(marshal.verified(round, block).await);
822
823 let verify_rx = inline.verify(verify_context, digest).await;
825 assert!(
826 verify_rx.await.unwrap(),
827 "verify should complete successfully before certify"
828 );
829
830 let certify_rx = inline.certify(round, digest).await;
832
833 select! {
834 result = certify_rx => {
835 assert!(
836 result.unwrap(),
837 "certify should return immediately once verify has fetched the block"
838 );
839 },
840 _ = context.sleep(Duration::from_secs(5)) => {
841 panic!("certify should not hang after local verify completed");
842 },
843 }
844 });
845 }
846
847 #[test_traced("INFO")]
848 fn test_certify_succeeds_without_verify_task() {
849 let runner = deterministic::Runner::timed(Duration::from_secs(30));
850 runner.start(|mut context| async move {
851 let Fixture {
852 participants,
853 schemes,
854 ..
855 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
856 let mut oracle = setup_network_with_participants(
857 context.child("network"),
858 NZUsize!(1),
859 participants.clone(),
860 )
861 .await;
862
863 let me = participants[0].clone();
864 let setup = StandardHarness::setup_validator(
865 context.child("validator").with_attribute("index", 0),
866 &mut oracle,
867 me.clone(),
868 ConstantProvider::new(schemes[0].clone()),
869 )
870 .await;
871 let marshal = setup.mailbox;
872
873 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
874 let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
875 let mut inline = Inline::new(
876 context.child("inline"),
877 mock_app,
878 marshal.clone(),
879 FixedEpocher::new(BLOCKS_PER_EPOCH),
880 );
881
882 let parent_round = Round::new(Epoch::zero(), View::new(1));
884 let parent_ctx = Ctx {
885 round: parent_round,
886 leader: default_leader(),
887 parent: (View::zero(), genesis.digest()),
888 };
889 let parent = B::new::<Sha256>(parent_ctx, genesis.digest(), Height::new(1), 100);
890 let parent_digest = parent.digest();
891 assert!(marshal.verified(parent_round, parent).await);
892
893 let round = Round::new(Epoch::zero(), View::new(2));
894 let verify_context = Ctx {
895 round,
896 leader: me,
897 parent: (View::new(1), parent_digest),
898 };
899 let block =
900 B::new::<Sha256>(verify_context.clone(), parent_digest, Height::new(2), 200);
901 let digest = block.digest();
902 assert!(marshal.verified(round, block).await);
903
904 let certify_rx = inline.certify(round, digest).await;
906
907 select! {
908 result = certify_rx => {
909 assert!(
910 result.unwrap(),
911 "certify should resolve once block availability is known"
912 );
913 },
914 _ = context.sleep(Duration::from_secs(5)) => {
915 panic!("certify should not hang when block is already available in marshal");
916 },
917 }
918 });
919 }
920
921 #[test_traced("INFO")]
922 fn test_certify_reproposal_uses_available_blocks_after_verify() {
923 let runner = deterministic::Runner::timed(Duration::from_secs(30));
924 runner.start(|mut context| async move {
925 let Fixture {
926 participants,
927 schemes,
928 ..
929 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
930 let mut oracle =
931 setup_network_with_participants(context.child("network"), NZUsize!(1), participants.clone())
932 .await;
933
934 let me = participants[0].clone();
935 let setup = StandardHarness::setup_validator(
936 context.child("validator").with_attribute("index", 0),
937 &mut oracle,
938 me.clone(),
939 ConstantProvider::new(schemes[0].clone()),
940 )
941 .await;
942 let marshal = setup.mailbox;
943 let marshal_actor_handle = setup.actor_handle;
944
945 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
946 let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
947 let mut inline = Inline::new(context.child("inline"),
948 mock_app,
949 marshal.clone(),
950 FixedEpocher::new(BLOCKS_PER_EPOCH),
951 );
952
953 let boundary_height = Height::new(BLOCKS_PER_EPOCH.get() - 1);
954 let boundary_round = Round::new(Epoch::zero(), View::new(boundary_height.get()));
955 let boundary_block = B::new::<Sha256>(
956 Ctx {
957 round: boundary_round,
958 leader: default_leader(),
959 parent: (View::zero(), genesis.digest()),
960 },
961 genesis.digest(),
962 boundary_height,
963 1900,
964 );
965 let boundary_digest = boundary_block.digest();
966 assert!(
967 marshal.verified(boundary_round, boundary_block).await
968 );
969
970 let reproposal_round = Round::new(Epoch::zero(), View::new(boundary_height.get() + 1));
971 let reproposal_context = Ctx {
972 round: reproposal_round,
973 leader: me,
974 parent: (View::new(boundary_height.get()), boundary_digest),
975 };
976
977 let verify_rx = inline.verify(reproposal_context, boundary_digest).await;
978 assert!(
979 verify_rx.await.unwrap(),
980 "verify should accept a valid boundary re-proposal"
981 );
982
983 marshal_actor_handle.abort();
984 drop(marshal);
985 context.sleep(Duration::from_millis(1)).await;
986
987 let certify_rx = inline.certify(reproposal_round, boundary_digest).await;
988 select! {
989 result = certify_rx => {
990 assert!(
991 result.unwrap(),
992 "certify should use the available_blocks fast path for verified re-proposals"
993 );
994 },
995 _ = context.sleep(Duration::from_secs(5)) => {
996 panic!("certify should not depend on marshal after verify cached a re-proposal");
997 },
998 }
999 });
1000 }
1001
1002 #[test_traced("WARN")]
1007 fn test_certify_recovers_from_invalid_reproposal_precheck() {
1008 let runner = deterministic::Runner::timed(Duration::from_secs(30));
1009 runner.start(|mut context| async move {
1010 let Fixture {
1011 participants,
1012 schemes,
1013 ..
1014 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1015 let mut oracle = setup_network_with_participants(
1016 context.child("network"),
1017 NZUsize!(1),
1018 participants.clone(),
1019 )
1020 .await;
1021
1022 let me = participants[0].clone();
1023 let setup = StandardHarness::setup_validator(
1024 context.child("validator").with_attribute("index", 0),
1025 &mut oracle,
1026 me.clone(),
1027 ConstantProvider::new(schemes[0].clone()),
1028 )
1029 .await;
1030 let marshal = setup.mailbox;
1031
1032 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1033 let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1034 let mut inline = Inline::new(
1035 context.child("inline"),
1036 mock_app,
1037 marshal.clone(),
1038 FixedEpocher::new(BLOCKS_PER_EPOCH),
1039 );
1040
1041 let block_round = Round::new(Epoch::zero(), View::new(2));
1043 let block = B::new::<Sha256>(
1044 Ctx {
1045 round: block_round,
1046 leader: default_leader(),
1047 parent: (View::zero(), genesis.digest()),
1048 },
1049 genesis.digest(),
1050 Height::new(2),
1051 200,
1052 );
1053 let digest = block.digest();
1054 assert!(marshal.verified(block_round, block).await);
1055
1056 let round = Round::new(Epoch::zero(), View::new(3));
1060 let reproposal_context = Ctx {
1061 round,
1062 leader: me,
1063 parent: (View::new(2), digest),
1064 };
1065 let verify_rx = inline.verify(reproposal_context, digest).await;
1066 assert!(
1067 !verify_rx.await.expect("verify result missing"),
1068 "a non-boundary re-proposal must be rejected"
1069 );
1070
1071 let certify_rx = inline.certify(round, digest).await;
1074 select! {
1075 result = certify_rx => {
1076 assert!(
1077 result.expect("certify result missing"),
1078 "certify must recover instead of adopting the precheck rejection"
1079 );
1080 },
1081 _ = context.sleep(Duration::from_secs(5)) => {
1082 panic!("certify should resolve promptly");
1083 },
1084 }
1085 });
1086 }
1087
1088 #[test_traced("WARN")]
1092 fn test_inline_certify_persists_block_before_resolving() {
1093 for seed in 0u64..16 {
1094 inline_certify_persists_block_before_resolving_at(seed);
1095 }
1096 }
1097
1098 fn inline_certify_persists_block_before_resolving_at(seed: u64) {
1099 let runner = deterministic::Runner::new(
1100 deterministic::Config::new()
1101 .with_seed(seed)
1102 .with_timeout(Some(Duration::from_secs(60))),
1103 );
1104 runner.start(|mut context| async move {
1105 let Fixture {
1106 participants,
1107 schemes,
1108 ..
1109 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1110 let mut oracle = setup_network_with_participants(
1111 context.child("network"),
1112 NZUsize!(1),
1113 participants.clone(),
1114 )
1115 .await;
1116
1117 let me = participants[0].clone();
1118
1119 let setup = StandardHarness::setup_validator(
1120 context.child("validator").with_attribute("index", 0),
1121 &mut oracle,
1122 me.clone(),
1123 ConstantProvider::new(schemes[0].clone()),
1124 )
1125 .await;
1126 let marshal = setup.mailbox;
1127 let buffer = setup.extra;
1128 let actor_handle = setup.actor_handle;
1129
1130 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1131 let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1132 let mut inline = Inline::new(
1133 context.child("inline"),
1134 mock_app,
1135 marshal.clone(),
1136 FixedEpocher::new(BLOCKS_PER_EPOCH),
1137 );
1138
1139 let parent = make_raw_block(genesis.digest(), Height::new(1), 100);
1140 let parent_digest = parent.digest();
1141
1142 let child_round = Round::new(Epoch::zero(), View::new(2));
1143 let child_ctx = Ctx {
1144 round: child_round,
1145 leader: me.clone(),
1146 parent: (View::new(1), parent_digest),
1147 };
1148 let child = B::new::<Sha256>(child_ctx.clone(), parent_digest, Height::new(2), 200);
1149 let child_digest = child.digest();
1150
1151 assert!(
1152 buffer
1153 .broadcast(commonware_p2p::Recipients::Some(vec![]), parent.clone())
1154 .accepted(),
1155 "buffer broadcast for parent should be accepted"
1156 );
1157 assert!(
1158 buffer
1159 .broadcast(commonware_p2p::Recipients::Some(vec![]), child.clone())
1160 .accepted(),
1161 "buffer broadcast for child should be accepted"
1162 );
1163
1164 let verify_rx = inline.verify(child_ctx, child_digest).await;
1165 let certify_result = inline
1166 .certify(child_round, child_digest)
1167 .await
1168 .await
1169 .expect("certify result missing");
1170 assert!(certify_result, "certify should succeed");
1171
1172 actor_handle.abort();
1173 drop(verify_rx);
1174 drop(inline);
1175 drop(marshal);
1176 drop(buffer);
1177
1178 let setup2 = StandardHarness::setup_validator(
1179 context
1180 .child("validator_restart")
1181 .with_attribute("index", 0),
1182 &mut oracle,
1183 me.clone(),
1184 ConstantProvider::new(schemes[0].clone()),
1185 )
1186 .await;
1187 let marshal2 = setup2.mailbox;
1188
1189 let post_restart = marshal2.get_block(&child_digest).await;
1190 assert!(
1191 post_restart.is_some(),
1192 "certify resolved true so block must be durably persisted (seed={seed})"
1193 );
1194 });
1195 }
1196
1197 #[test_traced("WARN")]
1203 fn test_inline_propose_then_certify_persists_block() {
1204 let runner = deterministic::Runner::timed(Duration::from_secs(30));
1205 runner.start(|mut context| async move {
1206 let Fixture {
1207 participants,
1208 schemes,
1209 ..
1210 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1211 let mut oracle = setup_network_with_participants(
1212 context.child("network"),
1213 NZUsize!(1),
1214 participants.clone(),
1215 )
1216 .await;
1217
1218 let me = participants[0].clone();
1219 let setup = StandardHarness::setup_validator(
1220 context.child("validator").with_attribute("index", 0),
1221 &mut oracle,
1222 me.clone(),
1223 ConstantProvider::new(schemes[0].clone()),
1224 )
1225 .await;
1226 let marshal = setup.mailbox;
1227 let actor_handle = setup.actor_handle;
1228
1229 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1230
1231 let parent_round = Round::new(Epoch::zero(), View::new(1));
1233 let parent_ctx = Ctx {
1234 round: parent_round,
1235 leader: default_leader(),
1236 parent: (View::zero(), genesis.digest()),
1237 };
1238 let parent = B::new::<Sha256>(parent_ctx, genesis.digest(), Height::new(1), 100);
1239 let parent_digest = parent.digest();
1240 assert!(marshal.verified(parent_round, parent).await);
1241
1242 let round = Round::new(Epoch::zero(), View::new(2));
1244 let ctx = Ctx {
1245 round,
1246 leader: me.clone(),
1247 parent: (View::new(1), parent_digest),
1248 };
1249 let child = B::new::<Sha256>(ctx.clone(), parent_digest, Height::new(2), 200);
1250 let child_digest = child.digest();
1251 let mock_app: MockVerifyingApp<B, S> =
1252 MockVerifyingApp::new().with_propose_result(child);
1253 let mut inline = Inline::new(
1254 context.child("inline"),
1255 mock_app,
1256 marshal.clone(),
1257 FixedEpocher::new(BLOCKS_PER_EPOCH),
1258 );
1259
1260 let digest = inline
1261 .propose(ctx)
1262 .await
1263 .await
1264 .expect("propose must return a digest");
1265 assert_eq!(
1266 digest, child_digest,
1267 "propose must return the built block's digest"
1268 );
1269
1270 assert!(
1272 inline
1273 .certify(round, child_digest)
1274 .await
1275 .await
1276 .expect("certify result missing"),
1277 "certify must succeed for the leader's own proposal"
1278 );
1279
1280 actor_handle.abort();
1282 drop(inline);
1283 drop(marshal);
1284
1285 let setup2 = StandardHarness::setup_validator(
1286 context
1287 .child("validator_restart")
1288 .with_attribute("index", 0),
1289 &mut oracle,
1290 me,
1291 ConstantProvider::new(schemes[0].clone()),
1292 )
1293 .await;
1294 let marshal2 = setup2.mailbox;
1295
1296 assert!(
1297 marshal2.get_block(&child_digest).await.is_some(),
1298 "certify resolved true for the leader's own proposal so the block must be durable"
1299 );
1300 });
1301 }
1302
1303 #[test_traced("WARN")]
1307 fn test_inline_certify_recovers_after_verify_receiver_drop() {
1308 let runner = deterministic::Runner::timed(Duration::from_secs(30));
1309 runner.start(|mut context| async move {
1310 let Fixture {
1311 participants,
1312 schemes,
1313 ..
1314 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1315 let mut oracle = setup_network_with_participants(
1316 context.child("network"),
1317 NZUsize!(1),
1318 participants.clone(),
1319 )
1320 .await;
1321
1322 let me = participants[0].clone();
1323 let setup = StandardHarness::setup_validator(
1324 context.child("validator").with_attribute("index", 0),
1325 &mut oracle,
1326 me.clone(),
1327 ConstantProvider::new(schemes[0].clone()),
1328 )
1329 .await;
1330 let marshal = setup.mailbox;
1331
1332 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1333 let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1334 let mut inline = Inline::new(
1335 context.child("inline"),
1336 mock_app,
1337 marshal.clone(),
1338 FixedEpocher::new(BLOCKS_PER_EPOCH),
1339 );
1340
1341 let round = Round::new(Epoch::zero(), View::new(1));
1342 let block_context = Ctx {
1343 round,
1344 leader: me,
1345 parent: (View::zero(), genesis.digest()),
1346 };
1347 let block =
1348 B::new::<Sha256>(block_context.clone(), genesis.digest(), Height::new(1), 100);
1349 let digest = block.digest();
1350
1351 let verify_rx = inline.verify(block_context, digest).await;
1352 drop(verify_rx);
1353
1354 context.sleep(Duration::from_millis(10)).await;
1357
1358 assert!(marshal.verified(round, block).await);
1359 let certify_rx = inline.certify(round, digest).await;
1360 select! {
1361 result = certify_rx => {
1362 assert!(
1363 result.expect("certify result missing"),
1364 "certify should recover after verify receiver drop"
1365 );
1366 },
1367 _ = context.sleep(Duration::from_secs(5)) => {
1368 panic!("certify should recover promptly after verify drop");
1369 },
1370 }
1371 });
1372 }
1373
1374 #[test_traced("WARN")]
1381 fn test_inline_store_overlaps_app_verify() {
1382 let runner = deterministic::Runner::timed(Duration::from_secs(30));
1383 runner.start(|mut context| async move {
1384 let Fixture {
1385 participants,
1386 schemes,
1387 ..
1388 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1389 let mut oracle = setup_network_with_participants(
1390 context.child("network"),
1391 NZUsize!(1),
1392 participants.clone(),
1393 )
1394 .await;
1395
1396 let me = participants[0].clone();
1397
1398 let setup = StandardHarness::setup_validator(
1399 context.child("validator").with_attribute("index", 0),
1400 &mut oracle,
1401 me.clone(),
1402 ConstantProvider::new(schemes[0].clone()),
1403 )
1404 .await;
1405 let marshal = setup.mailbox;
1406 let buffer = setup.extra;
1407
1408 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1409 let (mock_app, verify_started, release_verify): (GatedVerifyingApp<B, S>, _, _) =
1410 GatedVerifyingApp::new();
1411 let mut inline = Inline::new(
1412 context.child("inline"),
1413 mock_app,
1414 marshal.clone(),
1415 FixedEpocher::new(BLOCKS_PER_EPOCH),
1416 );
1417
1418 let parent = make_raw_block(genesis.digest(), Height::new(1), 100);
1419 let parent_digest = parent.digest();
1420
1421 let child_round = Round::new(Epoch::zero(), View::new(2));
1422 let child_ctx = Ctx {
1423 round: child_round,
1424 leader: me.clone(),
1425 parent: (View::new(1), parent_digest),
1426 };
1427 let child = B::new::<Sha256>(child_ctx.clone(), parent_digest, Height::new(2), 200);
1428 let child_digest = child.digest();
1429
1430 assert!(
1431 buffer
1432 .broadcast(commonware_p2p::Recipients::Some(vec![]), parent)
1433 .accepted(),
1434 "buffer broadcast for parent should be accepted"
1435 );
1436 assert!(
1437 buffer
1438 .broadcast(commonware_p2p::Recipients::Some(vec![]), child)
1439 .accepted(),
1440 "buffer broadcast for child should be accepted"
1441 );
1442
1443 let verify_rx = inline.verify(child_ctx, child_digest).await;
1444
1445 verify_started
1449 .await
1450 .expect("verify should reach the gated application");
1451 assert!(
1452 marshal.get_block(&child_digest).await.is_some(),
1453 "the store request runs concurrently with app.verify, so the block is locally queryable while verification is still gated"
1454 );
1455
1456 release_verify.send_lossy(());
1459 assert!(
1460 verify_rx.await.expect("verify result missing"),
1461 "inline verify should pass once verification is released"
1462 );
1463 let certify_rx = inline.certify(child_round, child_digest).await;
1464 select! {
1465 result = certify_rx => {
1466 assert!(
1467 result.expect("certify result missing"),
1468 "certify should succeed once verification passes"
1469 );
1470 },
1471 _ = context.sleep(Duration::from_secs(5)) => {
1472 panic!("certify should resolve after verification is released");
1473 },
1474 }
1475 });
1476 }
1477
1478 #[test_traced("WARN")]
1489 fn test_propose_skips_when_verified_block_exists_on_restart() {
1490 let runner = deterministic::Runner::timed(Duration::from_secs(30));
1491 runner.start(|mut context| async move {
1492 let Fixture {
1493 participants,
1494 schemes,
1495 ..
1496 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1497 let mut oracle = setup_network_with_participants(
1498 context.child("network"),
1499 NZUsize!(1),
1500 participants.clone(),
1501 )
1502 .await;
1503
1504 let me = participants[0].clone();
1505 let round = Round::new(Epoch::zero(), View::new(1));
1506 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1507 let ctx = Ctx {
1508 round,
1509 leader: me.clone(),
1510 parent: (View::zero(), genesis.digest()),
1511 };
1512
1513 let pre_setup = StandardHarness::setup_validator(
1517 context.child("validator").with_attribute("index", 0),
1518 &mut oracle,
1519 me.clone(),
1520 ConstantProvider::new(schemes[0].clone()),
1521 )
1522 .await;
1523 let pre_marshal = pre_setup.mailbox;
1524 let pre_actor = pre_setup.actor_handle;
1525 let pre_extra = pre_setup.extra;
1526 let pre_application = pre_setup.application;
1527
1528 let stale_block = B::new::<Sha256>(ctx.clone(), genesis.digest(), Height::new(1), 100);
1529 assert!(pre_marshal.verified(round, stale_block).await);
1530
1531 pre_actor.abort();
1534 drop(pre_marshal);
1535 drop(pre_extra);
1536 drop(pre_application);
1537
1538 let post_setup = StandardHarness::setup_validator(
1542 context
1543 .child("validator_restart")
1544 .with_attribute("index", 0),
1545 &mut oracle,
1546 me.clone(),
1547 ConstantProvider::new(schemes[0].clone()),
1548 )
1549 .await;
1550 let post_marshal = post_setup.mailbox;
1551
1552 let fresh_block = B::new::<Sha256>(ctx.clone(), genesis.digest(), Height::new(1), 200);
1553 let mock_app: MockVerifyingApp<B, S> =
1554 MockVerifyingApp::new().with_propose_result(fresh_block);
1555 let mut inline = Inline::new(
1556 context.child("inline"),
1557 mock_app,
1558 post_marshal.clone(),
1559 FixedEpocher::new(BLOCKS_PER_EPOCH),
1560 );
1561
1562 let digest_rx = inline.propose(ctx).await;
1563 assert!(
1564 digest_rx.await.is_err(),
1565 "propose must drop the receiver so the voter nullifies the round via timeout"
1566 );
1567 });
1568 }
1569
1570 #[test_traced("WARN")]
1577 fn test_certify_not_poisoned_by_equivocating_parent_verify() {
1578 let runner = deterministic::Runner::timed(Duration::from_secs(30));
1579 runner.start(|mut context| async move {
1580 let Fixture {
1581 participants,
1582 schemes,
1583 ..
1584 } = bls12381_threshold_vrf::fixture::<V, _>(&mut context, NAMESPACE, NUM_VALIDATORS);
1585 let mut oracle = setup_network_with_participants(
1586 context.child("network"),
1587 NZUsize!(1),
1588 participants.clone(),
1589 )
1590 .await;
1591
1592 let me = participants[0].clone();
1593 let setup = StandardHarness::setup_validator(
1594 context.child("validator").with_attribute("index", 0),
1595 &mut oracle,
1596 me,
1597 ConstantProvider::new(schemes[0].clone()),
1598 )
1599 .await;
1600 let marshal = setup.mailbox;
1601 let buffer = setup.extra;
1602
1603 let genesis = make_raw_block(Sha256::hash(&[b""]), Height::zero(), 0);
1604 let mock_app: MockVerifyingApp<B, S> = MockVerifyingApp::new();
1605 let mut inline = Inline::new(
1606 context.child("inline"),
1607 mock_app,
1608 marshal.clone(),
1609 FixedEpocher::new(BLOCKS_PER_EPOCH),
1610 );
1611
1612 let leader = participants[1].clone();
1613
1614 let certified_round = Round::new(Epoch::zero(), View::new(1));
1616 let certified_ctx = Ctx {
1617 round: certified_round,
1618 leader: default_leader(),
1619 parent: (View::zero(), genesis.digest()),
1620 };
1621 let certified = B::new::<Sha256>(certified_ctx, genesis.digest(), Height::new(1), 100);
1622 let certified_digest = certified.digest();
1623 assert!(marshal.verified(certified_round, certified).await);
1624
1625 let notarized_round = Round::new(Epoch::zero(), View::new(2));
1628 let notarized_ctx = Ctx {
1629 round: notarized_round,
1630 leader: leader.clone(),
1631 parent: (View::new(1), certified_digest),
1632 };
1633 let notarized = B::new::<Sha256>(notarized_ctx, certified_digest, Height::new(2), 200);
1634 let notarized_digest = notarized.digest();
1635 assert!(marshal.verified(notarized_round, notarized).await);
1636
1637 let round = Round::new(Epoch::zero(), View::new(3));
1641 let block_ctx = Ctx {
1642 round,
1643 leader: leader.clone(),
1644 parent: (View::new(2), notarized_digest),
1645 };
1646 let block = B::new::<Sha256>(block_ctx, notarized_digest, Height::new(3), 300);
1647 let digest = block.digest();
1648 assert!(
1649 buffer
1650 .broadcast(commonware_p2p::Recipients::Some(vec![]), block)
1651 .accepted(),
1652 "buffer broadcast for the candidate should be accepted"
1653 );
1654 context.sleep(Duration::from_millis(10)).await;
1655
1656 let equivocating_ctx = Ctx {
1661 round,
1662 leader,
1663 parent: (View::new(1), certified_digest),
1664 };
1665 let verify_rx = inline.verify(equivocating_ctx, digest).await;
1666 assert!(
1667 !verify_rx.await.expect("verify result missing"),
1668 "the equivocating proposal must not be notarized"
1669 );
1670
1671 let certify_rx = inline.certify(round, digest).await;
1673 select! {
1674 result = certify_rx => {
1675 assert!(
1676 result.expect("certify result missing"),
1677 "certify of the notarized digest must not adopt the verdict computed under the equivocating context"
1678 );
1679 },
1680 _ = context.sleep(Duration::from_secs(5)) => {
1681 panic!("certify should resolve promptly");
1682 },
1683 }
1684 });
1685 }
1686}