Skip to main content

commonware_consensus/marshal/core/
actor.rs

1use super::{
2    acks::{PendingAck, PendingAcks},
3    cache,
4    delivery::PendingVerification,
5    durability::{DispatchGate, Durable as _},
6    floor::Floor,
7    mailbox::{CommitmentFallback, Mailbox, Message},
8    stream::Stream,
9    subscriptions::{Key as SubscriptionKey, KeyFor as SubscriptionKeyFor, Subscriptions},
10    variant::NoBuffer,
11    Buffer, Variant,
12};
13use crate::{
14    marshal::{
15        resolver::handler::{self, Annotation, Key, Request},
16        store::{Blocks, Certificates},
17        Config, Identifier as BlockID, Start, Update,
18    },
19    simplex::{
20        scheme::Scheme,
21        types::{verify_certificates, Finalization, Notarization, Subject},
22    },
23    types::{Epoch, Epocher, Height, Round, ViewDelta},
24    Block, Epochable, Heightable, Reporter,
25};
26use bytes::Bytes;
27use commonware_actor::mailbox;
28use commonware_codec::{Decode, Encode, Read};
29use commonware_cryptography::{
30    certificate::{Provider, Verifier},
31    Digestible,
32};
33use commonware_macros::{boxed, select_loop};
34use commonware_p2p::Recipients;
35use commonware_parallel::Strategy;
36use commonware_resolver::{Delivery, Resolver, TargetedResolver};
37use commonware_runtime::{
38    spawn_cell,
39    telemetry::{
40        metrics::{Gauge, GaugeExt, MetricsExt as _},
41        traces::TracedExt as _,
42    },
43    BufferPooler, Clock, ContextCell, Handle, Metrics, Spawner, Storage,
44};
45use commonware_storage::archive::Identifier as ArchiveID;
46use commonware_utils::{
47    acknowledgement::Exact,
48    channel::{fallible::OneshotExt, oneshot},
49    futures::{AbortablePool, Pool},
50    Acknowledgement, BoxedError,
51};
52use futures::{
53    future::{join, join_all},
54    try_join,
55};
56use rand_core::CryptoRng;
57use std::{collections::BTreeMap, future::Future, num::NonZeroUsize, sync::Arc};
58use tracing::{debug, info_span, warn, Instrument as _, Span};
59
60// Resolver request keys are expressed in the variant commitment type, which
61// may differ from the block digest for coded variants.
62type ResolverRequestFor<V> = Key<<V as Variant>::Commitment>;
63
64// A resolver delivery plus the peer-validity response channel. Local
65// annotations on the delivery decide how accepted data is used.
66struct ResolverDelivery<V: Variant> {
67    delivery: Delivery<ResolverRequestFor<V>, Annotation>,
68    value: Bytes,
69    response: oneshot::Sender<bool>,
70}
71
72/// Completion marker for entries in the actor's durability sync pool.
73enum PooledSync {
74    /// A sync that requires no action on completion.
75    Observed,
76    /// A finalized-archive sync batch became durable. Carries the sequence
77    /// assigned by [`Actor::start_finalized_sync`] so the completion arm can
78    /// release every batch the sync covers (see [`DispatchGate::release`]).
79    Finalized(u64),
80}
81
82/// The [Actor] is responsible for receiving uncertified blocks from the broadcast mechanism,
83/// receiving notarizations and finalizations from consensus, and reconstructing a total order
84/// of blocks.
85///
86/// The actor is designed to be used in a view-based model. Each view corresponds to a
87/// potential block in the chain. The actor will only finalize a block if it has a
88/// corresponding finalization.
89///
90/// The actor also provides a backfill mechanism for missing blocks. If the actor receives a
91/// finalization for a block that is ahead of its current view, it will request the missing blocks
92/// from its peers. This ensures that the actor can catch up to the rest of the network if it falls
93/// behind.
94pub struct Actor<E, V, P, FC, FB, ES, T, A = Exact>
95where
96    E: BufferPooler + CryptoRng + Spawner + Metrics + Clock + Storage,
97    V: Variant,
98    P: Provider<Scope = Epoch, Scheme: Scheme<V::Commitment>>,
99    FC: Certificates<
100        BlockDigest = <V::Block as Digestible>::Digest,
101        Commitment = V::Commitment,
102        Scheme = P::Scheme,
103    >,
104    FB: Blocks<Block = V::StoredBlock>,
105    ES: Epocher,
106    T: Strategy,
107    A: Acknowledgement,
108{
109    // ---------- Context ----------
110    context: ContextCell<E>,
111
112    // ---------- Message Passing ----------
113    // Mailbox
114    mailbox: mailbox::Receiver<Message<P::Scheme, V>>,
115
116    // ---------- Configuration ----------
117    // Provider for epoch-specific signing schemes
118    provider: P,
119    // Epoch configuration
120    epocher: ES,
121    // Minimum number of views to retain temporary data after the application processes a block
122    view_retention_timeout: ViewDelta,
123    // Maximum number of blocks to repair at once
124    max_repair: NonZeroUsize,
125    // Codec configuration for block type
126    block_codec_config: <V::ApplicationBlock as Read>::Cfg,
127    // Strategy for parallel operations
128    strategy: T,
129
130    // ---------- State ----------
131    // Current processed floor and any pending floor update
132    floor: Floor<P::Scheme, V::Commitment>,
133    // Application delivery cursor
134    stream: Stream<E>,
135    // Pending application acknowledgements
136    pending_acks: PendingAcks<V, A>,
137    // Highest known finalized height
138    tip: Height,
139    // Outstanding subscriptions for blocks
140    block_subscriptions: Subscriptions<V>,
141    // Defers application dispatch of finalized-archive writes until a sync
142    // covering them completes
143    dispatch_gate: DispatchGate,
144
145    // ---------- Storage ----------
146    // Prunable cache
147    cache: cache::Manager<E, V, P::Scheme>,
148    // Finalizations stored by height
149    finalizations_by_height: FC,
150    // Finalized blocks stored by height
151    finalized_blocks: FB,
152
153    // ---------- Metrics ----------
154    // Latest height metric
155    finalized_height: Gauge,
156    // Latest processed height
157    processed_height: Gauge,
158}
159
160impl<E, V, P, FC, FB, ES, T, A> Actor<E, V, P, FC, FB, ES, T, A>
161where
162    E: BufferPooler + CryptoRng + Spawner + Metrics + Clock + Storage,
163    V: Variant,
164    P: Provider<Scope = Epoch, Scheme: Scheme<V::Commitment>>,
165    FC: Certificates<
166        BlockDigest = <V::Block as Digestible>::Digest,
167        Commitment = V::Commitment,
168        Scheme = P::Scheme,
169    >,
170    FB: Blocks<Block = V::StoredBlock>,
171    ES: Epocher,
172    T: Strategy,
173    A: Acknowledgement,
174{
175    /// Create a new application actor.
176    #[boxed]
177    pub async fn init(
178        context: E,
179        finalizations_by_height: FC,
180        mut finalized_blocks: FB,
181        config: Config<P, ES, T, V::ApplicationBlock, V::Block, V::Commitment>,
182    ) -> (Self, Mailbox<P::Scheme, V>, Option<Height>) {
183        // Initialize cache
184        let prunable_config = cache::Config {
185            partition_prefix: format!("{}-cache", config.partition_prefix),
186            prunable_items_per_section: config.prunable_items_per_section,
187            replay_buffer: config.replay_buffer,
188            key_write_buffer: config.key_write_buffer,
189            value_write_buffer: config.value_write_buffer,
190            key_page_cache: config.page_cache.clone(),
191        };
192        let cache = cache::Manager::init(
193            context.child("cache"),
194            prunable_config,
195            config.block_codec_config.clone(),
196        )
197        .await;
198
199        // The application metadata name is retained for legacy support.
200        let application_metadata_partition =
201            format!("{}-application-metadata", config.partition_prefix);
202        let stream = Stream::new(context.child("stream"), &application_metadata_partition).await;
203        let last_processed_height = stream.processed_height();
204
205        // Genesis is a local anchor. A floor finalization is verified and
206        // resolved after `run` receives the resolver and buffer.
207        let pending_floor_anchor = match config.start {
208            Start::Genesis(anchor) => {
209                assert_eq!(
210                    anchor.height(),
211                    Height::zero(),
212                    "genesis anchor must be at height zero"
213                );
214                Self::ensure_genesis_anchor(&mut finalized_blocks, anchor, last_processed_height)
215                    .await;
216                None
217            }
218            Start::Floor(finalization) => Some(finalization),
219        };
220        let last_processed_round =
221            Self::latest_processed_round(&finalizations_by_height, last_processed_height).await;
222
223        // Create metrics
224        let finalized_height = context.gauge("finalized_height", "Finalized height of application");
225        let processed_height = context.gauge("processed_height", "Processed height of application");
226        if let Some(last_processed_height) = last_processed_height {
227            let _ = processed_height.try_set(last_processed_height.get());
228        }
229        let floor = pending_floor_anchor.map_or_else(
230            || Floor::resolved(last_processed_height, last_processed_round),
231            |finalization| {
232                Floor::awaiting_anchor(last_processed_height, last_processed_round, finalization)
233            },
234        );
235
236        // Initialize mailbox
237        let (sender, mailbox) = mailbox::new(context.child("mailbox"), config.mailbox_size);
238        (
239            Self {
240                context: ContextCell::new(context),
241                mailbox,
242                provider: config.provider,
243                epocher: config.epocher,
244                view_retention_timeout: config.view_retention_timeout,
245                max_repair: config.max_repair,
246                block_codec_config: config.block_codec_config,
247                strategy: config.strategy,
248                floor,
249                stream,
250                pending_acks: PendingAcks::new(config.max_pending_acks.get()),
251                tip: Height::zero(),
252                block_subscriptions: Subscriptions::new(),
253                dispatch_gate: DispatchGate::default(),
254                cache,
255                finalizations_by_height,
256                finalized_blocks,
257                finalized_height,
258                processed_height,
259            },
260            Mailbox::new(sender),
261            last_processed_height,
262        )
263    }
264
265    async fn ensure_genesis_anchor(
266        finalized_blocks: &mut FB,
267        anchor: V::Block,
268        last_processed_height: Option<Height>,
269    ) {
270        let anchor_height = anchor.height();
271        let anchor_commitment = V::commitment(&anchor);
272        match finalized_blocks
273            .get(ArchiveID::Index(anchor_height.get()))
274            .await
275        {
276            Ok(Some(stored)) => {
277                let stored: V::Block = stored.into();
278                assert_eq!(
279                    stored.height(),
280                    anchor_height,
281                    "stored genesis block height mismatch"
282                );
283                assert!(
284                    V::commitment(&stored) == anchor_commitment,
285                    "stored genesis block does not match configured anchor"
286                );
287            }
288            Ok(None) => {
289                if let Some(existing) =
290                    last_processed_height.filter(|height| anchor_height < *height)
291                {
292                    warn!(
293                        height = %anchor_height,
294                        %existing,
295                        "ignoring stale anchor"
296                    );
297                    return;
298                }
299
300                finalized_blocks
301                    .put(anchor.into())
302                    .await
303                    .expect("failed to store startup anchor");
304                finalized_blocks
305                    .sync()
306                    .await
307                    .expect("failed to sync startup anchor");
308                debug!(height = %anchor_height, "stored genesis block");
309            }
310            Err(err) => panic!("failed to check startup anchor: {err}"),
311        }
312    }
313
314    /// Start the actor.
315    pub fn start<R, Buf>(
316        mut self,
317        application: impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
318        buffer: Buf,
319        resolver: (handler::Receiver<V::Commitment>, R),
320    ) -> Handle<()>
321    where
322        R: TargetedResolver<
323            Key = ResolverRequestFor<V>,
324            Subscriber = Annotation,
325            PublicKey = <P::Scheme as Verifier>::PublicKey,
326        >,
327        Buf: Buffer<V, PublicKey = <P::Scheme as Verifier>::PublicKey>,
328    {
329        spawn_cell!(self.context, self.run(application, buffer, resolver))
330    }
331
332    /// Start the actor without a broadcast buffer.
333    pub fn start_unbuffered<R>(
334        self,
335        application: impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
336        resolver: (handler::Receiver<V::Commitment>, R),
337    ) -> Handle<()>
338    where
339        R: TargetedResolver<
340            Key = ResolverRequestFor<V>,
341            Subscriber = Annotation,
342            PublicKey = <P::Scheme as Verifier>::PublicKey,
343        >,
344    {
345        self.start(
346            application,
347            NoBuffer::<<P::Scheme as Verifier>::PublicKey>::new(),
348            resolver,
349        )
350    }
351
352    /// Run the application actor.
353    async fn run<R, Buf>(
354        mut self,
355        mut application: impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
356        mut buffer: Buf,
357        (mut resolver_rx, mut resolver): (handler::Receiver<V::Commitment>, R),
358    ) where
359        R: TargetedResolver<
360            Key = ResolverRequestFor<V>,
361            Subscriber = Annotation,
362            PublicKey = <P::Scheme as Verifier>::PublicKey,
363        >,
364        Buf: Buffer<V, PublicKey = <P::Scheme as Verifier>::PublicKey>,
365    {
366        // Create a local pool for waiter futures.
367        let mut waiters = AbortablePool::<Result<Arc<V::Block>, SubscriptionKeyFor<V>>>::default();
368
369        // Observe durable syncs that no consensus caller awaits (the
370        // notarization and finalization paths). A flush failure inside
371        // `start_sync` is reported only through the returned handle, so every
372        // handle must be observed to apply the fatal policy. This pool does
373        // so without blocking the actor on a sync.
374        let mut syncs = Pool::<PooledSync>::default();
375
376        // Anchor all startup work under a single root span. Tip recovery, floor
377        // installation, gap repair, and the initial dispatch all run before any
378        // mailbox message arrives, so without this root their work would emit as
379        // orphan traces.
380        async {
381            // Get tip and send to application
382            let tip = self.get_latest().await;
383            if let Some((height, digest, round)) = tip {
384                application.report(Update::Tip(round, height, digest));
385                self.tip = height;
386                let _ = self.finalized_height.try_set(height.get());
387            }
388
389            // Load persisted cache epochs so find_block can discover blocks
390            // written before the last shutdown.
391            self.cache.load_persisted_epochs().await;
392
393            // A configured floor follows the same path as `SetFloor`: verify it,
394            // then apply a local anchor or fetch the anchor block.
395            if let Some(finalization) = self.floor.take_pending_anchor() {
396                self.install_floor(
397                    finalization,
398                    false,
399                    &mut resolver,
400                    &mut buffer,
401                    &mut application,
402                )
403                .await;
404            }
405
406            // Attempt to repair any gaps in the finalized blocks archive, if there are any.
407            if self
408                .try_repair_gaps(&mut buffer, &mut resolver, &mut application)
409                .await
410            {
411                self.sync_finalized().await;
412            }
413
414            // Attempt to dispatch the next finalized block to the application, if it is ready.
415            self.try_dispatch_blocks(&mut application).await;
416        }
417        .instrument(info_span!("marshal.actor.start"))
418        .await;
419
420        select_loop! {
421            self.context,
422            on_start => {
423                // Remove any dropped subscribers. If all subscribers dropped, abort the waiter.
424                self.block_subscriptions.retain_open();
425            },
426            on_stopped => {
427                debug!("context shutdown, stopping marshal");
428            },
429            // Drive durability syncs: a real sync failure panics inside the
430            // pooled future (the fatal policy), aborting the actor. A completed
431            // finalized-archive sync additionally releases the dispatch barrier
432            // for the batches it covers and resumes application dispatch.
433            sync = syncs.next_completed() => {
434                if let PooledSync::Finalized(seq) = sync {
435                    self.dispatch_gate.release(seq);
436                    self.try_dispatch_blocks(&mut application).await;
437                }
438            },
439            // Handle waiter completions first
440            Ok(completion) = waiters.next_completed() else continue => match completion {
441                Ok(block) => {
442                    self.ingest(block, &mut buffer, &mut application, &mut resolver)
443                        .await;
444                }
445                Err(key) => {
446                    match key {
447                        SubscriptionKey::Digest(digest) => {
448                            debug!(
449                                ?digest,
450                                "buffer subscription closed, canceling local subscribers"
451                            );
452                        }
453                        SubscriptionKey::Commitment(commitment) => {
454                            debug!(
455                                ?commitment,
456                                "buffer subscription closed, canceling local subscribers"
457                            );
458                        }
459                    }
460                    self.block_subscriptions.remove(&key);
461                }
462            },
463            // Handle application acknowledgements (drain all ready acks, sync once)
464            result = self.pending_acks.current() => {
465                if let Err((height, e)) = self
466                    .handle_ack(result, &mut application, &mut buffer, &mut resolver)
467                    .await
468                {
469                    debug!(
470                        ?e,
471                        %height,
472                        "application acknowledgement dropped, stopping marshal"
473                    );
474                    return;
475                }
476            },
477            // Handle consensus inputs before backfill or resolver traffic
478            Some(message) = self.mailbox.recv() else {
479                debug!("mailbox closed, shutting down");
480                break;
481            } => {
482                let span = info_span!(
483                    parent: message.span(),
484                    "marshal.actor.process",
485                    operation = message.name(),
486                );
487                self.handle_mailbox_message(
488                    message,
489                    &mut resolver,
490                    &mut waiters,
491                    &mut syncs,
492                    &mut buffer,
493                    &mut application,
494                )
495                .instrument(span)
496                .await;
497            },
498            // Handle resolver messages last (batched up to max_repair, sync once)
499            Some(message) = resolver_rx.recv() else {
500                debug!("handler closed, shutting down");
501                return;
502            } => {
503                self.handle_resolver_message(
504                    message,
505                    &mut resolver_rx,
506                    &mut resolver,
507                    &mut syncs,
508                    &mut buffer,
509                    &mut application,
510                )
511                .await;
512            },
513        }
514    }
515
516    /// Handles one ready application acknowledgement and drains any queued acks
517    /// that are already complete.
518    async fn handle_ack<Buf, R>(
519        &mut self,
520        result: <A::Waiter as Future>::Output,
521        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
522        buffer: &mut Buf,
523        resolver: &mut R,
524    ) -> Result<(), (Height, A::Error)>
525    where
526        Buf: Buffer<V>,
527        R: Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
528    {
529        // Start with the ack that woke this `select_loop!` arm.
530        let mut pending = Some(self.pending_acks.complete_current(result));
531        let last_acked_commitment = loop {
532            let (height, commitment, result) = pending.take().expect("pending ack must exist");
533            match result {
534                Ok(()) => {
535                    // Apply in-memory progress updates for this acknowledged
536                    // block. The metadata sync below makes drained updates durable.
537                    self.update_processed_height(height, resolver);
538                    self.update_processed_round(height, resolver).await;
539                }
540                Err(e) => return Err((height, e)),
541            }
542
543            // Opportunistically drain any additional already-ready acks so we
544            // can persist one metadata sync for the whole batch below.
545            match self.pending_acks.pop_ready() {
546                Some(next) => pending = Some(next),
547                None => break commitment,
548            }
549        };
550
551        // Persist buffered progress updates once after draining all ready acks.
552        self.stream
553            .sync()
554            .await
555            .expect("failed to sync application progress");
556
557        // Anything below the last acknowledged commitment is safe for the
558        // buffer to prune.
559        buffer.finalized(last_acked_commitment);
560
561        // Refill the application dispatch pipeline.
562        self.try_dispatch_blocks(application).await;
563
564        Ok(())
565    }
566
567    /// Handles a single mailbox message from local consensus/application callers.
568    async fn handle_mailbox_message<Buf, R>(
569        &mut self,
570        message: Message<P::Scheme, V>,
571        resolver: &mut R,
572        waiters: &mut AbortablePool<Result<Arc<V::Block>, SubscriptionKeyFor<V>>>,
573        syncs: &mut Pool<PooledSync>,
574        buffer: &mut Buf,
575        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
576    ) where
577        Buf: Buffer<V, PublicKey = <P::Scheme as Verifier>::PublicKey>,
578        R: TargetedResolver<
579            Key = ResolverRequestFor<V>,
580            Subscriber = Annotation,
581            PublicKey = <P::Scheme as Verifier>::PublicKey,
582        >,
583    {
584        if message.response_closed() {
585            return;
586        }
587
588        match message {
589            Message::GetInfo {
590                identifier,
591                response,
592                ..
593            } => {
594                let info = match identifier {
595                    // TODO: Instead of pulling out the entire block, determine the
596                    // height directly from the archive by mapping the digest to
597                    // the index, which is the same as the height.
598                    BlockID::Digest(digest) => self
599                        .finalized_blocks
600                        .get(ArchiveID::Key(&digest))
601                        .await
602                        .ok()
603                        .flatten()
604                        .map(|b| (b.height(), digest)),
605                    BlockID::Height(height) => self.get_info_by_height(height).await,
606                    BlockID::Latest => self.get_latest().await.map(|(h, d, _)| (h, d)),
607                };
608                response.send_lossy(info);
609            }
610            Message::GetVerified {
611                round, response, ..
612            } => {
613                let block = self.cache.get_verified(round).await.map(Into::into);
614                response.send_lossy(block);
615            }
616            Message::Forward {
617                round,
618                commitment,
619                recipients,
620                ..
621            } => {
622                if matches!(&recipients, Recipients::Some(peers) if peers.is_empty()) {
623                    return;
624                }
625                let Some(block) = self.find_block_by_commitment(buffer, commitment).await else {
626                    debug!(?commitment, "block not found for forwarding");
627                    return;
628                };
629                buffer.send(round, block, recipients);
630            }
631            Message::Proposed {
632                round,
633                block,
634                recipients,
635                ack,
636                ..
637            } => {
638                // To lower view latency as much as possible while preserving
639                // safety, we broadcast the block before persisting it
640                // (durability is not required until certify). A leader that
641                // crashes here may broadcast a conflicting block for the same
642                // round after restart. This is tolerated: extra block bytes
643                // cannot form a conflicting certificate (unlike votes), block
644                // storage tolerates multiple candidates per round (see
645                // [Mailbox::get_verified]), and the propose paths skip or
646                // reuse a recovered block on restart.
647                buffer.send(round, Arc::clone(&block), recipients);
648                self.persist_verified(round, block, ack, buffer, application, resolver)
649                    .await;
650            }
651            Message::Verified {
652                round, block, ack, ..
653            } => {
654                self.persist_verified(round, block, ack, buffer, application, resolver)
655                    .await;
656            }
657            Message::Certified {
658                round, block, ack, ..
659            } => {
660                self.ingest(Arc::clone(&block), buffer, application, resolver)
661                    .await;
662                let digest = block.digest();
663
664                // A block the verified archive already holds needs no second copy:
665                // the verified archive's covering sync handle vouches for it. At
666                // most one notarization exists per round, so the notarized slot can
667                // never belong to a different payload: a duplicate put is a no-op
668                // whose handle still covers the original write. If the round has
669                // already been pruned by tip advancement, both writes are no-ops
670                // because the round is below the retention floor.
671                let block_sync = if self.cache.has_verified(round, &digest).await {
672                    debug!(?round, "certified block covered by verified write");
673                    self.cache.start_sync_verified(round).await
674                } else {
675                    self.cache
676                        .put_notarized(round, digest, Arc::unwrap_or_clone(block).into())
677                        .await
678                };
679
680                // Hold the certify barrier until the round's notarization
681                // certificate (when one was accepted before this message) is
682                // durable alongside the block.
683                let notarization_sync = self.cache.start_sync_notarizations(round).await;
684                let handle = Handle::from_future(async move {
685                    let (notarization, block) = join(notarization_sync, block_sync).await;
686                    notarization.and(block)
687                });
688                ack.send_lossy(handle);
689            }
690            Message::Notarization { notarization, .. } => {
691                let round = notarization.round();
692                let commitment = notarization.proposal.payload;
693                let digest = V::commitment_to_inner(commitment);
694
695                // Persist the notarization; the certify barrier folds in its
696                // durability via `start_sync_notarizations`. The archive keeps a
697                // single notarization per round, so a re-delivery is a no-op whose
698                // handle still covers the original write. No consensus caller
699                // awaits this handle, so the pool observes it (applying the fatal
700                // policy) without blocking the actor.
701                let handle = self
702                    .cache
703                    .put_notarization(round, digest, notarization)
704                    .await;
705                syncs.push(async move {
706                    handle.durable(round, "notarization").await;
707                    PooledSync::Observed
708                });
709
710                // A notarization alone is not enough to fetch missing proposal
711                // data. If the block is not locally available, remember the
712                // certificate and wait for a later finalization/repair path.
713                if let Some(block) = self.find_block_by_commitment(buffer, commitment).await {
714                    self.ingest(Arc::clone(&block), buffer, application, resolver)
715                        .await;
716                    if self.cache.has_verified(round, &digest).await {
717                        debug!(?round, "notarized block covered by verified write");
718                    } else {
719                        let handle = self
720                            .cache
721                            .put_notarized(round, digest, Arc::unwrap_or_clone(block).into())
722                            .await;
723                        syncs.push(async move {
724                            handle.durable(round, "notarized").await;
725                            PooledSync::Observed
726                        });
727                    }
728                } else {
729                    debug!(?round, "notarized block unavailable locally");
730                }
731            }
732            Message::Finalization { finalization, .. } => {
733                let round = finalization.round();
734                let commitment = finalization.proposal.payload;
735                let digest = V::commitment_to_inner(commitment);
736
737                // Cache finalization by round.
738                self.cache
739                    .put_finalization(round, digest, finalization.clone())
740                    .await;
741
742                // Search for the finalized block locally, otherwise fetch it remotely.
743                if let Some(block) = self.find_block_by_commitment(buffer, commitment).await {
744                    // The anchor path stores the floor block and finalization,
745                    // advances floors, prunes below them, and resumes dispatch.
746                    if self
747                        .ingest(Arc::clone(&block), buffer, application, resolver)
748                        .await
749                    {
750                        return;
751                    }
752
753                    let height = block.height();
754                    self.update_processed_round_floor(height, round, resolver)
755                        .await;
756                    if self
757                        .store_finalization(
758                            height,
759                            digest,
760                            Arc::unwrap_or_clone(block),
761                            Some(finalization),
762                            application,
763                        )
764                        .await
765                    {
766                        // If a floor anchor is pending, repair and dispatch are
767                        // no-ops until the anchor block is stored.
768                        self.try_repair_gaps(buffer, resolver, application).await;
769                        self.start_finalized_sync(round, syncs).await;
770                        debug!(?round, %height, "finalized block stored");
771                    }
772                } else {
773                    // The finalization carries a round and commitment, but not a
774                    // height. Keep the request round-bound until the block is decoded.
775                    debug!(?round, ?commitment, "finalized block missing");
776                    self.floor
777                        .fetch_if_permitted(
778                            resolver,
779                            Request::finalized_block_by_round(commitment, round),
780                        )
781                        .ignore();
782                }
783            }
784            Message::GetBlock {
785                identifier,
786                response,
787                ..
788            } => match identifier {
789                BlockID::Digest(digest) => {
790                    let result = self
791                        .find_block_by_digest(buffer, digest)
792                        .await
793                        .map(Arc::unwrap_or_clone);
794                    response.send_lossy(result);
795                }
796                BlockID::Height(height) => {
797                    let result = self.get_finalized_block(height).await;
798                    response.send_lossy(result);
799                }
800                BlockID::Latest => {
801                    let block = match self.get_latest().await {
802                        Some((_, digest, _)) => self.find_block_by_digest(buffer, digest).await,
803                        None => None,
804                    }
805                    .map(Arc::unwrap_or_clone);
806                    response.send_lossy(block);
807                }
808            },
809            Message::GetFinalization {
810                height, response, ..
811            } => {
812                let finalization = self.get_finalization_by_height(height).await;
813                response.send_lossy(finalization);
814            }
815            Message::GetProcessedHeight { response, .. } => {
816                response.send_lossy(self.stream.processed_height());
817            }
818            Message::HintFinalized {
819                height, targets, ..
820            } => {
821                // Skip if finalization is already available locally.
822                if self.has_finalization_by_height(height).await {
823                    return;
824                }
825
826                self.floor
827                    .fetch_targeted_if_permitted(resolver, Request::finalized(height), targets)
828                    .ignore();
829            }
830            Message::SubscribeByDigest {
831                span,
832                digest,
833                fallback,
834                response,
835            } => {
836                self.handle_subscribe(
837                    span,
838                    fallback.into(),
839                    SubscriptionKey::Digest(digest),
840                    response,
841                    resolver,
842                    waiters,
843                    buffer,
844                )
845                .await;
846            }
847            Message::SubscribeByCommitment {
848                span,
849                commitment,
850                fallback,
851                response,
852            } => {
853                self.handle_subscribe(
854                    span,
855                    fallback,
856                    SubscriptionKey::Commitment(commitment),
857                    response,
858                    resolver,
859                    waiters,
860                    buffer,
861                )
862                .await;
863            }
864            Message::HintNotarized {
865                round, commitment, ..
866            } => {
867                if self
868                    .find_block_by_commitment(buffer, commitment)
869                    .await
870                    .is_none()
871                {
872                    self.floor
873                        .fetch_if_permitted(resolver, Request::notarized(round))
874                        .ignore();
875                }
876            }
877            Message::SetFloor { finalization, .. } => {
878                self.install_floor(finalization, true, resolver, buffer, application)
879                    .await;
880            }
881            Message::Prune { height, .. } => {
882                // Only allow pruning at or below the current floor.
883                if height > self.floor.processed_height() {
884                    warn!(%height, floor = %self.floor.processed_height(), "prune height above floor, ignoring");
885                    return;
886                }
887
888                self.prune_finalized_archives(height)
889                    .await
890                    .expect("failed to prune finalized archives");
891
892                // Intentionally keep existing block subscriptions alive. Canceling
893                // waiters can have catastrophic consequences because actors do not
894                // retry subscriptions on failed channels.
895            }
896        }
897    }
898
899    /// Handles a batch of resolver messages, starting one pooled
900    /// finalized-archive sync if any accepted delivery buffered a write.
901    async fn handle_resolver_message<Buf, R>(
902        &mut self,
903        message: handler::Message<V::Commitment>,
904        resolver_rx: &mut handler::Receiver<V::Commitment>,
905        resolver: &mut R,
906        syncs: &mut Pool<PooledSync>,
907        buffer: &mut Buf,
908        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
909    ) where
910        Buf: Buffer<V, PublicKey = <P::Scheme as Verifier>::PublicKey>,
911        R: Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
912    {
913        let mut handled = false;
914        let mut produces = Vec::new();
915        let mut delivers = Vec::new();
916
917        // Drain up to max_repair resolver messages. Block deliveries are handled
918        // immediately, certificate-bearing deliveries are batched for verification,
919        // and produce responses wait until repair has had a chance to fill gaps.
920        for msg in std::iter::once(message)
921            .chain(std::iter::from_fn(|| resolver_rx.try_recv().ok()))
922            .take(self.max_repair.get())
923        {
924            if msg.response_closed() {
925                continue;
926            }
927            handled = true;
928
929            match msg {
930                handler::Message::Produce { key, response } => {
931                    produces.push((key, response));
932                }
933                handler::Message::Deliver {
934                    delivery,
935                    value,
936                    response,
937                } => {
938                    let span = info_span!(
939                        parent: &delivery.subscribers.first().1,
940                        "marshal.resolver.deliver",
941                        key = %delivery.key
942                    );
943                    for (_, subscriber_span) in delivery.subscribers.iter().skip(1) {
944                        span.follows_from(subscriber_span.id());
945                    }
946                    self.handle_deliver(
947                        ResolverDelivery {
948                            delivery,
949                            value,
950                            response,
951                        },
952                        &mut delivers,
953                        buffer,
954                        application,
955                        resolver,
956                    )
957                    .instrument(span)
958                    .await;
959                }
960            }
961        }
962        if !handled {
963            return;
964        }
965
966        // Batch verify and process all certificate-bearing deliveries.
967        self.verify_delivered(delivers, buffer, application, resolver)
968            .await;
969
970        // Attempt to fill gaps before handling produce requests so we can serve
971        // data received earlier in the same batch.
972        self.try_repair_gaps(buffer, resolver, application).await;
973
974        // Start a pooled sync so any writes buffered by this batch become
975        // durable without blocking the mailbox. Dispatch of the written
976        // heights resumes when the sync completes. A batch has no single
977        // round, so the label is the node's processed round when it started.
978        self.start_finalized_sync(self.floor.processed_round(), syncs)
979            .await;
980
981        // Handle produce requests in parallel.
982        join_all(
983            produces
984                .into_iter()
985                .filter(|(_, response)| !response.is_closed())
986                .map(|(key, response)| self.handle_produce(key, response, buffer)),
987        )
988        .await;
989    }
990
991    /// Handle a produce request from a remote peer.
992    #[tracing::instrument(name = "marshal.resolver.produce", level = "debug", skip_all, fields(key = %key))]
993    async fn handle_produce<Buf: Buffer<V>>(
994        &self,
995        key: ResolverRequestFor<V>,
996        response: oneshot::Sender<Bytes>,
997        buffer: &Buf,
998    ) {
999        match key {
1000            Key::Block(commitment) => {
1001                let Some(block) = self.find_block_by_commitment(buffer, commitment).await else {
1002                    debug!(?commitment, "block missing on request");
1003                    return;
1004                };
1005                response.send_lossy(block.encode());
1006            }
1007            Key::Finalized { height } => {
1008                let Some(finalization) = self.get_finalization_by_height(height).await else {
1009                    debug!(%height, "finalization missing on request");
1010                    return;
1011                };
1012                let Some(block) = self.get_finalized_block(height).await else {
1013                    debug!(%height, "finalized block missing on request");
1014                    return;
1015                };
1016                response.send_lossy((finalization, V::into_inner(block)).encode());
1017            }
1018            Key::Notarized { round } => {
1019                let Some(notarization) = self.cache.get_notarization(round).await else {
1020                    debug!(?round, "notarization missing on request");
1021                    return;
1022                };
1023                let commitment = notarization.proposal.payload;
1024                let Some(block) = self.find_block_by_commitment(buffer, commitment).await else {
1025                    debug!(?commitment, "block missing on request");
1026                    return;
1027                };
1028                response.send_lossy((notarization, block).encode());
1029            }
1030        }
1031    }
1032
1033    /// Handle a local subscription request for a block.
1034    #[allow(clippy::too_many_arguments)]
1035    async fn handle_subscribe<Buf: Buffer<V>>(
1036        &mut self,
1037        span: Span,
1038        fallback: CommitmentFallback,
1039        key: SubscriptionKeyFor<V>,
1040        response: oneshot::Sender<Arc<V::Block>>,
1041        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
1042        waiters: &mut AbortablePool<Result<Arc<V::Block>, SubscriptionKeyFor<V>>>,
1043        buffer: &mut Buf,
1044    ) {
1045        let digest = match key {
1046            SubscriptionKey::Digest(digest) => digest,
1047            SubscriptionKey::Commitment(commitment) => V::commitment_to_inner(commitment),
1048        };
1049
1050        let block = match key {
1051            SubscriptionKey::Digest(digest) => self.find_block_by_digest(buffer, digest).await,
1052            SubscriptionKey::Commitment(commitment) => {
1053                self.find_block_by_commitment(buffer, commitment).await
1054            }
1055        };
1056        if let Some(block) = block {
1057            response.send_lossy(block);
1058            return;
1059        }
1060
1061        // We don't have the block locally. Local-only waits reach this point
1062        // without a round or height, so they only register a subscriber below.
1063        //
1064        // Round-based fetching is for notarized proposal lookups whose height is
1065        // not known before the request. Height-based fetching is only for callers
1066        // that already have a validated pruning height.
1067        match fallback {
1068            CommitmentFallback::FetchByRound { round } => {
1069                // Fetch the notarized proposal for this round. The response
1070                // must include a certificate so the commitment is tied to the
1071                // certified round context. The decoded block is heightable, but
1072                // that height is not known soon enough to key, coalesce, or prune
1073                // the in-flight resolver request.
1074                if self
1075                    .floor
1076                    .fetch_if_permitted(resolver, Request::notarized(round))
1077                    .denied()
1078                {
1079                    return;
1080                }
1081                debug!(?round, ?digest, "requested block missing");
1082            }
1083            CommitmentFallback::FetchByCommitment { height } => {
1084                let commitment = match key {
1085                    SubscriptionKey::Commitment(commitment) => commitment,
1086                    SubscriptionKey::Digest(_) => {
1087                        unreachable!("digest subscriptions cannot request commitment fallback")
1088                    }
1089                };
1090
1091                // This path is only for accepted ancestry or finalized repair,
1092                // never for a candidate block's immediate parent.
1093                if self
1094                    .floor
1095                    .fetch_if_permitted(resolver, Request::certified_block(commitment, height))
1096                    .denied()
1097                {
1098                    return;
1099                }
1100                debug!(%height, ?commitment, ?digest, "requested certified ancestry block missing");
1101            }
1102            CommitmentFallback::Wait => {}
1103        }
1104
1105        let round = match fallback {
1106            CommitmentFallback::FetchByRound { round } => Some(round),
1107            CommitmentFallback::Wait | CommitmentFallback::FetchByCommitment { .. } => None,
1108        };
1109
1110        // Register subscriber.
1111        match key {
1112            SubscriptionKey::Digest(digest) => {
1113                debug!(?round, ?digest, "registering subscriber");
1114            }
1115            SubscriptionKey::Commitment(commitment) => {
1116                debug!(?round, ?commitment, ?digest, "registering subscriber");
1117            }
1118        }
1119        self.block_subscriptions
1120            .insert(span, key, response, waiters, buffer);
1121    }
1122
1123    /// Verifies and installs a floor, fetching the anchor block if needed.
1124    async fn install_floor<Buf, R>(
1125        &mut self,
1126        finalization: Finalization<P::Scheme, V::Commitment>,
1127        skip_if_superseded: bool,
1128        resolver: &mut R,
1129        buffer: &mut Buf,
1130        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
1131    ) where
1132        Buf: Buffer<V, PublicKey = <P::Scheme as Verifier>::PublicKey>,
1133        R: Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
1134    {
1135        let round = finalization.round();
1136        if round <= self.floor.processed_round() {
1137            warn!(
1138                ?round,
1139                floor = ?self.floor.processed_round(),
1140                "floor not updated, below existing round floor"
1141            );
1142            return;
1143        }
1144
1145        let Some(scoped) = self.provider.scoped(finalization.epoch()) else {
1146            panic!("floor finalization epoch unavailable");
1147        };
1148        assert!(
1149            finalization.verify(self.context.as_mut(), &scoped, &self.strategy),
1150            "floor finalization must verify"
1151        );
1152
1153        let commitment = finalization.proposal.payload;
1154        let digest = V::commitment_to_inner(commitment);
1155        self.cache
1156            .put_finalization(round, digest, finalization.clone())
1157            .await;
1158
1159        // A pending anchor at the same or a newer floor already blocks
1160        // progress. Keep waiting for it instead of replacing it.
1161        if skip_if_superseded && self.floor.has_pending_anchor_at_or_after(round) {
1162            return;
1163        }
1164
1165        if let Some(block) = self.find_block_by_commitment(buffer, commitment).await {
1166            self.floor.await_anchor(finalization);
1167            assert!(self.ingest(block, buffer, application, resolver).await);
1168            return;
1169        }
1170
1171        // The pending floor owns the next application sync point. Drop any
1172        // in-flight acks before they can advance the processed height past it.
1173        self.pending_acks.clear();
1174
1175        debug!(?round, ?commitment, "starting fetch for floor block");
1176        self.floor.await_anchor(finalization);
1177        self.floor
1178            .fetch_if_permitted(
1179                resolver,
1180                Request::finalized_block_by_round(commitment, round),
1181            )
1182            .ignore();
1183    }
1184
1185    /// Ingests `block` and persists it as a verify-stage candidate for `round`,
1186    /// delivering the write's durable-sync handle through `ack`.
1187    ///
1188    /// If the round has already been pruned by tip advancement, `put_verified`
1189    /// is a no-op because the round is below the retention floor (and no longer
1190    /// is required by consensus to make progress). A duplicate delivery is also
1191    /// a no-op, with the handle still covering the original write's durability.
1192    async fn persist_verified<Buf: Buffer<V>>(
1193        &mut self,
1194        round: Round,
1195        block: Arc<V::Block>,
1196        ack: oneshot::Sender<Handle<()>>,
1197        buffer: &mut Buf,
1198        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
1199        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
1200    ) {
1201        self.ingest(Arc::clone(&block), buffer, application, resolver)
1202            .await;
1203        let digest = block.digest();
1204        let handle = self
1205            .cache
1206            .put_verified(round, digest, Arc::unwrap_or_clone(block).into())
1207            .await;
1208        ack.send_lossy(handle);
1209    }
1210
1211    /// Notifies subscribers of a validated block and applies it to any
1212    /// pending floor transition.
1213    ///
1214    /// Subscribers are notified before the block is persisted. This is not
1215    /// observable while running because mailbox requests are only served
1216    /// after the current `select_loop!` arm completes. After an unclean
1217    /// shutdown, however, a subscriber may hold a block that marshal never
1218    /// durably stored. Subscriptions make no durability promise. Durable
1219    /// height-ordered delivery is provided by application dispatch, which
1220    /// only sends blocks once the finalized archives are durable (see
1221    /// [`Self::try_dispatch_blocks`]).
1222    ///
1223    /// Returns true if the block was consumed as the floor anchor.
1224    async fn ingest<Buf: Buffer<V>>(
1225        &mut self,
1226        block: Arc<V::Block>,
1227        buffer: &mut Buf,
1228        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
1229        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
1230    ) -> bool {
1231        self.block_subscriptions.notify(Arc::clone(&block));
1232
1233        if !self.floor.matches_pending_anchor(V::commitment(&block)) {
1234            return false;
1235        }
1236
1237        self.apply_pending_floor(block, buffer, application, resolver)
1238            .await;
1239        true
1240    }
1241
1242    /// Applies the pending floor transition using its matching anchor block.
1243    ///
1244    /// # Panics
1245    ///
1246    /// Panics if no pending floor anchor is installed.
1247    async fn apply_pending_floor<Buf: Buffer<V>>(
1248        &mut self,
1249        block: Arc<V::Block>,
1250        buffer: &mut Buf,
1251        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
1252        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
1253    ) {
1254        // Floor anchors can bypass the local proposal-verification path. Check
1255        // the parent relationship before using a non-genesis anchor for walkback.
1256        let height = block.height();
1257        if height > Height::zero() {
1258            let parent_commitment = V::parent_commitment(&block);
1259            assert!(
1260                block.parent() == V::commitment_to_inner(parent_commitment),
1261                "floor block parent commitment mismatch"
1262            );
1263        }
1264
1265        // This anchor cannot move the application sync point, but its
1266        // finalization round can still prune round-bound resolver work.
1267        // Keep pending acks intact because processed_height is unchanged.
1268        if height <= self.floor.processed_height() {
1269            warn!(
1270                %height,
1271                existing = %self.floor.processed_height(),
1272                "floor not updated, at or below existing"
1273            );
1274            let finalization = self
1275                .floor
1276                .take_pending_anchor()
1277                .expect("pending floor anchor missing");
1278            self.update_processed_round_floor(height, finalization.round(), resolver)
1279                .await;
1280            if self.try_repair_gaps(buffer, resolver, application).await {
1281                self.sync_finalized().await;
1282            }
1283            self.try_dispatch_blocks(application).await;
1284            return;
1285        }
1286
1287        let digest = block.digest();
1288        let finalization = self
1289            .floor
1290            .take_pending_anchor()
1291            .expect("pending floor anchor missing");
1292        let round = finalization.round();
1293        try_join!(
1294            async {
1295                self.finalized_blocks
1296                    .put(Arc::unwrap_or_clone(block).into())
1297                    .await
1298                    .map_err(Box::new)?;
1299                Ok::<_, BoxedError>(())
1300            },
1301            async {
1302                self.finalizations_by_height
1303                    .put(height, digest, finalization)
1304                    .await
1305                    .map_err(Box::new)?;
1306                Ok::<_, BoxedError>(())
1307            }
1308        )
1309        .expect("failed to store floor anchor");
1310        self.sync_finalized().await;
1311
1312        if height > self.tip {
1313            application.report(Update::Tip(round, height, digest));
1314            self.tip = height;
1315            let _ = self.finalized_height.try_set(height.get());
1316        }
1317
1318        // The anchor is durable, but the application still needs to process it.
1319        // Record the previous height so dispatch resumes at the anchor itself.
1320        let dispatch_floor = height
1321            .previous()
1322            .expect("floor anchor above processed height must have predecessor");
1323        self.update_processed_height(dispatch_floor, resolver);
1324        self.update_processed_round_floor(dispatch_floor, round, resolver)
1325            .await;
1326        self.stream
1327            .sync()
1328            .await
1329            .expect("failed to sync floor metadata");
1330
1331        // Drop all pending acknowledgement waiters so any in-flight application
1332        // acks for blocks below the new floor cannot rewrite the processed floor.
1333        self.pending_acks.clear();
1334
1335        // The floor is durable, so cache/finalized data below it can be pruned.
1336        self.prune_after_floor(height)
1337            .await
1338            .expect("failed to prune data below floor");
1339
1340        // Intentionally keep existing block subscriptions alive. Canceling
1341        // waiters can have catastrophic consequences (nodes can get stuck in
1342        // different views) as actors do not retry subscriptions on failed channels.
1343        if self.try_repair_gaps(buffer, resolver, application).await {
1344            self.sync_finalized().await;
1345        }
1346        self.try_dispatch_blocks(application).await;
1347    }
1348
1349    /// Handle a deliver message from the resolver. Block delivers are handled
1350    /// immediately. Finalized/Notarized delivers are parsed and structurally
1351    /// validated, then collected into `delivers` for batch certificate verification.
1352    async fn handle_deliver<Buf: Buffer<V>>(
1353        &mut self,
1354        message: ResolverDelivery<V>,
1355        delivers: &mut Vec<PendingVerification<P::Scheme, V>>,
1356        buffer: &mut Buf,
1357        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
1358        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
1359    ) {
1360        let ResolverDelivery {
1361            delivery,
1362            mut value,
1363            response,
1364        } = message;
1365        let Delivery {
1366            key, subscribers, ..
1367        } = delivery;
1368        match key {
1369            Key::Block(commitment) => {
1370                let block_cfg = V::block_cfg(&self.block_codec_config, commitment);
1371                let Ok(block) = V::Block::decode_cfg(value.as_ref(), &block_cfg) else {
1372                    response.send_lossy(false);
1373                    return;
1374                };
1375                if V::commitment(&block) != commitment {
1376                    response.send_lossy(false);
1377                    return;
1378                }
1379
1380                // This block may match the pending floor request. Whether it
1381                // installs or is rejected as the floor anchor, do not also
1382                // process it as an ordinary block delivery.
1383                let block = Arc::new(block);
1384                if self
1385                    .ingest(Arc::clone(&block), buffer, application, resolver)
1386                    .await
1387                {
1388                    response.send_lossy(true);
1389                    return;
1390                }
1391
1392                // The peer-visible request only says "give me this block".
1393                // Local annotations explain why the block was requested and
1394                // therefore where, if anywhere, it should be stored.
1395                let height = block.height();
1396                let digest = block.digest();
1397                let annotations = subscribers
1398                    .map_into(|(annotation, _)| annotation)
1399                    .into_vec();
1400
1401                // Round-bound proposal-parent fetches are `Key::Notarized`
1402                // deliveries and are handled below. In this block-keyed path,
1403                // `Finalized` means the block belongs in the finalized chain.
1404                let finalization = self.cache.get_finalization_for(digest).await;
1405                if let Some(finalization) = &finalization {
1406                    self.update_processed_round_floor(height, finalization.round(), resolver)
1407                        .await;
1408                }
1409                if finalization.is_some()
1410                    || annotations
1411                        .iter()
1412                        .any(|annotation| matches!(annotation, Annotation::Finalized(_)))
1413                {
1414                    self.store_finalization(
1415                        height,
1416                        digest,
1417                        Arc::unwrap_or_clone(block),
1418                        finalization,
1419                        application,
1420                    )
1421                    .await;
1422                } else if annotations
1423                    .iter()
1424                    .any(|annotation| matches!(annotation, Annotation::Certified { .. }))
1425                    && height > self.floor.processed_height()
1426                {
1427                    if let Some(bounds) = self.epocher.containing(height) {
1428                        self.cache
1429                            .put_certified(
1430                                bounds.epoch(),
1431                                height,
1432                                digest,
1433                                Arc::unwrap_or_clone(block).into(),
1434                            )
1435                            .await;
1436                    }
1437                }
1438                debug!(?digest, %height, "received block");
1439                response.send_lossy(true);
1440            }
1441            Key::Finalized { height } => {
1442                let Some((epoch, certificate_codec_config)) =
1443                    self.certificate_codec_config_for_height(height)
1444                else {
1445                    debug!(
1446                        %height,
1447                        floor = %self.floor.processed_height(),
1448                        "ignoring stale delivery"
1449                    );
1450                    response.send_lossy(true);
1451                    return;
1452                };
1453
1454                let Ok(finalization) =
1455                    Finalization::read_cfg(&mut value, &certificate_codec_config)
1456                else {
1457                    response.send_lossy(false);
1458                    return;
1459                };
1460
1461                // We decoded the certificate with the codec config for the height's epoch, so the
1462                // finalization must claim that same epoch. A mismatch means the bytes were bounded
1463                // against the wrong participant set, so reject before verification.
1464                if finalization.epoch() != epoch {
1465                    response.send_lossy(false);
1466                    return;
1467                }
1468
1469                // Decode the block carried with the finalization. Below, it is checked against
1470                // the requested height and the finalization payload.
1471                let Ok(block) =
1472                    V::ApplicationBlock::decode_cfg(&mut value, &self.block_codec_config)
1473                else {
1474                    response.send_lossy(false);
1475                    return;
1476                };
1477
1478                // In contrast to the `Block` and `Notarization` deliveries, the finalization delivery
1479                // is guaranteed to be certified (assuming the certificate verifies). Because of this,
1480                // we can skip broader payload checks and just check that the application block matches
1481                // the commitment in the finalization proposal.
1482                //
1483                // TODO(https://github.com/commonwarexyz/monorepo/issues/3938): Apply this pattern
1484                // conditionally to `Request::Block` and `Request::Notarized`, if the requester knows
1485                // the requested block is certified.
1486                let commitment = finalization.proposal.payload;
1487                if block.height() != height || block.digest() != V::commitment_to_inner(commitment)
1488                {
1489                    response.send_lossy(false);
1490                    return;
1491                }
1492                delivers.push(PendingVerification::Finalized {
1493                    finalization,
1494                    block,
1495                    response,
1496                });
1497            }
1498            Key::Notarized { round } => {
1499                let Some(scheme) = self.provider.scheme(round.epoch()) else {
1500                    debug!(
1501                        ?round,
1502                        floor = %self.floor.processed_height(),
1503                        "ignoring stale delivery"
1504                    );
1505                    response.send_lossy(true);
1506                    return;
1507                };
1508                let certificate_codec_config = scheme.certificate_codec_config();
1509                let Ok(notarization) =
1510                    Notarization::read_cfg(&mut value, &certificate_codec_config)
1511                else {
1512                    response.send_lossy(false);
1513                    return;
1514                };
1515
1516                // The resolver key binds this response to `round`; a certificate for any other
1517                // round is a bad response even if it decodes correctly.
1518                if notarization.round() != round {
1519                    response.send_lossy(false);
1520                    return;
1521                }
1522
1523                // Use the notarization payload to derive the block decode config. Below, the
1524                // decoded block is checked against the same payload.
1525                let commitment = notarization.proposal.payload;
1526                if !V::check_payload(scheme.as_ref(), commitment) {
1527                    response.send_lossy(false);
1528                    return;
1529                }
1530                let block_cfg = V::block_cfg(&self.block_codec_config, commitment);
1531                let Ok(block) = V::Block::decode_cfg(value, &block_cfg) else {
1532                    response.send_lossy(false);
1533                    return;
1534                };
1535
1536                if V::commitment(&block) != notarization.proposal.payload {
1537                    response.send_lossy(false);
1538                    return;
1539                }
1540                delivers.push(PendingVerification::Notarized {
1541                    notarization,
1542                    block,
1543                    response,
1544                });
1545            }
1546        }
1547    }
1548
1549    /// Batch verify pending certificates and process valid items.
1550    #[tracing::instrument(name = "marshal.actor.verify_delivered", level = "info", skip_all, fields(count = delivers.len().traced()))]
1551    async fn verify_delivered<Buf: Buffer<V>>(
1552        &mut self,
1553        mut delivers: Vec<PendingVerification<P::Scheme, V>>,
1554        buffer: &mut Buf,
1555        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
1556        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
1557    ) {
1558        delivers.retain(|item| !item.response_closed());
1559        if delivers.is_empty() {
1560            return;
1561        }
1562
1563        // Extract (subject, certificate) pairs for batch verification.
1564        let certs: Vec<_> = delivers
1565            .iter()
1566            .map(|item| match item {
1567                PendingVerification::Finalized { finalization, .. } => (
1568                    Subject::Finalize {
1569                        proposal: &finalization.proposal,
1570                    },
1571                    &finalization.certificate,
1572                ),
1573                PendingVerification::Notarized { notarization, .. } => (
1574                    Subject::Notarize {
1575                        proposal: &notarization.proposal,
1576                    },
1577                    &notarization.certificate,
1578                ),
1579            })
1580            .collect();
1581
1582        // Group indices by epoch.
1583        let mut by_epoch: BTreeMap<Epoch, Vec<usize>> = BTreeMap::new();
1584        for (i, item) in delivers.iter().enumerate() {
1585            let epoch = match item {
1586                PendingVerification::Notarized { notarization, .. } => notarization.epoch(),
1587                PendingVerification::Finalized { finalization, .. } => finalization.epoch(),
1588            };
1589            by_epoch.entry(epoch).or_default().push(i);
1590        }
1591
1592        // Batch verify each epoch group.
1593        let mut verified = vec![false; delivers.len()];
1594        for (epoch, indices) in &by_epoch {
1595            let Some(scoped) = self.provider.scoped(*epoch) else {
1596                continue;
1597            };
1598            let group: Vec<_> = indices.iter().map(|&i| certs[i]).collect();
1599            let results =
1600                verify_certificates(self.context.as_mut(), &scoped, &group, &self.strategy);
1601            for (j, &idx) in indices.iter().enumerate() {
1602                verified[idx] = results[j];
1603            }
1604        }
1605
1606        // Process each verified item, rejecting unverified ones.
1607        for (index, item) in delivers.drain(..).enumerate() {
1608            if !verified[index] {
1609                match item {
1610                    PendingVerification::Finalized { response, .. }
1611                    | PendingVerification::Notarized { response, .. } => {
1612                        response.send_lossy(false);
1613                    }
1614                }
1615                continue;
1616            }
1617            match item {
1618                PendingVerification::Finalized {
1619                    finalization,
1620                    block,
1621                    response,
1622                } => {
1623                    // Valid finalization received.
1624                    response.send_lossy(true);
1625                    let block = Arc::new(V::from_application_block(
1626                        block,
1627                        finalization.proposal.payload,
1628                    ));
1629                    let round = finalization.round();
1630                    let height = block.height();
1631                    let digest = block.digest();
1632                    debug!(?round, %height, "received finalization");
1633
1634                    // The floor-anchor path fully handles this finalization
1635                    // and moves the lower bound past it.
1636                    if self
1637                        .ingest(Arc::clone(&block), buffer, application, resolver)
1638                        .await
1639                    {
1640                        continue;
1641                    }
1642
1643                    self.update_processed_round_floor(height, round, resolver)
1644                        .await;
1645
1646                    self.store_finalization(
1647                        height,
1648                        digest,
1649                        Arc::unwrap_or_clone(block),
1650                        Some(finalization),
1651                        application,
1652                    )
1653                    .await;
1654                }
1655                PendingVerification::Notarized {
1656                    notarization,
1657                    block,
1658                    response,
1659                } => {
1660                    // Valid notarization received.
1661                    response.send_lossy(true);
1662                    let round = notarization.round();
1663                    let commitment = notarization.proposal.payload;
1664                    let digest = V::commitment_to_inner(commitment);
1665                    debug!(?round, ?digest, "received notarization");
1666
1667                    // Cache the notarization and block, blocking until both are
1668                    // durable (or the runtime is shutting down) so the repair
1669                    // bookkeeping below never runs ahead of storage.
1670                    let height = block.height();
1671                    let block = Arc::new(block);
1672                    let block_sync = self
1673                        .cache
1674                        .put_notarized(round, digest, block.as_ref().clone().into())
1675                        .await;
1676                    let notarization_sync = self
1677                        .cache
1678                        .put_notarization(round, digest, notarization)
1679                        .await;
1680                    join(
1681                        block_sync.durable(round, "notarized"),
1682                        notarization_sync.durable(round, "notarization"),
1683                    )
1684                    .await;
1685
1686                    // A notarized delivery can carry the pending floor block
1687                    // after the finalization is cached.
1688                    if self
1689                        .ingest(Arc::clone(&block), buffer, application, resolver)
1690                        .await
1691                    {
1692                        continue;
1693                    }
1694
1695                    // If there exists a finalization certificate for this block, we
1696                    // should finalize it. This could finalize the block faster when
1697                    // a notarization then a finalization are received via consensus
1698                    // and we resolve the notarization request before the block request.
1699                    if let Some(finalization) = self.cache.get_finalization_for(digest).await {
1700                        self.update_processed_round_floor(height, finalization.round(), resolver)
1701                            .await;
1702
1703                        // SAFETY: `digest` identifies a unique `commitment`, so this
1704                        // cached finalization payload must match `V::commitment(&block)`.
1705                        self.store_finalization(
1706                            height,
1707                            digest,
1708                            Arc::unwrap_or_clone(block),
1709                            Some(finalization),
1710                            application,
1711                        )
1712                        .await;
1713                    }
1714                }
1715            }
1716        }
1717    }
1718
1719    /// Returns the certificate codec config for `epoch`.
1720    fn certificate_codec_config(
1721        &self,
1722        epoch: Epoch,
1723    ) -> Option<<<P::Scheme as Verifier>::Certificate as Read>::Cfg> {
1724        self.provider
1725            .scoped(epoch)
1726            .map(|scoped| scoped.certificate_codec_config())
1727    }
1728
1729    /// Returns the epoch containing `height` and its certificate codec config.
1730    fn certificate_codec_config_for_height(
1731        &self,
1732        height: Height,
1733    ) -> Option<(Epoch, <<P::Scheme as Verifier>::Certificate as Read>::Cfg)> {
1734        let epoch = self.epocher.containing(height)?.epoch();
1735        self.certificate_codec_config(epoch)
1736            .map(|config| (epoch, config))
1737    }
1738
1739    // -------------------- Application Dispatch --------------------
1740
1741    /// Attempt to dispatch the next finalized block to the application if ready.
1742    ///
1743    /// Dispatch finalized blocks to the application until the pipeline is full
1744    /// or no more blocks are available.
1745    ///
1746    /// This does NOT advance the processed floor height or sync metadata. It only
1747    /// sends blocks to the application and enqueues pending acks. Metadata is
1748    /// updated later, in a subsequent `select_loop!` iteration, when the ack
1749    /// handler updates the processed height.
1750    ///
1751    /// Blocks are dispatched only once durable. Every buffered
1752    /// finalized-archive write freezes dispatch at or above its height until
1753    /// a sync covering it completes (see [`DispatchGate`]). Dispatch is
1754    /// re-attempted by the pool-completion arm for pooled syncs and by the
1755    /// caller itself after a blocking sync. Callers that buffer writes must
1756    /// still call [`Self::sync_finalized`] or [`Self::start_finalized_sync`]
1757    /// before yielding to the `select_loop!` so the freeze is released.
1758    ///
1759    /// Acks are processed in FIFO order so the processed floor height always
1760    /// advances sequentially.
1761    ///
1762    /// # Crash safety
1763    ///
1764    /// Because `select_loop!` arms run to completion, archive data is always
1765    /// durable before the ack handler advances the processed floor height:
1766    ///
1767    /// ```text
1768    /// Iteration N (caller):
1769    ///   store_finalization   ->  Archive::put (buffered)
1770    ///   sync_finalized       ->  archive durable
1771    ///   try_dispatch_blocks  ->  sends durable blocks to app, enqueues pending acks
1772    ///
1773    /// Iteration M (ack handler, M > N):
1774    ///   ack handler       ->  update_processed_height  ->  metadata buffered
1775    ///   stream.sync       ->  metadata durable
1776    /// ```
1777    async fn try_dispatch_blocks(
1778        &mut self,
1779        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
1780    ) {
1781        // Dispatch resumes after the floor anchor is durably stored.
1782        if self.floor.blocks_progress() {
1783            return;
1784        }
1785
1786        // Durability barrier: buffered writes are readable from the archives
1787        // before they are durable. Never dispatch at or above the lowest
1788        // write not yet covered by a completed sync.
1789        let barrier = self.dispatch_gate.barrier();
1790        while self.pending_acks.has_capacity() {
1791            let next_height = self
1792                .pending_acks
1793                .next_dispatch_height(self.stream.next_height());
1794            if barrier.is_some_and(|lowest| next_height >= lowest) {
1795                return;
1796            }
1797            let Some(block) = self.get_finalized_block(next_height).await else {
1798                return;
1799            };
1800            assert_eq!(
1801                block.height(),
1802                next_height,
1803                "finalized block height mismatch"
1804            );
1805
1806            let (height, commitment) = (block.height(), V::commitment(&block));
1807            let (ack, ack_waiter) = A::handle();
1808            application.report(Update::Block(V::owned_into_inner_shared(block), ack));
1809            self.pending_acks.enqueue(PendingAck {
1810                height,
1811                commitment,
1812                receiver: ack_waiter,
1813            });
1814        }
1815    }
1816
1817    // -------------------- Prunable Storage --------------------
1818
1819    /// Sync both finalization archives to durable storage, blocking the actor
1820    /// until they are durable.
1821    ///
1822    /// Must be called within the same `select_loop!` arm as any preceding
1823    /// [`Self::store_finalization`] / [`Self::try_repair_gaps`] writes, before yielding back
1824    /// to the loop. This is the durability barrier for application delivery:
1825    /// [`Self::try_dispatch_blocks`] must run only after this sync completes.
1826    /// It also ensures archives are durable before the ack handler advances
1827    /// the processed floor height. See [`Self::try_dispatch_blocks`] for details.
1828    ///
1829    /// Blocking the actor stalls every mailbox caller behind the sync.
1830    /// Prefer [`Self::start_finalized_sync`] unless work later in the same
1831    /// arm requires the writes to already be durable.
1832    #[tracing::instrument(name = "marshal.actor.sync_finalized", level = "info", skip_all)]
1833    async fn sync_finalized(&mut self) {
1834        if let Err(e) = try_join!(
1835            async {
1836                self.finalized_blocks.sync().await.map_err(Box::new)?;
1837                Ok::<_, BoxedError>(())
1838            },
1839            async {
1840                self.finalizations_by_height
1841                    .sync()
1842                    .await
1843                    .map_err(Box::new)?;
1844                Ok::<_, BoxedError>(())
1845            },
1846        ) {
1847            panic!("failed to sync finalization archives: {e}");
1848        }
1849
1850        // Everything accepted before this sync is now durable, so nothing
1851        // remains to gate dispatch.
1852        self.dispatch_gate.clear();
1853    }
1854
1855    /// Start a non-blocking sync of both finalization archives on the
1856    /// durability pool. A no-op if nothing was written since the last sync
1857    /// (blocking or pooled) started.
1858    ///
1859    /// The pooled entry resolves to [`PooledSync::Finalized`] once every write
1860    /// accepted before this call is durable. The sync adopts every deferred
1861    /// write (see [`DispatchGate::adopt`]), and until the pool-completion arm
1862    /// observes the completion, [`Self::try_dispatch_blocks`] will not
1863    /// dispatch at or above the lowest height a pending batch wrote. This
1864    /// preserves the durability barrier described there without blocking the
1865    /// mailbox on a sync like [`Self::sync_finalized`].
1866    ///
1867    /// Like [`Self::sync_finalized`], this must be called within the same
1868    /// `select_loop!` arm as the writes it covers, before yielding back to the
1869    /// loop. `round` only labels the sync in diagnostics.
1870    #[tracing::instrument(name = "marshal.actor.start_finalized_sync", level = "info", skip_all)]
1871    async fn start_finalized_sync(&mut self, round: Round, syncs: &mut Pool<PooledSync>) {
1872        // If no write needs syncing, every accepted write is already covered
1873        // by a blocking or in-flight sync.
1874        let Some(seq) = self.dispatch_gate.adopt() else {
1875            return;
1876        };
1877
1878        let (blocks, finalizations) = match try_join!(
1879            async {
1880                let handle = self.finalized_blocks.start_sync().await.map_err(Box::new)?;
1881                Ok::<_, BoxedError>(handle)
1882            },
1883            async {
1884                let handle = self
1885                    .finalizations_by_height
1886                    .start_sync()
1887                    .await
1888                    .map_err(Box::new)?;
1889                Ok::<_, BoxedError>(handle)
1890            },
1891        ) {
1892            Ok(handles) => handles,
1893            Err(e) => panic!("failed to start finalization archive sync: {e}"),
1894        };
1895        syncs.push(async move {
1896            let (blocks, finalizations) = join(
1897                blocks.durable(round, "finalized blocks"),
1898                finalizations.durable(round, "finalizations"),
1899            )
1900            .await;
1901            if blocks && finalizations {
1902                PooledSync::Finalized(seq)
1903            } else {
1904                // Runtime shutdown before the sync completed: nothing may be
1905                // released for dispatch.
1906                PooledSync::Observed
1907            }
1908        });
1909    }
1910
1911    // -------------------- Immutable Storage --------------------
1912
1913    /// Get a finalized block from the immutable archive.
1914    async fn get_finalized_block(&self, height: Height) -> Option<V::Block> {
1915        match self
1916            .finalized_blocks
1917            .get(ArchiveID::Index(height.get()))
1918            .await
1919        {
1920            Ok(stored) => stored.map(|stored| stored.into()),
1921            Err(e) => panic!("failed to get block: {e}"),
1922        }
1923    }
1924
1925    /// Get a finalization from the archive by height.
1926    async fn get_finalization_by_height(
1927        &self,
1928        height: Height,
1929    ) -> Option<Finalization<P::Scheme, V::Commitment>> {
1930        match self
1931            .finalizations_by_height
1932            .get(ArchiveID::Index(height.get()))
1933            .await
1934        {
1935            Ok(finalization) => finalization,
1936            Err(e) => panic!("failed to get finalization: {e}"),
1937        }
1938    }
1939
1940    /// Check whether a finalization exists in the archive at `height` without
1941    /// fetching it.
1942    async fn has_finalization_by_height(&self, height: Height) -> bool {
1943        match self.finalizations_by_height.has(height).await {
1944            Ok(has) => has,
1945            Err(e) => panic!("failed to check finalization: {e}"),
1946        }
1947    }
1948
1949    /// Get finalized block information from either the finalization archive or
1950    /// the finalized-block archive.
1951    async fn get_info_by_height(
1952        &self,
1953        height: Height,
1954    ) -> Option<(Height, <V::Block as Digestible>::Digest)> {
1955        if let Some(finalization) = self.get_finalization_by_height(height).await {
1956            return Some((
1957                height,
1958                V::commitment_to_inner(finalization.proposal.payload),
1959            ));
1960        }
1961
1962        self.get_finalized_block(height)
1963            .await
1964            .map(|block| (block.height(), block.digest()))
1965    }
1966
1967    /// Add a finalized block, and optionally a finalization, to the archive.
1968    ///
1969    /// After persisting the block, the caller must sync finalized archives
1970    /// before dispatching the next contiguous block to the application. The
1971    /// buffered archive writes from this method are not a sufficient durability
1972    /// guarantee for downstream application state transitions on their own.
1973    ///
1974    /// Writes are buffered and not synced. The caller must call
1975    /// [sync_finalized](Self::sync_finalized) (blocking) or
1976    /// [start_finalized_sync](Self::start_finalized_sync) (pooled) before
1977    /// yielding to the `select_loop!` so that archive data is durable before
1978    /// the ack handler advances the processed floor height. See
1979    /// [`Self::try_dispatch_blocks`] for the crash safety invariant.
1980    async fn store_finalization(
1981        &mut self,
1982        height: Height,
1983        digest: <V::Block as Digestible>::Digest,
1984        block: V::Block,
1985        finalization: Option<Finalization<P::Scheme, V::Commitment>>,
1986        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
1987    ) -> bool {
1988        // Blocks below the last processed height are not useful to us, so we ignore them (this
1989        // has the nice byproduct of ensuring we don't call a backing store with a block below the
1990        // pruning boundary)
1991        if height <= self.floor.processed_height() {
1992            debug!(
1993                %height,
1994                floor = %self.floor.processed_height(),
1995                ?digest,
1996                "dropping finalization at or below processed height floor"
1997            );
1998            return false;
1999        }
2000
2001        // Convert block to storage format
2002        let stored: V::StoredBlock = block.into();
2003        let round = finalization.as_ref().map(|f| f.round());
2004
2005        // In parallel, update the finalized blocks and finalizations archives
2006        if let Err(e) = try_join!(
2007            // Update the finalized blocks archive
2008            async {
2009                self.finalized_blocks.put(stored).await.map_err(Box::new)?;
2010                Ok::<_, BoxedError>(())
2011            },
2012            // Update the finalizations archive (if provided)
2013            async {
2014                if let Some(finalization) = finalization {
2015                    self.finalizations_by_height
2016                        .put(height, digest, finalization)
2017                        .await
2018                        .map_err(Box::new)?;
2019                }
2020                Ok::<_, BoxedError>(())
2021            }
2022        ) {
2023            panic!("failed to finalize: {e}");
2024        }
2025
2026        // The write above is buffered and readable before it is durable, so
2027        // hold dispatch at or above it until a sync covers it.
2028        self.dispatch_gate.defer(height);
2029
2030        // Update metrics and application
2031        if let Some(round) = round.filter(|_| height > self.tip) {
2032            application.report(Update::Tip(round, height, digest));
2033            self.tip = height;
2034            let _ = self.finalized_height.try_set(height.get());
2035        }
2036
2037        true
2038    }
2039
2040    /// Get the latest finalized block information (height and digest tuple).
2041    ///
2042    /// Blocks are only finalized directly with a finalization or indirectly via a descendant
2043    /// block's finalization. Thus, the highest known finalized block must itself have a direct
2044    /// finalization.
2045    ///
2046    /// We return the height and digest using the highest known finalization that we know the
2047    /// block height for. While it's possible that we have a later finalization, if we do not have
2048    /// the full block for that finalization, we do not know its height and therefore it would not
2049    /// yet be found in the `finalizations_by_height` archive. While not checked explicitly, we
2050    /// should have the associated block (in the `finalized_blocks` archive) for the information
2051    /// returned.
2052    async fn get_latest(&mut self) -> Option<(Height, <V::Block as Digestible>::Digest, Round)> {
2053        let height = self.finalizations_by_height.last_index()?;
2054        let finalization = self
2055            .get_finalization_by_height(height)
2056            .await
2057            .expect("finalization missing");
2058        Some((
2059            height,
2060            V::commitment_to_inner(finalization.proposal.payload),
2061            finalization.round(),
2062        ))
2063    }
2064
2065    // -------------------- Mixed Storage --------------------
2066
2067    /// Looks for a block in cache and finalized storage by digest.
2068    async fn find_block_in_storage(
2069        &self,
2070        digest: <V::Block as Digestible>::Digest,
2071    ) -> Option<V::Block> {
2072        // Check verified / notarized blocks via cache manager.
2073        if let Some(block) = self.cache.find_block_matching(digest, |_| true).await {
2074            return Some(block.into());
2075        }
2076        // Check finalized blocks.
2077        match self.finalized_blocks.get(ArchiveID::Key(&digest)).await {
2078            Ok(stored) => stored.map(|stored| stored.into()),
2079            Err(e) => panic!("failed to get block: {e}"),
2080        }
2081    }
2082
2083    /// Looks for a block in cache and finalized storage by full consensus commitment.
2084    async fn find_block_in_storage_by_commitment(
2085        &self,
2086        commitment: V::Commitment,
2087    ) -> Option<V::Block> {
2088        let digest = V::commitment_to_inner(commitment);
2089        if let Some(block) = self
2090            .cache
2091            .find_block_matching(digest, |stored| V::stored_commitment(stored) == commitment)
2092            .await
2093        {
2094            return Some(block.into());
2095        }
2096
2097        match self.finalized_blocks.get(ArchiveID::Key(&digest)).await {
2098            Ok(Some(stored)) => {
2099                (V::stored_commitment(&stored) == commitment).then(|| stored.into())
2100            }
2101            Ok(None) => None,
2102            Err(e) => panic!("failed to get block: {e}"),
2103        }
2104    }
2105
2106    /// Looks for a block anywhere in local storage using only the digest.
2107    ///
2108    /// This is used when we only have a digest (during gap repair following
2109    /// parent links).
2110    async fn find_block_by_digest<Buf: Buffer<V>>(
2111        &self,
2112        buffer: &Buf,
2113        digest: <V::Block as Digestible>::Digest,
2114    ) -> Option<Arc<V::Block>> {
2115        if let Some(block) = buffer.find_by_digest(digest).await {
2116            return Some(block);
2117        }
2118        self.find_block_in_storage(digest).await.map(Arc::new)
2119    }
2120
2121    /// Looks for a block anywhere in local storage using the full commitment.
2122    ///
2123    /// This is used when we have a full commitment (from notarizations/finalizations).
2124    /// Having the full commitment may enable additional retrieval mechanisms.
2125    async fn find_block_by_commitment<Buf: Buffer<V>>(
2126        &self,
2127        buffer: &Buf,
2128        commitment: V::Commitment,
2129    ) -> Option<Arc<V::Block>> {
2130        if let Some(block) = buffer.find_by_commitment(commitment).await {
2131            return Some(block);
2132        }
2133        self.find_block_in_storage_by_commitment(commitment)
2134            .await
2135            .map(Arc::new)
2136    }
2137
2138    /// Attempt to repair any identified gaps in the finalized blocks archive. The total
2139    /// number of missing heights that can be repaired at once is bounded by `self.max_repair`,
2140    /// though multiple gaps may be spanned.
2141    ///
2142    /// This also handles the "trailing" case where finalizations exist beyond
2143    /// the last stored block (the block data was lost before a crash). The
2144    /// trailing block is anchored first so that backward gap repair can fill
2145    /// inward from it.
2146    ///
2147    /// Writes are buffered. Returns `true` if this call wrote repaired blocks and
2148    /// needs a subsequent [`sync_finalized`](Self::sync_finalized).
2149    #[tracing::instrument(name = "marshal.actor.try_repair_gaps", level = "info", skip_all)]
2150    async fn try_repair_gaps<Buf: Buffer<V>>(
2151        &mut self,
2152        buffer: &mut Buf,
2153        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
2154        application: &mut impl Reporter<Activity = Update<V::ApplicationBlock, A>>,
2155    ) -> bool {
2156        // Gap repair needs a known processed floor. A floor transition may
2157        // jump the lower bound once its anchor block arrives.
2158        if self.floor.blocks_progress() {
2159            return false;
2160        }
2161
2162        let mut wrote = false;
2163        let start = self.floor.processed_height().next();
2164
2165        // If finalizations extend beyond the last stored block, anchor the
2166        // trailing block so the gap repair loop below can walk backward from it.
2167        if let Some(last_finalized) = self.finalizations_by_height.last_index() {
2168            let have_block = self
2169                .finalized_blocks
2170                .last_index()
2171                .is_some_and(|last| last >= last_finalized);
2172            if last_finalized > self.floor.processed_height() && !have_block {
2173                // Get the finalization for the last finalized block.
2174                let finalization = self
2175                    .get_finalization_by_height(last_finalized)
2176                    .await
2177                    .expect("finalization missing");
2178                let commitment = finalization.proposal.payload;
2179                if let Some(block) = self.find_block_by_commitment(buffer, commitment).await {
2180                    // If found, persist the block.
2181                    let digest = block.digest();
2182                    wrote |= self
2183                        .store_finalization(
2184                            last_finalized,
2185                            digest,
2186                            Arc::unwrap_or_clone(block),
2187                            Some(finalization),
2188                            application,
2189                        )
2190                        .await;
2191                } else {
2192                    // Request the missing block.
2193                    self.floor
2194                        .fetch_if_permitted(
2195                            resolver,
2196                            Request::finalized_block_by_height(commitment, last_finalized),
2197                        )
2198                        .ignore();
2199                }
2200            }
2201        }
2202
2203        // Fill internal gaps by walking backward from each gap's end block.
2204        'cache_repair: loop {
2205            let (gap_start, Some(gap_end)) = self.finalized_blocks.next_gap(start) else {
2206                // No gaps detected
2207                return wrote;
2208            };
2209
2210            // Attempt to repair the gap backwards from the end of the gap, using
2211            // blocks from our local storage. The walkback only needs each
2212            // block's height and parent linkage.
2213            let Some(cursor) = self.get_finalized_block(gap_end).await else {
2214                panic!("gapped block missing that should exist: {gap_end}");
2215            };
2216            let (mut height, mut parent_digest, mut parent_commitment) = (
2217                cursor.height(),
2218                cursor.parent(),
2219                V::parent_commitment(&cursor),
2220            );
2221
2222            // Compute the lower bound of the recursive repair. `gap_start` is `Some`
2223            // if `start` is not in a gap. We add one to it to ensure we don't
2224            // re-persist it to the database in the repair loop below.
2225            let gap_start = gap_start.map(Height::next).unwrap_or(start);
2226
2227            // Iterate backwards, repairing blocks as we go.
2228            while height > gap_start {
2229                if let Some(block) = self
2230                    .find_block_by_commitment(buffer, parent_commitment)
2231                    .await
2232                {
2233                    let finalization = self.cache.get_finalization_for(parent_digest).await;
2234                    let next = (block.height(), block.parent(), V::parent_commitment(&block));
2235                    wrote |= self
2236                        .store_finalization(
2237                            next.0,
2238                            parent_digest,
2239                            Arc::unwrap_or_clone(block),
2240                            finalization,
2241                            application,
2242                        )
2243                        .await;
2244                    debug!(height = %next.0, "repaired block");
2245                    (height, parent_digest, parent_commitment) = next;
2246                } else {
2247                    // Request the next missing commitment.
2248                    //
2249                    // SAFETY: Finalized blocks are archived only after the
2250                    // parent relationship needed for walkback has been
2251                    // validated by marshal.
2252                    let parent_height = height
2253                        .previous()
2254                        .expect("cursor above gap start has a parent");
2255                    self.floor
2256                        .fetch_if_permitted(
2257                            resolver,
2258                            Request::finalized_block_by_height(parent_commitment, parent_height),
2259                        )
2260                        .ignore();
2261                    break 'cache_repair;
2262                }
2263            }
2264        }
2265
2266        // Request any finalizations for missing items in the archive, up to
2267        // the `max_repair` quota. This may help shrink the size of the gap
2268        // closest to the application's processed height if finalizations
2269        // for the requests' heights exist. If not, we rely on the recursive
2270        // digest fetches above.
2271        let missing_items = self
2272            .finalized_blocks
2273            .missing_items(start, self.max_repair.get());
2274        let requests: Vec<_> = missing_items.into_iter().map(Request::finalized).collect();
2275        if !requests.is_empty() {
2276            self.floor
2277                .fetch_all_if_permitted(resolver, requests)
2278                .ignore();
2279        }
2280        wrote
2281    }
2282
2283    /// Buffers a processed height update in memory and metrics. Does NOT sync
2284    /// to durable storage. Sync metadata after buffered updates to make them durable.
2285    fn update_processed_height(
2286        &mut self,
2287        height: Height,
2288        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
2289    ) {
2290        self.stream.acknowledge(height);
2291        self.floor.set_processed_height(height);
2292        let _ = self
2293            .processed_height
2294            .try_set(self.floor.processed_height().get());
2295
2296        // Prune any existing requests below the new floor.
2297        resolver.retain(handler::above_height_floor::<V::Commitment>(height));
2298    }
2299
2300    /// Returns the latest known finalization round at or below the processed height.
2301    async fn latest_processed_round(finalizations_by_height: &FC, height: Option<Height>) -> Round {
2302        let Some(height) = height else {
2303            return Round::zero();
2304        };
2305        let Some(finalization_height) = finalizations_by_height
2306            .ranges_from(Height::zero())
2307            .filter_map(|(start, end)| (start <= height).then_some(end.min(height)))
2308            .max()
2309        else {
2310            return Round::zero();
2311        };
2312
2313        match finalizations_by_height
2314            .get(ArchiveID::Index(finalization_height.get()))
2315            .await
2316        {
2317            Ok(Some(finalization)) => finalization.round(),
2318            Ok(None) => panic!("processed finalization missing from stored range"),
2319            Err(err) => panic!("failed to get processed finalization: {err}"),
2320        }
2321    }
2322
2323    /// Buffers a processed round update in memory and prunes round-bound requests.
2324    async fn update_processed_round(
2325        &mut self,
2326        height: Height,
2327        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
2328    ) {
2329        let Some(finalization) = self.get_finalization_by_height(height).await else {
2330            return;
2331        };
2332        self.update_processed_round_floor(height, finalization.round(), resolver)
2333            .await;
2334    }
2335
2336    /// Buffers a processed round floor update in memory and prunes round-bound requests.
2337    async fn update_processed_round_floor(
2338        &mut self,
2339        height: Height,
2340        round: Round,
2341        resolver: &mut impl Resolver<Key = ResolverRequestFor<V>, Subscriber = Annotation>,
2342    ) {
2343        if height > self.floor.processed_height() || round <= self.floor.processed_round() {
2344            return;
2345        }
2346
2347        let previous = self.floor.processed_round();
2348        self.floor.set_processed_round(round);
2349
2350        // Retain view-indexed cache data for a window behind the previously
2351        // processed finalized block.
2352        let prune_round = Round::new(
2353            previous.epoch(),
2354            previous.view().saturating_sub(self.view_retention_timeout),
2355        );
2356        self.cache.prune_by_view(prune_round).await;
2357
2358        // Prune round-bound requests at or below the processed round.
2359        resolver.retain(handler::above_round_floor::<V::Commitment>(
2360            self.floor.processed_round(),
2361        ));
2362    }
2363
2364    /// Prunes finalized blocks and certificates below the given height.
2365    async fn prune_finalized_archives(&mut self, height: Height) -> Result<(), BoxedError> {
2366        // Prune the finalized block and finalization certificate archives in parallel.
2367        try_join!(
2368            async {
2369                self.finalized_blocks
2370                    .prune(height)
2371                    .await
2372                    .map_err(Box::new)?;
2373                Ok::<_, BoxedError>(())
2374            },
2375            async {
2376                self.finalizations_by_height
2377                    .prune(height)
2378                    .await
2379                    .map_err(Box::new)?;
2380                Ok::<_, BoxedError>(())
2381            }
2382        )?;
2383        Ok(())
2384    }
2385
2386    /// Prunes finalized archives and height-indexed certified cache data below the durable floor.
2387    async fn prune_after_floor(&mut self, height: Height) -> Result<(), BoxedError> {
2388        let cache = &mut self.cache;
2389        let finalized_blocks = &mut self.finalized_blocks;
2390        let finalizations_by_height = &mut self.finalizations_by_height;
2391        try_join!(
2392            async {
2393                cache.prune_by_height(height).await;
2394                Ok::<_, BoxedError>(())
2395            },
2396            async {
2397                finalized_blocks.prune(height).await.map_err(Box::new)?;
2398                Ok::<_, BoxedError>(())
2399            },
2400            async {
2401                finalizations_by_height
2402                    .prune(height)
2403                    .await
2404                    .map_err(Box::new)?;
2405                Ok::<_, BoxedError>(())
2406            }
2407        )?;
2408        Ok(())
2409    }
2410}