Skip to main content

snarkos_node_bft/
primary.rs

1// Copyright (c) 2019-2026 Provable Inc.
2// This file is part of the snarkOS library.
3
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at:
7
8// http://www.apache.org/licenses/LICENSE-2.0
9
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15
16mod proposal_task;
17pub use proposal_task::ProposalTask;
18
19use crate::{
20    Gateway,
21    MAX_BATCH_DELAY,
22    MAX_LEADER_CERTIFICATE_DELAY,
23    MAX_WORKERS,
24    MIN_BATCH_DELAY,
25    PRIMARY_PING_INTERVAL,
26    Sync,
27    Transport,
28    WORKER_PING_INTERVAL,
29    Worker,
30    events::{BatchPropose, BatchSignature, Event},
31    helpers::{
32        PrimaryReceiver,
33        PrimarySender,
34        Proposal,
35        ProposalCache,
36        SignedProposals,
37        Storage,
38        assign_to_worker,
39        assign_to_workers,
40        fmt_id,
41        init_sync_channels,
42        init_worker_channels,
43        now,
44    },
45    spawn_blocking,
46    sync::SyncCallback,
47};
48
49use snarkos_account::Account;
50use snarkos_node_bft_events::PrimaryPing;
51use snarkos_node_bft_ledger_service::{LedgerService, deserialize_transaction_strict};
52#[cfg(test)]
53use snarkos_node_network::ConnectionMode;
54use snarkos_node_network::PeerPoolHandling;
55use snarkos_node_sync::{BlockSync, DUMMY_SELF_IP, Ping};
56use snarkos_utilities::{CallbackHandle, NodeDataDir};
57
58use snarkvm::{
59    console::{
60        prelude::*,
61        types::{Address, Field},
62    },
63    ledger::{
64        block::Transaction,
65        narwhal::{BatchCertificate, BatchHeader, Data, Transmission, TransmissionID},
66        puzzle::{Solution, SolutionID},
67    },
68    prelude::{Signature, committee::Committee},
69    utilities::flatten_error,
70};
71
72use anyhow::Context;
73use colored::Colorize;
74use futures::stream::{FuturesUnordered, StreamExt};
75use indexmap::{IndexMap, IndexSet};
76#[cfg(feature = "locktick")]
77use locktick::{
78    parking_lot::{Mutex, RwLock},
79    tokio::RwLock as TRwLock,
80};
81#[cfg(not(feature = "locktick"))]
82use parking_lot::{Mutex, RwLock};
83#[cfg(not(feature = "serial"))]
84use rayon::prelude::*;
85use std::{
86    collections::{HashMap, HashSet},
87    future::Future,
88    net::SocketAddr,
89    pin::Pin,
90    sync::{Arc, OnceLock},
91    time::Instant,
92};
93#[cfg(not(feature = "locktick"))]
94use tokio::sync::RwLock as TRwLock;
95use tokio::{sync::Notify, task::JoinHandle};
96
97/// The state of the primary's batch proposal.
98#[derive(Debug, PartialEq, Eq)]
99pub enum ProposedBatchState<N: Network> {
100    /// No batch is currently being proposed.
101    None,
102    /// A batch is being proposed and awaiting signatures.
103    Certifying(Box<Proposal<N>>),
104    /// A batch has reached quorum and is being inserted into storage.
105    /// Carries the batch ID so late-arriving signatures can be recognized and silently dropped.
106    Certified(Field<N>),
107}
108
109impl<N: Network> Default for ProposedBatchState<N> {
110    fn default() -> Self {
111        Self::None
112    }
113}
114
115impl<N: Network> ProposedBatchState<N> {
116    /// Returns `true` if the primary has no active batch proposal.
117    pub fn is_none(&self) -> bool {
118        matches!(self, Self::None)
119    }
120
121    /// Returns `true` if a batch is currently being proposed (awaiting signatures).
122    pub fn is_proposed(&self) -> bool {
123        matches!(self, Self::Certifying(_))
124    }
125
126    /// Returns a reference to the in-progress proposal, or `None` if not in the `Certifying` state.
127    pub fn as_proposal(&self) -> Option<&Proposal<N>> {
128        match self {
129            Self::Certifying(p) => Some(p.as_ref()),
130            _ => None,
131        }
132    }
133}
134
135/// A helper type to keep track of the state of the primary's batch proposal.
136pub type ProposedBatch<N> = RwLock<ProposedBatchState<N>>;
137
138/// This callback trait allows listening to changes in the Primary, such as round advancement.
139/// This is implemented by [`BFT`].
140#[async_trait::async_trait]
141pub trait PrimaryCallback<N: Network>: Send + std::marker::Sync {
142    /// Asks the callback to if we can move to the next round.
143    ///
144    /// # Arguments
145    /// * `current_round` - the round the Primary is in (to avoid race conditions)
146    ///
147    /// # Returns
148    /// `true` if we moved to the next round.
149    fn try_advance_to_next_round(&self, current_round: u64) -> bool;
150
151    /// Add a certificated that was created by the primary or received from a peer.
152    async fn add_new_certificate(&self, certificate: BatchCertificate<N>) -> Result<()>;
153}
154
155/// The primary logic of a node.
156/// AleoBFT adopts a primary-worker architecture as described in the Narwhal and Tusk paper (Section 4.2).
157#[derive(Clone)]
158pub struct Primary<N: Network> {
159    /// The sync module enables fetching data from other validators.
160    sync: Sync<N>,
161    /// The gateway allows talking to other nodes in the validator set.
162    gateway: Gateway<N>,
163    /// The storage.
164    storage: Storage<N>,
165    /// The ledger service.
166    ledger: Arc<dyn LedgerService<N>>,
167    /// The workers.
168    workers: Arc<OnceLock<Vec<Worker<N>>>>,
169
170    /// The primary callback (used by [`BFT`]).
171    primary_callback: Arc<CallbackHandle<Arc<dyn PrimaryCallback<N>>>>,
172
173    /// The batch proposal, if the primary is currently proposing a batch.
174    proposed_batch: Arc<ProposedBatch<N>>,
175
176    /// The instant at which the current batch was proposed (used to measure certification latency).
177    /// (used for higher precision in the metrics compared to the batch timestamp)
178    #[cfg(feature = "metrics")]
179    batch_propose_start: Arc<Mutex<Option<Instant>>>,
180
181    /// Holds the most recent round and timestamp that the primary proposed a batch for.
182    /// TODO(kaimast): avoiding using an async lock here, so this can be merged with the `proposed_batch`,
183    /// to have a unified `primary_state` field.
184    latest_proposal_timestamp: Arc<TRwLock<Option<(u64, i64)>>>,
185
186    /// The recently-signed batch proposals.
187    signed_proposals: Arc<RwLock<SignedProposals<N>>>,
188
189    /// The handles for all background tasks spawned by this primary.
190    handles: Arc<Mutex<Vec<JoinHandle<()>>>>,
191
192    /// The node configuration directory.
193    node_data_dir: NodeDataDir,
194
195    /// Manages proposal readiness state and drives the batch proposal loop.
196    proposal_task: ProposalTask<N>,
197
198    /// Used to wake up a the dedicated round-increment task, if we may be able to advance to the next round.
199    /// This is used, so the timeout for round advancement is reset on every round increment.
200    round_increment_notify: Arc<Notify>,
201}
202
203impl<N: Network> Primary<N> {
204    /// The maximum number of unconfirmed transmissions to send to the primary.
205    pub const MAX_TRANSMISSIONS_TOLERANCE: usize = BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH * 2;
206
207    /// Initializes a new primary instance.
208    #[allow(clippy::too_many_arguments)]
209    pub fn new(
210        account: Account<N>,
211        storage: Storage<N>,
212        ledger: Arc<dyn LedgerService<N>>,
213        block_sync: Arc<BlockSync<N>>,
214        ip: Option<SocketAddr>,
215        trusted_validators: &[SocketAddr],
216        trusted_peers_only: bool,
217        node_data_dir: NodeDataDir,
218        dev: Option<u16>,
219    ) -> Result<Self> {
220        // Initialize the gateway.
221        let gateway = Gateway::new(
222            account,
223            storage.clone(),
224            ledger.clone(),
225            ip,
226            trusted_validators,
227            trusted_peers_only,
228            node_data_dir.clone(),
229            dev,
230        )?;
231        // Initialize the sync module.
232        let sync = Sync::new(gateway.clone(), storage.clone(), ledger.clone(), block_sync);
233
234        // Initialize the primary instance.
235        Ok(Self {
236            sync,
237            gateway,
238            storage,
239            ledger,
240            node_data_dir,
241            workers: Default::default(),
242            primary_callback: Default::default(),
243            proposed_batch: Default::default(),
244            #[cfg(feature = "metrics")]
245            batch_propose_start: Default::default(),
246            latest_proposal_timestamp: Default::default(),
247            signed_proposals: Default::default(),
248            handles: Default::default(),
249            proposal_task: Default::default(),
250            round_increment_notify: Default::default(),
251        })
252    }
253
254    /// Load the proposal cache file and update the Primary state with the stored data.
255    async fn load_proposal_cache(&self) -> Result<()> {
256        // Fetch the signed proposals from the file system if it exists.
257        match ProposalCache::<N>::exists(&self.node_data_dir) {
258            // If the proposal cache exists, then process the proposal cache.
259            true => match ProposalCache::<N>::load(self.gateway.account().address(), &self.node_data_dir) {
260                Ok(proposal_cache) => {
261                    // Extract the proposal and signed proposals.
262                    let (latest_certificate_round, proposed_batch, signed_proposals, pending_certificates) =
263                        proposal_cache.into();
264
265                    *self.latest_proposal_timestamp.write().await = Some((latest_certificate_round, now()));
266                    *self.proposed_batch.write() = match proposed_batch {
267                        Some(p) => ProposedBatchState::Certifying(Box::new(p)),
268                        None => ProposedBatchState::None,
269                    };
270                    *self.signed_proposals.write() = signed_proposals;
271
272                    // Update the storage with the pending certificates.
273                    for certificate in pending_certificates {
274                        let batch_id = certificate.batch_id();
275                        // We use a dummy IP because the node should not need to request from any peers.
276                        // The storage should have stored all the transmissions. If not, we simply
277                        // skip the certificate.
278                        if let Err(err) = self.sync_with_certificate_from_peer::<true>(DUMMY_SELF_IP, certificate).await
279                        {
280                            let err = err.context(format!(
281                                "Failed to load stored certificate {} from proposal cache",
282                                fmt_id(batch_id)
283                            ));
284                            warn!("{}", &flatten_error(err));
285                        }
286                    }
287                    Ok(())
288                }
289                Err(err) => Err(err.context("Failed to read the signed proposals from the file system")),
290            },
291            // If the proposal cache does not exist, then return early.
292            false => Ok(()),
293        }
294    }
295
296    /// Run the primary instance.
297    pub async fn run(
298        &self,
299        ping: Option<Arc<Ping<N>>>,
300        primary_callback: Option<Arc<dyn PrimaryCallback<N>>>,
301        sync_callback: Option<Arc<dyn SyncCallback<N>>>,
302        primary_sender: PrimarySender<N>,
303        primary_receiver: PrimaryReceiver<N>,
304    ) -> Result<()> {
305        info!("Starting the primary instance of the memory pool...");
306
307        // Set the BFT sender.
308        if let Some(callback) = primary_callback {
309            self.primary_callback.set(callback)?;
310        }
311
312        // Construct a map of the worker senders.
313        let mut worker_senders = IndexMap::new();
314        // Construct a map for the workers.
315        let mut workers = Vec::new();
316        // Initialize the workers.
317        for id in 0..MAX_WORKERS {
318            // Construct the worker channels.
319            let (tx_worker, rx_worker) = init_worker_channels();
320            // Construct the worker instance.
321            let worker = Worker::new(
322                id,
323                Arc::new(self.gateway.clone()),
324                self.storage.clone(),
325                self.ledger.clone(),
326                self.proposed_batch.clone(),
327            )?;
328            // Run the worker instance.
329            worker.run(rx_worker);
330            // Add the worker to the list of workers.
331            workers.push(worker);
332            // Add the worker sender to the map.
333            worker_senders.insert(id, tx_worker);
334        }
335        // Set the workers.
336        if self.workers.set(workers).is_err() {
337            bail!("Workers already set. `Primary::run` cannot be called more than once.");
338        }
339
340        // First, initialize the sync channels.
341        let (sync_sender, sync_receiver) = init_sync_channels();
342        // Next, initialize the sync module and sync the storage from ledger.
343        self.sync.initialize(sync_callback)?;
344        // Next, load and process the proposal cache before running the sync module.
345        self.load_proposal_cache().await?;
346        // Next, run the sync module.
347        self.sync.run(ping, sync_receiver).await?;
348        // Next, initialize the gateway.
349        self.gateway.run(primary_sender, worker_senders, Some(sync_sender)).await;
350        // Lastly, start the primary handlers.
351        // Note: This ensures the primary does not start communicating before syncing is complete.
352        self.start_handlers(primary_receiver);
353
354        Ok(())
355    }
356
357    /// Returns the current round.
358    pub fn current_round(&self) -> u64 {
359        self.storage.current_round()
360    }
361
362    /// Returns `true` if the primary is synced.
363    pub fn is_synced(&self) -> bool {
364        self.sync.is_synced()
365    }
366
367    /// Returns the gateway.
368    pub const fn gateway(&self) -> &Gateway<N> {
369        &self.gateway
370    }
371
372    /// Returns the storage.
373    pub const fn storage(&self) -> &Storage<N> {
374        &self.storage
375    }
376
377    /// Returns the ledger.
378    pub const fn ledger(&self) -> &Arc<dyn LedgerService<N>> {
379        &self.ledger
380    }
381
382    /// Returns the number of workers.
383    pub fn num_workers(&self) -> u8 {
384        u8::try_from(self.workers.get().expect("Primary is not running yet").len()).expect("Too many workers")
385    }
386
387    /// Returns the workers.
388    pub fn workers(&self) -> &[Worker<N>] {
389        self.workers.get().expect("Primary is not running yet")
390    }
391}
392
393impl<N: Network> Primary<N> {
394    /// Returns the number of unconfirmed transmissions.
395    pub fn num_unconfirmed_transmissions(&self) -> usize {
396        self.workers().iter().map(|worker| worker.num_transmissions()).sum()
397    }
398
399    /// Returns the number of unconfirmed ratifications.
400    pub fn num_unconfirmed_ratifications(&self) -> usize {
401        self.workers().iter().map(|worker| worker.num_ratifications()).sum()
402    }
403
404    /// Returns the number of unconfirmed solutions.
405    pub fn num_unconfirmed_solutions(&self) -> usize {
406        self.workers().iter().map(|worker| worker.num_solutions()).sum()
407    }
408
409    /// Returns the number of unconfirmed transactions.
410    pub fn num_unconfirmed_transactions(&self) -> usize {
411        self.workers().iter().map(|worker| worker.num_transactions()).sum()
412    }
413}
414
415impl<N: Network> Primary<N> {
416    /// Returns the worker transmission IDs.
417    pub fn worker_transmission_ids(&self) -> impl '_ + Iterator<Item = TransmissionID<N>> {
418        self.workers().iter().flat_map(|worker| worker.transmission_ids())
419    }
420
421    /// Returns the worker transmissions.
422    pub fn worker_transmissions(&self) -> impl '_ + Iterator<Item = (TransmissionID<N>, Transmission<N>)> {
423        self.workers().iter().flat_map(|worker| worker.transmissions())
424    }
425
426    /// Returns the worker solutions.
427    pub fn worker_solutions(&self) -> impl '_ + Iterator<Item = (SolutionID<N>, Data<Solution<N>>)> {
428        self.workers().iter().flat_map(|worker| worker.solutions())
429    }
430
431    /// Returns the worker transactions.
432    pub fn worker_transactions(&self) -> impl '_ + Iterator<Item = (N::TransactionID, Data<Transaction<N>>)> {
433        self.workers().iter().flat_map(|worker| worker.transactions())
434    }
435}
436
437impl<N: Network> Primary<N> {
438    /// Clears the worker solutions.
439    pub fn clear_worker_solutions(&self) {
440        self.workers().iter().for_each(Worker::clear_solutions);
441    }
442}
443
444#[async_trait::async_trait]
445impl<N: Network> proposal_task::BatchPropose for Primary<N> {
446    fn current_round(&self) -> u64 {
447        Primary::current_round(self)
448    }
449
450    fn wait_for_synced_if_syncing(&self) -> Option<futures::future::BoxFuture<'_, ()>> {
451        self.sync.wait_for_synced_if_syncing()
452    }
453
454    fn is_synced(&self) -> bool {
455        self.sync.is_synced()
456    }
457
458    /// Proposes the batch for the current round.
459    ///
460    /// This method performs the following steps:
461    /// 1. Drain the workers.
462    /// 2. Sign the batch.
463    /// 3. Set the batch proposal in the primary.
464    /// 4. Broadcast the batch header to all validators for signing.
465    ///
466    /// # Returns
467    /// - `Ok(true)` if the batch was proposed.
468    /// - `Ok(false)` if the batch was not proposed for a benign reason, e.g., the timestamp is too soon after the previous certificate.
469    /// - `Err(err)` if an unexpected error occured.
470    async fn propose_batch(&self) -> Result<bool> {
471        // Ensure there are not concurrent executions of this function.
472        //
473        // Note, in the current design, this function is only invoked from the batch proposal task, and it is technically
474        // not possible for there to be concurrent invocations of the function, but we keep this lock for now.
475        let mut lock_guard = self.latest_proposal_timestamp.write().await;
476
477        // Check if the proposed batch has expired, and clear it if it has expired.
478        if let Err(err) = self
479            .check_proposed_batch_for_expiration()
480            .with_context(|| "Failed to check the proposed batch for expiration")
481        {
482            warn!("{}", flatten_error(&err));
483            return Ok(false);
484        }
485
486        // Retrieve the current round.
487        let round = self.current_round();
488        // Compute the previous round.
489        let previous_round = round.saturating_sub(1);
490
491        // If the current round is 0, return early.
492        // This can actually never happen, because of the invariant that the current round is never 0
493        // (see [`StorageInner::current_round`]).
494        ensure!(round > 0, "Round 0 cannot have transaction batches");
495
496        // If the current storage round is below the latest proposal round, then return early.
497        if let Some((latest_round, _)) = &*lock_guard
498            && round < *latest_round
499        {
500            warn!("Cannot propose a batch for round {round} - the latest proposal cache round is {latest_round}");
501            return Ok(false);
502        }
503
504        // If there is a batch being proposed or certified already, handle accordingly.
505        match &*self.proposed_batch.read() {
506            ProposedBatchState::Certifying(proposal) => {
507                // Ensure that the storage is caught up to the proposal before proceeding to rebroadcast this.
508                if round < proposal.round()
509                    || proposal
510                        .batch_header()
511                        .previous_certificate_ids()
512                        .iter()
513                        .any(|id| !self.storage.contains_certificate(*id))
514                {
515                    warn!(
516                        "Cannot propose a batch for round {} - the current storage (round {round}) is not caught up to the proposed batch.",
517                        proposal.round(),
518                    );
519                    return Ok(false);
520                }
521                // Construct the event.
522                // TODO(ljedrz): the BatchHeader should be serialized only once in advance before being sent to non-signers.
523                let event = Event::BatchPropose(proposal.batch_header().clone().into());
524                // Iterate through the non-signers.
525                for address in proposal.nonsigners(&self.ledger.get_committee_lookback_for_round(proposal.round())?) {
526                    // Resolve the address to the peer IP.
527                    match self.gateway.resolver().read().get_peer_ip_for_address(address) {
528                        // Resend the batch proposal to the validator for signing.
529                        Some(peer_ip) => {
530                            let (gateway, event_, round) = (self.gateway.clone(), event.clone(), proposal.round());
531                            tokio::spawn(async move {
532                                debug!("Resending batch proposal for round {round} to peer '{peer_ip}'");
533                                // Resend the batch proposal to the peer.
534                                if gateway.send(peer_ip, event_).await.is_none() {
535                                    warn!("Failed to resend batch proposal for round {round} to peer '{peer_ip}'");
536                                }
537                            });
538                        }
539                        None => continue,
540                    }
541                }
542                debug!("Proposed batch for round {} is still valid", proposal.round());
543                return Ok(false);
544            }
545            // A batch is being certified; wait until it completes before proposing another.
546            ProposedBatchState::Certified(_) => {
547                debug!("Cannot propose a batch for round {round} - a batch is currently being certified");
548                return Ok(false);
549            }
550            ProposedBatchState::None => {
551                // No batch in progress, so it is save to propose a new one.
552            }
553        }
554
555        #[cfg(feature = "metrics")]
556        metrics::gauge(metrics::bft::PROPOSAL_ROUND, round as f64);
557
558        // Ensure that the primary does not create a new proposal too quickly.
559        if let Some((_, latest_timestamp)) = &*lock_guard
560            && !self.check_own_proposal_timestamp(previous_round, *latest_timestamp, now())?
561        {
562            return Ok(false);
563        }
564
565        // Ensure the primary has not proposed a batch for this round before.
566        if self.storage.contains_certificate_in_round_from(round, self.gateway.account().address()) {
567            // If a BFT sender was provided, attempt to advance the current round.
568            if let Some(cb) = &*self.primary_callback.get_ref() {
569                match cb.try_advance_to_next_round(self.current_round()) {
570                    true => (), // continue,
571                    false => return Ok(false),
572                }
573            }
574            debug!("Primary is safely skipping {}", format!("(round {round} was already certified)").dimmed());
575            return Ok(false);
576        }
577
578        // Determine if the current round has been proposed.
579        // Note: Do NOT make this judgment in advance before rebroadcast and round update. Rebroadcasting is
580        // good for network reliability and should not be prevented for the already existing proposed_batch.
581        // If a certificate already exists for the current round, an attempt should be made to advance the
582        // round as early as possible.
583        if let Some((latest_round, _)) = &*lock_guard
584            && *latest_round == round
585        {
586            debug!("Primary is safely skipping a batch proposal - round {round} already proposed");
587            return Ok(false);
588        }
589
590        // Retrieve the committee to check against.
591        let committee_lookback = self.ledger.get_committee_lookback_for_round(round)?;
592        // Check if the primary is connected to enough validators to reach quorum threshold.
593        {
594            // Retrieve the connected validator addresses.
595            let mut connected_validators = self.gateway.connected_addresses();
596            // Append the primary to the set.
597            connected_validators.insert(self.gateway.account().address());
598            // If quorum threshold is not reached, return early.
599            if !committee_lookback.is_quorum_threshold_reached(&connected_validators) {
600                debug!(
601                    "Primary is safely skipping a batch proposal for round {round} {}",
602                    "(please connect to more validators)".dimmed()
603                );
604                trace!("Primary is connected to {} validators", connected_validators.len() - 1);
605                return Ok(false);
606            }
607        }
608
609        // Retrieve the previous certificates.
610        let previous_certificates = self.storage.get_certificates_for_round(previous_round);
611
612        // Check if the batch is ready to be proposed.
613        // Note: The primary starts at round 1, and round 0 contains no certificates, by definition.
614        let mut is_ready = previous_round == 0;
615        // If the previous round is not 0, check if the previous certificates have reached the quorum threshold.
616        if previous_round > 0 {
617            // Retrieve the committee lookback for the round.
618            let Ok(previous_committee_lookback) = self.ledger.get_committee_lookback_for_round(previous_round) else {
619                bail!("Cannot propose a batch for round {round}: the committee lookback is not known yet")
620            };
621            // Construct a set over the authors.
622            let authors = previous_certificates.iter().map(BatchCertificate::author).collect();
623            // Check if the previous certificates have reached the quorum threshold.
624            if previous_committee_lookback.is_quorum_threshold_reached(&authors) {
625                is_ready = true;
626            }
627            #[cfg(feature = "test_network")]
628            {
629                // If we are using a hotswapped dev committee, use simplified checks to more easily advance.
630                if let Some(dev_committee) = self.ledger.dev_committee_for_round(previous_round)? {
631                    if round <= dev_committee.starting_round() {
632                        is_ready = true;
633                    }
634                }
635            }
636        }
637        // If the batch is not ready to be proposed, return early.
638        if !is_ready {
639            debug!(
640                "Primary is safely skipping a batch proposal for round {round} {}",
641                format!("(previous round {previous_round} has not reached quorum)").dimmed()
642            );
643            return Ok(false);
644        }
645
646        // Initialize the map of transmissions.
647        let mut transmissions: IndexMap<_, _> = Default::default();
648        // Track the total execution costs of the batch proposal as it is being constructed.
649        let mut proposal_cost = 0u64;
650        // Note: worker draining and transaction inclusion needs to be thought
651        // through carefully when there is more than one worker. The fairness
652        // provided by one worker (FIFO) is no longer guaranteed with multiple workers.
653        debug_assert_eq!(MAX_WORKERS, 1);
654
655        'outer: for worker in self.workers().iter() {
656            let mut num_worker_transmissions = 0usize;
657
658            while let Some((id, transmission)) = worker.remove_front() {
659                // Check the selected transmissions are below the batch limit.
660                if transmissions.len() >= BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH {
661                    // Reinsert the transmission into the worker.
662                    worker.insert_front(id, transmission);
663                    break 'outer;
664                }
665
666                // Check the max transmissions per worker is not exceeded.
667                if num_worker_transmissions >= Worker::<N>::MAX_TRANSMISSIONS_PER_WORKER {
668                    // Reinsert the transmission into the worker.
669                    worker.insert_front(id, transmission);
670                    continue 'outer;
671                }
672
673                // Check if the ledger already contains the transmission.
674                if self.ledger.contains_transmission(&id).unwrap_or(true) {
675                    trace!("Proposing - Skipping transmission '{}' - Already in ledger", fmt_id(id));
676                    continue;
677                }
678
679                // Check if the storage already contain the transmission.
680                // Note: We do not skip if this is the first transmission in the proposal, to ensure that
681                // the primary does not propose a batch with no transmissions.
682                if !transmissions.is_empty() && self.storage.contains_transmission(id) {
683                    trace!("Proposing - Skipping transmission '{}' - Already in storage", fmt_id(id));
684                    continue;
685                }
686
687                // Check the transmission is still valid.
688                match (id, transmission.clone()) {
689                    (TransmissionID::Solution(solution_id, checksum), Transmission::Solution(solution)) => {
690                        // Ensure the checksum matches. If not, skip the solution.
691                        if !matches!(solution.to_checksum::<N>(), Ok(solution_checksum) if solution_checksum == checksum)
692                        {
693                            trace!("Proposing - Skipping solution '{}' - Checksum mismatch", fmt_id(solution_id));
694                            continue;
695                        }
696                        // Check if the solution is still valid.
697                        if let Err(e) = self.ledger.check_solution_basic(solution_id, solution).await {
698                            trace!("Proposing - Skipping solution '{}' - {e}", fmt_id(solution_id));
699                            continue;
700                        }
701                    }
702                    (TransmissionID::Transaction(transaction_id, checksum), Transmission::Transaction(transaction)) => {
703                        // Ensure the checksum matches. If not, skip the transaction.
704                        if !matches!(transaction.to_checksum::<N>(), Ok(transaction_checksum) if transaction_checksum == checksum )
705                        {
706                            trace!("Proposing - Skipping transaction '{}' - Checksum mismatch", fmt_id(transaction_id));
707                            continue;
708                        }
709
710                        // Deserialize the transaction. If the transaction exceeds the maximum size, then return an error.
711                        let transaction = spawn_blocking!(deserialize_transaction_strict(transaction))?;
712
713                        // Fetch the current block height and consensus version.
714                        let current_block_height = self.ledger.latest_block_height();
715                        let consensus_version = N::CONSENSUS_VERSION(current_block_height)?;
716
717                        // Compute the transaction spent cost (in microcredits).
718                        // Note: We purposefully discard this transaction if we are unable to compute the spent cost.
719                        let Ok(cost) = self.ledger.transaction_spend_in_microcredits(&transaction, consensus_version)
720                        else {
721                            debug!(
722                                "Proposing - Skipping and discarding transaction '{}' - Unable to compute transaction spent cost",
723                                fmt_id(transaction_id)
724                            );
725                            continue;
726                        };
727
728                        // Check if the transaction is still valid.
729                        if let Err(e) = self.ledger.check_transaction_basic(transaction_id, transaction).await {
730                            trace!("Proposing - Skipping transaction '{}' - {e}", fmt_id(transaction_id));
731                            continue;
732                        }
733
734                        // Compute the next proposal cost.
735                        // Note: We purposefully discard this transaction if the proposal cost overflows.
736                        let Some(next_proposal_cost) = proposal_cost.checked_add(cost) else {
737                            debug!(
738                                "Proposing - Skipping and discarding transaction '{}' - Proposal cost overflowed",
739                                fmt_id(transaction_id)
740                            );
741                            continue;
742                        };
743
744                        // Check if the next proposal cost exceeds the batch proposal spend limit.
745                        let batch_spend_limit = BatchHeader::<N>::batch_spend_limit(current_block_height);
746                        if next_proposal_cost > batch_spend_limit {
747                            debug!(
748                                "Proposing - Skipping transaction '{}' - Batch spend limit surpassed ({next_proposal_cost} > {})",
749                                fmt_id(transaction_id),
750                                batch_spend_limit
751                            );
752
753                            // Reinsert the transmission into the worker.
754                            worker.insert_front(id, transmission);
755                            break 'outer;
756                        }
757
758                        // Update the proposal cost.
759                        proposal_cost = next_proposal_cost;
760                    }
761
762                    // Note: We explicitly forbid including ratifications,
763                    // as the protocol currently does not support ratifications.
764                    (TransmissionID::Ratification, Transmission::Ratification) => continue,
765                    // All other combinations are clearly invalid.
766                    _ => continue,
767                }
768
769                // If the transmission is valid, insert it into the proposal's transmission list.
770                transmissions.insert(id, transmission);
771                num_worker_transmissions = num_worker_transmissions.saturating_add(1);
772            }
773        }
774
775        // Determine the current timestamp.
776        let current_timestamp = now();
777
778        /* Proceeding to sign & propose the batch. */
779        info!("Proposing a batch with {} transmissions for round {round}...", transmissions.len());
780
781        // Update the latest proposed round and timestamp.
782        *lock_guard = Some((round, current_timestamp));
783        // Retrieve the private key.
784        let private_key = *self.gateway.account().private_key();
785        // Retrieve the committee ID.
786        let committee_id = committee_lookback.id();
787        // Prepare the transmission IDs.
788        let transmission_ids = transmissions.keys().copied().collect();
789        // Prepare the previous batch certificate IDs.
790        let previous_certificate_ids = previous_certificates.into_iter().map(|c| c.id()).collect();
791        // Sign the batch header and construct the proposal.
792        let (batch_header, proposal) = spawn_blocking!(BatchHeader::new(
793            &private_key,
794            round,
795            current_timestamp,
796            committee_id,
797            transmission_ids,
798            previous_certificate_ids,
799            &mut rand::rng()
800        ))
801        .and_then(|batch_header| {
802            Proposal::new(committee_lookback, batch_header.clone(), transmissions.clone())
803                .map(|proposal| (batch_header, proposal))
804        })
805        .inspect_err(|_| {
806            // On error, reinsert the transmissions and then propagate the error.
807            if let Err(err) = self.reinsert_transmissions_into_workers(transmissions) {
808                error!("{}", flatten_error(err.context("Failed to reinsert transmissions")));
809            }
810        })?;
811
812        // Broadcast the batch to all validators for signing.
813        self.gateway.broadcast(Event::BatchPropose(batch_header.into()));
814        // Store the proposal in memory.
815        *self.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(proposal));
816        // Record the wall-clock time at which the batch was proposed.
817        #[cfg(feature = "metrics")]
818        {
819            *self.batch_propose_start.lock() = Some(Instant::now());
820        }
821
822        Ok(true)
823    }
824}
825
826impl<N: Network> Primary<N> {
827    /// Processes a batch propose from a peer.
828    ///
829    /// This method performs the following steps:
830    /// 1. Verify the batch.
831    /// 2. Sign the batch.
832    /// 3. Broadcast the signature back to the validator.
833    ///
834    /// If our primary is ahead of the peer, we will not sign the batch.
835    /// If our primary is behind the peer, but within GC range, we will sync up to the peer's round, and then sign the batch.
836    async fn process_batch_propose_from_peer(&self, peer_ip: SocketAddr, batch_propose: BatchPropose<N>) -> Result<()> {
837        let BatchPropose { round: batch_round, batch_header } = batch_propose;
838
839        // Deserialize the batch header.
840        let batch_header = spawn_blocking!(batch_header.deserialize_blocking())?;
841        // Ensure the round matches in the batch header.
842        if batch_round != batch_header.round() {
843            // Proceed to disconnect the validator.
844            self.gateway.disconnect(peer_ip);
845            bail!("Malicious peer - proposed round {batch_round}, but sent batch for round {}", batch_header.round());
846        }
847
848        // Retrieve the batch author.
849        let batch_author = batch_header.author();
850
851        // Ensure the batch proposal is from the validator.
852        match self.gateway.resolve_to_aleo_addr(peer_ip) {
853            // If the peer is a validator, then ensure the batch proposal is from the validator.
854            Some(address) => {
855                if address != batch_author {
856                    // Proceed to disconnect the validator.
857                    self.gateway.disconnect(peer_ip);
858                    bail!("Malicious peer - proposed batch from a different validator ({batch_author})");
859                }
860            }
861            None => bail!("Batch proposal from a disconnected validator"),
862        }
863        // Ensure the batch author is a current committee member.
864        if !self.gateway.is_authorized_validator_address(batch_author) {
865            // Proceed to disconnect the validator.
866            self.gateway.disconnect(peer_ip);
867            bail!("Malicious peer - proposed batch from a non-committee member ({batch_author})");
868        }
869        // Ensure the batch proposal is not from the current primary.
870        if self.gateway.account().address() == batch_author {
871            bail!("Invalid peer - proposed batch from myself ({batch_author})");
872        }
873
874        // Ensure that the batch proposal's committee ID matches the expected committee ID.
875        // This may happen when the network forks. A transaction referencing a
876        // state root from one side of the fork will be aborted on the other
877        // side of the fork. This leads to a different view of stake and
878        // therefore a different view of the committee ID.
879        let expected_committee_id = self.ledger.get_committee_lookback_for_round(batch_round)?.id();
880        if expected_committee_id != batch_header.committee_id() {
881            // Proceed to disconnect the validator.
882            self.gateway.disconnect(peer_ip);
883            bail!(
884                "Malicious peer - proposed batch has a different committee ID ({expected_committee_id} != {})",
885                batch_header.committee_id()
886            );
887        }
888
889        // Retrieve the cached round and batch ID for this validator.
890        if let Some((signed_round, signed_batch_id, signature)) =
891            self.signed_proposals.read().get(&batch_author).copied()
892        {
893            // If the signed round is ahead of the peer's batch round, do not sign the proposal.
894            // Note: while this may be valid behavior, additional formal analysis and testing will need to be done before allowing it.
895            if signed_round > batch_header.round() {
896                bail!(
897                    "Peer ({batch_author}) proposed a batch for a previous round ({}), latest signed round: {signed_round}",
898                    batch_header.round()
899                );
900            }
901
902            // If the round matches and the batch ID differs, then the validator is malicious.
903            if signed_round == batch_header.round() && signed_batch_id != batch_header.batch_id() {
904                bail!("Peer ({batch_author}) proposed another batch for the same round ({signed_round})");
905            }
906            // If the round and batch ID matches, then skip signing the batch a second time.
907            // Instead, rebroadcast the cached signature to the peer.
908            if signed_round == batch_header.round() && signed_batch_id == batch_header.batch_id() {
909                let gateway = self.gateway.clone();
910                tokio::spawn(async move {
911                    debug!("Resending a signature for a batch in round {batch_round} from '{peer_ip}'");
912                    let event = Event::BatchSignature(BatchSignature::new(batch_header.batch_id(), signature));
913                    // Resend the batch signature to the peer.
914                    if gateway.send(peer_ip, event).await.is_none() {
915                        warn!("Failed to resend a signature for a batch in round {batch_round} to '{peer_ip}'");
916                    }
917                });
918                // Return early.
919                return Ok(());
920            }
921        }
922
923        // Ensure that the batch header doesn't already exist in storage.
924        // Note this is already checked in `check_batch_header`, however we can return early here without creating a blocking task.
925        if self.storage.contains_batch(batch_header.batch_id()) {
926            debug!(
927                "Primary is safely skipping a batch proposal from '{peer_ip}' - {}",
928                format!("batch for round {batch_round} already exists in storage").dimmed()
929            );
930            return Ok(());
931        }
932
933        // Compute the previous round.
934        let previous_round = batch_round.saturating_sub(1);
935        // Ensure that the peer did not propose a batch too quickly.
936        if let Err(err) = self.check_peer_proposal_timestamp(previous_round, batch_author, batch_header.timestamp()) {
937            // Proceed to disconnect the validator.
938            self.gateway.disconnect(peer_ip);
939            return Err(err.context(format!("Malicious behavior of peer '{peer_ip}'")));
940        }
941
942        // Ensure the batch header does not contain any ratifications.
943        if batch_header.contains(TransmissionID::Ratification) {
944            // Proceed to disconnect the validator.
945            self.gateway.disconnect(peer_ip);
946            bail!(
947                "Malicious peer - proposed batch contains an unsupported ratification transmissionID from '{peer_ip}'",
948            );
949        }
950
951        // If the peer is ahead, use the batch header to sync up to the peer.
952        let mut missing_transmissions =
953            self.sync_with_batch_header_from_peer::<false, true>(peer_ip, &batch_header).await?;
954
955        // Check that the transmission ids match and are not fee transactions.
956        if let Err(err) = cfg_iter_mut!(&mut missing_transmissions).try_for_each(|(transmission_id, transmission)| {
957            // If the transmission is not well-formed, then return early.
958            self.ledger.ensure_transmission_is_well_formed(*transmission_id, transmission)
959        }) {
960            let err = err.context(format!(
961                "Batch propose at round {batch_round} from '{peer_ip}' contains an invalid transmission"
962            ));
963            debug!("{}", flatten_error(err));
964            return Ok(());
965        }
966
967        // Ensure the batch is for the current round.
968        // This method must be called after fetching previous certificates (above),
969        // and prior to checking the batch header (below).
970        if let Err(e) = self.ensure_is_signing_round(batch_round) {
971            // If the primary is not signing for the peer's round, then return early.
972            debug!("{e} from '{peer_ip}'");
973            return Ok(());
974        }
975
976        // Ensure the batch header from the peer is valid.
977        let (storage, header) = (self.storage.clone(), batch_header.clone());
978
979        // Check the batch header, and return early if it already exists in storage.
980        let Some(missing_transmissions) =
981            spawn_blocking!(storage.check_batch_header(&header, missing_transmissions, Default::default()))?
982        else {
983            return Ok(());
984        };
985
986        // Inserts the missing transmissions into the workers.
987        self.insert_missing_transmissions_into_workers(peer_ip, missing_transmissions.into_iter())?;
988
989        /* Proceeding to sign the batch. */
990
991        // Retrieve the batch ID.
992        let batch_id = batch_header.batch_id();
993        // Sign the batch ID.
994        let account = self.gateway.account().clone();
995        let signature = spawn_blocking!(account.sign(&[batch_id], &mut rand::rng()))?;
996
997        // Ensure the proposal has not already been signed.
998        //
999        // Note: Due to the need to sync the batch header with the peer, it is possible
1000        // for the primary to receive the same 'BatchPropose' event again, whereby only
1001        // one instance of this handler should sign the batch. This check guarantees this.
1002        if !self.cache_signed_proposal(batch_author, batch_round, batch_id, signature) {
1003            return Ok(());
1004        }
1005
1006        // Broadcast the signature back to the validator.
1007        let self_ = self.clone();
1008        tokio::spawn(async move {
1009            let event = Event::BatchSignature(BatchSignature::new(batch_id, signature));
1010            // Send the batch signature to the peer.
1011            if self_.gateway.send(peer_ip, event).await.is_some() {
1012                debug!("Signed a batch for round {batch_round} from '{peer_ip}'");
1013            }
1014        });
1015
1016        Ok(())
1017    }
1018
1019    /// Records that this primary signed `batch_id` at `batch_round`, on behalf of `batch_author`,
1020    /// returning whether the proposal may be signed at all.
1021    ///
1022    /// For correctness we must endorse at most one batch per author per round, and the cached round
1023    /// is the only record of that. So the cached round must never move backwards: if it did, for
1024    /// example due to a race, we would forget having signed the newer round and could go on to
1025    /// endorse a second, conflicting batch for it. Two signatures over different batch IDs in one
1026    /// round are enough to build conflicting certificates, and so to fork the DAG.
1027    ///
1028    /// This runs under a single write lock, and either atomically caches the proposal and returns
1029    /// `true`, or leaves the cache untouched and returns `false` to say the proposal must not be
1030    /// signed.
1031    ///
1032    /// `false` is also returned when the cached round already equals `batch_round`, even if the
1033    /// cached batch ID matches: exactly one signature is produced per author per round, and it stays
1034    /// in the cache, so a repeat of a proposal we already signed is answered from the cache rather
1035    /// than signed afresh.
1036    fn cache_signed_proposal(
1037        &self,
1038        batch_author: Address<N>,
1039        batch_round: u64,
1040        batch_id: Field<N>,
1041        signature: Signature<N>,
1042    ) -> bool {
1043        match self.signed_proposals.write().0.entry(batch_author) {
1044            std::collections::hash_map::Entry::Occupied(mut entry) => {
1045                // Only ever advance the cached round.
1046                if entry.get().0 >= batch_round {
1047                    return false;
1048                }
1049                entry.insert((batch_round, batch_id, signature));
1050                true
1051            }
1052            // If the validator has not signed a batch before, then continue.
1053            std::collections::hash_map::Entry::Vacant(entry) => {
1054                // Cache the round, batch ID, and signature for this validator.
1055                entry.insert((batch_round, batch_id, signature));
1056                true
1057            }
1058        }
1059    }
1060
1061    /// Attempts to add a peer's `signature` for `batch_id` to the current proposal.
1062    ///
1063    /// Consumes `state` and always returns the (possibly updated) state alongside a result so that
1064    /// the caller can restore it unconditionally, keeping `proposed_batch` consistent even on the
1065    /// error path.
1066    ///
1067    /// # Returns
1068    /// * `(Ok(Some(proposal)), Certified(id))` — quorum reached; caller should certify.
1069    /// * `(Ok(None), <restored state>)` — signature accepted or silently dropped; nothing to do.
1070    /// * `(Err(e), <restored state>)` — signature rejected; caller should propagate the error.
1071    fn add_signature_to_batch(
1072        &self,
1073        state: ProposedBatchState<N>,
1074        peer_ip: SocketAddr,
1075        batch_id: Field<N>,
1076        signature: Signature<N>,
1077    ) -> (Result<Option<Proposal<N>>>, ProposedBatchState<N>) {
1078        match state {
1079            ProposedBatchState::Certifying(mut proposal) if proposal.batch_id() == batch_id => {
1080                // This signature is for our currently active proposal.
1081                // Use an inner closure to keep `?` ergonomics while returning a tuple.
1082                let inner: Result<bool> = (|| {
1083                    let committee_lookback = self.ledger.get_committee_lookback_for_round(proposal.round())?;
1084                    let Some(signer) = self.gateway.resolve_to_aleo_addr(peer_ip) else {
1085                        bail!("Signature is from a disconnected validator");
1086                    };
1087                    let new_signature = proposal.add_signature(signer, signature, &committee_lookback)?;
1088                    if new_signature {
1089                        info!("Received a batch signature for round {} from '{peer_ip}'", proposal.round());
1090                        Ok(proposal.is_quorum_threshold_reached(&committee_lookback))
1091                    } else {
1092                        debug!(
1093                            "Received duplicated signature from '{peer_ip}' for batch \
1094                                {batch_id} in round {round}",
1095                            round = proposal.round()
1096                        );
1097                        Ok(false)
1098                    }
1099                })();
1100                match inner {
1101                    Ok(true) => {
1102                        let certified_id = proposal.batch_id();
1103                        (Ok(Some(*proposal)), ProposedBatchState::Certified(certified_id))
1104                    }
1105                    Ok(false) => (Ok(None), ProposedBatchState::Certifying(proposal)),
1106                    Err(e) => (Err(e), ProposedBatchState::Certifying(proposal)),
1107                }
1108            }
1109            ProposedBatchState::Certifying(proposal) => {
1110                // Certifying a different proposal — check if batch_id is already in storage.
1111                if self.storage.contains_batch(batch_id) {
1112                    debug!(
1113                        "Primary is safely skipping a batch signature from {peer_ip} for \
1114                            round {} - batch is already certified",
1115                        proposal.round()
1116                    );
1117                    (Ok(None), ProposedBatchState::Certifying(proposal))
1118                } else {
1119                    let expected_id = proposal.batch_id();
1120                    let round = proposal.round();
1121                    (
1122                        Err(anyhow!("Unknown batch ID '{batch_id}', expected '{expected_id}' for round {round}")),
1123                        ProposedBatchState::Certifying(proposal),
1124                    )
1125                }
1126            }
1127            ProposedBatchState::Certified(id) if id == batch_id => {
1128                // Quorum already reached; late-arriving signature is harmless.
1129                debug!(
1130                    "Skipping batch signature from {peer_ip} for batch '{batch_id}' - \
1131                        already received sufficient signatures"
1132                );
1133                (Ok(None), ProposedBatchState::Certified(id))
1134            }
1135            ProposedBatchState::Certified(id) => {
1136                let result = if self.storage.contains_batch(batch_id) {
1137                    // This is most likely not malicious, but could indicate connectivity issues.
1138                    warn!("Received signature for an older batch {batch_id}");
1139                    Ok(None)
1140                } else {
1141                    Err(anyhow!("Unknown batch ID '{batch_id}'"))
1142                };
1143
1144                (result, ProposedBatchState::Certified(id))
1145            }
1146            ProposedBatchState::None => {
1147                let result = if self.storage.contains_batch(batch_id) {
1148                    // This is most likely not malicious, but could indicate connectivity issues.
1149                    warn!("Received signature for an older batch {batch_id}");
1150                    Ok(None)
1151                } else {
1152                    Err(anyhow!("Unknown batch ID '{batch_id}'"))
1153                };
1154
1155                (result, ProposedBatchState::None)
1156            }
1157        }
1158    }
1159
1160    /// Processes a batch signature from a peer.
1161    ///
1162    /// This method performs the following steps:
1163    /// 1. Ensure the proposed batch has not expired.
1164    /// 2. Verify the signature, ensuring it corresponds to the proposed batch.
1165    /// 3. Store the signature.
1166    /// 4. Certify the batch if enough signatures have been received.
1167    /// 5. Broadcast the batch certificate to all validators.
1168    async fn process_batch_signature_from_peer(
1169        &self,
1170        peer_ip: SocketAddr,
1171        batch_signature: BatchSignature<N>,
1172    ) -> Result<()> {
1173        // Ensure the proposed batch has not expired, and clear the proposed batch if it has expired.
1174        self.check_proposed_batch_for_expiration()?;
1175
1176        // Retrieve the signature and timestamp.
1177        let BatchSignature { batch_id, signature } = batch_signature;
1178
1179        // Retrieve the signer.
1180        let signer = signature.to_address();
1181
1182        // Ensure the batch signature is signed by the validator.
1183        match self.gateway.resolve_to_aleo_addr(peer_ip) {
1184            // If the peer is a validator, then ensure the batch signature is from the validator.
1185            Some(address) => {
1186                if address != signer {
1187                    // Proceed to disconnect the validator.
1188                    self.gateway.disconnect(peer_ip);
1189                    bail!("Malicious peer - batch signature is from a different validator ({signer})");
1190                }
1191            }
1192            None => bail!("Batch signature from a disconnected validator"),
1193        }
1194        // Ensure the batch signature is not from the current primary.
1195        if self.gateway.account().address() == signer {
1196            bail!("Invalid peer - received a batch signature from myself ({signer})");
1197        }
1198
1199        let self_ = self.clone();
1200        let Some(proposal) = spawn_blocking!({
1201            // Acquire the write lock.
1202            let mut proposed_batch = self_.proposed_batch.write();
1203
1204            let (result, new_state) =
1205                self_.add_signature_to_batch(std::mem::take(&mut *proposed_batch), peer_ip, batch_id, signature);
1206            *proposed_batch = new_state;
1207            result
1208        })?
1209        else {
1210            return Ok(());
1211        };
1212
1213        /* Proceeding to certify the batch. */
1214
1215        info!("Quorum threshold reached - Preparing to certify our batch for round {}...", proposal.round());
1216
1217        // Retrieve the committee lookback for the round.
1218        let committee_lookback = self.ledger.get_committee_lookback_for_round(proposal.round())?;
1219        // Store the certified batch and broadcast it to all validators.
1220        // If there was an error storing the certificate, reinsert the transmissions back into the ready queue.
1221        if let Err(e) = self.store_and_broadcast_certificate(&proposal, &committee_lookback).await {
1222            // Reinsert the transmissions back into the ready queue for the next proposal.
1223            self.reinsert_transmissions_into_workers(proposal.into_transmissions())?;
1224            return Err(e);
1225        }
1226
1227        #[cfg(feature = "metrics")]
1228        metrics::increment_gauge(metrics::bft::CERTIFIED_BATCHES, 1.0);
1229        Ok(())
1230    }
1231
1232    /// Processes a batch certificate from a peer.
1233    ///
1234    /// This method performs the following steps:
1235    /// 1. Stores the given batch certificate, after ensuring it is valid.
1236    /// 2. If there are enough certificates to reach quorum threshold for the current round,
1237    ///    then proceed to advance to the next round.
1238    async fn process_batch_certificate_from_peer(
1239        &self,
1240        peer_ip: SocketAddr,
1241        certificate: BatchCertificate<N>,
1242    ) -> Result<()> {
1243        // Ensure the batch certificate is from an authorized validator.
1244        if !self.gateway.is_authorized_validator_ip(peer_ip) {
1245            // Proceed to disconnect the validator.
1246            self.gateway.disconnect(peer_ip);
1247            bail!("Malicious peer - Received a batch certificate from an unauthorized validator IP ({peer_ip})");
1248        }
1249        // Ensure storage does not already contain the certificate.
1250        if self.storage.contains_certificate(certificate.id()) {
1251            return Ok(());
1252        // Otherwise, ensure ephemeral storage contains the certificate.
1253        } else if !self.storage.contains_unprocessed_certificate(certificate.id()) {
1254            self.storage.insert_unprocessed_certificate(certificate.clone())?;
1255        }
1256
1257        // Retrieve the batch certificate author.
1258        let author = certificate.author();
1259        // Retrieve the batch certificate round.
1260        let certificate_round = certificate.round();
1261        // Retrieve the batch certificate committee ID.
1262        let committee_id = certificate.committee_id();
1263
1264        // Ensure the batch certificate is not from the current primary.
1265        if self.gateway.account().address() == author {
1266            bail!("Received a batch certificate for myself ({author})");
1267        }
1268
1269        // Ensure that the incoming certificate is valid.
1270        self.storage.check_incoming_certificate(&certificate)?;
1271
1272        // Store the certificate, after ensuring it is valid above.
1273        // The following call recursively fetches and stores
1274        // the previous certificates referenced from this certificate.
1275        // It is critical to make the following call this after validating the certificate above.
1276        // The reason is that a sequence of malformed certificates,
1277        // with references to previous certificates with non-decreasing rounds,
1278        // cause the recursive fetching of certificates to crash the validator due to resource exhaustion.
1279        // Note that if the following call, if not returning an error, guarantees the backward closure of the DAG
1280        // (i.e. that all the referenced previous certificates are in the DAG before storing this one),
1281        // then all the validity checks in [`Storage::check_certificate`] should be redundant.
1282        // TODO: eliminate those redundant checks
1283        self.sync_with_certificate_from_peer::<false>(peer_ip, certificate).await?;
1284
1285        // If there are enough certificates to reach quorum threshold for the certificate round,
1286        // then proceed to advance to the next round.
1287
1288        // Retrieve the committee lookback.
1289        let committee_lookback = self.ledger.get_committee_lookback_for_round(certificate_round)?;
1290
1291        // Retrieve the certificate authors.
1292        let authors = self.storage.get_certificate_authors_for_round(certificate_round);
1293        // Check if the certificates have reached the quorum threshold.
1294        let is_quorum = committee_lookback.is_quorum_threshold_reached(&authors);
1295
1296        // Ensure that the batch certificate's committee ID matches the expected committee ID.
1297        let expected_committee_id = committee_lookback.id();
1298        if expected_committee_id != committee_id {
1299            // Proceed to disconnect the validator.
1300            self.gateway.disconnect(peer_ip);
1301            bail!("Batch certificate has a different committee ID ({expected_committee_id} != {committee_id})");
1302        }
1303
1304        // Determine if we are currently proposing a round that is relevant.
1305        // Note: This is important, because while our peers have advanced,
1306        // they may not be proposing yet, and thus still able to sign our proposed batch.
1307        let should_advance = match &*self.latest_proposal_timestamp.read().await {
1308            // We advance if the proposal round is less than the current round that was just certified.
1309            Some((latest_round, _)) => *latest_round < certificate_round,
1310            // If there's no proposal, we consider advancing.
1311            None => true,
1312        };
1313
1314        // Retrieve the current round.
1315        let current_round = self.current_round();
1316
1317        // Determine whether to advance to the next round.
1318        if is_quorum && should_advance && certificate_round >= current_round {
1319            // If we have reached the quorum threshold and the round should advance, then proceed to the next round.
1320            self.round_increment_notify.notify_one();
1321        }
1322        Ok(())
1323    }
1324}
1325
1326impl<N: Network> Primary<N> {
1327    /// Starts the primary handlers.
1328    ///
1329    /// For each receiver in the `primary_receiver` struct, there will be a dedicated task
1330    /// that awaits new data and handles it accordingly.
1331    /// Additionally, this spawns a task that periodically issues PrimaryPings and one that
1332    /// tries to move to the next round when triggered (e.g. after a certificate is stored) or on a timeout.
1333    ///
1334    /// This function is called exactly once, in `Self::run()`.
1335    fn start_handlers(&self, primary_receiver: PrimaryReceiver<N>) {
1336        let PrimaryReceiver {
1337            mut rx_batch_propose,
1338            mut rx_batch_signature,
1339            mut rx_batch_certified,
1340            mut rx_primary_ping,
1341            mut rx_unconfirmed_solution,
1342            mut rx_unconfirmed_transaction,
1343        } = primary_receiver;
1344
1345        // Start the primary ping sender.
1346        let self_ = self.clone();
1347        self.spawn(async move {
1348            loop {
1349                // Sleep briefly.
1350                tokio::time::sleep(PRIMARY_PING_INTERVAL).await;
1351
1352                // Retrieve the block locators.
1353                let self__ = self_.clone();
1354                let block_locators = match spawn_blocking!(self__.sync.get_block_locators()) {
1355                    Ok(block_locators) => block_locators,
1356                    Err(e) => {
1357                        warn!("Failed to retrieve block locators - {e}");
1358                        continue;
1359                    }
1360                };
1361
1362                // Retrieve the latest certificate of the primary.
1363                let primary_certificate = {
1364                    // Retrieve the primary address.
1365                    let primary_address = self_.gateway.account().address();
1366
1367                    // Iterate backwards from the latest round to find the primary certificate.
1368                    let mut certificate = None;
1369                    let mut current_round = self_.current_round();
1370                    while certificate.is_none() {
1371                        // If the current round is 0, then break the while loop.
1372                        if current_round == 0 {
1373                            break;
1374                        }
1375                        // Retrieve the primary certificates.
1376                        if let Some(primary_certificate) =
1377                            self_.storage.get_certificate_for_round_with_author(current_round, primary_address)
1378                        {
1379                            certificate = Some(primary_certificate);
1380                        // If the primary certificate was not found, decrement the round.
1381                        } else {
1382                            current_round = current_round.saturating_sub(1);
1383                        }
1384                    }
1385
1386                    // Determine if the primary certificate was found.
1387                    match certificate {
1388                        Some(certificate) => certificate,
1389                        // Skip this iteration of the loop (do not send a primary ping).
1390                        None => continue,
1391                    }
1392                };
1393
1394                // Construct the primary ping.
1395                let primary_ping = PrimaryPing::from((<Event<N>>::VERSION, block_locators, primary_certificate));
1396                // Broadcast the event.
1397                self_.gateway.broadcast(Event::PrimaryPing(primary_ping));
1398            }
1399        });
1400
1401        // Start the primary ping handler.
1402        let self_ = self.clone();
1403        self.spawn(async move {
1404            while let Some((peer_ip, primary_certificate)) = rx_primary_ping.recv().await {
1405                // If the primary is not synced, then do not process the primary ping.
1406                if self_.sync.is_synced() {
1407                    trace!("Processing new primary ping from '{peer_ip}'");
1408                } else {
1409                    trace!("Skipping a primary ping from '{peer_ip}' {}", "(node is syncing)".dimmed());
1410                    continue;
1411                }
1412
1413                // Spawn a task to process the primary certificate.
1414                {
1415                    let self_ = self_.clone();
1416                    tokio::spawn(async move {
1417                        // Deserialize the primary certificate in the primary ping.
1418                        let Ok(primary_certificate) = spawn_blocking!(primary_certificate.deserialize_blocking())
1419                        else {
1420                            warn!("Failed to deserialize primary certificate in 'PrimaryPing' from '{peer_ip}'");
1421                            return;
1422                        };
1423                        // Process the primary certificate.
1424                        let id = fmt_id(primary_certificate.id());
1425                        let round = primary_certificate.round();
1426                        if let Err(e) = self_.process_batch_certificate_from_peer(peer_ip, primary_certificate).await {
1427                            warn!("Cannot process a primary certificate '{id}' at round {round} in a 'PrimaryPing' from '{peer_ip}' - {e}");
1428                        }
1429                    });
1430                }
1431            }
1432        });
1433
1434        // Start the worker ping(s).
1435        let self_ = self.clone();
1436        self.spawn(async move {
1437            loop {
1438                tokio::time::sleep(WORKER_PING_INTERVAL).await;
1439                // If the primary is not synced, then do not broadcast the worker ping(s).
1440                if !self_.sync.is_synced() {
1441                    trace!("Skipping worker ping(s) {}", "(node is syncing)".dimmed());
1442                    continue;
1443                }
1444                // Broadcast the worker ping(s).
1445                for worker in self_.workers() {
1446                    worker.broadcast_ping();
1447                }
1448            }
1449        });
1450
1451        // Start the batch proposal task.
1452        let proposal_task = self.proposal_task.clone();
1453        let self_ = self.clone();
1454        self.spawn(async move { proposal_task.run(self_).await });
1455
1456        // Start the proposed batch handler.
1457        let self_ = self.clone();
1458        self.spawn(async move {
1459            while let Some((peer_ip, batch_propose)) = rx_batch_propose.recv().await {
1460                // If the primary is not synced, then do not sign the batch.
1461                if !self_.sync.is_synced() {
1462                    trace!("Skipping a batch proposal from '{peer_ip}' {}", "(node is syncing)".dimmed());
1463                    continue;
1464                }
1465
1466                // Spawn a task to process the proposed batch.
1467                let self_ = self_.clone();
1468                tokio::spawn(async move {
1469                    // Process the batch proposal.
1470                    let round = batch_propose.round;
1471                    if let Err(err) = self_.process_batch_propose_from_peer(peer_ip, batch_propose).await {
1472                        let err = err.context(format!("Cannot sign a batch at round {round} from '{peer_ip}'"));
1473                        warn!("{}", flatten_error(err));
1474                    }
1475                });
1476            }
1477        });
1478
1479        // Start the batch signature handler.
1480        let self_ = self.clone();
1481        self.spawn(async move {
1482            while let Some((peer_ip, batch_signature)) = rx_batch_signature.recv().await {
1483                // If the primary is not synced, then do not store the signature.
1484                if !self_.sync.is_synced() {
1485                    trace!("Skipping a batch signature from '{peer_ip}' {}", "(node is syncing)".dimmed());
1486                    continue;
1487                }
1488                // Process the batch signature.
1489                // Note: Do NOT spawn a task around this function call. Processing signatures from peers
1490                // is a critical path, and we should only store the minimum required number of signatures.
1491                // In addition, spawning a task can cause concurrent processing of signatures (even with a lock),
1492                // which means the RwLock for the proposed batch must become a 'tokio::sync' to be safe.
1493                let id = fmt_id(batch_signature.batch_id);
1494                if let Err(err) = self_.process_batch_signature_from_peer(peer_ip, batch_signature).await {
1495                    let err = err.context(format!("Cannot store a signature for batch '{id}' from '{peer_ip}'"));
1496                    warn!("{}", flatten_error(err));
1497                }
1498            }
1499        });
1500
1501        // Start the certified batch handler.
1502        let self_ = self.clone();
1503        self.spawn(async move {
1504            while let Some((peer_ip, batch_certificate)) = rx_batch_certified.recv().await {
1505                // If the primary is not synced, then do not store the certificate.
1506                if !self_.sync.is_synced() {
1507                    trace!("Skipping a certified batch from '{peer_ip}' {}", "(node is syncing)".dimmed());
1508                    continue;
1509                }
1510                // Spawn a task to process the batch certificate.
1511                let self_ = self_.clone();
1512                tokio::spawn(async move {
1513                    // Deserialize the batch certificate.
1514                    let Ok(batch_certificate) = spawn_blocking!(batch_certificate.deserialize_blocking()) else {
1515                        warn!("Failed to deserialize the batch certificate from '{peer_ip}'");
1516                        return;
1517                    };
1518                    // Process the batch certificate.
1519                    let id = fmt_id(batch_certificate.id());
1520                    let round = batch_certificate.round();
1521                    if let Err(err) = self_.process_batch_certificate_from_peer(peer_ip, batch_certificate).await {
1522                        warn!(
1523                            "{}",
1524                            flatten_error(err.context(format!(
1525                                "Cannot store a certificate '{id}' for round {round} from '{peer_ip}'"
1526                            )))
1527                        );
1528                    }
1529                });
1530            }
1531        });
1532
1533        // This task tries to move to the next round when triggered (e.g. after a certificate is stored)
1534        // or after a timeout, so we are not stuck on a previous round despite having quorum.
1535        let self_ = self.clone();
1536        self.spawn(async move {
1537            loop {
1538                let round_start = Instant::now();
1539                let current_round = self_.current_round();
1540
1541                // Inner loop: wait and try to increment while we're still in the same round.
1542                while self_.current_round() == current_round {
1543                    let mut futures: Vec<Pin<Box<dyn Future<Output = ()> + Send>>> =
1544                        vec![Box::pin(self_.round_increment_notify.notified())];
1545
1546                    if let Some(remaining_delay) = MAX_BATCH_DELAY.checked_sub(round_start.elapsed())
1547                        && !remaining_delay.is_zero()
1548                    {
1549                        futures.push(Box::pin(tokio::time::sleep(remaining_delay)));
1550                    }
1551                    // Always ensure a wakeup no later than MAX_LEADER_CERTIFICATE_DELAY so that
1552                    // try_advance_to_next_round is called after the leader-certificate timer
1553                    // expires, even when no further certificates arrive (e.g. an even round where
1554                    // the elected leader was absent and quorum was reached without their cert).
1555                    futures.push(Box::pin(tokio::time::sleep(MAX_LEADER_CERTIFICATE_DELAY)));
1556                    if !self_.sync.is_synced() {
1557                        futures.push(Box::pin(self_.sync.wait_for_synced()));
1558                    }
1559                    let _ = futures::future::select_all(futures).await;
1560
1561                    if !self_.sync.is_synced() {
1562                        trace!("Skipping round increment {}", "(node is syncing)".dimmed());
1563                        continue;
1564                    }
1565
1566                    let next_round = current_round.saturating_add(1);
1567                    let is_quorum_threshold_reached = {
1568                        let authors = self_.storage.get_certificate_authors_for_round(current_round);
1569                        if authors.is_empty() {
1570                            continue;
1571                        }
1572                        let Ok(committee_lookback) = self_.ledger.get_committee_lookback_for_round(current_round)
1573                        else {
1574                            warn!("Failed to retrieve the committee lookback for round {current_round}");
1575                            continue;
1576                        };
1577                        committee_lookback.is_quorum_threshold_reached(&authors)
1578                    };
1579
1580                    if is_quorum_threshold_reached {
1581                        debug!("Quorum threshold reached for round {current_round}");
1582                        if let Err(err) = self_.try_increment_to_the_next_round(next_round).await {
1583                            warn!("{}", flatten_error(err.context("Failed to increment to the next round")));
1584                        }
1585                    }
1586                }
1587            }
1588        });
1589
1590        // Start a handler to process new unconfirmed solutions.
1591        let self_ = self.clone();
1592        self.spawn(async move {
1593            while let Some((solution_id, solution, callback)) = rx_unconfirmed_solution.recv().await {
1594                // Compute the checksum for the solution.
1595                let Ok(checksum) = solution.to_checksum::<N>() else {
1596                    error!("Failed to compute the checksum for the unconfirmed solution");
1597                    continue;
1598                };
1599                // Compute the worker ID.
1600                let Ok(worker_id) = assign_to_worker((solution_id, checksum), self_.num_workers()) else {
1601                    error!("Unable to determine the worker ID for the unconfirmed solution");
1602                    continue;
1603                };
1604                let self_ = self_.clone();
1605                tokio::spawn(async move {
1606                    // Retrieve the worker.
1607                    let worker = &self_.workers()[worker_id as usize];
1608                    // Process the unconfirmed solution.
1609                    let result = worker.process_unconfirmed_solution(solution_id, solution).await;
1610                    // Send the result to the callback.
1611                    callback.send(result).ok();
1612                });
1613            }
1614        });
1615
1616        // Start a handler to process new unconfirmed transactions.
1617        let self_ = self.clone();
1618        self.spawn(async move {
1619            while let Some((transaction_id, transaction, callback)) = rx_unconfirmed_transaction.recv().await {
1620                trace!("Primary - Received an unconfirmed transaction '{}'", fmt_id(transaction_id));
1621                // Compute the checksum for the transaction.
1622                let Ok(checksum) = transaction.to_checksum::<N>() else {
1623                    error!("Failed to compute the checksum for the unconfirmed transaction");
1624                    continue;
1625                };
1626                // Compute the worker ID.
1627                let Ok(worker_id) = assign_to_worker::<N>((&transaction_id, &checksum), self_.num_workers()) else {
1628                    error!("Unable to determine the worker ID for the unconfirmed transaction");
1629                    continue;
1630                };
1631                let self_ = self_.clone();
1632                tokio::spawn(async move {
1633                    // Retrieve the worker.
1634                    let worker = &self_.workers().get(worker_id as usize).expect("Invalid worker ID");
1635                    // Process the unconfirmed transaction.
1636                    let result = worker.process_unconfirmed_transaction(transaction_id, transaction).await;
1637                    // Send the result to the callback.
1638                    callback.send(result).ok();
1639                });
1640            }
1641        });
1642    }
1643
1644    /// Checks if the proposed batch is expired, and clears the proposed batch if it has expired.
1645    ///
1646    /// The proposal is inspected and cleared under a single write lock, so that the proposal which
1647    /// is discarded is exactly the one that was found to be expired.
1648    fn check_proposed_batch_for_expiration(&self) -> Result<()> {
1649        // Read the current round before taking the lock below, so that no other lock is acquired
1650        // while it is held. The round only ever increases, so reading it early is conservative: at
1651        // worst a proposal that just expired is left for the next call to clear.
1652        let current_round = self.current_round();
1653
1654        // Take the proposed batch only if it has expired; taking it leaves the state cleared.
1655        // A batch being certified is not considered expired.
1656        let expired = {
1657            let mut proposed_batch = self.proposed_batch.write();
1658            let is_expired = matches!(
1659                &*proposed_batch,
1660                ProposedBatchState::Certifying(proposal) if proposal.round() < current_round
1661            );
1662            is_expired.then(|| std::mem::take(&mut *proposed_batch))
1663        };
1664
1665        // Reinsert the transmissions once the lock above has been released, since this acquires the
1666        // workers' locks in turn.
1667        if let Some(ProposedBatchState::Certifying(proposal)) = expired {
1668            debug!("Cleared expired proposal for round {}", proposal.round());
1669            self.reinsert_transmissions_into_workers(proposal.into_transmissions())?;
1670        }
1671        Ok(())
1672    }
1673
1674    /// Increments to the next round.
1675    async fn try_increment_to_the_next_round(&self, next_round: u64) -> Result<()> {
1676        // If the next round is within GC range, then iterate to the penultimate round.
1677        if self.current_round() + self.storage.max_gc_rounds() >= next_round {
1678            let mut fast_forward_round = self.current_round();
1679            // Iterate until the penultimate round is reached.
1680            while fast_forward_round < next_round.saturating_sub(1) {
1681                // Update to the next round in storage.
1682                fast_forward_round = self.storage.increment_to_next_round(fast_forward_round)?;
1683                // Clear the proposed batch.
1684                *self.proposed_batch.write() = ProposedBatchState::None;
1685            }
1686        }
1687
1688        // Retrieve the current round.
1689        let current_round = self.current_round();
1690        // Attempt to advance to the next round.
1691        if current_round < next_round {
1692            // If a BFT sender was provided, send the current round to the BFT.
1693            let is_ready = if let Some(cb) = self.primary_callback.get() {
1694                cb.try_advance_to_next_round(current_round)
1695            }
1696            // Otherwise, handle the Narwhal case.
1697            else {
1698                // Update to the next round in storage.
1699                self.storage.increment_to_next_round(current_round)?;
1700                // Set 'is_ready' to 'true'.
1701                true
1702            };
1703
1704            // Notify the proposal task if the new round is ready.
1705            if is_ready && self.is_synced() {
1706                debug!("Primary is ready to propose the next round");
1707                self.proposal_task.signal();
1708            } else {
1709                debug!("Primary is not ready to propose the next round");
1710            }
1711        }
1712        Ok(())
1713    }
1714
1715    /// Ensures the primary is signing for the specified batch round.
1716    /// This method is used to ensure: for a given round, as soon as the primary starts proposing,
1717    /// it will no longer sign for the previous round (as it has enough previous certificates to proceed).
1718    fn ensure_is_signing_round(&self, batch_round: u64) -> Result<()> {
1719        // Retrieve the current round.
1720        let current_round = self.current_round();
1721        // Ensure the batch round is within GC range of the current round.
1722        if current_round + self.storage.max_gc_rounds() <= batch_round {
1723            bail!("Round {batch_round} is too far in the future")
1724        }
1725        // Ensure the batch round is at or one before the current round.
1726        // Intuition: Our primary has moved on to the next round, but has not necessarily started proposing,
1727        // so we can still sign for the previous round. If we have started proposing, the next check will fail.
1728        if current_round > batch_round + 1 {
1729            bail!("Primary is on round {current_round}, and no longer signing for round {batch_round}")
1730        }
1731        // Check if the primary is still signing for the batch round.
1732        if let ProposedBatchState::Certifying(proposal) = &*self.proposed_batch.read()
1733            && proposal.round() > batch_round
1734        {
1735            bail!("Our primary at round {} is no longer signing for round {batch_round}", proposal.round())
1736        }
1737        Ok(())
1738    }
1739
1740    /// Ensure the primary is not creating batch proposals too frequently.
1741    /// This checks that the certificate timestamp for the previous round is within the expected range.
1742    fn check_peer_proposal_timestamp(&self, previous_round: u64, author: Address<N>, timestamp: i64) -> Result<()> {
1743        ensure!(author != self.gateway.account().address(), "Peer cannot propose a batch that is authored by myself");
1744
1745        // Retrieve the timestamp of the previous timestamp to check against.
1746        let previous_timestamp = match self.storage.get_certificate_for_round_with_author(previous_round, author) {
1747            // Ensure that the previous certificate was created at least `MIN_BATCH_DELAY` seconds ago.
1748            Some(certificate) => certificate.timestamp(),
1749            // If we do not see a previous certificate for the author, then proceed optimistically.
1750            None => return Ok(()),
1751        };
1752
1753        // Determine the elapsed time since the previous timestamp.
1754        let elapsed = timestamp
1755            .checked_sub(previous_timestamp)
1756            .ok_or_else(|| anyhow!("Timestamp cannot be before the previous certificate at round {previous_round}"))?;
1757        // Ensure that the previous certificate was created at least `MIN_BATCH_DELAY` seconds ago.
1758        match elapsed < MIN_BATCH_DELAY.as_secs() as i64 {
1759            true => bail!("Timestamp is too soon after the previous certificate at round {previous_round}"),
1760            false => Ok(()),
1761        }
1762    }
1763
1764    /// Ensure the primary is not creating batch proposals too frequently.
1765    /// This checks that the certificate timestamp for the previous round is within the expected range.
1766    ///
1767    /// # Returns
1768    /// - `Ok(true)` if the timestamp allows a new proposal.
1769    /// - `Ok(false)` if the timestamp is valid but too soon after the previous proposal.
1770    /// - `Err(err)` if an unexpected error occured, such as the timestamp being before the previous certificate.
1771    fn check_own_proposal_timestamp(
1772        &self,
1773        previous_round: u64,
1774        previous_timestamp: i64,
1775        timestamp: i64,
1776    ) -> Result<bool> {
1777        // Determine the elapsed time since the previous timestamp.
1778        let elapsed = timestamp
1779            .checked_sub(previous_timestamp)
1780            .ok_or_else(|| anyhow!("Timestamp cannot be before the previous certificate at round {previous_round}"))?;
1781
1782        Ok(elapsed >= MIN_BATCH_DELAY.as_secs() as i64)
1783    }
1784
1785    /// Stores the certified batch and broadcasts it to all validators, returning the certificate.
1786    async fn store_and_broadcast_certificate(&self, proposal: &Proposal<N>, committee: &Committee<N>) -> Result<()> {
1787        // Create the batch certificate and transmissions.
1788        let (certificate, transmissions) = tokio::task::block_in_place(|| proposal.to_certificate(committee))?;
1789
1790        // Convert the transmissions into a HashMap.
1791        // Note: Do not change the `Proposal` to use a HashMap. The ordering there is necessary for safety.
1792        let transmissions = transmissions.into_iter().collect::<HashMap<_, _>>();
1793
1794        // Store some metadata about the certified batch.
1795        let round = certificate.round();
1796        let num_transmissions = certificate.transmission_ids().len();
1797
1798        // Store the certified batch.
1799        let (storage, certificate_) = (self.storage.clone(), certificate.clone());
1800        spawn_blocking!(storage.insert_certificate(certificate_, transmissions, Default::default()))?;
1801        debug!("Stored a batch certificate for round {}", certificate.round());
1802        // The batch is now in storage, so late-arriving signatures can find it via contains_batch.
1803        // Transition from Certified back to None.
1804        *self.proposed_batch.write() = ProposedBatchState::None;
1805
1806        // If a BFT sender was provided, send the certificate to the BFT.
1807        if let Some(cb) = self.primary_callback.get() {
1808            // Await the callback to continue.
1809            cb.add_new_certificate(certificate.clone()).await.with_context(|| {
1810                format!("Failed to insert our newly certified batch for round {round} into the DAG")
1811            })?;
1812        }
1813        // Broadcast the certified batch to all validators.
1814        self.gateway.broadcast(Event::BatchCertified(certificate.into()));
1815
1816        // Log the certified batch.
1817        info!("Our batch with {num_transmissions} transmissions for round {round} was certified!");
1818
1819        // Record the certification latency (time from batch proposal to certification).
1820        #[cfg(feature = "metrics")]
1821        if let Some(start) = self.batch_propose_start.lock().take() {
1822            metrics::histogram(metrics::bft::BATCH_CERTIFICATION_LATENCY, start.elapsed().as_secs_f64());
1823        }
1824
1825        // Wake up the round increment task to re-check quorum.
1826        self.round_increment_notify.notify_one();
1827
1828        Ok(())
1829    }
1830
1831    /// Inserts the missing transmissions from the proposal into the workers.
1832    fn insert_missing_transmissions_into_workers(
1833        &self,
1834        peer_ip: SocketAddr,
1835        transmissions: impl Iterator<Item = (TransmissionID<N>, Transmission<N>)>,
1836    ) -> Result<()> {
1837        // Insert the transmissions into the workers.
1838        assign_to_workers(self.workers(), transmissions, |worker, transmission_id, transmission| {
1839            worker.process_transmission_from_peer(peer_ip, transmission_id, transmission);
1840        })
1841    }
1842
1843    /// Re-inserts the transmissions from the proposal into the workers.
1844    fn reinsert_transmissions_into_workers(
1845        &self,
1846        transmissions: IndexMap<TransmissionID<N>, Transmission<N>>,
1847    ) -> Result<()> {
1848        // Re-insert the transmissions into the workers.
1849        assign_to_workers(self.workers(), transmissions.into_iter(), |worker, transmission_id, transmission| {
1850            worker.reinsert(transmission_id, transmission);
1851        })
1852    }
1853
1854    /// Recursively stores a given batch certificate, after ensuring:
1855    ///   - Ensure the round matches the committee round.
1856    ///   - Ensure the address is a member of the committee.
1857    ///   - Ensure the timestamp is within range.
1858    ///   - Ensure we have all of the transmissions.
1859    ///   - Ensure we have all of the previous certificates.
1860    ///   - Ensure the previous certificates are for the previous round (i.e. round - 1).
1861    ///   - Ensure the previous certificates have reached the quorum threshold.
1862    ///   - Ensure we have not already signed the batch ID.
1863    #[async_recursion::async_recursion]
1864    async fn sync_with_certificate_from_peer<const IS_SYNCING: bool>(
1865        &self,
1866        peer_ip: SocketAddr,
1867        certificate: BatchCertificate<N>,
1868    ) -> Result<()> {
1869        // Retrieve the batch header.
1870        let batch_header = certificate.batch_header();
1871        // Retrieve the batch round.
1872        let batch_round = batch_header.round();
1873
1874        // If the certificate round is outdated, do not store it.
1875        if batch_round <= self.storage.gc_round() {
1876            return Ok(());
1877        }
1878        // If the certificate already exists in storage, return early.
1879        if self.storage.contains_certificate(certificate.id()) {
1880            return Ok(());
1881        }
1882
1883        // If node is not in sync mode and the node is not synced. Then return an error.
1884        if !IS_SYNCING && !self.is_synced() {
1885            bail!(
1886                "Failed to process certificate `{}` at round {batch_round} from '{peer_ip}' (node is syncing)",
1887                fmt_id(certificate.id())
1888            );
1889        }
1890
1891        // If the peer is ahead, use the batch header to sync up to the peer.
1892        let missing_transmissions =
1893            self.sync_with_batch_header_from_peer::<IS_SYNCING, false>(peer_ip, batch_header).await?;
1894
1895        // Check if the certificate needs to be stored.
1896        if !self.storage.contains_certificate(certificate.id()) {
1897            // Store the batch certificate.
1898            let (storage, certificate_) = (self.storage.clone(), certificate.clone());
1899            spawn_blocking!(storage.insert_certificate(certificate_, missing_transmissions, Default::default()))?;
1900            debug!("Stored a batch certificate for round {batch_round} from '{peer_ip}'");
1901            // If a BFT sender was provided, send the round and certificate to the BFT.
1902            if let Some(cb) = self.primary_callback.get() {
1903                cb.add_new_certificate(certificate).await.with_context(|| "Failed to update the DAG from sync")?;
1904            }
1905            // Wake the round-increment task to re-check quorum.
1906            self.round_increment_notify.notify_one();
1907        }
1908        Ok(())
1909    }
1910
1911    /// Recursively syncs using the given batch header.
1912    async fn sync_with_batch_header_from_peer<const IS_SYNCING: bool, const CHECK_PREVIOUS_CERTIFICATES: bool>(
1913        &self,
1914        peer_ip: SocketAddr,
1915        batch_header: &BatchHeader<N>,
1916    ) -> Result<HashMap<TransmissionID<N>, Transmission<N>>> {
1917        // Retrieve the batch round.
1918        let batch_round = batch_header.round();
1919
1920        // If the certificate round is outdated, do not store it.
1921        if batch_round <= self.storage.gc_round() {
1922            bail!("Round {batch_round} is too far in the past")
1923        }
1924
1925        // If node is not in sync mode and the node is not synced, then return an error.
1926        if !IS_SYNCING && !self.is_synced() {
1927            bail!(
1928                "Failed to process batch header `{}` at round {batch_round} from '{peer_ip}' (node is syncing)",
1929                fmt_id(batch_header.batch_id())
1930            );
1931        }
1932
1933        // Determine if quorum threshold is reached on the batch round.
1934        let is_quorum_threshold_reached = {
1935            let authors = self.storage.get_certificate_authors_for_round(batch_round);
1936            let committee_lookback = self.ledger.get_committee_lookback_for_round(batch_round)?;
1937            committee_lookback.is_quorum_threshold_reached(&authors)
1938        };
1939
1940        // Check if our primary should move to the next round.
1941        // Note: Checking that quorum threshold is reached is important for mitigating a race condition,
1942        // whereby Narwhal requires N-f, however the BFT only requires f+1. Without this check, the primary
1943        // will advance to the next round assuming f+1, not N-f, which can lead to a network stall.
1944        let is_behind_schedule = is_quorum_threshold_reached && batch_round > self.current_round();
1945        // Check if our primary is far behind the peer.
1946        let is_peer_far_in_future = batch_round > self.current_round() + self.storage.max_gc_rounds();
1947        // If our primary is far behind the peer, update our committee to the batch round.
1948        if is_behind_schedule || is_peer_far_in_future {
1949            // If the batch round is greater than the current committee round, update the committee.
1950            self.try_increment_to_the_next_round(batch_round)
1951                .await
1952                .with_context(|| "Failed to fast forward current round")?;
1953        }
1954
1955        // Ensure the primary has all of the transmissions.
1956        let missing_transmissions_handle = self.fetch_missing_transmissions(peer_ip, batch_header);
1957
1958        // Ensure the primary has all of the previous certificates.
1959        let missing_previous_certificates_handle = self.fetch_missing_previous_certificates(peer_ip, batch_header);
1960
1961        // Wait for the missing transmissions and previous certificates to be fetched.
1962        let (missing_transmissions, missing_previous_certificates) = tokio::try_join!(
1963            missing_transmissions_handle,
1964            missing_previous_certificates_handle,
1965        ).with_context(|| format!("Failed to fetch missing transmissions and previous certificates for round {batch_round} from '{peer_ip}"))?;
1966
1967        // Iterate through the missing previous certificates sequentially.
1968        // This is done sequentially to avoid requesting a large number of certificates from peers all at once.
1969        // TODO (raychu86): Optimize this by parallelizing requests, but avoiding duplicated requests since certificates are likely shared across batches.
1970        for batch_certificate in missing_previous_certificates {
1971            // Check if the missing previous certificate is valid. This is only
1972            // needed if we are processing an incoming batch header from a peer.
1973            // For incoming certificates, validity is assured by checking the
1974            // root certificate in `process_batch_certificate_from_peer`.
1975            if CHECK_PREVIOUS_CERTIFICATES {
1976                self.storage.check_incoming_certificate(&batch_certificate)?;
1977            }
1978            // Store the batch certificate (recursively fetching any missing previous certificates).
1979            self.sync_with_certificate_from_peer::<IS_SYNCING>(peer_ip, batch_certificate).await?;
1980        }
1981        Ok(missing_transmissions)
1982    }
1983
1984    /// Fetches any missing transmissions for the specified batch header.
1985    /// If a transmission does not exist, it will be fetched from the specified peer IP.
1986    async fn fetch_missing_transmissions(
1987        &self,
1988        peer_ip: SocketAddr,
1989        batch_header: &BatchHeader<N>,
1990    ) -> Result<HashMap<TransmissionID<N>, Transmission<N>>> {
1991        // If the round is <= the GC round, return early.
1992        if batch_header.round() <= self.storage.gc_round() {
1993            return Ok(Default::default());
1994        }
1995
1996        // Ensure this batch ID is new, otherwise return early.
1997        if self.storage.contains_batch(batch_header.batch_id()) {
1998            trace!("Batch for round {} from peer has already been processed", batch_header.round());
1999            return Ok(Default::default());
2000        }
2001
2002        // Retrieve the workers.
2003        let workers = self.workers.clone();
2004
2005        // Initialize a list for the transmissions.
2006        let mut fetch_transmissions = FuturesUnordered::new();
2007
2008        // Retrieve the number of workers.
2009        let num_workers = self.num_workers();
2010        // Iterate through the transmission IDs.
2011        for transmission_id in batch_header.transmission_ids() {
2012            // If the transmission does not exist in storage, proceed to fetch the transmission.
2013            if !self.storage.contains_transmission(*transmission_id) {
2014                // Determine the worker ID.
2015                let Ok(worker_id) = assign_to_worker(*transmission_id, num_workers) else {
2016                    bail!("Unable to assign transmission ID '{transmission_id}' to a worker")
2017                };
2018                // Retrieve the worker.
2019                let Some(worker) = workers.get().expect("No workers set").get(worker_id as usize) else {
2020                    bail!("Unable to find worker {worker_id}")
2021                };
2022                // Push the callback onto the list.
2023                fetch_transmissions.push(worker.get_or_fetch_transmission(peer_ip, *transmission_id));
2024            }
2025        }
2026
2027        // Initialize a set for the transmissions.
2028        let mut transmissions = HashMap::with_capacity(fetch_transmissions.len());
2029        // Wait for all of the transmissions to be fetched.
2030        while let Some(result) = fetch_transmissions.next().await {
2031            // Retrieve the transmission.
2032            let (transmission_id, transmission) = result?;
2033            // Insert the transmission into the set.
2034            transmissions.insert(transmission_id, transmission);
2035        }
2036        // Return the transmissions.
2037        Ok(transmissions)
2038    }
2039
2040    /// Fetches any missing previous certificates for the specified batch header from the specified peer.
2041    // `BatchCertificate` contains a `OnceLock` cache which does not affect its `Hash` or `Eq`.
2042    #[allow(clippy::mutable_key_type)]
2043    async fn fetch_missing_previous_certificates(
2044        &self,
2045        peer_ip: SocketAddr,
2046        batch_header: &BatchHeader<N>,
2047    ) -> Result<HashSet<BatchCertificate<N>>> {
2048        // Retrieve the round.
2049        let round = batch_header.round();
2050        // If the previous round is 0, or is <= the GC round, return early.
2051        if round == 1 || round <= self.storage.gc_round() + 1 {
2052            return Ok(Default::default());
2053        }
2054
2055        // Fetch the missing previous certificates.
2056        let missing_previous_certificates =
2057            self.fetch_missing_certificates(peer_ip, round, batch_header.previous_certificate_ids()).await?;
2058        if !missing_previous_certificates.is_empty() {
2059            debug!(
2060                "Fetched {} missing previous certificates for round {round} from '{peer_ip}'",
2061                missing_previous_certificates.len(),
2062            );
2063        }
2064        // Return the missing previous certificates.
2065        Ok(missing_previous_certificates)
2066    }
2067
2068    /// Fetches any missing certificates for the specified batch header from the specified peer.
2069    // `BatchCertificate` contains a `OnceLock` cache which does not affect its `Hash` or `Eq`.
2070    #[allow(clippy::mutable_key_type)]
2071    async fn fetch_missing_certificates(
2072        &self,
2073        peer_ip: SocketAddr,
2074        round: u64,
2075        certificate_ids: &IndexSet<Field<N>>,
2076    ) -> Result<HashSet<BatchCertificate<N>>> {
2077        // Initialize a list for the missing certificates.
2078        let mut fetch_certificates = FuturesUnordered::new();
2079        // Initialize a set for the missing certificates.
2080        let mut missing_certificates = HashSet::default();
2081        // Iterate through the certificate IDs.
2082        for certificate_id in certificate_ids {
2083            // Check if the certificate already exists in the ledger.
2084            if self.ledger.contains_certificate(certificate_id)? {
2085                continue;
2086            }
2087            // Check if the certificate already exists in storage.
2088            if self.storage.contains_certificate(*certificate_id) {
2089                continue;
2090            }
2091            // If we have not fully processed the certificate yet, store it.
2092            if let Some(certificate) = self.storage.get_unprocessed_certificate(*certificate_id) {
2093                missing_certificates.insert(certificate);
2094            } else {
2095                // If we do not have the certificate, request it.
2096                trace!("Primary - Found a new certificate ID for round {round} from '{peer_ip}'");
2097                // TODO (howardwu): Limit the number of open requests we send to a peer.
2098                // Send an certificate request to the peer.
2099                fetch_certificates.push(self.sync.send_certificate_request(peer_ip, *certificate_id));
2100            }
2101        }
2102
2103        // If there are no certificates to fetch, return early with the existing unprocessed certificates.
2104        match fetch_certificates.is_empty() {
2105            true => return Ok(missing_certificates),
2106            false => trace!(
2107                "Fetching {} missing certificates for round {round} from '{peer_ip}'...",
2108                fetch_certificates.len(),
2109            ),
2110        }
2111
2112        // Wait for all of the missing certificates to be fetched.
2113        while let Some(result) = fetch_certificates.next().await {
2114            // Insert the missing certificate into the set.
2115            missing_certificates.insert(result?);
2116        }
2117        // Return the missing certificates.
2118        Ok(missing_certificates)
2119    }
2120}
2121
2122impl<N: Network> Primary<N> {
2123    /// Spawns a task with the given future; it should only be used for long-running tasks.
2124    fn spawn<T: Future<Output = ()> + Send + 'static>(&self, future: T) {
2125        self.handles.lock().push(tokio::spawn(future));
2126    }
2127
2128    /// Shuts down the primary.
2129    pub async fn shut_down(&self) {
2130        info!("Shutting down the primary...");
2131        // Remove the callback.
2132        self.primary_callback.clear();
2133        // Shut down the sync service.
2134        self.sync.shut_down().await;
2135        // Shut down the workers.
2136        self.workers().iter().for_each(|worker| worker.shut_down());
2137        // Abort the tasks.
2138        self.handles.lock().drain(..).for_each(|handle| handle.abort());
2139        // Save the current proposal cache to disk.
2140        let proposal_cache = {
2141            // Only persist a Certifying batch; a Certified batch will appear in pending_certificates.
2142            // Note: it is guaranteed that there are no concurrent accesses to `proposed_batch` as all
2143            // background tasks already terminated at this point.
2144            let proposal = match std::mem::replace(&mut *self.proposed_batch.write(), ProposedBatchState::None) {
2145                ProposedBatchState::Certifying(p) => Some(*p),
2146                _ => None,
2147            };
2148            let signed_proposals = self.signed_proposals.read().clone();
2149            let latest_round = proposal
2150                .as_ref()
2151                .map(Proposal::round)
2152                .unwrap_or(self.latest_proposal_timestamp.read().await.map(|(round, _)| round).unwrap_or(0));
2153            let pending_certificates = self.storage.get_pending_certificates();
2154            ProposalCache::new(latest_round, proposal, signed_proposals, pending_certificates)
2155        };
2156        if let Err(err) = proposal_cache.store(&self.node_data_dir) {
2157            error!("{}", flatten_error(err.context("Failed to store the current proposal cache")));
2158        }
2159        // Close the gateway.
2160        self.gateway.shut_down().await;
2161    }
2162}
2163
2164#[cfg(test)]
2165mod tests {
2166    use super::{proposal_task::BatchPropose as _, *};
2167
2168    use snarkos_node_bft_ledger_service::MockLedgerService;
2169    use snarkos_node_bft_storage_service::BFTMemoryService;
2170    use snarkos_node_sync::{BlockSync, locators::test_helpers::sample_block_locators};
2171    use snarkvm::{
2172        ledger::{
2173            committee::{Committee, MIN_VALIDATOR_STAKE},
2174            test_helpers::sample_execution_transaction_with_fee,
2175        },
2176        prelude::{Address, Signature},
2177    };
2178
2179    use bytes::Bytes;
2180    use indexmap::IndexSet;
2181    use rand::RngExt;
2182
2183    type CurrentNetwork = snarkvm::prelude::MainnetV0;
2184
2185    fn sample_committee(rng: &mut TestRng) -> (Vec<(SocketAddr, Account<CurrentNetwork>)>, Committee<CurrentNetwork>) {
2186        // Create a committee containing the primary's account.
2187        const COMMITTEE_SIZE: usize = 4;
2188        let mut accounts = Vec::with_capacity(COMMITTEE_SIZE);
2189        let mut members = IndexMap::new();
2190
2191        for i in 0..COMMITTEE_SIZE {
2192            let socket_addr = format!("127.0.0.1:{}", 5000 + i).parse().unwrap();
2193            let account = Account::new(rng).unwrap();
2194
2195            members.insert(account.address(), (MIN_VALIDATOR_STAKE, true, rng.random_range(0..100)));
2196            accounts.push((socket_addr, account));
2197        }
2198
2199        (accounts, Committee::<CurrentNetwork>::new(1, members).unwrap())
2200    }
2201
2202    // Returns a primary and a list of accounts in the configured committee.
2203    fn primary_with_committee(
2204        account_index: usize,
2205        accounts: &[(SocketAddr, Account<CurrentNetwork>)],
2206        committee: Committee<CurrentNetwork>,
2207        height: u32,
2208    ) -> Primary<CurrentNetwork> {
2209        let ledger = Arc::new(MockLedgerService::new_at_height(committee, height));
2210        let storage = Storage::new(ledger.clone(), Arc::new(BFTMemoryService::new()), 10).unwrap();
2211
2212        // Initialize the primary.
2213        let account = accounts[account_index].1.clone();
2214        let block_sync = Arc::new(BlockSync::new(ledger.clone(), ConnectionMode::Gateway));
2215        let primary =
2216            Primary::new(account, storage, ledger, block_sync, None, &[], false, NodeDataDir::new_test(None), None)
2217                .unwrap();
2218
2219        // Construct a worker instance.
2220        let worker = Worker::new(
2221            0, // id
2222            Arc::new(primary.gateway.clone()),
2223            primary.storage.clone(),
2224            primary.ledger.clone(),
2225            primary.proposed_batch.clone(),
2226        )
2227        .unwrap();
2228        let _ = primary.workers.set(vec![worker]);
2229        for a in accounts.iter().skip(account_index) {
2230            primary.gateway.insert_connected_peer(a.0, a.0, a.1.address());
2231        }
2232
2233        primary
2234    }
2235
2236    fn primary_without_handlers(
2237        rng: &mut TestRng,
2238    ) -> (Primary<CurrentNetwork>, Vec<(SocketAddr, Account<CurrentNetwork>)>) {
2239        let (accounts, committee) = sample_committee(rng);
2240        let primary = primary_with_committee(
2241            0, // index of primary's account
2242            &accounts,
2243            committee,
2244            CurrentNetwork::CONSENSUS_HEIGHT(ConsensusVersion::V1).unwrap(),
2245        );
2246
2247        (primary, accounts)
2248    }
2249
2250    // Creates a mock solution.
2251    fn sample_unconfirmed_solution(rng: &mut TestRng) -> (SolutionID<CurrentNetwork>, Data<Solution<CurrentNetwork>>) {
2252        // Sample a random fake solution ID.
2253        let solution_id = rng.random::<u64>().into();
2254        // Vary the size of the solutions.
2255        let size = rng.random_range(1024..10 * 1024);
2256        // Sample random fake solution bytes.
2257        let vec: Vec<u8> = (0..size).map(|_| rng.random::<u8>()).collect();
2258        let solution = Data::Buffer(Bytes::from(vec));
2259        // Return the solution ID and solution.
2260        (solution_id, solution)
2261    }
2262
2263    // Samples a test transaction.
2264    fn sample_unconfirmed_transaction(
2265        rng: &mut TestRng,
2266    ) -> (<CurrentNetwork as Network>::TransactionID, Data<Transaction<CurrentNetwork>>) {
2267        let transaction = sample_execution_transaction_with_fee(false, rng, 0);
2268        let id = transaction.id();
2269
2270        (id, Data::Object(transaction))
2271    }
2272
2273    // Creates a batch proposal with one solution and one transaction.
2274    fn create_test_proposal(
2275        author: &Account<CurrentNetwork>,
2276        committee: Committee<CurrentNetwork>,
2277        round: u64,
2278        previous_certificate_ids: IndexSet<Field<CurrentNetwork>>,
2279        timestamp: i64,
2280        num_transactions: u64,
2281        rng: &mut TestRng,
2282    ) -> Proposal<CurrentNetwork> {
2283        let mut transmission_ids = IndexSet::new();
2284        let mut transmissions = IndexMap::new();
2285
2286        // Prepare the solution and insert into the sets.
2287        let (solution_id, solution) = sample_unconfirmed_solution(rng);
2288        let solution_checksum = solution.to_checksum::<CurrentNetwork>().unwrap();
2289        let solution_transmission_id = (solution_id, solution_checksum).into();
2290        transmission_ids.insert(solution_transmission_id);
2291        transmissions.insert(solution_transmission_id, Transmission::Solution(solution));
2292
2293        // Prepare the transactions and insert into the sets.
2294        for _ in 0..num_transactions {
2295            let (transaction_id, transaction) = sample_unconfirmed_transaction(rng);
2296            let transaction_checksum = transaction.to_checksum::<CurrentNetwork>().unwrap();
2297            let transaction_transmission_id = (&transaction_id, &transaction_checksum).into();
2298            transmission_ids.insert(transaction_transmission_id);
2299            transmissions.insert(transaction_transmission_id, Transmission::Transaction(transaction));
2300        }
2301
2302        // Retrieve the private key.
2303        let private_key = author.private_key();
2304        // Sign the batch header.
2305        let batch_header = BatchHeader::new(
2306            private_key,
2307            round,
2308            timestamp,
2309            committee.id(),
2310            transmission_ids,
2311            previous_certificate_ids,
2312            rng,
2313        )
2314        .unwrap();
2315        // Construct the proposal.
2316        Proposal::new(committee, batch_header, transmissions).unwrap()
2317    }
2318
2319    // Creates a signature of the primary's current proposal for each committee member (excluding
2320    // the primary).
2321    fn peer_signatures_for_proposal(
2322        primary: &Primary<CurrentNetwork>,
2323        accounts: &[(SocketAddr, Account<CurrentNetwork>)],
2324        rng: &mut TestRng,
2325    ) -> Vec<(SocketAddr, BatchSignature<CurrentNetwork>)> {
2326        // Each committee member signs the batch.
2327        let mut signatures = Vec::with_capacity(accounts.len() - 1);
2328        for (socket_addr, account) in accounts {
2329            if account.address() == primary.gateway.account().address() {
2330                continue;
2331            }
2332            let batch_id = primary.proposed_batch.read().as_proposal().unwrap().batch_id();
2333            let signature = account.sign(&[batch_id], rng).unwrap();
2334            signatures.push((*socket_addr, BatchSignature::new(batch_id, signature)));
2335        }
2336
2337        signatures
2338    }
2339
2340    /// Creates a signature of the batch ID for each committee member (excluding the primary).
2341    fn peer_signatures_for_batch(
2342        primary_address: Address<CurrentNetwork>,
2343        accounts: &[(SocketAddr, Account<CurrentNetwork>)],
2344        batch_id: Field<CurrentNetwork>,
2345        rng: &mut TestRng,
2346    ) -> IndexSet<Signature<CurrentNetwork>> {
2347        let mut signatures = IndexSet::new();
2348        for (_, account) in accounts {
2349            if account.address() == primary_address {
2350                continue;
2351            }
2352            let signature = account.sign(&[batch_id], rng).unwrap();
2353            signatures.insert(signature);
2354        }
2355        signatures
2356    }
2357
2358    // Creates a batch certificate.
2359    fn create_batch_certificate(
2360        primary_address: Address<CurrentNetwork>,
2361        accounts: &[(SocketAddr, Account<CurrentNetwork>)],
2362        round: u64,
2363        previous_certificate_ids: IndexSet<Field<CurrentNetwork>>,
2364        rng: &mut TestRng,
2365    ) -> (BatchCertificate<CurrentNetwork>, HashMap<TransmissionID<CurrentNetwork>, Transmission<CurrentNetwork>>) {
2366        let timestamp = now();
2367
2368        let author =
2369            accounts.iter().find(|&(_, acct)| acct.address() == primary_address).map(|(_, acct)| acct.clone()).unwrap();
2370        let private_key = author.private_key();
2371
2372        let committee_id = Field::rand(rng);
2373        let (solution_id, solution) = sample_unconfirmed_solution(rng);
2374        let (transaction_id, transaction) = sample_unconfirmed_transaction(rng);
2375        let solution_checksum = solution.to_checksum::<CurrentNetwork>().unwrap();
2376        let transaction_checksum = transaction.to_checksum::<CurrentNetwork>().unwrap();
2377
2378        let solution_transmission_id = (solution_id, solution_checksum).into();
2379        let transaction_transmission_id = (&transaction_id, &transaction_checksum).into();
2380
2381        let transmission_ids = [solution_transmission_id, transaction_transmission_id].into();
2382        let transmissions = [
2383            (solution_transmission_id, Transmission::Solution(solution)),
2384            (transaction_transmission_id, Transmission::Transaction(transaction)),
2385        ]
2386        .into();
2387
2388        let batch_header = BatchHeader::new(
2389            private_key,
2390            round,
2391            timestamp,
2392            committee_id,
2393            transmission_ids,
2394            previous_certificate_ids,
2395            rng,
2396        )
2397        .unwrap();
2398        let signatures = peer_signatures_for_batch(primary_address, accounts, batch_header.batch_id(), rng);
2399        let certificate = BatchCertificate::<CurrentNetwork>::from(batch_header, signatures).unwrap();
2400        (certificate, transmissions)
2401    }
2402
2403    // Create a certificate chain up to, but not including, the specified round in the primary storage.
2404    fn store_certificate_chain(
2405        primary: &Primary<CurrentNetwork>,
2406        accounts: &[(SocketAddr, Account<CurrentNetwork>)],
2407        round: u64,
2408        rng: &mut TestRng,
2409    ) -> IndexSet<Field<CurrentNetwork>> {
2410        let mut previous_certificates = IndexSet::<Field<CurrentNetwork>>::new();
2411        let mut next_certificates = IndexSet::<Field<CurrentNetwork>>::new();
2412        for cur_round in 1..round {
2413            for (_, account) in accounts.iter() {
2414                let (certificate, transmissions) = create_batch_certificate(
2415                    account.address(),
2416                    accounts,
2417                    cur_round,
2418                    previous_certificates.clone(),
2419                    rng,
2420                );
2421                next_certificates.insert(certificate.id());
2422                assert!(primary.storage.insert_certificate(certificate, transmissions, Default::default()).is_ok());
2423            }
2424
2425            assert!(primary.storage.increment_to_next_round(cur_round).is_ok());
2426            previous_certificates = next_certificates;
2427            next_certificates = IndexSet::<Field<CurrentNetwork>>::new();
2428        }
2429
2430        previous_certificates
2431    }
2432
2433    // Insert the account socket addresses into the resolver so that
2434    // they are recognized as "connected".
2435    fn map_account_addresses(primary: &Primary<CurrentNetwork>, accounts: &[(SocketAddr, Account<CurrentNetwork>)]) {
2436        // First account is primary, which doesn't need to resolve.
2437        for (addr, acct) in accounts.iter().skip(1) {
2438            primary.gateway.resolver().write().insert_peer(*addr, *addr, Some(acct.address()));
2439        }
2440    }
2441
2442    #[test_log::test(tokio::test)]
2443    async fn test_propose_batch() {
2444        let mut rng = TestRng::default();
2445        let (primary, _) = primary_without_handlers(&mut rng);
2446
2447        // Check there is no batch currently proposed.
2448        assert!(primary.proposed_batch.read().is_none());
2449
2450        // Generate a solution and a transaction.
2451        let (solution_id, solution) = sample_unconfirmed_solution(&mut rng);
2452        let (transaction_id, transaction) = sample_unconfirmed_transaction(&mut rng);
2453
2454        // Store it on one of the workers.
2455        primary.workers()[0].process_unconfirmed_solution(solution_id, solution).await.unwrap();
2456        primary.workers()[0].process_unconfirmed_transaction(transaction_id, transaction).await.unwrap();
2457
2458        // Try to propose a batch again. This time, it should succeed.
2459        assert!(primary.propose_batch().await.is_ok());
2460        assert!(primary.proposed_batch.read().is_proposed());
2461    }
2462
2463    #[test_log::test(tokio::test)]
2464    async fn test_propose_batch_with_no_transmissions() {
2465        let mut rng = TestRng::default();
2466        let (primary, _) = primary_without_handlers(&mut rng);
2467
2468        // Check there is no batch currently proposed.
2469        assert!(primary.proposed_batch.read().is_none());
2470
2471        // Try to propose a batch with no transmissions.
2472        assert!(primary.propose_batch().await.is_ok());
2473        assert!(primary.proposed_batch.read().is_proposed());
2474    }
2475
2476    #[test_log::test(tokio::test)]
2477    async fn test_propose_batch_in_round() {
2478        let round = 3;
2479        let mut rng = TestRng::default();
2480        let (primary, accounts) = primary_without_handlers(&mut rng);
2481
2482        // Fill primary storage.
2483        store_certificate_chain(&primary, &accounts, round, &mut rng);
2484
2485        // Sleep for a while to ensure the primary is ready to propose the next round.
2486        tokio::time::sleep(MIN_BATCH_DELAY).await;
2487
2488        // Generate a solution and a transaction.
2489        let (solution_id, solution) = sample_unconfirmed_solution(&mut rng);
2490        let (transaction_id, transaction) = sample_unconfirmed_transaction(&mut rng);
2491
2492        // Store it on one of the workers.
2493        primary.workers()[0].process_unconfirmed_solution(solution_id, solution).await.unwrap();
2494        primary.workers()[0].process_unconfirmed_transaction(transaction_id, transaction).await.unwrap();
2495
2496        // Propose a batch again. This time, it should succeed.
2497        assert!(primary.propose_batch().await.is_ok());
2498        assert!(primary.proposed_batch.read().is_proposed());
2499    }
2500
2501    #[test_log::test(tokio::test)]
2502    async fn test_propose_batch_skip_transmissions_from_previous_certificates() {
2503        let round = 3;
2504        let prev_round = round - 1;
2505        let mut rng = TestRng::default();
2506        let (primary, accounts) = primary_without_handlers(&mut rng);
2507        let peer_account = &accounts[1];
2508        let peer_ip = peer_account.0;
2509
2510        // Fill primary storage.
2511        store_certificate_chain(&primary, &accounts, round, &mut rng);
2512
2513        // Get transmissions from previous certificates.
2514        let previous_certificate_ids: IndexSet<_> = primary.storage.get_certificate_ids_for_round(prev_round);
2515
2516        // Track the number of transmissions in the previous round.
2517        let mut num_transmissions_in_previous_round = 0;
2518
2519        // Generate a solution and a transaction.
2520        let (solution_commitment, solution) = sample_unconfirmed_solution(&mut rng);
2521        let (transaction_id, transaction) = sample_unconfirmed_transaction(&mut rng);
2522        let solution_checksum = solution.to_checksum::<CurrentNetwork>().unwrap();
2523        let transaction_checksum = transaction.to_checksum::<CurrentNetwork>().unwrap();
2524
2525        // Store it on one of the workers.
2526        primary.workers()[0].process_unconfirmed_solution(solution_commitment, solution).await.unwrap();
2527        primary.workers()[0].process_unconfirmed_transaction(transaction_id, transaction).await.unwrap();
2528
2529        // Check that the worker has 2 transmissions.
2530        assert_eq!(primary.workers()[0].num_transmissions(), 2);
2531
2532        // Create certificates for the current round and add the transmissions to the worker before inserting the certificate to storage.
2533        for (_, account) in accounts.iter() {
2534            let (certificate, transmissions) = create_batch_certificate(
2535                account.address(),
2536                &accounts,
2537                round,
2538                previous_certificate_ids.clone(),
2539                &mut rng,
2540            );
2541
2542            // Add the transmissions to the worker.
2543            for (transmission_id, transmission) in transmissions.iter() {
2544                primary.workers()[0].process_transmission_from_peer(peer_ip, *transmission_id, transmission.clone());
2545            }
2546
2547            // Insert the certificate to storage.
2548            num_transmissions_in_previous_round += transmissions.len();
2549            primary.storage.insert_certificate(certificate, transmissions, Default::default()).unwrap();
2550        }
2551
2552        // Sleep for a while to ensure the primary is ready to propose the next round.
2553        tokio::time::sleep(MIN_BATCH_DELAY).await;
2554
2555        // Advance to the next round.
2556        assert!(primary.storage.increment_to_next_round(round).is_ok());
2557
2558        // Check that the worker has `num_transmissions_in_previous_round + 2` transmissions.
2559        assert_eq!(primary.workers()[0].num_transmissions(), num_transmissions_in_previous_round + 2);
2560
2561        // Propose the batch.
2562        assert!(primary.propose_batch().await.is_ok());
2563
2564        // Check that the proposal only contains the new transmissions that were not in previous certificates.
2565        let proposed_transmissions = primary.proposed_batch.read().as_proposal().unwrap().transmissions().clone();
2566        assert_eq!(proposed_transmissions.len(), 2);
2567        assert!(proposed_transmissions.contains_key(&TransmissionID::Solution(solution_commitment, solution_checksum)));
2568        assert!(
2569            proposed_transmissions.contains_key(&TransmissionID::Transaction(transaction_id, transaction_checksum))
2570        );
2571    }
2572
2573    #[test_log::test(tokio::test)]
2574    async fn test_propose_batch_over_spend_limit() {
2575        let mut rng = TestRng::default();
2576
2577        // Create a primary to test spend limit backwards compatibility with V4.
2578        let (accounts, committee) = sample_committee(&mut rng);
2579        let primary = primary_with_committee(
2580            0,
2581            &accounts,
2582            committee.clone(),
2583            CurrentNetwork::CONSENSUS_HEIGHT(ConsensusVersion::V4).unwrap(),
2584        );
2585
2586        // Check there is no batch currently proposed.
2587        assert!(primary.proposed_batch.read().is_none());
2588        // Check the workers are empty.
2589        primary.workers().iter().for_each(|worker| assert!(worker.transmissions().is_empty()));
2590
2591        // Generate a solution and transactions.
2592        let (solution_id, solution) = sample_unconfirmed_solution(&mut rng);
2593        primary.workers()[0].process_unconfirmed_solution(solution_id, solution).await.unwrap();
2594
2595        for _i in 0..5 {
2596            let (transaction_id, transaction) = sample_unconfirmed_transaction(&mut rng);
2597            // Store it on one of the workers.
2598            primary.workers()[0].process_unconfirmed_transaction(transaction_id, transaction).await.unwrap();
2599        }
2600
2601        // Try to propose a batch again. This time, it should succeed.
2602        assert!(primary.propose_batch().await.is_ok());
2603        // Expect 2/5 transactions to be included in the proposal in addition to the solution.
2604        assert_eq!(primary.proposed_batch.read().as_proposal().unwrap().transmissions().len(), 3);
2605        // Check the transmissions were correctly drained from the workers.
2606        assert_eq!(primary.workers().iter().map(|worker| worker.transmissions().len()).sum::<usize>(), 3);
2607    }
2608
2609    #[test_log::test(tokio::test)]
2610    async fn test_batch_propose_from_peer() {
2611        let mut rng = TestRng::default();
2612        let (primary, accounts) = primary_without_handlers(&mut rng);
2613
2614        // Create a valid proposal with an author that isn't the primary.
2615        let round = 1;
2616        let peer_account = &accounts[1];
2617        let peer_ip = peer_account.0;
2618        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
2619        let proposal = create_test_proposal(
2620            &peer_account.1,
2621            primary.ledger.current_committee().unwrap(),
2622            round,
2623            Default::default(),
2624            timestamp,
2625            1,
2626            &mut rng,
2627        );
2628
2629        // Make sure the primary is aware of the transmissions in the proposal.
2630        for (transmission_id, transmission) in proposal.transmissions() {
2631            primary.workers()[0].process_transmission_from_peer(peer_ip, *transmission_id, transmission.clone())
2632        }
2633
2634        // The author must be known to resolver to pass propose checks.
2635        primary.gateway.resolver().write().insert_peer(peer_ip, peer_ip, Some(peer_account.1.address()));
2636
2637        // The primary will only consider itself synced if we received
2638        // block locators from a peer.
2639        primary.sync.testing_only_update_peer_locators_testing_only(peer_ip, sample_block_locators(20)).unwrap();
2640        primary.sync.testing_only_set_sync_height_testing_only(20);
2641
2642        // Try to process the batch proposal from the peer, should succeed.
2643        assert!(
2644            primary.process_batch_propose_from_peer(peer_ip, (*proposal.batch_header()).clone().into()).await.is_ok()
2645        );
2646    }
2647
2648    /// A proposal for the current round is left in place; once the round advances past it, the same
2649    /// proposal is cleared and its transmissions are returned to the workers.
2650    #[test_log::test(tokio::test)]
2651    async fn test_check_proposed_batch_for_expiration() {
2652        let mut rng = TestRng::default();
2653        let (primary, accounts) = primary_without_handlers(&mut rng);
2654
2655        // Advance storage to the round the proposal is made for.
2656        let round = 3;
2657        let previous_certificates = store_certificate_chain(&primary, &accounts, round, &mut rng);
2658        assert_eq!(primary.current_round(), round);
2659
2660        // Propose a batch for the current round.
2661        let proposal = create_test_proposal(
2662            &accounts[0].1,
2663            primary.ledger.current_committee().unwrap(),
2664            round,
2665            previous_certificates,
2666            now(),
2667            1,
2668            &mut rng,
2669        );
2670        let batch_id = proposal.batch_id();
2671        let transmission_ids: Vec<_> = proposal.transmissions().keys().copied().collect();
2672        assert!(!transmission_ids.is_empty());
2673        *primary.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(proposal));
2674
2675        // The proposal is for the current round, so it is not expired and must be left alone.
2676        primary.check_proposed_batch_for_expiration().unwrap();
2677        assert_eq!(primary.proposed_batch.read().as_proposal().unwrap().batch_id(), batch_id);
2678
2679        // Advance the round past the proposal.
2680        primary.storage.increment_to_next_round(round).unwrap();
2681        assert!(primary.current_round() > round);
2682
2683        // The proposal is now stale, so it is cleared and its transmissions go back to the workers.
2684        primary.check_proposed_batch_for_expiration().unwrap();
2685        assert!(primary.proposed_batch.read().is_none());
2686        for transmission_id in transmission_ids {
2687            assert!(primary.workers()[0].contains_transmission(transmission_id));
2688        }
2689    }
2690
2691    /// The signed-proposal cache advances, and only advances: an older round must never overwrite a
2692    /// newer one, and a round already cached must never be signed twice.
2693    #[test_log::test(tokio::test)]
2694    async fn test_signed_proposal_cache_never_moves_backwards() {
2695        let mut rng = TestRng::default();
2696        let (primary, accounts) = primary_without_handlers(&mut rng);
2697        let account = primary.gateway.account().clone();
2698        let author = accounts[1].1.address();
2699
2700        // The first proposal seen for a validator is cached.
2701        let id_2 = Field::rand(&mut rng);
2702        let sig_2 = account.sign(&[id_2], &mut rng).unwrap();
2703        assert!(primary.cache_signed_proposal(author, 2, id_2, sig_2));
2704        assert_eq!(primary.signed_proposals.read().get(&author).copied().unwrap(), (2, id_2, sig_2));
2705
2706        // A newer round advances the cache.
2707        let id_3 = Field::rand(&mut rng);
2708        let sig_3 = account.sign(&[id_3], &mut rng).unwrap();
2709        assert!(primary.cache_signed_proposal(author, 3, id_3, sig_3));
2710        assert_eq!(primary.signed_proposals.read().get(&author).copied().unwrap(), (3, id_3, sig_3));
2711
2712        // A conflicting batch for the cached round is refused, and leaves the cache intact.
2713        let conflicting_id = Field::rand(&mut rng);
2714        let conflicting_sig = account.sign(&[conflicting_id], &mut rng).unwrap();
2715        assert!(!primary.cache_signed_proposal(author, 3, conflicting_id, conflicting_sig));
2716        assert_eq!(primary.signed_proposals.read().get(&author).copied().unwrap(), (3, id_3, sig_3));
2717
2718        // The regression: a handler for an older round must not downgrade the cache.
2719        let stale_id = Field::rand(&mut rng);
2720        let stale_sig = account.sign(&[stale_id], &mut rng).unwrap();
2721        assert!(!primary.cache_signed_proposal(author, 2, stale_id, stale_sig));
2722        assert_eq!(
2723            primary.signed_proposals.read().get(&author).copied().unwrap(),
2724            (3, id_3, sig_3),
2725            "a handler for an older round downgraded the signed-proposal cache"
2726        );
2727
2728        // Entries for other validators are independent.
2729        let other_author = accounts[2].1.address();
2730        let other_id = Field::rand(&mut rng);
2731        let other_sig = account.sign(&[other_id], &mut rng).unwrap();
2732        assert!(primary.cache_signed_proposal(other_author, 1, other_id, other_sig));
2733        assert_eq!(primary.signed_proposals.read().get(&author).copied().unwrap().0, 3);
2734    }
2735
2736    #[test_log::test(tokio::test)]
2737    async fn test_batch_propose_from_peer_when_not_synced() {
2738        let mut rng = TestRng::default();
2739        let (primary, accounts) = primary_without_handlers(&mut rng);
2740
2741        // Create a valid proposal with an author that isn't the primary.
2742        let round = 1;
2743        let peer_account = &accounts[1];
2744        let peer_ip = peer_account.0;
2745        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
2746        let proposal = create_test_proposal(
2747            &peer_account.1,
2748            primary.ledger.current_committee().unwrap(),
2749            round,
2750            Default::default(),
2751            timestamp,
2752            1,
2753            &mut rng,
2754        );
2755
2756        // Make sure the primary is aware of the transmissions in the proposal.
2757        for (transmission_id, transmission) in proposal.transmissions() {
2758            primary.workers()[0].process_transmission_from_peer(peer_ip, *transmission_id, transmission.clone())
2759        }
2760
2761        // The author must be known to resolver to pass propose checks.
2762        primary.gateway.resolver().write().insert_peer(peer_ip, peer_ip, Some(peer_account.1.address()));
2763
2764        // Add a high block locator to indicate we are not synced.
2765        primary.sync.testing_only_update_peer_locators_testing_only(peer_ip, sample_block_locators(20)).unwrap();
2766
2767        // Try to process the batch proposal from the peer, should fail
2768        assert!(
2769            primary.process_batch_propose_from_peer(peer_ip, (*proposal.batch_header()).clone().into()).await.is_err()
2770        );
2771    }
2772
2773    #[test_log::test(tokio::test)]
2774    async fn test_batch_propose_from_peer_in_round() {
2775        let round = 2;
2776        let mut rng = TestRng::default();
2777        let (primary, accounts) = primary_without_handlers(&mut rng);
2778
2779        // Generate certificates.
2780        let previous_certificates = store_certificate_chain(&primary, &accounts, round, &mut rng);
2781
2782        // Create a valid proposal with an author that isn't the primary.
2783        let peer_account = &accounts[1];
2784        let peer_ip = peer_account.0;
2785        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
2786        let proposal = create_test_proposal(
2787            &peer_account.1,
2788            primary.ledger.current_committee().unwrap(),
2789            round,
2790            previous_certificates,
2791            timestamp,
2792            1,
2793            &mut rng,
2794        );
2795
2796        // Make sure the primary is aware of the transmissions in the proposal.
2797        for (transmission_id, transmission) in proposal.transmissions() {
2798            primary.workers()[0].process_transmission_from_peer(peer_ip, *transmission_id, transmission.clone())
2799        }
2800
2801        // The author must be known to resolver to pass propose checks.
2802        primary.gateway.resolver().write().insert_peer(peer_ip, peer_ip, Some(peer_account.1.address()));
2803
2804        // The primary will only consider itself synced if we received
2805        // block locators from a peer.
2806        primary.sync.testing_only_update_peer_locators_testing_only(peer_ip, sample_block_locators(20)).unwrap();
2807        primary.sync.testing_only_set_sync_height_testing_only(20);
2808
2809        // Try to process the batch proposal from the peer, should succeed.
2810        primary.process_batch_propose_from_peer(peer_ip, (*proposal.batch_header()).clone().into()).await.unwrap();
2811    }
2812
2813    #[test_log::test(tokio::test)]
2814    async fn test_batch_propose_from_peer_wrong_round() {
2815        let mut rng = TestRng::default();
2816        let (primary, accounts) = primary_without_handlers(&mut rng);
2817
2818        // Create a valid proposal with an author that isn't the primary.
2819        let round = 1;
2820        let peer_account = &accounts[1];
2821        let peer_ip = peer_account.0;
2822        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
2823        let proposal = create_test_proposal(
2824            &peer_account.1,
2825            primary.ledger.current_committee().unwrap(),
2826            round,
2827            Default::default(),
2828            timestamp,
2829            1,
2830            &mut rng,
2831        );
2832
2833        // Make sure the primary is aware of the transmissions in the proposal.
2834        for (transmission_id, transmission) in proposal.transmissions() {
2835            primary.workers()[0].process_transmission_from_peer(peer_ip, *transmission_id, transmission.clone())
2836        }
2837
2838        // The author must be known to resolver to pass propose checks.
2839        primary.gateway.resolver().write().insert_peer(peer_ip, peer_ip, Some(peer_account.1.address()));
2840        // The primary must be considered synced.
2841        primary.sync.testing_only_update_peer_locators_testing_only(peer_ip, sample_block_locators(20)).unwrap();
2842        primary.sync.testing_only_set_sync_height_testing_only(20);
2843
2844        // Try to process the batch proposal from the peer, should error.
2845        assert!(
2846            primary
2847                .process_batch_propose_from_peer(peer_ip, BatchPropose {
2848                    round: round + 1,
2849                    batch_header: Data::Object(proposal.batch_header().clone())
2850                })
2851                .await
2852                .is_err()
2853        );
2854    }
2855
2856    #[test_log::test(tokio::test)]
2857    async fn test_batch_propose_from_peer_in_round_wrong_round() {
2858        let round = 4;
2859        let mut rng = TestRng::default();
2860        let (primary, accounts) = primary_without_handlers(&mut rng);
2861
2862        // Generate certificates.
2863        let previous_certificates = store_certificate_chain(&primary, &accounts, round, &mut rng);
2864
2865        // Create a valid proposal with an author that isn't the primary.
2866        let peer_account = &accounts[1];
2867        let peer_ip = peer_account.0;
2868        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
2869        let proposal = create_test_proposal(
2870            &peer_account.1,
2871            primary.ledger.current_committee().unwrap(),
2872            round,
2873            previous_certificates,
2874            timestamp,
2875            1,
2876            &mut rng,
2877        );
2878
2879        // Make sure the primary is aware of the transmissions in the proposal.
2880        for (transmission_id, transmission) in proposal.transmissions() {
2881            primary.workers()[0].process_transmission_from_peer(peer_ip, *transmission_id, transmission.clone())
2882        }
2883
2884        // The author must be known to resolver to pass propose checks.
2885        primary.gateway.resolver().write().insert_peer(peer_ip, peer_ip, Some(peer_account.1.address()));
2886        // The primary must be considered synced.
2887        primary.sync.testing_only_update_peer_locators_testing_only(peer_ip, sample_block_locators(0)).unwrap();
2888        primary.sync.testing_only_set_sync_height_testing_only(0);
2889
2890        // Try to process the batch proposal from the peer, should error.
2891        assert!(
2892            primary
2893                .process_batch_propose_from_peer(peer_ip, BatchPropose {
2894                    round: round + 1,
2895                    batch_header: Data::Object(proposal.batch_header().clone())
2896                })
2897                .await
2898                .is_err()
2899        );
2900    }
2901
2902    /// Tests that the minimum batch delay is enforced as expected, i.e., that proposals with timestamps that are too close to the previous proposal are rejected.
2903    #[test_log::test(tokio::test)]
2904    async fn test_batch_propose_from_peer_with_past_timestamp() {
2905        let round = 2;
2906        let mut rng = TestRng::default();
2907        let (primary, accounts) = primary_without_handlers(&mut rng);
2908
2909        // Generate certificates.
2910        let previous_certificates = store_certificate_chain(&primary, &accounts, round, &mut rng);
2911
2912        // Create a valid proposal with an author that isn't the primary.
2913        let peer_account = &accounts[1];
2914        let peer_ip = peer_account.0;
2915
2916        // Use a timestamp that is too early.
2917        // Set it to something that is less than the minimum batch delay
2918        // Note, that the minimum delay is currently 1, so this will be equal to the last timestamp
2919        let last_timestamp = primary
2920            .storage
2921            .get_certificate_for_round_with_author(round - 1, peer_account.1.address())
2922            .expect("No previous proposal exists")
2923            .timestamp();
2924        let invalid_timestamp = last_timestamp + (MIN_BATCH_DELAY.as_secs() as i64) - 1;
2925
2926        let proposal = create_test_proposal(
2927            &peer_account.1,
2928            primary.ledger.current_committee().unwrap(),
2929            round,
2930            previous_certificates,
2931            invalid_timestamp,
2932            1,
2933            &mut rng,
2934        );
2935
2936        // Make sure the primary is aware of the transmissions in the proposal.
2937        for (transmission_id, transmission) in proposal.transmissions() {
2938            primary.workers()[0].process_transmission_from_peer(peer_ip, *transmission_id, transmission.clone())
2939        }
2940
2941        // The author must be known to resolver to pass propose checks.
2942        primary.gateway.resolver().write().insert_peer(peer_ip, peer_ip, Some(peer_account.1.address()));
2943        // The primary must be considered synced.
2944        primary.sync.testing_only_update_peer_locators_testing_only(peer_ip, sample_block_locators(0)).unwrap();
2945        primary.sync.testing_only_set_sync_height_testing_only(0);
2946
2947        // Try to process the batch proposal from the peer, should error.
2948        assert!(
2949            primary.process_batch_propose_from_peer(peer_ip, (*proposal.batch_header()).clone().into()).await.is_err()
2950        );
2951    }
2952
2953    #[test_log::test(tokio::test)]
2954    async fn test_propose_batch_with_storage_round_behind_proposal_lock() {
2955        let round = 3;
2956        let mut rng = TestRng::default();
2957        let (primary, _) = primary_without_handlers(&mut rng);
2958
2959        // Check there is no batch currently proposed.
2960        assert!(primary.proposed_batch.read().is_none());
2961
2962        // Generate a solution and a transaction.
2963        let (solution_id, solution) = sample_unconfirmed_solution(&mut rng);
2964        let (transaction_id, transaction) = sample_unconfirmed_transaction(&mut rng);
2965
2966        // Store it on one of the workers.
2967        primary.workers()[0].process_unconfirmed_solution(solution_id, solution).await.unwrap();
2968        primary.workers()[0].process_unconfirmed_transaction(transaction_id, transaction).await.unwrap();
2969
2970        // Set the proposal lock to a round ahead of the storage.
2971        let (old_proposal_round, old_proposal_timestamp) = primary
2972            .latest_proposal_timestamp
2973            .read()
2974            .await
2975            .map(|(round, timestamp)| (round, timestamp))
2976            .unwrap_or((0, 0));
2977        *primary.latest_proposal_timestamp.write().await =
2978            Some((round + 1, old_proposal_timestamp + MIN_BATCH_DELAY.as_secs() as i64));
2979
2980        // Propose a batch and enforce that it fails.
2981        assert!(primary.propose_batch().await.is_ok());
2982        assert!(primary.proposed_batch.read().is_none());
2983
2984        // Set the proposal lock back to the old round.
2985        *primary.latest_proposal_timestamp.write().await = Some((old_proposal_round, old_proposal_timestamp));
2986
2987        // Try to propose a batch again. This time, it should succeed.
2988        assert!(primary.propose_batch().await.is_ok());
2989        assert!(primary.proposed_batch.read().is_proposed());
2990    }
2991
2992    #[test_log::test(tokio::test)]
2993    async fn test_propose_batch_with_storage_round_behind_proposal() {
2994        let round = 5;
2995        let mut rng = TestRng::default();
2996        let (primary, accounts) = primary_without_handlers(&mut rng);
2997
2998        // Generate previous certificates.
2999        let previous_certificates = store_certificate_chain(&primary, &accounts, round, &mut rng);
3000
3001        // Create a valid proposal.
3002        let timestamp = now();
3003        let proposal = create_test_proposal(
3004            primary.gateway.account(),
3005            primary.ledger.current_committee().unwrap(),
3006            round + 1,
3007            previous_certificates,
3008            timestamp,
3009            1,
3010            &mut rng,
3011        );
3012
3013        // Store the proposal on the primary.
3014        *primary.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(proposal));
3015
3016        // Try to propose a batch will terminate early because the storage is behind the proposal.
3017        assert!(primary.propose_batch().await.is_ok());
3018        assert!(primary.proposed_batch.read().is_proposed());
3019        assert!(primary.proposed_batch.read().as_proposal().unwrap().round() > primary.current_round());
3020    }
3021
3022    #[test_log::test(tokio::test(flavor = "multi_thread"))]
3023    async fn test_batch_signature_from_peer() {
3024        let mut rng = TestRng::default();
3025        let (primary, accounts) = primary_without_handlers(&mut rng);
3026        map_account_addresses(&primary, &accounts);
3027
3028        // Create a valid proposal.
3029        let round = 1;
3030        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
3031        let proposal = create_test_proposal(
3032            primary.gateway.account(),
3033            primary.ledger.current_committee().unwrap(),
3034            round,
3035            Default::default(),
3036            timestamp,
3037            1,
3038            &mut rng,
3039        );
3040
3041        // Store the proposal on the primary.
3042        *primary.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(proposal));
3043
3044        // Each committee member signs the batch.
3045        let signatures = peer_signatures_for_proposal(&primary, &accounts, &mut rng);
3046
3047        // Have the primary process the signatures.
3048        for (socket_addr, signature) in signatures {
3049            primary.process_batch_signature_from_peer(socket_addr, signature).await.unwrap();
3050        }
3051
3052        // Check the certificate was created and stored by the primary.
3053        assert!(primary.storage.contains_certificate_in_round_from(round, primary.gateway.account().address()));
3054        // Manually attempt round advancement (because the handler is not running).
3055        primary.try_increment_to_the_next_round(round + 1).await.unwrap();
3056        // Check the round was incremented.
3057        assert_eq!(primary.current_round(), round + 1);
3058    }
3059
3060    #[test_log::test(tokio::test(flavor = "multi_thread"))]
3061    async fn test_batch_signature_from_peer_in_round() {
3062        let round = 5;
3063        let mut rng = TestRng::default();
3064        let (primary, accounts) = primary_without_handlers(&mut rng);
3065        map_account_addresses(&primary, &accounts);
3066
3067        // Generate certificates.
3068        let previous_certificates = store_certificate_chain(&primary, &accounts, round, &mut rng);
3069
3070        // Create a valid proposal.
3071        let timestamp = now();
3072        let proposal = create_test_proposal(
3073            primary.gateway.account(),
3074            primary.ledger.current_committee().unwrap(),
3075            round,
3076            previous_certificates,
3077            timestamp,
3078            1,
3079            &mut rng,
3080        );
3081
3082        // Store the proposal on the primary.
3083        *primary.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(proposal));
3084
3085        // Each committee member signs the batch.
3086        let signatures = peer_signatures_for_proposal(&primary, &accounts, &mut rng);
3087
3088        // Have the primary process the signatures.
3089        for (socket_addr, signature) in signatures {
3090            primary.process_batch_signature_from_peer(socket_addr, signature).await.unwrap();
3091        }
3092
3093        // Check the certificate was created and stored by the primary.
3094        assert!(primary.storage.contains_certificate_in_round_from(round, primary.gateway.account().address()));
3095        // Manually attempt round advancement (because the handler is not running).
3096        primary.try_increment_to_the_next_round(round + 1).await.unwrap();
3097        // Check the round was incremented.
3098        assert_eq!(primary.current_round(), round + 1);
3099    }
3100
3101    #[test_log::test(tokio::test)]
3102    async fn test_batch_signature_from_peer_no_quorum() {
3103        let mut rng = TestRng::default();
3104        let (primary, accounts) = primary_without_handlers(&mut rng);
3105        map_account_addresses(&primary, &accounts);
3106
3107        // Create a valid proposal.
3108        let round = 1;
3109        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
3110        let proposal = create_test_proposal(
3111            primary.gateway.account(),
3112            primary.ledger.current_committee().unwrap(),
3113            round,
3114            Default::default(),
3115            timestamp,
3116            1,
3117            &mut rng,
3118        );
3119
3120        // Store the proposal on the primary.
3121        *primary.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(proposal));
3122
3123        // Each committee member signs the batch.
3124        let signatures = peer_signatures_for_proposal(&primary, &accounts, &mut rng);
3125
3126        // Have the primary process only one signature, mimicking a lack of quorum.
3127        let (socket_addr, signature) = signatures.first().unwrap();
3128        primary.process_batch_signature_from_peer(*socket_addr, *signature).await.unwrap();
3129
3130        // Check the certificate was not created and stored by the primary.
3131        assert!(!primary.storage.contains_certificate_in_round_from(round, primary.gateway.account().address()));
3132        // Check the round was incremented.
3133        assert_eq!(primary.current_round(), round);
3134    }
3135
3136    #[test_log::test(tokio::test)]
3137    async fn test_batch_signature_from_peer_in_round_no_quorum() {
3138        let round = 7;
3139        let mut rng = TestRng::default();
3140        let (primary, accounts) = primary_without_handlers(&mut rng);
3141        map_account_addresses(&primary, &accounts);
3142
3143        // Generate certificates.
3144        let previous_certificates = store_certificate_chain(&primary, &accounts, round, &mut rng);
3145
3146        // Create a valid proposal.
3147        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
3148        let proposal = create_test_proposal(
3149            primary.gateway.account(),
3150            primary.ledger.current_committee().unwrap(),
3151            round,
3152            previous_certificates,
3153            timestamp,
3154            1,
3155            &mut rng,
3156        );
3157
3158        // Store the proposal on the primary.
3159        *primary.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(proposal));
3160
3161        // Each committee member signs the batch.
3162        let signatures = peer_signatures_for_proposal(&primary, &accounts, &mut rng);
3163
3164        // Have the primary process only one signature, mimicking a lack of quorum.
3165        let (socket_addr, signature) = signatures.first().unwrap();
3166        primary.process_batch_signature_from_peer(*socket_addr, *signature).await.unwrap();
3167
3168        // Check the certificate was not created and stored by the primary.
3169        assert!(!primary.storage.contains_certificate_in_round_from(round, primary.gateway.account().address()));
3170        // Check the round was incremented.
3171        assert_eq!(primary.current_round(), round);
3172    }
3173
3174    // Tests that a late-arriving signature for a batch that is currently being certified
3175    // (ProposedBatchState::Certified) is silently dropped without error.
3176    // This exercises the race condition where proposed_batch.take() has been called but
3177    // insert_certificate has not yet completed.
3178    #[test_log::test(tokio::test)]
3179    async fn test_batch_signature_from_peer_batch_being_certified() {
3180        let mut rng = TestRng::default();
3181        let (primary, accounts) = primary_without_handlers(&mut rng);
3182        map_account_addresses(&primary, &accounts);
3183
3184        // Create a valid proposal.
3185        let round = 1;
3186        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
3187        let proposal = create_test_proposal(
3188            primary.gateway.account(),
3189            primary.ledger.current_committee().unwrap(),
3190            round,
3191            Default::default(),
3192            timestamp,
3193            1,
3194            &mut rng,
3195        );
3196        let batch_id = proposal.batch_id();
3197
3198        // Simulate the race: the batch has been taken for certification but not yet stored.
3199        *primary.proposed_batch.write() = ProposedBatchState::Certified(batch_id);
3200
3201        // Send a late signature for the batch being certified.
3202        let (socket_addr, account) =
3203            accounts.iter().find(|(_, a)| a.address() != primary.gateway.account().address()).unwrap();
3204        let signature = account.sign(&[batch_id], &mut rng).unwrap();
3205        let batch_signature = BatchSignature::new(batch_id, signature);
3206
3207        // The signature should be accepted without error (silently dropped).
3208        assert!(primary.process_batch_signature_from_peer(*socket_addr, batch_signature).await.is_ok());
3209        // The batch state is unchanged (still BeingCertified — no new proposal was set).
3210        assert!(matches!(&*primary.proposed_batch.read(), ProposedBatchState::Certified(id) if *id == batch_id));
3211    }
3212
3213    // Tests that a signature for a completely unknown batch ID is rejected even when another
3214    // batch is being certified. The BeingCertified state only suppresses errors for its own ID.
3215    #[test_log::test(tokio::test)]
3216    async fn test_batch_signature_from_peer_unknown_id_while_certifying() {
3217        let mut rng = TestRng::default();
3218        let (primary, accounts) = primary_without_handlers(&mut rng);
3219        map_account_addresses(&primary, &accounts);
3220
3221        // Create two proposals so we have two distinct batch IDs.
3222        let round = 1;
3223        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
3224        let proposal_a = create_test_proposal(
3225            primary.gateway.account(),
3226            primary.ledger.current_committee().unwrap(),
3227            round,
3228            Default::default(),
3229            timestamp,
3230            1,
3231            &mut rng,
3232        );
3233        let proposal_b = create_test_proposal(
3234            primary.gateway.account(),
3235            primary.ledger.current_committee().unwrap(),
3236            round,
3237            Default::default(),
3238            timestamp,
3239            1,
3240            &mut rng,
3241        );
3242        let batch_id_a = proposal_a.batch_id();
3243        let batch_id_b = proposal_b.batch_id();
3244        assert_ne!(batch_id_a, batch_id_b);
3245
3246        // Simulate certifying batch A.
3247        *primary.proposed_batch.write() = ProposedBatchState::Certified(batch_id_a);
3248
3249        // Send a signature for batch B (a genuinely unknown ID).
3250        let (socket_addr, account) =
3251            accounts.iter().find(|(_, a)| a.address() != primary.gateway.account().address()).unwrap();
3252        let signature = account.sign(&[batch_id_b], &mut rng).unwrap();
3253        let batch_signature = BatchSignature::new(batch_id_b, signature);
3254
3255        // The signature is for a genuinely unknown ID — should be rejected with an error.
3256        assert!(primary.process_batch_signature_from_peer(*socket_addr, batch_signature).await.is_err());
3257    }
3258
3259    // Tests the "already certified" path: a signature arrives after the batch is fully in
3260    // storage and the primary has moved on to a new proposal.
3261    #[test_log::test(tokio::test(flavor = "multi_thread"))]
3262    async fn test_batch_signature_from_peer_already_certified() {
3263        let mut rng = TestRng::default();
3264        let (primary, accounts) = primary_without_handlers(&mut rng);
3265        map_account_addresses(&primary, &accounts);
3266
3267        // Create and certify a batch so it lands in storage.
3268        let round = 1;
3269        let timestamp = now() + MIN_BATCH_DELAY.as_secs() as i64;
3270        let old_proposal = create_test_proposal(
3271            primary.gateway.account(),
3272            primary.ledger.current_committee().unwrap(),
3273            round,
3274            Default::default(),
3275            timestamp,
3276            1,
3277            &mut rng,
3278        );
3279        let old_batch_id = old_proposal.batch_id();
3280        *primary.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(old_proposal));
3281        let signatures = peer_signatures_for_proposal(&primary, &accounts, &mut rng);
3282        for (socket_addr, signature) in signatures {
3283            primary.process_batch_signature_from_peer(socket_addr, signature).await.unwrap();
3284        }
3285        // The batch is now in storage.
3286        assert!(primary.storage.contains_certificate_in_round_from(round, primary.gateway.account().address()));
3287
3288        // Simulate a new proposal being active.
3289        let new_proposal = create_test_proposal(
3290            primary.gateway.account(),
3291            primary.ledger.current_committee().unwrap(),
3292            round,
3293            Default::default(),
3294            timestamp,
3295            1,
3296            &mut rng,
3297        );
3298        assert_ne!(new_proposal.batch_id(), old_batch_id);
3299        *primary.proposed_batch.write() = ProposedBatchState::Certifying(Box::new(new_proposal));
3300
3301        // Send a late signature for the already-certified old batch.
3302        let (socket_addr, account) =
3303            accounts.iter().find(|(_, a)| a.address() != primary.gateway.account().address()).unwrap();
3304        let signature = account.sign(&[old_batch_id], &mut rng).unwrap();
3305        let batch_signature = BatchSignature::new(old_batch_id, signature);
3306
3307        // Should be silently accepted (already certified path).
3308        assert!(primary.process_batch_signature_from_peer(*socket_addr, batch_signature).await.is_ok());
3309    }
3310
3311    #[test_log::test(tokio::test)]
3312    async fn test_insert_certificate_with_aborted_transmissions() {
3313        let round = 3;
3314        let prev_round = round - 1;
3315        let mut rng = TestRng::default();
3316        let (primary, accounts) = primary_without_handlers(&mut rng);
3317        let peer_account = &accounts[1];
3318        let peer_ip = peer_account.0;
3319
3320        // Fill primary storage.
3321        store_certificate_chain(&primary, &accounts, round, &mut rng);
3322
3323        // Get transmissions from previous certificates.
3324        let previous_certificate_ids: IndexSet<_> = primary.storage.get_certificate_ids_for_round(prev_round);
3325
3326        // Generate a solution and a transaction.
3327        let (solution_commitment, solution) = sample_unconfirmed_solution(&mut rng);
3328        let (transaction_id, transaction) = sample_unconfirmed_transaction(&mut rng);
3329
3330        // Store it on one of the workers.
3331        primary.workers()[0].process_unconfirmed_solution(solution_commitment, solution).await.unwrap();
3332        primary.workers()[0].process_unconfirmed_transaction(transaction_id, transaction).await.unwrap();
3333
3334        // Check that the worker has 2 transmissions.
3335        assert_eq!(primary.workers()[0].num_transmissions(), 2);
3336
3337        // Create certificates for the current round.
3338        let account = accounts[0].1.clone();
3339        let (certificate, transmissions) =
3340            create_batch_certificate(account.address(), &accounts, round, previous_certificate_ids.clone(), &mut rng);
3341        let certificate_id = certificate.id();
3342
3343        // Randomly abort some of the transmissions.
3344        let mut aborted_transmissions = HashSet::new();
3345        let mut transmissions_without_aborted = HashMap::new();
3346        for (transmission_id, transmission) in transmissions.clone() {
3347            match rng.random::<bool>() || aborted_transmissions.is_empty() {
3348                true => {
3349                    // Insert the aborted transmission.
3350                    aborted_transmissions.insert(transmission_id);
3351                }
3352                false => {
3353                    // Insert the transmission without the aborted transmission.
3354                    transmissions_without_aborted.insert(transmission_id, transmission);
3355                }
3356            };
3357        }
3358
3359        // Add the non-aborted transmissions to the worker.
3360        for (transmission_id, transmission) in transmissions_without_aborted.iter() {
3361            primary.workers()[0].process_transmission_from_peer(peer_ip, *transmission_id, transmission.clone());
3362        }
3363
3364        // Check that inserting the transmission with missing transmissions fails.
3365        assert!(
3366            primary
3367                .storage
3368                .check_certificate(&certificate, transmissions_without_aborted.clone(), Default::default())
3369                .is_err()
3370        );
3371        assert!(
3372            primary
3373                .storage
3374                .insert_certificate(certificate.clone(), transmissions_without_aborted.clone(), Default::default())
3375                .is_err()
3376        );
3377
3378        // Insert the certificate to storage.
3379        primary
3380            .storage
3381            .insert_certificate(certificate, transmissions_without_aborted, aborted_transmissions.clone())
3382            .unwrap();
3383
3384        // Ensure the certificate exists in storage.
3385        assert!(primary.storage.contains_certificate(certificate_id));
3386        // Ensure that the aborted transmission IDs exist in storage.
3387        for aborted_transmission_id in aborted_transmissions {
3388            assert!(primary.storage.contains_transmission(aborted_transmission_id));
3389            assert!(primary.storage.get_transmission(aborted_transmission_id).is_none());
3390        }
3391    }
3392
3393    // -----------------------------------------------------------------------
3394    // add_signature_to_batch
3395    // -----------------------------------------------------------------------
3396
3397    /// State is `None` and the batch is not in storage — returns an error, state stays `None`.
3398    #[test]
3399    fn test_add_signature_to_batch_none_state() {
3400        let mut rng = TestRng::default();
3401        let (primary, accounts) = primary_without_handlers(&mut rng);
3402
3403        let peer_ip = accounts[1].0;
3404        let batch_id = Field::rand(&mut rng);
3405        let signature = accounts[1].1.sign(&[batch_id], &mut rng).unwrap();
3406
3407        let (result, new_state) =
3408            primary.add_signature_to_batch(ProposedBatchState::None, peer_ip, batch_id, signature);
3409
3410        assert!(result.is_err());
3411        assert_eq!(new_state, ProposedBatchState::None);
3412    }
3413
3414    /// State is `Certified` with a matching batch ID — silently dropped, state restored.
3415    #[test]
3416    fn test_add_signature_to_batch_certified_matching_id() {
3417        let mut rng = TestRng::default();
3418        let (primary, accounts) = primary_without_handlers(&mut rng);
3419
3420        let peer_ip = accounts[1].0;
3421        let batch_id = Field::rand(&mut rng);
3422        let signature = accounts[1].1.sign(&[batch_id], &mut rng).unwrap();
3423
3424        let (result, new_state) =
3425            primary.add_signature_to_batch(ProposedBatchState::Certified(batch_id), peer_ip, batch_id, signature);
3426
3427        assert!(result.unwrap().is_none());
3428        assert_eq!(new_state, ProposedBatchState::Certified(batch_id));
3429    }
3430
3431    /// State is `Certified` with a *different* batch ID — error returned, state becomes `None`.
3432    #[test]
3433    fn test_add_signature_to_batch_certified_different_id() {
3434        let mut rng = TestRng::default();
3435        let (primary, accounts) = primary_without_handlers(&mut rng);
3436
3437        let peer_ip = accounts[1].0;
3438        let certified_id = Field::rand(&mut rng);
3439        let other_id = Field::rand(&mut rng);
3440        let signature = accounts[1].1.sign(&[other_id], &mut rng).unwrap();
3441
3442        let (result, new_state) =
3443            primary.add_signature_to_batch(ProposedBatchState::Certified(certified_id), peer_ip, other_id, signature);
3444
3445        assert!(result.is_err());
3446        assert_eq!(new_state, ProposedBatchState::Certified(certified_id));
3447    }
3448
3449    /// State is `Certifying` for a *different* batch ID that **is already in storage** — silently
3450    /// dropped, state restored.
3451    #[tokio::test(flavor = "multi_thread")]
3452    async fn test_add_signature_to_batch_certifying_different_id_in_storage() {
3453        let round = 1;
3454        let mut rng = TestRng::default();
3455        let (primary, accounts) = primary_without_handlers(&mut rng);
3456        map_account_addresses(&primary, &accounts);
3457
3458        // Create a proposal owned by the primary.
3459        let proposal = create_test_proposal(
3460            primary.gateway.account(),
3461            primary.ledger.current_committee().unwrap(),
3462            round,
3463            Default::default(),
3464            now(),
3465            0,
3466            &mut rng,
3467        );
3468        let proposal_batch_id = proposal.batch_id();
3469
3470        // Create and store a *different* certificate so `contains_batch` returns true for it.
3471        let (certificate, transmissions) =
3472            create_batch_certificate(accounts[1].1.address(), &accounts, round, Default::default(), &mut rng);
3473        let stored_batch_id = certificate.batch_id();
3474        primary.storage.insert_certificate(certificate, transmissions, Default::default()).unwrap();
3475
3476        let peer_ip = accounts[1].0;
3477        let signature = accounts[1].1.sign(&[stored_batch_id], &mut rng).unwrap();
3478
3479        let (result, new_state) = primary.add_signature_to_batch(
3480            ProposedBatchState::Certifying(Box::new(proposal)),
3481            peer_ip,
3482            stored_batch_id,
3483            signature,
3484        );
3485
3486        assert!(result.unwrap().is_none());
3487        // State is restored with the original proposal.
3488        assert_eq!(new_state.as_proposal().unwrap().batch_id(), proposal_batch_id);
3489    }
3490
3491    /// State is `Certifying` for a *different* batch ID that is **not in storage** — error
3492    /// returned, state restored.
3493    #[test]
3494    fn test_add_signature_to_batch_certifying_different_id_unknown() {
3495        let mut rng = TestRng::default();
3496        let (primary, accounts) = primary_without_handlers(&mut rng);
3497
3498        let proposal = create_test_proposal(
3499            primary.gateway.account(),
3500            primary.ledger.current_committee().unwrap(),
3501            1,
3502            Default::default(),
3503            now(),
3504            0,
3505            &mut rng,
3506        );
3507        let proposal_batch_id = proposal.batch_id();
3508
3509        let peer_ip = accounts[1].0;
3510        let unknown_id = Field::rand(&mut rng);
3511        let signature = accounts[1].1.sign(&[unknown_id], &mut rng).unwrap();
3512
3513        let (result, new_state) = primary.add_signature_to_batch(
3514            ProposedBatchState::Certifying(Box::new(proposal)),
3515            peer_ip,
3516            unknown_id,
3517            signature,
3518        );
3519
3520        assert!(result.is_err());
3521        assert_eq!(new_state.as_proposal().unwrap().batch_id(), proposal_batch_id);
3522    }
3523
3524    /// Matching batch ID, valid signature, quorum **not yet** reached — state stays `Certifying`.
3525    #[test]
3526    fn test_add_signature_to_batch_certifying_matching_no_quorum() {
3527        let mut rng = TestRng::default();
3528        let (primary, accounts) = primary_without_handlers(&mut rng);
3529        map_account_addresses(&primary, &accounts);
3530
3531        let proposal = create_test_proposal(
3532            primary.gateway.account(),
3533            primary.ledger.current_committee().unwrap(),
3534            1,
3535            Default::default(),
3536            now(),
3537            0,
3538            &mut rng,
3539        );
3540        let batch_id = proposal.batch_id();
3541
3542        // Only one peer signs — not enough for quorum.
3543        let peer_ip = accounts[1].0;
3544        let signature = accounts[1].1.sign(&[batch_id], &mut rng).unwrap();
3545
3546        let (result, new_state) = primary.add_signature_to_batch(
3547            ProposedBatchState::Certifying(Box::new(proposal)),
3548            peer_ip,
3549            batch_id,
3550            signature,
3551        );
3552
3553        assert!(result.unwrap().is_none());
3554        assert_eq!(new_state.as_proposal().unwrap().batch_id(), batch_id);
3555    }
3556
3557    /// Matching batch ID, all peers sign — quorum reached, proposal extracted and state becomes
3558    /// `Certified`.
3559    #[test]
3560    fn test_add_signature_to_batch_certifying_matching_quorum_reached() {
3561        let mut rng = TestRng::default();
3562        let (primary, accounts) = primary_without_handlers(&mut rng);
3563        map_account_addresses(&primary, &accounts);
3564
3565        let proposal = create_test_proposal(
3566            primary.gateway.account(),
3567            primary.ledger.current_committee().unwrap(),
3568            1,
3569            Default::default(),
3570            now(),
3571            0,
3572            &mut rng,
3573        );
3574        let batch_id = proposal.batch_id();
3575
3576        // Add all peer signatures one by one until quorum is reached.
3577        let peers: Vec<_> =
3578            accounts.iter().filter(|(_, a)| a.address() != primary.gateway.account().address()).collect();
3579        let mut state = ProposedBatchState::Certifying(Box::new(proposal));
3580        let mut final_result = None;
3581
3582        for (peer_ip, peer_account) in &peers {
3583            let signature = peer_account.sign(&[batch_id], &mut rng).unwrap();
3584            let (result, new_state) = primary.add_signature_to_batch(state, *peer_ip, batch_id, signature);
3585            state = new_state;
3586            if result.as_ref().unwrap().is_some() {
3587                final_result = Some(result);
3588                break;
3589            }
3590        }
3591
3592        // Quorum must have been reached with the committee's peers.
3593        let proposal = final_result.expect("quorum should be reached").unwrap().unwrap();
3594        assert_eq!(proposal.batch_id(), batch_id);
3595        assert_eq!(state, ProposedBatchState::Certified(batch_id));
3596    }
3597}