Skip to main content

snarkos_node_bft/
gateway.rs

1// Copyright (c) 2019-2026 Provable Inc.
2// This file is part of the snarkOS library.
3
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at:
7
8// http://www.apache.org/licenses/LICENSE-2.0
9
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15
16#[cfg(feature = "metrics")]
17use crate::helpers::{Telemetry, TelemetryWorker};
18use crate::{
19    CONTEXT,
20    MAX_BATCH_DELAY,
21    MAX_FETCH_TIMEOUT,
22    MEMORY_POOL_PORT,
23    Worker,
24    events::{DisconnectReason, EventCodec, PrimaryPing},
25    helpers::{Cache, PrimarySender, Storage, SyncSender, WorkerSender, assign_to_worker},
26    spawn_blocking,
27};
28use smol_str::SmolStr;
29use snarkos_account::Account;
30use snarkos_node_bft_events::{
31    BlockRequest,
32    BlockResponse,
33    CertificateRequest,
34    CertificateResponse,
35    ChallengeRequest,
36    ChallengeResponse,
37    DataBlocks,
38    Event,
39    EventTrait,
40    HANDSHAKE_DOMAIN,
41    HandshakeHint,
42    InitiatorInfo,
43    PeerInfo,
44    ResponderProof,
45    TransmissionRequest,
46    TransmissionResponse,
47    ValidatorsRequest,
48    ValidatorsResponse,
49};
50use snarkos_node_bft_ledger_service::LedgerService;
51use snarkos_node_network::{
52    ConnectionMode,
53    NodeType,
54    Peer,
55    PeerPoolHandling,
56    Resolver,
57    bootstrap_peers,
58    get_repo_commit_hash,
59    harden_socket,
60    log_repo_sha_comparison,
61    noise::{
62        HandshakeProtocol,
63        NoiseSession,
64        PendingSession,
65        Role,
66        binding_message,
67        detect_handshake_protocol,
68        prepare_framed,
69        write_noise_magic,
70    },
71    shorten_snarkos_sha,
72};
73use snarkos_node_sync::{MAX_BLOCKS_BEHIND, communication_service::CommunicationService};
74use snarkos_node_tcp::{
75    Config,
76    ConnectError,
77    Connection,
78    ConnectionSide,
79    P2P,
80    Tcp,
81    connections::DisconnectOrigin,
82    protocols::{Disconnect, Handshake, OnConnect, Reading, Writing},
83};
84use snarkos_utilities::NodeDataDir;
85use snarkvm::{
86    console::prelude::*,
87    ledger::{
88        committee::Committee,
89        narwhal::{BatchHeader, Data},
90    },
91    prelude::{Address, Field, Signature},
92    utilities::flatten_error,
93};
94
95use colored::Colorize;
96use futures::{SinkExt, future::join_all};
97use indexmap::IndexMap;
98#[cfg(feature = "locktick")]
99use locktick::parking_lot::{Mutex, RwLock};
100#[cfg(not(feature = "locktick"))]
101use parking_lot::{Mutex, RwLock};
102use rand::seq::{IteratorRandom, SliceRandom};
103use std::{
104    collections::{HashMap, HashSet},
105    future::Future,
106    io,
107    net::{Ipv4Addr, SocketAddr, SocketAddrV4},
108    sync::Arc,
109    time::Duration,
110};
111use tokio::{
112    net::TcpStream,
113    sync::{OnceCell, oneshot},
114    task::{self, JoinHandle},
115};
116use tokio_stream::StreamExt;
117use tokio_util::codec::Framed;
118
119/// The maximum interval of events to cache.
120const CACHE_EVENTS_INTERVAL: i64 = (MAX_BATCH_DELAY.as_secs()) as i64; // seconds
121/// The maximum interval of requests to cache.
122const CACHE_REQUESTS_INTERVAL: i64 = (MAX_BATCH_DELAY.as_secs()) as i64; // seconds
123
124/// The number of consensus rounds of traffic that a per-connection message queue must absorb.
125///
126/// A message that has not made it onto (or off of) the wire within `MAX_FETCH_TIMEOUT` is already
127/// useless to the peer: a requester has given up on it by then (see
128/// `Worker::send_transmission_request`), and any consensus event it carried is stale. Sizing the
129/// per-connection queues to this window therefore bounds them to the traffic that can still be
130/// delivered in time, rather than to the entire garbage-collection window (`MAX_GC_ROUNDS`, which
131/// is ~100 rounds, i.e. several minutes of traffic).
132const QUEUE_WINDOW_ROUNDS: usize = (MAX_FETCH_TIMEOUT.as_millis() / MAX_BATCH_DELAY.as_millis()) as usize;
133
134/// Computes the depth of the per-connection inbound and outbound message queues.
135///
136/// These queues are transient send/receive buffers, not backlogs: they only need to hold the
137/// traffic a peer can legitimately exchange with us over `QUEUE_WINDOW_ROUNDS` (see above).
138/// Per round, the worst case is every certificate in the round plus every transmission each of
139/// those certificates contains — that is, a peer that is missing an entire round and fetches all
140/// of it from us. The leading factor of 2 is headroom for the remaining, far smaller, event
141/// traffic (batch proposals and signatures, primary and worker pings, block and validator
142/// requests) and for requests that straddle a round boundary.
143///
144/// Note that each slot can hold a full `Transmission`, so this value is a direct multiplier on the
145/// heap a single peer can pin. It must stay small enough that `depth * max transmission size` is
146/// survivable on a commodity validator.
147fn per_connection_queue_depth<N: Network>() -> usize {
148    2 * QUEUE_WINDOW_ROUNDS
149        * N::LATEST_MAX_CERTIFICATES() as usize
150        * (BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH + 1)
151}
152
153/// The maximum number of connection attempts in an interval.
154#[cfg(not(test))]
155const MAX_CONNECTION_ATTEMPTS: usize = 10;
156
157/// The maximum number of validators to send in a validators response event.
158pub const MAX_VALIDATORS_TO_SEND: usize = 200;
159
160/// The minimum permitted interval between connection attempts for an IP; anything shorter is considered malicious.
161#[cfg(not(test))]
162const CONNECTION_ATTEMPTS_SINCE_SECS: i64 = 10;
163
164/// The amount of time an IP address is prohibited from connecting.
165const IP_BAN_TIME_IN_SECS: u64 = 300;
166
167/// The consensus version at which this node starts *initiating* Noise handshakes, if one is
168/// scheduled.
169///
170/// Only the initiator's choice is gated: a responder accepts either protocol as soon as this code
171/// ships, which is what allows validators to be upgraded one at a time. `None` means no switchover
172/// has been scheduled yet - except in development, where the Noise path is always taken so that
173/// devnets exercise it, and in tests, which pin the choice explicitly.
174///
175/// Setting this is not the end of the migration, only the middle of it; see
176/// [`LEGACY_HANDSHAKE_EXPIRY`].
177const NOISE_HANDSHAKE_ACTIVATION: Option<ConsensusVersion> = Some(ConsensusVersion::V20);
178
179/// The consensus version at which this node stops *accepting* the legacy handshake, if one is
180/// scheduled.
181///
182/// This is the step that actually collects what the conversion is for. For as long as the responder
183/// still accepts the legacy handshake, two things remain reachable through it: the relay that the
184/// handshake binding exists to prevent, and the legacy handshake codec's 1 MiB frame limit - sixteen
185/// times what a Noise message may be, on a buffer an unauthenticated peer gets to size. Preferring
186/// the new path does not close either; refusing the old one does.
187///
188/// It must trail [`NOISE_HANDSHAKE_ACTIVATION`] by enough for every peer to have switched, as a
189/// validator that has not yet reached the activation still dials with the legacy handshake and would
190/// be shut out. Note also that the same relay is reachable through the router's handshake, which
191/// signs a byte-identical message with the same account key, so the gateway cannot be the last part
192/// of this to be converted.
193const LEGACY_HANDSHAKE_EXPIRY: Option<ConsensusVersion> = Some(ConsensusVersion::V21);
194
195/// Part of the Gateway API that deals with networking.
196/// This is a separate trait to allow for easier testing/mocking.
197#[async_trait]
198pub trait Transport<N: Network>: Send + Sync {
199    async fn send(&self, peer_ip: SocketAddr, event: Event<N>) -> Option<oneshot::Receiver<io::Result<()>>>;
200    fn broadcast(&self, event: Event<N>);
201}
202
203/// The gateway maintains connections to other validators.
204/// For connections with clients and provers, the Router logic is used.
205#[derive(Clone)]
206pub struct Gateway<N: Network>(Arc<InnerGateway<N>>);
207
208impl<N: Network> Deref for Gateway<N> {
209    type Target = Arc<InnerGateway<N>>;
210
211    fn deref(&self) -> &Self::Target {
212        &self.0
213    }
214}
215
216pub struct InnerGateway<N: Network> {
217    /// The account of the node.
218    account: Account<N>,
219    /// The storage.
220    storage: Storage<N>,
221    /// The ledger service.
222    ledger: Arc<dyn LedgerService<N>>,
223    /// The TCP stack.
224    tcp: Tcp,
225    /// The cache.
226    cache: Cache<N>,
227    /// The resolver.
228    resolver: RwLock<Resolver<N>>,
229    /// The collection of both candidate and connected peers.
230    peer_pool: RwLock<HashMap<SocketAddr, Peer<N>>>,
231    /// The handle to the validator telemetry tracker.
232    #[cfg(feature = "metrics")]
233    validator_telemetry: Telemetry<N>,
234    /// The telemetry worker, taken and spawned by `run`.
235    ///
236    /// This is a one-shot handoff from `new` to `run`, not a hot-path lock: the telemetry
237    /// state itself lives in the worker task, and is never shared.
238    #[cfg(feature = "metrics")]
239    telemetry_worker: Mutex<Option<TelemetryWorker<N>>>,
240    /// The primary sender.
241    primary_sender: OnceCell<PrimarySender<N>>,
242    /// The worker senders.
243    worker_senders: OnceCell<IndexMap<u8, WorkerSender<N>>>,
244    /// The sync sender.
245    sync_sender: OnceCell<SyncSender<N>>,
246    /// The spawned handles.
247    handles: Mutex<Vec<JoinHandle<()>>>,
248    /// The storage mode.
249    node_data_dir: NodeDataDir,
250    /// If the flag is set, the node will only connect to trusted peers.
251    trusted_peers_only: bool,
252    /// The development mode.
253    dev: Option<u16>,
254    /// Pins which handshake this node offers when it dials, bypassing
255    /// [`NOISE_HANDSHAKE_ACTIVATION`].
256    ///
257    /// Tests default to the Noise handshake, since that is the one under test, and set this to
258    /// `false` to cover the other half of the transition: a converted node has to keep talking to
259    /// unconverted ones, which means the legacy path must stay exercised for as long as it exists.
260    #[cfg(any(test, feature = "test"))]
261    initiates_noise_handshake: std::sync::atomic::AtomicBool,
262}
263
264impl<N: Network> PeerPoolHandling<N> for Gateway<N> {
265    const MAXIMUM_POOL_SIZE: usize = 200;
266    const OWNER: &str = CONTEXT;
267    const PEER_SLASHING_COUNT: usize = 20;
268
269    fn peer_pool(&self) -> &RwLock<HashMap<SocketAddr, Peer<N>>> {
270        &self.peer_pool
271    }
272
273    fn resolver(&self) -> &RwLock<Resolver<N>> {
274        &self.resolver
275    }
276
277    fn is_dev(&self) -> bool {
278        self.dev.is_some()
279    }
280
281    fn trusted_peers_only(&self) -> bool {
282        self.trusted_peers_only
283    }
284
285    fn node_type(&self) -> NodeType {
286        NodeType::Validator
287    }
288}
289
290impl<N: Network> Gateway<N> {
291    /// Initializes a new gateway.
292    #[allow(clippy::too_many_arguments)]
293    pub fn new(
294        account: Account<N>,
295        storage: Storage<N>,
296        ledger: Arc<dyn LedgerService<N>>,
297        ip: Option<SocketAddr>,
298        trusted_validators: &[SocketAddr],
299        trusted_peers_only: bool,
300        node_data_dir: NodeDataDir,
301        dev: Option<u16>,
302    ) -> Result<Self> {
303        // Initialize the gateway IP.
304        let ip = match (ip, dev) {
305            (None, Some(dev)) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, MEMORY_POOL_PORT + dev)),
306            (None, None) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, MEMORY_POOL_PORT)),
307            (Some(ip), _) => ip,
308        };
309        // Initialize the TCP stack.
310        //
311        // The 10x multiplier allows for more TCP connections than the maximum
312        // committee size to prevent "connection refused" errors when two nodes
313        // simultaneous attempt to connect to each other. Note, that later,
314        // during handshake, the Gateway applies its own limit to the number of
315        // active connections and removes duplicates.
316        let tcp = Tcp::new(Config::new(ip, Committee::<N>::max_committee_size() * 10));
317
318        // Prepare the collection of the initial peers.
319        let mut initial_peers = HashMap::new();
320
321        // Load entries from the validator cache (if present and if we are not in trusted peers only mode).
322        if !trusted_peers_only {
323            let cached_peers = Self::load_cached_peers(&node_data_dir.gateway_peer_cache_path())?;
324            for addr in cached_peers {
325                initial_peers.insert(addr, Peer::new_candidate(addr, false));
326            }
327        }
328
329        // Add the trusted peers to the list of the initial peers; this may promote
330        // some of the cached validators to trusted ones.
331        initial_peers.extend(trusted_validators.iter().copied().map(|addr| (addr, Peer::new_candidate(addr, true))));
332
333        // Initialize the validator telemetry. The worker is spawned later, by `run`.
334        #[cfg(feature = "metrics")]
335        let (validator_telemetry, telemetry_worker) = Telemetry::new();
336
337        // Return the gateway.
338        Ok(Self(Arc::new(InnerGateway {
339            account,
340            storage,
341            ledger,
342            tcp,
343            cache: Default::default(),
344            resolver: Default::default(),
345            peer_pool: RwLock::new(initial_peers),
346            #[cfg(feature = "metrics")]
347            validator_telemetry,
348            #[cfg(feature = "metrics")]
349            telemetry_worker: Mutex::new(Some(telemetry_worker)),
350            primary_sender: Default::default(),
351            worker_senders: Default::default(),
352            sync_sender: Default::default(),
353            handles: Default::default(),
354            node_data_dir,
355            trusted_peers_only,
356            dev,
357            // See the field's documentation for why the tests start out on the Noise handshake.
358            #[cfg(any(test, feature = "test"))]
359            initiates_noise_handshake: std::sync::atomic::AtomicBool::new(true),
360        })))
361    }
362
363    /// Run the gateway.
364    pub async fn run(
365        &self,
366        primary_sender: PrimarySender<N>,
367        worker_senders: IndexMap<u8, WorkerSender<N>>,
368        sync_sender: Option<SyncSender<N>>,
369    ) {
370        debug!("Starting the gateway for the memory pool...");
371
372        // Set the primary sender.
373        self.primary_sender.set(primary_sender).expect("Primary sender already set in gateway");
374
375        // Set the worker senders.
376        self.worker_senders.set(worker_senders).expect("The worker senders are already set");
377
378        // If the sync sender was provided, set the sync sender.
379        if let Some(sync_sender) = sync_sender {
380            self.sync_sender.set(sync_sender).expect("Sync sender already set in gateway");
381        }
382
383        // Spawn the validator telemetry worker, which owns all telemetry state.
384        // It is registered in `handles`, so `shut_down` aborts it along with everything else.
385        #[cfg(feature = "metrics")]
386        if let Some(telemetry_worker) = self.telemetry_worker.lock().take() {
387            self.spawn(telemetry_worker.run());
388        }
389
390        // Enable the TCP protocols.
391        self.enable_handshake().await;
392        self.enable_reading().await;
393        self.enable_writing().await;
394        self.enable_disconnect().await;
395        self.enable_on_connect().await;
396
397        // Spawn a loop for periodic metrics.
398        #[cfg(feature = "metrics")]
399        {
400            let gateway = self.clone();
401            self.spawn(async move {
402                loop {
403                    tokio::time::sleep(Duration::from_secs(1)).await;
404                    gateway.update_metrics();
405                }
406            });
407        }
408
409        // Enable the TCP listener. Note: This must be called after the above protocols.
410        let listen_addr = self.tcp.enable_listener().await.expect("Failed to enable the TCP listener");
411        debug!("Listening for validator connections at address {listen_addr:?}");
412
413        // Initialize the heartbeat.
414        self.initialize_heartbeat();
415
416        info!("Started the gateway for the memory pool at '{}'", self.local_ip());
417    }
418}
419
420// Dynamic rate limiting.
421impl<N: Network> Gateway<N> {
422    /// The current maximum committee size.
423    fn max_committee_size(&self) -> usize {
424        self.ledger
425            .current_committee()
426            .map_or_else(|_e| Committee::<N>::max_committee_size() as usize, |committee| committee.num_members())
427    }
428
429    /// The maximum number of events to cache.
430    fn max_cache_events(&self) -> usize {
431        self.max_cache_transmissions()
432    }
433
434    /// The maximum number of certificate requests to cache.
435    fn max_cache_certificates(&self) -> usize {
436        2 * BatchHeader::<N>::MAX_GC_ROUNDS * self.max_committee_size()
437    }
438
439    /// The maximum number of transmission requests to cache.
440    fn max_cache_transmissions(&self) -> usize {
441        self.max_cache_certificates() * BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH
442    }
443
444    /// The maximum number of duplicates for any particular request.
445    fn max_cache_duplicates(&self) -> usize {
446        self.max_committee_size().pow(2)
447    }
448}
449
450#[async_trait]
451impl<N: Network> CommunicationService for Gateway<N> {
452    /// The message type.
453    type Message = Event<N>;
454
455    /// Prepares a block request to be sent.
456    fn prepare_block_request(start_height: u32, end_height: u32) -> Self::Message {
457        debug_assert!(start_height < end_height, "Invalid block request format");
458        Event::BlockRequest(BlockRequest { start_height, end_height })
459    }
460
461    /// Sends the given message to specified peer.
462    ///
463    /// This function returns as soon as the message is queued to be sent,
464    /// without waiting for the actual delivery; instead, the caller is provided with a [`oneshot::Receiver`]
465    /// which can be used to determine when and whether the message has been delivered.
466    async fn send(&self, peer_ip: SocketAddr, message: Self::Message) -> Option<oneshot::Receiver<io::Result<()>>> {
467        Transport::send(self, peer_ip, message).await
468    }
469}
470
471impl<N: Network> Gateway<N> {
472    /// Returns the account of the node.
473    pub fn account(&self) -> &Account<N> {
474        &self.account
475    }
476
477    /// Returns the dev identifier of the node.
478    pub fn dev(&self) -> Option<u16> {
479        self.dev
480    }
481
482    /// Returns a reference to the ledger.
483    pub fn ledger(&self) -> &Arc<dyn LedgerService<N>> {
484        &self.ledger
485    }
486
487    /// Returns the resolver.
488    pub fn resolver(&self) -> &RwLock<Resolver<N>> {
489        &self.resolver
490    }
491
492    /// Returns the listener IP address from the (ambiguous) peer address.
493    pub fn resolve_to_listener(&self, connected_addr: &SocketAddr) -> Option<SocketAddr> {
494        self.resolver.read().get_listener(*connected_addr)
495    }
496
497    /// Returns the validator telemetry.
498    #[cfg(feature = "metrics")]
499    pub fn validator_telemetry(&self) -> &Telemetry<N> {
500        &self.validator_telemetry
501    }
502
503    /// Returns the primary sender.
504    pub fn primary_sender(&self) -> &PrimarySender<N> {
505        self.primary_sender.get().expect("Primary sender not set in gateway")
506    }
507
508    /// Returns the number of workers.
509    pub fn num_workers(&self) -> u8 {
510        u8::try_from(self.worker_senders.get().expect("Missing worker senders in gateway").len())
511            .expect("Too many workers")
512    }
513
514    /// Returns the worker sender for the given worker ID.
515    pub fn get_worker_sender(&self, worker_id: u8) -> Option<&WorkerSender<N>> {
516        self.worker_senders.get().and_then(|senders| senders.get(&worker_id))
517    }
518
519    /// Returns `true` if the given peer IP is an authorized validator.
520    pub fn is_authorized_validator_ip(&self, ip: SocketAddr) -> bool {
521        // If the peer IP is in the trusted validators, return early.
522        if self.trusted_peers().contains(&ip) {
523            return true;
524        }
525        // Retrieve the Aleo address of the peer IP.
526        match self.resolve_to_aleo_addr(ip) {
527            // Determine if the peer IP is an authorized validator.
528            Some(address) => self.is_authorized_validator_address(address),
529            None => {
530                warn!("{CONTEXT} Could not resolve the Aleo address for '{ip}'");
531                false
532            }
533        }
534    }
535
536    /// Returns `true` if the given address is an authorized validator.
537    pub fn is_authorized_validator_address(&self, validator_address: Address<N>) -> bool {
538        // Determine if the validator address is a member of the committee lookback,
539        // the current committee, or the previous committee lookbacks.
540        // We allow leniency in this validation check in order to accommodate these two scenarios:
541        //  1. New validators should be able to connect immediately once bonded as a committee member.
542        //  2. Existing validators must remain connected until they are no longer bonded as a committee member.
543        //     (i.e. meaning they must stay online until the next block has been produced)
544
545        // Determine if the validator is in the current committee with lookback.
546        if self
547            .ledger
548            .get_committee_lookback_for_round(self.storage.current_round())
549            .is_ok_and(|committee| committee.is_committee_member(validator_address))
550        {
551            return true;
552        }
553
554        // Determine if the validator is in the latest committee on the ledger.
555        if self.ledger.current_committee().is_ok_and(|committee| committee.is_committee_member(validator_address)) {
556            return true;
557        }
558
559        // Retrieve the previous block height to consider from the sync tolerance.
560        let previous_block_height = self.ledger.latest_block_height().saturating_sub(MAX_BLOCKS_BEHIND);
561        // Determine if the validator is in any of the previous committee lookbacks.
562        match self.ledger.get_block_round(previous_block_height) {
563            Ok(block_round) => (block_round..self.storage.current_round()).step_by(2).any(|round| {
564                self.ledger
565                    .get_committee_lookback_for_round(round)
566                    .is_ok_and(|committee| committee.is_committee_member(validator_address))
567            }),
568            Err(_) => false,
569        }
570    }
571
572    /// Returns the list of connected addresses.
573    pub fn connected_addresses(&self) -> HashSet<Address<N>> {
574        self.get_connected_peers().into_iter().map(|peer| peer.aleo_addr).collect()
575    }
576
577    /// Ensure the peer is allowed to connect.
578    fn ensure_peer_is_allowed(&self, listener_addr: SocketAddr) -> Result<(), DisconnectReason> {
579        // Ensure the peer IP is not this node.
580        if self.is_local_ip(listener_addr) {
581            return Err(DisconnectReason::SelfConnect);
582        }
583
584        Ok(())
585    }
586
587    /// Updates the connection metrics for the gateway. Ignores the bootstrap clients.
588    #[cfg(feature = "metrics")]
589    fn update_metrics(&self) {
590        if let Some(count) = self.number_of_connected_validators() {
591            metrics::gauge(metrics::bft::CONNECTED, count as f64);
592        }
593        if let Some(count) = self.number_of_connecting_peers() {
594            metrics::gauge(metrics::bft::CONNECTING, count as f64);
595        }
596    }
597
598    /// Inserts the given peer into the connected peers. This is only used in testing.
599    #[cfg(test)]
600    pub fn insert_connected_peer(&self, peer_ip: SocketAddr, peer_addr: SocketAddr, address: Address<N>) {
601        // Adds a bidirectional map between the listener address and (ambiguous) peer address.
602        self.resolver.write().insert_peer(peer_ip, peer_addr, Some(address));
603        // Add a transmission for this peer in the connected peers.
604        self.peer_pool.write().insert(peer_ip, Peer::new_connecting(peer_ip, false));
605        if let Some(peer) = self.peer_pool.write().get_mut(&peer_ip) {
606            peer.upgrade_to_connected(
607                peer_addr,
608                peer_ip.port(),
609                address,
610                NodeType::Validator,
611                0,
612                get_repo_commit_hash(),
613                ConnectionMode::Gateway,
614            );
615        }
616    }
617
618    /// Sends the given event to specified peer.
619    ///
620    /// This function returns as soon as the event is queued to be sent,
621    /// without waiting for the actual delivery; instead, the caller is provided with a [`oneshot::Receiver`]
622    /// which can be used to determine when and whether the event has been delivered.
623    fn send_inner(&self, peer_ip: SocketAddr, event: Event<N>) -> Option<oneshot::Receiver<io::Result<()>>> {
624        // Resolve the listener IP to the (ambiguous) peer address.
625        let Some(peer_addr) = self.resolve_to_ambiguous(peer_ip) else {
626            warn!("Unable to resolve the listener IP address '{peer_ip}'");
627            return None;
628        };
629        // Retrieve the event name.
630        let name = event.name();
631        // Send the event to the peer.
632        trace!("{CONTEXT} Sending '{name}' to '{peer_ip}'");
633        let result = self.unicast(peer_addr, event);
634        // If the event was unable to be sent, disconnect.
635        if let Err(err) = &result {
636            warn!("{CONTEXT} Failed to send '{name}' to '{peer_ip}': {err:?}");
637            debug!("{CONTEXT} Disconnecting from '{peer_ip}' (unable to send)");
638            self.disconnect(peer_ip);
639        }
640        result.ok()
641    }
642
643    /// Handles the inbound event from the peer. The returned value indicates whether
644    /// the connection is still active, and errors cause a disconnect once they are
645    /// propagated to the caller.
646    async fn inbound(&self, peer_addr: SocketAddr, event: Event<N>) -> Result<bool> {
647        // Retrieve the listener IP for the peer.
648        let Some(peer_ip) = self.resolver.read().get_listener(peer_addr) else {
649            // No longer connected to the peer.
650            trace!("Dropping a {} from {peer_addr} - no longer connected.", event.name());
651            return Ok(false);
652        };
653        // Ensure that the peer is an authorized committee member or a bootstrapper.
654        if !(self.is_authorized_validator_ip(peer_ip)
655            || self
656                .get_connected_peer(peer_ip)
657                .map(|peer| peer.node_type == NodeType::BootstrapClient)
658                .unwrap_or(false))
659        {
660            bail!("{CONTEXT} Dropping '{}' from '{peer_ip}' (not authorized)", event.name())
661        }
662        // Drop the peer, if they have exceeded the rate limit (i.e. they are requesting too much from us).
663        let num_events = self.cache.insert_inbound_event(peer_ip, CACHE_EVENTS_INTERVAL);
664        if num_events >= self.max_cache_events() {
665            bail!("Dropping '{peer_ip}' for spamming events (num_events = {num_events})")
666        }
667        // Rate limit for duplicate requests.
668        match event {
669            Event::CertificateRequest(_) | Event::CertificateResponse(_) => {
670                // Retrieve the certificate ID.
671                let certificate_id = match &event {
672                    Event::CertificateRequest(CertificateRequest { certificate_id }) => *certificate_id,
673                    Event::CertificateResponse(CertificateResponse { certificate }) => certificate.id(),
674                    _ => unreachable!(),
675                };
676                // Skip processing this certificate if the rate limit was exceed (i.e. someone is spamming a specific certificate).
677                let num_events = self.cache.insert_inbound_certificate(certificate_id, CACHE_REQUESTS_INTERVAL);
678                if num_events >= self.max_cache_duplicates() {
679                    return Ok(true);
680                }
681            }
682            Event::TransmissionRequest(TransmissionRequest { transmission_id })
683            | Event::TransmissionResponse(TransmissionResponse { transmission_id, .. }) => {
684                // Skip processing this certificate if the rate limit was exceeded (i.e. someone is spamming a specific certificate).
685                let num_events = self.cache.insert_inbound_transmission(transmission_id, CACHE_REQUESTS_INTERVAL);
686                if num_events >= self.max_cache_duplicates() {
687                    return Ok(true);
688                }
689            }
690            Event::BlockRequest(_) => {
691                let num_events = self.cache.insert_inbound_block_request(peer_ip, CACHE_REQUESTS_INTERVAL);
692                if num_events >= self.max_cache_duplicates() {
693                    return Ok(true);
694                }
695            }
696            _ => {}
697        }
698        trace!("{CONTEXT} Received '{}' from '{peer_ip}'", event.name());
699
700        // This match statement handles the inbound event by deserializing the event,
701        // checking the event is valid, and then calling the appropriate (trait) handler.
702        match event {
703            Event::BatchPropose(batch_propose) => {
704                // Send the batch propose to the primary.
705                let _ = self.primary_sender().tx_batch_propose.send((peer_ip, batch_propose)).await;
706                Ok(true)
707            }
708            Event::BatchSignature(batch_signature) => {
709                // Send the batch signature to the primary.
710                let _ = self.primary_sender().tx_batch_signature.send((peer_ip, batch_signature)).await;
711                Ok(true)
712            }
713            Event::BatchCertified(batch_certified) => {
714                // Send the batch certificate to the primary.
715                let _ = self.primary_sender().tx_batch_certified.send((peer_ip, batch_certified.certificate)).await;
716                Ok(true)
717            }
718            Event::BlockRequest(block_request) => {
719                let BlockRequest { start_height, end_height } = block_request;
720
721                // Ensure the block request is well-formed.
722                if start_height >= end_height {
723                    bail!("Block request from '{peer_ip}' has an invalid range ({start_height}..{end_height})")
724                }
725                // Ensure that the block request is within the allowed bounds.
726                if end_height - start_height > DataBlocks::<N>::MAXIMUM_NUMBER_OF_BLOCKS as u32 {
727                    bail!("Block request from '{peer_ip}' has an excessive range ({start_height}..{end_height})")
728                }
729
730                // End height is exclusive.
731                let latest_consensus_version = N::CONSENSUS_VERSION(end_height - 1)?;
732
733                let self_ = self.clone();
734                let blocks = match task::spawn_blocking(move || {
735                    // Retrieve the blocks within the requested range.
736                    match self_.ledger.get_blocks(start_height..end_height) {
737                        Ok(blocks) => Ok(DataBlocks(blocks)),
738                        Err(error) => bail!("Missing blocks {start_height} to {end_height} from ledger - {error}"),
739                    }
740                })
741                .await
742                {
743                    Ok(Ok(blocks)) => blocks,
744                    Ok(Err(error)) => return Err(error),
745                    Err(error) => return Err(anyhow!("[BlockRequest] {error}")),
746                };
747
748                let self_ = self.clone();
749                tokio::spawn(async move {
750                    // Send the `BlockResponse` message to the peer.
751                    let event =
752                        Event::BlockResponse(BlockResponse::new(block_request, blocks, latest_consensus_version));
753                    Transport::send(&self_, peer_ip, event).await;
754                });
755                Ok(true)
756            }
757            Event::BlockResponse(BlockResponse { request, latest_consensus_version, blocks, .. }) => {
758                // Process the block response. Except for some tests, there is always a sync sender.
759                if let Some(sync_sender) = self.sync_sender.get() {
760                    // Check the response corresponds to a request.
761                    if !self.cache.remove_outbound_block_request(peer_ip, &request) {
762                        bail!("Unsolicited block response from '{peer_ip}'")
763                    }
764
765                    // Perform the deferred non-blocking deserialization of the blocks.
766                    // The deserialization can take a long time (minutes). We should not be running
767                    // this on a blocking task, but on a rayon thread pool.
768                    let (send, recv) = tokio::sync::oneshot::channel();
769                    rayon::spawn_fifo(move || {
770                        let blocks = blocks.deserialize_blocking().map_err(|error| anyhow!("[BlockResponse] {error}"));
771                        let _ = send.send(blocks);
772                    });
773                    let blocks = match recv.await {
774                        Ok(Ok(blocks)) => blocks,
775                        Ok(Err(error)) => bail!("Peer '{peer_ip}' sent an invalid block response - {error}"),
776                        Err(error) => bail!("Peer '{peer_ip}' sent an invalid block response - {error}"),
777                    };
778
779                    // Ensure the block response is well-formed.
780                    blocks.ensure_response_is_well_formed(peer_ip, request.start_height, request.end_height)?;
781                    // Send the blocks to the sync module.
782                    match sync_sender.insert_block_response(peer_ip, blocks.0, latest_consensus_version).await {
783                        Ok(_) => Ok(true),
784                        Err(err) if err.is_benign() => {
785                            let err: anyhow::Error = err.into();
786                            let err = err.context(format!("Ignoring block response from peer '{peer_ip}'"));
787                            debug!("{}", flatten_error(err));
788                            Ok(true)
789                        }
790                        Err(err) if err.is_consensus_version_ahead() => {
791                            let err: anyhow::Error = err.into();
792                            let err = err.context(format!(
793                                "Peer sent a block response with a newer consensus version '{peer_ip}'"
794                            ));
795                            warn!("{}", flatten_error(&err));
796                            Ok(true)
797                        }
798                        Err(err) if err.is_consensus_version_behind() => {
799                            let err: anyhow::Error = err.into();
800                            let err = err.context(format!("Peer sent an invalid block response '{peer_ip}'"));
801
802                            let msg = flatten_error(&err);
803                            error!("{msg}");
804                            self.ip_ban_peer(peer_ip, Some(&msg));
805                            Err(err)
806                        }
807                        Err(err) => {
808                            let err: anyhow::Error = err.into();
809                            let err = err.context(format!("Peer '{peer_ip}' sent an invalid block response"));
810                            warn!("{}", flatten_error(err));
811
812                            // TODO(kaimast): This needs more testing to ensure disconnect is the correct action.
813                            Ok(true)
814                        }
815                    }
816                } else {
817                    debug!("Ignoring block response from '{peer_ip}' - no sync sender");
818                    Ok(true)
819                }
820            }
821            Event::CertificateRequest(certificate_request) => {
822                // Send the certificate request to the sync module.
823                // Except for some tests, there is always a sync sender.
824                if let Some(sync_sender) = self.sync_sender.get() {
825                    // Send the certificate request to the sync module.
826                    let _ = sync_sender.tx_certificate_request.send((peer_ip, certificate_request)).await;
827                }
828                Ok(true)
829            }
830            Event::CertificateResponse(certificate_response) => {
831                // Send the certificate response to the sync module.
832                // Except for some tests, there is always a sync sender.
833                if let Some(sync_sender) = self.sync_sender.get() {
834                    // Send the certificate response to the sync module.
835                    let _ = sync_sender.tx_certificate_response.send((peer_ip, certificate_response)).await;
836                }
837                Ok(true)
838            }
839            Event::ChallengeRequest(..) | Event::ChallengeResponse(..) => {
840                // Disconnect as the peer is not following the protocol.
841                bail!("{CONTEXT} Peer '{peer_ip}' is not following the protocol")
842            }
843            Event::Disconnect(message) => {
844                // The peer informs us that they had disconnected. Disconnect from them too.
845                debug!("Peer '{peer_ip}' decided to disconnect due to '{}'", message.reason);
846                self.disconnect(peer_ip);
847                Ok(false)
848            }
849            Event::PrimaryPing(ping) => {
850                let PrimaryPing { version, block_locators, primary_certificate } = ping;
851
852                // Ensure the event version is not outdated.
853                if version < Event::<N>::VERSION {
854                    bail!("Dropping '{peer_ip}' on event version {version} (outdated)");
855                }
856
857                // Log the validator's height.
858                debug!("Validator '{peer_ip}' is at height {}", block_locators.latest_locator_height());
859
860                // Update the peer locators. Except for some tests, there is always a sync sender.
861                if let Some(sync_sender) = self.sync_sender.get() {
862                    // Check the block locators are valid, and update the validators in the sync module.
863                    if let Err(error) = sync_sender.update_peer_locators(peer_ip, block_locators).await {
864                        bail!("Validator '{peer_ip}' sent invalid block locators - {error}");
865                    }
866                }
867
868                // Send the batch certificates to the primary.
869                let _ = self.primary_sender().tx_primary_ping.send((peer_ip, primary_certificate)).await;
870                Ok(true)
871            }
872            Event::TransmissionRequest(request) => {
873                // TODO (howardwu): Add rate limiting checks on this event, on a per-peer basis.
874                // Determine the worker ID.
875                let Ok(worker_id) = assign_to_worker(request.transmission_id, self.num_workers()) else {
876                    warn!("{CONTEXT} Unable to assign transmission ID '{}' to a worker", request.transmission_id);
877                    return Ok(true);
878                };
879                // Send the transmission request to the worker.
880                if let Some(sender) = self.get_worker_sender(worker_id) {
881                    // Send the transmission request to the worker.
882                    let _ = sender.tx_transmission_request.send((peer_ip, request)).await;
883                }
884                Ok(true)
885            }
886            Event::TransmissionResponse(response) => {
887                // Determine the worker ID.
888                let Ok(worker_id) = assign_to_worker(response.transmission_id, self.num_workers()) else {
889                    warn!("{CONTEXT} Unable to assign transmission ID '{}' to a worker", response.transmission_id);
890                    return Ok(true);
891                };
892                // Send the transmission response to the worker.
893                if let Some(sender) = self.get_worker_sender(worker_id) {
894                    // Send the transmission response to the worker.
895                    let _ = sender.tx_transmission_response.send((peer_ip, response)).await;
896                }
897                Ok(true)
898            }
899            Event::ValidatorsRequest(_) => {
900                let mut connected_peers = self.get_best_connected_peers(Some(MAX_VALIDATORS_TO_SEND));
901                connected_peers.shuffle(&mut rand::rng());
902
903                let self_ = self.clone();
904                tokio::spawn(async move {
905                    // Initialize the validators.
906                    let mut validators = IndexMap::with_capacity(MAX_VALIDATORS_TO_SEND);
907                    // Iterate over the validators.
908                    for validator in connected_peers.into_iter() {
909                        // Add the validator to the list of validators.
910                        validators.insert(validator.listener_addr, validator.aleo_addr);
911                    }
912                    // Send the validators response to the peer.
913                    let event = Event::ValidatorsResponse(ValidatorsResponse { validators });
914                    Transport::send(&self_, peer_ip, event).await;
915                });
916                Ok(true)
917            }
918            Event::ValidatorsResponse(response) => {
919                if self.trusted_peers_only {
920                    bail!("{CONTEXT} Not accepting validators response from '{peer_ip}' (trusted peers only)");
921                }
922                let ValidatorsResponse { validators } = response;
923                // Ensure the number of validators is not too large.
924                ensure!(validators.len() <= MAX_VALIDATORS_TO_SEND, "{CONTEXT} Received too many validators");
925                // Ensure the cache contains a validators request for this peer.
926                if !self.cache.contains_outbound_validators_request(peer_ip) {
927                    bail!("{CONTEXT} Received validators response from '{peer_ip}' without a validators request")
928                }
929                // Decrement the number of validators requests for this peer.
930                self.cache.decrement_outbound_validators_requests(peer_ip);
931
932                // Add valid validators as candidates to the peer pool; only validator-related
933                // filters need to be applied, the rest is handled by `PeerPoolHandling`.
934                let valid_addrs = validators
935                    .into_iter()
936                    .filter_map(|(listener_addr, aleo_addr)| {
937                        (self.account.address() != aleo_addr
938                            && !self.is_connected_address(aleo_addr)
939                            && self.is_authorized_validator_address(aleo_addr))
940                        .then_some((listener_addr, None))
941                    })
942                    .collect::<Vec<_>>();
943                if !valid_addrs.is_empty() {
944                    self.insert_candidate_peers(valid_addrs);
945                }
946
947                Ok(true)
948            }
949            Event::WorkerPing(ping) => {
950                // Ensure the number of transmissions is not too large.
951                ensure!(
952                    ping.transmission_ids.len() <= Worker::<N>::MAX_TRANSMISSIONS_PER_WORKER_PING,
953                    "{CONTEXT} Received too many transmissions"
954                );
955                // Retrieve the number of workers.
956                let num_workers = self.num_workers();
957                // Iterate over the transmission IDs.
958                for transmission_id in ping.transmission_ids.into_iter() {
959                    // Determine the worker ID.
960                    let Ok(worker_id) = assign_to_worker(transmission_id, num_workers) else {
961                        warn!("{CONTEXT} Unable to assign transmission ID '{transmission_id}' to a worker");
962                        continue;
963                    };
964                    // Send the transmission ID to the worker.
965                    if let Some(sender) = self.get_worker_sender(worker_id) {
966                        // Send the transmission ID to the worker.
967                        let _ = sender.tx_worker_ping.send((peer_ip, transmission_id)).await;
968                    }
969                }
970                Ok(true)
971            }
972        }
973    }
974
975    /// Initialize a new instance of the heartbeat.
976    fn initialize_heartbeat(&self) {
977        let self_clone = self.clone();
978        self.spawn(async move {
979            // Sleep briefly to ensure the other nodes are ready to connect.
980            tokio::time::sleep(Duration::from_millis(1000)).await;
981            info!("Starting the heartbeat of the gateway...");
982            loop {
983                // Process a heartbeat in the gateway.
984                self_clone.heartbeat().await;
985                // Sleep for the heartbeat interval.
986                tokio::time::sleep(Duration::from_secs(15)).await;
987            }
988        });
989    }
990
991    /// Spawns a task with the given future; it should only be used for long-running tasks.
992    #[allow(dead_code)]
993    fn spawn<T: Future<Output = ()> + Send + 'static>(&self, future: T) {
994        self.handles.lock().push(tokio::spawn(future));
995    }
996
997    /// Shuts down the gateway.
998    pub async fn shut_down(&self) {
999        info!("Shutting down the gateway...");
1000        // Save the best peers for future use.
1001        if let Err(e) = self.save_best_peers(&self.node_data_dir.gateway_peer_cache_path(), None, true) {
1002            warn!("Failed to persist best validators to disk: {e}");
1003        }
1004        // Abort the tasks.
1005        self.handles.lock().iter().for_each(|handle| handle.abort());
1006        // Close the listener.
1007        self.tcp.shut_down().await;
1008    }
1009}
1010
1011impl<N: Network> Gateway<N> {
1012    /// The minimum time between connection attempts to a peer.
1013    const MINIMUM_TIME_BETWEEN_CONNECTION_ATTEMPTS: Duration = Duration::from_secs(10);
1014    /// The uptime after which nodes log a warning about missing validator connections.
1015    const MISSING_VALIDATOR_CONNECTIONS_GRACE_PERIOD: Duration = Duration::from_secs(60);
1016
1017    /// Handles the heartbeat request.
1018    async fn heartbeat(&self) {
1019        // Log the connected validators.
1020        self.log_connected_validators();
1021        // Log the validator participation scores.
1022        #[cfg(feature = "metrics")]
1023        self.log_participation_scores();
1024        // Keep the trusted validators connected.
1025        self.handle_trusted_validators();
1026        // Keep the bootstrap peers within the allowed range.
1027        self.handle_bootstrap_peers().await;
1028        // Removes any validators that not in the current committee.
1029        self.handle_unauthorized_validators();
1030        // If the number of connected validators is less than the minimum, send a `ValidatorsRequest`.
1031        self.handle_min_connected_validators().await;
1032        // Unban any addresses whose ban time has expired.
1033        self.handle_banned_ips();
1034    }
1035
1036    /// Logs the connected validators.
1037    fn log_connected_validators(&self) {
1038        // Retrieve the connected validators and current committee.
1039        // The gatway may also be connected to bootstrap clients, which we should not log as connected validators.
1040        let connected_validators = self.filter_connected_peers(|peer| peer.node_type == NodeType::Validator);
1041
1042        let committee = match self.ledger.current_committee() {
1043            Ok(c) => c,
1044            Err(err) => {
1045                error!("Failed to get current committee: {err}");
1046                return;
1047            }
1048        };
1049
1050        // Resolve the total number of connectable validators.
1051        let validators_total = committee.num_members().saturating_sub(1);
1052        // Format the total validators message.
1053        let total_validators = format!("(of {validators_total} bonded validators)").dimmed();
1054        // Construct the connections message.
1055        let connections_msg = match connected_validators.len() {
1056            0 => "No connected validators".to_string(),
1057            num_connected => format!("Connected to {num_connected} validators {total_validators}"),
1058        };
1059        info!("{connections_msg}");
1060
1061        // Collect the connected validator addresses and stake.
1062        let mut connected_validator_addresses = HashSet::with_capacity(connected_validators.len());
1063        let mut connected_validator_shas: HashMap<SmolStr, u64> = HashMap::with_capacity(connected_validators.len());
1064        // Insert our sha.
1065        let our_sha = shorten_snarkos_sha(&get_repo_commit_hash());
1066        let our_stake = committee.get_stake(self.account.address());
1067        connected_validator_shas.insert(our_sha.clone(), our_stake);
1068        // Include our own address.
1069        connected_validator_addresses.insert(self.account.address());
1070        // Include and log the connected validators.
1071        for peer in &connected_validators {
1072            // Register the Aleo address.
1073            let address = peer.aleo_addr;
1074            connected_validator_addresses.insert(address);
1075            // Register the snarkOS commit SHA and the associated stake.
1076            let address_stake = committee.get_stake(address);
1077            let short_peer_sha = shorten_snarkos_sha(&peer.snarkos_sha);
1078            *connected_validator_shas.entry(short_peer_sha.clone()).or_default() += address_stake;
1079
1080            debug!(
1081                "{}",
1082                format!(
1083                    "  Connected to: {} - {} (connection age {:?})",
1084                    peer.listener_addr,
1085                    peer.aleo_addr,
1086                    peer.first_seen.elapsed()
1087                )
1088                .dimmed()
1089            );
1090        }
1091
1092        // Log how much of the stake uses our git commit hash.
1093        if let Some(combined_stake) = connected_validator_shas.get(&our_sha) {
1094            let percentage = *combined_stake as f64 / committee.total_stake() as f64 * 100.0;
1095            debug!("{}", format!("  Combined stake @ {our_sha}: {percentage:.2}%").dimmed());
1096            #[cfg(feature = "metrics")]
1097            metrics::gauge(metrics::bft::CONNECTED_STAKE_WITH_MATCHING_SHA, percentage);
1098        }
1099
1100        // Log the validators that are not connected.
1101        let num_not_connected = validators_total.saturating_sub(connected_validators.len());
1102        if num_not_connected > 0 && self.tcp().uptime() > Self::MISSING_VALIDATOR_CONNECTIONS_GRACE_PERIOD {
1103            // Cache the total stake for computing percentages.
1104            let total_stake = committee.total_stake();
1105            let total_stake_f64 = total_stake as f64;
1106
1107            // Collect the committee members.
1108            let committee_members: HashSet<_> =
1109                self.ledger.current_committee().map(|c| c.members().keys().copied().collect()).unwrap_or_default();
1110
1111            let not_connected_stake: u64 = committee_members
1112                .difference(&connected_validator_addresses)
1113                .map(|address| {
1114                    let address_stake = committee.get_stake(*address);
1115                    let address_stake_as_percentage =
1116                        if total_stake == 0 { 0.0 } else { address_stake as f64 / total_stake_f64 * 100.0 };
1117                    debug!(
1118                        "{}",
1119                        format!("  Not connected to {address} ({address_stake_as_percentage:.2}% of total stake)")
1120                            .dimmed()
1121                    );
1122                    address_stake
1123                })
1124                .sum();
1125
1126            let not_connected_stake_as_percentage =
1127                if total_stake == 0 { 0.0 } else { not_connected_stake as f64 / total_stake_f64 * 100.0 };
1128            warn!(
1129                "Not connected to {num_not_connected} validators {total_validators} ({not_connected_stake_as_percentage:.2}% of total stake not connected)"
1130            );
1131            #[cfg(feature = "metrics")]
1132            {
1133                let connected_stake_as_percentage = 100.0 - not_connected_stake_as_percentage;
1134                metrics::gauge(metrics::bft::CONNECTED_STAKE, connected_stake_as_percentage);
1135            }
1136        } else {
1137            #[cfg(feature = "metrics")]
1138            metrics::gauge(metrics::bft::CONNECTED_STAKE, 100.0);
1139        };
1140
1141        if !committee.is_quorum_threshold_reached(&connected_validator_addresses) {
1142            // Not being connected to a quorum of validators is begning during startup.
1143            if self.tcp().uptime() > Self::MISSING_VALIDATOR_CONNECTIONS_GRACE_PERIOD {
1144                error!("Not connected to a quorum of validators");
1145            } else {
1146                debug!("Not connected to a quorum of validators");
1147            }
1148        }
1149    }
1150
1151    // Logs the validator participation scores.
1152    #[cfg(feature = "metrics")]
1153    fn log_participation_scores(&self) {
1154        if let Ok(committee_lookback) = self.ledger.get_committee_lookback_for_round(self.storage.current_round()) {
1155            // Retrieve the participation scores.
1156            let participation_scores = self.validator_telemetry().get_participation_scores(&committee_lookback);
1157
1158            // Log the participation scores.
1159            debug!("Participation Scores (in the last {} rounds):", self.storage.max_gc_rounds());
1160            for (address, (cert_score, sig_score)) in participation_scores {
1161                debug!(
1162                    "{}",
1163                    format!("  {address} - certificates: {cert_score:.2}%  signatures: {sig_score:.2}%").dimmed()
1164                );
1165            }
1166        }
1167    }
1168
1169    /// This function attempts to connect to any disconnected trusted validators.
1170    fn handle_trusted_validators(&self) {
1171        let trusted_peers = self.trusted_peers();
1172
1173        // Attempt to re-establish connections with any trusted peer that is not connected already.
1174        let handles: Vec<JoinHandle<_>> = trusted_peers
1175            .iter()
1176            .filter_map(|validator_ip| {
1177                // Attempt to connect to the trusted validator.
1178                match self.connect(*validator_ip) {
1179                    Ok(hdl) => Some(hdl),
1180                    Err(ConnectError::SelfConnect { .. })
1181                    | Err(ConnectError::AlreadyConnected { .. })
1182                    | Err(ConnectError::AlreadyConnecting { .. }) => None,
1183                    Err(err) => {
1184                        warn!("Could not initiate connection to trusted validator at '{validator_ip}' - {err}");
1185                        None
1186                    }
1187                }
1188            })
1189            .collect();
1190
1191        if !handles.is_empty() {
1192            info!("Reconnecting to {} out of {} trusted validators", handles.len(), trusted_peers.len());
1193        }
1194    }
1195
1196    /// This function keeps the number of bootstrap peers within the allowed range.
1197    async fn handle_bootstrap_peers(&self) {
1198        // Return early if we are in trusted peers only mode.
1199        if self.trusted_peers_only {
1200            return;
1201        }
1202        // Split the bootstrap peers into connected and candidate lists.
1203        let mut candidate_bootstrap = Vec::new();
1204        let connected_bootstrap = self.filter_connected_peers(|peer| peer.node_type == NodeType::BootstrapClient);
1205        for bootstrap_ip in bootstrap_peers::<N>(self.is_dev()) {
1206            if !connected_bootstrap.iter().any(|peer| peer.listener_addr == bootstrap_ip) {
1207                candidate_bootstrap.push(bootstrap_ip);
1208            }
1209        }
1210        // If there are not enough connected bootstrap peers, connect to more.
1211        if connected_bootstrap.is_empty() {
1212            // Sample a random bootstrap peer to connect to (drop rng before any await).
1213            let peer_to_connect = candidate_bootstrap.into_iter().choose(&mut rand::rng());
1214            if let Some(peer_ip) = peer_to_connect {
1215                match self.connect(peer_ip) {
1216                    Ok(hdl) => {
1217                        debug!("{CONTEXT} (Re-)connecting to bootstrap peer at '{peer_ip}'");
1218                        let result = hdl.await;
1219                        if let Err(err) = result {
1220                            warn!("{CONTEXT} Failed to connect to bootstrap peer at '{peer_ip}' - {err}");
1221                        }
1222                    }
1223                    Err(ConnectError::AlreadyConnected { .. }) | Err(ConnectError::AlreadyConnecting { .. }) => {}
1224                    Err(err) => {
1225                        warn!("{CONTEXT} Could not initiate connection to bootstrap peer at '{peer_ip}' - {err}")
1226                    }
1227                }
1228            }
1229        }
1230        // Determine if the node is connected to more bootstrap peers than allowed.
1231        let num_surplus = connected_bootstrap.len().saturating_sub(1);
1232        if num_surplus > 0 {
1233            // Sample peers to disconnect (drop rng before any await).
1234            let peers_to_disconnect = connected_bootstrap.into_iter().sample(&mut rand::rng(), num_surplus);
1235            for peer in peers_to_disconnect {
1236                info!("{CONTEXT} Disconnecting from '{}' (exceeded maximum bootstrap)", peer.listener_addr);
1237                <Self as Transport<N>>::send(
1238                    self,
1239                    peer.listener_addr,
1240                    Event::Disconnect(DisconnectReason::NoReasonGiven.into()),
1241                )
1242                .await;
1243                // Disconnect from this peer.
1244                self.disconnect(peer.listener_addr);
1245            }
1246        }
1247    }
1248
1249    /// This function attempts to disconnect any validators that are not in the current committee.
1250    fn handle_unauthorized_validators(&self) {
1251        let self_ = self.clone();
1252        tokio::spawn(async move {
1253            // Retrieve the connected validators.
1254            let validators = self_.get_connected_peers();
1255            // Iterate over the validator IPs.
1256            for peer in validators {
1257                // Skip bootstrapper peers.
1258                if peer.node_type == NodeType::BootstrapClient {
1259                    continue;
1260                }
1261                // Disconnect any validator that is not in the current committee.
1262                if !self_.is_authorized_validator_ip(peer.listener_addr) {
1263                    warn!(
1264                        "{CONTEXT} Disconnecting from '{}' - Validator is not in the current committee",
1265                        peer.listener_addr
1266                    );
1267                    Transport::send(&self_, peer.listener_addr, DisconnectReason::ProtocolViolation.into()).await;
1268                    // Disconnect from this peer.
1269                    self_.disconnect(peer.listener_addr);
1270                }
1271            }
1272        });
1273    }
1274
1275    /// This function sends a `ValidatorsRequest` to a random validator,
1276    /// if the number of connected validators is less than the minimum.
1277    /// It also attempts to connect to known unconnected validators.
1278    async fn handle_min_connected_validators(&self) {
1279        // Attempt to connect to untrusted validators we're not connected to yet.
1280        // The trusted ones are already handled by `handle_trusted_validators`.
1281        let trusted_validators = self.trusted_peers();
1282        if self.number_of_connected_peers() < N::LATEST_MAX_CERTIFICATES() as usize {
1283            let (addrs, handles): (Vec<_>, Vec<_>) = self
1284                .get_candidate_peers()
1285                .iter()
1286                .filter_map(|peer| {
1287                    if trusted_validators.contains(&peer.listener_addr) {
1288                        return None;
1289                    }
1290
1291                    if let Some(previous_attempt) = peer.last_connection_attempt
1292                        && previous_attempt.elapsed() < Self::MINIMUM_TIME_BETWEEN_CONNECTION_ATTEMPTS
1293                    {
1294                        return None;
1295                    }
1296
1297                    match self.connect(peer.listener_addr) {
1298                        Ok(hdl) => Some((peer.listener_addr, hdl)),
1299                        Err(ConnectError::AlreadyConnected { .. })
1300                        | Err(ConnectError::AlreadyConnecting { .. })
1301                        | Err(ConnectError::SelfConnect { .. }) => None,
1302                        Err(err) => {
1303                            warn!(
1304                                "{CONTEXT} Could not initiate connection to validator at '{}' - {err}",
1305                                peer.listener_addr
1306                            );
1307                            None
1308                        }
1309                    }
1310                })
1311                .unzip();
1312
1313            for (addr, result) in addrs.into_iter().zip(join_all(handles).await) {
1314                if let Err(err) = result {
1315                    warn!("{CONTEXT} Failed to connect to validator at '{addr}' - {err}");
1316                }
1317            }
1318
1319            // Retrieve the connected validators.
1320            let validators = self.connected_peers();
1321            // If there are no validator IPs to connect to, return early.
1322            if validators.is_empty() {
1323                return;
1324            }
1325            // Select a random validator IP.
1326            if let Some(validator_ip) = validators.into_iter().choose(&mut rand::rng()) {
1327                let self_ = self.clone();
1328                tokio::spawn(async move {
1329                    // Increment the number of outbound validators requests for this validator.
1330                    self_.cache.increment_outbound_validators_requests(validator_ip);
1331                    // Send a `ValidatorsRequest` to the validator.
1332                    let _ = Transport::send(&self_, validator_ip, Event::ValidatorsRequest(ValidatorsRequest)).await;
1333                });
1334            }
1335        }
1336    }
1337
1338    /// Processes a message received from the network.
1339    async fn process_message_inner(&self, peer_addr: SocketAddr, message: Event<N>) {
1340        // Process the message. Disconnect if the peer violated the protocol.
1341        if let Err(error) = self.inbound(peer_addr, message).await
1342            && let Some(peer_ip) = self.resolver.read().get_listener(peer_addr)
1343        {
1344            warn!("{CONTEXT} Disconnecting from '{peer_ip}' - {error}");
1345            let self_ = self.clone();
1346            tokio::spawn(async move {
1347                Transport::send(&self_, peer_ip, DisconnectReason::ProtocolViolation.into()).await;
1348                // Disconnect from this peer.
1349                self_.disconnect(peer_ip);
1350            });
1351        }
1352    }
1353
1354    // Remove addresses whose ban time has expired.
1355    fn handle_banned_ips(&self) {
1356        self.tcp.banned_peers().remove_old_bans(IP_BAN_TIME_IN_SECS);
1357    }
1358}
1359
1360#[async_trait]
1361impl<N: Network> Transport<N> for Gateway<N> {
1362    /// Sends the given event to specified peer.
1363    ///
1364    /// This method is rate limited to prevent spamming the peer.
1365    ///
1366    /// This function returns as soon as the event is queued to be sent,
1367    /// without waiting for the actual delivery; instead, the caller is provided with a [`oneshot::Receiver`]
1368    /// which can be used to determine when and whether the event has been delivered.
1369    async fn send(&self, peer_ip: SocketAddr, mut event: Event<N>) -> Option<oneshot::Receiver<io::Result<()>>> {
1370        // Serialize the payload here, rather than leaving it for the connection's writer task.
1371        //
1372        // `Data` defers serialization until the event is written to the stream, which puts it on
1373        // the writer task -- a Tokio worker -- and inside the write timeout. For the large
1374        // responses (`BlockResponse` in particular, which can carry `DataBlocks::MAXIMUM_NUMBER_
1375        // OF_BLOCKS`) that is a substantial amount of compute on the reactor.
1376        //
1377        // The check keeps the hop off the common path: most events carry no payload, and a
1378        // broadcast has already serialized its payload before fanning out here, so this is a no-op
1379        // for every recipient after the first.
1380        if event.has_unserialized_payload() {
1381            let name = event.name();
1382            event = match spawn_blocking!({
1383                let mut event = event;
1384                event.serialize_payload()?;
1385                Ok(event)
1386            }) {
1387                Ok(event) => event,
1388                Err(err) => {
1389                    error!("{CONTEXT} Unable to serialize '{name}' for '{peer_ip}' - {err}");
1390                    return None;
1391                }
1392            };
1393        }
1394
1395        macro_rules! send {
1396            ($self:ident, $cache_map:ident, $interval:expr, $freq:ident) => {{
1397                // Rate limit the number of certificate requests sent to the peer.
1398                while $self.cache.$cache_map(peer_ip, $interval) > $self.$freq() {
1399                    // Sleep for a short period of time to allow the cache to clear.
1400                    tokio::time::sleep(Duration::from_millis(10)).await;
1401                }
1402                // Send the event to the peer.
1403                $self.send_inner(peer_ip, event)
1404            }};
1405        }
1406
1407        // Increment the cache for certificate, transmission and block events.
1408        match event {
1409            Event::CertificateRequest(_) | Event::CertificateResponse(_) => {
1410                // Update the outbound event cache. This is necessary to ensure we don't under count the outbound events.
1411                self.cache.insert_outbound_event(peer_ip, CACHE_EVENTS_INTERVAL);
1412                // Send the event to the peer.
1413                send!(self, insert_outbound_certificate, CACHE_REQUESTS_INTERVAL, max_cache_certificates)
1414            }
1415            Event::TransmissionRequest(_) | Event::TransmissionResponse(_) => {
1416                // Update the outbound event cache. This is necessary to ensure we don't under count the outbound events.
1417                self.cache.insert_outbound_event(peer_ip, CACHE_EVENTS_INTERVAL);
1418                // Send the event to the peer.
1419                send!(self, insert_outbound_transmission, CACHE_REQUESTS_INTERVAL, max_cache_transmissions)
1420            }
1421            Event::BlockRequest(request) => {
1422                // Insert the outbound request so we can match it to responses.
1423                self.cache.insert_outbound_block_request(peer_ip, request);
1424                // Send the event to the peer and update the outbound event cache, use the general rate limit.
1425                send!(self, insert_outbound_event, CACHE_EVENTS_INTERVAL, max_cache_events)
1426            }
1427            _ => {
1428                // Send the event to the peer, use the general rate limit.
1429                send!(self, insert_outbound_event, CACHE_EVENTS_INTERVAL, max_cache_events)
1430            }
1431        }
1432    }
1433
1434    /// Broadcasts the given event to all connected peers.
1435    fn broadcast(&self, mut event: Event<N>) {
1436        // Ensure there are connected peers.
1437        if self.number_of_connected_peers() > 0 {
1438            let self_ = self.clone();
1439            let connected_peers = self.connected_peers();
1440            tokio::spawn(async move {
1441                // Serialize the event's payload once, rather than once per recipient; every
1442                // recipient then shares the resulting buffer. `Transport::send` would otherwise do
1443                // this separately for each peer below.
1444                if event.has_unserialized_payload() {
1445                    let name = event.name();
1446                    event = match spawn_blocking!({
1447                        let mut event = event;
1448                        event.serialize_payload()?;
1449                        Ok(event)
1450                    }) {
1451                        Ok(event) => event,
1452                        Err(err) => {
1453                            error!("{CONTEXT} Unable to serialize '{name}' for broadcast - {err}");
1454                            return;
1455                        }
1456                    };
1457                }
1458                // Iterate through all connected peers.
1459                for peer_ip in connected_peers {
1460                    // Send the event to the peer.
1461                    let _ = Transport::send(&self_, peer_ip, event.clone()).await;
1462                }
1463            });
1464        }
1465    }
1466}
1467
1468impl<N: Network> P2P for Gateway<N> {
1469    /// Returns a reference to the TCP instance.
1470    fn tcp(&self) -> &Tcp {
1471        &self.tcp
1472    }
1473}
1474
1475#[async_trait]
1476impl<N: Network> Reading for Gateway<N> {
1477    type Codec = EventCodec<N>;
1478    type Message = Event<N>;
1479
1480    /// Creates a [`Decoder`] used to interpret messages from the network.
1481    /// The `side` param indicates the connection side **from the node's perspective**.
1482    fn codec(&self, _peer_addr: SocketAddr, _side: ConnectionSide) -> Self::Codec {
1483        Default::default()
1484    }
1485
1486    /// Processes a message received from the network.
1487    async fn process_message(&self, peer_addr: SocketAddr, message: Self::Message) -> io::Result<()> {
1488        if matches!(message, Event::BlockRequest(_) | Event::BlockResponse(_)) {
1489            let self_ = self.clone();
1490            // Handle BlockRequest and BlockResponse messages in a separate task to not block the
1491            // inbound queue.
1492            tokio::spawn(async move {
1493                self_.process_message_inner(peer_addr, message).await;
1494            });
1495        } else {
1496            self.process_message_inner(peer_addr, message).await;
1497        }
1498        Ok(())
1499    }
1500
1501    /// Computes the depth of per-connection queues used to process inbound messages, sufficient to process the maximum expected load at any given moment.
1502    /// The greater it is, the more inbound messages the node can enqueue, but a too large value can make the node more susceptible to DoS attacks.
1503    /// See [`per_connection_queue_depth`] for the derivation.
1504    fn message_queue_depth(&self) -> usize {
1505        per_connection_queue_depth::<N>()
1506    }
1507}
1508
1509#[async_trait]
1510impl<N: Network> Writing for Gateway<N> {
1511    type Codec = EventCodec<N>;
1512    type Message = Event<N>;
1513
1514    /// Creates an [`Encoder`] used to write the outbound messages to the target stream.
1515    /// The `side` parameter indicates the connection side **from the node's perspective**.
1516    fn codec(&self, _peer_addr: SocketAddr, _side: ConnectionSide) -> Self::Codec {
1517        Default::default()
1518    }
1519
1520    /// Computes the depth of per-connection queues used to send outbound messages, sufficient to process the maximum expected load at any given moment.
1521    /// The greater it is, the more outbound messages the node can enqueue. A too large value might obscure potential issues with your implementation
1522    /// (like slow serialization) or network, and lets a peer that stops reading its socket pin an unreasonable amount of our heap.
1523    /// See [`per_connection_queue_depth`] for the derivation.
1524    fn message_queue_depth(&self) -> usize {
1525        per_connection_queue_depth::<N>()
1526    }
1527}
1528
1529#[async_trait]
1530impl<N: Network> Disconnect for Gateway<N> {
1531    /// Any extra operations to be performed during a disconnect.
1532    async fn handle_disconnect(&self, peer_addr: SocketAddr, origin: DisconnectOrigin) {
1533        debug!("Physically disconnecting from {peer_addr}; origin: {origin:?}");
1534
1535        if let Some(peer_ip) = self.resolve_to_listener(&peer_addr) {
1536            // TODO(kaimast): This can, in theory, still lead to race conditions, if we immediately reconnect to the same peer.
1537            // In practice, there should always be a significant delay between those two delays, so it is not an immediate issue.
1538            //
1539            // To properly fix this, we either needk hold a lock here, or add a dedicated "disconnecting" state, so that
1540            // a peer is not re-added while the rest of the disconnect logic is running.
1541            let was_fully_connected = self.downgrade_peer_to_candidate(peer_ip);
1542
1543            // Remove the peer from the sync module. Except for some tests, there is always a sync sender.
1544            if was_fully_connected && let Some(sync_sender) = self.sync_sender.get() {
1545                let (tx, rx) = oneshot::channel();
1546
1547                if let Err(err) = sync_sender.tx_block_sync_remove_peer.send((peer_ip, tx)).await {
1548                    let err: anyhow::Error = err.into();
1549                    let err =
1550                        err.context(format!("Unable to remove disconnecting peer '{peer_ip}' from the sync module"));
1551                    warn!("{CONTEXT} {}", flatten_error(err));
1552                }
1553
1554                if let Err(err) = rx.await {
1555                    let err: anyhow::Error = err.into();
1556                    let err =
1557                        err.context(format!("Unable to remove disconnecting peer '{peer_ip}' from the sync module"));
1558                    warn!("{CONTEXT} {}", flatten_error(err));
1559                }
1560            }
1561            // We don't clear this map based on time but only on peer disconnect.
1562            // This is sufficient to avoid infinite growth as the committee has a fixed number
1563            // of members.
1564            self.cache.clear_outbound_validators_requests(peer_ip);
1565            self.cache.clear_outbound_block_requests(peer_ip);
1566        } else {
1567            warn!("{CONTEXT} Got disconnect for a peer '{peer_addr}' that is not in the peer pool");
1568        }
1569    }
1570}
1571
1572#[async_trait]
1573impl<N: Network> OnConnect for Gateway<N> {
1574    async fn on_connect(&self, peer_addr: SocketAddr) {
1575        if let Some(listener_addr) = self.resolve_to_listener(&peer_addr) {
1576            if let Some(peer) = self.get_connected_peer(listener_addr) {
1577                if peer.node_type == NodeType::BootstrapClient {
1578                    self.cache.increment_outbound_validators_requests(listener_addr);
1579                    let _ =
1580                        <Self as Transport<N>>::send(self, listener_addr, Event::ValidatorsRequest(ValidatorsRequest))
1581                            .await;
1582                }
1583            }
1584        }
1585    }
1586}
1587
1588#[async_trait]
1589impl<N: Network> Handshake for Gateway<N> {
1590    /// Performs the handshake protocol.
1591    async fn perform_handshake(&self, mut connection: Connection) -> Result<Connection, ConnectError> {
1592        // Perform the handshake.
1593        let peer_addr = connection.addr();
1594        let peer_side = connection.side();
1595
1596        // Check (or impose) IP-level bans.
1597        #[cfg(not(test))]
1598        if self.dev().is_none() && peer_side == ConnectionSide::Initiator {
1599            // If the IP is already banned reject the connection.
1600            if self.is_ip_banned(peer_addr.ip()) {
1601                trace!("{CONTEXT} Rejected a connection request from banned IP '{}'", peer_addr.ip());
1602                return Err(ConnectError::BannedIp { ip: peer_addr.ip() });
1603            }
1604
1605            let num_attempts = self.cache.insert_inbound_connection(peer_addr.ip(), CONNECTION_ATTEMPTS_SINCE_SECS);
1606
1607            debug!("Number of connection attempts from '{}': {}", peer_addr.ip(), num_attempts);
1608            if num_attempts > MAX_CONNECTION_ATTEMPTS {
1609                self.update_ip_ban(peer_addr.ip());
1610                trace!("{CONTEXT} Rejected a consecutive connection request from IP '{}'", peer_addr.ip());
1611                return Err(ConnectError::other(anyhow!("'{}' appears to be spamming connections", peer_addr.ip())));
1612            }
1613        }
1614
1615        let stream = self.borrow_stream(&mut connection);
1616        // Make the socket more robust; every other handshake in the node does this too.
1617        harden_socket(stream)?;
1618
1619        // If this is an inbound connection, we log it, but don't know the listening address yet.
1620        // Otherwise, we can immediately register the listening address.
1621        let mut listener_addr = if peer_side == ConnectionSide::Initiator {
1622            debug!("{CONTEXT} Received a connection request from '{peer_addr}'");
1623            None
1624        } else {
1625            debug!("{CONTEXT} Shaking hands with {peer_addr}...");
1626            Some(peer_addr)
1627        };
1628
1629        // Retrieve the restrictions ID.
1630        let restrictions_id = self.ledger.latest_restrictions_id();
1631
1632        // Perform the handshake; we pass on a mutable reference to peer_ip in case the process is broken at any point in time.
1633        //
1634        // The initiator picks the handshake protocol, gated on the block height so that validators
1635        // can be upgraded one at a time; the responder goes along with whichever one it is offered.
1636        let handshake_result = if peer_side == ConnectionSide::Responder {
1637            if self.initiates_noise_handshake() {
1638                write_noise_magic(stream).await?;
1639                self.handshake_inner_initiator_noise(peer_addr, restrictions_id, stream).await
1640            } else {
1641                self.handshake_inner_initiator(peer_addr, restrictions_id, stream).await
1642            }
1643        } else {
1644            match detect_handshake_protocol(stream).await? {
1645                (HandshakeProtocol::Noise, _) => {
1646                    self.handshake_inner_responder_noise(peer_addr, &mut listener_addr, restrictions_id, stream).await
1647                }
1648                (HandshakeProtocol::Legacy, _) if !self.accepts_legacy_handshake() => {
1649                    Err(ConnectError::other(format!("'{peer_addr}' offered the legacy handshake, which has expired")))
1650                }
1651                (HandshakeProtocol::Legacy, prefix) => {
1652                    self.handshake_inner_responder(peer_addr, &mut listener_addr, restrictions_id, stream, &prefix)
1653                        .await
1654                }
1655            }
1656        };
1657
1658        // Register the peer, or roll it back, if the handshake got far enough to learn its listening
1659        // address.
1660        match (&handshake_result, listener_addr) {
1661            (Ok(peer_info), Some(addr)) => {
1662                let node_type = if bootstrap_peers::<N>(self.is_dev()).contains(&addr) {
1663                    NodeType::BootstrapClient
1664                } else {
1665                    NodeType::Validator
1666                };
1667
1668                let mut peer_pool = self.peer_pool.write();
1669
1670                // Validators may change their listening address, but not the Aleo address; traverse
1671                // the peer pool, and retain previously connected (the prior Aleo address is known)
1672                // candidate peers with the same Aleo address only if their listening address is the
1673                // same; otherwise, it may be concluded that a known validator has changed their
1674                // listening address, and thus the old entry should be removed as outdated.
1675                peer_pool.retain(|_, peer| {
1676                    if let Peer::Candidate(peer) = peer
1677                        && let Some(old_aleo_addr) = peer.last_known_aleo_addr
1678                    {
1679                        old_aleo_addr != peer_info.address || peer.listener_addr == addr
1680                    } else {
1681                        true
1682                    }
1683                });
1684
1685                if let Some(peer) = peer_pool.get_mut(&addr) {
1686                    self.resolver.write().insert_peer(addr, peer_addr, Some(peer_info.address));
1687                    peer.upgrade_to_connected(
1688                        peer_addr,
1689                        peer_info.listener_port,
1690                        peer_info.address,
1691                        node_type,
1692                        peer_info.version,
1693                        peer_info.snarkos_sha,
1694                        ConnectionMode::Gateway,
1695                    );
1696                }
1697                info!("{CONTEXT} Connected to '{addr}'");
1698            }
1699            (Err(_), Some(addr)) => {
1700                if let Some(peer) = self.peer_pool.write().get_mut(&addr) {
1701                    // The peer may only be downgraded if it's a ConnectingPeer.
1702                    if peer.is_connecting() {
1703                        peer.downgrade_to_candidate(addr);
1704                    }
1705                }
1706            }
1707            // Neither handshake can succeed before it has learned the peer's listening address, so
1708            // this is unreachable; if it ever happened, the connection would go live with neither a
1709            // peer pool nor a resolver entry, and every event on it would be discarded as coming
1710            // from an unknown peer. Refuse it instead of leaving it in that state.
1711            (Ok(_), None) => {
1712                return Err(ConnectError::other(format!(
1713                    "the handshake with '{peer_addr}' succeeded without a listening address"
1714                )));
1715            }
1716            // The handshake failed before the peer's listening address was known, so there is nothing
1717            // in the pool to roll back.
1718            (Err(_), None) => {}
1719        }
1720
1721        // Abort the connection on failure whether or not the peer reached the pool.
1722        handshake_result?;
1723
1724        Ok(connection)
1725    }
1726}
1727
1728/// A macro unwrapping the expected handshake event or returning an error for unexpected events.
1729macro_rules! expect_event {
1730    ($event_ty:path, $framed:expr, $peer_addr:expr) => {
1731        match $framed.try_next().await? {
1732            // Received the expected event, proceed.
1733            Some($event_ty(data)) => {
1734                trace!("{CONTEXT} Received '{}' from '{}'", data.name(), $peer_addr);
1735                data
1736            }
1737            // Received a disconnect event, abort.
1738            Some(Event::Disconnect($crate::events::Disconnect { reason })) => {
1739                return Err(ConnectError::other(format!("'{}' disconnected with reason \"{reason}\"", $peer_addr)));
1740            }
1741            // Received an unexpected event, abort.
1742            Some(ty) => {
1743                return Err(ConnectError::other(format!(
1744                    "'{}' did not follow the handshake protocol: received {:?} instead of {}",
1745                    $peer_addr,
1746                    ty.name(),
1747                    stringify!($msg_ty),
1748                )));
1749            }
1750            // Received nothing.
1751            None => return Err(ConnectError::IoError(io::ErrorKind::BrokenPipe.into())),
1752        }
1753    };
1754}
1755
1756/// Send the given message to the peer.
1757async fn send_event<N: Network>(
1758    framed: &mut Framed<&mut TcpStream, EventCodec<N>>,
1759    peer_addr: SocketAddr,
1760    event: Event<N>,
1761) -> io::Result<()> {
1762    trace!("{CONTEXT} Sending '{}' to '{peer_addr}'", event.name());
1763    framed.send(event).await
1764}
1765
1766/// Serializes a handshake payload for transmission inside a Noise message.
1767fn encode_payload<T: ToBytes>(payload: &T) -> Result<Vec<u8>, ConnectError> {
1768    snarkos_node_bft_events::encode_payload(payload)
1769}
1770
1771/// Deserializes a handshake payload received inside a Noise message.
1772fn decode_payload<T: FromBytes>(peer_addr: SocketAddr, bytes: &[u8]) -> Result<T, ConnectError> {
1773    snarkos_node_bft_events::decode_payload(peer_addr, bytes)
1774}
1775
1776/// Verifies a peer's proof that it owns the Aleo address it claims: its signature over the binding
1777/// message for this Noise session.
1778///
1779/// This is by far the most expensive step of the handshake, which is why both sides only reach it
1780/// once every cheap check has already passed.
1781#[must_use]
1782async fn verify_binding_signature<N: Network>(
1783    peer_addr: SocketAddr,
1784    signature: Data<Signature<N>>,
1785    address: Address<N>,
1786    binding: &[u8],
1787) -> Option<DisconnectReason> {
1788    // Perform the deferred non-blocking deserialization of the signature.
1789    let Ok(signature) = spawn_blocking!(signature.deserialize_blocking()) else {
1790        warn!("{CONTEXT} Handshake with '{peer_addr}' failed (cannot deserialize the signature)");
1791        return Some(DisconnectReason::InvalidChallengeResponse);
1792    };
1793    // Verify the signature.
1794    if !signature.verify_bytes(&address, binding) {
1795        warn!("{CONTEXT} Handshake with '{peer_addr}' failed (invalid signature)");
1796        return Some(DisconnectReason::InvalidChallengeResponse);
1797    }
1798
1799    None
1800}
1801
1802/// Concludes a Noise handshake, handing the stream back to the connection.
1803///
1804/// The stream deliberately goes back unframed. [`Reading`] builds a codec of its own, capped at the
1805/// 256 MiB an event may be rather than the 64 KiB a Noise message may be, and takes full
1806/// responsibility for the stream from here on. Handing it a bare stream is only sound because the
1807/// session reads its messages exactly: anything the peer pipelined behind the last handshake message
1808/// is still on the socket rather than in a buffer about to be dropped.
1809fn finish_noise_handshake(noise: NoiseSession<&mut TcpStream>) {
1810    // Note: the transport keys are discarded here, leaving the resulting connection unencrypted.
1811    let _stream = noise.into_inner();
1812}
1813
1814/// Concludes a legacy handshake, dropping its codec along with anything the peer had pipelined behind
1815/// the last handshake message.
1816///
1817/// Unlike the Noise handshake, this one reads through a buffering codec, so it can pull bytes off the
1818/// socket that belong to the events which follow - and those are lost here, leaving the event codec to
1819/// start in the middle of a frame. That has always been the case, and carrying them across would mean
1820/// wrapping the stream for a protocol that is being retired, so it is logged rather than fixed.
1821fn note_legacy_handshake_end<N: Network>(framed: Framed<&mut TcpStream, EventCodec<N>>, peer_addr: SocketAddr) {
1822    let leftover = framed.into_parts().read_buf;
1823
1824    if !leftover.is_empty() {
1825        debug!("{CONTEXT} Discarding {} bytes '{peer_addr}' sent before the handshake was over", leftover.len());
1826    }
1827}
1828
1829/// Distills a legacy challenge request into the protocol-agnostic peer information.
1830///
1831/// The peer's restrictions ID is not part of its challenge request, but by the time this is called
1832/// `verify_challenge_response` has established that it matches the one passed in.
1833fn peer_info_from_challenge_request<N: Network>(
1834    request: ChallengeRequest<N>,
1835    restrictions_id: Field<N>,
1836) -> PeerInfo<N> {
1837    let ChallengeRequest { version, listener_port, address, nonce: _, snarkos_sha } = request;
1838    PeerInfo { version, listener_port, address, restrictions_id, snarkos_sha }
1839}
1840
1841impl<N: Network> Gateway<N> {
1842    /// Returns `true` if this node should offer the Noise handshake when it dials the given peer.
1843    fn initiates_noise_handshake(&self) -> bool {
1844        // Tests pin the choice, so that both sides of the transition can be covered.
1845        if let Some(initiates) = self.pinned_handshake_protocol() {
1846            return initiates;
1847        }
1848
1849        // Development nodes always take the new path, so that devnets exercise it.
1850        self.is_dev() || self.consensus_version_reached(NOISE_HANDSHAKE_ACTIVATION)
1851    }
1852
1853    /// Returns `true` if this node still accepts the legacy handshake from a peer that dials it.
1854    ///
1855    /// Unlike the choice of what to offer, this is not pinned in tests and not forced in development:
1856    /// a converted node has to keep answering unconverted ones for the whole of the transition, and
1857    /// the tests covering that rely on it.
1858    fn accepts_legacy_handshake(&self) -> bool {
1859        !self.consensus_version_reached(LEGACY_HANDSHAKE_EXPIRY)
1860    }
1861
1862    /// Returns `true` if the given consensus version is scheduled and the ledger has reached it.
1863    fn consensus_version_reached(&self, version: Option<ConsensusVersion>) -> bool {
1864        version.is_some_and(|version| {
1865            N::CONSENSUS_HEIGHT(version).is_ok_and(|height| self.ledger.latest_block_height() >= height)
1866        })
1867    }
1868
1869    /// The pinned choice of handshake protocol; always `None` outside tests.
1870    #[cfg(not(any(test, feature = "test")))]
1871    fn pinned_handshake_protocol(&self) -> Option<bool> {
1872        None
1873    }
1874
1875    /// The pinned choice of handshake protocol; see `InnerGateway::initiates_noise_handshake`.
1876    #[cfg(any(test, feature = "test"))]
1877    fn pinned_handshake_protocol(&self) -> Option<bool> {
1878        Some(self.initiates_noise_handshake.load(std::sync::atomic::Ordering::Relaxed))
1879    }
1880
1881    /// Pins whether this node offers the Noise handshake when it dials, regardless of the activation
1882    /// height.
1883    ///
1884    /// This exists so that tests can cover the transition, during which a converted node still has
1885    /// to be able to shake hands with unconverted ones.
1886    #[cfg(any(test, feature = "test"))]
1887    pub fn set_initiates_noise_handshake(&self, initiates: bool) {
1888        self.initiates_noise_handshake.store(initiates, std::sync::atomic::Ordering::Relaxed);
1889    }
1890
1891    /// Returns the snarkOS commit hash to disclose to a peer, if any.
1892    fn snarkos_sha(&self) -> Option<[u8; 40]> {
1893        let current_block_height = self.ledger.latest_block_height();
1894        let consensus_version = N::CONSENSUS_VERSION(current_block_height).unwrap();
1895        match (self.is_dev(), consensus_version >= ConsensusVersion::V12, get_repo_commit_hash()) {
1896            (true, _, Some(sha)) => Some(sha),
1897            (_, true, Some(sha)) => Some(sha),
1898            _ => None,
1899        }
1900    }
1901
1902    /// The connection initiator side of the Noise handshake.
1903    ///
1904    /// The initiator does the expensive work first: it signs before the responder has committed to
1905    /// anything, and only learns whether it was accepted when the fourth message arrives. See
1906    /// [`Gateway::handshake_inner_responder_noise`] for the other half of that bargain.
1907    async fn handshake_inner_initiator_noise<'a>(
1908        &'a self,
1909        peer_addr: SocketAddr,
1910        restrictions_id: Field<N>,
1911        stream: &'a mut TcpStream,
1912    ) -> Result<PeerInfo<N>, ConnectError> {
1913        // Note who answered here last time, before the pool entry becomes a connecting one and stops
1914        // carrying it.
1915        let expected_address = match self.peer_pool.read().get(&peer_addr) {
1916            Some(Peer::Candidate(peer)) => peer.last_known_aleo_addr,
1917            _ => None,
1918        };
1919
1920        // Introduce the peer into the peer pool.
1921        self.add_connecting_peer(peer_addr)?;
1922
1923        let mut noise = NoiseSession::new(stream, Role::Initiator)?;
1924        let our_info =
1925            PeerInfo::new(self.local_ip().port(), self.account.address(), restrictions_id, self.snarkos_sha());
1926
1927        /* Message 1: announce ourselves in the clear, so the responder can turn us away cheaply. */
1928
1929        let hint = HandshakeHint {
1930            version: our_info.version,
1931            listener_port: our_info.listener_port,
1932            address: our_info.address,
1933        };
1934        noise.send(&encode_payload(&hint)?).await?;
1935
1936        /* Message 2: receive the responder's metadata, which deliberately carries no signature. */
1937
1938        let peer_info: PeerInfo<N> = decode_payload(peer_addr, &noise.recv().await?)?;
1939
1940        // The handshake hash at this point already commits to both ephemeral keys, the responder's
1941        // static key and every payload exchanged so far, so a signature over it is only valid for
1942        // this session with this responder, and cannot be relayed into another one.
1943        let binding = binding_message(HANDSHAKE_DOMAIN, Role::Initiator, &noise.handshake_hash()?);
1944
1945        // Check the peer over before signing anything for it.
1946        if let Some(reason) = self.verify_peer_info(peer_addr, &peer_info, restrictions_id) {
1947            return Err(reason.into_connect_error(peer_addr));
1948        }
1949
1950        // A validator is expected to keep its Aleo address and change its listening address rather
1951        // than the other way around (the peer pool is pruned on that assumption), so a mismatch here
1952        // is not a peer that has legitimately moved.
1953        if let Some(expected) = expected_address
1954            && peer_info.address != expected
1955        {
1956            warn!("{CONTEXT} Dropping '{peer_addr}': expected validator {expected}, got {}", peer_info.address);
1957            return Err(DisconnectReason::InvalidChallengeResponse.into_connect_error(peer_addr));
1958        }
1959
1960        /* Message 3: disclose ourselves and prove that we own the Aleo address we claim. */
1961
1962        let Ok(our_signature) = self.account.sign_bytes(&binding, &mut rand::rng()) else {
1963            return Err(ConnectError::other(anyhow!("Failed to sign the handshake binding")));
1964        };
1965        let our_message = InitiatorInfo { info: our_info, signature: Data::Object(our_signature) };
1966        noise.send(&encode_payload(&our_message)?).await?;
1967
1968        // Capture the binding for the responder's proof before the hash becomes unavailable; it
1969        // additionally commits to our static key and to the message we have just sent.
1970        let peer_binding = binding_message(HANDSHAKE_DOMAIN, Role::Responder, &noise.handshake_hash()?);
1971
1972        /* Message 4: receive the responder's verdict. */
1973
1974        let mut noise = noise.into_transport_mode()?;
1975        let verdict = decode_payload::<ResponderProof<N>>(peer_addr, &noise.recv().await?)?;
1976
1977        // The pattern is over, so the stream goes back to the connection. The responder considers the
1978        // handshake done the moment it sent this message and is free to start sending events while we
1979        // are still verifying below; those bytes wait on the socket for the event codec.
1980        finish_noise_handshake(noise);
1981
1982        let peer_signature = match verdict {
1983            ResponderProof::Accepted { signature } => signature,
1984            ResponderProof::Rejected { reason } => {
1985                warn!("{CONTEXT} '{peer_addr}' rejected the handshake with reason \"{reason}\"");
1986                return Err(reason.into_connect_error(peer_addr));
1987            }
1988        };
1989
1990        if let Some(reason) =
1991            verify_binding_signature(peer_addr, peer_signature, peer_info.address, &peer_binding).await
1992        {
1993            return Err(reason.into_connect_error(peer_addr));
1994        }
1995
1996        Ok(peer_info)
1997    }
1998
1999    /// The connection responder side of the Noise handshake.
2000    ///
2001    /// Every expensive operation is deferred for as long as the protocol allows: the responder
2002    /// verifies a signature only once the initiator's authenticated metadata has passed all of the
2003    /// cheap checks, and produces one only once that verification has succeeded.
2004    ///
2005    /// The legacy handshake also runs its cheap checks first, so a peer that fails one of those was
2006    /// never expensive under either protocol. What changes is the price of *claiming* an identity
2007    /// that passes them - committee membership is public, so anyone can claim it. Under the legacy
2008    /// handshake that claim alone bought a signature from this node, for the cost of one packet.
2009    /// Here it buys a handful of Diffie-Hellman operations and a signature verification; extracting
2010    /// a signature requires actually holding the committee key, and even reaching the verification
2011    /// requires completing the pattern, which a peer that cannot receive our reply - a spoofed
2012    /// source address - cannot do.
2013    async fn handshake_inner_responder_noise<'a>(
2014        &'a self,
2015        peer_addr: SocketAddr,
2016        peer_ip: &mut Option<SocketAddr>,
2017        restrictions_id: Field<N>,
2018        stream: &'a mut TcpStream,
2019    ) -> Result<PeerInfo<N>, ConnectError> {
2020        /* Message 1: the peer's cleartext hint. Everything it claims is re-checked in message 3. */
2021
2022        // The first message is read without deriving any keys, so that everything below costs the
2023        // responder no more than parsing and lookups until it has decided the peer is worth talking
2024        // to.
2025        let pending = PendingSession::accept(stream).await?;
2026        let hint: HandshakeHint<N> = decode_payload(peer_addr, pending.first_payload()?)?;
2027        let listener_addr = SocketAddr::new(peer_addr.ip(), hint.listener_port);
2028
2029        // Turn the peer away before performing any Diffie-Hellman if we already know we do not want
2030        // it. Nothing here is trustworthy yet - message 3 runs all of it again against the
2031        // authenticated copy - but every one of these is a peer that would be refused either way,
2032        // and the committee check in particular is one an attacker cannot talk its way past.
2033        if let Err(reason) = self.ensure_peer_is_allowed(listener_addr) {
2034            return Err(reason.into_connect_error(peer_addr));
2035        }
2036        if let Some(reason) = self.verify_peer_identity(peer_addr, hint.version, hint.listener_port, hint.address) {
2037            return Err(reason.into_connect_error(peer_addr));
2038        }
2039
2040        // The peer pool is the only thing mutated here, so it goes last: there is no point admitting
2041        // a peer that one of the checks above was about to turn away. Recording the listening address
2042        // only once that has succeeded also stops a peer claiming somebody else's port from getting
2043        // their entry downgraded by failing this handshake.
2044        self.add_connecting_peer(listener_addr)?;
2045        *peer_ip = Some(listener_addr);
2046
2047        /* Message 2: disclose ourselves, but do not sign anything yet. */
2048
2049        // The peer has passed every free check, so it is now worth deriving keys for.
2050        let mut noise = pending.into_session()?;
2051
2052        let our_info =
2053            PeerInfo::new(self.local_ip().port(), self.account.address(), restrictions_id, self.snarkos_sha());
2054        noise.send(&encode_payload(&our_info)?).await?;
2055
2056        // The binding the initiator is expected to have signed.
2057        let peer_binding = binding_message(HANDSHAKE_DOMAIN, Role::Initiator, &noise.handshake_hash()?);
2058
2059        /* Message 3: the peer's authenticated metadata and its proof of identity. */
2060
2061        let InitiatorInfo { info: peer_info, signature: peer_signature } =
2062            decode_payload::<InitiatorInfo<N>>(peer_addr, &noise.recv().await?)?;
2063
2064        // Our own binding additionally commits to the peer's static key and to the message it has
2065        // just sent, so it can only be captured after that message has been processed.
2066        let binding = binding_message(HANDSHAKE_DOMAIN, Role::Responder, &noise.handshake_hash()?);
2067        let mut noise = noise.into_transport_mode()?;
2068
2069        // The cleartext hint is a claim, not a fact: reject the peer if it does not match what it
2070        // has now authenticated, as otherwise the hint would be a way to bypass the checks above.
2071        if (hint.version, hint.listener_port, hint.address)
2072            != (peer_info.version, peer_info.listener_port, peer_info.address)
2073        {
2074            warn!("{CONTEXT} Handshake with '{peer_addr}' failed (the handshake hint was contradicted)");
2075            return self.reject_noise_handshake(peer_addr, noise, DisconnectReason::ProtocolViolation).await;
2076        }
2077
2078        // Everything below is a lookup or a comparison; only once all of it passes is the peer
2079        // worth the cost of a signature verification.
2080        if let Some(reason) = self.verify_peer_info(peer_addr, &peer_info, restrictions_id) {
2081            return self.reject_noise_handshake(peer_addr, noise, reason).await;
2082        }
2083
2084        /* Message 4: having checked the peer over, verify its proof and produce our own. */
2085
2086        if let Some(reason) =
2087            verify_binding_signature(peer_addr, peer_signature, peer_info.address, &peer_binding).await
2088        {
2089            return self.reject_noise_handshake(peer_addr, noise, reason).await;
2090        }
2091
2092        let Ok(our_signature) = self.account.sign_bytes(&binding, &mut rand::rng()) else {
2093            return Err(ConnectError::other(anyhow!("Failed to sign the handshake binding")));
2094        };
2095        noise.send(&encode_payload(&ResponderProof::Accepted { signature: Data::Object(our_signature) })?).await?;
2096
2097        finish_noise_handshake(noise);
2098
2099        Ok(peer_info)
2100    }
2101
2102    /// Tells the initiator why it was turned away, and fails the handshake with that reason.
2103    async fn reject_noise_handshake(
2104        &self,
2105        peer_addr: SocketAddr,
2106        mut noise: NoiseSession<&mut TcpStream>,
2107        reason: DisconnectReason,
2108    ) -> Result<PeerInfo<N>, ConnectError> {
2109        noise.send(&encode_payload(&ResponderProof::<N>::Rejected { reason })?).await?;
2110
2111        Err(reason.into_connect_error(peer_addr))
2112    }
2113
2114    /// The connection initiator side of the legacy handshake.
2115    async fn handshake_inner_initiator<'a>(
2116        &'a self,
2117        peer_addr: SocketAddr,
2118        restrictions_id: Field<N>,
2119        stream: &'a mut TcpStream,
2120    ) -> Result<PeerInfo<N>, ConnectError> {
2121        // Introduce the peer into the peer pool.
2122        self.add_connecting_peer(peer_addr)?;
2123
2124        // Construct the stream.
2125        let mut framed = Framed::new(stream, EventCodec::<N>::handshake());
2126
2127        /* Step 1: Send the challenge request. */
2128
2129        // Sample a random nonce.
2130        let our_nonce: u64 = rand::random();
2131        // Determine the snarkOS SHA to send to the peer.
2132        let snarkos_sha = self.snarkos_sha();
2133        // Send a challenge request to the peer.
2134        let our_request = ChallengeRequest::new(self.local_ip().port(), self.account.address(), our_nonce, snarkos_sha);
2135        send_event(&mut framed, peer_addr, Event::ChallengeRequest(our_request)).await?;
2136
2137        /* Step 2: Receive the peer's challenge response followed by the challenge request. */
2138
2139        // Listen for the challenge response message.
2140        let peer_response = expect_event!(Event::ChallengeResponse, framed, peer_addr);
2141        // Listen for the challenge request message.
2142        let peer_request = expect_event!(Event::ChallengeRequest, framed, peer_addr);
2143
2144        // Verify the challenge response. If a disconnect reason was returned, send the disconnect message and abort.
2145        if let Some(reason) = self
2146            .verify_challenge_response(peer_addr, peer_request.address, peer_response, restrictions_id, our_nonce)
2147            .await
2148        {
2149            send_event(&mut framed, peer_addr, reason.into()).await?;
2150            return Err(ConnectError::application(reason));
2151        }
2152
2153        // Verify the challenge request. If a disconnect reason was returned, send the disconnect message and abort.
2154        if let Some(reason) = self.verify_challenge_request(peer_addr, &peer_request) {
2155            send_event(&mut framed, peer_addr, reason.into()).await?;
2156            return Err(reason.into_connect_error(peer_addr));
2157        }
2158
2159        /* Step 3: Send the challenge response. */
2160
2161        // Sign the counterparty nonce.
2162        let response_nonce: u64 = rand::random();
2163        let data = [peer_request.nonce.to_le_bytes(), response_nonce.to_le_bytes()].concat();
2164        let Ok(our_signature) = self.account.sign_bytes(&data, &mut rand::rng()) else {
2165            return Err(ConnectError::other(anyhow!("Failed to sign the challenge request nonce")));
2166        };
2167        // Send the challenge response.
2168        let our_response =
2169            ChallengeResponse { restrictions_id, signature: Data::Object(our_signature), nonce: response_nonce };
2170        send_event(&mut framed, peer_addr, Event::ChallengeResponse(our_response)).await?;
2171
2172        note_legacy_handshake_end(framed, peer_addr);
2173
2174        Ok(peer_info_from_challenge_request(peer_request, restrictions_id))
2175    }
2176
2177    /// The connection responder side of the legacy handshake.
2178    ///
2179    /// `prefix` holds the bytes that were consumed from the stream while determining which
2180    /// handshake protocol the peer speaks; they are the beginning of its first frame.
2181    async fn handshake_inner_responder<'a>(
2182        &'a self,
2183        peer_addr: SocketAddr,
2184        peer_ip: &mut Option<SocketAddr>,
2185        restrictions_id: Field<N>,
2186        stream: &'a mut TcpStream,
2187        prefix: &[u8],
2188    ) -> Result<PeerInfo<N>, ConnectError> {
2189        // Construct the stream.
2190        let mut framed = prepare_framed(stream, EventCodec::<N>::handshake(), prefix);
2191
2192        /* Step 1: Receive the challenge request. */
2193
2194        // Listen for the challenge request message.
2195        let peer_request = expect_event!(Event::ChallengeRequest, framed, peer_addr);
2196
2197        // Ensure the address is not the same as this node.
2198        if self.account.address() == peer_request.address {
2199            return Err(ConnectError::SelfConnect { address: peer_addr });
2200        }
2201
2202        // Obtain the peer's listening address.
2203        *peer_ip = Some(SocketAddr::new(peer_addr.ip(), peer_request.listener_port));
2204        let peer_ip = peer_ip.unwrap();
2205
2206        // Knowing the peer's listening address, ensure it is allowed to connect.
2207        if let Err(reason) = self.ensure_peer_is_allowed(peer_ip) {
2208            send_event(&mut framed, peer_addr, reason.into()).await?;
2209            return Err(reason.into_connect_error(peer_addr));
2210        }
2211
2212        // Introduce the peer into the peer pool.
2213        self.add_connecting_peer(peer_ip)?;
2214
2215        // Verify the challenge request. If a disconnect reason was returned, send the disconnect message and abort.
2216        if let Some(reason) = self.verify_challenge_request(peer_addr, &peer_request) {
2217            send_event(&mut framed, peer_addr, reason.into()).await?;
2218            return Err(reason.into_connect_error(peer_addr));
2219        }
2220
2221        /* Step 2: Send the challenge response followed by own challenge request. */
2222
2223        // Sign the counterparty nonce.
2224        let response_nonce: u64 = rand::random();
2225        let data = [peer_request.nonce.to_le_bytes(), response_nonce.to_le_bytes()].concat();
2226        let Ok(our_signature) = self.account.sign_bytes(&data, &mut rand::rng()) else {
2227            return Err(ConnectError::other(anyhow!("Failed to sign the challenge request nonce")));
2228        };
2229        // Send the challenge response.
2230        let our_response =
2231            ChallengeResponse { restrictions_id, signature: Data::Object(our_signature), nonce: response_nonce };
2232        send_event(&mut framed, peer_addr, Event::ChallengeResponse(our_response)).await?;
2233
2234        // Sample a random nonce.
2235        let our_nonce: u64 = rand::random();
2236        // Determine the snarkOS SHA to send to the peer.
2237        let snarkos_sha = self.snarkos_sha();
2238        // Send the challenge request.
2239        let our_request = ChallengeRequest::new(self.local_ip().port(), self.account.address(), our_nonce, snarkos_sha);
2240        send_event(&mut framed, peer_addr, Event::ChallengeRequest(our_request)).await?;
2241
2242        /* Step 3: Receive the challenge response. */
2243
2244        // Listen for the challenge response message.
2245        let peer_response = expect_event!(Event::ChallengeResponse, framed, peer_addr);
2246        // Verify the challenge response. If a disconnect reason was returned, send the disconnect message and abort.
2247        if let Some(reason) = self
2248            .verify_challenge_response(peer_addr, peer_request.address, peer_response, restrictions_id, our_nonce)
2249            .await
2250        {
2251            send_event(&mut framed, peer_addr, reason.into()).await?;
2252            Err(reason.into_connect_error(peer_addr))
2253        } else {
2254            note_legacy_handshake_end(framed, peer_addr);
2255
2256            Ok(peer_info_from_challenge_request(peer_request, restrictions_id))
2257        }
2258    }
2259
2260    /// Verifies the given challenge request. Returns a disconnect reason if the request is invalid.
2261    #[must_use]
2262    fn verify_challenge_request(&self, peer_addr: SocketAddr, event: &ChallengeRequest<N>) -> Option<DisconnectReason> {
2263        // Retrieve the components of the challenge request.
2264        let &ChallengeRequest { version, listener_port, address, nonce: _, ref snarkos_sha } = event;
2265        log_repo_sha_comparison(peer_addr, snarkos_sha, CONTEXT);
2266
2267        self.verify_peer_claims(peer_addr, version, listener_port, address)
2268    }
2269
2270    /// Verifies the metadata a peer disclosed during the Noise handshake. Returns a disconnect
2271    /// reason if the peer is not acceptable.
2272    ///
2273    /// Every check reachable from here is a lookup or a comparison, which is precisely what makes
2274    /// it safe to run before verifying the peer's signature.
2275    #[must_use]
2276    fn verify_peer_info(
2277        &self,
2278        peer_addr: SocketAddr,
2279        info: &PeerInfo<N>,
2280        expected_restrictions_id: Field<N>,
2281    ) -> Option<DisconnectReason> {
2282        log_repo_sha_comparison(peer_addr, &info.snarkos_sha, CONTEXT);
2283
2284        // Verify the restrictions ID. This is the only check here that the cleartext hint cannot
2285        // carry, as the peer would simply state the expected value; it is left to this point.
2286        if info.restrictions_id != expected_restrictions_id {
2287            warn!("{CONTEXT} Handshake with '{peer_addr}' failed (incorrect restrictions ID)");
2288            return Some(DisconnectReason::InvalidChallengeResponse);
2289        }
2290
2291        self.verify_peer_identity(peer_addr, info.version, info.listener_port, info.address)
2292    }
2293
2294    /// Everything a peer's claimed identity can be held to without any cryptography.
2295    ///
2296    /// These are all lookups and comparisons, which is what makes them safe to run against the
2297    /// unauthenticated hint in the Noise handshake's first message, and worth running there: a peer
2298    /// that fails one of them would fail it again against the authenticated copy, so there is no
2299    /// reason to spend a Diffie-Hellman on it first.
2300    #[must_use]
2301    fn verify_peer_identity(
2302        &self,
2303        peer_addr: SocketAddr,
2304        version: u32,
2305        listener_port: u16,
2306        address: Address<N>,
2307    ) -> Option<DisconnectReason> {
2308        // Ensure the address is not the same as this node's.
2309        if self.account.address() == address {
2310            return Some(DisconnectReason::SelfConnect);
2311        }
2312
2313        self.verify_peer_claims(peer_addr, version, listener_port, address)
2314    }
2315
2316    /// The peer checks shared by both handshakes.
2317    #[must_use]
2318    fn verify_peer_claims(
2319        &self,
2320        peer_addr: SocketAddr,
2321        version: u32,
2322        listener_port: u16,
2323        address: Address<N>,
2324    ) -> Option<DisconnectReason> {
2325        let listener_addr = SocketAddr::new(peer_addr.ip(), listener_port);
2326
2327        // Ensure the event protocol version is not outdated.
2328        if version < Event::<N>::VERSION {
2329            return Some(DisconnectReason::OutdatedClientVersion);
2330        }
2331        // If the node is in trusted peers only mode, ensure the peer is trusted.
2332        if self.trusted_peers_only && !self.is_trusted(listener_addr) {
2333            warn!("{CONTEXT} Dropping '{peer_addr}' for being an untrusted validator ({address})");
2334            return Some(DisconnectReason::NoExternalPeersAllowed);
2335        }
2336        if !bootstrap_peers::<N>(self.dev().is_some()).contains(&listener_addr) {
2337            // Ensure the address is a current committee member.
2338            if !self.is_authorized_validator_address(address) {
2339                return Some(DisconnectReason::UnauthorizedValidator);
2340            }
2341        }
2342
2343        // Ensure the address is not already connected.
2344        if self.is_connected_address(address) {
2345            return Some(DisconnectReason::AlreadyConnectedToAleoAddress);
2346        }
2347
2348        None
2349    }
2350
2351    /// Verifies the given challenge response. Returns a disconnect reason if the response is invalid.
2352    #[must_use]
2353    async fn verify_challenge_response(
2354        &self,
2355        peer_addr: SocketAddr,
2356        peer_address: Address<N>,
2357        response: ChallengeResponse<N>,
2358        expected_restrictions_id: Field<N>,
2359        expected_nonce: u64,
2360    ) -> Option<DisconnectReason> {
2361        // Retrieve the components of the challenge response.
2362        let ChallengeResponse { restrictions_id, signature, nonce } = response;
2363
2364        // Verify the restrictions ID.
2365        if restrictions_id != expected_restrictions_id {
2366            warn!("{CONTEXT} Handshake with '{peer_addr}' failed (incorrect restrictions ID)");
2367            return Some(DisconnectReason::InvalidChallengeResponse);
2368        }
2369        // Perform the deferred non-blocking deserialization of the signature.
2370        let Ok(signature) = spawn_blocking!(signature.deserialize_blocking()) else {
2371            warn!("{CONTEXT} Handshake with '{peer_addr}' failed (cannot deserialize the signature)");
2372            return Some(DisconnectReason::InvalidChallengeResponse);
2373        };
2374        // Verify the signature.
2375        if !signature.verify_bytes(&peer_address, &[expected_nonce.to_le_bytes(), nonce.to_le_bytes()].concat()) {
2376            warn!("{CONTEXT} Handshake with '{peer_addr}' failed (invalid signature)");
2377            return Some(DisconnectReason::InvalidChallengeResponse);
2378        }
2379        None
2380    }
2381}
2382
2383#[cfg(test)]
2384mod prop_tests {
2385    use crate::{
2386        Gateway,
2387        MAX_WORKERS,
2388        MEMORY_POOL_PORT,
2389        Worker,
2390        helpers::{Storage, init_primary_channels, init_worker_channels},
2391    };
2392
2393    use snarkos_account::Account;
2394    use snarkos_node_bft_ledger_service::MockLedgerService;
2395    use snarkos_node_bft_storage_service::BFTMemoryService;
2396    use snarkos_node_network::PeerPoolHandling;
2397    use snarkos_node_tcp::P2P;
2398    use snarkos_utilities::NodeDataDir;
2399
2400    use snarkos_node_bft_events::committee_prop_tests::{CommitteeContext, ValidatorSet};
2401    use snarkvm::{
2402        ledger::{
2403            committee::{Committee, test_helpers::sample_committee_for_round_and_members},
2404            narwhal::{BatchHeader, batch_certificate::test_helpers::sample_batch_certificate_for_round},
2405        },
2406        prelude::{MainnetV0, PrivateKey},
2407        utilities::TestRng,
2408    };
2409
2410    use indexmap::{IndexMap, IndexSet};
2411    use proptest::{
2412        prelude::{Arbitrary, BoxedStrategy, Just, Strategy, any, any_with},
2413        sample::Selector,
2414    };
2415    use std::{
2416        fmt::{Debug, Formatter},
2417        net::{IpAddr, Ipv4Addr, SocketAddr},
2418        sync::Arc,
2419    };
2420    use test_strategy::proptest;
2421
2422    type CurrentNetwork = MainnetV0;
2423
2424    /// The per-connection queues are sized from the fetch-timeout window, not the GC window.
2425    ///
2426    /// Each slot can hold a full `Transmission`, so the depth is a direct multiplier on the heap a
2427    /// single peer can pin by not reading its socket. Pin the derivation so a regression is loud.
2428    #[test]
2429    fn test_per_connection_queue_depth() {
2430        use crate::gateway::{QUEUE_WINDOW_ROUNDS, per_connection_queue_depth};
2431        use snarkvm::console::network::Network;
2432
2433        // The window is `MAX_FETCH_TIMEOUT` expressed in rounds.
2434        assert_eq!(QUEUE_WINDOW_ROUNDS, 3);
2435
2436        let certificates = CurrentNetwork::LATEST_MAX_CERTIFICATES() as usize;
2437        let transmissions = BatchHeader::<CurrentNetwork>::MAX_TRANSMISSIONS_PER_BATCH;
2438        let depth = per_connection_queue_depth::<CurrentNetwork>();
2439
2440        assert_eq!(depth, 2 * QUEUE_WINDOW_ROUNDS * certificates * (transmissions + 1));
2441
2442        // It must cover a peer fetching every transmission of every certificate for the window...
2443        assert!(depth >= QUEUE_WINDOW_ROUNDS * certificates * transmissions);
2444        // ...but stay far below the old `MAX_GC_ROUNDS`-derived depth, which let one peer pin
2445        // 400,000 transmissions.
2446        assert!(depth < 2 * BatchHeader::<CurrentNetwork>::MAX_GC_ROUNDS * certificates * transmissions / 8);
2447    }
2448
2449    impl Debug for Gateway<CurrentNetwork> {
2450        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
2451            // TODO implement Debug properly and move it over to production code
2452            f.debug_tuple("Gateway").field(&self.account.address()).field(&self.tcp.config()).finish()
2453        }
2454    }
2455
2456    #[derive(Debug, test_strategy::Arbitrary)]
2457    enum GatewayAddress {
2458        Dev(u8),
2459        Prod(Option<SocketAddr>),
2460    }
2461
2462    impl GatewayAddress {
2463        fn ip(&self) -> Option<SocketAddr> {
2464            if let GatewayAddress::Prod(ip) = self {
2465                return *ip;
2466            }
2467            None
2468        }
2469
2470        fn port(&self) -> Option<u16> {
2471            if let GatewayAddress::Dev(port) = self {
2472                return Some(*port as u16);
2473            }
2474            None
2475        }
2476    }
2477
2478    impl Arbitrary for Gateway<CurrentNetwork> {
2479        type Parameters = ();
2480        type Strategy = BoxedStrategy<Gateway<CurrentNetwork>>;
2481
2482        fn arbitrary_with(_: Self::Parameters) -> Self::Strategy {
2483            any_valid_dev_gateway()
2484                .prop_map(|(storage, _, private_key, address)| {
2485                    Gateway::new(
2486                        Account::try_from(private_key).unwrap(),
2487                        storage.clone(),
2488                        storage.ledger().clone(),
2489                        address.ip(),
2490                        &[],
2491                        false,
2492                        NodeDataDir::new_test(None),
2493                        address.port(),
2494                    )
2495                    .unwrap()
2496                })
2497                .boxed()
2498        }
2499    }
2500
2501    type GatewayInput = (Storage<CurrentNetwork>, CommitteeContext, PrivateKey<CurrentNetwork>, GatewayAddress);
2502
2503    fn any_valid_dev_gateway() -> BoxedStrategy<GatewayInput> {
2504        (any::<CommitteeContext>(), any::<Selector>())
2505            .prop_flat_map(|(context, account_selector)| {
2506                let CommitteeContext(_, ValidatorSet(validators)) = context.clone();
2507                (
2508                    any_with::<Storage<CurrentNetwork>>(context.clone()),
2509                    Just(context),
2510                    Just(account_selector.select(validators)),
2511                    0u8..,
2512                )
2513                    .prop_map(|(a, b, c, d)| (a, b, c.private_key, GatewayAddress::Dev(d)))
2514            })
2515            .boxed()
2516    }
2517
2518    fn any_valid_prod_gateway() -> BoxedStrategy<GatewayInput> {
2519        (any::<CommitteeContext>(), any::<Selector>())
2520            .prop_flat_map(|(context, account_selector)| {
2521                let CommitteeContext(_, ValidatorSet(validators)) = context.clone();
2522                (
2523                    any_with::<Storage<CurrentNetwork>>(context.clone()),
2524                    Just(context),
2525                    Just(account_selector.select(validators)),
2526                    any::<Option<SocketAddr>>(),
2527                )
2528                    .prop_map(|(a, b, c, d)| (a, b, c.private_key, GatewayAddress::Prod(d)))
2529            })
2530            .boxed()
2531    }
2532
2533    #[proptest]
2534    fn gateway_dev_initialization(#[strategy(any_valid_dev_gateway())] input: GatewayInput) {
2535        let (storage, _, private_key, dev) = input;
2536        let account = Account::try_from(private_key).unwrap();
2537
2538        let gateway = Gateway::new(
2539            account.clone(),
2540            storage.clone(),
2541            storage.ledger().clone(),
2542            dev.ip(),
2543            &[],
2544            false,
2545            NodeDataDir::new_test(None),
2546            dev.port(),
2547        )
2548        .unwrap();
2549        let tcp_config = gateway.tcp().config();
2550        assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::LOCALHOST)));
2551        assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT + dev.port().unwrap()));
2552
2553        let tcp_config = gateway.tcp().config();
2554        assert_eq!(tcp_config.max_connections, Committee::<CurrentNetwork>::max_committee_size() * 10);
2555        assert_eq!(gateway.account().address(), account.address());
2556    }
2557
2558    #[proptest]
2559    fn gateway_prod_initialization(#[strategy(any_valid_prod_gateway())] input: GatewayInput) {
2560        let (storage, _, private_key, dev) = input;
2561        let account = Account::try_from(private_key).unwrap();
2562
2563        let gateway = Gateway::new(
2564            account.clone(),
2565            storage.clone(),
2566            storage.ledger().clone(),
2567            dev.ip(),
2568            &[],
2569            false,
2570            NodeDataDir::new_test(None),
2571            dev.port(),
2572        )
2573        .unwrap();
2574        let tcp_config = gateway.tcp().config();
2575        if let Some(socket_addr) = dev.ip() {
2576            assert_eq!(tcp_config.listener_ip, Some(socket_addr.ip()));
2577            assert_eq!(tcp_config.desired_listening_port, Some(socket_addr.port()));
2578        } else {
2579            assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED)));
2580            assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT));
2581        }
2582
2583        let tcp_config = gateway.tcp().config();
2584        assert_eq!(tcp_config.max_connections, Committee::<CurrentNetwork>::max_committee_size() * 10);
2585        assert_eq!(gateway.account().address(), account.address());
2586    }
2587
2588    #[proptest(async = "tokio")]
2589    async fn gateway_start(
2590        #[strategy(any_valid_dev_gateway())] input: GatewayInput,
2591        #[strategy(0..MAX_WORKERS)] workers_count: u8,
2592    ) {
2593        let (storage, committee, private_key, dev) = input;
2594        let committee = committee.0;
2595        let worker_storage = storage.clone();
2596        let account = Account::try_from(private_key).unwrap();
2597
2598        let gateway = Gateway::new(
2599            account,
2600            storage.clone(),
2601            storage.ledger().clone(),
2602            dev.ip(),
2603            &[],
2604            false,
2605            NodeDataDir::new_test(None),
2606            dev.port(),
2607        )
2608        .unwrap();
2609
2610        let (primary_sender, _) = init_primary_channels();
2611
2612        let (workers, worker_senders) = {
2613            // Construct a map of the worker senders.
2614            let mut tx_workers = IndexMap::new();
2615            let mut workers = IndexMap::new();
2616
2617            // Initialize the workers.
2618            for id in 0..workers_count {
2619                // Construct the worker channels.
2620                let (tx_worker, rx_worker) = init_worker_channels();
2621                // Construct the worker instance.
2622                let ledger = Arc::new(MockLedgerService::new(committee.clone()));
2623                let worker =
2624                    Worker::new(id, Arc::new(gateway.clone()), worker_storage.clone(), ledger, Default::default())
2625                        .unwrap();
2626                // Run the worker instance.
2627                worker.run(rx_worker);
2628
2629                // Add the worker and the worker sender to maps
2630                workers.insert(id, worker);
2631                tx_workers.insert(id, tx_worker);
2632            }
2633            (workers, tx_workers)
2634        };
2635
2636        gateway.run(primary_sender, worker_senders, None).await;
2637        assert_eq!(
2638            gateway.local_ip(),
2639            SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), MEMORY_POOL_PORT + dev.port().unwrap())
2640        );
2641        assert_eq!(gateway.num_workers(), workers.len() as u8);
2642    }
2643
2644    #[proptest]
2645    fn test_is_authorized_validator(#[strategy(any_valid_dev_gateway())] input: GatewayInput) {
2646        let rng = &mut TestRng::default();
2647
2648        // Initialize the round parameters.
2649        let current_round = 2;
2650        let committee_size = 4;
2651        let max_gc_rounds = BatchHeader::<CurrentNetwork>::MAX_GC_ROUNDS as u64;
2652        let (_, _, private_key, dev) = input;
2653        let account = Account::try_from(private_key).unwrap();
2654
2655        // Sample the certificates.
2656        let mut certificates = IndexSet::new();
2657        for _ in 0..committee_size {
2658            certificates.insert(sample_batch_certificate_for_round(current_round, rng));
2659        }
2660        let addresses: Vec<_> = certificates.iter().map(|certificate| certificate.author()).collect();
2661        // Initialize the committee.
2662        let committee = sample_committee_for_round_and_members(current_round, addresses, rng);
2663        // Sample extra certificates from non-committee members.
2664        for _ in 0..committee_size {
2665            certificates.insert(sample_batch_certificate_for_round(current_round, rng));
2666        }
2667        // Initialize the ledger.
2668        let ledger = Arc::new(MockLedgerService::new(committee.clone()));
2669        // Initialize the storage.
2670        let storage = Storage::new(ledger.clone(), Arc::new(BFTMemoryService::new()), max_gc_rounds).unwrap();
2671        // Initialize the gateway.
2672        let gateway = Gateway::new(
2673            account.clone(),
2674            storage.clone(),
2675            ledger.clone(),
2676            dev.ip(),
2677            &[],
2678            false,
2679            NodeDataDir::new_test(None),
2680            dev.port(),
2681        )
2682        .unwrap();
2683        // Insert certificate to the storage.
2684        for certificate in certificates.iter() {
2685            storage.testing_only_insert_certificate_testing_only(certificate.clone());
2686        }
2687        // Check that the current committee members are authorized validators.
2688        for i in 0..certificates.clone().len() {
2689            let is_authorized = gateway.is_authorized_validator_address(certificates[i].author());
2690            if i < committee_size {
2691                assert!(is_authorized);
2692            } else {
2693                assert!(!is_authorized);
2694            }
2695        }
2696    }
2697}