1#[cfg(feature = "metrics")]
17use crate::helpers::{Telemetry, TelemetryWorker};
18use crate::{
19 CONTEXT,
20 MAX_BATCH_DELAY,
21 MAX_FETCH_TIMEOUT,
22 MEMORY_POOL_PORT,
23 Worker,
24 events::{DisconnectReason, EventCodec, PrimaryPing},
25 helpers::{Cache, PrimarySender, Storage, SyncSender, WorkerSender, assign_to_worker},
26 spawn_blocking,
27};
28use smol_str::SmolStr;
29use snarkos_account::Account;
30use snarkos_node_bft_events::{
31 BlockRequest,
32 BlockResponse,
33 CertificateRequest,
34 CertificateResponse,
35 ChallengeRequest,
36 ChallengeResponse,
37 DataBlocks,
38 Event,
39 EventTrait,
40 HANDSHAKE_DOMAIN,
41 HandshakeHint,
42 InitiatorInfo,
43 PeerInfo,
44 ResponderProof,
45 TransmissionRequest,
46 TransmissionResponse,
47 ValidatorsRequest,
48 ValidatorsResponse,
49};
50use snarkos_node_bft_ledger_service::LedgerService;
51use snarkos_node_network::{
52 ConnectionMode,
53 NodeType,
54 Peer,
55 PeerPoolHandling,
56 Resolver,
57 bootstrap_peers,
58 get_repo_commit_hash,
59 harden_socket,
60 log_repo_sha_comparison,
61 noise::{
62 HandshakeProtocol,
63 NoiseSession,
64 PendingSession,
65 Role,
66 binding_message,
67 detect_handshake_protocol,
68 prepare_framed,
69 write_noise_magic,
70 },
71 shorten_snarkos_sha,
72};
73use snarkos_node_sync::{MAX_BLOCKS_BEHIND, communication_service::CommunicationService};
74use snarkos_node_tcp::{
75 Config,
76 ConnectError,
77 Connection,
78 ConnectionSide,
79 P2P,
80 Tcp,
81 connections::DisconnectOrigin,
82 protocols::{Disconnect, Handshake, OnConnect, Reading, Writing},
83};
84use snarkos_utilities::NodeDataDir;
85use snarkvm::{
86 console::prelude::*,
87 ledger::{
88 committee::Committee,
89 narwhal::{BatchHeader, Data},
90 },
91 prelude::{Address, Field, Signature},
92 utilities::flatten_error,
93};
94
95use colored::Colorize;
96use futures::{SinkExt, future::join_all};
97use indexmap::IndexMap;
98#[cfg(feature = "locktick")]
99use locktick::parking_lot::{Mutex, RwLock};
100#[cfg(not(feature = "locktick"))]
101use parking_lot::{Mutex, RwLock};
102use rand::seq::{IteratorRandom, SliceRandom};
103use std::{
104 collections::{HashMap, HashSet},
105 future::Future,
106 io,
107 net::{Ipv4Addr, SocketAddr, SocketAddrV4},
108 sync::Arc,
109 time::Duration,
110};
111use tokio::{
112 net::TcpStream,
113 sync::{OnceCell, oneshot},
114 task::{self, JoinHandle},
115};
116use tokio_stream::StreamExt;
117use tokio_util::codec::Framed;
118
119const CACHE_EVENTS_INTERVAL: i64 = (MAX_BATCH_DELAY.as_secs()) as i64; const CACHE_REQUESTS_INTERVAL: i64 = (MAX_BATCH_DELAY.as_secs()) as i64; const QUEUE_WINDOW_ROUNDS: usize = (MAX_FETCH_TIMEOUT.as_millis() / MAX_BATCH_DELAY.as_millis()) as usize;
133
134fn per_connection_queue_depth<N: Network>() -> usize {
148 2 * QUEUE_WINDOW_ROUNDS
149 * N::LATEST_MAX_CERTIFICATES() as usize
150 * (BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH + 1)
151}
152
153#[cfg(not(test))]
155const MAX_CONNECTION_ATTEMPTS: usize = 10;
156
157pub const MAX_VALIDATORS_TO_SEND: usize = 200;
159
160#[cfg(not(test))]
162const CONNECTION_ATTEMPTS_SINCE_SECS: i64 = 10;
163
164const IP_BAN_TIME_IN_SECS: u64 = 300;
166
167const NOISE_HANDSHAKE_ACTIVATION: Option<ConsensusVersion> = Some(ConsensusVersion::V20);
178
179const LEGACY_HANDSHAKE_EXPIRY: Option<ConsensusVersion> = Some(ConsensusVersion::V21);
194
195#[async_trait]
198pub trait Transport<N: Network>: Send + Sync {
199 async fn send(&self, peer_ip: SocketAddr, event: Event<N>) -> Option<oneshot::Receiver<io::Result<()>>>;
200 fn broadcast(&self, event: Event<N>);
201}
202
203#[derive(Clone)]
206pub struct Gateway<N: Network>(Arc<InnerGateway<N>>);
207
208impl<N: Network> Deref for Gateway<N> {
209 type Target = Arc<InnerGateway<N>>;
210
211 fn deref(&self) -> &Self::Target {
212 &self.0
213 }
214}
215
216pub struct InnerGateway<N: Network> {
217 account: Account<N>,
219 storage: Storage<N>,
221 ledger: Arc<dyn LedgerService<N>>,
223 tcp: Tcp,
225 cache: Cache<N>,
227 resolver: RwLock<Resolver<N>>,
229 peer_pool: RwLock<HashMap<SocketAddr, Peer<N>>>,
231 #[cfg(feature = "metrics")]
233 validator_telemetry: Telemetry<N>,
234 #[cfg(feature = "metrics")]
239 telemetry_worker: Mutex<Option<TelemetryWorker<N>>>,
240 primary_sender: OnceCell<PrimarySender<N>>,
242 worker_senders: OnceCell<IndexMap<u8, WorkerSender<N>>>,
244 sync_sender: OnceCell<SyncSender<N>>,
246 handles: Mutex<Vec<JoinHandle<()>>>,
248 node_data_dir: NodeDataDir,
250 trusted_peers_only: bool,
252 dev: Option<u16>,
254 #[cfg(any(test, feature = "test"))]
261 initiates_noise_handshake: std::sync::atomic::AtomicBool,
262}
263
264impl<N: Network> PeerPoolHandling<N> for Gateway<N> {
265 const MAXIMUM_POOL_SIZE: usize = 200;
266 const OWNER: &str = CONTEXT;
267 const PEER_SLASHING_COUNT: usize = 20;
268
269 fn peer_pool(&self) -> &RwLock<HashMap<SocketAddr, Peer<N>>> {
270 &self.peer_pool
271 }
272
273 fn resolver(&self) -> &RwLock<Resolver<N>> {
274 &self.resolver
275 }
276
277 fn is_dev(&self) -> bool {
278 self.dev.is_some()
279 }
280
281 fn trusted_peers_only(&self) -> bool {
282 self.trusted_peers_only
283 }
284
285 fn node_type(&self) -> NodeType {
286 NodeType::Validator
287 }
288}
289
290impl<N: Network> Gateway<N> {
291 #[allow(clippy::too_many_arguments)]
293 pub fn new(
294 account: Account<N>,
295 storage: Storage<N>,
296 ledger: Arc<dyn LedgerService<N>>,
297 ip: Option<SocketAddr>,
298 trusted_validators: &[SocketAddr],
299 trusted_peers_only: bool,
300 node_data_dir: NodeDataDir,
301 dev: Option<u16>,
302 ) -> Result<Self> {
303 let ip = match (ip, dev) {
305 (None, Some(dev)) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, MEMORY_POOL_PORT + dev)),
306 (None, None) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, MEMORY_POOL_PORT)),
307 (Some(ip), _) => ip,
308 };
309 let tcp = Tcp::new(Config::new(ip, Committee::<N>::max_committee_size() * 10));
317
318 let mut initial_peers = HashMap::new();
320
321 if !trusted_peers_only {
323 let cached_peers = Self::load_cached_peers(&node_data_dir.gateway_peer_cache_path())?;
324 for addr in cached_peers {
325 initial_peers.insert(addr, Peer::new_candidate(addr, false));
326 }
327 }
328
329 initial_peers.extend(trusted_validators.iter().copied().map(|addr| (addr, Peer::new_candidate(addr, true))));
332
333 #[cfg(feature = "metrics")]
335 let (validator_telemetry, telemetry_worker) = Telemetry::new();
336
337 Ok(Self(Arc::new(InnerGateway {
339 account,
340 storage,
341 ledger,
342 tcp,
343 cache: Default::default(),
344 resolver: Default::default(),
345 peer_pool: RwLock::new(initial_peers),
346 #[cfg(feature = "metrics")]
347 validator_telemetry,
348 #[cfg(feature = "metrics")]
349 telemetry_worker: Mutex::new(Some(telemetry_worker)),
350 primary_sender: Default::default(),
351 worker_senders: Default::default(),
352 sync_sender: Default::default(),
353 handles: Default::default(),
354 node_data_dir,
355 trusted_peers_only,
356 dev,
357 #[cfg(any(test, feature = "test"))]
359 initiates_noise_handshake: std::sync::atomic::AtomicBool::new(true),
360 })))
361 }
362
363 pub async fn run(
365 &self,
366 primary_sender: PrimarySender<N>,
367 worker_senders: IndexMap<u8, WorkerSender<N>>,
368 sync_sender: Option<SyncSender<N>>,
369 ) {
370 debug!("Starting the gateway for the memory pool...");
371
372 self.primary_sender.set(primary_sender).expect("Primary sender already set in gateway");
374
375 self.worker_senders.set(worker_senders).expect("The worker senders are already set");
377
378 if let Some(sync_sender) = sync_sender {
380 self.sync_sender.set(sync_sender).expect("Sync sender already set in gateway");
381 }
382
383 #[cfg(feature = "metrics")]
386 if let Some(telemetry_worker) = self.telemetry_worker.lock().take() {
387 self.spawn(telemetry_worker.run());
388 }
389
390 self.enable_handshake().await;
392 self.enable_reading().await;
393 self.enable_writing().await;
394 self.enable_disconnect().await;
395 self.enable_on_connect().await;
396
397 #[cfg(feature = "metrics")]
399 {
400 let gateway = self.clone();
401 self.spawn(async move {
402 loop {
403 tokio::time::sleep(Duration::from_secs(1)).await;
404 gateway.update_metrics();
405 }
406 });
407 }
408
409 let listen_addr = self.tcp.enable_listener().await.expect("Failed to enable the TCP listener");
411 debug!("Listening for validator connections at address {listen_addr:?}");
412
413 self.initialize_heartbeat();
415
416 info!("Started the gateway for the memory pool at '{}'", self.local_ip());
417 }
418}
419
420impl<N: Network> Gateway<N> {
422 fn max_committee_size(&self) -> usize {
424 self.ledger
425 .current_committee()
426 .map_or_else(|_e| Committee::<N>::max_committee_size() as usize, |committee| committee.num_members())
427 }
428
429 fn max_cache_events(&self) -> usize {
431 self.max_cache_transmissions()
432 }
433
434 fn max_cache_certificates(&self) -> usize {
436 2 * BatchHeader::<N>::MAX_GC_ROUNDS * self.max_committee_size()
437 }
438
439 fn max_cache_transmissions(&self) -> usize {
441 self.max_cache_certificates() * BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH
442 }
443
444 fn max_cache_duplicates(&self) -> usize {
446 self.max_committee_size().pow(2)
447 }
448}
449
450#[async_trait]
451impl<N: Network> CommunicationService for Gateway<N> {
452 type Message = Event<N>;
454
455 fn prepare_block_request(start_height: u32, end_height: u32) -> Self::Message {
457 debug_assert!(start_height < end_height, "Invalid block request format");
458 Event::BlockRequest(BlockRequest { start_height, end_height })
459 }
460
461 async fn send(&self, peer_ip: SocketAddr, message: Self::Message) -> Option<oneshot::Receiver<io::Result<()>>> {
467 Transport::send(self, peer_ip, message).await
468 }
469}
470
471impl<N: Network> Gateway<N> {
472 pub fn account(&self) -> &Account<N> {
474 &self.account
475 }
476
477 pub fn dev(&self) -> Option<u16> {
479 self.dev
480 }
481
482 pub fn ledger(&self) -> &Arc<dyn LedgerService<N>> {
484 &self.ledger
485 }
486
487 pub fn resolver(&self) -> &RwLock<Resolver<N>> {
489 &self.resolver
490 }
491
492 pub fn resolve_to_listener(&self, connected_addr: &SocketAddr) -> Option<SocketAddr> {
494 self.resolver.read().get_listener(*connected_addr)
495 }
496
497 #[cfg(feature = "metrics")]
499 pub fn validator_telemetry(&self) -> &Telemetry<N> {
500 &self.validator_telemetry
501 }
502
503 pub fn primary_sender(&self) -> &PrimarySender<N> {
505 self.primary_sender.get().expect("Primary sender not set in gateway")
506 }
507
508 pub fn num_workers(&self) -> u8 {
510 u8::try_from(self.worker_senders.get().expect("Missing worker senders in gateway").len())
511 .expect("Too many workers")
512 }
513
514 pub fn get_worker_sender(&self, worker_id: u8) -> Option<&WorkerSender<N>> {
516 self.worker_senders.get().and_then(|senders| senders.get(&worker_id))
517 }
518
519 pub fn is_authorized_validator_ip(&self, ip: SocketAddr) -> bool {
521 if self.trusted_peers().contains(&ip) {
523 return true;
524 }
525 match self.resolve_to_aleo_addr(ip) {
527 Some(address) => self.is_authorized_validator_address(address),
529 None => {
530 warn!("{CONTEXT} Could not resolve the Aleo address for '{ip}'");
531 false
532 }
533 }
534 }
535
536 pub fn is_authorized_validator_address(&self, validator_address: Address<N>) -> bool {
538 if self
547 .ledger
548 .get_committee_lookback_for_round(self.storage.current_round())
549 .is_ok_and(|committee| committee.is_committee_member(validator_address))
550 {
551 return true;
552 }
553
554 if self.ledger.current_committee().is_ok_and(|committee| committee.is_committee_member(validator_address)) {
556 return true;
557 }
558
559 let previous_block_height = self.ledger.latest_block_height().saturating_sub(MAX_BLOCKS_BEHIND);
561 match self.ledger.get_block_round(previous_block_height) {
563 Ok(block_round) => (block_round..self.storage.current_round()).step_by(2).any(|round| {
564 self.ledger
565 .get_committee_lookback_for_round(round)
566 .is_ok_and(|committee| committee.is_committee_member(validator_address))
567 }),
568 Err(_) => false,
569 }
570 }
571
572 pub fn connected_addresses(&self) -> HashSet<Address<N>> {
574 self.get_connected_peers().into_iter().map(|peer| peer.aleo_addr).collect()
575 }
576
577 fn ensure_peer_is_allowed(&self, listener_addr: SocketAddr) -> Result<(), DisconnectReason> {
579 if self.is_local_ip(listener_addr) {
581 return Err(DisconnectReason::SelfConnect);
582 }
583
584 Ok(())
585 }
586
587 #[cfg(feature = "metrics")]
589 fn update_metrics(&self) {
590 if let Some(count) = self.number_of_connected_validators() {
591 metrics::gauge(metrics::bft::CONNECTED, count as f64);
592 }
593 if let Some(count) = self.number_of_connecting_peers() {
594 metrics::gauge(metrics::bft::CONNECTING, count as f64);
595 }
596 }
597
598 #[cfg(test)]
600 pub fn insert_connected_peer(&self, peer_ip: SocketAddr, peer_addr: SocketAddr, address: Address<N>) {
601 self.resolver.write().insert_peer(peer_ip, peer_addr, Some(address));
603 self.peer_pool.write().insert(peer_ip, Peer::new_connecting(peer_ip, false));
605 if let Some(peer) = self.peer_pool.write().get_mut(&peer_ip) {
606 peer.upgrade_to_connected(
607 peer_addr,
608 peer_ip.port(),
609 address,
610 NodeType::Validator,
611 0,
612 get_repo_commit_hash(),
613 ConnectionMode::Gateway,
614 );
615 }
616 }
617
618 fn send_inner(&self, peer_ip: SocketAddr, event: Event<N>) -> Option<oneshot::Receiver<io::Result<()>>> {
624 let Some(peer_addr) = self.resolve_to_ambiguous(peer_ip) else {
626 warn!("Unable to resolve the listener IP address '{peer_ip}'");
627 return None;
628 };
629 let name = event.name();
631 trace!("{CONTEXT} Sending '{name}' to '{peer_ip}'");
633 let result = self.unicast(peer_addr, event);
634 if let Err(err) = &result {
636 warn!("{CONTEXT} Failed to send '{name}' to '{peer_ip}': {err:?}");
637 debug!("{CONTEXT} Disconnecting from '{peer_ip}' (unable to send)");
638 self.disconnect(peer_ip);
639 }
640 result.ok()
641 }
642
643 async fn inbound(&self, peer_addr: SocketAddr, event: Event<N>) -> Result<bool> {
647 let Some(peer_ip) = self.resolver.read().get_listener(peer_addr) else {
649 trace!("Dropping a {} from {peer_addr} - no longer connected.", event.name());
651 return Ok(false);
652 };
653 if !(self.is_authorized_validator_ip(peer_ip)
655 || self
656 .get_connected_peer(peer_ip)
657 .map(|peer| peer.node_type == NodeType::BootstrapClient)
658 .unwrap_or(false))
659 {
660 bail!("{CONTEXT} Dropping '{}' from '{peer_ip}' (not authorized)", event.name())
661 }
662 let num_events = self.cache.insert_inbound_event(peer_ip, CACHE_EVENTS_INTERVAL);
664 if num_events >= self.max_cache_events() {
665 bail!("Dropping '{peer_ip}' for spamming events (num_events = {num_events})")
666 }
667 match event {
669 Event::CertificateRequest(_) | Event::CertificateResponse(_) => {
670 let certificate_id = match &event {
672 Event::CertificateRequest(CertificateRequest { certificate_id }) => *certificate_id,
673 Event::CertificateResponse(CertificateResponse { certificate }) => certificate.id(),
674 _ => unreachable!(),
675 };
676 let num_events = self.cache.insert_inbound_certificate(certificate_id, CACHE_REQUESTS_INTERVAL);
678 if num_events >= self.max_cache_duplicates() {
679 return Ok(true);
680 }
681 }
682 Event::TransmissionRequest(TransmissionRequest { transmission_id })
683 | Event::TransmissionResponse(TransmissionResponse { transmission_id, .. }) => {
684 let num_events = self.cache.insert_inbound_transmission(transmission_id, CACHE_REQUESTS_INTERVAL);
686 if num_events >= self.max_cache_duplicates() {
687 return Ok(true);
688 }
689 }
690 Event::BlockRequest(_) => {
691 let num_events = self.cache.insert_inbound_block_request(peer_ip, CACHE_REQUESTS_INTERVAL);
692 if num_events >= self.max_cache_duplicates() {
693 return Ok(true);
694 }
695 }
696 _ => {}
697 }
698 trace!("{CONTEXT} Received '{}' from '{peer_ip}'", event.name());
699
700 match event {
703 Event::BatchPropose(batch_propose) => {
704 let _ = self.primary_sender().tx_batch_propose.send((peer_ip, batch_propose)).await;
706 Ok(true)
707 }
708 Event::BatchSignature(batch_signature) => {
709 let _ = self.primary_sender().tx_batch_signature.send((peer_ip, batch_signature)).await;
711 Ok(true)
712 }
713 Event::BatchCertified(batch_certified) => {
714 let _ = self.primary_sender().tx_batch_certified.send((peer_ip, batch_certified.certificate)).await;
716 Ok(true)
717 }
718 Event::BlockRequest(block_request) => {
719 let BlockRequest { start_height, end_height } = block_request;
720
721 if start_height >= end_height {
723 bail!("Block request from '{peer_ip}' has an invalid range ({start_height}..{end_height})")
724 }
725 if end_height - start_height > DataBlocks::<N>::MAXIMUM_NUMBER_OF_BLOCKS as u32 {
727 bail!("Block request from '{peer_ip}' has an excessive range ({start_height}..{end_height})")
728 }
729
730 let latest_consensus_version = N::CONSENSUS_VERSION(end_height - 1)?;
732
733 let self_ = self.clone();
734 let blocks = match task::spawn_blocking(move || {
735 match self_.ledger.get_blocks(start_height..end_height) {
737 Ok(blocks) => Ok(DataBlocks(blocks)),
738 Err(error) => bail!("Missing blocks {start_height} to {end_height} from ledger - {error}"),
739 }
740 })
741 .await
742 {
743 Ok(Ok(blocks)) => blocks,
744 Ok(Err(error)) => return Err(error),
745 Err(error) => return Err(anyhow!("[BlockRequest] {error}")),
746 };
747
748 let self_ = self.clone();
749 tokio::spawn(async move {
750 let event =
752 Event::BlockResponse(BlockResponse::new(block_request, blocks, latest_consensus_version));
753 Transport::send(&self_, peer_ip, event).await;
754 });
755 Ok(true)
756 }
757 Event::BlockResponse(BlockResponse { request, latest_consensus_version, blocks, .. }) => {
758 if let Some(sync_sender) = self.sync_sender.get() {
760 if !self.cache.remove_outbound_block_request(peer_ip, &request) {
762 bail!("Unsolicited block response from '{peer_ip}'")
763 }
764
765 let (send, recv) = tokio::sync::oneshot::channel();
769 rayon::spawn_fifo(move || {
770 let blocks = blocks.deserialize_blocking().map_err(|error| anyhow!("[BlockResponse] {error}"));
771 let _ = send.send(blocks);
772 });
773 let blocks = match recv.await {
774 Ok(Ok(blocks)) => blocks,
775 Ok(Err(error)) => bail!("Peer '{peer_ip}' sent an invalid block response - {error}"),
776 Err(error) => bail!("Peer '{peer_ip}' sent an invalid block response - {error}"),
777 };
778
779 blocks.ensure_response_is_well_formed(peer_ip, request.start_height, request.end_height)?;
781 match sync_sender.insert_block_response(peer_ip, blocks.0, latest_consensus_version).await {
783 Ok(_) => Ok(true),
784 Err(err) if err.is_benign() => {
785 let err: anyhow::Error = err.into();
786 let err = err.context(format!("Ignoring block response from peer '{peer_ip}'"));
787 debug!("{}", flatten_error(err));
788 Ok(true)
789 }
790 Err(err) if err.is_consensus_version_ahead() => {
791 let err: anyhow::Error = err.into();
792 let err = err.context(format!(
793 "Peer sent a block response with a newer consensus version '{peer_ip}'"
794 ));
795 warn!("{}", flatten_error(&err));
796 Ok(true)
797 }
798 Err(err) if err.is_consensus_version_behind() => {
799 let err: anyhow::Error = err.into();
800 let err = err.context(format!("Peer sent an invalid block response '{peer_ip}'"));
801
802 let msg = flatten_error(&err);
803 error!("{msg}");
804 self.ip_ban_peer(peer_ip, Some(&msg));
805 Err(err)
806 }
807 Err(err) => {
808 let err: anyhow::Error = err.into();
809 let err = err.context(format!("Peer '{peer_ip}' sent an invalid block response"));
810 warn!("{}", flatten_error(err));
811
812 Ok(true)
814 }
815 }
816 } else {
817 debug!("Ignoring block response from '{peer_ip}' - no sync sender");
818 Ok(true)
819 }
820 }
821 Event::CertificateRequest(certificate_request) => {
822 if let Some(sync_sender) = self.sync_sender.get() {
825 let _ = sync_sender.tx_certificate_request.send((peer_ip, certificate_request)).await;
827 }
828 Ok(true)
829 }
830 Event::CertificateResponse(certificate_response) => {
831 if let Some(sync_sender) = self.sync_sender.get() {
834 let _ = sync_sender.tx_certificate_response.send((peer_ip, certificate_response)).await;
836 }
837 Ok(true)
838 }
839 Event::ChallengeRequest(..) | Event::ChallengeResponse(..) => {
840 bail!("{CONTEXT} Peer '{peer_ip}' is not following the protocol")
842 }
843 Event::Disconnect(message) => {
844 debug!("Peer '{peer_ip}' decided to disconnect due to '{}'", message.reason);
846 self.disconnect(peer_ip);
847 Ok(false)
848 }
849 Event::PrimaryPing(ping) => {
850 let PrimaryPing { version, block_locators, primary_certificate } = ping;
851
852 if version < Event::<N>::VERSION {
854 bail!("Dropping '{peer_ip}' on event version {version} (outdated)");
855 }
856
857 debug!("Validator '{peer_ip}' is at height {}", block_locators.latest_locator_height());
859
860 if let Some(sync_sender) = self.sync_sender.get() {
862 if let Err(error) = sync_sender.update_peer_locators(peer_ip, block_locators).await {
864 bail!("Validator '{peer_ip}' sent invalid block locators - {error}");
865 }
866 }
867
868 let _ = self.primary_sender().tx_primary_ping.send((peer_ip, primary_certificate)).await;
870 Ok(true)
871 }
872 Event::TransmissionRequest(request) => {
873 let Ok(worker_id) = assign_to_worker(request.transmission_id, self.num_workers()) else {
876 warn!("{CONTEXT} Unable to assign transmission ID '{}' to a worker", request.transmission_id);
877 return Ok(true);
878 };
879 if let Some(sender) = self.get_worker_sender(worker_id) {
881 let _ = sender.tx_transmission_request.send((peer_ip, request)).await;
883 }
884 Ok(true)
885 }
886 Event::TransmissionResponse(response) => {
887 let Ok(worker_id) = assign_to_worker(response.transmission_id, self.num_workers()) else {
889 warn!("{CONTEXT} Unable to assign transmission ID '{}' to a worker", response.transmission_id);
890 return Ok(true);
891 };
892 if let Some(sender) = self.get_worker_sender(worker_id) {
894 let _ = sender.tx_transmission_response.send((peer_ip, response)).await;
896 }
897 Ok(true)
898 }
899 Event::ValidatorsRequest(_) => {
900 let mut connected_peers = self.get_best_connected_peers(Some(MAX_VALIDATORS_TO_SEND));
901 connected_peers.shuffle(&mut rand::rng());
902
903 let self_ = self.clone();
904 tokio::spawn(async move {
905 let mut validators = IndexMap::with_capacity(MAX_VALIDATORS_TO_SEND);
907 for validator in connected_peers.into_iter() {
909 validators.insert(validator.listener_addr, validator.aleo_addr);
911 }
912 let event = Event::ValidatorsResponse(ValidatorsResponse { validators });
914 Transport::send(&self_, peer_ip, event).await;
915 });
916 Ok(true)
917 }
918 Event::ValidatorsResponse(response) => {
919 if self.trusted_peers_only {
920 bail!("{CONTEXT} Not accepting validators response from '{peer_ip}' (trusted peers only)");
921 }
922 let ValidatorsResponse { validators } = response;
923 ensure!(validators.len() <= MAX_VALIDATORS_TO_SEND, "{CONTEXT} Received too many validators");
925 if !self.cache.contains_outbound_validators_request(peer_ip) {
927 bail!("{CONTEXT} Received validators response from '{peer_ip}' without a validators request")
928 }
929 self.cache.decrement_outbound_validators_requests(peer_ip);
931
932 let valid_addrs = validators
935 .into_iter()
936 .filter_map(|(listener_addr, aleo_addr)| {
937 (self.account.address() != aleo_addr
938 && !self.is_connected_address(aleo_addr)
939 && self.is_authorized_validator_address(aleo_addr))
940 .then_some((listener_addr, None))
941 })
942 .collect::<Vec<_>>();
943 if !valid_addrs.is_empty() {
944 self.insert_candidate_peers(valid_addrs);
945 }
946
947 Ok(true)
948 }
949 Event::WorkerPing(ping) => {
950 ensure!(
952 ping.transmission_ids.len() <= Worker::<N>::MAX_TRANSMISSIONS_PER_WORKER_PING,
953 "{CONTEXT} Received too many transmissions"
954 );
955 let num_workers = self.num_workers();
957 for transmission_id in ping.transmission_ids.into_iter() {
959 let Ok(worker_id) = assign_to_worker(transmission_id, num_workers) else {
961 warn!("{CONTEXT} Unable to assign transmission ID '{transmission_id}' to a worker");
962 continue;
963 };
964 if let Some(sender) = self.get_worker_sender(worker_id) {
966 let _ = sender.tx_worker_ping.send((peer_ip, transmission_id)).await;
968 }
969 }
970 Ok(true)
971 }
972 }
973 }
974
975 fn initialize_heartbeat(&self) {
977 let self_clone = self.clone();
978 self.spawn(async move {
979 tokio::time::sleep(Duration::from_millis(1000)).await;
981 info!("Starting the heartbeat of the gateway...");
982 loop {
983 self_clone.heartbeat().await;
985 tokio::time::sleep(Duration::from_secs(15)).await;
987 }
988 });
989 }
990
991 #[allow(dead_code)]
993 fn spawn<T: Future<Output = ()> + Send + 'static>(&self, future: T) {
994 self.handles.lock().push(tokio::spawn(future));
995 }
996
997 pub async fn shut_down(&self) {
999 info!("Shutting down the gateway...");
1000 if let Err(e) = self.save_best_peers(&self.node_data_dir.gateway_peer_cache_path(), None, true) {
1002 warn!("Failed to persist best validators to disk: {e}");
1003 }
1004 self.handles.lock().iter().for_each(|handle| handle.abort());
1006 self.tcp.shut_down().await;
1008 }
1009}
1010
1011impl<N: Network> Gateway<N> {
1012 const MINIMUM_TIME_BETWEEN_CONNECTION_ATTEMPTS: Duration = Duration::from_secs(10);
1014 const MISSING_VALIDATOR_CONNECTIONS_GRACE_PERIOD: Duration = Duration::from_secs(60);
1016
1017 async fn heartbeat(&self) {
1019 self.log_connected_validators();
1021 #[cfg(feature = "metrics")]
1023 self.log_participation_scores();
1024 self.handle_trusted_validators();
1026 self.handle_bootstrap_peers().await;
1028 self.handle_unauthorized_validators();
1030 self.handle_min_connected_validators().await;
1032 self.handle_banned_ips();
1034 }
1035
1036 fn log_connected_validators(&self) {
1038 let connected_validators = self.filter_connected_peers(|peer| peer.node_type == NodeType::Validator);
1041
1042 let committee = match self.ledger.current_committee() {
1043 Ok(c) => c,
1044 Err(err) => {
1045 error!("Failed to get current committee: {err}");
1046 return;
1047 }
1048 };
1049
1050 let validators_total = committee.num_members().saturating_sub(1);
1052 let total_validators = format!("(of {validators_total} bonded validators)").dimmed();
1054 let connections_msg = match connected_validators.len() {
1056 0 => "No connected validators".to_string(),
1057 num_connected => format!("Connected to {num_connected} validators {total_validators}"),
1058 };
1059 info!("{connections_msg}");
1060
1061 let mut connected_validator_addresses = HashSet::with_capacity(connected_validators.len());
1063 let mut connected_validator_shas: HashMap<SmolStr, u64> = HashMap::with_capacity(connected_validators.len());
1064 let our_sha = shorten_snarkos_sha(&get_repo_commit_hash());
1066 let our_stake = committee.get_stake(self.account.address());
1067 connected_validator_shas.insert(our_sha.clone(), our_stake);
1068 connected_validator_addresses.insert(self.account.address());
1070 for peer in &connected_validators {
1072 let address = peer.aleo_addr;
1074 connected_validator_addresses.insert(address);
1075 let address_stake = committee.get_stake(address);
1077 let short_peer_sha = shorten_snarkos_sha(&peer.snarkos_sha);
1078 *connected_validator_shas.entry(short_peer_sha.clone()).or_default() += address_stake;
1079
1080 debug!(
1081 "{}",
1082 format!(
1083 " Connected to: {} - {} (connection age {:?})",
1084 peer.listener_addr,
1085 peer.aleo_addr,
1086 peer.first_seen.elapsed()
1087 )
1088 .dimmed()
1089 );
1090 }
1091
1092 if let Some(combined_stake) = connected_validator_shas.get(&our_sha) {
1094 let percentage = *combined_stake as f64 / committee.total_stake() as f64 * 100.0;
1095 debug!("{}", format!(" Combined stake @ {our_sha}: {percentage:.2}%").dimmed());
1096 #[cfg(feature = "metrics")]
1097 metrics::gauge(metrics::bft::CONNECTED_STAKE_WITH_MATCHING_SHA, percentage);
1098 }
1099
1100 let num_not_connected = validators_total.saturating_sub(connected_validators.len());
1102 if num_not_connected > 0 && self.tcp().uptime() > Self::MISSING_VALIDATOR_CONNECTIONS_GRACE_PERIOD {
1103 let total_stake = committee.total_stake();
1105 let total_stake_f64 = total_stake as f64;
1106
1107 let committee_members: HashSet<_> =
1109 self.ledger.current_committee().map(|c| c.members().keys().copied().collect()).unwrap_or_default();
1110
1111 let not_connected_stake: u64 = committee_members
1112 .difference(&connected_validator_addresses)
1113 .map(|address| {
1114 let address_stake = committee.get_stake(*address);
1115 let address_stake_as_percentage =
1116 if total_stake == 0 { 0.0 } else { address_stake as f64 / total_stake_f64 * 100.0 };
1117 debug!(
1118 "{}",
1119 format!(" Not connected to {address} ({address_stake_as_percentage:.2}% of total stake)")
1120 .dimmed()
1121 );
1122 address_stake
1123 })
1124 .sum();
1125
1126 let not_connected_stake_as_percentage =
1127 if total_stake == 0 { 0.0 } else { not_connected_stake as f64 / total_stake_f64 * 100.0 };
1128 warn!(
1129 "Not connected to {num_not_connected} validators {total_validators} ({not_connected_stake_as_percentage:.2}% of total stake not connected)"
1130 );
1131 #[cfg(feature = "metrics")]
1132 {
1133 let connected_stake_as_percentage = 100.0 - not_connected_stake_as_percentage;
1134 metrics::gauge(metrics::bft::CONNECTED_STAKE, connected_stake_as_percentage);
1135 }
1136 } else {
1137 #[cfg(feature = "metrics")]
1138 metrics::gauge(metrics::bft::CONNECTED_STAKE, 100.0);
1139 };
1140
1141 if !committee.is_quorum_threshold_reached(&connected_validator_addresses) {
1142 if self.tcp().uptime() > Self::MISSING_VALIDATOR_CONNECTIONS_GRACE_PERIOD {
1144 error!("Not connected to a quorum of validators");
1145 } else {
1146 debug!("Not connected to a quorum of validators");
1147 }
1148 }
1149 }
1150
1151 #[cfg(feature = "metrics")]
1153 fn log_participation_scores(&self) {
1154 if let Ok(committee_lookback) = self.ledger.get_committee_lookback_for_round(self.storage.current_round()) {
1155 let participation_scores = self.validator_telemetry().get_participation_scores(&committee_lookback);
1157
1158 debug!("Participation Scores (in the last {} rounds):", self.storage.max_gc_rounds());
1160 for (address, (cert_score, sig_score)) in participation_scores {
1161 debug!(
1162 "{}",
1163 format!(" {address} - certificates: {cert_score:.2}% signatures: {sig_score:.2}%").dimmed()
1164 );
1165 }
1166 }
1167 }
1168
1169 fn handle_trusted_validators(&self) {
1171 let trusted_peers = self.trusted_peers();
1172
1173 let handles: Vec<JoinHandle<_>> = trusted_peers
1175 .iter()
1176 .filter_map(|validator_ip| {
1177 match self.connect(*validator_ip) {
1179 Ok(hdl) => Some(hdl),
1180 Err(ConnectError::SelfConnect { .. })
1181 | Err(ConnectError::AlreadyConnected { .. })
1182 | Err(ConnectError::AlreadyConnecting { .. }) => None,
1183 Err(err) => {
1184 warn!("Could not initiate connection to trusted validator at '{validator_ip}' - {err}");
1185 None
1186 }
1187 }
1188 })
1189 .collect();
1190
1191 if !handles.is_empty() {
1192 info!("Reconnecting to {} out of {} trusted validators", handles.len(), trusted_peers.len());
1193 }
1194 }
1195
1196 async fn handle_bootstrap_peers(&self) {
1198 if self.trusted_peers_only {
1200 return;
1201 }
1202 let mut candidate_bootstrap = Vec::new();
1204 let connected_bootstrap = self.filter_connected_peers(|peer| peer.node_type == NodeType::BootstrapClient);
1205 for bootstrap_ip in bootstrap_peers::<N>(self.is_dev()) {
1206 if !connected_bootstrap.iter().any(|peer| peer.listener_addr == bootstrap_ip) {
1207 candidate_bootstrap.push(bootstrap_ip);
1208 }
1209 }
1210 if connected_bootstrap.is_empty() {
1212 let peer_to_connect = candidate_bootstrap.into_iter().choose(&mut rand::rng());
1214 if let Some(peer_ip) = peer_to_connect {
1215 match self.connect(peer_ip) {
1216 Ok(hdl) => {
1217 debug!("{CONTEXT} (Re-)connecting to bootstrap peer at '{peer_ip}'");
1218 let result = hdl.await;
1219 if let Err(err) = result {
1220 warn!("{CONTEXT} Failed to connect to bootstrap peer at '{peer_ip}' - {err}");
1221 }
1222 }
1223 Err(ConnectError::AlreadyConnected { .. }) | Err(ConnectError::AlreadyConnecting { .. }) => {}
1224 Err(err) => {
1225 warn!("{CONTEXT} Could not initiate connection to bootstrap peer at '{peer_ip}' - {err}")
1226 }
1227 }
1228 }
1229 }
1230 let num_surplus = connected_bootstrap.len().saturating_sub(1);
1232 if num_surplus > 0 {
1233 let peers_to_disconnect = connected_bootstrap.into_iter().sample(&mut rand::rng(), num_surplus);
1235 for peer in peers_to_disconnect {
1236 info!("{CONTEXT} Disconnecting from '{}' (exceeded maximum bootstrap)", peer.listener_addr);
1237 <Self as Transport<N>>::send(
1238 self,
1239 peer.listener_addr,
1240 Event::Disconnect(DisconnectReason::NoReasonGiven.into()),
1241 )
1242 .await;
1243 self.disconnect(peer.listener_addr);
1245 }
1246 }
1247 }
1248
1249 fn handle_unauthorized_validators(&self) {
1251 let self_ = self.clone();
1252 tokio::spawn(async move {
1253 let validators = self_.get_connected_peers();
1255 for peer in validators {
1257 if peer.node_type == NodeType::BootstrapClient {
1259 continue;
1260 }
1261 if !self_.is_authorized_validator_ip(peer.listener_addr) {
1263 warn!(
1264 "{CONTEXT} Disconnecting from '{}' - Validator is not in the current committee",
1265 peer.listener_addr
1266 );
1267 Transport::send(&self_, peer.listener_addr, DisconnectReason::ProtocolViolation.into()).await;
1268 self_.disconnect(peer.listener_addr);
1270 }
1271 }
1272 });
1273 }
1274
1275 async fn handle_min_connected_validators(&self) {
1279 let trusted_validators = self.trusted_peers();
1282 if self.number_of_connected_peers() < N::LATEST_MAX_CERTIFICATES() as usize {
1283 let (addrs, handles): (Vec<_>, Vec<_>) = self
1284 .get_candidate_peers()
1285 .iter()
1286 .filter_map(|peer| {
1287 if trusted_validators.contains(&peer.listener_addr) {
1288 return None;
1289 }
1290
1291 if let Some(previous_attempt) = peer.last_connection_attempt
1292 && previous_attempt.elapsed() < Self::MINIMUM_TIME_BETWEEN_CONNECTION_ATTEMPTS
1293 {
1294 return None;
1295 }
1296
1297 match self.connect(peer.listener_addr) {
1298 Ok(hdl) => Some((peer.listener_addr, hdl)),
1299 Err(ConnectError::AlreadyConnected { .. })
1300 | Err(ConnectError::AlreadyConnecting { .. })
1301 | Err(ConnectError::SelfConnect { .. }) => None,
1302 Err(err) => {
1303 warn!(
1304 "{CONTEXT} Could not initiate connection to validator at '{}' - {err}",
1305 peer.listener_addr
1306 );
1307 None
1308 }
1309 }
1310 })
1311 .unzip();
1312
1313 for (addr, result) in addrs.into_iter().zip(join_all(handles).await) {
1314 if let Err(err) = result {
1315 warn!("{CONTEXT} Failed to connect to validator at '{addr}' - {err}");
1316 }
1317 }
1318
1319 let validators = self.connected_peers();
1321 if validators.is_empty() {
1323 return;
1324 }
1325 if let Some(validator_ip) = validators.into_iter().choose(&mut rand::rng()) {
1327 let self_ = self.clone();
1328 tokio::spawn(async move {
1329 self_.cache.increment_outbound_validators_requests(validator_ip);
1331 let _ = Transport::send(&self_, validator_ip, Event::ValidatorsRequest(ValidatorsRequest)).await;
1333 });
1334 }
1335 }
1336 }
1337
1338 async fn process_message_inner(&self, peer_addr: SocketAddr, message: Event<N>) {
1340 if let Err(error) = self.inbound(peer_addr, message).await
1342 && let Some(peer_ip) = self.resolver.read().get_listener(peer_addr)
1343 {
1344 warn!("{CONTEXT} Disconnecting from '{peer_ip}' - {error}");
1345 let self_ = self.clone();
1346 tokio::spawn(async move {
1347 Transport::send(&self_, peer_ip, DisconnectReason::ProtocolViolation.into()).await;
1348 self_.disconnect(peer_ip);
1350 });
1351 }
1352 }
1353
1354 fn handle_banned_ips(&self) {
1356 self.tcp.banned_peers().remove_old_bans(IP_BAN_TIME_IN_SECS);
1357 }
1358}
1359
1360#[async_trait]
1361impl<N: Network> Transport<N> for Gateway<N> {
1362 async fn send(&self, peer_ip: SocketAddr, mut event: Event<N>) -> Option<oneshot::Receiver<io::Result<()>>> {
1370 if event.has_unserialized_payload() {
1381 let name = event.name();
1382 event = match spawn_blocking!({
1383 let mut event = event;
1384 event.serialize_payload()?;
1385 Ok(event)
1386 }) {
1387 Ok(event) => event,
1388 Err(err) => {
1389 error!("{CONTEXT} Unable to serialize '{name}' for '{peer_ip}' - {err}");
1390 return None;
1391 }
1392 };
1393 }
1394
1395 macro_rules! send {
1396 ($self:ident, $cache_map:ident, $interval:expr, $freq:ident) => {{
1397 while $self.cache.$cache_map(peer_ip, $interval) > $self.$freq() {
1399 tokio::time::sleep(Duration::from_millis(10)).await;
1401 }
1402 $self.send_inner(peer_ip, event)
1404 }};
1405 }
1406
1407 match event {
1409 Event::CertificateRequest(_) | Event::CertificateResponse(_) => {
1410 self.cache.insert_outbound_event(peer_ip, CACHE_EVENTS_INTERVAL);
1412 send!(self, insert_outbound_certificate, CACHE_REQUESTS_INTERVAL, max_cache_certificates)
1414 }
1415 Event::TransmissionRequest(_) | Event::TransmissionResponse(_) => {
1416 self.cache.insert_outbound_event(peer_ip, CACHE_EVENTS_INTERVAL);
1418 send!(self, insert_outbound_transmission, CACHE_REQUESTS_INTERVAL, max_cache_transmissions)
1420 }
1421 Event::BlockRequest(request) => {
1422 self.cache.insert_outbound_block_request(peer_ip, request);
1424 send!(self, insert_outbound_event, CACHE_EVENTS_INTERVAL, max_cache_events)
1426 }
1427 _ => {
1428 send!(self, insert_outbound_event, CACHE_EVENTS_INTERVAL, max_cache_events)
1430 }
1431 }
1432 }
1433
1434 fn broadcast(&self, mut event: Event<N>) {
1436 if self.number_of_connected_peers() > 0 {
1438 let self_ = self.clone();
1439 let connected_peers = self.connected_peers();
1440 tokio::spawn(async move {
1441 if event.has_unserialized_payload() {
1445 let name = event.name();
1446 event = match spawn_blocking!({
1447 let mut event = event;
1448 event.serialize_payload()?;
1449 Ok(event)
1450 }) {
1451 Ok(event) => event,
1452 Err(err) => {
1453 error!("{CONTEXT} Unable to serialize '{name}' for broadcast - {err}");
1454 return;
1455 }
1456 };
1457 }
1458 for peer_ip in connected_peers {
1460 let _ = Transport::send(&self_, peer_ip, event.clone()).await;
1462 }
1463 });
1464 }
1465 }
1466}
1467
1468impl<N: Network> P2P for Gateway<N> {
1469 fn tcp(&self) -> &Tcp {
1471 &self.tcp
1472 }
1473}
1474
1475#[async_trait]
1476impl<N: Network> Reading for Gateway<N> {
1477 type Codec = EventCodec<N>;
1478 type Message = Event<N>;
1479
1480 fn codec(&self, _peer_addr: SocketAddr, _side: ConnectionSide) -> Self::Codec {
1483 Default::default()
1484 }
1485
1486 async fn process_message(&self, peer_addr: SocketAddr, message: Self::Message) -> io::Result<()> {
1488 if matches!(message, Event::BlockRequest(_) | Event::BlockResponse(_)) {
1489 let self_ = self.clone();
1490 tokio::spawn(async move {
1493 self_.process_message_inner(peer_addr, message).await;
1494 });
1495 } else {
1496 self.process_message_inner(peer_addr, message).await;
1497 }
1498 Ok(())
1499 }
1500
1501 fn message_queue_depth(&self) -> usize {
1505 per_connection_queue_depth::<N>()
1506 }
1507}
1508
1509#[async_trait]
1510impl<N: Network> Writing for Gateway<N> {
1511 type Codec = EventCodec<N>;
1512 type Message = Event<N>;
1513
1514 fn codec(&self, _peer_addr: SocketAddr, _side: ConnectionSide) -> Self::Codec {
1517 Default::default()
1518 }
1519
1520 fn message_queue_depth(&self) -> usize {
1525 per_connection_queue_depth::<N>()
1526 }
1527}
1528
1529#[async_trait]
1530impl<N: Network> Disconnect for Gateway<N> {
1531 async fn handle_disconnect(&self, peer_addr: SocketAddr, origin: DisconnectOrigin) {
1533 debug!("Physically disconnecting from {peer_addr}; origin: {origin:?}");
1534
1535 if let Some(peer_ip) = self.resolve_to_listener(&peer_addr) {
1536 let was_fully_connected = self.downgrade_peer_to_candidate(peer_ip);
1542
1543 if was_fully_connected && let Some(sync_sender) = self.sync_sender.get() {
1545 let (tx, rx) = oneshot::channel();
1546
1547 if let Err(err) = sync_sender.tx_block_sync_remove_peer.send((peer_ip, tx)).await {
1548 let err: anyhow::Error = err.into();
1549 let err =
1550 err.context(format!("Unable to remove disconnecting peer '{peer_ip}' from the sync module"));
1551 warn!("{CONTEXT} {}", flatten_error(err));
1552 }
1553
1554 if let Err(err) = rx.await {
1555 let err: anyhow::Error = err.into();
1556 let err =
1557 err.context(format!("Unable to remove disconnecting peer '{peer_ip}' from the sync module"));
1558 warn!("{CONTEXT} {}", flatten_error(err));
1559 }
1560 }
1561 self.cache.clear_outbound_validators_requests(peer_ip);
1565 self.cache.clear_outbound_block_requests(peer_ip);
1566 } else {
1567 warn!("{CONTEXT} Got disconnect for a peer '{peer_addr}' that is not in the peer pool");
1568 }
1569 }
1570}
1571
1572#[async_trait]
1573impl<N: Network> OnConnect for Gateway<N> {
1574 async fn on_connect(&self, peer_addr: SocketAddr) {
1575 if let Some(listener_addr) = self.resolve_to_listener(&peer_addr) {
1576 if let Some(peer) = self.get_connected_peer(listener_addr) {
1577 if peer.node_type == NodeType::BootstrapClient {
1578 self.cache.increment_outbound_validators_requests(listener_addr);
1579 let _ =
1580 <Self as Transport<N>>::send(self, listener_addr, Event::ValidatorsRequest(ValidatorsRequest))
1581 .await;
1582 }
1583 }
1584 }
1585 }
1586}
1587
1588#[async_trait]
1589impl<N: Network> Handshake for Gateway<N> {
1590 async fn perform_handshake(&self, mut connection: Connection) -> Result<Connection, ConnectError> {
1592 let peer_addr = connection.addr();
1594 let peer_side = connection.side();
1595
1596 #[cfg(not(test))]
1598 if self.dev().is_none() && peer_side == ConnectionSide::Initiator {
1599 if self.is_ip_banned(peer_addr.ip()) {
1601 trace!("{CONTEXT} Rejected a connection request from banned IP '{}'", peer_addr.ip());
1602 return Err(ConnectError::BannedIp { ip: peer_addr.ip() });
1603 }
1604
1605 let num_attempts = self.cache.insert_inbound_connection(peer_addr.ip(), CONNECTION_ATTEMPTS_SINCE_SECS);
1606
1607 debug!("Number of connection attempts from '{}': {}", peer_addr.ip(), num_attempts);
1608 if num_attempts > MAX_CONNECTION_ATTEMPTS {
1609 self.update_ip_ban(peer_addr.ip());
1610 trace!("{CONTEXT} Rejected a consecutive connection request from IP '{}'", peer_addr.ip());
1611 return Err(ConnectError::other(anyhow!("'{}' appears to be spamming connections", peer_addr.ip())));
1612 }
1613 }
1614
1615 let stream = self.borrow_stream(&mut connection);
1616 harden_socket(stream)?;
1618
1619 let mut listener_addr = if peer_side == ConnectionSide::Initiator {
1622 debug!("{CONTEXT} Received a connection request from '{peer_addr}'");
1623 None
1624 } else {
1625 debug!("{CONTEXT} Shaking hands with {peer_addr}...");
1626 Some(peer_addr)
1627 };
1628
1629 let restrictions_id = self.ledger.latest_restrictions_id();
1631
1632 let handshake_result = if peer_side == ConnectionSide::Responder {
1637 if self.initiates_noise_handshake() {
1638 write_noise_magic(stream).await?;
1639 self.handshake_inner_initiator_noise(peer_addr, restrictions_id, stream).await
1640 } else {
1641 self.handshake_inner_initiator(peer_addr, restrictions_id, stream).await
1642 }
1643 } else {
1644 match detect_handshake_protocol(stream).await? {
1645 (HandshakeProtocol::Noise, _) => {
1646 self.handshake_inner_responder_noise(peer_addr, &mut listener_addr, restrictions_id, stream).await
1647 }
1648 (HandshakeProtocol::Legacy, _) if !self.accepts_legacy_handshake() => {
1649 Err(ConnectError::other(format!("'{peer_addr}' offered the legacy handshake, which has expired")))
1650 }
1651 (HandshakeProtocol::Legacy, prefix) => {
1652 self.handshake_inner_responder(peer_addr, &mut listener_addr, restrictions_id, stream, &prefix)
1653 .await
1654 }
1655 }
1656 };
1657
1658 match (&handshake_result, listener_addr) {
1661 (Ok(peer_info), Some(addr)) => {
1662 let node_type = if bootstrap_peers::<N>(self.is_dev()).contains(&addr) {
1663 NodeType::BootstrapClient
1664 } else {
1665 NodeType::Validator
1666 };
1667
1668 let mut peer_pool = self.peer_pool.write();
1669
1670 peer_pool.retain(|_, peer| {
1676 if let Peer::Candidate(peer) = peer
1677 && let Some(old_aleo_addr) = peer.last_known_aleo_addr
1678 {
1679 old_aleo_addr != peer_info.address || peer.listener_addr == addr
1680 } else {
1681 true
1682 }
1683 });
1684
1685 if let Some(peer) = peer_pool.get_mut(&addr) {
1686 self.resolver.write().insert_peer(addr, peer_addr, Some(peer_info.address));
1687 peer.upgrade_to_connected(
1688 peer_addr,
1689 peer_info.listener_port,
1690 peer_info.address,
1691 node_type,
1692 peer_info.version,
1693 peer_info.snarkos_sha,
1694 ConnectionMode::Gateway,
1695 );
1696 }
1697 info!("{CONTEXT} Connected to '{addr}'");
1698 }
1699 (Err(_), Some(addr)) => {
1700 if let Some(peer) = self.peer_pool.write().get_mut(&addr) {
1701 if peer.is_connecting() {
1703 peer.downgrade_to_candidate(addr);
1704 }
1705 }
1706 }
1707 (Ok(_), None) => {
1712 return Err(ConnectError::other(format!(
1713 "the handshake with '{peer_addr}' succeeded without a listening address"
1714 )));
1715 }
1716 (Err(_), None) => {}
1719 }
1720
1721 handshake_result?;
1723
1724 Ok(connection)
1725 }
1726}
1727
1728macro_rules! expect_event {
1730 ($event_ty:path, $framed:expr, $peer_addr:expr) => {
1731 match $framed.try_next().await? {
1732 Some($event_ty(data)) => {
1734 trace!("{CONTEXT} Received '{}' from '{}'", data.name(), $peer_addr);
1735 data
1736 }
1737 Some(Event::Disconnect($crate::events::Disconnect { reason })) => {
1739 return Err(ConnectError::other(format!("'{}' disconnected with reason \"{reason}\"", $peer_addr)));
1740 }
1741 Some(ty) => {
1743 return Err(ConnectError::other(format!(
1744 "'{}' did not follow the handshake protocol: received {:?} instead of {}",
1745 $peer_addr,
1746 ty.name(),
1747 stringify!($msg_ty),
1748 )));
1749 }
1750 None => return Err(ConnectError::IoError(io::ErrorKind::BrokenPipe.into())),
1752 }
1753 };
1754}
1755
1756async fn send_event<N: Network>(
1758 framed: &mut Framed<&mut TcpStream, EventCodec<N>>,
1759 peer_addr: SocketAddr,
1760 event: Event<N>,
1761) -> io::Result<()> {
1762 trace!("{CONTEXT} Sending '{}' to '{peer_addr}'", event.name());
1763 framed.send(event).await
1764}
1765
1766fn encode_payload<T: ToBytes>(payload: &T) -> Result<Vec<u8>, ConnectError> {
1768 snarkos_node_bft_events::encode_payload(payload)
1769}
1770
1771fn decode_payload<T: FromBytes>(peer_addr: SocketAddr, bytes: &[u8]) -> Result<T, ConnectError> {
1773 snarkos_node_bft_events::decode_payload(peer_addr, bytes)
1774}
1775
1776#[must_use]
1782async fn verify_binding_signature<N: Network>(
1783 peer_addr: SocketAddr,
1784 signature: Data<Signature<N>>,
1785 address: Address<N>,
1786 binding: &[u8],
1787) -> Option<DisconnectReason> {
1788 let Ok(signature) = spawn_blocking!(signature.deserialize_blocking()) else {
1790 warn!("{CONTEXT} Handshake with '{peer_addr}' failed (cannot deserialize the signature)");
1791 return Some(DisconnectReason::InvalidChallengeResponse);
1792 };
1793 if !signature.verify_bytes(&address, binding) {
1795 warn!("{CONTEXT} Handshake with '{peer_addr}' failed (invalid signature)");
1796 return Some(DisconnectReason::InvalidChallengeResponse);
1797 }
1798
1799 None
1800}
1801
1802fn finish_noise_handshake(noise: NoiseSession<&mut TcpStream>) {
1810 let _stream = noise.into_inner();
1812}
1813
1814fn note_legacy_handshake_end<N: Network>(framed: Framed<&mut TcpStream, EventCodec<N>>, peer_addr: SocketAddr) {
1822 let leftover = framed.into_parts().read_buf;
1823
1824 if !leftover.is_empty() {
1825 debug!("{CONTEXT} Discarding {} bytes '{peer_addr}' sent before the handshake was over", leftover.len());
1826 }
1827}
1828
1829fn peer_info_from_challenge_request<N: Network>(
1834 request: ChallengeRequest<N>,
1835 restrictions_id: Field<N>,
1836) -> PeerInfo<N> {
1837 let ChallengeRequest { version, listener_port, address, nonce: _, snarkos_sha } = request;
1838 PeerInfo { version, listener_port, address, restrictions_id, snarkos_sha }
1839}
1840
1841impl<N: Network> Gateway<N> {
1842 fn initiates_noise_handshake(&self) -> bool {
1844 if let Some(initiates) = self.pinned_handshake_protocol() {
1846 return initiates;
1847 }
1848
1849 self.is_dev() || self.consensus_version_reached(NOISE_HANDSHAKE_ACTIVATION)
1851 }
1852
1853 fn accepts_legacy_handshake(&self) -> bool {
1859 !self.consensus_version_reached(LEGACY_HANDSHAKE_EXPIRY)
1860 }
1861
1862 fn consensus_version_reached(&self, version: Option<ConsensusVersion>) -> bool {
1864 version.is_some_and(|version| {
1865 N::CONSENSUS_HEIGHT(version).is_ok_and(|height| self.ledger.latest_block_height() >= height)
1866 })
1867 }
1868
1869 #[cfg(not(any(test, feature = "test")))]
1871 fn pinned_handshake_protocol(&self) -> Option<bool> {
1872 None
1873 }
1874
1875 #[cfg(any(test, feature = "test"))]
1877 fn pinned_handshake_protocol(&self) -> Option<bool> {
1878 Some(self.initiates_noise_handshake.load(std::sync::atomic::Ordering::Relaxed))
1879 }
1880
1881 #[cfg(any(test, feature = "test"))]
1887 pub fn set_initiates_noise_handshake(&self, initiates: bool) {
1888 self.initiates_noise_handshake.store(initiates, std::sync::atomic::Ordering::Relaxed);
1889 }
1890
1891 fn snarkos_sha(&self) -> Option<[u8; 40]> {
1893 let current_block_height = self.ledger.latest_block_height();
1894 let consensus_version = N::CONSENSUS_VERSION(current_block_height).unwrap();
1895 match (self.is_dev(), consensus_version >= ConsensusVersion::V12, get_repo_commit_hash()) {
1896 (true, _, Some(sha)) => Some(sha),
1897 (_, true, Some(sha)) => Some(sha),
1898 _ => None,
1899 }
1900 }
1901
1902 async fn handshake_inner_initiator_noise<'a>(
1908 &'a self,
1909 peer_addr: SocketAddr,
1910 restrictions_id: Field<N>,
1911 stream: &'a mut TcpStream,
1912 ) -> Result<PeerInfo<N>, ConnectError> {
1913 let expected_address = match self.peer_pool.read().get(&peer_addr) {
1916 Some(Peer::Candidate(peer)) => peer.last_known_aleo_addr,
1917 _ => None,
1918 };
1919
1920 self.add_connecting_peer(peer_addr)?;
1922
1923 let mut noise = NoiseSession::new(stream, Role::Initiator)?;
1924 let our_info =
1925 PeerInfo::new(self.local_ip().port(), self.account.address(), restrictions_id, self.snarkos_sha());
1926
1927 let hint = HandshakeHint {
1930 version: our_info.version,
1931 listener_port: our_info.listener_port,
1932 address: our_info.address,
1933 };
1934 noise.send(&encode_payload(&hint)?).await?;
1935
1936 let peer_info: PeerInfo<N> = decode_payload(peer_addr, &noise.recv().await?)?;
1939
1940 let binding = binding_message(HANDSHAKE_DOMAIN, Role::Initiator, &noise.handshake_hash()?);
1944
1945 if let Some(reason) = self.verify_peer_info(peer_addr, &peer_info, restrictions_id) {
1947 return Err(reason.into_connect_error(peer_addr));
1948 }
1949
1950 if let Some(expected) = expected_address
1954 && peer_info.address != expected
1955 {
1956 warn!("{CONTEXT} Dropping '{peer_addr}': expected validator {expected}, got {}", peer_info.address);
1957 return Err(DisconnectReason::InvalidChallengeResponse.into_connect_error(peer_addr));
1958 }
1959
1960 let Ok(our_signature) = self.account.sign_bytes(&binding, &mut rand::rng()) else {
1963 return Err(ConnectError::other(anyhow!("Failed to sign the handshake binding")));
1964 };
1965 let our_message = InitiatorInfo { info: our_info, signature: Data::Object(our_signature) };
1966 noise.send(&encode_payload(&our_message)?).await?;
1967
1968 let peer_binding = binding_message(HANDSHAKE_DOMAIN, Role::Responder, &noise.handshake_hash()?);
1971
1972 let mut noise = noise.into_transport_mode()?;
1975 let verdict = decode_payload::<ResponderProof<N>>(peer_addr, &noise.recv().await?)?;
1976
1977 finish_noise_handshake(noise);
1981
1982 let peer_signature = match verdict {
1983 ResponderProof::Accepted { signature } => signature,
1984 ResponderProof::Rejected { reason } => {
1985 warn!("{CONTEXT} '{peer_addr}' rejected the handshake with reason \"{reason}\"");
1986 return Err(reason.into_connect_error(peer_addr));
1987 }
1988 };
1989
1990 if let Some(reason) =
1991 verify_binding_signature(peer_addr, peer_signature, peer_info.address, &peer_binding).await
1992 {
1993 return Err(reason.into_connect_error(peer_addr));
1994 }
1995
1996 Ok(peer_info)
1997 }
1998
1999 async fn handshake_inner_responder_noise<'a>(
2014 &'a self,
2015 peer_addr: SocketAddr,
2016 peer_ip: &mut Option<SocketAddr>,
2017 restrictions_id: Field<N>,
2018 stream: &'a mut TcpStream,
2019 ) -> Result<PeerInfo<N>, ConnectError> {
2020 let pending = PendingSession::accept(stream).await?;
2026 let hint: HandshakeHint<N> = decode_payload(peer_addr, pending.first_payload()?)?;
2027 let listener_addr = SocketAddr::new(peer_addr.ip(), hint.listener_port);
2028
2029 if let Err(reason) = self.ensure_peer_is_allowed(listener_addr) {
2034 return Err(reason.into_connect_error(peer_addr));
2035 }
2036 if let Some(reason) = self.verify_peer_identity(peer_addr, hint.version, hint.listener_port, hint.address) {
2037 return Err(reason.into_connect_error(peer_addr));
2038 }
2039
2040 self.add_connecting_peer(listener_addr)?;
2045 *peer_ip = Some(listener_addr);
2046
2047 let mut noise = pending.into_session()?;
2051
2052 let our_info =
2053 PeerInfo::new(self.local_ip().port(), self.account.address(), restrictions_id, self.snarkos_sha());
2054 noise.send(&encode_payload(&our_info)?).await?;
2055
2056 let peer_binding = binding_message(HANDSHAKE_DOMAIN, Role::Initiator, &noise.handshake_hash()?);
2058
2059 let InitiatorInfo { info: peer_info, signature: peer_signature } =
2062 decode_payload::<InitiatorInfo<N>>(peer_addr, &noise.recv().await?)?;
2063
2064 let binding = binding_message(HANDSHAKE_DOMAIN, Role::Responder, &noise.handshake_hash()?);
2067 let mut noise = noise.into_transport_mode()?;
2068
2069 if (hint.version, hint.listener_port, hint.address)
2072 != (peer_info.version, peer_info.listener_port, peer_info.address)
2073 {
2074 warn!("{CONTEXT} Handshake with '{peer_addr}' failed (the handshake hint was contradicted)");
2075 return self.reject_noise_handshake(peer_addr, noise, DisconnectReason::ProtocolViolation).await;
2076 }
2077
2078 if let Some(reason) = self.verify_peer_info(peer_addr, &peer_info, restrictions_id) {
2081 return self.reject_noise_handshake(peer_addr, noise, reason).await;
2082 }
2083
2084 if let Some(reason) =
2087 verify_binding_signature(peer_addr, peer_signature, peer_info.address, &peer_binding).await
2088 {
2089 return self.reject_noise_handshake(peer_addr, noise, reason).await;
2090 }
2091
2092 let Ok(our_signature) = self.account.sign_bytes(&binding, &mut rand::rng()) else {
2093 return Err(ConnectError::other(anyhow!("Failed to sign the handshake binding")));
2094 };
2095 noise.send(&encode_payload(&ResponderProof::Accepted { signature: Data::Object(our_signature) })?).await?;
2096
2097 finish_noise_handshake(noise);
2098
2099 Ok(peer_info)
2100 }
2101
2102 async fn reject_noise_handshake(
2104 &self,
2105 peer_addr: SocketAddr,
2106 mut noise: NoiseSession<&mut TcpStream>,
2107 reason: DisconnectReason,
2108 ) -> Result<PeerInfo<N>, ConnectError> {
2109 noise.send(&encode_payload(&ResponderProof::<N>::Rejected { reason })?).await?;
2110
2111 Err(reason.into_connect_error(peer_addr))
2112 }
2113
2114 async fn handshake_inner_initiator<'a>(
2116 &'a self,
2117 peer_addr: SocketAddr,
2118 restrictions_id: Field<N>,
2119 stream: &'a mut TcpStream,
2120 ) -> Result<PeerInfo<N>, ConnectError> {
2121 self.add_connecting_peer(peer_addr)?;
2123
2124 let mut framed = Framed::new(stream, EventCodec::<N>::handshake());
2126
2127 let our_nonce: u64 = rand::random();
2131 let snarkos_sha = self.snarkos_sha();
2133 let our_request = ChallengeRequest::new(self.local_ip().port(), self.account.address(), our_nonce, snarkos_sha);
2135 send_event(&mut framed, peer_addr, Event::ChallengeRequest(our_request)).await?;
2136
2137 let peer_response = expect_event!(Event::ChallengeResponse, framed, peer_addr);
2141 let peer_request = expect_event!(Event::ChallengeRequest, framed, peer_addr);
2143
2144 if let Some(reason) = self
2146 .verify_challenge_response(peer_addr, peer_request.address, peer_response, restrictions_id, our_nonce)
2147 .await
2148 {
2149 send_event(&mut framed, peer_addr, reason.into()).await?;
2150 return Err(ConnectError::application(reason));
2151 }
2152
2153 if let Some(reason) = self.verify_challenge_request(peer_addr, &peer_request) {
2155 send_event(&mut framed, peer_addr, reason.into()).await?;
2156 return Err(reason.into_connect_error(peer_addr));
2157 }
2158
2159 let response_nonce: u64 = rand::random();
2163 let data = [peer_request.nonce.to_le_bytes(), response_nonce.to_le_bytes()].concat();
2164 let Ok(our_signature) = self.account.sign_bytes(&data, &mut rand::rng()) else {
2165 return Err(ConnectError::other(anyhow!("Failed to sign the challenge request nonce")));
2166 };
2167 let our_response =
2169 ChallengeResponse { restrictions_id, signature: Data::Object(our_signature), nonce: response_nonce };
2170 send_event(&mut framed, peer_addr, Event::ChallengeResponse(our_response)).await?;
2171
2172 note_legacy_handshake_end(framed, peer_addr);
2173
2174 Ok(peer_info_from_challenge_request(peer_request, restrictions_id))
2175 }
2176
2177 async fn handshake_inner_responder<'a>(
2182 &'a self,
2183 peer_addr: SocketAddr,
2184 peer_ip: &mut Option<SocketAddr>,
2185 restrictions_id: Field<N>,
2186 stream: &'a mut TcpStream,
2187 prefix: &[u8],
2188 ) -> Result<PeerInfo<N>, ConnectError> {
2189 let mut framed = prepare_framed(stream, EventCodec::<N>::handshake(), prefix);
2191
2192 let peer_request = expect_event!(Event::ChallengeRequest, framed, peer_addr);
2196
2197 if self.account.address() == peer_request.address {
2199 return Err(ConnectError::SelfConnect { address: peer_addr });
2200 }
2201
2202 *peer_ip = Some(SocketAddr::new(peer_addr.ip(), peer_request.listener_port));
2204 let peer_ip = peer_ip.unwrap();
2205
2206 if let Err(reason) = self.ensure_peer_is_allowed(peer_ip) {
2208 send_event(&mut framed, peer_addr, reason.into()).await?;
2209 return Err(reason.into_connect_error(peer_addr));
2210 }
2211
2212 self.add_connecting_peer(peer_ip)?;
2214
2215 if let Some(reason) = self.verify_challenge_request(peer_addr, &peer_request) {
2217 send_event(&mut framed, peer_addr, reason.into()).await?;
2218 return Err(reason.into_connect_error(peer_addr));
2219 }
2220
2221 let response_nonce: u64 = rand::random();
2225 let data = [peer_request.nonce.to_le_bytes(), response_nonce.to_le_bytes()].concat();
2226 let Ok(our_signature) = self.account.sign_bytes(&data, &mut rand::rng()) else {
2227 return Err(ConnectError::other(anyhow!("Failed to sign the challenge request nonce")));
2228 };
2229 let our_response =
2231 ChallengeResponse { restrictions_id, signature: Data::Object(our_signature), nonce: response_nonce };
2232 send_event(&mut framed, peer_addr, Event::ChallengeResponse(our_response)).await?;
2233
2234 let our_nonce: u64 = rand::random();
2236 let snarkos_sha = self.snarkos_sha();
2238 let our_request = ChallengeRequest::new(self.local_ip().port(), self.account.address(), our_nonce, snarkos_sha);
2240 send_event(&mut framed, peer_addr, Event::ChallengeRequest(our_request)).await?;
2241
2242 let peer_response = expect_event!(Event::ChallengeResponse, framed, peer_addr);
2246 if let Some(reason) = self
2248 .verify_challenge_response(peer_addr, peer_request.address, peer_response, restrictions_id, our_nonce)
2249 .await
2250 {
2251 send_event(&mut framed, peer_addr, reason.into()).await?;
2252 Err(reason.into_connect_error(peer_addr))
2253 } else {
2254 note_legacy_handshake_end(framed, peer_addr);
2255
2256 Ok(peer_info_from_challenge_request(peer_request, restrictions_id))
2257 }
2258 }
2259
2260 #[must_use]
2262 fn verify_challenge_request(&self, peer_addr: SocketAddr, event: &ChallengeRequest<N>) -> Option<DisconnectReason> {
2263 let &ChallengeRequest { version, listener_port, address, nonce: _, ref snarkos_sha } = event;
2265 log_repo_sha_comparison(peer_addr, snarkos_sha, CONTEXT);
2266
2267 self.verify_peer_claims(peer_addr, version, listener_port, address)
2268 }
2269
2270 #[must_use]
2276 fn verify_peer_info(
2277 &self,
2278 peer_addr: SocketAddr,
2279 info: &PeerInfo<N>,
2280 expected_restrictions_id: Field<N>,
2281 ) -> Option<DisconnectReason> {
2282 log_repo_sha_comparison(peer_addr, &info.snarkos_sha, CONTEXT);
2283
2284 if info.restrictions_id != expected_restrictions_id {
2287 warn!("{CONTEXT} Handshake with '{peer_addr}' failed (incorrect restrictions ID)");
2288 return Some(DisconnectReason::InvalidChallengeResponse);
2289 }
2290
2291 self.verify_peer_identity(peer_addr, info.version, info.listener_port, info.address)
2292 }
2293
2294 #[must_use]
2301 fn verify_peer_identity(
2302 &self,
2303 peer_addr: SocketAddr,
2304 version: u32,
2305 listener_port: u16,
2306 address: Address<N>,
2307 ) -> Option<DisconnectReason> {
2308 if self.account.address() == address {
2310 return Some(DisconnectReason::SelfConnect);
2311 }
2312
2313 self.verify_peer_claims(peer_addr, version, listener_port, address)
2314 }
2315
2316 #[must_use]
2318 fn verify_peer_claims(
2319 &self,
2320 peer_addr: SocketAddr,
2321 version: u32,
2322 listener_port: u16,
2323 address: Address<N>,
2324 ) -> Option<DisconnectReason> {
2325 let listener_addr = SocketAddr::new(peer_addr.ip(), listener_port);
2326
2327 if version < Event::<N>::VERSION {
2329 return Some(DisconnectReason::OutdatedClientVersion);
2330 }
2331 if self.trusted_peers_only && !self.is_trusted(listener_addr) {
2333 warn!("{CONTEXT} Dropping '{peer_addr}' for being an untrusted validator ({address})");
2334 return Some(DisconnectReason::NoExternalPeersAllowed);
2335 }
2336 if !bootstrap_peers::<N>(self.dev().is_some()).contains(&listener_addr) {
2337 if !self.is_authorized_validator_address(address) {
2339 return Some(DisconnectReason::UnauthorizedValidator);
2340 }
2341 }
2342
2343 if self.is_connected_address(address) {
2345 return Some(DisconnectReason::AlreadyConnectedToAleoAddress);
2346 }
2347
2348 None
2349 }
2350
2351 #[must_use]
2353 async fn verify_challenge_response(
2354 &self,
2355 peer_addr: SocketAddr,
2356 peer_address: Address<N>,
2357 response: ChallengeResponse<N>,
2358 expected_restrictions_id: Field<N>,
2359 expected_nonce: u64,
2360 ) -> Option<DisconnectReason> {
2361 let ChallengeResponse { restrictions_id, signature, nonce } = response;
2363
2364 if restrictions_id != expected_restrictions_id {
2366 warn!("{CONTEXT} Handshake with '{peer_addr}' failed (incorrect restrictions ID)");
2367 return Some(DisconnectReason::InvalidChallengeResponse);
2368 }
2369 let Ok(signature) = spawn_blocking!(signature.deserialize_blocking()) else {
2371 warn!("{CONTEXT} Handshake with '{peer_addr}' failed (cannot deserialize the signature)");
2372 return Some(DisconnectReason::InvalidChallengeResponse);
2373 };
2374 if !signature.verify_bytes(&peer_address, &[expected_nonce.to_le_bytes(), nonce.to_le_bytes()].concat()) {
2376 warn!("{CONTEXT} Handshake with '{peer_addr}' failed (invalid signature)");
2377 return Some(DisconnectReason::InvalidChallengeResponse);
2378 }
2379 None
2380 }
2381}
2382
2383#[cfg(test)]
2384mod prop_tests {
2385 use crate::{
2386 Gateway,
2387 MAX_WORKERS,
2388 MEMORY_POOL_PORT,
2389 Worker,
2390 helpers::{Storage, init_primary_channels, init_worker_channels},
2391 };
2392
2393 use snarkos_account::Account;
2394 use snarkos_node_bft_ledger_service::MockLedgerService;
2395 use snarkos_node_bft_storage_service::BFTMemoryService;
2396 use snarkos_node_network::PeerPoolHandling;
2397 use snarkos_node_tcp::P2P;
2398 use snarkos_utilities::NodeDataDir;
2399
2400 use snarkos_node_bft_events::committee_prop_tests::{CommitteeContext, ValidatorSet};
2401 use snarkvm::{
2402 ledger::{
2403 committee::{Committee, test_helpers::sample_committee_for_round_and_members},
2404 narwhal::{BatchHeader, batch_certificate::test_helpers::sample_batch_certificate_for_round},
2405 },
2406 prelude::{MainnetV0, PrivateKey},
2407 utilities::TestRng,
2408 };
2409
2410 use indexmap::{IndexMap, IndexSet};
2411 use proptest::{
2412 prelude::{Arbitrary, BoxedStrategy, Just, Strategy, any, any_with},
2413 sample::Selector,
2414 };
2415 use std::{
2416 fmt::{Debug, Formatter},
2417 net::{IpAddr, Ipv4Addr, SocketAddr},
2418 sync::Arc,
2419 };
2420 use test_strategy::proptest;
2421
2422 type CurrentNetwork = MainnetV0;
2423
2424 #[test]
2429 fn test_per_connection_queue_depth() {
2430 use crate::gateway::{QUEUE_WINDOW_ROUNDS, per_connection_queue_depth};
2431 use snarkvm::console::network::Network;
2432
2433 assert_eq!(QUEUE_WINDOW_ROUNDS, 3);
2435
2436 let certificates = CurrentNetwork::LATEST_MAX_CERTIFICATES() as usize;
2437 let transmissions = BatchHeader::<CurrentNetwork>::MAX_TRANSMISSIONS_PER_BATCH;
2438 let depth = per_connection_queue_depth::<CurrentNetwork>();
2439
2440 assert_eq!(depth, 2 * QUEUE_WINDOW_ROUNDS * certificates * (transmissions + 1));
2441
2442 assert!(depth >= QUEUE_WINDOW_ROUNDS * certificates * transmissions);
2444 assert!(depth < 2 * BatchHeader::<CurrentNetwork>::MAX_GC_ROUNDS * certificates * transmissions / 8);
2447 }
2448
2449 impl Debug for Gateway<CurrentNetwork> {
2450 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
2451 f.debug_tuple("Gateway").field(&self.account.address()).field(&self.tcp.config()).finish()
2453 }
2454 }
2455
2456 #[derive(Debug, test_strategy::Arbitrary)]
2457 enum GatewayAddress {
2458 Dev(u8),
2459 Prod(Option<SocketAddr>),
2460 }
2461
2462 impl GatewayAddress {
2463 fn ip(&self) -> Option<SocketAddr> {
2464 if let GatewayAddress::Prod(ip) = self {
2465 return *ip;
2466 }
2467 None
2468 }
2469
2470 fn port(&self) -> Option<u16> {
2471 if let GatewayAddress::Dev(port) = self {
2472 return Some(*port as u16);
2473 }
2474 None
2475 }
2476 }
2477
2478 impl Arbitrary for Gateway<CurrentNetwork> {
2479 type Parameters = ();
2480 type Strategy = BoxedStrategy<Gateway<CurrentNetwork>>;
2481
2482 fn arbitrary_with(_: Self::Parameters) -> Self::Strategy {
2483 any_valid_dev_gateway()
2484 .prop_map(|(storage, _, private_key, address)| {
2485 Gateway::new(
2486 Account::try_from(private_key).unwrap(),
2487 storage.clone(),
2488 storage.ledger().clone(),
2489 address.ip(),
2490 &[],
2491 false,
2492 NodeDataDir::new_test(None),
2493 address.port(),
2494 )
2495 .unwrap()
2496 })
2497 .boxed()
2498 }
2499 }
2500
2501 type GatewayInput = (Storage<CurrentNetwork>, CommitteeContext, PrivateKey<CurrentNetwork>, GatewayAddress);
2502
2503 fn any_valid_dev_gateway() -> BoxedStrategy<GatewayInput> {
2504 (any::<CommitteeContext>(), any::<Selector>())
2505 .prop_flat_map(|(context, account_selector)| {
2506 let CommitteeContext(_, ValidatorSet(validators)) = context.clone();
2507 (
2508 any_with::<Storage<CurrentNetwork>>(context.clone()),
2509 Just(context),
2510 Just(account_selector.select(validators)),
2511 0u8..,
2512 )
2513 .prop_map(|(a, b, c, d)| (a, b, c.private_key, GatewayAddress::Dev(d)))
2514 })
2515 .boxed()
2516 }
2517
2518 fn any_valid_prod_gateway() -> BoxedStrategy<GatewayInput> {
2519 (any::<CommitteeContext>(), any::<Selector>())
2520 .prop_flat_map(|(context, account_selector)| {
2521 let CommitteeContext(_, ValidatorSet(validators)) = context.clone();
2522 (
2523 any_with::<Storage<CurrentNetwork>>(context.clone()),
2524 Just(context),
2525 Just(account_selector.select(validators)),
2526 any::<Option<SocketAddr>>(),
2527 )
2528 .prop_map(|(a, b, c, d)| (a, b, c.private_key, GatewayAddress::Prod(d)))
2529 })
2530 .boxed()
2531 }
2532
2533 #[proptest]
2534 fn gateway_dev_initialization(#[strategy(any_valid_dev_gateway())] input: GatewayInput) {
2535 let (storage, _, private_key, dev) = input;
2536 let account = Account::try_from(private_key).unwrap();
2537
2538 let gateway = Gateway::new(
2539 account.clone(),
2540 storage.clone(),
2541 storage.ledger().clone(),
2542 dev.ip(),
2543 &[],
2544 false,
2545 NodeDataDir::new_test(None),
2546 dev.port(),
2547 )
2548 .unwrap();
2549 let tcp_config = gateway.tcp().config();
2550 assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::LOCALHOST)));
2551 assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT + dev.port().unwrap()));
2552
2553 let tcp_config = gateway.tcp().config();
2554 assert_eq!(tcp_config.max_connections, Committee::<CurrentNetwork>::max_committee_size() * 10);
2555 assert_eq!(gateway.account().address(), account.address());
2556 }
2557
2558 #[proptest]
2559 fn gateway_prod_initialization(#[strategy(any_valid_prod_gateway())] input: GatewayInput) {
2560 let (storage, _, private_key, dev) = input;
2561 let account = Account::try_from(private_key).unwrap();
2562
2563 let gateway = Gateway::new(
2564 account.clone(),
2565 storage.clone(),
2566 storage.ledger().clone(),
2567 dev.ip(),
2568 &[],
2569 false,
2570 NodeDataDir::new_test(None),
2571 dev.port(),
2572 )
2573 .unwrap();
2574 let tcp_config = gateway.tcp().config();
2575 if let Some(socket_addr) = dev.ip() {
2576 assert_eq!(tcp_config.listener_ip, Some(socket_addr.ip()));
2577 assert_eq!(tcp_config.desired_listening_port, Some(socket_addr.port()));
2578 } else {
2579 assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED)));
2580 assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT));
2581 }
2582
2583 let tcp_config = gateway.tcp().config();
2584 assert_eq!(tcp_config.max_connections, Committee::<CurrentNetwork>::max_committee_size() * 10);
2585 assert_eq!(gateway.account().address(), account.address());
2586 }
2587
2588 #[proptest(async = "tokio")]
2589 async fn gateway_start(
2590 #[strategy(any_valid_dev_gateway())] input: GatewayInput,
2591 #[strategy(0..MAX_WORKERS)] workers_count: u8,
2592 ) {
2593 let (storage, committee, private_key, dev) = input;
2594 let committee = committee.0;
2595 let worker_storage = storage.clone();
2596 let account = Account::try_from(private_key).unwrap();
2597
2598 let gateway = Gateway::new(
2599 account,
2600 storage.clone(),
2601 storage.ledger().clone(),
2602 dev.ip(),
2603 &[],
2604 false,
2605 NodeDataDir::new_test(None),
2606 dev.port(),
2607 )
2608 .unwrap();
2609
2610 let (primary_sender, _) = init_primary_channels();
2611
2612 let (workers, worker_senders) = {
2613 let mut tx_workers = IndexMap::new();
2615 let mut workers = IndexMap::new();
2616
2617 for id in 0..workers_count {
2619 let (tx_worker, rx_worker) = init_worker_channels();
2621 let ledger = Arc::new(MockLedgerService::new(committee.clone()));
2623 let worker =
2624 Worker::new(id, Arc::new(gateway.clone()), worker_storage.clone(), ledger, Default::default())
2625 .unwrap();
2626 worker.run(rx_worker);
2628
2629 workers.insert(id, worker);
2631 tx_workers.insert(id, tx_worker);
2632 }
2633 (workers, tx_workers)
2634 };
2635
2636 gateway.run(primary_sender, worker_senders, None).await;
2637 assert_eq!(
2638 gateway.local_ip(),
2639 SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), MEMORY_POOL_PORT + dev.port().unwrap())
2640 );
2641 assert_eq!(gateway.num_workers(), workers.len() as u8);
2642 }
2643
2644 #[proptest]
2645 fn test_is_authorized_validator(#[strategy(any_valid_dev_gateway())] input: GatewayInput) {
2646 let rng = &mut TestRng::default();
2647
2648 let current_round = 2;
2650 let committee_size = 4;
2651 let max_gc_rounds = BatchHeader::<CurrentNetwork>::MAX_GC_ROUNDS as u64;
2652 let (_, _, private_key, dev) = input;
2653 let account = Account::try_from(private_key).unwrap();
2654
2655 let mut certificates = IndexSet::new();
2657 for _ in 0..committee_size {
2658 certificates.insert(sample_batch_certificate_for_round(current_round, rng));
2659 }
2660 let addresses: Vec<_> = certificates.iter().map(|certificate| certificate.author()).collect();
2661 let committee = sample_committee_for_round_and_members(current_round, addresses, rng);
2663 for _ in 0..committee_size {
2665 certificates.insert(sample_batch_certificate_for_round(current_round, rng));
2666 }
2667 let ledger = Arc::new(MockLedgerService::new(committee.clone()));
2669 let storage = Storage::new(ledger.clone(), Arc::new(BFTMemoryService::new()), max_gc_rounds).unwrap();
2671 let gateway = Gateway::new(
2673 account.clone(),
2674 storage.clone(),
2675 ledger.clone(),
2676 dev.ip(),
2677 &[],
2678 false,
2679 NodeDataDir::new_test(None),
2680 dev.port(),
2681 )
2682 .unwrap();
2683 for certificate in certificates.iter() {
2685 storage.testing_only_insert_certificate_testing_only(certificate.clone());
2686 }
2687 for i in 0..certificates.clone().len() {
2689 let is_authorized = gateway.is_authorized_validator_address(certificates[i].author());
2690 if i < committee_size {
2691 assert!(is_authorized);
2692 } else {
2693 assert!(!is_authorized);
2694 }
2695 }
2696 }
2697}