Skip to main content

federation_net/
engine.rs

1//! The network engine: endpoint lifecycle, connection registry and messaging.
2
3use std::collections::HashMap;
4use std::fmt;
5use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
6use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak};
7use std::time::{Duration, Instant};
8
9use iroh::endpoint::{Connection, RecvStream, SendStream, VarInt, presets};
10use iroh::protocol::{AcceptError, ProtocolHandler, Router};
11use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey, Signature};
12use serde::Serialize;
13use serde::de::DeserializeOwned;
14use tokio::sync::{Semaphore, mpsc, watch};
15use tokio::task::JoinSet;
16use tokio::time::timeout;
17use tracing::{debug, info, warn};
18
19use crate::config::NetworkConfig;
20use crate::error::{NetworkError, Result};
21use crate::event::{ConnectionDirection, NetworkEvent, NetworkEventReceiver};
22use crate::health::{NetworkHealthSnapshot, NetworkHealthState};
23use crate::identity;
24use crate::protocol::{
25    ALPN, Handshake, HandshakeAck, HandshakeErrorCode, MAX_HANDSHAKE_FRAME_SIZE,
26    MessageRejectReason, MessageRequest, MessageResponse, NetworkId, PROTOCOL_VERSION, SchemaId,
27};
28use crate::rendezvous::{RendezvousClient, RendezvousConfig, now_ms};
29use crate::ticket::{PeerTicket, TICKET_VERSION};
30use crate::wire;
31
32/// Close code for a graceful, error-free close.
33const CLOSE_CODE_OK: VarInt = VarInt::from_u32(0);
34/// Close code used when the handshake was rejected by either side.
35const CLOSE_CODE_HANDSHAKE_REJECTED: VarInt = VarInt::from_u32(1);
36/// Close code used when the engine shuts down.
37const CLOSE_CODE_SHUTDOWN: VarInt = VarInt::from_u32(2);
38/// Close code used when a connection is replaced by a newer one.
39const CLOSE_CODE_REPLACED: VarInt = VarInt::from_u32(3);
40
41/// Extra bytes allowed on top of `max_message_size` for message framing
42/// (request id, length varints and enum tags).
43const MESSAGE_FRAME_OVERHEAD: usize = 64;
44/// Maximum size of an encoded [`MessageResponse`] frame.
45const MAX_RESPONSE_FRAME_SIZE: usize = 1024;
46/// How long the accepting side keeps a rejected connection open so the
47/// rejection ack reaches the initiator before the connection is torn down.
48const REJECT_LINGER: Duration = Duration::from_secs(3);
49/// How long [`NetworkEngine::shutdown`] waits for background tasks before
50/// aborting them.
51const SHUTDOWN_TASK_GRACE: Duration = Duration::from_secs(5);
52/// Upper bound of the rendezvous round delay while no peer is connected yet.
53const RENDEZVOUS_LONELY_INTERVAL: Duration = Duration::from_secs(15);
54/// A stuck Mainline-DHT operation must not stall discovery forever.
55const RENDEZVOUS_ROUND_TIMEOUT: Duration = Duration::from_secs(30);
56/// Recreate the Mainline-DHT client after this many consecutive failures.
57const RENDEZVOUS_REBUILD_AFTER_FAILURES: u32 = 2;
58/// Recommend an application-owned full restart after sustained discovery failure.
59const RENDEZVOUS_RESTART_AFTER_FAILURES: u32 = 4;
60/// Capacity of the queue of accepted-but-not-yet-consumed incoming byte
61/// streams, per stream protocol. A full queue delays the handshake ack of
62/// further incoming streams (natural backpressure).
63const STREAM_ACCEPT_QUEUE: usize = 16;
64
65/// Bounds required of a domain message type.
66///
67/// This trait is implemented automatically for every type that satisfies the
68/// bounds; applications never implement it by hand.
69pub trait Message: Serialize + DeserializeOwned + Send + Sync + 'static {}
70
71impl<T> Message for T where T: Serialize + DeserializeOwned + Send + Sync + 'static {}
72
73/// Locks a mutex, recovering the guard if the mutex was poisoned.
74///
75/// The engine never panics while holding one of its locks, so poisoning can
76/// only originate from a panic in unrelated user code; recovering is safe.
77fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
78    mutex.lock().unwrap_or_else(PoisonError::into_inner)
79}
80
81/// State kept for every active peer connection.
82struct PeerState {
83    connection: Connection,
84    direction: ConnectionDirection,
85    semaphore: Arc<Semaphore>,
86    /// Monotonic registration counter, used so a stale connection's cleanup
87    /// never removes a newer connection to the same peer.
88    generation: u64,
89}
90
91/// A raw bidirectional byte stream to a peer, running over a dedicated
92/// connection on an auxiliary ALPN.
93///
94/// Returned by [`NetworkEngine::open_stream`] (outgoing) and by
95/// [`StreamAcceptor::accept`] (incoming). Both sides passed the regular
96/// network handshake, so the peer is authenticated and belongs to the same
97/// network and schema. The connection behind the stream closes when both
98/// halves are dropped.
99pub struct ByteStream {
100    /// The authenticated peer on the other side of the stream.
101    pub peer_id: EndpointId,
102    /// The sending half of the stream.
103    pub send: SendStream,
104    /// The receiving half of the stream.
105    pub recv: RecvStream,
106    /// Keeps the dedicated connection alive for the lifetime of the stream.
107    _connection: Connection,
108}
109
110/// The currently selected transport path kind for a byte stream connection.
111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
112pub enum ConnectionPathKind {
113    /// A direct UDP path to the remote peer.
114    Direct,
115    /// A path through the peer's home relay.
116    Relay,
117    /// A custom transport path.
118    Custom,
119    /// Iroh has not reported a selected path yet.
120    Unknown,
121}
122
123impl ConnectionPathKind {
124    /// Returns a compact stable label for logs and UI.
125    pub fn as_str(self) -> &'static str {
126        match self {
127            Self::Direct => "direct",
128            Self::Relay => "relay",
129            Self::Custom => "custom",
130            Self::Unknown => "unknown",
131        }
132    }
133}
134
135impl fmt::Display for ConnectionPathKind {
136    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137        f.write_str(self.as_str())
138    }
139}
140
141/// Snapshot of low-level transport statistics for a byte stream connection.
142#[derive(Debug, Clone, Copy, PartialEq, Eq)]
143pub struct ByteStreamConnectionStats {
144    /// The authenticated peer on the other side of the stream.
145    pub peer_id: EndpointId,
146    /// The transport path currently selected for application traffic.
147    pub selected_path: ConnectionPathKind,
148    /// Number of open paths on the underlying Iroh connection.
149    pub open_paths: usize,
150    /// Number of open direct paths.
151    pub direct_paths: usize,
152    /// Number of open relay paths.
153    pub relay_paths: usize,
154    /// Number of open custom transport paths.
155    pub custom_paths: usize,
156    /// RTT of the selected path, when available.
157    pub selected_rtt: Option<Duration>,
158    /// UDP bytes transmitted on the selected path.
159    pub selected_tx_bytes: u64,
160    /// UDP bytes received on the selected path.
161    pub selected_rx_bytes: u64,
162    /// UDP bytes transmitted on the whole connection.
163    pub total_tx_bytes: u64,
164    /// UDP bytes received on the whole connection.
165    pub total_rx_bytes: u64,
166    /// Packets lost on the whole connection.
167    pub lost_packets: u64,
168    /// Bytes lost on the whole connection.
169    pub lost_bytes: u64,
170}
171
172impl ByteStreamConnectionStats {
173    fn from_connection(peer_id: EndpointId, connection: &Connection) -> Self {
174        let totals = connection.stats();
175        let paths = connection.paths();
176        let mut stats = Self {
177            peer_id,
178            selected_path: ConnectionPathKind::Unknown,
179            open_paths: paths.len(),
180            direct_paths: 0,
181            relay_paths: 0,
182            custom_paths: 0,
183            selected_rtt: None,
184            selected_tx_bytes: 0,
185            selected_rx_bytes: 0,
186            total_tx_bytes: totals.udp_tx.bytes,
187            total_rx_bytes: totals.udp_rx.bytes,
188            lost_packets: totals.lost_packets,
189            lost_bytes: totals.lost_bytes,
190        };
191
192        for path in paths.iter() {
193            let path_kind = if path.is_ip() {
194                stats.direct_paths += 1;
195                ConnectionPathKind::Direct
196            } else if path.is_relay() {
197                stats.relay_paths += 1;
198                ConnectionPathKind::Relay
199            } else {
200                stats.custom_paths += 1;
201                ConnectionPathKind::Custom
202            };
203            if path.is_selected() {
204                let path_stats = path.stats();
205                stats.selected_path = path_kind;
206                stats.selected_rtt = Some(path_stats.rtt);
207                stats.selected_tx_bytes = path_stats.udp_tx.bytes;
208                stats.selected_rx_bytes = path_stats.udp_rx.bytes;
209            }
210        }
211
212        stats
213    }
214}
215
216impl ByteStream {
217    /// Returns a point-in-time snapshot of the underlying Iroh connection.
218    pub fn connection_stats(&self) -> ByteStreamConnectionStats {
219        ByteStreamConnectionStats::from_connection(self.peer_id, &self._connection)
220    }
221}
222
223impl fmt::Debug for ByteStream {
224    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
225        f.debug_struct("ByteStream")
226            .field("peer_id", &self.peer_id)
227            .finish_non_exhaustive()
228    }
229}
230
231/// Receiver of incoming [`ByteStream`]s for one declared stream protocol.
232///
233/// Obtained once per protocol from [`NetworkEngine::stream_acceptor`].
234/// Dropping the acceptor makes the engine refuse further incoming streams
235/// on that protocol.
236pub struct StreamAcceptor {
237    alpn: Vec<u8>,
238    rx: mpsc::Receiver<ByteStream>,
239}
240
241impl StreamAcceptor {
242    /// Waits for the next incoming stream.
243    ///
244    /// Returns `None` after the engine shut down.
245    pub async fn accept(&mut self) -> Option<ByteStream> {
246        self.rx.recv().await
247    }
248
249    /// The ALPN of the stream protocol this acceptor serves.
250    pub fn alpn(&self) -> &[u8] {
251        &self.alpn
252    }
253}
254
255impl fmt::Debug for StreamAcceptor {
256    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
257        f.debug_struct("StreamAcceptor")
258            .field("alpn", &String::from_utf8_lossy(&self.alpn))
259            .finish_non_exhaustive()
260    }
261}
262
263/// Producer/consumer pair behind one declared stream protocol.
264struct StreamAcceptorSlot {
265    sender: mpsc::Sender<ByteStream>,
266    /// Taken (once) by [`NetworkEngine::stream_acceptor`].
267    receiver: Option<mpsc::Receiver<ByteStream>>,
268}
269
270#[derive(Debug)]
271struct RendezvousHealth {
272    last_success: Option<Instant>,
273    consecutive_failures: u32,
274    consecutive_peer_dial_failures: u32,
275    client_restarts: u64,
276    last_error: Option<String>,
277}
278
279impl RendezvousHealth {
280    fn new() -> Self {
281        Self {
282            last_success: None,
283            consecutive_failures: 0,
284            consecutive_peer_dial_failures: 0,
285            client_restarts: 0,
286            last_error: None,
287        }
288    }
289}
290
291fn alpn_display(alpn: &[u8]) -> String {
292    String::from_utf8_lossy(alpn).into_owned()
293}
294
295/// State shared between the engine handles, the protocol handler and all
296/// background tasks.
297struct Shared<M> {
298    config: NetworkConfig,
299    endpoint: Endpoint,
300    router: Mutex<Option<Router>>,
301    peers: Mutex<HashMap<EndpointId, PeerState>>,
302    stream_acceptors: Mutex<HashMap<Vec<u8>, StreamAcceptorSlot>>,
303    events: Mutex<Option<mpsc::Sender<NetworkEvent<M>>>>,
304    tasks: Mutex<JoinSet<()>>,
305    next_generation: AtomicU64,
306    shutting_down: AtomicBool,
307    rendezvous_health: Mutex<Option<RendezvousHealth>>,
308    /// Broadcasts the start of the shutdown to long-running background
309    /// loops so they can stop promptly instead of being aborted.
310    shutdown_signal: watch::Sender<bool>,
311}
312
313impl<M: Message> Shared<M> {
314    fn is_shutting_down(&self) -> bool {
315        self.shutting_down.load(Ordering::SeqCst)
316    }
317
318    fn ensure_running(&self) -> Result<()> {
319        if self.is_shutting_down() {
320            Err(NetworkError::ShuttingDown)
321        } else {
322            Ok(())
323        }
324    }
325
326    fn record_rendezvous_success(&self) {
327        if let Some(health) = lock(&self.rendezvous_health).as_mut() {
328            health.last_success = Some(Instant::now());
329            health.consecutive_failures = 0;
330            health.last_error = None;
331        }
332    }
333
334    fn record_rendezvous_failure(&self, error: String) -> u32 {
335        let mut guard = lock(&self.rendezvous_health);
336        let Some(health) = guard.as_mut() else {
337            return 0;
338        };
339        health.consecutive_failures = health.consecutive_failures.saturating_add(1);
340        health.last_error = Some(error);
341        health.consecutive_failures
342    }
343
344    fn record_rendezvous_restart(&self) {
345        if let Some(health) = lock(&self.rendezvous_health).as_mut() {
346            health.client_restarts = health.client_restarts.saturating_add(1);
347        }
348    }
349
350    fn record_peer_connection_success(&self) {
351        if let Some(health) = lock(&self.rendezvous_health).as_mut() {
352            health.consecutive_peer_dial_failures = 0;
353        }
354    }
355
356    fn record_rendezvous_peer_dials(&self, candidates: usize, attempts: usize, successes: usize) {
357        let mut guard = lock(&self.rendezvous_health);
358        let Some(health) = guard.as_mut() else {
359            return;
360        };
361        if candidates == 0 || successes > 0 {
362            health.consecutive_peer_dial_failures = 0;
363        } else if attempts > 0 {
364            health.consecutive_peer_dial_failures =
365                health.consecutive_peer_dial_failures.saturating_add(1);
366            health.last_error = Some(format!(
367                "failed to connect to all {attempts} peer(s) discovered by rendezvous"
368            ));
369        }
370    }
371
372    fn health(&self) -> NetworkHealthSnapshot {
373        let connected_peers = lock(&self.peers).len();
374        let guard = lock(&self.rendezvous_health);
375        let Some(health) = guard.as_ref() else {
376            return NetworkHealthSnapshot {
377                state: if connected_peers == 0 {
378                    NetworkHealthState::Discovering
379                } else {
380                    NetworkHealthState::Healthy
381                },
382                connected_peers,
383                rendezvous_enabled: false,
384                consecutive_rendezvous_failures: 0,
385                consecutive_peer_dial_failures: 0,
386                rendezvous_restarts: 0,
387                last_rendezvous_success_ago: None,
388                last_rendezvous_error: None,
389                restart_recommended: false,
390            };
391        };
392        let restart_recommended = connected_peers == 0
393            && (health.consecutive_failures >= RENDEZVOUS_RESTART_AFTER_FAILURES
394                || health.consecutive_peer_dial_failures >= RENDEZVOUS_RESTART_AFTER_FAILURES);
395        let state = if restart_recommended {
396            NetworkHealthState::Degraded
397        } else if health.consecutive_failures > 0 || health.consecutive_peer_dial_failures > 0 {
398            NetworkHealthState::Recovering
399        } else if health.last_success.is_some() {
400            NetworkHealthState::Healthy
401        } else {
402            NetworkHealthState::Discovering
403        };
404        NetworkHealthSnapshot {
405            state,
406            connected_peers,
407            rendezvous_enabled: true,
408            consecutive_rendezvous_failures: health.consecutive_failures,
409            consecutive_peer_dial_failures: health.consecutive_peer_dial_failures,
410            rendezvous_restarts: health.client_restarts,
411            last_rendezvous_success_ago: health.last_success.map(|at| at.elapsed()),
412            last_rendezvous_error: health.last_error.clone(),
413            restart_recommended,
414        }
415    }
416
417    /// Delivers an event to the application.
418    ///
419    /// The channel is bounded; if it is full this awaits until the
420    /// application consumes events, applying natural back-pressure without
421    /// unbounded buffering.
422    async fn emit(&self, event: NetworkEvent<M>) {
423        let sender = lock(&self.events).clone();
424        if let Some(sender) = sender
425            && sender.send(event).await.is_err()
426        {
427            debug!("event receiver dropped; event discarded");
428        }
429    }
430
431    /// Spawns a background task tracked until shutdown.
432    fn spawn_task<F>(&self, future: F)
433    where
434        F: Future<Output = ()> + Send + 'static,
435    {
436        let mut tasks = lock(&self.tasks);
437        // Reap already-finished tasks so the set does not grow unboundedly.
438        while tasks.try_join_next().is_some() {}
439        tasks.spawn(future);
440    }
441
442    /// Adds a connection to the registry, replacing (and closing) any previous
443    /// connection to the same peer. Returns the registration generation.
444    fn register_connection(
445        &self,
446        peer_id: EndpointId,
447        connection: Connection,
448        direction: ConnectionDirection,
449    ) -> u64 {
450        let generation = self.next_generation.fetch_add(1, Ordering::Relaxed);
451        let state = PeerState {
452            connection,
453            direction,
454            semaphore: Arc::new(Semaphore::new(self.config.max_concurrent_streams_per_peer)),
455            generation,
456        };
457        let replaced = lock(&self.peers).insert(peer_id, state);
458        self.record_peer_connection_success();
459        if let Some(old) = replaced {
460            debug!(peer = %peer_id, "replacing existing connection");
461            old.connection
462                .close(CLOSE_CODE_REPLACED, b"replaced by a new connection");
463        }
464        generation
465    }
466
467    /// Removes the connection from the registry and reports the disconnect,
468    /// unless a newer connection to the same peer took its place.
469    async fn cleanup_connection(
470        &self,
471        peer_id: EndpointId,
472        generation: u64,
473        reason: Option<String>,
474    ) {
475        let removed = {
476            let mut peers = lock(&self.peers);
477            match peers.get(&peer_id) {
478                Some(state) if state.generation == generation => peers.remove(&peer_id),
479                _ => None,
480            }
481        };
482        if let Some(state) = removed {
483            info!(
484                peer = %peer_id,
485                direction = ?state.direction,
486                reason = reason.as_deref().unwrap_or("unknown"),
487                "peer disconnected"
488            );
489            self.emit(NetworkEvent::PeerDisconnected { peer_id, reason })
490                .await;
491        }
492    }
493
494    /// Validates a remote handshake against the local configuration.
495    fn validate_handshake(&self, handshake: &Handshake) -> Option<HandshakeErrorCode> {
496        if handshake.protocol_version != PROTOCOL_VERSION {
497            Some(HandshakeErrorCode::UnsupportedProtocolVersion)
498        } else if handshake.network_id != self.config.network_id {
499            Some(HandshakeErrorCode::NetworkMismatch)
500        } else if handshake.schema_id != self.config.schema_id {
501            Some(HandshakeErrorCode::SchemaMismatch)
502        } else {
503            None
504        }
505    }
506
507    fn validate_stream_handshake(
508        &self,
509        alpn: &[u8],
510        handshake: &Handshake,
511    ) -> Option<HandshakeErrorCode> {
512        if handshake.protocol_version != PROTOCOL_VERSION {
513            Some(HandshakeErrorCode::UnsupportedProtocolVersion)
514        } else if handshake.network_id != self.config.network_id {
515            Some(HandshakeErrorCode::NetworkMismatch)
516        } else if !self
517            .config
518            .schema_independent_stream_protocols
519            .iter()
520            .any(|protocol| protocol.as_slice() == alpn)
521            && handshake.schema_id != self.config.schema_id
522        {
523            Some(HandshakeErrorCode::SchemaMismatch)
524        } else {
525            None
526        }
527    }
528
529    /// Drives an established connection: accepts message streams until the
530    /// connection closes, then cleans up the registry entry.
531    async fn connection_loop(
532        self: Arc<Self>,
533        peer_id: EndpointId,
534        connection: Connection,
535        generation: u64,
536    ) {
537        let semaphore = {
538            let peers = lock(&self.peers);
539            match peers.get(&peer_id) {
540                Some(state) if state.generation == generation => state.semaphore.clone(),
541                // The connection was already replaced; nothing to drive.
542                _ => return,
543            }
544        };
545        loop {
546            let (send, recv) = match connection.accept_bi().await {
547                Ok(streams) => streams,
548                Err(_) => break,
549            };
550            let permit = match semaphore.clone().acquire_owned().await {
551                Ok(permit) => permit,
552                Err(_) => break,
553            };
554            let shared = self.clone();
555            self.spawn_task(async move {
556                shared
557                    .handle_message_stream(peer_id, send, recv, permit)
558                    .await;
559                // The permit is released when the task finishes.
560            });
561        }
562        let reason = connection.close_reason().map(|err| err.to_string());
563        self.cleanup_connection(peer_id, generation, reason).await;
564    }
565
566    /// Handles a single incoming message stream: decodes the request, hands
567    /// the message to the application and acknowledges it.
568    async fn handle_message_stream(
569        &self,
570        peer_id: EndpointId,
571        mut send: SendStream,
572        mut recv: RecvStream,
573        _permit: tokio::sync::OwnedSemaphorePermit,
574    ) {
575        let request_timeout = self.config.request_timeout;
576        let max_frame = self.config.max_message_size + MESSAGE_FRAME_OVERHEAD;
577        let request: MessageRequest =
578            match timeout(request_timeout, wire::read_frame(&mut recv, max_frame)).await {
579                Ok(Ok(request)) => request,
580                Ok(Err(err)) => {
581                    self.report_protocol_error(Some(peer_id), format!("bad message frame: {err}"))
582                        .await;
583                    return;
584                }
585                Err(_) => {
586                    self.report_protocol_error(
587                        Some(peer_id),
588                        "timed out reading message frame".to_string(),
589                    )
590                    .await;
591                    return;
592                }
593            };
594        let request_id = request.request_id;
595        let response = if self.is_shutting_down() {
596            MessageResponse::Rejected {
597                request_id,
598                reason: MessageRejectReason::ShuttingDown,
599            }
600        } else {
601            match wire::decode::<M>(&request.payload) {
602                Ok(message) => {
603                    self.emit(NetworkEvent::MessageReceived { peer_id, message })
604                        .await;
605                    MessageResponse::Accepted { request_id }
606                }
607                Err(err) => {
608                    self.report_protocol_error(
609                        Some(peer_id),
610                        format!("failed to decode domain message: {err}"),
611                    )
612                    .await;
613                    MessageResponse::Rejected {
614                        request_id,
615                        reason: MessageRejectReason::MalformedPayload,
616                    }
617                }
618            }
619        };
620        let write = timeout(
621            request_timeout,
622            wire::write_frame(&mut send, &response, MAX_RESPONSE_FRAME_SIZE),
623        )
624        .await;
625        match write {
626            Ok(Ok(())) => {
627                let _ = send.finish();
628            }
629            Ok(Err(err)) => {
630                debug!(peer = %peer_id, error = %err, "failed to send message response");
631            }
632            Err(_) => {
633                debug!(peer = %peer_id, "timed out sending message response");
634            }
635        }
636    }
637
638    async fn report_protocol_error(&self, peer_id: Option<EndpointId>, error: String) {
639        match peer_id {
640            Some(peer) => warn!(peer = %peer, error = %error, "protocol error"),
641            None => warn!(error = %error, "protocol error"),
642        }
643        self.emit(NetworkEvent::ProtocolError { peer_id, error })
644            .await;
645    }
646
647    /// Handles a freshly accepted incoming connection: performs the handshake
648    /// and, if accepted, drives the connection until it closes.
649    async fn handle_incoming(self: &Arc<Self>, connection: &Connection) -> Result<()> {
650        let peer_id = connection.remote_id();
651        if self.is_shutting_down() {
652            connection.close(CLOSE_CODE_SHUTDOWN, b"engine is shutting down");
653            return Err(NetworkError::ShuttingDown);
654        }
655        debug!(peer = %peer_id, "incoming connection");
656        let request_timeout = self.config.request_timeout;
657
658        let (mut send, mut recv) = timeout(request_timeout, connection.accept_bi())
659            .await
660            .map_err(|_| NetworkError::Timeout)
661            .and_then(|res| {
662                res.map_err(|err| {
663                    NetworkError::Transport(format!("failed to accept handshake stream: {err}"))
664                })
665            })
666            .inspect_err(|_| {
667                connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"handshake failed");
668            })?;
669
670        let handshake: Result<Handshake> = timeout(
671            request_timeout,
672            wire::read_frame(&mut recv, MAX_HANDSHAKE_FRAME_SIZE),
673        )
674        .await
675        .map_err(|_| NetworkError::Timeout)
676        .and_then(|res| res);
677
678        let verdict = match &handshake {
679            Ok(handshake) => self.validate_handshake(handshake),
680            Err(_) => Some(HandshakeErrorCode::InvalidHandshake),
681        };
682
683        if let Some(code) = verdict {
684            warn!(peer = %peer_id, code = ?code, "handshake rejected");
685            let ack = HandshakeAck {
686                accepted: false,
687                error: Some(code),
688            };
689            let sent = timeout(
690                request_timeout,
691                wire::write_frame(&mut send, &ack, MAX_HANDSHAKE_FRAME_SIZE),
692            )
693            .await;
694            if matches!(sent, Ok(Ok(()))) {
695                let _ = send.finish();
696                // Keep the connection open briefly so QUIC delivers the ack;
697                // the initiator closes as soon as it has read the rejection.
698                let _ = timeout(REJECT_LINGER, connection.closed()).await;
699            }
700            connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"handshake rejected");
701            return Err(handshake_code_to_error(code));
702        }
703
704        let ack = HandshakeAck {
705            accepted: true,
706            error: None,
707        };
708        timeout(
709            request_timeout,
710            wire::write_frame(&mut send, &ack, MAX_HANDSHAKE_FRAME_SIZE),
711        )
712        .await
713        .map_err(|_| NetworkError::Timeout)
714        .and_then(|res| res)
715        .inspect_err(|_| {
716            connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"handshake failed");
717        })?;
718        let _ = send.finish();
719
720        let generation =
721            self.register_connection(peer_id, connection.clone(), ConnectionDirection::Incoming);
722        info!(peer = %peer_id, "peer connected (incoming)");
723        self.emit(NetworkEvent::PeerConnected {
724            peer_id,
725            direction: ConnectionDirection::Incoming,
726        })
727        .await;
728        // Drive the connection inside the router's accept task; it runs for
729        // as long as the connection lives.
730        self.clone()
731            .connection_loop(peer_id, connection.clone(), generation)
732            .await;
733        Ok(())
734    }
735
736    /// Handles a freshly accepted connection on an auxiliary stream ALPN:
737    /// validates the stream handshake and hands the byte stream over to the
738    /// application through the protocol's acceptor.
739    ///
740    /// Unlike message connections, stream connections are not registered in
741    /// the peer registry: their lifecycle belongs entirely to the returned
742    /// [`ByteStream`].
743    async fn handle_incoming_stream(&self, connection: &Connection, alpn: &[u8]) -> Result<()> {
744        let peer_id = connection.remote_id();
745        if self.is_shutting_down() {
746            connection.close(CLOSE_CODE_SHUTDOWN, b"engine is shutting down");
747            return Err(NetworkError::ShuttingDown);
748        }
749        debug!(peer = %peer_id, alpn = %alpn_display(alpn), "incoming byte stream");
750        let request_timeout = self.config.request_timeout;
751
752        let (mut send, mut recv) = timeout(request_timeout, connection.accept_bi())
753            .await
754            .map_err(|_| NetworkError::Timeout)
755            .and_then(|res| {
756                res.map_err(|err| {
757                    NetworkError::Transport(format!("failed to accept stream handshake: {err}"))
758                })
759            })
760            .inspect_err(|_| {
761                connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"stream handshake failed");
762            })?;
763
764        let handshake: Result<Handshake> = timeout(
765            request_timeout,
766            wire::read_frame(&mut recv, MAX_HANDSHAKE_FRAME_SIZE),
767        )
768        .await
769        .map_err(|_| NetworkError::Timeout)
770        .and_then(|res| res);
771
772        let verdict = match &handshake {
773            Ok(handshake) => self.validate_stream_handshake(alpn, handshake),
774            Err(_) => Some(HandshakeErrorCode::InvalidHandshake),
775        };
776        if let Some(code) = verdict {
777            warn!(peer = %peer_id, code = ?code, "stream handshake rejected");
778            let ack = HandshakeAck {
779                accepted: false,
780                error: Some(code),
781            };
782            let sent = timeout(
783                request_timeout,
784                wire::write_frame(&mut send, &ack, MAX_HANDSHAKE_FRAME_SIZE),
785            )
786            .await;
787            if matches!(sent, Ok(Ok(()))) {
788                let _ = send.finish();
789                let _ = timeout(REJECT_LINGER, connection.closed()).await;
790            }
791            connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"stream handshake rejected");
792            return Err(handshake_code_to_error(code));
793        }
794
795        // Reserve a queue slot before acking so a stream the application
796        // will never consume is refused instead of silently dying after an
797        // accepted handshake. A full queue delays the ack (backpressure).
798        let sender = lock(&self.stream_acceptors)
799            .get(alpn)
800            .map(|slot| slot.sender.clone())
801            .ok_or_else(|| NetworkError::UnknownStreamProtocol(alpn_display(alpn)))?;
802        let Ok(permit) = sender.reserve().await else {
803            connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"stream acceptor closed");
804            return Err(NetworkError::Transport(
805                "the stream acceptor was dropped by the application".to_string(),
806            ));
807        };
808
809        let ack = HandshakeAck {
810            accepted: true,
811            error: None,
812        };
813        timeout(
814            request_timeout,
815            wire::write_frame(&mut send, &ack, MAX_HANDSHAKE_FRAME_SIZE),
816        )
817        .await
818        .map_err(|_| NetworkError::Timeout)
819        .and_then(|res| res)
820        .inspect_err(|_| {
821            connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"stream handshake failed");
822        })?;
823        // The stream is deliberately not finished: the application payload
824        // follows on the same stream pair.
825
826        info!(peer = %peer_id, alpn = %alpn_display(alpn), "byte stream accepted");
827        permit.send(ByteStream {
828            peer_id,
829            send,
830            recv,
831            _connection: connection.clone(),
832        });
833        Ok(())
834    }
835
836    /// Dials `addr`, runs the client handshake and registers the connection.
837    ///
838    /// An existing healthy connection to the same peer is reused. The remote
839    /// identity is taken from the authenticated Iroh connection, never from
840    /// `addr` itself.
841    async fn connect_to_addr(self: &Arc<Self>, addr: EndpointAddr) -> Result<EndpointId> {
842        self.ensure_running()?;
843        let target = addr.id;
844        if lock(&self.peers).contains_key(&target) {
845            debug!(peer = %target, "reusing existing connection");
846            return Ok(target);
847        }
848
849        let request_timeout = self.config.request_timeout;
850        let connection = timeout(request_timeout, self.endpoint.connect(addr, ALPN))
851            .await
852            .map_err(|_| NetworkError::Timeout)?
853            .map_err(|err| NetworkError::Transport(format!("failed to connect: {err}")))?;
854        let peer_id = connection.remote_id();
855
856        let handshake = timeout(request_timeout, self.run_client_handshake(&connection))
857            .await
858            .map_err(|_| NetworkError::Timeout)
859            .and_then(|res| res);
860        if let Err(err) = handshake {
861            connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"handshake failed");
862            return Err(err);
863        }
864
865        let generation =
866            self.register_connection(peer_id, connection.clone(), ConnectionDirection::Outgoing);
867        info!(peer = %peer_id, "peer connected (outgoing)");
868        self.emit(NetworkEvent::PeerConnected {
869            peer_id,
870            direction: ConnectionDirection::Outgoing,
871        })
872        .await;
873        let loop_shared = self.clone();
874        self.spawn_task(async move {
875            loop_shared
876                .connection_loop(peer_id, connection, generation)
877                .await;
878        });
879        Ok(peer_id)
880    }
881
882    /// Runs the client side of the handshake on a fresh outgoing connection.
883    async fn run_client_handshake(&self, connection: &Connection) -> Result<()> {
884        let (mut send, mut recv) = connection.open_bi().await.map_err(|err| {
885            NetworkError::Transport(format!("failed to open handshake stream: {err}"))
886        })?;
887        let handshake = Handshake {
888            protocol_version: PROTOCOL_VERSION,
889            network_id: self.config.network_id,
890            schema_id: self.config.schema_id,
891        };
892        wire::write_frame(&mut send, &handshake, MAX_HANDSHAKE_FRAME_SIZE).await?;
893        let _ = send.finish();
894        let ack: HandshakeAck = wire::read_frame(&mut recv, MAX_HANDSHAKE_FRAME_SIZE).await?;
895        if ack.accepted {
896            Ok(())
897        } else {
898            let code = ack.error.unwrap_or(HandshakeErrorCode::InvalidHandshake);
899            Err(handshake_code_to_error(code))
900        }
901    }
902}
903
904/// Periodically publishes this peer to the network's rendezvous record and
905/// dials every other peer found there.
906///
907/// Failures of a single round or dial are logged and retried on the next
908/// round; the loop only ends when the engine shuts down (the task is
909/// aborted).
910async fn rendezvous_loop<M: Message>(shared: Arc<Shared<M>>, config: RendezvousConfig) {
911    let mut shutdown = shared.shutdown_signal.subscribe();
912    // Give the endpoint a moment to learn its relay and direct addresses so
913    // the very first published record is already dialable.
914    let _ = timeout(shared.config.request_timeout, shared.endpoint.online()).await;
915    let self_id = shared.endpoint.id();
916    let mut client = None;
917    loop {
918        if shared.is_shutting_down() {
919            return;
920        }
921        if client.is_none() {
922            match RendezvousClient::new(shared.config.network_id, &config) {
923                Ok(new_client) => client = Some(new_client),
924                Err(err) => {
925                    let failures = shared.record_rendezvous_failure(err.to_string());
926                    warn!(error = %err, failures, "failed to start peer rendezvous; retrying");
927                }
928            }
929        }
930        let round = async {
931            let Some(active_client) = client.as_ref() else {
932                return;
933            };
934            let addr = shared.endpoint.addr();
935            let self_addr = (!addr.is_empty()).then_some(addr);
936            if self_addr.is_none() {
937                let failures = shared.record_rendezvous_failure(
938                    "local endpoint has no dialable address".to_string(),
939                );
940                debug!(failures, "rendezvous waiting for a dialable local address");
941                return;
942            }
943            match timeout(
944                RENDEZVOUS_ROUND_TIMEOUT,
945                active_client.round(self_addr, now_ms()),
946            )
947            .await
948            {
949                Ok(Ok(round)) => {
950                    let candidates = round.peers.len();
951                    let mut attempts = 0;
952                    let mut successes = 0;
953                    for peer_addr in round.peers {
954                        let peer = peer_addr.id;
955                        if peer == self_id || lock(&shared.peers).contains_key(&peer) {
956                            continue;
957                        }
958                        debug!(peer = %peer, "rendezvous discovered a peer; connecting");
959                        attempts += 1;
960                        match shared.connect_to_addr(peer_addr).await {
961                            Ok(_) => successes += 1,
962                            Err(err) => {
963                                // Stale entries (peers that left) fail here;
964                                // they age out of the record by TTL.
965                                debug!(peer = %peer, error = %err, "rendezvous connect attempt failed");
966                            }
967                        }
968                    }
969                    if let Some(error) = round.publish_error {
970                        let failures = shared.record_rendezvous_failure(error.clone());
971                        debug!(%error, failures, "rendezvous publish failed");
972                    } else {
973                        shared.record_rendezvous_success();
974                    }
975                    shared.record_rendezvous_peer_dials(candidates, attempts, successes);
976                }
977                Ok(Err(err)) => {
978                    let failures = shared.record_rendezvous_failure(err.to_string());
979                    debug!(error = %err, failures, "rendezvous round failed");
980                }
981                Err(_) => {
982                    let failures = shared.record_rendezvous_failure(format!(
983                        "rendezvous round timed out after {}s",
984                        RENDEZVOUS_ROUND_TIMEOUT.as_secs()
985                    ));
986                    warn!(failures, "rendezvous round timed out");
987                }
988            }
989            let failures = lock(&shared.rendezvous_health)
990                .as_ref()
991                .map_or(0, |health| health.consecutive_failures);
992            if failures >= RENDEZVOUS_REBUILD_AFTER_FAILURES {
993                client = None;
994                shared.record_rendezvous_restart();
995                info!(failures, "rebuilding the rendezvous client");
996            }
997        };
998        tokio::select! {
999            _ = round => {}
1000            _ = shutdown.changed() => return,
1001        }
1002        // Poll faster while the peer is still alone: joining a network
1003        // should not have to wait a full interval for the next round.
1004        let delay = if lock(&shared.peers).is_empty() {
1005            config.interval.min(RENDEZVOUS_LONELY_INTERVAL)
1006        } else {
1007            config.interval
1008        };
1009        tokio::select! {
1010            _ = tokio::time::sleep(delay) => {}
1011            _ = shutdown.changed() => return,
1012        }
1013    }
1014}
1015
1016fn handshake_code_to_error(code: HandshakeErrorCode) -> NetworkError {
1017    match code {
1018        HandshakeErrorCode::UnsupportedProtocolVersion => NetworkError::UnsupportedProtocolVersion,
1019        HandshakeErrorCode::NetworkMismatch => NetworkError::NetworkMismatch,
1020        HandshakeErrorCode::SchemaMismatch => NetworkError::SchemaMismatch,
1021        HandshakeErrorCode::InvalidHandshake => NetworkError::HandshakeRejected,
1022    }
1023}
1024
1025/// Protocol handler registered with the Iroh [`Router`] for [`ALPN`].
1026///
1027/// Holds only a weak reference to the shared state so the router does not
1028/// keep the engine alive after all engine handles are dropped.
1029struct FederationProtocol<M> {
1030    shared: Weak<Shared<M>>,
1031}
1032
1033impl<M> fmt::Debug for FederationProtocol<M> {
1034    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1035        f.debug_struct("FederationProtocol").finish_non_exhaustive()
1036    }
1037}
1038
1039impl<M: Message> ProtocolHandler for FederationProtocol<M> {
1040    async fn accept(&self, connection: Connection) -> Result<(), AcceptError> {
1041        let Some(shared) = self.shared.upgrade() else {
1042            connection.close(CLOSE_CODE_SHUTDOWN, b"engine is gone");
1043            return Ok(());
1044        };
1045        let peer_id = connection.remote_id();
1046        if let Err(err) = shared.handle_incoming(&connection).await {
1047            shared
1048                .report_protocol_error(Some(peer_id), format!("incoming connection failed: {err}"))
1049                .await;
1050        }
1051        // Errors of a single connection never bring down the router; the
1052        // connection has already been closed with an appropriate code.
1053        Ok(())
1054    }
1055}
1056
1057/// Protocol handler registered with the Iroh [`Router`] for one auxiliary
1058/// stream ALPN declared in the configuration.
1059struct StreamProtocol<M> {
1060    shared: Weak<Shared<M>>,
1061    alpn: Vec<u8>,
1062}
1063
1064impl<M> fmt::Debug for StreamProtocol<M> {
1065    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1066        f.debug_struct("StreamProtocol")
1067            .field("alpn", &alpn_display(&self.alpn))
1068            .finish_non_exhaustive()
1069    }
1070}
1071
1072impl<M: Message> ProtocolHandler for StreamProtocol<M> {
1073    async fn accept(&self, connection: Connection) -> Result<(), AcceptError> {
1074        let Some(shared) = self.shared.upgrade() else {
1075            connection.close(CLOSE_CODE_SHUTDOWN, b"engine is gone");
1076            return Ok(());
1077        };
1078        let peer_id = connection.remote_id();
1079        if let Err(err) = shared.handle_incoming_stream(&connection, &self.alpn).await {
1080            shared
1081                .report_protocol_error(
1082                    Some(peer_id),
1083                    format!(
1084                        "incoming byte stream on {} failed: {err}",
1085                        alpn_display(&self.alpn)
1086                    ),
1087                )
1088                .await;
1089        }
1090        Ok(())
1091    }
1092}
1093
1094/// A generic peer-to-peer network engine on top of Iroh.
1095///
1096/// `M` is the application-defined domain message type; the engine treats it
1097/// as an opaque, postcard-serializable payload. The engine is cheaply
1098/// clonable: all clones share the same endpoint and connection registry.
1099pub struct NetworkEngine<M> {
1100    shared: Arc<Shared<M>>,
1101}
1102
1103impl<M> Clone for NetworkEngine<M> {
1104    fn clone(&self) -> Self {
1105        Self {
1106            shared: self.shared.clone(),
1107        }
1108    }
1109}
1110
1111impl<M> fmt::Debug for NetworkEngine<M> {
1112    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1113        f.debug_struct("NetworkEngine").finish_non_exhaustive()
1114    }
1115}
1116
1117impl<M: Message> NetworkEngine<M> {
1118    /// Starts the engine: loads (or creates) the persistent identity, binds
1119    /// the Iroh endpoint and starts accepting connections.
1120    ///
1121    /// Returns the engine handle together with the single receiver for
1122    /// [`NetworkEvent`]s.
1123    pub async fn start(config: NetworkConfig) -> Result<(Self, NetworkEventReceiver<M>)> {
1124        let secret_key = identity::load_or_create(&config.data_dir).await?;
1125        Self::start_with_secret_key(config, secret_key).await
1126    }
1127
1128    /// Starts the engine with an application-provided identity key.
1129    ///
1130    /// Use this when the caller already stores the peer identity in durable
1131    /// application storage. The regular [`NetworkEngine::start`] method
1132    /// remains the file-backed default.
1133    pub async fn start_with_secret_key(
1134        config: NetworkConfig,
1135        secret_key: SecretKey,
1136    ) -> Result<(Self, NetworkEventReceiver<M>)> {
1137        let rendezvous_enabled = config.rendezvous.is_some();
1138        let endpoint = Endpoint::builder(presets::N0)
1139            .secret_key(secret_key)
1140            .bind()
1141            .await
1142            .map_err(|err| NetworkError::Transport(format!("failed to bind endpoint: {err}")))?;
1143        let endpoint_id = endpoint.id();
1144        let (sender, receiver) = mpsc::channel(config.event_channel_capacity);
1145        let stream_acceptors = config
1146            .stream_protocols
1147            .iter()
1148            .chain(config.schema_independent_stream_protocols.iter())
1149            .map(|alpn| {
1150                let (sender, receiver) = mpsc::channel(STREAM_ACCEPT_QUEUE);
1151                (
1152                    alpn.clone(),
1153                    StreamAcceptorSlot {
1154                        sender,
1155                        receiver: Some(receiver),
1156                    },
1157                )
1158            })
1159            .collect();
1160        let shared = Arc::new(Shared {
1161            config,
1162            endpoint: endpoint.clone(),
1163            router: Mutex::new(None),
1164            peers: Mutex::new(HashMap::new()),
1165            stream_acceptors: Mutex::new(stream_acceptors),
1166            events: Mutex::new(Some(sender)),
1167            tasks: Mutex::new(JoinSet::new()),
1168            next_generation: AtomicU64::new(0),
1169            shutting_down: AtomicBool::new(false),
1170            rendezvous_health: Mutex::new(rendezvous_enabled.then(RendezvousHealth::new)),
1171            shutdown_signal: watch::Sender::new(false),
1172        });
1173        let handler = FederationProtocol {
1174            shared: Arc::downgrade(&shared),
1175        };
1176        let mut router_builder = Router::builder(endpoint).accept(ALPN, handler);
1177        for alpn in &shared.config.stream_protocols {
1178            let handler = StreamProtocol::<M> {
1179                shared: Arc::downgrade(&shared),
1180                alpn: alpn.clone(),
1181            };
1182            router_builder = router_builder.accept(alpn.clone(), handler);
1183        }
1184        for alpn in &shared.config.schema_independent_stream_protocols {
1185            let handler = StreamProtocol::<M> {
1186                shared: Arc::downgrade(&shared),
1187                alpn: alpn.clone(),
1188            };
1189            router_builder = router_builder.accept(alpn.clone(), handler);
1190        }
1191        let router = router_builder.spawn();
1192        *lock(&shared.router) = Some(router);
1193        if let Some(rendezvous) = shared.config.rendezvous.clone() {
1194            let loop_shared = shared.clone();
1195            shared.spawn_task(async move {
1196                rendezvous_loop(loop_shared, rendezvous).await;
1197            });
1198        }
1199        info!(
1200            endpoint_id = %endpoint_id,
1201            network_id = %shared.config.network_id,
1202            schema_id = %shared.config.schema_id,
1203            "network engine started"
1204        );
1205        Ok((Self { shared }, NetworkEventReceiver::new(receiver)))
1206    }
1207
1208    /// Returns the stable identifier of this peer.
1209    pub fn endpoint_id(&self) -> EndpointId {
1210        self.shared.endpoint.id()
1211    }
1212
1213    /// Signs application data with this peer's persistent transport identity.
1214    ///
1215    /// The secret key never leaves the engine. Protocols can attach the
1216    /// returned signature to records that may be forwarded by other peers;
1217    /// recipients verify it against [`Self::endpoint_id`].
1218    pub fn sign_identity(&self, message: &[u8]) -> Signature {
1219        self.shared.endpoint.secret_key().sign(message)
1220    }
1221
1222    /// Returns the network this engine participates in.
1223    pub fn network_id(&self) -> NetworkId {
1224        self.shared.config.network_id
1225    }
1226
1227    /// Returns the message schema this engine uses.
1228    pub fn schema_id(&self) -> SchemaId {
1229        self.shared.config.schema_id
1230    }
1231
1232    /// Creates a shareable [`PeerTicket`] for this peer.
1233    ///
1234    /// Waits (bounded by the configured request timeout) for the endpoint to
1235    /// come online so the ticket contains relay information whenever
1236    /// possible; on an isolated network the ticket falls back to the locally
1237    /// known direct addresses.
1238    pub async fn ticket(&self) -> Result<PeerTicket> {
1239        self.shared.ensure_running()?;
1240        let endpoint = &self.shared.endpoint;
1241        let has_relay = endpoint.addr().relay_urls().next().is_some();
1242        if !has_relay {
1243            let _ = timeout(self.shared.config.request_timeout, endpoint.online()).await;
1244        }
1245        let endpoint_addr = endpoint.addr();
1246        if endpoint_addr.is_empty() {
1247            return Err(NetworkError::Transport(
1248                "endpoint has no reachable addresses yet".to_string(),
1249            ));
1250        }
1251        Ok(PeerTicket {
1252            ticket_version: TICKET_VERSION,
1253            protocol_version: PROTOCOL_VERSION,
1254            network_id: self.shared.config.network_id,
1255            schema_id: self.shared.config.schema_id,
1256            endpoint_addr,
1257        })
1258    }
1259
1260    /// Connects to the peer described by `ticket` and performs the handshake.
1261    ///
1262    /// Incompatible tickets (different network, schema or protocol version)
1263    /// are rejected before any connection is attempted. An existing healthy
1264    /// connection to the same peer is reused.
1265    pub async fn connect(&self, ticket: PeerTicket) -> Result<EndpointId> {
1266        let shared = &self.shared;
1267        shared.ensure_running()?;
1268        if ticket.ticket_version != TICKET_VERSION {
1269            return Err(NetworkError::InvalidTicket(format!(
1270                "unsupported ticket version {}",
1271                ticket.ticket_version
1272            )));
1273        }
1274        if ticket.protocol_version != PROTOCOL_VERSION {
1275            return Err(NetworkError::UnsupportedProtocolVersion);
1276        }
1277        if ticket.network_id != shared.config.network_id {
1278            return Err(NetworkError::NetworkMismatch);
1279        }
1280        if ticket.schema_id != shared.config.schema_id {
1281            return Err(NetworkError::SchemaMismatch);
1282        }
1283        if ticket.endpoint_id() == self.endpoint_id() {
1284            return Err(NetworkError::InvalidTicket(
1285                "the ticket points to this peer itself".to_string(),
1286            ));
1287        }
1288        shared.connect_to_addr(ticket.endpoint_addr).await
1289    }
1290
1291    /// Sends a domain message to a connected peer.
1292    ///
1293    /// Returns only after the peer acknowledged the message. Fails with
1294    /// [`NetworkError::PeerNotConnected`] if there is no active connection to
1295    /// the peer; no implicit dialing or peer lookup is performed.
1296    pub async fn send(&self, peer: EndpointId, message: &M) -> Result<()> {
1297        let shared = &self.shared;
1298        shared.ensure_running()?;
1299        let connection = lock(&shared.peers)
1300            .get(&peer)
1301            .map(|state| state.connection.clone())
1302            .ok_or(NetworkError::PeerNotConnected(peer))?;
1303        let payload = wire::encode(message, shared.config.max_message_size)?;
1304        let request_id: [u8; 16] = rand::random();
1305        let request = MessageRequest {
1306            request_id,
1307            payload,
1308        };
1309        let max_frame = shared.config.max_message_size + MESSAGE_FRAME_OVERHEAD;
1310        timeout(shared.config.request_timeout, async move {
1311            let (mut send, mut recv) = connection.open_bi().await.map_err(|err| {
1312                NetworkError::Transport(format!("failed to open message stream: {err}"))
1313            })?;
1314            wire::write_frame(&mut send, &request, max_frame).await?;
1315            let _ = send.finish();
1316            let response: MessageResponse =
1317                wire::read_frame(&mut recv, MAX_RESPONSE_FRAME_SIZE).await?;
1318            match response {
1319                MessageResponse::Accepted { request_id: id } if id == request_id => Ok(()),
1320                MessageResponse::Rejected {
1321                    request_id: id,
1322                    reason,
1323                } if id == request_id => Err(NetworkError::MessageRejected(reason.to_string())),
1324                _ => Err(NetworkError::Transport(
1325                    "peer responded to a different request".to_string(),
1326                )),
1327            }
1328        })
1329        .await
1330        .map_err(|_| NetworkError::Timeout)?
1331    }
1332
1333    /// Opens a raw bidirectional byte stream to a peer over a dedicated
1334    /// connection on the auxiliary ALPN `alpn`.
1335    ///
1336    /// The remote peer must have declared `alpn` in its configuration
1337    /// ([`crate::NetworkConfigBuilder::stream_protocol`]) and hold on to the
1338    /// matching [`StreamAcceptor`]. The stream begins with the regular
1339    /// network handshake, so it can only be established between peers of the
1340    /// same network and schema; afterwards the stream carries opaque
1341    /// application bytes with no framing imposed by the engine.
1342    ///
1343    /// `target` can be a full [`EndpointAddr`] (e.g. from a ticket) or a
1344    /// bare [`EndpointId`] when the transport already knows how to reach the
1345    /// peer (an active or recent connection, or discovery).
1346    pub async fn open_stream(
1347        &self,
1348        target: impl Into<EndpointAddr>,
1349        alpn: &[u8],
1350    ) -> Result<ByteStream> {
1351        let shared = &self.shared;
1352        shared.ensure_running()?;
1353        let addr: EndpointAddr = target.into();
1354        let request_timeout = shared.config.request_timeout;
1355        let connection = timeout(request_timeout, shared.endpoint.connect(addr, alpn))
1356            .await
1357            .map_err(|_| NetworkError::Timeout)?
1358            .map_err(|err| {
1359                NetworkError::Transport(format!("failed to connect for a byte stream: {err}"))
1360            })?;
1361        let peer_id = connection.remote_id();
1362        let handshake = Handshake {
1363            protocol_version: PROTOCOL_VERSION,
1364            network_id: shared.config.network_id,
1365            schema_id: shared.config.schema_id,
1366        };
1367        let result = timeout(request_timeout, async {
1368            let (mut send, mut recv) = connection.open_bi().await.map_err(|err| {
1369                NetworkError::Transport(format!("failed to open the byte stream: {err}"))
1370            })?;
1371            wire::write_frame(&mut send, &handshake, MAX_HANDSHAKE_FRAME_SIZE).await?;
1372            // The stream is deliberately not finished: application payload
1373            // follows the handshake on the same stream.
1374            let ack: HandshakeAck = wire::read_frame(&mut recv, MAX_HANDSHAKE_FRAME_SIZE).await?;
1375            if ack.accepted {
1376                Ok((send, recv))
1377            } else {
1378                let code = ack.error.unwrap_or(HandshakeErrorCode::InvalidHandshake);
1379                Err(handshake_code_to_error(code))
1380            }
1381        })
1382        .await
1383        .map_err(|_| NetworkError::Timeout)
1384        .and_then(|res| res);
1385        match result {
1386            Ok((send, recv)) => {
1387                debug!(peer = %peer_id, alpn = %alpn_display(alpn), "byte stream opened");
1388                Ok(ByteStream {
1389                    peer_id,
1390                    send,
1391                    recv,
1392                    _connection: connection,
1393                })
1394            }
1395            Err(err) => {
1396                connection.close(CLOSE_CODE_HANDSHAKE_REJECTED, b"stream handshake failed");
1397                Err(err)
1398            }
1399        }
1400    }
1401
1402    /// Takes the acceptor of incoming byte streams for the declared stream
1403    /// protocol `alpn`.
1404    ///
1405    /// The acceptor for a protocol can be taken exactly once. Streams whose
1406    /// handshake succeeded are delivered in accept order; the queue is small
1407    /// and bounded, so an unconsumed acceptor applies backpressure to
1408    /// remote peers.
1409    pub fn stream_acceptor(&self, alpn: &[u8]) -> Result<StreamAcceptor> {
1410        let mut acceptors = lock(&self.shared.stream_acceptors);
1411        let slot = acceptors
1412            .get_mut(alpn)
1413            .ok_or_else(|| NetworkError::UnknownStreamProtocol(alpn_display(alpn)))?;
1414        let rx = slot
1415            .receiver
1416            .take()
1417            .ok_or_else(|| NetworkError::StreamAcceptorTaken(alpn_display(alpn)))?;
1418        Ok(StreamAcceptor {
1419            alpn: alpn.to_vec(),
1420            rx,
1421        })
1422    }
1423
1424    /// Returns the ids of all currently connected peers.
1425    pub fn connected_peers(&self) -> Vec<EndpointId> {
1426        lock(&self.shared.peers).keys().copied().collect()
1427    }
1428
1429    /// Returns a point-in-time snapshot of transport and rendezvous health.
1430    pub fn health(&self) -> NetworkHealthSnapshot {
1431        self.shared.health()
1432    }
1433
1434    /// Returns `true` if there is an active connection to `peer`.
1435    pub fn is_connected(&self, peer: EndpointId) -> bool {
1436        lock(&self.shared.peers).contains_key(&peer)
1437    }
1438
1439    /// Closes the connection to `peer` and removes it from the registry.
1440    pub async fn disconnect(&self, peer: EndpointId) -> Result<()> {
1441        let state = lock(&self.shared.peers)
1442            .remove(&peer)
1443            .ok_or(NetworkError::PeerNotConnected(peer))?;
1444        state
1445            .connection
1446            .close(CLOSE_CODE_OK, b"disconnected by local peer");
1447        info!(peer = %peer, "peer disconnected (local request)");
1448        self.shared
1449            .emit(NetworkEvent::PeerDisconnected {
1450                peer_id: peer,
1451                reason: Some("disconnected by local peer".to_string()),
1452            })
1453            .await;
1454        Ok(())
1455    }
1456
1457    /// Shuts the engine down gracefully.
1458    ///
1459    /// Rejects new operations, closes all connections, stops the protocol
1460    /// router, closes the endpoint, waits for background tasks and finally
1461    /// closes the event channel. Calling `shutdown` on several clones of the
1462    /// same engine is safe; only the first call does the work.
1463    pub async fn shutdown(self) -> Result<()> {
1464        let shared = &self.shared;
1465        if shared.shutting_down.swap(true, Ordering::SeqCst) {
1466            return Ok(());
1467        }
1468        info!("network engine shutting down");
1469        let _ = shared.shutdown_signal.send(true);
1470
1471        // Close all active connections.
1472        let states: Vec<PeerState> = lock(&shared.peers)
1473            .drain()
1474            .map(|(_, state)| state)
1475            .collect();
1476        for state in &states {
1477            state
1478                .connection
1479                .close(CLOSE_CODE_SHUTDOWN, b"engine is shutting down");
1480        }
1481        drop(states);
1482
1483        // Shut down the router; this stops the protocol handler and closes
1484        // the endpoint.
1485        let router = lock(&shared.router).take();
1486        if let Some(router) = router
1487            && let Err(err) = router.shutdown().await
1488        {
1489            warn!(error = %err, "router shutdown reported an error");
1490        }
1491        shared.endpoint.close().await;
1492
1493        // Wait for the remaining background tasks; abort stragglers.
1494        let mut tasks = std::mem::take(&mut *lock(&shared.tasks));
1495        let drained = timeout(SHUTDOWN_TASK_GRACE, async {
1496            while tasks.join_next().await.is_some() {}
1497        })
1498        .await;
1499        if drained.is_err() {
1500            warn!("background tasks did not finish in time; aborting them");
1501            tasks.abort_all();
1502        }
1503
1504        // Close the event channel: dropping the last sender makes
1505        // `NetworkEventReceiver::recv` return `None` once drained.
1506        *lock(&shared.events) = None;
1507        info!("network engine shut down");
1508        Ok(())
1509    }
1510}