Skip to main content

openrtc/
native_node.rs

1#![cfg(not(target_arch = "wasm32"))]
2
3use anyhow::Result;
4use futures::StreamExt;
5use iroh::{
6    endpoint::{Connection, RecvStream, SendStream},
7    protocol::{AcceptError, ProtocolHandler, Router},
8    Endpoint, EndpointAddr, EndpointId, Watcher,
9};
10use serde::{Deserialize, Serialize};
11use std::collections::{HashMap, HashSet};
12use std::sync::{Arc, Mutex as StdMutex, Weak};
13use std::time::Instant;
14use tokio::sync::{broadcast, mpsc, oneshot, RwLock};
15use tokio_stream::wrappers::BroadcastStream;
16
17use crate::heartbeat::{
18    classify_incoming_uni, parse_incoming_pong, respond_to_ping, HealthTransition, HeartbeatConfig,
19    IncomingUniClassification, IrohConnectionProbe, IrohHeartbeatManager, IrohProbeRegistry,
20    PrefixedRecvStream,
21};
22use crate::iroh_connection_policy::{
23    decide_inbound_install, decide_outbound_install, should_start_outbound_dial,
24    should_wait_for_canonical_inbound, ExistingConnectionState, IrohConnectionDirection,
25    IrohConnectionInstallDecision, NONCANONICAL_OUTBOUND_DIAL_GRACE_MS,
26};
27
28/// Native Iroh endpoint policy shared by OpenRTC's primary endpoint and any
29/// separately-built custom-transport endpoint. Keeping this configuration in
30/// the lifecycle owner prevents a BLE/LAN adapter from inheriting Iroh's short
31/// default idle timeout and silently dropping an otherwise healthy route.
32pub fn native_iroh_transport_config() -> iroh::endpoint::QuicTransportConfig {
33    use iroh::endpoint::VarInt;
34
35    let idle_timeout = std::time::Duration::from_secs(120)
36        .try_into()
37        .expect("120 seconds is a valid Iroh idle timeout");
38    let keep_alive_secs = if cfg!(any(target_os = "ios", target_os = "android")) {
39        90
40    } else {
41        25
42    };
43
44    iroh::endpoint::QuicTransportConfig::builder()
45        .max_idle_timeout(Some(idle_timeout))
46        .keep_alive_interval(std::time::Duration::from_secs(keep_alive_secs))
47        .stream_receive_window(VarInt::from_u32(8 * 1024 * 1024))
48        .receive_window(VarInt::from_u32(16 * 1024 * 1024))
49        .send_window(16 * 1024 * 1024)
50        .build()
51}
52
53const INSTALL_COLLISION_PROBE_TIMEOUT: std::time::Duration =
54    std::time::Duration::from_millis(1_000);
55
56pub(crate) fn transport_generation_for_connection(connection: &Connection) -> u64 {
57    crate::transport_generation::for_connection(connection)
58}
59
60fn forget_transport_generation(connection: &Connection) {
61    crate::transport_generation::forget_connection(connection);
62}
63
64async fn existing_transport_is_live_for_install(
65    endpoint_id: EndpointId,
66    previous: &Connection,
67    fresh_stable_id: u64,
68    fresh_direction: IrohConnectionDirection,
69    probe_registry: &IrohProbeRegistry,
70) -> bool {
71    let previous_stable_id = transport_generation_for_connection(previous);
72    if previous_stable_id == fresh_stable_id {
73        return previous.close_reason().is_none();
74    }
75    if previous.close_reason().is_some() {
76        return false;
77    }
78
79    let responsive = probe_registry
80        .probe(
81            &endpoint_id.to_string(),
82            previous_stable_id,
83            previous,
84            INSTALL_COLLISION_PROBE_TIMEOUT,
85        )
86        .await;
87    if !responsive {
88        let reason = match fresh_direction {
89            IrohConnectionDirection::Inbound => {
90                crate::lifecycle_reason::REASON_ZOMBIE_REPLACED_BY_FRESH_ACCEPT
91            }
92            IrohConnectionDirection::Outbound => {
93                crate::lifecycle_reason::REASON_ZOMBIE_REPLACED_BY_FRESH_DIAL
94            }
95        };
96        println!(
97            "[OpenRTC][iroh-install] replacing unresponsive transport endpoint_id={} previous_stable_id={} fresh_stable_id={} fresh_direction={:?}",
98            endpoint_id,
99            previous_stable_id,
100            fresh_stable_id,
101            fresh_direction,
102        );
103        previous.close(0u8.into(), reason.as_bytes());
104    }
105    responsive
106}
107
108async fn should_break_accept_loop(connection: &Connection) -> bool {
109    if connection.close_reason().is_some() {
110        return true;
111    }
112    tokio::time::sleep(std::time::Duration::from_millis(25)).await;
113    connection.close_reason().is_some()
114}
115
116fn local_prefers_outbound(local_endpoint_id: EndpointId, remote_endpoint_id: EndpointId) -> bool {
117    local_endpoint_id.to_string() > remote_endpoint_id.to_string()
118}
119
120async fn outbound_dial_precheck(
121    endpoint: &Endpoint,
122    endpoint_id: EndpointId,
123    connections: &Arc<RwLock<HashMap<EndpointId, Connection>>>,
124    connection_inserted_at: &Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
125) -> bool {
126    let prefers_outbound = local_prefers_outbound(endpoint.id(), endpoint_id);
127    let existing = {
128        let conns = connections.read().await;
129        if let Some(existing) = conns.get(&endpoint_id) {
130            let direction = connection_inserted_at
131                .read()
132                .await
133                .get(&endpoint_id)
134                .map(|metadata| metadata.direction)
135                .unwrap_or(IrohConnectionDirection::Outbound);
136            Some((
137                existing.close_reason().is_none(),
138                direction,
139                selected_custom_transport_id(existing),
140            ))
141        } else {
142            None
143        }
144    };
145    let existing_state = existing.map(|(alive, direction, _)| (alive, direction));
146    let existing_custom_transport_id =
147        existing.and_then(|(_, _, custom_transport_id)| custom_transport_id);
148
149    if !should_start_outbound_dial_with_transport_preference(
150        existing_state,
151        existing_custom_transport_id,
152        prefers_outbound,
153    ) {
154        return true;
155    }
156    if !should_wait_for_canonical_inbound(existing_state, prefers_outbound) {
157        return false;
158    }
159
160    tokio::time::sleep(std::time::Duration::from_millis(
161        NONCANONICAL_OUTBOUND_DIAL_GRACE_MS,
162    ))
163    .await;
164    connections
165        .read()
166        .await
167        .get(&endpoint_id)
168        .is_some_and(|connection| connection.close_reason().is_none())
169}
170
171fn manual_disconnect_notice_key(endpoint_id: EndpointId, transport_stable_id: u64) -> String {
172    format!("{endpoint_id}:{transport_stable_id}")
173}
174
175fn heartbeat_connection_key(endpoint_id: EndpointId, transport_stable_id: u64) -> String {
176    format!("{endpoint_id}:{transport_stable_id}")
177}
178
179#[derive(Debug, Clone, Copy)]
180struct IrohConnectionMetadata {
181    inserted_at: Instant,
182    direction: IrohConnectionDirection,
183}
184
185#[derive(Debug)]
186pub enum IncomingStreamType {
187    Bi(SendStream, RecvStream),
188    Uni(PrefixedRecvStream),
189}
190
191#[derive(Debug)]
192pub struct IncomingStream {
193    pub endpoint_id: EndpointId,
194    /// Physical Iroh connection generation that accepted this stream. This is
195    /// distinct from the logical peer/session id and must follow the stream
196    /// through admission so stale accept loops cannot replace current owners.
197    pub transport_stable_id: u64,
198    /// Wire bytes inspected by the Rust admission router and preserved for the
199    /// eventual application-crypto decoder. Empty for ordinary streams.
200    pub recv_prefix: Vec<u8>,
201    pub stream: IncomingStreamType,
202}
203
204#[derive(Debug, Clone, Serialize, Deserialize)]
205#[serde(tag = "type", rename_all = "camelCase")]
206pub enum ConnectEvent {
207    Connected,
208    Closed { error: Option<String> },
209}
210
211#[derive(Debug, Clone, Serialize, Deserialize)]
212#[serde(tag = "type", rename_all = "camelCase")]
213pub enum ReplacementConnectEvent {
214    Connected { transport_stable_id: u64 },
215    Closed { error: Option<String> },
216}
217
218#[derive(Debug, Clone, Serialize, Deserialize)]
219#[serde(tag = "type", rename_all = "camelCase")]
220pub enum AcceptEvent {
221    Accepted {
222        endpoint_id: EndpointId,
223        transport_stable_id: u64,
224        replaced_transport_stable_id: Option<u64>,
225    },
226    Closed {
227        endpoint_id: EndpointId,
228        transport_stable_id: u64,
229        error: Option<String>,
230        /// True when this runtime dialed the physical leg. External-router
231        /// integrations still need OpenRTC to reconcile closes for outbound
232        /// legs, while their host-owned inbound handler performs its own
233        /// exact-generation reconciliation.
234        was_outbound: bool,
235        /// True when the local side initiated the close (e.g. `disconnect()`
236        /// after auth rejection). Mirrors the wasm32 variant so consumers can
237        /// short-circuit replacement-wait logic that only makes sense for
238        /// peer-initiated or transport-failure closes.
239        was_locally_closed: bool,
240    },
241}
242
243/// Immediate result of the native node's atomic physical-connection arbitration.
244/// Higher lifecycle layers must only promote `Installed`; `KeptExisting` means
245/// the fresh connection lost and the existing stable ID remains authoritative.
246#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247pub(crate) enum ExternalConnectionInstallOutcome {
248    Installed {
249        transport_stable_id: u64,
250        replaced_transport_stable_id: Option<u64>,
251    },
252    KeptExisting {
253        fresh_transport_stable_id: u64,
254        kept_transport_stable_id: u64,
255    },
256}
257
258#[derive(Debug, Clone, Copy)]
259struct InboundReplacementAuthorization {
260    transport_id: u64,
261    expires_at: Instant,
262}
263
264type InboundReplacementAuthorizations =
265    Arc<RwLock<HashMap<EndpointId, InboundReplacementAuthorization>>>;
266
267type OutboundDialGate = tokio::sync::Mutex<()>;
268type OutboundDialGates = Arc<StdMutex<HashMap<EndpointId, Weak<OutboundDialGate>>>>;
269
270struct OutboundDialGateLease {
271    endpoint_id: EndpointId,
272    gate: Arc<OutboundDialGate>,
273    gates: OutboundDialGates,
274}
275
276impl Drop for OutboundDialGateLease {
277    fn drop(&mut self) {
278        let mut gates = self
279            .gates
280            .lock()
281            .unwrap_or_else(|poisoned| poisoned.into_inner());
282        let is_current = gates
283            .get(&self.endpoint_id)
284            .is_some_and(|current| current.ptr_eq(&Arc::downgrade(&self.gate)));
285        // The registry holds a Weak reference, so one strong reference means
286        // this is the last active/waiting lease. Remove its key immediately;
287        // cancelled dials follow the same Drop path.
288        if is_current && Arc::strong_count(&self.gate) == 1 {
289            gates.remove(&self.endpoint_id);
290        }
291    }
292}
293
294fn outbound_dial_gate(
295    endpoint_id: EndpointId,
296    registry: &OutboundDialGates,
297) -> OutboundDialGateLease {
298    let mut gates = registry
299        .lock()
300        .unwrap_or_else(|poisoned| poisoned.into_inner());
301    // The registry owns only weak references. Pruning on every acquisition
302    // means completed or cancelled peer dials cannot leave an ever-growing
303    // per-endpoint map behind.
304    gates.retain(|_, gate| gate.strong_count() > 0);
305    let gate = gates
306        .get(&endpoint_id)
307        .and_then(Weak::upgrade)
308        .unwrap_or_else(|| {
309            let gate = Arc::new(OutboundDialGate::new(()));
310            gates.insert(endpoint_id, Arc::downgrade(&gate));
311            gate
312        });
313    drop(gates);
314    OutboundDialGateLease {
315        endpoint_id,
316        gate,
317        gates: Arc::clone(registry),
318    }
319}
320
321fn selected_custom_transport_id(connection: &Connection) -> Option<u64> {
322    connection.paths().iter().find_map(|path| {
323        if !path.is_selected() {
324            return None;
325        }
326        match path.remote_addr() {
327            iroh::TransportAddr::Custom(addr) => Some(addr.id()),
328            _ => None,
329        }
330    })
331}
332
333fn custom_transport_arbitration_override(
334    existing_alive: bool,
335    existing_custom_transport_id: Option<u64>,
336    fresh_custom_transport_id: Option<u64>,
337    authorized_replacement_transport_id: Option<u64>,
338) -> Option<IrohConnectionInstallDecision> {
339    if authorized_replacement_transport_id
340        .is_some_and(|expected| fresh_custom_transport_id == Some(expected))
341    {
342        return Some(IrohConnectionInstallDecision::ReplaceExisting {
343            close_existing_reason: crate::lifecycle_reason::REASON_NATIVE_CUSTOM_TRANSPORT_UPGRADE,
344        });
345    }
346    if existing_alive
347        && existing_custom_transport_id.is_some()
348        && fresh_custom_transport_id != existing_custom_transport_id
349    {
350        return Some(IrohConnectionInstallDecision::KeepExisting {
351            close_fresh_reason: "existing-custom-transport-preferred",
352        });
353    }
354    None
355}
356
357fn should_start_outbound_dial_with_transport_preference(
358    existing: Option<(bool, IrohConnectionDirection)>,
359    existing_custom_transport_id: Option<u64>,
360    local_prefers_outbound: bool,
361) -> bool {
362    if existing.is_some_and(|(alive, _)| alive) && existing_custom_transport_id.is_some() {
363        return false;
364    }
365    should_start_outbound_dial(existing, local_prefers_outbound)
366}
367
368fn decide_outbound_install_with_transport_preference(
369    existing: Option<ExistingConnectionState>,
370    existing_custom_transport_id: Option<u64>,
371    fresh_custom_transport_id: Option<u64>,
372    local_prefers_outbound: bool,
373) -> IrohConnectionInstallDecision {
374    custom_transport_arbitration_override(
375        existing.as_ref().is_some_and(|state| state.alive),
376        existing_custom_transport_id,
377        fresh_custom_transport_id,
378        None,
379    )
380    .unwrap_or_else(|| decide_outbound_install(existing, local_prefers_outbound))
381}
382
383async fn consume_authorized_inbound_replacement(
384    connection: &Connection,
385    authorizations: &InboundReplacementAuthorizations,
386) -> Option<u64> {
387    let endpoint_id = connection.remote_id();
388    let deadline = Instant::now() + std::time::Duration::from_secs(1);
389    loop {
390        let authorization = authorizations.read().await.get(&endpoint_id).copied();
391        let Some(authorization) = authorization else {
392            return None;
393        };
394        if Instant::now() >= authorization.expires_at {
395            let mut authorizations = authorizations.write().await;
396            if authorizations.get(&endpoint_id).is_some_and(|current| {
397                current.transport_id == authorization.transport_id
398                    && current.expires_at == authorization.expires_at
399            }) {
400                authorizations.remove(&endpoint_id);
401            }
402            return None;
403        }
404        if selected_custom_transport_id(connection) == Some(authorization.transport_id) {
405            let mut authorizations = authorizations.write().await;
406            if authorizations.get(&endpoint_id).is_some_and(|current| {
407                current.transport_id == authorization.transport_id
408                    && current.expires_at == authorization.expires_at
409            }) {
410                authorizations.remove(&endpoint_id);
411                return Some(authorization.transport_id);
412            }
413            return None;
414        }
415        if Instant::now() >= deadline || connection.close_reason().is_some() {
416            return None;
417        }
418        tokio::time::sleep(std::time::Duration::from_millis(25)).await;
419    }
420}
421
422#[derive(Debug, Clone)]
423pub struct PlutoniumProtocol {
424    event_sender: broadcast::Sender<AcceptEvent>,
425    stream_sender: async_channel::Sender<IncomingStream>,
426    connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
427    connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
428    local_endpoint_id: EndpointId,
429    manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
430    probe_registry: IrohProbeRegistry,
431    heartbeat_manager: Option<IrohHeartbeatManager>,
432    heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
433    heartbeat_config: HeartbeatConfig,
434    inbound_replacement_authorizations: InboundReplacementAuthorizations,
435}
436
437async fn run_connection_loop(
438    connection: Connection,
439    event_sender: broadcast::Sender<AcceptEvent>,
440    stream_sender: async_channel::Sender<IncomingStream>,
441    connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
442    connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
443    local_endpoint_id: EndpointId,
444    connection_direction: IrohConnectionDirection,
445    manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
446    probe_registry: IrohProbeRegistry,
447    heartbeat_manager: Option<IrohHeartbeatManager>,
448    heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
449    heartbeat_config: HeartbeatConfig,
450    replacement_transport_id: Option<u64>,
451    mut install_outcome_sender: Option<oneshot::Sender<ExternalConnectionInstallOutcome>>,
452) -> std::result::Result<(), AcceptError> {
453    let endpoint_id = connection.remote_id();
454    let raw_stable_id = connection.stable_id();
455    let stable_id = transport_generation_for_connection(&connection);
456    let endpoint_key = endpoint_id.to_string();
457    let heartbeat_key = heartbeat_connection_key(endpoint_id, stable_id);
458
459    let mut kept_existing = None;
460    let mut replaced_transport_stable_id = None;
461    {
462        let mut conns = connections.write().await;
463        let mut insert_times = connection_inserted_at.write().await;
464        if let Some(previous) = conns.get(&endpoint_id).cloned() {
465            let fresh_custom_transport_id = selected_custom_transport_id(&connection);
466            let existing_custom_transport_id = selected_custom_transport_id(&previous);
467            let forced_replacement = replacement_transport_id
468                .is_some_and(|expected| fresh_custom_transport_id == Some(expected));
469            let previous_alive = if forced_replacement {
470                previous.close_reason().is_none()
471            } else {
472                existing_transport_is_live_for_install(
473                    endpoint_id,
474                    &previous,
475                    stable_id,
476                    connection_direction,
477                    &probe_registry,
478                )
479                .await
480            };
481            let previous_direction = insert_times
482                .get(&endpoint_id)
483                .map(|metadata| metadata.direction)
484                .unwrap_or(connection_direction);
485            let previous_age_ms = insert_times
486                .get(&endpoint_id)
487                .map(|metadata| metadata.inserted_at.elapsed().as_millis() as u64)
488                .unwrap_or(0);
489            let existing = Some(ExistingConnectionState {
490                same_stable_id: previous.stable_id() == raw_stable_id,
491                alive: previous_alive,
492                direction: previous_direction,
493                age_ms: previous_age_ms,
494            });
495            let prefers_outbound = local_prefers_outbound(local_endpoint_id, endpoint_id);
496            let decision = if forced_replacement && previous.stable_id() == raw_stable_id {
497                IrohConnectionInstallDecision::Install
498            } else if let Some(decision) = custom_transport_arbitration_override(
499                previous_alive,
500                existing_custom_transport_id,
501                fresh_custom_transport_id,
502                replacement_transport_id,
503            ) {
504                decision
505            } else {
506                match connection_direction {
507                    IrohConnectionDirection::Inbound => {
508                        decide_inbound_install(existing, prefers_outbound)
509                    }
510                    IrohConnectionDirection::Outbound => {
511                        decide_outbound_install(existing, prefers_outbound)
512                    }
513                }
514            };
515            match decision {
516                IrohConnectionInstallDecision::Install => {
517                    let previous_stable_id = transport_generation_for_connection(&previous);
518                    if previous_stable_id != stable_id {
519                        replaced_transport_stable_id = Some(previous_stable_id);
520                    }
521                }
522                IrohConnectionInstallDecision::ReplaceExisting {
523                    close_existing_reason,
524                } => {
525                    replaced_transport_stable_id =
526                        Some(transport_generation_for_connection(&previous));
527                    previous.close(0u8.into(), close_existing_reason.as_bytes());
528                }
529                IrohConnectionInstallDecision::KeepExisting { close_fresh_reason } => {
530                    kept_existing = Some(ExternalConnectionInstallOutcome::KeptExisting {
531                        fresh_transport_stable_id: stable_id,
532                        kept_transport_stable_id: transport_generation_for_connection(&previous),
533                    });
534                    connection.close(0u8.into(), close_fresh_reason.as_bytes());
535                }
536            }
537        }
538        if kept_existing.is_none() {
539            conns.insert(endpoint_id, connection.clone());
540            insert_times.insert(
541                endpoint_id,
542                IrohConnectionMetadata {
543                    inserted_at: Instant::now(),
544                    direction: connection_direction,
545                },
546            );
547        }
548    }
549
550    if let Some(outcome) = kept_existing {
551        if let Some(sender) = install_outcome_sender.take() {
552            let _ = sender.send(outcome);
553        }
554        forget_transport_generation(&connection);
555        return Ok(());
556    }
557
558    if let Some(sender) = install_outcome_sender.take() {
559        let _ = sender.send(ExternalConnectionInstallOutcome::Installed {
560            transport_stable_id: stable_id,
561            replaced_transport_stable_id,
562        });
563    }
564
565    // Start iroh heartbeat for this connection if the manager is configured.
566    if let (Some(mgr), Some(tx)) = (&heartbeat_manager, &heartbeat_health_tx) {
567        mgr.start_connection(
568            heartbeat_key.clone(),
569            connection.clone(),
570            heartbeat_config.clone(),
571            tx.clone(),
572        )
573        .await;
574    }
575
576    event_sender
577        .send(AcceptEvent::Accepted {
578            endpoint_id,
579            transport_stable_id: stable_id,
580            replaced_transport_stable_id,
581        })
582        .ok();
583
584    loop {
585        tokio::select! {
586            // Application bi streams and control/application uni streams share
587            // this physical connection. A biased bi-first dispatcher can
588            // indefinitely starve heartbeat pings under sustained application
589            // traffic, causing the health owner to tear down a live transport.
590            // Tokio's default fair selection keeps both protocol classes
591            // progressing without introducing another liveness owner.
592            res = connection.accept_bi() => {
593                match res {
594                    Ok((send, recv)) => {
595                        probe_registry
596                            .record_inbound_activity(&endpoint_key, stable_id)
597                            .await;
598                        let _ = stream_sender.send(IncomingStream {
599                            endpoint_id,
600                            transport_stable_id: stable_id,
601                            recv_prefix: Vec::new(),
602                            stream: IncomingStreamType::Bi(send, recv),
603                        }).await;
604                    }
605                    Err(_) => {
606                        if should_break_accept_loop(&connection).await {
607                            break;
608                        }
609                    },
610                }
611            }
612            res = connection.accept_uni() => {
613                match res {
614                    Ok(recv) => {
615                        probe_registry
616                            .record_inbound_activity(&endpoint_key, stable_id)
617                            .await;
618                        // Classification runs independently so a quiet application
619                        // uni stream cannot stall acceptance of later streams.
620                        let conn_clone = connection.clone();
621                        let mgr_clone = heartbeat_manager.clone();
622                        let probe_registry = probe_registry.clone();
623                        let endpoint_key = endpoint_key.clone();
624                        let heartbeat_key = heartbeat_key.clone();
625                        let transport_stable_id = stable_id;
626                        let notice_key = manual_disconnect_notice_key(
627                            endpoint_id,
628                            transport_stable_id,
629                        );
630                        let notices = manual_disconnect_notices.clone();
631                        let application_streams = stream_sender.clone();
632                        tokio::spawn(async move {
633                            match classify_incoming_uni(recv).await {
634                                IncomingUniClassification::Control { type_id, payload } => {
635                                use crate::heartbeat::codec;
636                                if type_id == codec::TYPE_PING {
637                                    if !respond_to_ping(&conn_clone, &payload).await {
638                                        eprintln!(
639                                            "[OpenRTC][iroh-probe] failed to send pong endpoint_id={} transport_stable_id={}",
640                                            endpoint_key,
641                                            transport_stable_id,
642                                        );
643                                    }
644                                } else if type_id == codec::TYPE_PONG {
645                                    if let Some(pong) = parse_incoming_pong(&payload) {
646                                        probe_registry
647                                            .deliver_pong(
648                                                &endpoint_key,
649                                                transport_stable_id,
650                                                &pong,
651                                            )
652                                            .await;
653                                        if let Some(manager) = mgr_clone {
654                                            manager.deliver_pong(&heartbeat_key, pong).await;
655                                        }
656                                    }
657                                } else if type_id == codec::TYPE_MANUAL_DISCONNECT {
658                                    notices.write().await.insert(notice_key);
659                                    conn_clone.close(
660                                        0u8.into(),
661                                        crate::lifecycle_reason::REASON_MANUAL_DISCONNECT
662                                            .as_bytes(),
663                                    );
664                                }
665                                }
666                                IncomingUniClassification::Application(recv) => {
667                                    let _ = application_streams
668                                        .send(IncomingStream {
669                                            endpoint_id,
670                                            transport_stable_id,
671                                            recv_prefix: Vec::new(),
672                                            stream: IncomingStreamType::Uni(recv),
673                                        })
674                                        .await;
675                                }
676                                IncomingUniClassification::MalformedControl => {}
677                            }
678                        });
679                    }
680                    Err(_) => {
681                        if should_break_accept_loop(&connection).await {
682                            break;
683                        }
684                    },
685                }
686            }
687            _ = connection.closed() => break,
688        }
689    }
690
691    // Stop heartbeat for this connection.
692    if let Some(ref mgr) = heartbeat_manager {
693        mgr.stop_connection(&heartbeat_key).await;
694    }
695    probe_registry
696        .forget_transport(&endpoint_key, stable_id)
697        .await;
698
699    let close_reason = connection.close_reason();
700    let close_reason_debug = close_reason.as_ref().map(|reason| format!("{:?}", reason));
701    let was_locally_closed = matches!(
702        close_reason,
703        Some(iroh::endpoint::ConnectionError::LocallyClosed)
704    );
705    let current_transport_stable_id = connections
706        .read()
707        .await
708        .get(&endpoint_id)
709        .map(transport_generation_for_connection);
710    eprintln!(
711        "[OpenRTC][iroh-connection][closed] endpoint_id={} transport_stable_id={} direction={:?} close_reason={:?} current_transport_stable_id={:?}",
712        endpoint_id,
713        stable_id,
714        connection_direction,
715        close_reason_debug,
716        current_transport_stable_id,
717    );
718    event_sender
719        .send(AcceptEvent::Closed {
720            endpoint_id,
721            transport_stable_id: stable_id,
722            error: close_reason_debug,
723            was_outbound: matches!(connection_direction, IrohConnectionDirection::Outbound),
724            was_locally_closed,
725        })
726        .ok();
727
728    {
729        let mut conns = connections.write().await;
730        let should_remove = conns
731            .get(&endpoint_id)
732            .map(|current| current.stable_id() == raw_stable_id)
733            .unwrap_or(false);
734        if should_remove {
735            conns.remove(&endpoint_id);
736            connection_inserted_at.write().await.remove(&endpoint_id);
737        }
738    }
739
740    forget_transport_generation(&connection);
741    Ok(())
742}
743
744impl PlutoniumProtocol {
745    pub const ALPN: &[u8] = b"plutonium/p2p/1";
746
747    fn new(
748        event_sender: broadcast::Sender<AcceptEvent>,
749        stream_sender: async_channel::Sender<IncomingStream>,
750        connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
751        connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
752        local_endpoint_id: EndpointId,
753        manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
754        probe_registry: IrohProbeRegistry,
755        inbound_replacement_authorizations: InboundReplacementAuthorizations,
756    ) -> Self {
757        Self {
758            event_sender,
759            stream_sender,
760            connections,
761            connection_inserted_at,
762            local_endpoint_id,
763            manual_disconnect_notices,
764            probe_registry,
765            heartbeat_manager: None,
766            heartbeat_health_tx: None,
767            heartbeat_config: HeartbeatConfig::default(),
768            inbound_replacement_authorizations,
769        }
770    }
771
772    pub fn with_heartbeat(
773        mut self,
774        manager: IrohHeartbeatManager,
775        health_tx: mpsc::Sender<HealthTransition>,
776        config: HeartbeatConfig,
777    ) -> Self {
778        self.heartbeat_manager = Some(manager);
779        self.heartbeat_health_tx = Some(health_tx);
780        self.heartbeat_config = config;
781        self
782    }
783
784    async fn handle_connection(
785        self,
786        connection: Connection,
787    ) -> std::result::Result<(), AcceptError> {
788        let replacement_transport_id = consume_authorized_inbound_replacement(
789            &connection,
790            &self.inbound_replacement_authorizations,
791        )
792        .await;
793        run_connection_loop(
794            connection,
795            self.event_sender.clone(),
796            self.stream_sender.clone(),
797            self.connections.clone(),
798            self.connection_inserted_at.clone(),
799            self.local_endpoint_id,
800            IrohConnectionDirection::Inbound,
801            self.manual_disconnect_notices.clone(),
802            self.probe_registry.clone(),
803            self.heartbeat_manager.clone(),
804            self.heartbeat_health_tx.clone(),
805            self.heartbeat_config.clone(),
806            replacement_transport_id,
807            None,
808        )
809        .await
810    }
811}
812
813impl ProtocolHandler for PlutoniumProtocol {
814    #[allow(refining_impl_trait)]
815    fn accept(
816        &self,
817        connection: Connection,
818    ) -> impl n0_future::Future<Output = std::result::Result<(), AcceptError>> + std::marker::Send
819    {
820        let proto = self.clone();
821        async move { proto.handle_connection(connection).await }
822    }
823}
824
825async fn retire_unobserved_outbound_connection(
826    endpoint_id: EndpointId,
827    connection: &Connection,
828    connections: &Arc<RwLock<HashMap<EndpointId, Connection>>>,
829    connection_inserted_at: &Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
830) {
831    connection.close(0u8.into(), b"connect-event-receiver-closed");
832    let raw_stable_id = connection.stable_id();
833    let mut conns = connections.write().await;
834    if conns
835        .get(&endpoint_id)
836        .is_some_and(|current| current.stable_id() == raw_stable_id)
837    {
838        conns.remove(&endpoint_id);
839        connection_inserted_at.write().await.remove(&endpoint_id);
840    }
841    forget_transport_generation(connection);
842}
843
844async fn connect(
845    endpoint: &Endpoint,
846    endpoint_id: EndpointId,
847    event_sender: async_channel::Sender<ConnectEvent>,
848    outbound_dial_gates: OutboundDialGates,
849    connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
850    connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
851    stream_sender: async_channel::Sender<IncomingStream>,
852    manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
853    probe_registry: IrohProbeRegistry,
854    heartbeat_manager: Option<IrohHeartbeatManager>,
855    heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
856    heartbeat_config: HeartbeatConfig,
857) -> Result<()> {
858    let connection = {
859        // Only the physical Iroh owner can make the precheck + dial + install
860        // decision atomic. The gate is per endpoint, so unrelated peer dials
861        // remain fully parallel.
862        let dial_gate = outbound_dial_gate(endpoint_id, &outbound_dial_gates);
863        let _dial_guard = dial_gate.gate.lock().await;
864        if outbound_dial_precheck(endpoint, endpoint_id, &connections, &connection_inserted_at)
865            .await
866        {
867            event_sender.send(ConnectEvent::Connected).await?;
868            return Ok(());
869        }
870
871        let connection = endpoint
872            .connect(endpoint_id, PlutoniumProtocol::ALPN)
873            .await?;
874        let raw_stable_id = connection.stable_id();
875        let stable_id = transport_generation_for_connection(&connection);
876
877        {
878            let mut conns = connections.write().await;
879            let mut insert_times = connection_inserted_at.write().await;
880            if let Some(previous) = conns.get(&endpoint_id).cloned() {
881                let previous_alive = existing_transport_is_live_for_install(
882                    endpoint_id,
883                    &previous,
884                    stable_id,
885                    IrohConnectionDirection::Outbound,
886                    &probe_registry,
887                )
888                .await;
889                let previous_direction = insert_times
890                    .get(&endpoint_id)
891                    .map(|metadata| metadata.direction)
892                    .unwrap_or(IrohConnectionDirection::Outbound);
893                let previous_age_ms = insert_times
894                    .get(&endpoint_id)
895                    .map(|metadata| metadata.inserted_at.elapsed().as_millis() as u64)
896                    .unwrap_or(0);
897                let existing = Some(ExistingConnectionState {
898                    same_stable_id: previous.stable_id() == raw_stable_id,
899                    alive: previous_alive,
900                    direction: previous_direction,
901                    age_ms: previous_age_ms,
902                });
903                match decide_outbound_install_with_transport_preference(
904                    existing,
905                    selected_custom_transport_id(&previous),
906                    selected_custom_transport_id(&connection),
907                    local_prefers_outbound(endpoint.id(), endpoint_id),
908                ) {
909                    IrohConnectionInstallDecision::Install => {}
910                    IrohConnectionInstallDecision::ReplaceExisting {
911                        close_existing_reason,
912                    } => {
913                        previous.close(0u8.into(), close_existing_reason.as_bytes());
914                    }
915                    IrohConnectionInstallDecision::KeepExisting { close_fresh_reason } => {
916                        connection.close(0u8.into(), close_fresh_reason.as_bytes());
917                        forget_transport_generation(&connection);
918                        event_sender.send(ConnectEvent::Connected).await?;
919                        let _ = event_sender
920                            .send(ConnectEvent::Closed { error: None })
921                            .await;
922                        return Ok(());
923                    }
924                }
925            }
926            conns.insert(endpoint_id, connection.clone());
927            insert_times.insert(
928                endpoint_id,
929                IrohConnectionMetadata {
930                    inserted_at: Instant::now(),
931                    direction: IrohConnectionDirection::Outbound,
932                },
933            );
934        }
935        connection
936    };
937
938    if let Err(error) = event_sender.send(ConnectEvent::Connected).await {
939        retire_unobserved_outbound_connection(
940            endpoint_id,
941            &connection,
942            &connections,
943            &connection_inserted_at,
944        )
945        .await;
946        return Err(error.into());
947    }
948
949    // Use the shared loop (handles heartbeat interception for Uni streams).
950    let (accept_tx, _) = broadcast::channel(1);
951    run_connection_loop(
952        connection,
953        accept_tx,
954        stream_sender,
955        connections.clone(),
956        connection_inserted_at,
957        endpoint.id(),
958        IrohConnectionDirection::Outbound,
959        manual_disconnect_notices,
960        probe_registry,
961        heartbeat_manager,
962        heartbeat_health_tx,
963        heartbeat_config,
964        None,
965        None,
966    )
967    .await
968    .ok();
969
970    event_sender
971        .send(ConnectEvent::Closed { error: None })
972        .await?;
973
974    Ok(())
975}
976
977async fn connect_addr(
978    endpoint: &Endpoint,
979    endpoint_id: EndpointId,
980    endpoint_addr: EndpointAddr,
981    event_sender: async_channel::Sender<ConnectEvent>,
982    outbound_dial_gates: OutboundDialGates,
983    connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
984    connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
985    stream_sender: async_channel::Sender<IncomingStream>,
986    manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
987    probe_registry: IrohProbeRegistry,
988    heartbeat_manager: Option<IrohHeartbeatManager>,
989    heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
990    heartbeat_config: HeartbeatConfig,
991) -> Result<()> {
992    let connection = {
993        let dial_gate = outbound_dial_gate(endpoint_id, &outbound_dial_gates);
994        let _dial_guard = dial_gate.gate.lock().await;
995        if outbound_dial_precheck(endpoint, endpoint_id, &connections, &connection_inserted_at)
996            .await
997        {
998            event_sender.send(ConnectEvent::Connected).await?;
999            return Ok(());
1000        }
1001
1002        let connection = endpoint
1003            .connect(endpoint_addr, PlutoniumProtocol::ALPN)
1004            .await?;
1005        let raw_stable_id = connection.stable_id();
1006        let stable_id = transport_generation_for_connection(&connection);
1007
1008        {
1009            let mut conns = connections.write().await;
1010            let mut insert_times = connection_inserted_at.write().await;
1011            if let Some(previous) = conns.get(&endpoint_id).cloned() {
1012                let previous_alive = existing_transport_is_live_for_install(
1013                    endpoint_id,
1014                    &previous,
1015                    stable_id,
1016                    IrohConnectionDirection::Outbound,
1017                    &probe_registry,
1018                )
1019                .await;
1020                let previous_direction = insert_times
1021                    .get(&endpoint_id)
1022                    .map(|metadata| metadata.direction)
1023                    .unwrap_or(IrohConnectionDirection::Outbound);
1024                let previous_age_ms = insert_times
1025                    .get(&endpoint_id)
1026                    .map(|metadata| metadata.inserted_at.elapsed().as_millis() as u64)
1027                    .unwrap_or(0);
1028                let existing = Some(ExistingConnectionState {
1029                    same_stable_id: previous.stable_id() == raw_stable_id,
1030                    alive: previous_alive,
1031                    direction: previous_direction,
1032                    age_ms: previous_age_ms,
1033                });
1034                match decide_outbound_install_with_transport_preference(
1035                    existing,
1036                    selected_custom_transport_id(&previous),
1037                    selected_custom_transport_id(&connection),
1038                    local_prefers_outbound(endpoint.id(), endpoint_id),
1039                ) {
1040                    IrohConnectionInstallDecision::Install => {}
1041                    IrohConnectionInstallDecision::ReplaceExisting {
1042                        close_existing_reason,
1043                    } => {
1044                        previous.close(0u8.into(), close_existing_reason.as_bytes());
1045                    }
1046                    IrohConnectionInstallDecision::KeepExisting { close_fresh_reason } => {
1047                        connection.close(0u8.into(), close_fresh_reason.as_bytes());
1048                        forget_transport_generation(&connection);
1049                        event_sender.send(ConnectEvent::Connected).await?;
1050                        let _ = event_sender
1051                            .send(ConnectEvent::Closed { error: None })
1052                            .await;
1053                        return Ok(());
1054                    }
1055                }
1056            }
1057            conns.insert(endpoint_id, connection.clone());
1058            insert_times.insert(
1059                endpoint_id,
1060                IrohConnectionMetadata {
1061                    inserted_at: Instant::now(),
1062                    direction: IrohConnectionDirection::Outbound,
1063                },
1064            );
1065        }
1066        connection
1067    };
1068
1069    if let Err(error) = event_sender.send(ConnectEvent::Connected).await {
1070        retire_unobserved_outbound_connection(
1071            endpoint_id,
1072            &connection,
1073            &connections,
1074            &connection_inserted_at,
1075        )
1076        .await;
1077        return Err(error.into());
1078    }
1079
1080    let (accept_tx, _) = broadcast::channel(1);
1081    run_connection_loop(
1082        connection,
1083        accept_tx,
1084        stream_sender,
1085        connections.clone(),
1086        connection_inserted_at,
1087        endpoint.id(),
1088        IrohConnectionDirection::Outbound,
1089        manual_disconnect_notices,
1090        probe_registry,
1091        heartbeat_manager,
1092        heartbeat_health_tx,
1093        heartbeat_config,
1094        None,
1095        None,
1096    )
1097    .await
1098    .ok();
1099
1100    event_sender
1101        .send(ConnectEvent::Closed { error: None })
1102        .await?;
1103
1104    Ok(())
1105}
1106
1107async fn install_replacement_connection(
1108    endpoint: &Endpoint,
1109    connection: Connection,
1110    replacement_transport_id: u64,
1111    event_sender: async_channel::Sender<ReplacementConnectEvent>,
1112    connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
1113    connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
1114    stream_sender: async_channel::Sender<IncomingStream>,
1115    manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
1116    probe_registry: IrohProbeRegistry,
1117    heartbeat_manager: Option<IrohHeartbeatManager>,
1118    heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
1119    heartbeat_config: HeartbeatConfig,
1120) -> Result<()> {
1121    fn replacement_path_summary(connection: &Connection) -> Vec<String> {
1122        connection
1123            .paths()
1124            .iter()
1125            .map(|path| {
1126                format!(
1127                    "selected={} remote_addr={:?}",
1128                    path.is_selected(),
1129                    path.remote_addr()
1130                )
1131            })
1132            .collect()
1133    }
1134
1135    eprintln!(
1136        "[OpenRTC][custom-transport] replacement admission waiting stable_id={} remote_endpoint_id={} expected_transport_id={} paths={:?}",
1137        connection.stable_id(),
1138        connection.remote_id(),
1139        replacement_transport_id,
1140        replacement_path_summary(&connection)
1141    );
1142    let selected_expected_transport = || {
1143        connection.paths().iter().any(|path| {
1144            path.is_selected()
1145                && matches!(
1146                    path.remote_addr(),
1147                    iroh::TransportAddr::Custom(addr) if addr.id() == replacement_transport_id
1148                )
1149        })
1150    };
1151    let selected = tokio::time::timeout(std::time::Duration::from_secs(5), async {
1152        loop {
1153            if selected_expected_transport() {
1154                break;
1155            }
1156            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1157        }
1158    })
1159    .await
1160    .is_ok();
1161    if !selected {
1162        eprintln!(
1163            "[OpenRTC][custom-transport] replacement admission rejected stable_id={} remote_endpoint_id={} expected_transport_id={} paths={:?}",
1164            connection.stable_id(),
1165            connection.remote_id(),
1166            replacement_transport_id,
1167            replacement_path_summary(&connection)
1168        );
1169        connection.close(0u8.into(), b"custom-transport-not-selected");
1170        anyhow::bail!(
1171            "replacement connection did not select custom transport id {replacement_transport_id}"
1172        );
1173    }
1174
1175    eprintln!(
1176        "[OpenRTC][custom-transport] replacement admission accepted stable_id={} remote_endpoint_id={} transport_id={} paths={:?}",
1177        connection.stable_id(),
1178        connection.remote_id(),
1179        replacement_transport_id,
1180        replacement_path_summary(&connection)
1181    );
1182
1183    let (accept_tx, _) = broadcast::channel(1);
1184    let (install_outcome_sender, install_outcome_receiver) = oneshot::channel();
1185    let connection_loop = tokio::spawn(run_connection_loop(
1186        connection,
1187        accept_tx,
1188        stream_sender,
1189        connections,
1190        connection_inserted_at,
1191        endpoint.id(),
1192        IrohConnectionDirection::Outbound,
1193        manual_disconnect_notices,
1194        probe_registry,
1195        heartbeat_manager,
1196        heartbeat_health_tx,
1197        heartbeat_config,
1198        Some(replacement_transport_id),
1199        Some(install_outcome_sender),
1200    ));
1201    let install_outcome = install_outcome_receiver
1202        .await
1203        .map_err(|_| anyhow::anyhow!("replacement connection install task ended"))?;
1204    let transport_stable_id = match install_outcome {
1205        ExternalConnectionInstallOutcome::Installed {
1206            transport_stable_id,
1207            ..
1208        } => transport_stable_id,
1209        ExternalConnectionInstallOutcome::KeptExisting {
1210            fresh_transport_stable_id,
1211            kept_transport_stable_id,
1212        } => {
1213            anyhow::bail!(
1214                "replacement connection lost arbitration fresh_stable_id={} kept_stable_id={}",
1215                fresh_transport_stable_id,
1216                kept_transport_stable_id,
1217            );
1218        }
1219    };
1220    event_sender
1221        .send(ReplacementConnectEvent::Connected {
1222            transport_stable_id,
1223        })
1224        .await?;
1225    let _ = connection_loop.await;
1226
1227    let _ = event_sender
1228        .send(ReplacementConnectEvent::Closed { error: None })
1229        .await;
1230    Ok(())
1231}
1232
1233#[allow(unused_variables)]
1234async fn connect_replacement_addr(
1235    endpoint: &Endpoint,
1236    _endpoint_id: EndpointId,
1237    endpoint_addr: EndpointAddr,
1238    replacement_transport_id: u64,
1239    event_sender: async_channel::Sender<ReplacementConnectEvent>,
1240    connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
1241    connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
1242    stream_sender: async_channel::Sender<IncomingStream>,
1243    manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
1244    probe_registry: IrohProbeRegistry,
1245    heartbeat_manager: Option<IrohHeartbeatManager>,
1246    heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
1247    heartbeat_config: HeartbeatConfig,
1248) -> Result<()> {
1249    #[cfg(not(openrtc_iroh_preferred_transport_api))]
1250    anyhow::bail!(
1251        "custom transport replacement requires a host Iroh build with the preferred-transport API"
1252    );
1253
1254    #[cfg(openrtc_iroh_preferred_transport_api)]
1255    {
1256        let preferred_transport_addr = endpoint_addr
1257        .addrs
1258        .iter()
1259        .find(|addr| {
1260            matches!(
1261                addr,
1262                iroh::TransportAddr::Custom(custom)
1263                    if custom.id() == replacement_transport_id
1264            )
1265        })
1266        .cloned()
1267        .ok_or_else(|| {
1268            anyhow::anyhow!(
1269                "replacement address does not contain custom transport id {replacement_transport_id}"
1270            )
1271        })?;
1272        eprintln!(
1273        "[OpenRTC][custom-transport] replacement dial pinned remote_endpoint_id={} transport_id={} addr={:?}",
1274        endpoint_addr.id, replacement_transport_id, preferred_transport_addr
1275    );
1276        let connection = endpoint
1277            .connect_with_opts(
1278                endpoint_addr,
1279                PlutoniumProtocol::ALPN,
1280                iroh::endpoint::ConnectOptions::new()
1281                    .with_preferred_transport_addr(preferred_transport_addr),
1282            )
1283            .await?
1284            .await?;
1285        install_replacement_connection(
1286            endpoint,
1287            connection,
1288            replacement_transport_id,
1289            event_sender,
1290            connections,
1291            connection_inserted_at,
1292            stream_sender,
1293            manual_disconnect_notices,
1294            probe_registry,
1295            heartbeat_manager,
1296            heartbeat_health_tx,
1297            heartbeat_config,
1298        )
1299        .await
1300    }
1301}
1302
1303#[derive(Debug, Clone)]
1304pub struct IrohNativeNode {
1305    endpoint: Endpoint,
1306    // Keep the router alive for endpoints that run pluto-rtc's internal accept loop.
1307    #[allow(dead_code)]
1308    router: Option<Router>,
1309    accept_events: broadcast::Sender<AcceptEvent>,
1310    connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
1311    connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
1312    outbound_dial_gates: OutboundDialGates,
1313    incoming_streams: async_channel::Sender<IncomingStream>,
1314    incoming_streams_receiver: async_channel::Receiver<IncomingStream>,
1315    manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
1316    probe_registry: IrohProbeRegistry,
1317    heartbeat_manager: Option<IrohHeartbeatManager>,
1318    heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
1319    heartbeat_config: HeartbeatConfig,
1320    inbound_replacement_authorizations: InboundReplacementAuthorizations,
1321}
1322
1323impl IrohNativeNode {
1324    pub async fn spawn_with_endpoint(endpoint: Endpoint) -> Result<Self> {
1325        Self::spawn_with_endpoint_config(endpoint, true).await
1326    }
1327
1328    /// Spawn a native node wrapper without creating an internal Router accept loop.
1329    ///
1330    /// This mode is used when the embedding application owns the single iroh
1331    /// Router and forwards accepted plutonium connections into pluto-rtc via
1332    /// `Client::handle_incoming_connection`.
1333    pub async fn spawn_with_endpoint_no_router(endpoint: Endpoint) -> Result<Self> {
1334        Self::spawn_with_endpoint_config(endpoint, false).await
1335    }
1336
1337    async fn spawn_with_endpoint_config(endpoint: Endpoint, spawn_router: bool) -> Result<Self> {
1338        Self::spawn_with_endpoint_config_and_heartbeat(
1339            endpoint,
1340            spawn_router,
1341            None,
1342            None,
1343            HeartbeatConfig::default(),
1344        )
1345        .await
1346    }
1347
1348    /// Spawn with an explicit heartbeat manager for iroh-level liveness monitoring.
1349    pub async fn spawn_with_heartbeat(
1350        endpoint: Endpoint,
1351        heartbeat_manager: IrohHeartbeatManager,
1352        heartbeat_health_tx: mpsc::Sender<HealthTransition>,
1353        heartbeat_config: HeartbeatConfig,
1354    ) -> Result<Self> {
1355        Self::spawn_with_endpoint_config_and_heartbeat(
1356            endpoint,
1357            true,
1358            Some(heartbeat_manager),
1359            Some(heartbeat_health_tx),
1360            heartbeat_config,
1361        )
1362        .await
1363    }
1364
1365    async fn spawn_with_endpoint_config_and_heartbeat(
1366        endpoint: Endpoint,
1367        spawn_router: bool,
1368        heartbeat_manager: Option<IrohHeartbeatManager>,
1369        heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
1370        heartbeat_config: HeartbeatConfig,
1371    ) -> Result<Self> {
1372        let (event_sender, _) = broadcast::channel(128);
1373        let (stream_sender, stream_receiver) = async_channel::bounded(64);
1374        let connections = Arc::new(RwLock::new(HashMap::new()));
1375        let connection_inserted_at = Arc::new(RwLock::new(HashMap::new()));
1376        let outbound_dial_gates = Arc::new(StdMutex::new(HashMap::new()));
1377        let manual_disconnect_notices = Arc::new(RwLock::new(HashSet::new()));
1378        let probe_registry = IrohProbeRegistry::new();
1379        let inbound_replacement_authorizations = Arc::new(RwLock::new(HashMap::new()));
1380
1381        let router = if spawn_router {
1382            let proto = PlutoniumProtocol::new(
1383                event_sender.clone(),
1384                stream_sender.clone(),
1385                connections.clone(),
1386                connection_inserted_at.clone(),
1387                endpoint.id(),
1388                manual_disconnect_notices.clone(),
1389                probe_registry.clone(),
1390                inbound_replacement_authorizations.clone(),
1391            );
1392            let proto = if let (Some(mgr), Some(tx)) =
1393                (heartbeat_manager.clone(), heartbeat_health_tx.clone())
1394            {
1395                proto.with_heartbeat(mgr, tx, heartbeat_config.clone())
1396            } else {
1397                proto
1398            };
1399            Some(
1400                Router::builder(endpoint.clone())
1401                    .accept(PlutoniumProtocol::ALPN, proto)
1402                    .spawn(),
1403            )
1404        } else {
1405            None
1406        };
1407
1408        Ok(Self {
1409            endpoint,
1410            router,
1411            accept_events: event_sender,
1412            connections,
1413            connection_inserted_at,
1414            outbound_dial_gates,
1415            incoming_streams: stream_sender,
1416            incoming_streams_receiver: stream_receiver,
1417            manual_disconnect_notices,
1418            probe_registry,
1419            heartbeat_manager,
1420            heartbeat_health_tx,
1421            heartbeat_config,
1422            inbound_replacement_authorizations,
1423        })
1424    }
1425
1426    pub fn endpoint(&self) -> &Endpoint {
1427        &self.endpoint
1428    }
1429
1430    pub async fn node_addr(&self) -> Result<EndpointAddr> {
1431        // Client initialization owns the bounded relay-online wait and arranges
1432        // a presence republish when a relay arrives later. Ticket reads must use
1433        // the latest address snapshot immediately: waiting here again stalls
1434        // presence and every ticket refresh during relay or DNS outages.
1435        Ok(self.endpoint.watch_addr().get())
1436    }
1437
1438    pub async fn is_connected(&self, endpoint_id: EndpointId) -> bool {
1439        let conns = self.connections.read().await;
1440        if let Some(conn) = conns.get(&endpoint_id) {
1441            conn.close_reason().is_none()
1442        } else {
1443            false
1444        }
1445    }
1446
1447    pub fn accept_events(&self) -> futures::stream::BoxStream<'static, AcceptEvent> {
1448        let receiver = self.accept_events.subscribe();
1449        Box::pin(
1450            BroadcastStream::new(receiver).filter_map(|event| futures::future::ready(event.ok())),
1451        )
1452    }
1453
1454    pub fn connect(
1455        &self,
1456        endpoint_id: EndpointId,
1457    ) -> futures::stream::BoxStream<'static, ConnectEvent> {
1458        let (event_sender, event_receiver) = async_channel::bounded(16);
1459        let endpoint = self.endpoint.clone();
1460        let connections = self.connections.clone();
1461        let connection_inserted_at = self.connection_inserted_at.clone();
1462        let outbound_dial_gates = self.outbound_dial_gates.clone();
1463        let stream_sender = self.incoming_streams.clone();
1464        let hb_mgr = self.heartbeat_manager.clone();
1465        let hb_tx = self.heartbeat_health_tx.clone();
1466        let hb_cfg = self.heartbeat_config.clone();
1467        let manual_notices = self.manual_disconnect_notices.clone();
1468        let probe_registry = self.probe_registry.clone();
1469
1470        tokio::spawn(async move {
1471            let result = connect(
1472                &endpoint,
1473                endpoint_id,
1474                event_sender.clone(),
1475                outbound_dial_gates,
1476                connections,
1477                connection_inserted_at,
1478                stream_sender,
1479                manual_notices,
1480                probe_registry,
1481                hb_mgr,
1482                hb_tx,
1483                hb_cfg,
1484            )
1485            .await;
1486
1487            if let Err(error) = result {
1488                let _ = event_sender
1489                    .send(ConnectEvent::Closed {
1490                        error: Some(error.to_string()),
1491                    })
1492                    .await;
1493            }
1494        });
1495
1496        Box::pin(event_receiver)
1497    }
1498
1499    pub fn connect_addr(
1500        &self,
1501        endpoint_id: EndpointId,
1502        endpoint_addr: EndpointAddr,
1503    ) -> futures::stream::BoxStream<'static, ConnectEvent> {
1504        let (event_sender, event_receiver) = async_channel::bounded(16);
1505        let endpoint = self.endpoint.clone();
1506        let connections = self.connections.clone();
1507        let connection_inserted_at = self.connection_inserted_at.clone();
1508        let outbound_dial_gates = self.outbound_dial_gates.clone();
1509        let stream_sender = self.incoming_streams.clone();
1510        let hb_mgr = self.heartbeat_manager.clone();
1511        let hb_tx = self.heartbeat_health_tx.clone();
1512        let hb_cfg = self.heartbeat_config.clone();
1513        let manual_notices = self.manual_disconnect_notices.clone();
1514        let probe_registry = self.probe_registry.clone();
1515
1516        tokio::spawn(async move {
1517            let result = connect_addr(
1518                &endpoint,
1519                endpoint_id,
1520                endpoint_addr,
1521                event_sender.clone(),
1522                outbound_dial_gates,
1523                connections,
1524                connection_inserted_at,
1525                stream_sender,
1526                manual_notices,
1527                probe_registry,
1528                hb_mgr,
1529                hb_tx,
1530                hb_cfg,
1531            )
1532            .await;
1533
1534            if let Err(error) = result {
1535                let _ = event_sender
1536                    .send(ConnectEvent::Closed {
1537                        error: Some(error.to_string()),
1538                    })
1539                    .await;
1540            }
1541        });
1542
1543        Box::pin(event_receiver)
1544    }
1545
1546    /// Establish a fresh Iroh generation through one prepared custom transport.
1547    /// The existing base generation remains available until the replacement is
1548    /// authenticated and the requested custom path is selected.
1549    pub fn connect_replacement_addr(
1550        &self,
1551        endpoint_id: EndpointId,
1552        endpoint_addr: EndpointAddr,
1553        replacement_transport_id: u64,
1554    ) -> futures::stream::BoxStream<'static, ReplacementConnectEvent> {
1555        let (event_sender, event_receiver) = async_channel::bounded(16);
1556        let endpoint = self.endpoint.clone();
1557        let connections = self.connections.clone();
1558        let connection_inserted_at = self.connection_inserted_at.clone();
1559        let stream_sender = self.incoming_streams.clone();
1560        let hb_mgr = self.heartbeat_manager.clone();
1561        let hb_tx = self.heartbeat_health_tx.clone();
1562        let hb_cfg = self.heartbeat_config.clone();
1563        let manual_notices = self.manual_disconnect_notices.clone();
1564        let probe_registry = self.probe_registry.clone();
1565
1566        tokio::spawn(async move {
1567            if let Err(error) = connect_replacement_addr(
1568                &endpoint,
1569                endpoint_id,
1570                endpoint_addr,
1571                replacement_transport_id,
1572                event_sender.clone(),
1573                connections,
1574                connection_inserted_at,
1575                stream_sender,
1576                manual_notices,
1577                probe_registry,
1578                hb_mgr,
1579                hb_tx,
1580                hb_cfg,
1581            )
1582            .await
1583            {
1584                let _ = event_sender
1585                    .send(ReplacementConnectEvent::Closed {
1586                        error: Some(error.to_string()),
1587                    })
1588                    .await;
1589            }
1590        });
1591
1592        Box::pin(event_receiver)
1593    }
1594
1595    pub async fn disconnect(&self, endpoint_id: EndpointId) -> Result<()> {
1596        self.disconnect_with_reason(
1597            endpoint_id,
1598            crate::lifecycle_reason::REASON_DISCONNECTED_BY_USER,
1599        )
1600        .await
1601    }
1602
1603    pub async fn disconnect_with_reason(
1604        &self,
1605        endpoint_id: EndpointId,
1606        reason: &str,
1607    ) -> Result<()> {
1608        let connection = {
1609            let mut conns = self.connections.write().await;
1610            conns.remove(&endpoint_id)
1611        };
1612
1613        if let Some(conn) = connection {
1614            if std::env::var("PLUTO_RTC_TEARDOWN_TRACE").is_ok() {
1615                eprintln!(
1616                    "[PlutoRTC][teardown-trace] NativeNode::disconnect endpoint_id={}",
1617                    endpoint_id
1618                );
1619            }
1620            conn.close(1u8.into(), reason.as_bytes());
1621        }
1622
1623        Ok(())
1624    }
1625
1626    pub async fn disconnect_with_reason_if_current(
1627        &self,
1628        endpoint_id: EndpointId,
1629        expected_transport_stable_id: u64,
1630        reason: &str,
1631    ) -> Result<bool> {
1632        let connection = {
1633            let mut connections = self.connections.write().await;
1634            let is_current = connections.get(&endpoint_id).is_some_and(|connection| {
1635                transport_generation_for_connection(connection) == expected_transport_stable_id
1636            });
1637            if !is_current {
1638                return Ok(false);
1639            }
1640            connections.remove(&endpoint_id)
1641        };
1642
1643        if let Some(connection) = connection {
1644            connection.close(1u8.into(), reason.as_bytes());
1645            return Ok(true);
1646        }
1647        Ok(false)
1648    }
1649
1650    pub async fn open_bi(&self, endpoint_id: EndpointId) -> Result<(SendStream, RecvStream)> {
1651        let (_, send, recv) = self.open_bi_with_transport_stable_id(endpoint_id).await?;
1652        Ok((send, recv))
1653    }
1654
1655    /// Open a stream and return the physical connection generation that owns
1656    /// it. Reading the id from the same cloned `Connection` avoids labeling a
1657    /// stream with a replacement that raced into the endpoint map.
1658    pub async fn open_bi_with_transport_stable_id(
1659        &self,
1660        endpoint_id: EndpointId,
1661    ) -> Result<(u64, SendStream, RecvStream)> {
1662        let connection = {
1663            let conns = self.connections.read().await;
1664            conns.get(&endpoint_id).cloned()
1665        };
1666
1667        if let Some(conn) = connection {
1668            let transport_stable_id = transport_generation_for_connection(&conn);
1669            let (send, recv) = conn.open_bi().await?;
1670            Ok((transport_stable_id, send, recv))
1671        } else {
1672            Err(anyhow::anyhow!("No active connection to {}", endpoint_id))
1673        }
1674    }
1675
1676    pub async fn open_uni(&self, endpoint_id: EndpointId) -> Result<SendStream> {
1677        let connection = {
1678            let conns = self.connections.read().await;
1679            conns.get(&endpoint_id).cloned()
1680        };
1681
1682        if let Some(conn) = connection {
1683            let send = conn.open_uni().await?;
1684            Ok(send)
1685        } else {
1686            Err(anyhow::anyhow!("No active connection to {}", endpoint_id))
1687        }
1688    }
1689
1690    /// Perform a real remote round-trip on the current physical generation.
1691    /// A replacement that wins during the probe is preserved and checked on
1692    /// the next health pass instead of inheriting the retired leg's failure.
1693    pub async fn probe_connection(
1694        &self,
1695        endpoint_id: EndpointId,
1696        timeout: std::time::Duration,
1697    ) -> Option<IrohConnectionProbe> {
1698        let connection = self.connections.read().await.get(&endpoint_id).cloned()?;
1699        let probed_stable_id = transport_generation_for_connection(&connection);
1700        let responsive = self
1701            .probe_registry
1702            .probe(
1703                &endpoint_id.to_string(),
1704                probed_stable_id,
1705                &connection,
1706                timeout,
1707            )
1708            .await;
1709        let last_inbound_activity_age = self
1710            .probe_registry
1711            .last_inbound_activity_age(&endpoint_id.to_string(), probed_stable_id)
1712            .await;
1713        let current_stable_id = self
1714            .connections
1715            .read()
1716            .await
1717            .get(&endpoint_id)
1718            .map(transport_generation_for_connection)?;
1719        if current_stable_id != probed_stable_id {
1720            return Some(IrohConnectionProbe {
1721                // No pong was observed on the replacement generation. Return
1722                // the generation that was actually probed so callers can treat
1723                // the race as indeterminate and fence any delayed failure.
1724                transport_stable_id: probed_stable_id,
1725                responsive: false,
1726                last_inbound_activity_age,
1727            });
1728        }
1729        Some(IrohConnectionProbe {
1730            transport_stable_id: probed_stable_id,
1731            responsive,
1732            last_inbound_activity_age,
1733        })
1734    }
1735
1736    pub fn incoming_streams_stream(&self) -> async_channel::Receiver<IncomingStream> {
1737        self.incoming_streams_receiver.clone()
1738    }
1739
1740    /// Authorize one protected, peer-scoped inbound custom-transport replacement.
1741    /// The authorization is consumed only by the exact peer and transport id.
1742    pub(crate) async fn authorize_inbound_replacement(
1743        &self,
1744        endpoint_id: EndpointId,
1745        transport_id: u64,
1746        ttl: std::time::Duration,
1747    ) -> Instant {
1748        let expires_at = Instant::now() + ttl;
1749        self.inbound_replacement_authorizations
1750            .write()
1751            .await
1752            .insert(
1753                endpoint_id,
1754                InboundReplacementAuthorization {
1755                    transport_id,
1756                    expires_at,
1757                },
1758            );
1759        expires_at
1760    }
1761
1762    /// Revoke an authorization only when it still names this exact transport.
1763    /// A delayed cleanup task therefore cannot remove a newer replacement grant.
1764    pub(crate) async fn revoke_inbound_replacement_if_current(
1765        &self,
1766        endpoint_id: EndpointId,
1767        transport_id: u64,
1768        expires_at: Instant,
1769    ) -> bool {
1770        let mut authorizations = self.inbound_replacement_authorizations.write().await;
1771        if authorizations.get(&endpoint_id).is_some_and(|current| {
1772            current.transport_id == transport_id && current.expires_at == expires_at
1773        }) {
1774            authorizations.remove(&endpoint_id);
1775            true
1776        } else {
1777            false
1778        }
1779    }
1780
1781    /// Ingest an already-accepted connection into the pluto-rtc native node's
1782    /// internal connection/event/stream pipeline.
1783    pub async fn accept_external_connection(&self, connection: Connection) -> Result<()> {
1784        let replacement_transport_id = consume_authorized_inbound_replacement(
1785            &connection,
1786            &self.inbound_replacement_authorizations,
1787        )
1788        .await;
1789        run_connection_loop(
1790            connection,
1791            self.accept_events.clone(),
1792            self.incoming_streams.clone(),
1793            self.connections.clone(),
1794            self.connection_inserted_at.clone(),
1795            self.endpoint.id(),
1796            IrohConnectionDirection::Inbound,
1797            self.manual_disconnect_notices.clone(),
1798            self.probe_registry.clone(),
1799            self.heartbeat_manager.clone(),
1800            self.heartbeat_health_tx.clone(),
1801            self.heartbeat_config.clone(),
1802            replacement_transport_id,
1803            None,
1804        )
1805        .await
1806        .map_err(|e| anyhow::anyhow!(e.to_string()))
1807    }
1808
1809    /// Ingest an externally accepted connection while reporting the atomic
1810    /// install decision before the stream loop starts. The caller must keep
1811    /// polling this future while awaiting `install_outcome_sender`.
1812    pub(crate) async fn accept_external_connection_with_install_notifier(
1813        &self,
1814        connection: Connection,
1815        install_outcome_sender: oneshot::Sender<ExternalConnectionInstallOutcome>,
1816    ) -> Result<()> {
1817        let replacement_transport_id = consume_authorized_inbound_replacement(
1818            &connection,
1819            &self.inbound_replacement_authorizations,
1820        )
1821        .await;
1822        run_connection_loop(
1823            connection,
1824            self.accept_events.clone(),
1825            self.incoming_streams.clone(),
1826            self.connections.clone(),
1827            self.connection_inserted_at.clone(),
1828            self.endpoint.id(),
1829            IrohConnectionDirection::Inbound,
1830            self.manual_disconnect_notices.clone(),
1831            self.probe_registry.clone(),
1832            self.heartbeat_manager.clone(),
1833            self.heartbeat_health_tx.clone(),
1834            self.heartbeat_config.clone(),
1835            replacement_transport_id,
1836            Some(install_outcome_sender),
1837        )
1838        .await
1839        .map_err(|e| anyhow::anyhow!(e.to_string()))
1840    }
1841
1842    /// Get a raw iroh::Connection for a given EndpointId, if one exists.
1843    /// Used by external protocol handlers (handshake, bucket_sync) to attach
1844    /// application-level logic on top of pluto-rtc-managed connections.
1845    pub async fn get_connection(&self, endpoint_id: EndpointId) -> Option<Connection> {
1846        let conns = self.connections.read().await;
1847        conns.get(&endpoint_id).cloned()
1848    }
1849
1850    /// List all currently active EndpointIds with connections.
1851    pub async fn active_endpoint_ids(&self) -> Vec<EndpointId> {
1852        let conns = self.connections.read().await;
1853        conns.keys().cloned().collect()
1854    }
1855
1856    pub async fn take_manual_disconnect_notice(
1857        &self,
1858        endpoint_id: EndpointId,
1859        transport_stable_id: u64,
1860    ) -> bool {
1861        self.manual_disconnect_notices
1862            .write()
1863            .await
1864            .remove(&manual_disconnect_notice_key(
1865                endpoint_id,
1866                transport_stable_id,
1867            ))
1868    }
1869
1870    /// Observe an explicit peer disconnect without consuming the generation-
1871    /// scoped notice. Both the generic physical-close watcher and the
1872    /// incoming-handler lifecycle may observe the same close; only the latter
1873    /// removes the notice after its terminal transition is fenced.
1874    pub async fn has_manual_disconnect_notice(
1875        &self,
1876        endpoint_id: EndpointId,
1877        transport_stable_id: u64,
1878    ) -> bool {
1879        self.manual_disconnect_notices
1880            .read()
1881            .await
1882            .contains(&manual_disconnect_notice_key(
1883                endpoint_id,
1884                transport_stable_id,
1885            ))
1886    }
1887}
1888
1889#[cfg(test)]
1890mod tests {
1891    use super::*;
1892    use iroh::endpoint::Endpoint;
1893    use tokio::io::AsyncWriteExt;
1894    use tokio::time::{sleep, timeout, Duration};
1895
1896    async fn setup_protocol_endpoint() -> (
1897        Router,
1898        PlutoniumProtocol,
1899        broadcast::Receiver<AcceptEvent>,
1900        async_channel::Receiver<IncomingStream>,
1901        Arc<RwLock<HashMap<EndpointId, Connection>>>,
1902    ) {
1903        let (event_tx, event_rx) = broadcast::channel(16);
1904        let (stream_tx, stream_rx) = async_channel::unbounded();
1905        let connections = Arc::new(RwLock::new(HashMap::new()));
1906        let connection_inserted_at = Arc::new(RwLock::new(HashMap::new()));
1907
1908        let endpoint = Endpoint::builder(iroh::endpoint::presets::N0)
1909            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
1910            .bind()
1911            .await
1912            .unwrap();
1913
1914        let protocol = PlutoniumProtocol::new(
1915            event_tx.clone(),
1916            stream_tx.clone(),
1917            connections.clone(),
1918            connection_inserted_at,
1919            endpoint.id(),
1920            Arc::new(RwLock::new(HashSet::new())),
1921            IrohProbeRegistry::new(),
1922            Arc::new(RwLock::new(HashMap::new())),
1923        );
1924
1925        let router = Router::builder(endpoint)
1926            .accept(PlutoniumProtocol::ALPN, Arc::new(protocol.clone()))
1927            .spawn();
1928
1929        (router, protocol, event_rx, stream_rx, connections)
1930    }
1931
1932    #[test]
1933    fn manual_disconnect_notice_is_scoped_to_transport_generation() {
1934        let endpoint_id = iroh::SecretKey::generate().public();
1935        let old_transport = manual_disconnect_notice_key(endpoint_id, 41);
1936        let replacement_transport = manual_disconnect_notice_key(endpoint_id, 42);
1937        let mut notices = HashSet::from([old_transport.clone()]);
1938
1939        assert_ne!(old_transport, replacement_transport);
1940        assert!(!notices.remove(&replacement_transport));
1941        assert!(notices.remove(&old_transport));
1942    }
1943
1944    #[test]
1945    fn authorized_custom_transport_replaces_live_base_connection() {
1946        assert_eq!(
1947            custom_transport_arbitration_override(true, None, Some(4344901), Some(4344901)),
1948            Some(IrohConnectionInstallDecision::ReplaceExisting {
1949                close_existing_reason:
1950                    crate::lifecycle_reason::REASON_NATIVE_CUSTOM_TRANSPORT_UPGRADE,
1951            })
1952        );
1953    }
1954
1955    #[test]
1956    fn live_custom_transport_wins_against_late_non_custom_duplicate() {
1957        assert_eq!(
1958            custom_transport_arbitration_override(true, Some(4344901), None, None),
1959            Some(IrohConnectionInstallDecision::KeepExisting {
1960                close_fresh_reason: "existing-custom-transport-preferred",
1961            })
1962        );
1963    }
1964
1965    #[test]
1966    fn canonical_outbound_dial_cannot_preempt_live_custom_transport() {
1967        let existing = Some(ExistingConnectionState {
1968            same_stable_id: false,
1969            alive: true,
1970            direction: IrohConnectionDirection::Inbound,
1971            age_ms: 0,
1972        });
1973
1974        assert!(!should_start_outbound_dial_with_transport_preference(
1975            Some((true, IrohConnectionDirection::Inbound)),
1976            Some(4344901),
1977            true,
1978        ));
1979        assert_eq!(
1980            decide_outbound_install_with_transport_preference(existing, Some(4344901), None, true,),
1981            IrohConnectionInstallDecision::KeepExisting {
1982                close_fresh_reason: "existing-custom-transport-preferred",
1983            },
1984        );
1985    }
1986
1987    #[test]
1988    fn unauthorized_custom_transport_uses_normal_direction_arbitration() {
1989        assert_eq!(
1990            custom_transport_arbitration_override(true, None, Some(4344901), None),
1991            None
1992        );
1993    }
1994
1995    #[tokio::test]
1996    async fn delayed_replacement_cleanup_cannot_revoke_a_newer_grant() {
1997        let endpoint = Endpoint::builder(iroh::endpoint::presets::N0)
1998            .relay_mode(iroh::RelayMode::Disabled)
1999            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2000            .bind()
2001            .await
2002            .expect("endpoint");
2003        let node = IrohNativeNode::spawn_with_endpoint_no_router(endpoint)
2004            .await
2005            .expect("native node");
2006        let remote = iroh::SecretKey::generate().public();
2007        let old_expiry = node
2008            .authorize_inbound_replacement(remote, 4344901, Duration::from_secs(1))
2009            .await;
2010        let new_expiry = node
2011            .authorize_inbound_replacement(remote, 4344901, Duration::from_secs(2))
2012            .await;
2013
2014        assert!(
2015            !node
2016                .revoke_inbound_replacement_if_current(remote, 4344901, old_expiry)
2017                .await
2018        );
2019        assert!(
2020            node.revoke_inbound_replacement_if_current(remote, 4344901, new_expiry)
2021                .await
2022        );
2023    }
2024
2025    #[tokio::test]
2026    async fn node_addr_does_not_wait_for_an_unavailable_relay() {
2027        let endpoint = Endpoint::builder(iroh::endpoint::presets::N0)
2028            .relay_mode(iroh::RelayMode::Disabled)
2029            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2030            .bind()
2031            .await
2032            .expect("bind relay-disabled endpoint");
2033        let expected_id = endpoint.id();
2034        let node = IrohNativeNode::spawn_with_endpoint(endpoint)
2035            .await
2036            .expect("spawn relay-disabled node");
2037
2038        let address = timeout(Duration::from_secs(1), node.node_addr())
2039            .await
2040            .expect("node_addr must not wait for relay readiness")
2041            .expect("read current endpoint address");
2042
2043        assert_eq!(address.id, expected_id);
2044    }
2045
2046    #[tokio::test]
2047    async fn test_two_nodes_connect_and_exchange_streams() {
2048        let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2049        let (r2, _proto2, mut events2, streams2, _) = setup_protocol_endpoint().await;
2050
2051        let ep1 = r1.endpoint();
2052        let ep2 = r2.endpoint();
2053
2054        // Exchange addr
2055        let addr2 = ep2.addr();
2056
2057        // Node 1 connects to Node 2
2058        let conn_res = ep1.connect(addr2, PlutoniumProtocol::ALPN).await.unwrap();
2059
2060        // Wait for connect event on Node 2
2061        let event = timeout(Duration::from_secs(5), events2.recv())
2062            .await
2063            .unwrap()
2064            .unwrap();
2065        match event {
2066            AcceptEvent::Accepted { endpoint_id, .. } => assert_eq!(endpoint_id, ep1.id()),
2067            _ => panic!("Expected AcceptEvent::Accepted"),
2068        }
2069
2070        // Node 1 opens stream
2071        let (mut send1, _recv1) = conn_res.open_bi().await.unwrap();
2072        send1.write_all(b"hello node2").await.unwrap();
2073
2074        // Node 2 receives stream
2075        let incoming = timeout(Duration::from_secs(5), streams2.recv())
2076            .await
2077            .unwrap()
2078            .unwrap();
2079        assert_eq!(incoming.endpoint_id, ep1.id());
2080
2081        let mut recv2 = match incoming.stream {
2082            IncomingStreamType::Bi(_, r) => r,
2083            _ => panic!("Expected Bi stream"),
2084        };
2085
2086        let mut buf = [0u8; 11];
2087        recv2.read_exact(&mut buf).await.unwrap();
2088        assert_eq!(&buf, b"hello node2");
2089    }
2090
2091    #[tokio::test]
2092    async fn retiring_losing_control_candidate_preserves_buffered_verdict() {
2093        let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2094        let (r2, _proto2, mut events2, streams2, _) = setup_protocol_endpoint().await;
2095        let ep1 = r1.endpoint();
2096        let ep2 = r2.endpoint();
2097        let connection = ep1
2098            .connect(ep2.addr(), PlutoniumProtocol::ALPN)
2099            .await
2100            .expect("connect protocol endpoints");
2101        timeout(Duration::from_secs(5), events2.recv())
2102            .await
2103            .expect("host accept timeout")
2104            .expect("host accept event");
2105
2106        let (mut dialer_send, mut dialer_recv) =
2107            connection.open_bi().await.expect("open candidate stream");
2108        dialer_send
2109            .write_all(b"candidate")
2110            .await
2111            .expect("activate candidate stream");
2112        dialer_send.flush().await.expect("flush candidate stream");
2113        let incoming = timeout(Duration::from_secs(5), streams2.recv())
2114            .await
2115            .expect("host stream timeout")
2116            .expect("host stream");
2117        let IncomingStreamType::Bi(mut host_send, host_recv) = incoming.stream else {
2118            panic!("expected bidirectional candidate stream");
2119        };
2120        let verdict = b"session-token-approved";
2121        host_send
2122            .write_all(verdict)
2123            .await
2124            .expect("write approval verdict");
2125        host_send.flush().await.expect("flush approval verdict");
2126
2127        let (retired, received) = tokio::join!(
2128            crate::client::retire_losing_native_control_candidate(host_send, host_recv),
2129            dialer_recv.read_to_end(256),
2130        );
2131        retired.expect("retire candidate after peer acknowledges verdict");
2132        assert_eq!(
2133            received.expect("dialer reads losing-stream verdict"),
2134            verdict,
2135        );
2136    }
2137
2138    #[tokio::test]
2139    async fn retiring_replaced_control_send_preserves_buffered_verdict() {
2140        let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2141        let (r2, _proto2, mut events2, streams2, _) = setup_protocol_endpoint().await;
2142        let connection = r1
2143            .endpoint()
2144            .connect(r2.endpoint().addr(), PlutoniumProtocol::ALPN)
2145            .await
2146            .expect("connect protocol endpoints");
2147        timeout(Duration::from_secs(5), events2.recv())
2148            .await
2149            .expect("host accept timeout")
2150            .expect("host accept event");
2151
2152        let (mut dialer_send, mut dialer_recv) =
2153            connection.open_bi().await.expect("open displaced stream");
2154        dialer_send
2155            .write_all(b"candidate")
2156            .await
2157            .expect("activate displaced stream");
2158        dialer_send.flush().await.expect("flush displaced stream");
2159        let incoming = timeout(Duration::from_secs(5), streams2.recv())
2160            .await
2161            .expect("host stream timeout")
2162            .expect("host stream");
2163        let IncomingStreamType::Bi(mut host_send, _host_recv) = incoming.stream else {
2164            panic!("expected bidirectional displaced stream");
2165        };
2166        let verdict = b"session-token-approved";
2167        host_send
2168            .write_all(verdict)
2169            .await
2170            .expect("write approval verdict");
2171        host_send.flush().await.expect("host should flush approval");
2172        let displaced_send = Arc::new(tokio::sync::Mutex::new(host_send));
2173
2174        let (retired, received) = tokio::join!(
2175            crate::client::retire_replaced_native_control_send(displaced_send),
2176            dialer_recv.read_to_end(256),
2177        );
2178        retired.expect("retire replaced stream after peer acknowledges verdict");
2179        assert_eq!(
2180            received.expect("dialer reads replaced-stream verdict"),
2181            verdict,
2182        );
2183    }
2184
2185    #[tokio::test]
2186    async fn test_healthy_connection_not_replaced() {
2187        let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2188        let (r2, _proto2, mut events2, _streams2, conns2) = setup_protocol_endpoint().await;
2189
2190        let ep1 = r1.endpoint();
2191        let ep2 = r2.endpoint();
2192
2193        let addr2 = ep2.addr();
2194        let addr1 = ep1.addr();
2195
2196        // 1. First connection
2197        let _conn1 = ep1.connect(addr2, PlutoniumProtocol::ALPN).await.unwrap();
2198
2199        // Wait for connection to be registered in Protocol 2
2200        let _ = timeout(Duration::from_secs(5), events2.recv())
2201            .await
2202            .unwrap()
2203            .unwrap();
2204
2205        let active_count = conns2.read().await.len();
2206        assert_eq!(active_count, 1);
2207
2208        let original_stable_id = conns2.read().await.get(&ep1.id()).unwrap().stable_id();
2209
2210        // 2. Dual-dial: Node 2 connects to Node 1 while connection is still healthy
2211        let _conn2 = ep2.connect(addr1, PlutoniumProtocol::ALPN).await.unwrap();
2212
2213        // Allow time for the second connection to process
2214        sleep(Duration::from_millis(100)).await;
2215
2216        // The original connection should still be intact because it wasn't closed
2217        let current_conn = conns2.read().await.get(&ep1.id()).unwrap().clone();
2218        assert_eq!(current_conn.stable_id(), original_stable_id);
2219    }
2220
2221    #[tokio::test]
2222    async fn fresh_connection_replaces_an_unresponsive_same_identity_transport() {
2223        let (host, _protocol, mut host_events, _streams, host_connections) =
2224            setup_protocol_endpoint().await;
2225        let host_addr = host.endpoint().addr();
2226        let remote_secret = iroh::SecretKey::generate();
2227
2228        let first_remote = Endpoint::builder(iroh::endpoint::presets::N0)
2229            .secret_key(remote_secret.clone())
2230            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2231            .bind()
2232            .await
2233            .expect("bind first remote runtime");
2234        let _first_connection = first_remote
2235            .connect(host_addr.clone(), PlutoniumProtocol::ALPN)
2236            .await
2237            .expect("connect first remote runtime");
2238        timeout(Duration::from_secs(5), host_events.recv())
2239            .await
2240            .expect("first host accept timeout")
2241            .expect("first host accept");
2242        let first_stable_id = host_connections
2243            .read()
2244            .await
2245            .get(&remote_secret.public())
2246            .expect("first transport installed")
2247            .stable_id();
2248
2249        // Model a process crash: the old QUIC handle still looks locally open,
2250        // but the old runtime is no longer consuming OpenRTC control streams.
2251        // A replacement process starts with the same durable endpoint identity.
2252        let replacement_remote = Endpoint::builder(iroh::endpoint::presets::N0)
2253            .secret_key(remote_secret.clone())
2254            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2255            .bind()
2256            .await
2257            .expect("bind replacement remote runtime");
2258        let _replacement_connection = replacement_remote
2259            .connect(host_addr, PlutoniumProtocol::ALPN)
2260            .await
2261            .expect("connect replacement remote runtime");
2262        timeout(Duration::from_secs(5), host_events.recv())
2263            .await
2264            .expect("replacement host accept timeout")
2265            .expect("replacement host accept");
2266
2267        let replacement_stable_id = host_connections
2268            .read()
2269            .await
2270            .get(&remote_secret.public())
2271            .expect("replacement transport installed")
2272            .stable_id();
2273        assert_ne!(
2274            replacement_stable_id, first_stable_id,
2275            "an actively unresponsive old leg must not block a same-identity process restart",
2276        );
2277    }
2278
2279    /// Symmetric idle iroh heartbeat should open a **bounded** number of uni streams
2280    /// (ping + pong on each side). Bursts far above this model usually mean non-heartbeat
2281    /// traffic or regressions.
2282    #[tokio::test]
2283    async fn idle_symmetric_iroh_heartbeat_send_uni_open_rate_bounded() {
2284        use crate::heartbeat::idle_symmetric_heartbeat_max_send_uni_opens_upper_bound;
2285        use crate::heartbeat::iroh_heartbeat::test_counters;
2286        use std::sync::atomic::Ordering;
2287
2288        test_counters::reset_heartbeat_send_uni_count();
2289
2290        let tick = Duration::from_millis(200);
2291        let heartbeat_config = HeartbeatConfig {
2292            tick_interval: tick,
2293            suspect_after: Duration::from_secs(10),
2294            stale_after: Duration::from_secs(30),
2295            send_timeout: Duration::from_secs(2),
2296        };
2297
2298        let (tx1, _rx1) = mpsc::channel::<HealthTransition>(32);
2299        let (tx2, _rx2) = mpsc::channel::<HealthTransition>(32);
2300        let mgr1 = IrohHeartbeatManager::new();
2301        let mgr2 = IrohHeartbeatManager::new();
2302
2303        let ep1 = Endpoint::builder(iroh::endpoint::presets::N0)
2304            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2305            .bind()
2306            .await
2307            .unwrap();
2308        let ep2 = Endpoint::builder(iroh::endpoint::presets::N0)
2309            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2310            .bind()
2311            .await
2312            .unwrap();
2313
2314        let node1 = IrohNativeNode::spawn_with_heartbeat(ep1, mgr1, tx1, heartbeat_config.clone())
2315            .await
2316            .unwrap();
2317        let node2 = IrohNativeNode::spawn_with_heartbeat(ep2, mgr2, tx2, heartbeat_config)
2318            .await
2319            .unwrap();
2320
2321        let remote_id = node2.endpoint().id();
2322        let remote_addr = node2.node_addr().await.unwrap();
2323
2324        let mut conn_stream = node1.connect_addr(remote_id, remote_addr);
2325        let connected = timeout(Duration::from_secs(5), async {
2326            while let Some(ev) = conn_stream.next().await {
2327                match ev {
2328                    ConnectEvent::Connected => return true,
2329                    ConnectEvent::Closed { .. } => return false,
2330                }
2331            }
2332            false
2333        })
2334        .await
2335        .unwrap();
2336        assert!(connected, "expected outbound connect to reach Connected");
2337
2338        let observe = Duration::from_millis(900);
2339        sleep(observe).await;
2340
2341        let observed = test_counters::HEARTBEAT_SEND_UNI_COUNT.load(Ordering::SeqCst);
2342        let bound = idle_symmetric_heartbeat_max_send_uni_opens_upper_bound(observe, tick);
2343        assert!(
2344            observed <= bound,
2345            "heartbeat send_uni opens should stay within idle symmetric model (observed={} bound={})",
2346            observed,
2347            bound
2348        );
2349        assert!(
2350            observed >= 4,
2351            "expected some heartbeat uni traffic after idle window (observed={})",
2352            observed
2353        );
2354    }
2355
2356    #[tokio::test]
2357    async fn active_probe_requires_remote_control_loop_round_trip() {
2358        let ep1 = Endpoint::builder(iroh::endpoint::presets::N0)
2359            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2360            .bind()
2361            .await
2362            .expect("bind first endpoint");
2363        let ep2 = Endpoint::builder(iroh::endpoint::presets::N0)
2364            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2365            .bind()
2366            .await
2367            .expect("bind second endpoint");
2368
2369        let node1 = IrohNativeNode::spawn_with_endpoint(ep1)
2370            .await
2371            .expect("spawn first node");
2372        let node2 = IrohNativeNode::spawn_with_endpoint(ep2)
2373            .await
2374            .expect("spawn second node");
2375        let remote_id = node2.endpoint().id();
2376        let mut events = node1.connect_addr(
2377            remote_id,
2378            node2.node_addr().await.expect("second node address"),
2379        );
2380
2381        let connected = timeout(Duration::from_secs(5), async {
2382            while let Some(event) = events.next().await {
2383                if matches!(event, ConnectEvent::Connected) {
2384                    return true;
2385                }
2386            }
2387            false
2388        })
2389        .await
2390        .expect("connect timeout");
2391        assert!(connected, "expected physical connection");
2392
2393        let probe = node1
2394            .probe_connection(remote_id, Duration::from_secs(2))
2395            .await
2396            .expect("current physical generation");
2397        assert!(
2398            probe.responsive,
2399            "remote runtime must return the typed pong"
2400        );
2401        assert_ne!(probe.transport_stable_id, 0);
2402        assert!(
2403            probe.last_inbound_activity_age.is_some(),
2404            "the accepted pong stream must also be recorded as generation-bound activity",
2405        );
2406    }
2407
2408    #[tokio::test]
2409    async fn active_probe_is_not_starved_by_bidirectional_stream_load() {
2410        let ep1 = Endpoint::builder(iroh::endpoint::presets::N0)
2411            .relay_mode(iroh::RelayMode::Disabled)
2412            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2413            .bind()
2414            .await
2415            .expect("bind first endpoint");
2416        let ep2 = Endpoint::builder(iroh::endpoint::presets::N0)
2417            .relay_mode(iroh::RelayMode::Disabled)
2418            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2419            .bind()
2420            .await
2421            .expect("bind second endpoint");
2422
2423        let node1 = IrohNativeNode::spawn_with_endpoint(ep1)
2424            .await
2425            .expect("spawn first node");
2426        let node2 = IrohNativeNode::spawn_with_endpoint(ep2)
2427            .await
2428            .expect("spawn second node");
2429        let remote_id = node2.endpoint().id();
2430        let mut events = node1.connect_addr(
2431            remote_id,
2432            node2.node_addr().await.expect("second node address"),
2433        );
2434        timeout(Duration::from_secs(5), async {
2435            while let Some(event) = events.next().await {
2436                if matches!(event, ConnectEvent::Connected) {
2437                    return;
2438                }
2439            }
2440            panic!("connection event stream ended");
2441        })
2442        .await
2443        .expect("connect timeout");
2444
2445        let incoming = node2.incoming_streams_stream();
2446        let drain = tokio::spawn(async move { while incoming.recv().await.is_ok() {} });
2447        let flood_node = node1.clone();
2448        let flood = tokio::spawn(async move {
2449            loop {
2450                let Ok((mut send, _recv)) = flood_node.open_bi(remote_id).await else {
2451                    break;
2452                };
2453                if send.write_all(b"x").await.is_err() {
2454                    break;
2455                }
2456                let _ = send.finish();
2457                tokio::task::yield_now().await;
2458            }
2459        });
2460
2461        sleep(Duration::from_millis(25)).await;
2462        let probe = node1
2463            .probe_connection(remote_id, Duration::from_secs(2))
2464            .await
2465            .expect("current physical generation");
2466        flood.abort();
2467        drain.abort();
2468
2469        assert!(
2470            probe.responsive,
2471            "bidirectional application/control load must not starve the uni-stream liveness control plane",
2472        );
2473    }
2474
2475    #[tokio::test]
2476    async fn concurrent_native_address_dials_install_one_physical_generation() {
2477        let ep1 = Endpoint::builder(iroh::endpoint::presets::N0)
2478            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2479            .bind()
2480            .await
2481            .expect("bind first endpoint");
2482        let ep2 = Endpoint::builder(iroh::endpoint::presets::N0)
2483            .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2484            .bind()
2485            .await
2486            .expect("bind second endpoint");
2487
2488        let node1 = IrohNativeNode::spawn_with_endpoint(ep1)
2489            .await
2490            .expect("spawn first node");
2491        let node2 = IrohNativeNode::spawn_with_endpoint(ep2)
2492            .await
2493            .expect("spawn second node");
2494        let remote_id = node2.endpoint().id();
2495        let remote_addr = node2.node_addr().await.expect("second node address");
2496        let mut remote_events = node2.accept_events();
2497
2498        // Exercise the physical owner directly. All callers begin before any
2499        // one of them has observed Connected, matching concurrent recovery
2500        // wakes after a network change.
2501        let mut dial_streams = (0..16)
2502            .map(|_| node1.connect_addr(remote_id, remote_addr.clone()))
2503            .collect::<Vec<_>>();
2504        let connected = futures::future::join_all(dial_streams.iter_mut().map(|events| async {
2505            timeout(Duration::from_secs(5), async {
2506                while let Some(event) = events.next().await {
2507                    match event {
2508                        ConnectEvent::Connected => return true,
2509                        ConnectEvent::Closed { .. } => return false,
2510                    }
2511                }
2512                false
2513            })
2514            .await
2515            .expect("concurrent dial timeout")
2516        }))
2517        .await;
2518        assert!(
2519            connected.iter().all(|connected| *connected),
2520            "every caller should reuse the one installed physical connection"
2521        );
2522
2523        let accepted = timeout(Duration::from_secs(5), remote_events.next())
2524            .await
2525            .expect("remote accept timeout")
2526            .expect("remote accept event");
2527        assert!(matches!(accepted, AcceptEvent::Accepted { .. }));
2528        let unexpected_second_accept = timeout(Duration::from_millis(300), async {
2529            while let Some(event) = remote_events.next().await {
2530                if matches!(event, AcceptEvent::Accepted { .. }) {
2531                    return Some(event);
2532                }
2533            }
2534            None
2535        })
2536        .await;
2537        assert!(
2538            unexpected_second_accept.is_err(),
2539            "concurrent callers must not create a second remote generation"
2540        );
2541        assert_eq!(
2542            node1.active_endpoint_ids().await,
2543            vec![remote_id],
2544            "exactly one physical endpoint mapping should remain"
2545        );
2546        assert!(
2547            node1
2548                .outbound_dial_gates
2549                .lock()
2550                .unwrap_or_else(|poisoned| poisoned.into_inner())
2551                .is_empty(),
2552            "completed single-flight entries must be retired"
2553        );
2554    }
2555
2556    #[tokio::test]
2557    async fn application_uni_stream_prefix_is_replayed_after_control_classification() {
2558        let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2559        let (r2, _proto2, mut events2, streams2, _) = setup_protocol_endpoint().await;
2560        let connection = r1
2561            .endpoint()
2562            .connect(r2.endpoint().addr(), PlutoniumProtocol::ALPN)
2563            .await
2564            .expect("connect endpoints");
2565        timeout(Duration::from_secs(5), events2.recv())
2566            .await
2567            .expect("accept event timeout")
2568            .expect("accept event");
2569
2570        let payload = b"application-uni-payload-that-is-not-control";
2571        let mut send = connection.open_uni().await.expect("open app uni stream");
2572        send.write_all(payload)
2573            .await
2574            .expect("write app uni payload");
2575        send.finish().expect("finish app uni payload");
2576
2577        let incoming = timeout(Duration::from_secs(5), streams2.recv())
2578            .await
2579            .expect("application stream timeout")
2580            .expect("application stream");
2581        let IncomingStreamType::Uni(mut recv) = incoming.stream else {
2582            panic!("expected application uni stream");
2583        };
2584        let mut received = Vec::new();
2585        tokio::io::AsyncReadExt::read_to_end(&mut recv, &mut received)
2586            .await
2587            .expect("read replayed application payload");
2588        assert_eq!(received, payload);
2589    }
2590}