1#![cfg(not(target_arch = "wasm32"))]
2
3use anyhow::Result;
4use futures::StreamExt;
5use iroh::{
6 endpoint::{Connection, RecvStream, SendStream},
7 protocol::{AcceptError, ProtocolHandler, Router},
8 Endpoint, EndpointAddr, EndpointId, Watcher,
9};
10use serde::{Deserialize, Serialize};
11use std::collections::{HashMap, HashSet};
12use std::sync::{Arc, Mutex as StdMutex, Weak};
13use std::time::Instant;
14use tokio::sync::{broadcast, mpsc, oneshot, RwLock};
15use tokio_stream::wrappers::BroadcastStream;
16
17use crate::heartbeat::{
18 classify_incoming_uni, parse_incoming_pong, respond_to_ping, HealthTransition, HeartbeatConfig,
19 IncomingUniClassification, IrohConnectionProbe, IrohHeartbeatManager, IrohProbeRegistry,
20 PrefixedRecvStream,
21};
22use crate::iroh_connection_policy::{
23 decide_inbound_install, decide_outbound_install, should_start_outbound_dial,
24 should_wait_for_canonical_inbound, ExistingConnectionState, IrohConnectionDirection,
25 IrohConnectionInstallDecision, NONCANONICAL_OUTBOUND_DIAL_GRACE_MS,
26};
27
28pub fn native_iroh_transport_config() -> iroh::endpoint::QuicTransportConfig {
33 use iroh::endpoint::VarInt;
34
35 let idle_timeout = std::time::Duration::from_secs(120)
36 .try_into()
37 .expect("120 seconds is a valid Iroh idle timeout");
38 let keep_alive_secs = if cfg!(any(target_os = "ios", target_os = "android")) {
39 90
40 } else {
41 25
42 };
43
44 iroh::endpoint::QuicTransportConfig::builder()
45 .max_idle_timeout(Some(idle_timeout))
46 .keep_alive_interval(std::time::Duration::from_secs(keep_alive_secs))
47 .stream_receive_window(VarInt::from_u32(8 * 1024 * 1024))
48 .receive_window(VarInt::from_u32(16 * 1024 * 1024))
49 .send_window(16 * 1024 * 1024)
50 .build()
51}
52
53const INSTALL_COLLISION_PROBE_TIMEOUT: std::time::Duration =
54 std::time::Duration::from_millis(1_000);
55
56pub(crate) fn transport_generation_for_connection(connection: &Connection) -> u64 {
57 crate::transport_generation::for_connection(connection)
58}
59
60fn forget_transport_generation(connection: &Connection) {
61 crate::transport_generation::forget_connection(connection);
62}
63
64async fn existing_transport_is_live_for_install(
65 endpoint_id: EndpointId,
66 previous: &Connection,
67 fresh_stable_id: u64,
68 fresh_direction: IrohConnectionDirection,
69 probe_registry: &IrohProbeRegistry,
70) -> bool {
71 let previous_stable_id = transport_generation_for_connection(previous);
72 if previous_stable_id == fresh_stable_id {
73 return previous.close_reason().is_none();
74 }
75 if previous.close_reason().is_some() {
76 return false;
77 }
78
79 let responsive = probe_registry
80 .probe(
81 &endpoint_id.to_string(),
82 previous_stable_id,
83 previous,
84 INSTALL_COLLISION_PROBE_TIMEOUT,
85 )
86 .await;
87 if !responsive {
88 let reason = match fresh_direction {
89 IrohConnectionDirection::Inbound => {
90 crate::lifecycle_reason::REASON_ZOMBIE_REPLACED_BY_FRESH_ACCEPT
91 }
92 IrohConnectionDirection::Outbound => {
93 crate::lifecycle_reason::REASON_ZOMBIE_REPLACED_BY_FRESH_DIAL
94 }
95 };
96 println!(
97 "[OpenRTC][iroh-install] replacing unresponsive transport endpoint_id={} previous_stable_id={} fresh_stable_id={} fresh_direction={:?}",
98 endpoint_id,
99 previous_stable_id,
100 fresh_stable_id,
101 fresh_direction,
102 );
103 previous.close(0u8.into(), reason.as_bytes());
104 }
105 responsive
106}
107
108async fn should_break_accept_loop(connection: &Connection) -> bool {
109 if connection.close_reason().is_some() {
110 return true;
111 }
112 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
113 connection.close_reason().is_some()
114}
115
116fn local_prefers_outbound(local_endpoint_id: EndpointId, remote_endpoint_id: EndpointId) -> bool {
117 local_endpoint_id.to_string() > remote_endpoint_id.to_string()
118}
119
120async fn outbound_dial_precheck(
121 endpoint: &Endpoint,
122 endpoint_id: EndpointId,
123 connections: &Arc<RwLock<HashMap<EndpointId, Connection>>>,
124 connection_inserted_at: &Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
125) -> bool {
126 let prefers_outbound = local_prefers_outbound(endpoint.id(), endpoint_id);
127 let existing = {
128 let conns = connections.read().await;
129 if let Some(existing) = conns.get(&endpoint_id) {
130 let direction = connection_inserted_at
131 .read()
132 .await
133 .get(&endpoint_id)
134 .map(|metadata| metadata.direction)
135 .unwrap_or(IrohConnectionDirection::Outbound);
136 Some((
137 existing.close_reason().is_none(),
138 direction,
139 selected_custom_transport_id(existing),
140 ))
141 } else {
142 None
143 }
144 };
145 let existing_state = existing.map(|(alive, direction, _)| (alive, direction));
146 let existing_custom_transport_id =
147 existing.and_then(|(_, _, custom_transport_id)| custom_transport_id);
148
149 if !should_start_outbound_dial_with_transport_preference(
150 existing_state,
151 existing_custom_transport_id,
152 prefers_outbound,
153 ) {
154 return true;
155 }
156 if !should_wait_for_canonical_inbound(existing_state, prefers_outbound) {
157 return false;
158 }
159
160 tokio::time::sleep(std::time::Duration::from_millis(
161 NONCANONICAL_OUTBOUND_DIAL_GRACE_MS,
162 ))
163 .await;
164 connections
165 .read()
166 .await
167 .get(&endpoint_id)
168 .is_some_and(|connection| connection.close_reason().is_none())
169}
170
171fn manual_disconnect_notice_key(endpoint_id: EndpointId, transport_stable_id: u64) -> String {
172 format!("{endpoint_id}:{transport_stable_id}")
173}
174
175fn heartbeat_connection_key(endpoint_id: EndpointId, transport_stable_id: u64) -> String {
176 format!("{endpoint_id}:{transport_stable_id}")
177}
178
179#[derive(Debug, Clone, Copy)]
180struct IrohConnectionMetadata {
181 inserted_at: Instant,
182 direction: IrohConnectionDirection,
183}
184
185#[derive(Debug)]
186pub enum IncomingStreamType {
187 Bi(SendStream, RecvStream),
188 Uni(PrefixedRecvStream),
189}
190
191#[derive(Debug)]
192pub struct IncomingStream {
193 pub endpoint_id: EndpointId,
194 pub transport_stable_id: u64,
198 pub recv_prefix: Vec<u8>,
201 pub stream: IncomingStreamType,
202}
203
204#[derive(Debug, Clone, Serialize, Deserialize)]
205#[serde(tag = "type", rename_all = "camelCase")]
206pub enum ConnectEvent {
207 Connected,
208 Closed { error: Option<String> },
209}
210
211#[derive(Debug, Clone, Serialize, Deserialize)]
212#[serde(tag = "type", rename_all = "camelCase")]
213pub enum ReplacementConnectEvent {
214 Connected { transport_stable_id: u64 },
215 Closed { error: Option<String> },
216}
217
218#[derive(Debug, Clone, Serialize, Deserialize)]
219#[serde(tag = "type", rename_all = "camelCase")]
220pub enum AcceptEvent {
221 Accepted {
222 endpoint_id: EndpointId,
223 transport_stable_id: u64,
224 replaced_transport_stable_id: Option<u64>,
225 },
226 Closed {
227 endpoint_id: EndpointId,
228 transport_stable_id: u64,
229 error: Option<String>,
230 was_outbound: bool,
235 was_locally_closed: bool,
240 },
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247pub(crate) enum ExternalConnectionInstallOutcome {
248 Installed {
249 transport_stable_id: u64,
250 replaced_transport_stable_id: Option<u64>,
251 },
252 KeptExisting {
253 fresh_transport_stable_id: u64,
254 kept_transport_stable_id: u64,
255 },
256}
257
258#[derive(Debug, Clone, Copy)]
259struct InboundReplacementAuthorization {
260 transport_id: u64,
261 expires_at: Instant,
262}
263
264type InboundReplacementAuthorizations =
265 Arc<RwLock<HashMap<EndpointId, InboundReplacementAuthorization>>>;
266
267type OutboundDialGate = tokio::sync::Mutex<()>;
268type OutboundDialGates = Arc<StdMutex<HashMap<EndpointId, Weak<OutboundDialGate>>>>;
269
270struct OutboundDialGateLease {
271 endpoint_id: EndpointId,
272 gate: Arc<OutboundDialGate>,
273 gates: OutboundDialGates,
274}
275
276impl Drop for OutboundDialGateLease {
277 fn drop(&mut self) {
278 let mut gates = self
279 .gates
280 .lock()
281 .unwrap_or_else(|poisoned| poisoned.into_inner());
282 let is_current = gates
283 .get(&self.endpoint_id)
284 .is_some_and(|current| current.ptr_eq(&Arc::downgrade(&self.gate)));
285 if is_current && Arc::strong_count(&self.gate) == 1 {
289 gates.remove(&self.endpoint_id);
290 }
291 }
292}
293
294fn outbound_dial_gate(
295 endpoint_id: EndpointId,
296 registry: &OutboundDialGates,
297) -> OutboundDialGateLease {
298 let mut gates = registry
299 .lock()
300 .unwrap_or_else(|poisoned| poisoned.into_inner());
301 gates.retain(|_, gate| gate.strong_count() > 0);
305 let gate = gates
306 .get(&endpoint_id)
307 .and_then(Weak::upgrade)
308 .unwrap_or_else(|| {
309 let gate = Arc::new(OutboundDialGate::new(()));
310 gates.insert(endpoint_id, Arc::downgrade(&gate));
311 gate
312 });
313 drop(gates);
314 OutboundDialGateLease {
315 endpoint_id,
316 gate,
317 gates: Arc::clone(registry),
318 }
319}
320
321fn selected_custom_transport_id(connection: &Connection) -> Option<u64> {
322 connection.paths().iter().find_map(|path| {
323 if !path.is_selected() {
324 return None;
325 }
326 match path.remote_addr() {
327 iroh::TransportAddr::Custom(addr) => Some(addr.id()),
328 _ => None,
329 }
330 })
331}
332
333fn custom_transport_arbitration_override(
334 existing_alive: bool,
335 existing_custom_transport_id: Option<u64>,
336 fresh_custom_transport_id: Option<u64>,
337 authorized_replacement_transport_id: Option<u64>,
338) -> Option<IrohConnectionInstallDecision> {
339 if authorized_replacement_transport_id
340 .is_some_and(|expected| fresh_custom_transport_id == Some(expected))
341 {
342 return Some(IrohConnectionInstallDecision::ReplaceExisting {
343 close_existing_reason: crate::lifecycle_reason::REASON_NATIVE_CUSTOM_TRANSPORT_UPGRADE,
344 });
345 }
346 if existing_alive
347 && existing_custom_transport_id.is_some()
348 && fresh_custom_transport_id != existing_custom_transport_id
349 {
350 return Some(IrohConnectionInstallDecision::KeepExisting {
351 close_fresh_reason: "existing-custom-transport-preferred",
352 });
353 }
354 None
355}
356
357fn should_start_outbound_dial_with_transport_preference(
358 existing: Option<(bool, IrohConnectionDirection)>,
359 existing_custom_transport_id: Option<u64>,
360 local_prefers_outbound: bool,
361) -> bool {
362 if existing.is_some_and(|(alive, _)| alive) && existing_custom_transport_id.is_some() {
363 return false;
364 }
365 should_start_outbound_dial(existing, local_prefers_outbound)
366}
367
368fn decide_outbound_install_with_transport_preference(
369 existing: Option<ExistingConnectionState>,
370 existing_custom_transport_id: Option<u64>,
371 fresh_custom_transport_id: Option<u64>,
372 local_prefers_outbound: bool,
373) -> IrohConnectionInstallDecision {
374 custom_transport_arbitration_override(
375 existing.as_ref().is_some_and(|state| state.alive),
376 existing_custom_transport_id,
377 fresh_custom_transport_id,
378 None,
379 )
380 .unwrap_or_else(|| decide_outbound_install(existing, local_prefers_outbound))
381}
382
383async fn consume_authorized_inbound_replacement(
384 connection: &Connection,
385 authorizations: &InboundReplacementAuthorizations,
386) -> Option<u64> {
387 let endpoint_id = connection.remote_id();
388 let deadline = Instant::now() + std::time::Duration::from_secs(1);
389 loop {
390 let authorization = authorizations.read().await.get(&endpoint_id).copied();
391 let Some(authorization) = authorization else {
392 return None;
393 };
394 if Instant::now() >= authorization.expires_at {
395 let mut authorizations = authorizations.write().await;
396 if authorizations.get(&endpoint_id).is_some_and(|current| {
397 current.transport_id == authorization.transport_id
398 && current.expires_at == authorization.expires_at
399 }) {
400 authorizations.remove(&endpoint_id);
401 }
402 return None;
403 }
404 if selected_custom_transport_id(connection) == Some(authorization.transport_id) {
405 let mut authorizations = authorizations.write().await;
406 if authorizations.get(&endpoint_id).is_some_and(|current| {
407 current.transport_id == authorization.transport_id
408 && current.expires_at == authorization.expires_at
409 }) {
410 authorizations.remove(&endpoint_id);
411 return Some(authorization.transport_id);
412 }
413 return None;
414 }
415 if Instant::now() >= deadline || connection.close_reason().is_some() {
416 return None;
417 }
418 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
419 }
420}
421
422#[derive(Debug, Clone)]
423pub struct PlutoniumProtocol {
424 event_sender: broadcast::Sender<AcceptEvent>,
425 stream_sender: async_channel::Sender<IncomingStream>,
426 connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
427 connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
428 local_endpoint_id: EndpointId,
429 manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
430 probe_registry: IrohProbeRegistry,
431 heartbeat_manager: Option<IrohHeartbeatManager>,
432 heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
433 heartbeat_config: HeartbeatConfig,
434 inbound_replacement_authorizations: InboundReplacementAuthorizations,
435}
436
437async fn run_connection_loop(
438 connection: Connection,
439 event_sender: broadcast::Sender<AcceptEvent>,
440 stream_sender: async_channel::Sender<IncomingStream>,
441 connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
442 connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
443 local_endpoint_id: EndpointId,
444 connection_direction: IrohConnectionDirection,
445 manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
446 probe_registry: IrohProbeRegistry,
447 heartbeat_manager: Option<IrohHeartbeatManager>,
448 heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
449 heartbeat_config: HeartbeatConfig,
450 replacement_transport_id: Option<u64>,
451 mut install_outcome_sender: Option<oneshot::Sender<ExternalConnectionInstallOutcome>>,
452) -> std::result::Result<(), AcceptError> {
453 let endpoint_id = connection.remote_id();
454 let raw_stable_id = connection.stable_id();
455 let stable_id = transport_generation_for_connection(&connection);
456 let endpoint_key = endpoint_id.to_string();
457 let heartbeat_key = heartbeat_connection_key(endpoint_id, stable_id);
458
459 let mut kept_existing = None;
460 let mut replaced_transport_stable_id = None;
461 {
462 let mut conns = connections.write().await;
463 let mut insert_times = connection_inserted_at.write().await;
464 if let Some(previous) = conns.get(&endpoint_id).cloned() {
465 let fresh_custom_transport_id = selected_custom_transport_id(&connection);
466 let existing_custom_transport_id = selected_custom_transport_id(&previous);
467 let forced_replacement = replacement_transport_id
468 .is_some_and(|expected| fresh_custom_transport_id == Some(expected));
469 let previous_alive = if forced_replacement {
470 previous.close_reason().is_none()
471 } else {
472 existing_transport_is_live_for_install(
473 endpoint_id,
474 &previous,
475 stable_id,
476 connection_direction,
477 &probe_registry,
478 )
479 .await
480 };
481 let previous_direction = insert_times
482 .get(&endpoint_id)
483 .map(|metadata| metadata.direction)
484 .unwrap_or(connection_direction);
485 let previous_age_ms = insert_times
486 .get(&endpoint_id)
487 .map(|metadata| metadata.inserted_at.elapsed().as_millis() as u64)
488 .unwrap_or(0);
489 let existing = Some(ExistingConnectionState {
490 same_stable_id: previous.stable_id() == raw_stable_id,
491 alive: previous_alive,
492 direction: previous_direction,
493 age_ms: previous_age_ms,
494 });
495 let prefers_outbound = local_prefers_outbound(local_endpoint_id, endpoint_id);
496 let decision = if forced_replacement && previous.stable_id() == raw_stable_id {
497 IrohConnectionInstallDecision::Install
498 } else if let Some(decision) = custom_transport_arbitration_override(
499 previous_alive,
500 existing_custom_transport_id,
501 fresh_custom_transport_id,
502 replacement_transport_id,
503 ) {
504 decision
505 } else {
506 match connection_direction {
507 IrohConnectionDirection::Inbound => {
508 decide_inbound_install(existing, prefers_outbound)
509 }
510 IrohConnectionDirection::Outbound => {
511 decide_outbound_install(existing, prefers_outbound)
512 }
513 }
514 };
515 match decision {
516 IrohConnectionInstallDecision::Install => {
517 let previous_stable_id = transport_generation_for_connection(&previous);
518 if previous_stable_id != stable_id {
519 replaced_transport_stable_id = Some(previous_stable_id);
520 }
521 }
522 IrohConnectionInstallDecision::ReplaceExisting {
523 close_existing_reason,
524 } => {
525 replaced_transport_stable_id =
526 Some(transport_generation_for_connection(&previous));
527 previous.close(0u8.into(), close_existing_reason.as_bytes());
528 }
529 IrohConnectionInstallDecision::KeepExisting { close_fresh_reason } => {
530 kept_existing = Some(ExternalConnectionInstallOutcome::KeptExisting {
531 fresh_transport_stable_id: stable_id,
532 kept_transport_stable_id: transport_generation_for_connection(&previous),
533 });
534 connection.close(0u8.into(), close_fresh_reason.as_bytes());
535 }
536 }
537 }
538 if kept_existing.is_none() {
539 conns.insert(endpoint_id, connection.clone());
540 insert_times.insert(
541 endpoint_id,
542 IrohConnectionMetadata {
543 inserted_at: Instant::now(),
544 direction: connection_direction,
545 },
546 );
547 }
548 }
549
550 if let Some(outcome) = kept_existing {
551 if let Some(sender) = install_outcome_sender.take() {
552 let _ = sender.send(outcome);
553 }
554 forget_transport_generation(&connection);
555 return Ok(());
556 }
557
558 if let Some(sender) = install_outcome_sender.take() {
559 let _ = sender.send(ExternalConnectionInstallOutcome::Installed {
560 transport_stable_id: stable_id,
561 replaced_transport_stable_id,
562 });
563 }
564
565 if let (Some(mgr), Some(tx)) = (&heartbeat_manager, &heartbeat_health_tx) {
567 mgr.start_connection(
568 heartbeat_key.clone(),
569 connection.clone(),
570 heartbeat_config.clone(),
571 tx.clone(),
572 )
573 .await;
574 }
575
576 event_sender
577 .send(AcceptEvent::Accepted {
578 endpoint_id,
579 transport_stable_id: stable_id,
580 replaced_transport_stable_id,
581 })
582 .ok();
583
584 loop {
585 tokio::select! {
586 res = connection.accept_bi() => {
593 match res {
594 Ok((send, recv)) => {
595 probe_registry
596 .record_inbound_activity(&endpoint_key, stable_id)
597 .await;
598 let _ = stream_sender.send(IncomingStream {
599 endpoint_id,
600 transport_stable_id: stable_id,
601 recv_prefix: Vec::new(),
602 stream: IncomingStreamType::Bi(send, recv),
603 }).await;
604 }
605 Err(_) => {
606 if should_break_accept_loop(&connection).await {
607 break;
608 }
609 },
610 }
611 }
612 res = connection.accept_uni() => {
613 match res {
614 Ok(recv) => {
615 probe_registry
616 .record_inbound_activity(&endpoint_key, stable_id)
617 .await;
618 let conn_clone = connection.clone();
621 let mgr_clone = heartbeat_manager.clone();
622 let probe_registry = probe_registry.clone();
623 let endpoint_key = endpoint_key.clone();
624 let heartbeat_key = heartbeat_key.clone();
625 let transport_stable_id = stable_id;
626 let notice_key = manual_disconnect_notice_key(
627 endpoint_id,
628 transport_stable_id,
629 );
630 let notices = manual_disconnect_notices.clone();
631 let application_streams = stream_sender.clone();
632 tokio::spawn(async move {
633 match classify_incoming_uni(recv).await {
634 IncomingUniClassification::Control { type_id, payload } => {
635 use crate::heartbeat::codec;
636 if type_id == codec::TYPE_PING {
637 if !respond_to_ping(&conn_clone, &payload).await {
638 eprintln!(
639 "[OpenRTC][iroh-probe] failed to send pong endpoint_id={} transport_stable_id={}",
640 endpoint_key,
641 transport_stable_id,
642 );
643 }
644 } else if type_id == codec::TYPE_PONG {
645 if let Some(pong) = parse_incoming_pong(&payload) {
646 probe_registry
647 .deliver_pong(
648 &endpoint_key,
649 transport_stable_id,
650 &pong,
651 )
652 .await;
653 if let Some(manager) = mgr_clone {
654 manager.deliver_pong(&heartbeat_key, pong).await;
655 }
656 }
657 } else if type_id == codec::TYPE_MANUAL_DISCONNECT {
658 notices.write().await.insert(notice_key);
659 conn_clone.close(
660 0u8.into(),
661 crate::lifecycle_reason::REASON_MANUAL_DISCONNECT
662 .as_bytes(),
663 );
664 }
665 }
666 IncomingUniClassification::Application(recv) => {
667 let _ = application_streams
668 .send(IncomingStream {
669 endpoint_id,
670 transport_stable_id,
671 recv_prefix: Vec::new(),
672 stream: IncomingStreamType::Uni(recv),
673 })
674 .await;
675 }
676 IncomingUniClassification::MalformedControl => {}
677 }
678 });
679 }
680 Err(_) => {
681 if should_break_accept_loop(&connection).await {
682 break;
683 }
684 },
685 }
686 }
687 _ = connection.closed() => break,
688 }
689 }
690
691 if let Some(ref mgr) = heartbeat_manager {
693 mgr.stop_connection(&heartbeat_key).await;
694 }
695 probe_registry
696 .forget_transport(&endpoint_key, stable_id)
697 .await;
698
699 let close_reason = connection.close_reason();
700 let close_reason_debug = close_reason.as_ref().map(|reason| format!("{:?}", reason));
701 let was_locally_closed = matches!(
702 close_reason,
703 Some(iroh::endpoint::ConnectionError::LocallyClosed)
704 );
705 let current_transport_stable_id = connections
706 .read()
707 .await
708 .get(&endpoint_id)
709 .map(transport_generation_for_connection);
710 eprintln!(
711 "[OpenRTC][iroh-connection][closed] endpoint_id={} transport_stable_id={} direction={:?} close_reason={:?} current_transport_stable_id={:?}",
712 endpoint_id,
713 stable_id,
714 connection_direction,
715 close_reason_debug,
716 current_transport_stable_id,
717 );
718 event_sender
719 .send(AcceptEvent::Closed {
720 endpoint_id,
721 transport_stable_id: stable_id,
722 error: close_reason_debug,
723 was_outbound: matches!(connection_direction, IrohConnectionDirection::Outbound),
724 was_locally_closed,
725 })
726 .ok();
727
728 {
729 let mut conns = connections.write().await;
730 let should_remove = conns
731 .get(&endpoint_id)
732 .map(|current| current.stable_id() == raw_stable_id)
733 .unwrap_or(false);
734 if should_remove {
735 conns.remove(&endpoint_id);
736 connection_inserted_at.write().await.remove(&endpoint_id);
737 }
738 }
739
740 forget_transport_generation(&connection);
741 Ok(())
742}
743
744impl PlutoniumProtocol {
745 pub const ALPN: &[u8] = b"plutonium/p2p/1";
746
747 fn new(
748 event_sender: broadcast::Sender<AcceptEvent>,
749 stream_sender: async_channel::Sender<IncomingStream>,
750 connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
751 connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
752 local_endpoint_id: EndpointId,
753 manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
754 probe_registry: IrohProbeRegistry,
755 inbound_replacement_authorizations: InboundReplacementAuthorizations,
756 ) -> Self {
757 Self {
758 event_sender,
759 stream_sender,
760 connections,
761 connection_inserted_at,
762 local_endpoint_id,
763 manual_disconnect_notices,
764 probe_registry,
765 heartbeat_manager: None,
766 heartbeat_health_tx: None,
767 heartbeat_config: HeartbeatConfig::default(),
768 inbound_replacement_authorizations,
769 }
770 }
771
772 pub fn with_heartbeat(
773 mut self,
774 manager: IrohHeartbeatManager,
775 health_tx: mpsc::Sender<HealthTransition>,
776 config: HeartbeatConfig,
777 ) -> Self {
778 self.heartbeat_manager = Some(manager);
779 self.heartbeat_health_tx = Some(health_tx);
780 self.heartbeat_config = config;
781 self
782 }
783
784 async fn handle_connection(
785 self,
786 connection: Connection,
787 ) -> std::result::Result<(), AcceptError> {
788 let replacement_transport_id = consume_authorized_inbound_replacement(
789 &connection,
790 &self.inbound_replacement_authorizations,
791 )
792 .await;
793 run_connection_loop(
794 connection,
795 self.event_sender.clone(),
796 self.stream_sender.clone(),
797 self.connections.clone(),
798 self.connection_inserted_at.clone(),
799 self.local_endpoint_id,
800 IrohConnectionDirection::Inbound,
801 self.manual_disconnect_notices.clone(),
802 self.probe_registry.clone(),
803 self.heartbeat_manager.clone(),
804 self.heartbeat_health_tx.clone(),
805 self.heartbeat_config.clone(),
806 replacement_transport_id,
807 None,
808 )
809 .await
810 }
811}
812
813impl ProtocolHandler for PlutoniumProtocol {
814 #[allow(refining_impl_trait)]
815 fn accept(
816 &self,
817 connection: Connection,
818 ) -> impl n0_future::Future<Output = std::result::Result<(), AcceptError>> + std::marker::Send
819 {
820 let proto = self.clone();
821 async move { proto.handle_connection(connection).await }
822 }
823}
824
825async fn retire_unobserved_outbound_connection(
826 endpoint_id: EndpointId,
827 connection: &Connection,
828 connections: &Arc<RwLock<HashMap<EndpointId, Connection>>>,
829 connection_inserted_at: &Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
830) {
831 connection.close(0u8.into(), b"connect-event-receiver-closed");
832 let raw_stable_id = connection.stable_id();
833 let mut conns = connections.write().await;
834 if conns
835 .get(&endpoint_id)
836 .is_some_and(|current| current.stable_id() == raw_stable_id)
837 {
838 conns.remove(&endpoint_id);
839 connection_inserted_at.write().await.remove(&endpoint_id);
840 }
841 forget_transport_generation(connection);
842}
843
844async fn connect(
845 endpoint: &Endpoint,
846 endpoint_id: EndpointId,
847 event_sender: async_channel::Sender<ConnectEvent>,
848 outbound_dial_gates: OutboundDialGates,
849 connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
850 connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
851 stream_sender: async_channel::Sender<IncomingStream>,
852 manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
853 probe_registry: IrohProbeRegistry,
854 heartbeat_manager: Option<IrohHeartbeatManager>,
855 heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
856 heartbeat_config: HeartbeatConfig,
857) -> Result<()> {
858 let connection = {
859 let dial_gate = outbound_dial_gate(endpoint_id, &outbound_dial_gates);
863 let _dial_guard = dial_gate.gate.lock().await;
864 if outbound_dial_precheck(endpoint, endpoint_id, &connections, &connection_inserted_at)
865 .await
866 {
867 event_sender.send(ConnectEvent::Connected).await?;
868 return Ok(());
869 }
870
871 let connection = endpoint
872 .connect(endpoint_id, PlutoniumProtocol::ALPN)
873 .await?;
874 let raw_stable_id = connection.stable_id();
875 let stable_id = transport_generation_for_connection(&connection);
876
877 {
878 let mut conns = connections.write().await;
879 let mut insert_times = connection_inserted_at.write().await;
880 if let Some(previous) = conns.get(&endpoint_id).cloned() {
881 let previous_alive = existing_transport_is_live_for_install(
882 endpoint_id,
883 &previous,
884 stable_id,
885 IrohConnectionDirection::Outbound,
886 &probe_registry,
887 )
888 .await;
889 let previous_direction = insert_times
890 .get(&endpoint_id)
891 .map(|metadata| metadata.direction)
892 .unwrap_or(IrohConnectionDirection::Outbound);
893 let previous_age_ms = insert_times
894 .get(&endpoint_id)
895 .map(|metadata| metadata.inserted_at.elapsed().as_millis() as u64)
896 .unwrap_or(0);
897 let existing = Some(ExistingConnectionState {
898 same_stable_id: previous.stable_id() == raw_stable_id,
899 alive: previous_alive,
900 direction: previous_direction,
901 age_ms: previous_age_ms,
902 });
903 match decide_outbound_install_with_transport_preference(
904 existing,
905 selected_custom_transport_id(&previous),
906 selected_custom_transport_id(&connection),
907 local_prefers_outbound(endpoint.id(), endpoint_id),
908 ) {
909 IrohConnectionInstallDecision::Install => {}
910 IrohConnectionInstallDecision::ReplaceExisting {
911 close_existing_reason,
912 } => {
913 previous.close(0u8.into(), close_existing_reason.as_bytes());
914 }
915 IrohConnectionInstallDecision::KeepExisting { close_fresh_reason } => {
916 connection.close(0u8.into(), close_fresh_reason.as_bytes());
917 forget_transport_generation(&connection);
918 event_sender.send(ConnectEvent::Connected).await?;
919 let _ = event_sender
920 .send(ConnectEvent::Closed { error: None })
921 .await;
922 return Ok(());
923 }
924 }
925 }
926 conns.insert(endpoint_id, connection.clone());
927 insert_times.insert(
928 endpoint_id,
929 IrohConnectionMetadata {
930 inserted_at: Instant::now(),
931 direction: IrohConnectionDirection::Outbound,
932 },
933 );
934 }
935 connection
936 };
937
938 if let Err(error) = event_sender.send(ConnectEvent::Connected).await {
939 retire_unobserved_outbound_connection(
940 endpoint_id,
941 &connection,
942 &connections,
943 &connection_inserted_at,
944 )
945 .await;
946 return Err(error.into());
947 }
948
949 let (accept_tx, _) = broadcast::channel(1);
951 run_connection_loop(
952 connection,
953 accept_tx,
954 stream_sender,
955 connections.clone(),
956 connection_inserted_at,
957 endpoint.id(),
958 IrohConnectionDirection::Outbound,
959 manual_disconnect_notices,
960 probe_registry,
961 heartbeat_manager,
962 heartbeat_health_tx,
963 heartbeat_config,
964 None,
965 None,
966 )
967 .await
968 .ok();
969
970 event_sender
971 .send(ConnectEvent::Closed { error: None })
972 .await?;
973
974 Ok(())
975}
976
977async fn connect_addr(
978 endpoint: &Endpoint,
979 endpoint_id: EndpointId,
980 endpoint_addr: EndpointAddr,
981 event_sender: async_channel::Sender<ConnectEvent>,
982 outbound_dial_gates: OutboundDialGates,
983 connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
984 connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
985 stream_sender: async_channel::Sender<IncomingStream>,
986 manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
987 probe_registry: IrohProbeRegistry,
988 heartbeat_manager: Option<IrohHeartbeatManager>,
989 heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
990 heartbeat_config: HeartbeatConfig,
991) -> Result<()> {
992 let connection = {
993 let dial_gate = outbound_dial_gate(endpoint_id, &outbound_dial_gates);
994 let _dial_guard = dial_gate.gate.lock().await;
995 if outbound_dial_precheck(endpoint, endpoint_id, &connections, &connection_inserted_at)
996 .await
997 {
998 event_sender.send(ConnectEvent::Connected).await?;
999 return Ok(());
1000 }
1001
1002 let connection = endpoint
1003 .connect(endpoint_addr, PlutoniumProtocol::ALPN)
1004 .await?;
1005 let raw_stable_id = connection.stable_id();
1006 let stable_id = transport_generation_for_connection(&connection);
1007
1008 {
1009 let mut conns = connections.write().await;
1010 let mut insert_times = connection_inserted_at.write().await;
1011 if let Some(previous) = conns.get(&endpoint_id).cloned() {
1012 let previous_alive = existing_transport_is_live_for_install(
1013 endpoint_id,
1014 &previous,
1015 stable_id,
1016 IrohConnectionDirection::Outbound,
1017 &probe_registry,
1018 )
1019 .await;
1020 let previous_direction = insert_times
1021 .get(&endpoint_id)
1022 .map(|metadata| metadata.direction)
1023 .unwrap_or(IrohConnectionDirection::Outbound);
1024 let previous_age_ms = insert_times
1025 .get(&endpoint_id)
1026 .map(|metadata| metadata.inserted_at.elapsed().as_millis() as u64)
1027 .unwrap_or(0);
1028 let existing = Some(ExistingConnectionState {
1029 same_stable_id: previous.stable_id() == raw_stable_id,
1030 alive: previous_alive,
1031 direction: previous_direction,
1032 age_ms: previous_age_ms,
1033 });
1034 match decide_outbound_install_with_transport_preference(
1035 existing,
1036 selected_custom_transport_id(&previous),
1037 selected_custom_transport_id(&connection),
1038 local_prefers_outbound(endpoint.id(), endpoint_id),
1039 ) {
1040 IrohConnectionInstallDecision::Install => {}
1041 IrohConnectionInstallDecision::ReplaceExisting {
1042 close_existing_reason,
1043 } => {
1044 previous.close(0u8.into(), close_existing_reason.as_bytes());
1045 }
1046 IrohConnectionInstallDecision::KeepExisting { close_fresh_reason } => {
1047 connection.close(0u8.into(), close_fresh_reason.as_bytes());
1048 forget_transport_generation(&connection);
1049 event_sender.send(ConnectEvent::Connected).await?;
1050 let _ = event_sender
1051 .send(ConnectEvent::Closed { error: None })
1052 .await;
1053 return Ok(());
1054 }
1055 }
1056 }
1057 conns.insert(endpoint_id, connection.clone());
1058 insert_times.insert(
1059 endpoint_id,
1060 IrohConnectionMetadata {
1061 inserted_at: Instant::now(),
1062 direction: IrohConnectionDirection::Outbound,
1063 },
1064 );
1065 }
1066 connection
1067 };
1068
1069 if let Err(error) = event_sender.send(ConnectEvent::Connected).await {
1070 retire_unobserved_outbound_connection(
1071 endpoint_id,
1072 &connection,
1073 &connections,
1074 &connection_inserted_at,
1075 )
1076 .await;
1077 return Err(error.into());
1078 }
1079
1080 let (accept_tx, _) = broadcast::channel(1);
1081 run_connection_loop(
1082 connection,
1083 accept_tx,
1084 stream_sender,
1085 connections.clone(),
1086 connection_inserted_at,
1087 endpoint.id(),
1088 IrohConnectionDirection::Outbound,
1089 manual_disconnect_notices,
1090 probe_registry,
1091 heartbeat_manager,
1092 heartbeat_health_tx,
1093 heartbeat_config,
1094 None,
1095 None,
1096 )
1097 .await
1098 .ok();
1099
1100 event_sender
1101 .send(ConnectEvent::Closed { error: None })
1102 .await?;
1103
1104 Ok(())
1105}
1106
1107async fn install_replacement_connection(
1108 endpoint: &Endpoint,
1109 connection: Connection,
1110 replacement_transport_id: u64,
1111 event_sender: async_channel::Sender<ReplacementConnectEvent>,
1112 connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
1113 connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
1114 stream_sender: async_channel::Sender<IncomingStream>,
1115 manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
1116 probe_registry: IrohProbeRegistry,
1117 heartbeat_manager: Option<IrohHeartbeatManager>,
1118 heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
1119 heartbeat_config: HeartbeatConfig,
1120) -> Result<()> {
1121 fn replacement_path_summary(connection: &Connection) -> Vec<String> {
1122 connection
1123 .paths()
1124 .iter()
1125 .map(|path| {
1126 format!(
1127 "selected={} remote_addr={:?}",
1128 path.is_selected(),
1129 path.remote_addr()
1130 )
1131 })
1132 .collect()
1133 }
1134
1135 eprintln!(
1136 "[OpenRTC][custom-transport] replacement admission waiting stable_id={} remote_endpoint_id={} expected_transport_id={} paths={:?}",
1137 connection.stable_id(),
1138 connection.remote_id(),
1139 replacement_transport_id,
1140 replacement_path_summary(&connection)
1141 );
1142 let selected_expected_transport = || {
1143 connection.paths().iter().any(|path| {
1144 path.is_selected()
1145 && matches!(
1146 path.remote_addr(),
1147 iroh::TransportAddr::Custom(addr) if addr.id() == replacement_transport_id
1148 )
1149 })
1150 };
1151 let selected = tokio::time::timeout(std::time::Duration::from_secs(5), async {
1152 loop {
1153 if selected_expected_transport() {
1154 break;
1155 }
1156 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1157 }
1158 })
1159 .await
1160 .is_ok();
1161 if !selected {
1162 eprintln!(
1163 "[OpenRTC][custom-transport] replacement admission rejected stable_id={} remote_endpoint_id={} expected_transport_id={} paths={:?}",
1164 connection.stable_id(),
1165 connection.remote_id(),
1166 replacement_transport_id,
1167 replacement_path_summary(&connection)
1168 );
1169 connection.close(0u8.into(), b"custom-transport-not-selected");
1170 anyhow::bail!(
1171 "replacement connection did not select custom transport id {replacement_transport_id}"
1172 );
1173 }
1174
1175 eprintln!(
1176 "[OpenRTC][custom-transport] replacement admission accepted stable_id={} remote_endpoint_id={} transport_id={} paths={:?}",
1177 connection.stable_id(),
1178 connection.remote_id(),
1179 replacement_transport_id,
1180 replacement_path_summary(&connection)
1181 );
1182
1183 let (accept_tx, _) = broadcast::channel(1);
1184 let (install_outcome_sender, install_outcome_receiver) = oneshot::channel();
1185 let connection_loop = tokio::spawn(run_connection_loop(
1186 connection,
1187 accept_tx,
1188 stream_sender,
1189 connections,
1190 connection_inserted_at,
1191 endpoint.id(),
1192 IrohConnectionDirection::Outbound,
1193 manual_disconnect_notices,
1194 probe_registry,
1195 heartbeat_manager,
1196 heartbeat_health_tx,
1197 heartbeat_config,
1198 Some(replacement_transport_id),
1199 Some(install_outcome_sender),
1200 ));
1201 let install_outcome = install_outcome_receiver
1202 .await
1203 .map_err(|_| anyhow::anyhow!("replacement connection install task ended"))?;
1204 let transport_stable_id = match install_outcome {
1205 ExternalConnectionInstallOutcome::Installed {
1206 transport_stable_id,
1207 ..
1208 } => transport_stable_id,
1209 ExternalConnectionInstallOutcome::KeptExisting {
1210 fresh_transport_stable_id,
1211 kept_transport_stable_id,
1212 } => {
1213 anyhow::bail!(
1214 "replacement connection lost arbitration fresh_stable_id={} kept_stable_id={}",
1215 fresh_transport_stable_id,
1216 kept_transport_stable_id,
1217 );
1218 }
1219 };
1220 event_sender
1221 .send(ReplacementConnectEvent::Connected {
1222 transport_stable_id,
1223 })
1224 .await?;
1225 let _ = connection_loop.await;
1226
1227 let _ = event_sender
1228 .send(ReplacementConnectEvent::Closed { error: None })
1229 .await;
1230 Ok(())
1231}
1232
1233#[allow(unused_variables)]
1234async fn connect_replacement_addr(
1235 endpoint: &Endpoint,
1236 _endpoint_id: EndpointId,
1237 endpoint_addr: EndpointAddr,
1238 replacement_transport_id: u64,
1239 event_sender: async_channel::Sender<ReplacementConnectEvent>,
1240 connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
1241 connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
1242 stream_sender: async_channel::Sender<IncomingStream>,
1243 manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
1244 probe_registry: IrohProbeRegistry,
1245 heartbeat_manager: Option<IrohHeartbeatManager>,
1246 heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
1247 heartbeat_config: HeartbeatConfig,
1248) -> Result<()> {
1249 #[cfg(not(openrtc_iroh_preferred_transport_api))]
1250 anyhow::bail!(
1251 "custom transport replacement requires a host Iroh build with the preferred-transport API"
1252 );
1253
1254 #[cfg(openrtc_iroh_preferred_transport_api)]
1255 {
1256 let preferred_transport_addr = endpoint_addr
1257 .addrs
1258 .iter()
1259 .find(|addr| {
1260 matches!(
1261 addr,
1262 iroh::TransportAddr::Custom(custom)
1263 if custom.id() == replacement_transport_id
1264 )
1265 })
1266 .cloned()
1267 .ok_or_else(|| {
1268 anyhow::anyhow!(
1269 "replacement address does not contain custom transport id {replacement_transport_id}"
1270 )
1271 })?;
1272 eprintln!(
1273 "[OpenRTC][custom-transport] replacement dial pinned remote_endpoint_id={} transport_id={} addr={:?}",
1274 endpoint_addr.id, replacement_transport_id, preferred_transport_addr
1275 );
1276 let connection = endpoint
1277 .connect_with_opts(
1278 endpoint_addr,
1279 PlutoniumProtocol::ALPN,
1280 iroh::endpoint::ConnectOptions::new()
1281 .with_preferred_transport_addr(preferred_transport_addr),
1282 )
1283 .await?
1284 .await?;
1285 install_replacement_connection(
1286 endpoint,
1287 connection,
1288 replacement_transport_id,
1289 event_sender,
1290 connections,
1291 connection_inserted_at,
1292 stream_sender,
1293 manual_disconnect_notices,
1294 probe_registry,
1295 heartbeat_manager,
1296 heartbeat_health_tx,
1297 heartbeat_config,
1298 )
1299 .await
1300 }
1301}
1302
1303#[derive(Debug, Clone)]
1304pub struct IrohNativeNode {
1305 endpoint: Endpoint,
1306 #[allow(dead_code)]
1308 router: Option<Router>,
1309 accept_events: broadcast::Sender<AcceptEvent>,
1310 connections: Arc<RwLock<HashMap<EndpointId, Connection>>>,
1311 connection_inserted_at: Arc<RwLock<HashMap<EndpointId, IrohConnectionMetadata>>>,
1312 outbound_dial_gates: OutboundDialGates,
1313 incoming_streams: async_channel::Sender<IncomingStream>,
1314 incoming_streams_receiver: async_channel::Receiver<IncomingStream>,
1315 manual_disconnect_notices: Arc<RwLock<HashSet<String>>>,
1316 probe_registry: IrohProbeRegistry,
1317 heartbeat_manager: Option<IrohHeartbeatManager>,
1318 heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
1319 heartbeat_config: HeartbeatConfig,
1320 inbound_replacement_authorizations: InboundReplacementAuthorizations,
1321}
1322
1323impl IrohNativeNode {
1324 pub async fn spawn_with_endpoint(endpoint: Endpoint) -> Result<Self> {
1325 Self::spawn_with_endpoint_config(endpoint, true).await
1326 }
1327
1328 pub async fn spawn_with_endpoint_no_router(endpoint: Endpoint) -> Result<Self> {
1334 Self::spawn_with_endpoint_config(endpoint, false).await
1335 }
1336
1337 async fn spawn_with_endpoint_config(endpoint: Endpoint, spawn_router: bool) -> Result<Self> {
1338 Self::spawn_with_endpoint_config_and_heartbeat(
1339 endpoint,
1340 spawn_router,
1341 None,
1342 None,
1343 HeartbeatConfig::default(),
1344 )
1345 .await
1346 }
1347
1348 pub async fn spawn_with_heartbeat(
1350 endpoint: Endpoint,
1351 heartbeat_manager: IrohHeartbeatManager,
1352 heartbeat_health_tx: mpsc::Sender<HealthTransition>,
1353 heartbeat_config: HeartbeatConfig,
1354 ) -> Result<Self> {
1355 Self::spawn_with_endpoint_config_and_heartbeat(
1356 endpoint,
1357 true,
1358 Some(heartbeat_manager),
1359 Some(heartbeat_health_tx),
1360 heartbeat_config,
1361 )
1362 .await
1363 }
1364
1365 async fn spawn_with_endpoint_config_and_heartbeat(
1366 endpoint: Endpoint,
1367 spawn_router: bool,
1368 heartbeat_manager: Option<IrohHeartbeatManager>,
1369 heartbeat_health_tx: Option<mpsc::Sender<HealthTransition>>,
1370 heartbeat_config: HeartbeatConfig,
1371 ) -> Result<Self> {
1372 let (event_sender, _) = broadcast::channel(128);
1373 let (stream_sender, stream_receiver) = async_channel::bounded(64);
1374 let connections = Arc::new(RwLock::new(HashMap::new()));
1375 let connection_inserted_at = Arc::new(RwLock::new(HashMap::new()));
1376 let outbound_dial_gates = Arc::new(StdMutex::new(HashMap::new()));
1377 let manual_disconnect_notices = Arc::new(RwLock::new(HashSet::new()));
1378 let probe_registry = IrohProbeRegistry::new();
1379 let inbound_replacement_authorizations = Arc::new(RwLock::new(HashMap::new()));
1380
1381 let router = if spawn_router {
1382 let proto = PlutoniumProtocol::new(
1383 event_sender.clone(),
1384 stream_sender.clone(),
1385 connections.clone(),
1386 connection_inserted_at.clone(),
1387 endpoint.id(),
1388 manual_disconnect_notices.clone(),
1389 probe_registry.clone(),
1390 inbound_replacement_authorizations.clone(),
1391 );
1392 let proto = if let (Some(mgr), Some(tx)) =
1393 (heartbeat_manager.clone(), heartbeat_health_tx.clone())
1394 {
1395 proto.with_heartbeat(mgr, tx, heartbeat_config.clone())
1396 } else {
1397 proto
1398 };
1399 Some(
1400 Router::builder(endpoint.clone())
1401 .accept(PlutoniumProtocol::ALPN, proto)
1402 .spawn(),
1403 )
1404 } else {
1405 None
1406 };
1407
1408 Ok(Self {
1409 endpoint,
1410 router,
1411 accept_events: event_sender,
1412 connections,
1413 connection_inserted_at,
1414 outbound_dial_gates,
1415 incoming_streams: stream_sender,
1416 incoming_streams_receiver: stream_receiver,
1417 manual_disconnect_notices,
1418 probe_registry,
1419 heartbeat_manager,
1420 heartbeat_health_tx,
1421 heartbeat_config,
1422 inbound_replacement_authorizations,
1423 })
1424 }
1425
1426 pub fn endpoint(&self) -> &Endpoint {
1427 &self.endpoint
1428 }
1429
1430 pub async fn node_addr(&self) -> Result<EndpointAddr> {
1431 Ok(self.endpoint.watch_addr().get())
1436 }
1437
1438 pub async fn is_connected(&self, endpoint_id: EndpointId) -> bool {
1439 let conns = self.connections.read().await;
1440 if let Some(conn) = conns.get(&endpoint_id) {
1441 conn.close_reason().is_none()
1442 } else {
1443 false
1444 }
1445 }
1446
1447 pub fn accept_events(&self) -> futures::stream::BoxStream<'static, AcceptEvent> {
1448 let receiver = self.accept_events.subscribe();
1449 Box::pin(
1450 BroadcastStream::new(receiver).filter_map(|event| futures::future::ready(event.ok())),
1451 )
1452 }
1453
1454 pub fn connect(
1455 &self,
1456 endpoint_id: EndpointId,
1457 ) -> futures::stream::BoxStream<'static, ConnectEvent> {
1458 let (event_sender, event_receiver) = async_channel::bounded(16);
1459 let endpoint = self.endpoint.clone();
1460 let connections = self.connections.clone();
1461 let connection_inserted_at = self.connection_inserted_at.clone();
1462 let outbound_dial_gates = self.outbound_dial_gates.clone();
1463 let stream_sender = self.incoming_streams.clone();
1464 let hb_mgr = self.heartbeat_manager.clone();
1465 let hb_tx = self.heartbeat_health_tx.clone();
1466 let hb_cfg = self.heartbeat_config.clone();
1467 let manual_notices = self.manual_disconnect_notices.clone();
1468 let probe_registry = self.probe_registry.clone();
1469
1470 tokio::spawn(async move {
1471 let result = connect(
1472 &endpoint,
1473 endpoint_id,
1474 event_sender.clone(),
1475 outbound_dial_gates,
1476 connections,
1477 connection_inserted_at,
1478 stream_sender,
1479 manual_notices,
1480 probe_registry,
1481 hb_mgr,
1482 hb_tx,
1483 hb_cfg,
1484 )
1485 .await;
1486
1487 if let Err(error) = result {
1488 let _ = event_sender
1489 .send(ConnectEvent::Closed {
1490 error: Some(error.to_string()),
1491 })
1492 .await;
1493 }
1494 });
1495
1496 Box::pin(event_receiver)
1497 }
1498
1499 pub fn connect_addr(
1500 &self,
1501 endpoint_id: EndpointId,
1502 endpoint_addr: EndpointAddr,
1503 ) -> futures::stream::BoxStream<'static, ConnectEvent> {
1504 let (event_sender, event_receiver) = async_channel::bounded(16);
1505 let endpoint = self.endpoint.clone();
1506 let connections = self.connections.clone();
1507 let connection_inserted_at = self.connection_inserted_at.clone();
1508 let outbound_dial_gates = self.outbound_dial_gates.clone();
1509 let stream_sender = self.incoming_streams.clone();
1510 let hb_mgr = self.heartbeat_manager.clone();
1511 let hb_tx = self.heartbeat_health_tx.clone();
1512 let hb_cfg = self.heartbeat_config.clone();
1513 let manual_notices = self.manual_disconnect_notices.clone();
1514 let probe_registry = self.probe_registry.clone();
1515
1516 tokio::spawn(async move {
1517 let result = connect_addr(
1518 &endpoint,
1519 endpoint_id,
1520 endpoint_addr,
1521 event_sender.clone(),
1522 outbound_dial_gates,
1523 connections,
1524 connection_inserted_at,
1525 stream_sender,
1526 manual_notices,
1527 probe_registry,
1528 hb_mgr,
1529 hb_tx,
1530 hb_cfg,
1531 )
1532 .await;
1533
1534 if let Err(error) = result {
1535 let _ = event_sender
1536 .send(ConnectEvent::Closed {
1537 error: Some(error.to_string()),
1538 })
1539 .await;
1540 }
1541 });
1542
1543 Box::pin(event_receiver)
1544 }
1545
1546 pub fn connect_replacement_addr(
1550 &self,
1551 endpoint_id: EndpointId,
1552 endpoint_addr: EndpointAddr,
1553 replacement_transport_id: u64,
1554 ) -> futures::stream::BoxStream<'static, ReplacementConnectEvent> {
1555 let (event_sender, event_receiver) = async_channel::bounded(16);
1556 let endpoint = self.endpoint.clone();
1557 let connections = self.connections.clone();
1558 let connection_inserted_at = self.connection_inserted_at.clone();
1559 let stream_sender = self.incoming_streams.clone();
1560 let hb_mgr = self.heartbeat_manager.clone();
1561 let hb_tx = self.heartbeat_health_tx.clone();
1562 let hb_cfg = self.heartbeat_config.clone();
1563 let manual_notices = self.manual_disconnect_notices.clone();
1564 let probe_registry = self.probe_registry.clone();
1565
1566 tokio::spawn(async move {
1567 if let Err(error) = connect_replacement_addr(
1568 &endpoint,
1569 endpoint_id,
1570 endpoint_addr,
1571 replacement_transport_id,
1572 event_sender.clone(),
1573 connections,
1574 connection_inserted_at,
1575 stream_sender,
1576 manual_notices,
1577 probe_registry,
1578 hb_mgr,
1579 hb_tx,
1580 hb_cfg,
1581 )
1582 .await
1583 {
1584 let _ = event_sender
1585 .send(ReplacementConnectEvent::Closed {
1586 error: Some(error.to_string()),
1587 })
1588 .await;
1589 }
1590 });
1591
1592 Box::pin(event_receiver)
1593 }
1594
1595 pub async fn disconnect(&self, endpoint_id: EndpointId) -> Result<()> {
1596 self.disconnect_with_reason(
1597 endpoint_id,
1598 crate::lifecycle_reason::REASON_DISCONNECTED_BY_USER,
1599 )
1600 .await
1601 }
1602
1603 pub async fn disconnect_with_reason(
1604 &self,
1605 endpoint_id: EndpointId,
1606 reason: &str,
1607 ) -> Result<()> {
1608 let connection = {
1609 let mut conns = self.connections.write().await;
1610 conns.remove(&endpoint_id)
1611 };
1612
1613 if let Some(conn) = connection {
1614 if std::env::var("PLUTO_RTC_TEARDOWN_TRACE").is_ok() {
1615 eprintln!(
1616 "[PlutoRTC][teardown-trace] NativeNode::disconnect endpoint_id={}",
1617 endpoint_id
1618 );
1619 }
1620 conn.close(1u8.into(), reason.as_bytes());
1621 }
1622
1623 Ok(())
1624 }
1625
1626 pub async fn disconnect_with_reason_if_current(
1627 &self,
1628 endpoint_id: EndpointId,
1629 expected_transport_stable_id: u64,
1630 reason: &str,
1631 ) -> Result<bool> {
1632 let connection = {
1633 let mut connections = self.connections.write().await;
1634 let is_current = connections.get(&endpoint_id).is_some_and(|connection| {
1635 transport_generation_for_connection(connection) == expected_transport_stable_id
1636 });
1637 if !is_current {
1638 return Ok(false);
1639 }
1640 connections.remove(&endpoint_id)
1641 };
1642
1643 if let Some(connection) = connection {
1644 connection.close(1u8.into(), reason.as_bytes());
1645 return Ok(true);
1646 }
1647 Ok(false)
1648 }
1649
1650 pub async fn open_bi(&self, endpoint_id: EndpointId) -> Result<(SendStream, RecvStream)> {
1651 let (_, send, recv) = self.open_bi_with_transport_stable_id(endpoint_id).await?;
1652 Ok((send, recv))
1653 }
1654
1655 pub async fn open_bi_with_transport_stable_id(
1659 &self,
1660 endpoint_id: EndpointId,
1661 ) -> Result<(u64, SendStream, RecvStream)> {
1662 let connection = {
1663 let conns = self.connections.read().await;
1664 conns.get(&endpoint_id).cloned()
1665 };
1666
1667 if let Some(conn) = connection {
1668 let transport_stable_id = transport_generation_for_connection(&conn);
1669 let (send, recv) = conn.open_bi().await?;
1670 Ok((transport_stable_id, send, recv))
1671 } else {
1672 Err(anyhow::anyhow!("No active connection to {}", endpoint_id))
1673 }
1674 }
1675
1676 pub async fn open_uni(&self, endpoint_id: EndpointId) -> Result<SendStream> {
1677 let connection = {
1678 let conns = self.connections.read().await;
1679 conns.get(&endpoint_id).cloned()
1680 };
1681
1682 if let Some(conn) = connection {
1683 let send = conn.open_uni().await?;
1684 Ok(send)
1685 } else {
1686 Err(anyhow::anyhow!("No active connection to {}", endpoint_id))
1687 }
1688 }
1689
1690 pub async fn probe_connection(
1694 &self,
1695 endpoint_id: EndpointId,
1696 timeout: std::time::Duration,
1697 ) -> Option<IrohConnectionProbe> {
1698 let connection = self.connections.read().await.get(&endpoint_id).cloned()?;
1699 let probed_stable_id = transport_generation_for_connection(&connection);
1700 let responsive = self
1701 .probe_registry
1702 .probe(
1703 &endpoint_id.to_string(),
1704 probed_stable_id,
1705 &connection,
1706 timeout,
1707 )
1708 .await;
1709 let last_inbound_activity_age = self
1710 .probe_registry
1711 .last_inbound_activity_age(&endpoint_id.to_string(), probed_stable_id)
1712 .await;
1713 let current_stable_id = self
1714 .connections
1715 .read()
1716 .await
1717 .get(&endpoint_id)
1718 .map(transport_generation_for_connection)?;
1719 if current_stable_id != probed_stable_id {
1720 return Some(IrohConnectionProbe {
1721 transport_stable_id: probed_stable_id,
1725 responsive: false,
1726 last_inbound_activity_age,
1727 });
1728 }
1729 Some(IrohConnectionProbe {
1730 transport_stable_id: probed_stable_id,
1731 responsive,
1732 last_inbound_activity_age,
1733 })
1734 }
1735
1736 pub fn incoming_streams_stream(&self) -> async_channel::Receiver<IncomingStream> {
1737 self.incoming_streams_receiver.clone()
1738 }
1739
1740 pub(crate) async fn authorize_inbound_replacement(
1743 &self,
1744 endpoint_id: EndpointId,
1745 transport_id: u64,
1746 ttl: std::time::Duration,
1747 ) -> Instant {
1748 let expires_at = Instant::now() + ttl;
1749 self.inbound_replacement_authorizations
1750 .write()
1751 .await
1752 .insert(
1753 endpoint_id,
1754 InboundReplacementAuthorization {
1755 transport_id,
1756 expires_at,
1757 },
1758 );
1759 expires_at
1760 }
1761
1762 pub(crate) async fn revoke_inbound_replacement_if_current(
1765 &self,
1766 endpoint_id: EndpointId,
1767 transport_id: u64,
1768 expires_at: Instant,
1769 ) -> bool {
1770 let mut authorizations = self.inbound_replacement_authorizations.write().await;
1771 if authorizations.get(&endpoint_id).is_some_and(|current| {
1772 current.transport_id == transport_id && current.expires_at == expires_at
1773 }) {
1774 authorizations.remove(&endpoint_id);
1775 true
1776 } else {
1777 false
1778 }
1779 }
1780
1781 pub async fn accept_external_connection(&self, connection: Connection) -> Result<()> {
1784 let replacement_transport_id = consume_authorized_inbound_replacement(
1785 &connection,
1786 &self.inbound_replacement_authorizations,
1787 )
1788 .await;
1789 run_connection_loop(
1790 connection,
1791 self.accept_events.clone(),
1792 self.incoming_streams.clone(),
1793 self.connections.clone(),
1794 self.connection_inserted_at.clone(),
1795 self.endpoint.id(),
1796 IrohConnectionDirection::Inbound,
1797 self.manual_disconnect_notices.clone(),
1798 self.probe_registry.clone(),
1799 self.heartbeat_manager.clone(),
1800 self.heartbeat_health_tx.clone(),
1801 self.heartbeat_config.clone(),
1802 replacement_transport_id,
1803 None,
1804 )
1805 .await
1806 .map_err(|e| anyhow::anyhow!(e.to_string()))
1807 }
1808
1809 pub(crate) async fn accept_external_connection_with_install_notifier(
1813 &self,
1814 connection: Connection,
1815 install_outcome_sender: oneshot::Sender<ExternalConnectionInstallOutcome>,
1816 ) -> Result<()> {
1817 let replacement_transport_id = consume_authorized_inbound_replacement(
1818 &connection,
1819 &self.inbound_replacement_authorizations,
1820 )
1821 .await;
1822 run_connection_loop(
1823 connection,
1824 self.accept_events.clone(),
1825 self.incoming_streams.clone(),
1826 self.connections.clone(),
1827 self.connection_inserted_at.clone(),
1828 self.endpoint.id(),
1829 IrohConnectionDirection::Inbound,
1830 self.manual_disconnect_notices.clone(),
1831 self.probe_registry.clone(),
1832 self.heartbeat_manager.clone(),
1833 self.heartbeat_health_tx.clone(),
1834 self.heartbeat_config.clone(),
1835 replacement_transport_id,
1836 Some(install_outcome_sender),
1837 )
1838 .await
1839 .map_err(|e| anyhow::anyhow!(e.to_string()))
1840 }
1841
1842 pub async fn get_connection(&self, endpoint_id: EndpointId) -> Option<Connection> {
1846 let conns = self.connections.read().await;
1847 conns.get(&endpoint_id).cloned()
1848 }
1849
1850 pub async fn active_endpoint_ids(&self) -> Vec<EndpointId> {
1852 let conns = self.connections.read().await;
1853 conns.keys().cloned().collect()
1854 }
1855
1856 pub async fn take_manual_disconnect_notice(
1857 &self,
1858 endpoint_id: EndpointId,
1859 transport_stable_id: u64,
1860 ) -> bool {
1861 self.manual_disconnect_notices
1862 .write()
1863 .await
1864 .remove(&manual_disconnect_notice_key(
1865 endpoint_id,
1866 transport_stable_id,
1867 ))
1868 }
1869
1870 pub async fn has_manual_disconnect_notice(
1875 &self,
1876 endpoint_id: EndpointId,
1877 transport_stable_id: u64,
1878 ) -> bool {
1879 self.manual_disconnect_notices
1880 .read()
1881 .await
1882 .contains(&manual_disconnect_notice_key(
1883 endpoint_id,
1884 transport_stable_id,
1885 ))
1886 }
1887}
1888
1889#[cfg(test)]
1890mod tests {
1891 use super::*;
1892 use iroh::endpoint::Endpoint;
1893 use tokio::io::AsyncWriteExt;
1894 use tokio::time::{sleep, timeout, Duration};
1895
1896 async fn setup_protocol_endpoint() -> (
1897 Router,
1898 PlutoniumProtocol,
1899 broadcast::Receiver<AcceptEvent>,
1900 async_channel::Receiver<IncomingStream>,
1901 Arc<RwLock<HashMap<EndpointId, Connection>>>,
1902 ) {
1903 let (event_tx, event_rx) = broadcast::channel(16);
1904 let (stream_tx, stream_rx) = async_channel::unbounded();
1905 let connections = Arc::new(RwLock::new(HashMap::new()));
1906 let connection_inserted_at = Arc::new(RwLock::new(HashMap::new()));
1907
1908 let endpoint = Endpoint::builder(iroh::endpoint::presets::N0)
1909 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
1910 .bind()
1911 .await
1912 .unwrap();
1913
1914 let protocol = PlutoniumProtocol::new(
1915 event_tx.clone(),
1916 stream_tx.clone(),
1917 connections.clone(),
1918 connection_inserted_at,
1919 endpoint.id(),
1920 Arc::new(RwLock::new(HashSet::new())),
1921 IrohProbeRegistry::new(),
1922 Arc::new(RwLock::new(HashMap::new())),
1923 );
1924
1925 let router = Router::builder(endpoint)
1926 .accept(PlutoniumProtocol::ALPN, Arc::new(protocol.clone()))
1927 .spawn();
1928
1929 (router, protocol, event_rx, stream_rx, connections)
1930 }
1931
1932 #[test]
1933 fn manual_disconnect_notice_is_scoped_to_transport_generation() {
1934 let endpoint_id = iroh::SecretKey::generate().public();
1935 let old_transport = manual_disconnect_notice_key(endpoint_id, 41);
1936 let replacement_transport = manual_disconnect_notice_key(endpoint_id, 42);
1937 let mut notices = HashSet::from([old_transport.clone()]);
1938
1939 assert_ne!(old_transport, replacement_transport);
1940 assert!(!notices.remove(&replacement_transport));
1941 assert!(notices.remove(&old_transport));
1942 }
1943
1944 #[test]
1945 fn authorized_custom_transport_replaces_live_base_connection() {
1946 assert_eq!(
1947 custom_transport_arbitration_override(true, None, Some(4344901), Some(4344901)),
1948 Some(IrohConnectionInstallDecision::ReplaceExisting {
1949 close_existing_reason:
1950 crate::lifecycle_reason::REASON_NATIVE_CUSTOM_TRANSPORT_UPGRADE,
1951 })
1952 );
1953 }
1954
1955 #[test]
1956 fn live_custom_transport_wins_against_late_non_custom_duplicate() {
1957 assert_eq!(
1958 custom_transport_arbitration_override(true, Some(4344901), None, None),
1959 Some(IrohConnectionInstallDecision::KeepExisting {
1960 close_fresh_reason: "existing-custom-transport-preferred",
1961 })
1962 );
1963 }
1964
1965 #[test]
1966 fn canonical_outbound_dial_cannot_preempt_live_custom_transport() {
1967 let existing = Some(ExistingConnectionState {
1968 same_stable_id: false,
1969 alive: true,
1970 direction: IrohConnectionDirection::Inbound,
1971 age_ms: 0,
1972 });
1973
1974 assert!(!should_start_outbound_dial_with_transport_preference(
1975 Some((true, IrohConnectionDirection::Inbound)),
1976 Some(4344901),
1977 true,
1978 ));
1979 assert_eq!(
1980 decide_outbound_install_with_transport_preference(existing, Some(4344901), None, true,),
1981 IrohConnectionInstallDecision::KeepExisting {
1982 close_fresh_reason: "existing-custom-transport-preferred",
1983 },
1984 );
1985 }
1986
1987 #[test]
1988 fn unauthorized_custom_transport_uses_normal_direction_arbitration() {
1989 assert_eq!(
1990 custom_transport_arbitration_override(true, None, Some(4344901), None),
1991 None
1992 );
1993 }
1994
1995 #[tokio::test]
1996 async fn delayed_replacement_cleanup_cannot_revoke_a_newer_grant() {
1997 let endpoint = Endpoint::builder(iroh::endpoint::presets::N0)
1998 .relay_mode(iroh::RelayMode::Disabled)
1999 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2000 .bind()
2001 .await
2002 .expect("endpoint");
2003 let node = IrohNativeNode::spawn_with_endpoint_no_router(endpoint)
2004 .await
2005 .expect("native node");
2006 let remote = iroh::SecretKey::generate().public();
2007 let old_expiry = node
2008 .authorize_inbound_replacement(remote, 4344901, Duration::from_secs(1))
2009 .await;
2010 let new_expiry = node
2011 .authorize_inbound_replacement(remote, 4344901, Duration::from_secs(2))
2012 .await;
2013
2014 assert!(
2015 !node
2016 .revoke_inbound_replacement_if_current(remote, 4344901, old_expiry)
2017 .await
2018 );
2019 assert!(
2020 node.revoke_inbound_replacement_if_current(remote, 4344901, new_expiry)
2021 .await
2022 );
2023 }
2024
2025 #[tokio::test]
2026 async fn node_addr_does_not_wait_for_an_unavailable_relay() {
2027 let endpoint = Endpoint::builder(iroh::endpoint::presets::N0)
2028 .relay_mode(iroh::RelayMode::Disabled)
2029 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2030 .bind()
2031 .await
2032 .expect("bind relay-disabled endpoint");
2033 let expected_id = endpoint.id();
2034 let node = IrohNativeNode::spawn_with_endpoint(endpoint)
2035 .await
2036 .expect("spawn relay-disabled node");
2037
2038 let address = timeout(Duration::from_secs(1), node.node_addr())
2039 .await
2040 .expect("node_addr must not wait for relay readiness")
2041 .expect("read current endpoint address");
2042
2043 assert_eq!(address.id, expected_id);
2044 }
2045
2046 #[tokio::test]
2047 async fn test_two_nodes_connect_and_exchange_streams() {
2048 let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2049 let (r2, _proto2, mut events2, streams2, _) = setup_protocol_endpoint().await;
2050
2051 let ep1 = r1.endpoint();
2052 let ep2 = r2.endpoint();
2053
2054 let addr2 = ep2.addr();
2056
2057 let conn_res = ep1.connect(addr2, PlutoniumProtocol::ALPN).await.unwrap();
2059
2060 let event = timeout(Duration::from_secs(5), events2.recv())
2062 .await
2063 .unwrap()
2064 .unwrap();
2065 match event {
2066 AcceptEvent::Accepted { endpoint_id, .. } => assert_eq!(endpoint_id, ep1.id()),
2067 _ => panic!("Expected AcceptEvent::Accepted"),
2068 }
2069
2070 let (mut send1, _recv1) = conn_res.open_bi().await.unwrap();
2072 send1.write_all(b"hello node2").await.unwrap();
2073
2074 let incoming = timeout(Duration::from_secs(5), streams2.recv())
2076 .await
2077 .unwrap()
2078 .unwrap();
2079 assert_eq!(incoming.endpoint_id, ep1.id());
2080
2081 let mut recv2 = match incoming.stream {
2082 IncomingStreamType::Bi(_, r) => r,
2083 _ => panic!("Expected Bi stream"),
2084 };
2085
2086 let mut buf = [0u8; 11];
2087 recv2.read_exact(&mut buf).await.unwrap();
2088 assert_eq!(&buf, b"hello node2");
2089 }
2090
2091 #[tokio::test]
2092 async fn retiring_losing_control_candidate_preserves_buffered_verdict() {
2093 let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2094 let (r2, _proto2, mut events2, streams2, _) = setup_protocol_endpoint().await;
2095 let ep1 = r1.endpoint();
2096 let ep2 = r2.endpoint();
2097 let connection = ep1
2098 .connect(ep2.addr(), PlutoniumProtocol::ALPN)
2099 .await
2100 .expect("connect protocol endpoints");
2101 timeout(Duration::from_secs(5), events2.recv())
2102 .await
2103 .expect("host accept timeout")
2104 .expect("host accept event");
2105
2106 let (mut dialer_send, mut dialer_recv) =
2107 connection.open_bi().await.expect("open candidate stream");
2108 dialer_send
2109 .write_all(b"candidate")
2110 .await
2111 .expect("activate candidate stream");
2112 dialer_send.flush().await.expect("flush candidate stream");
2113 let incoming = timeout(Duration::from_secs(5), streams2.recv())
2114 .await
2115 .expect("host stream timeout")
2116 .expect("host stream");
2117 let IncomingStreamType::Bi(mut host_send, host_recv) = incoming.stream else {
2118 panic!("expected bidirectional candidate stream");
2119 };
2120 let verdict = b"session-token-approved";
2121 host_send
2122 .write_all(verdict)
2123 .await
2124 .expect("write approval verdict");
2125 host_send.flush().await.expect("flush approval verdict");
2126
2127 let (retired, received) = tokio::join!(
2128 crate::client::retire_losing_native_control_candidate(host_send, host_recv),
2129 dialer_recv.read_to_end(256),
2130 );
2131 retired.expect("retire candidate after peer acknowledges verdict");
2132 assert_eq!(
2133 received.expect("dialer reads losing-stream verdict"),
2134 verdict,
2135 );
2136 }
2137
2138 #[tokio::test]
2139 async fn retiring_replaced_control_send_preserves_buffered_verdict() {
2140 let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2141 let (r2, _proto2, mut events2, streams2, _) = setup_protocol_endpoint().await;
2142 let connection = r1
2143 .endpoint()
2144 .connect(r2.endpoint().addr(), PlutoniumProtocol::ALPN)
2145 .await
2146 .expect("connect protocol endpoints");
2147 timeout(Duration::from_secs(5), events2.recv())
2148 .await
2149 .expect("host accept timeout")
2150 .expect("host accept event");
2151
2152 let (mut dialer_send, mut dialer_recv) =
2153 connection.open_bi().await.expect("open displaced stream");
2154 dialer_send
2155 .write_all(b"candidate")
2156 .await
2157 .expect("activate displaced stream");
2158 dialer_send.flush().await.expect("flush displaced stream");
2159 let incoming = timeout(Duration::from_secs(5), streams2.recv())
2160 .await
2161 .expect("host stream timeout")
2162 .expect("host stream");
2163 let IncomingStreamType::Bi(mut host_send, _host_recv) = incoming.stream else {
2164 panic!("expected bidirectional displaced stream");
2165 };
2166 let verdict = b"session-token-approved";
2167 host_send
2168 .write_all(verdict)
2169 .await
2170 .expect("write approval verdict");
2171 host_send.flush().await.expect("host should flush approval");
2172 let displaced_send = Arc::new(tokio::sync::Mutex::new(host_send));
2173
2174 let (retired, received) = tokio::join!(
2175 crate::client::retire_replaced_native_control_send(displaced_send),
2176 dialer_recv.read_to_end(256),
2177 );
2178 retired.expect("retire replaced stream after peer acknowledges verdict");
2179 assert_eq!(
2180 received.expect("dialer reads replaced-stream verdict"),
2181 verdict,
2182 );
2183 }
2184
2185 #[tokio::test]
2186 async fn test_healthy_connection_not_replaced() {
2187 let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2188 let (r2, _proto2, mut events2, _streams2, conns2) = setup_protocol_endpoint().await;
2189
2190 let ep1 = r1.endpoint();
2191 let ep2 = r2.endpoint();
2192
2193 let addr2 = ep2.addr();
2194 let addr1 = ep1.addr();
2195
2196 let _conn1 = ep1.connect(addr2, PlutoniumProtocol::ALPN).await.unwrap();
2198
2199 let _ = timeout(Duration::from_secs(5), events2.recv())
2201 .await
2202 .unwrap()
2203 .unwrap();
2204
2205 let active_count = conns2.read().await.len();
2206 assert_eq!(active_count, 1);
2207
2208 let original_stable_id = conns2.read().await.get(&ep1.id()).unwrap().stable_id();
2209
2210 let _conn2 = ep2.connect(addr1, PlutoniumProtocol::ALPN).await.unwrap();
2212
2213 sleep(Duration::from_millis(100)).await;
2215
2216 let current_conn = conns2.read().await.get(&ep1.id()).unwrap().clone();
2218 assert_eq!(current_conn.stable_id(), original_stable_id);
2219 }
2220
2221 #[tokio::test]
2222 async fn fresh_connection_replaces_an_unresponsive_same_identity_transport() {
2223 let (host, _protocol, mut host_events, _streams, host_connections) =
2224 setup_protocol_endpoint().await;
2225 let host_addr = host.endpoint().addr();
2226 let remote_secret = iroh::SecretKey::generate();
2227
2228 let first_remote = Endpoint::builder(iroh::endpoint::presets::N0)
2229 .secret_key(remote_secret.clone())
2230 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2231 .bind()
2232 .await
2233 .expect("bind first remote runtime");
2234 let _first_connection = first_remote
2235 .connect(host_addr.clone(), PlutoniumProtocol::ALPN)
2236 .await
2237 .expect("connect first remote runtime");
2238 timeout(Duration::from_secs(5), host_events.recv())
2239 .await
2240 .expect("first host accept timeout")
2241 .expect("first host accept");
2242 let first_stable_id = host_connections
2243 .read()
2244 .await
2245 .get(&remote_secret.public())
2246 .expect("first transport installed")
2247 .stable_id();
2248
2249 let replacement_remote = Endpoint::builder(iroh::endpoint::presets::N0)
2253 .secret_key(remote_secret.clone())
2254 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2255 .bind()
2256 .await
2257 .expect("bind replacement remote runtime");
2258 let _replacement_connection = replacement_remote
2259 .connect(host_addr, PlutoniumProtocol::ALPN)
2260 .await
2261 .expect("connect replacement remote runtime");
2262 timeout(Duration::from_secs(5), host_events.recv())
2263 .await
2264 .expect("replacement host accept timeout")
2265 .expect("replacement host accept");
2266
2267 let replacement_stable_id = host_connections
2268 .read()
2269 .await
2270 .get(&remote_secret.public())
2271 .expect("replacement transport installed")
2272 .stable_id();
2273 assert_ne!(
2274 replacement_stable_id, first_stable_id,
2275 "an actively unresponsive old leg must not block a same-identity process restart",
2276 );
2277 }
2278
2279 #[tokio::test]
2283 async fn idle_symmetric_iroh_heartbeat_send_uni_open_rate_bounded() {
2284 use crate::heartbeat::idle_symmetric_heartbeat_max_send_uni_opens_upper_bound;
2285 use crate::heartbeat::iroh_heartbeat::test_counters;
2286 use std::sync::atomic::Ordering;
2287
2288 test_counters::reset_heartbeat_send_uni_count();
2289
2290 let tick = Duration::from_millis(200);
2291 let heartbeat_config = HeartbeatConfig {
2292 tick_interval: tick,
2293 suspect_after: Duration::from_secs(10),
2294 stale_after: Duration::from_secs(30),
2295 send_timeout: Duration::from_secs(2),
2296 };
2297
2298 let (tx1, _rx1) = mpsc::channel::<HealthTransition>(32);
2299 let (tx2, _rx2) = mpsc::channel::<HealthTransition>(32);
2300 let mgr1 = IrohHeartbeatManager::new();
2301 let mgr2 = IrohHeartbeatManager::new();
2302
2303 let ep1 = Endpoint::builder(iroh::endpoint::presets::N0)
2304 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2305 .bind()
2306 .await
2307 .unwrap();
2308 let ep2 = Endpoint::builder(iroh::endpoint::presets::N0)
2309 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2310 .bind()
2311 .await
2312 .unwrap();
2313
2314 let node1 = IrohNativeNode::spawn_with_heartbeat(ep1, mgr1, tx1, heartbeat_config.clone())
2315 .await
2316 .unwrap();
2317 let node2 = IrohNativeNode::spawn_with_heartbeat(ep2, mgr2, tx2, heartbeat_config)
2318 .await
2319 .unwrap();
2320
2321 let remote_id = node2.endpoint().id();
2322 let remote_addr = node2.node_addr().await.unwrap();
2323
2324 let mut conn_stream = node1.connect_addr(remote_id, remote_addr);
2325 let connected = timeout(Duration::from_secs(5), async {
2326 while let Some(ev) = conn_stream.next().await {
2327 match ev {
2328 ConnectEvent::Connected => return true,
2329 ConnectEvent::Closed { .. } => return false,
2330 }
2331 }
2332 false
2333 })
2334 .await
2335 .unwrap();
2336 assert!(connected, "expected outbound connect to reach Connected");
2337
2338 let observe = Duration::from_millis(900);
2339 sleep(observe).await;
2340
2341 let observed = test_counters::HEARTBEAT_SEND_UNI_COUNT.load(Ordering::SeqCst);
2342 let bound = idle_symmetric_heartbeat_max_send_uni_opens_upper_bound(observe, tick);
2343 assert!(
2344 observed <= bound,
2345 "heartbeat send_uni opens should stay within idle symmetric model (observed={} bound={})",
2346 observed,
2347 bound
2348 );
2349 assert!(
2350 observed >= 4,
2351 "expected some heartbeat uni traffic after idle window (observed={})",
2352 observed
2353 );
2354 }
2355
2356 #[tokio::test]
2357 async fn active_probe_requires_remote_control_loop_round_trip() {
2358 let ep1 = Endpoint::builder(iroh::endpoint::presets::N0)
2359 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2360 .bind()
2361 .await
2362 .expect("bind first endpoint");
2363 let ep2 = Endpoint::builder(iroh::endpoint::presets::N0)
2364 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2365 .bind()
2366 .await
2367 .expect("bind second endpoint");
2368
2369 let node1 = IrohNativeNode::spawn_with_endpoint(ep1)
2370 .await
2371 .expect("spawn first node");
2372 let node2 = IrohNativeNode::spawn_with_endpoint(ep2)
2373 .await
2374 .expect("spawn second node");
2375 let remote_id = node2.endpoint().id();
2376 let mut events = node1.connect_addr(
2377 remote_id,
2378 node2.node_addr().await.expect("second node address"),
2379 );
2380
2381 let connected = timeout(Duration::from_secs(5), async {
2382 while let Some(event) = events.next().await {
2383 if matches!(event, ConnectEvent::Connected) {
2384 return true;
2385 }
2386 }
2387 false
2388 })
2389 .await
2390 .expect("connect timeout");
2391 assert!(connected, "expected physical connection");
2392
2393 let probe = node1
2394 .probe_connection(remote_id, Duration::from_secs(2))
2395 .await
2396 .expect("current physical generation");
2397 assert!(
2398 probe.responsive,
2399 "remote runtime must return the typed pong"
2400 );
2401 assert_ne!(probe.transport_stable_id, 0);
2402 assert!(
2403 probe.last_inbound_activity_age.is_some(),
2404 "the accepted pong stream must also be recorded as generation-bound activity",
2405 );
2406 }
2407
2408 #[tokio::test]
2409 async fn active_probe_is_not_starved_by_bidirectional_stream_load() {
2410 let ep1 = Endpoint::builder(iroh::endpoint::presets::N0)
2411 .relay_mode(iroh::RelayMode::Disabled)
2412 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2413 .bind()
2414 .await
2415 .expect("bind first endpoint");
2416 let ep2 = Endpoint::builder(iroh::endpoint::presets::N0)
2417 .relay_mode(iroh::RelayMode::Disabled)
2418 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2419 .bind()
2420 .await
2421 .expect("bind second endpoint");
2422
2423 let node1 = IrohNativeNode::spawn_with_endpoint(ep1)
2424 .await
2425 .expect("spawn first node");
2426 let node2 = IrohNativeNode::spawn_with_endpoint(ep2)
2427 .await
2428 .expect("spawn second node");
2429 let remote_id = node2.endpoint().id();
2430 let mut events = node1.connect_addr(
2431 remote_id,
2432 node2.node_addr().await.expect("second node address"),
2433 );
2434 timeout(Duration::from_secs(5), async {
2435 while let Some(event) = events.next().await {
2436 if matches!(event, ConnectEvent::Connected) {
2437 return;
2438 }
2439 }
2440 panic!("connection event stream ended");
2441 })
2442 .await
2443 .expect("connect timeout");
2444
2445 let incoming = node2.incoming_streams_stream();
2446 let drain = tokio::spawn(async move { while incoming.recv().await.is_ok() {} });
2447 let flood_node = node1.clone();
2448 let flood = tokio::spawn(async move {
2449 loop {
2450 let Ok((mut send, _recv)) = flood_node.open_bi(remote_id).await else {
2451 break;
2452 };
2453 if send.write_all(b"x").await.is_err() {
2454 break;
2455 }
2456 let _ = send.finish();
2457 tokio::task::yield_now().await;
2458 }
2459 });
2460
2461 sleep(Duration::from_millis(25)).await;
2462 let probe = node1
2463 .probe_connection(remote_id, Duration::from_secs(2))
2464 .await
2465 .expect("current physical generation");
2466 flood.abort();
2467 drain.abort();
2468
2469 assert!(
2470 probe.responsive,
2471 "bidirectional application/control load must not starve the uni-stream liveness control plane",
2472 );
2473 }
2474
2475 #[tokio::test]
2476 async fn concurrent_native_address_dials_install_one_physical_generation() {
2477 let ep1 = Endpoint::builder(iroh::endpoint::presets::N0)
2478 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2479 .bind()
2480 .await
2481 .expect("bind first endpoint");
2482 let ep2 = Endpoint::builder(iroh::endpoint::presets::N0)
2483 .alpns(vec![PlutoniumProtocol::ALPN.to_vec()])
2484 .bind()
2485 .await
2486 .expect("bind second endpoint");
2487
2488 let node1 = IrohNativeNode::spawn_with_endpoint(ep1)
2489 .await
2490 .expect("spawn first node");
2491 let node2 = IrohNativeNode::spawn_with_endpoint(ep2)
2492 .await
2493 .expect("spawn second node");
2494 let remote_id = node2.endpoint().id();
2495 let remote_addr = node2.node_addr().await.expect("second node address");
2496 let mut remote_events = node2.accept_events();
2497
2498 let mut dial_streams = (0..16)
2502 .map(|_| node1.connect_addr(remote_id, remote_addr.clone()))
2503 .collect::<Vec<_>>();
2504 let connected = futures::future::join_all(dial_streams.iter_mut().map(|events| async {
2505 timeout(Duration::from_secs(5), async {
2506 while let Some(event) = events.next().await {
2507 match event {
2508 ConnectEvent::Connected => return true,
2509 ConnectEvent::Closed { .. } => return false,
2510 }
2511 }
2512 false
2513 })
2514 .await
2515 .expect("concurrent dial timeout")
2516 }))
2517 .await;
2518 assert!(
2519 connected.iter().all(|connected| *connected),
2520 "every caller should reuse the one installed physical connection"
2521 );
2522
2523 let accepted = timeout(Duration::from_secs(5), remote_events.next())
2524 .await
2525 .expect("remote accept timeout")
2526 .expect("remote accept event");
2527 assert!(matches!(accepted, AcceptEvent::Accepted { .. }));
2528 let unexpected_second_accept = timeout(Duration::from_millis(300), async {
2529 while let Some(event) = remote_events.next().await {
2530 if matches!(event, AcceptEvent::Accepted { .. }) {
2531 return Some(event);
2532 }
2533 }
2534 None
2535 })
2536 .await;
2537 assert!(
2538 unexpected_second_accept.is_err(),
2539 "concurrent callers must not create a second remote generation"
2540 );
2541 assert_eq!(
2542 node1.active_endpoint_ids().await,
2543 vec![remote_id],
2544 "exactly one physical endpoint mapping should remain"
2545 );
2546 assert!(
2547 node1
2548 .outbound_dial_gates
2549 .lock()
2550 .unwrap_or_else(|poisoned| poisoned.into_inner())
2551 .is_empty(),
2552 "completed single-flight entries must be retired"
2553 );
2554 }
2555
2556 #[tokio::test]
2557 async fn application_uni_stream_prefix_is_replayed_after_control_classification() {
2558 let (r1, _proto1, _events1, _streams1, _) = setup_protocol_endpoint().await;
2559 let (r2, _proto2, mut events2, streams2, _) = setup_protocol_endpoint().await;
2560 let connection = r1
2561 .endpoint()
2562 .connect(r2.endpoint().addr(), PlutoniumProtocol::ALPN)
2563 .await
2564 .expect("connect endpoints");
2565 timeout(Duration::from_secs(5), events2.recv())
2566 .await
2567 .expect("accept event timeout")
2568 .expect("accept event");
2569
2570 let payload = b"application-uni-payload-that-is-not-control";
2571 let mut send = connection.open_uni().await.expect("open app uni stream");
2572 send.write_all(payload)
2573 .await
2574 .expect("write app uni payload");
2575 send.finish().expect("finish app uni payload");
2576
2577 let incoming = timeout(Duration::from_secs(5), streams2.recv())
2578 .await
2579 .expect("application stream timeout")
2580 .expect("application stream");
2581 let IncomingStreamType::Uni(mut recv) = incoming.stream else {
2582 panic!("expected application uni stream");
2583 };
2584 let mut received = Vec::new();
2585 tokio::io::AsyncReadExt::read_to_end(&mut recv, &mut received)
2586 .await
2587 .expect("read replayed application payload");
2588 assert_eq!(received, payload);
2589 }
2590}