use std::{
collections::{BTreeMap, BTreeSet},
net::SocketAddr,
sync::{Arc, RwLock},
time::Instant,
};
use futures_util::StreamExt;
use iroh::{
Endpoint, EndpointAddr, PublicKey,
address_lookup::memory::MemoryLookup,
endpoint::{AfterHandshakeOutcome, BeforeConnectOutcome, EndpointHooks, presets},
};
use iroh_mdns_address_lookup::{DiscoveryEvent, MdnsAddressLookup};
use tokio::sync::{Mutex, OnceCell};
use super::{ConnectionBudget, Error, NetworkProfile, PeerId, RelayOptions, Result, transport};
const MAX_ADDRESS_HINTS: usize = 4096;
const MAX_CACHED_CONNECTIONS: usize = 32;
const MAX_CONTROL_CONNECTIONS: usize = 32;
const UNREACHABLE_BACKOFF: std::time::Duration = std::time::Duration::from_secs(5);
const MAX_UNREACHABLE_BACKOFF: std::time::Duration = std::time::Duration::from_secs(120);
type PeerProtocol = (PeerId, Vec<u8>);
struct CachedConnection {
connection: Arc<OnceCell<Arc<iroh::endpoint::Connection>>>,
last_used: Instant,
}
#[derive(Debug, Default)]
struct ObservedConnections {
live: Vec<iroh::endpoint::WeakConnectionHandle>,
limited: bool,
}
#[derive(Clone, Debug, Default)]
struct ConnectionObserver(Arc<RwLock<ObservedConnections>>);
impl EndpointHooks for ConnectionObserver {
async fn after_handshake<'a>(
&'a self,
connection: &'a iroh::endpoint::Connection,
) -> AfterHandshakeOutcome {
let mut observed = self.0.write().unwrap();
observed
.live
.retain(|weak| weak.upgrade().is_some_and(|c| c.close_reason().is_none()));
if observed.live.len() < 256 {
observed.live.push(connection.weak_handle());
} else {
observed.limited = true;
}
AfterHandshakeOutcome::Accept
}
}
#[derive(Clone, Debug, Default)]
struct GossipAuthorization(Arc<RwLock<BTreeMap<Vec<u8>, BTreeSet<PeerId>>>>);
impl GossipAuthorization {
fn allows(&self, alpn: &[u8], peer: PeerId) -> bool {
self.0
.read()
.unwrap()
.get(alpn)
.is_some_and(|allowed| allowed.contains(&peer))
}
}
impl EndpointHooks for GossipAuthorization {
async fn before_connect<'a>(
&'a self,
remote: &'a EndpointAddr,
alpn: &'a [u8],
) -> BeforeConnectOutcome {
if !alpn.starts_with(super::overlay::ALPN_PREFIX)
|| self.allows(alpn, *remote.id.as_bytes())
{
BeforeConnectOutcome::Accept
} else {
BeforeConnectOutcome::Reject
}
}
async fn after_handshake<'a>(
&'a self,
connection: &'a iroh::endpoint::Connection,
) -> AfterHandshakeOutcome {
if !connection.alpn().starts_with(super::overlay::ALPN_PREFIX)
|| self.allows(connection.alpn(), *connection.remote_id().as_bytes())
{
AfterHandshakeOutcome::Accept
} else {
AfterHandshakeOutcome::Reject {
error_code: 403u32.into(),
reason: b"workspace endpoint denied".to_vec(),
}
}
}
}
fn unreachable_backoff(failures: u32) -> std::time::Duration {
UNREACHABLE_BACKOFF
.saturating_mul(1 << failures.saturating_sub(1).min(5))
.min(MAX_UNREACHABLE_BACKOFF)
}
#[derive(Clone)]
pub(super) struct Connections {
endpoint: Endpoint,
budget: ConnectionBudget,
bound_address: SocketAddr,
addresses: Arc<Mutex<BTreeMap<PeerId, SocketAddr>>>,
memory: MemoryLookup,
alpns: Arc<Mutex<BTreeSet<Vec<u8>>>>,
gossip_authorization: GossipAuthorization,
observer: ConnectionObserver,
outgoing: Arc<Mutex<BTreeMap<PeerProtocol, CachedConnection>>>,
nearby: Arc<Mutex<BTreeSet<PeerId>>>,
nearby_listener: Arc<Mutex<Option<tokio::task::JoinHandle<()>>>>,
unreachable: Arc<Mutex<BTreeMap<PeerId, (Instant, u32)>>>,
address_lookup: bool,
use_ip_hints: bool,
}
impl Connections {
pub(super) async fn bind(
address: SocketAddr,
profile: NetworkProfile,
secret: Option<iroh::SecretKey>,
budget: ConnectionBudget,
alpns: Vec<Vec<u8>>,
relay: Option<RelayOptions>,
) -> Result<Self> {
let (mdns_service, wan_lookup, relay_only, use_ip_hints) = profile.settings();
let gossip_authorization = GossipAuthorization::default();
let observer = ConnectionObserver::default();
let builder = if wan_lookup {
Endpoint::builder(presets::N0)
} else {
Endpoint::builder(presets::Minimal)
.clear_relay_transports()
.clear_ip_transports()
};
let memory = MemoryLookup::new();
let mut builder = if relay_only {
builder.clear_ip_transports()
} else {
builder.bind_addr(address).map_err(transport)?
}
.alpns(alpns.clone())
.address_lookup(memory.clone())
.hooks(gossip_authorization.clone())
.hooks(budget.clone())
.hooks(observer.clone());
#[cfg(all(target_os = "android", target_arch = "x86_64"))]
{
builder = builder.transport_config(
iroh::endpoint::QuicTransportConfig::builder()
.enable_segmentation_offload(false)
.build(),
);
}
if let Some(secret) = secret {
builder = builder.secret_key(secret);
}
if let Some(relay) = relay {
builder = builder
.relay_mode(iroh::RelayMode::Custom(relay.map))
.ca_tls_config(relay.tls);
}
let endpoint = builder.bind().await.map_err(transport)?;
let mdns = if let Some(service_name) = mdns_service {
let lookup = MdnsAddressLookup::builder()
.service_name(service_name)
.build(endpoint.id())
.map_err(transport)?;
endpoint
.address_lookup()
.map_err(transport)?
.add(lookup.clone());
Some(lookup)
} else {
None
};
let nearby = Arc::new(Mutex::new(BTreeSet::new()));
let nearby_listener = mdns.as_ref().map(|lookup| {
let lookup = lookup.clone();
let nearby = nearby.clone();
let own = endpoint.id();
tokio::spawn(async move {
let mut events = lookup.subscribe().await;
while let Some(event) = events.next().await {
match event {
DiscoveryEvent::Discovered { endpoint_info, .. } => {
let endpoint = endpoint_info.endpoint_id;
if endpoint != own {
let mut discovered = nearby.lock().await;
if discovered.len() < 16 {
discovered.insert(*endpoint.as_bytes());
}
}
}
DiscoveryEvent::Expired { endpoint_id } => {
nearby.lock().await.remove(endpoint_id.as_bytes());
}
_ => {}
}
}
})
});
let bound_address = endpoint.bound_sockets().first().copied().unwrap_or(address);
Ok(Self {
endpoint,
budget,
bound_address,
addresses: Arc::new(Mutex::new(BTreeMap::new())),
memory,
alpns: Arc::new(Mutex::new(alpns.into_iter().collect())),
gossip_authorization,
observer,
outgoing: Arc::new(Mutex::new(BTreeMap::new())),
nearby,
nearby_listener: Arc::new(Mutex::new(nearby_listener)),
unreachable: Arc::new(Mutex::new(BTreeMap::new())),
address_lookup: mdns_service.is_some() || wan_lookup,
use_ip_hints,
})
}
pub(super) async fn accept(&self) -> Option<iroh::endpoint::Incoming> {
self.endpoint.accept().await
}
pub(super) fn id(&self) -> PeerId {
*self.endpoint.id().as_bytes()
}
pub(super) fn address(&self) -> SocketAddr {
self.bound_address
}
pub(super) fn endpoint(&self) -> Endpoint {
self.endpoint.clone()
}
pub(super) async fn wait_online(&self) {
self.endpoint.online().await;
}
pub(super) fn capacity_counts(&self) -> super::budget::CapacityCounts {
self.budget.capacity_counts()
}
pub(super) fn mark_evictable(&self, id: usize) {
self.budget.mark_evictable(id);
}
pub(super) fn exchange(&self, id: usize) -> super::budget::ExchangeGuard {
self.budget.exchange(id)
}
pub(super) fn gossip_dial_capacity(&self) -> Arc<tokio::sync::Semaphore> {
self.budget.gossip_dials()
}
pub(super) fn metrics(&self) -> super::TransportMetrics {
let counters = &self.endpoint.metrics().socket;
let mut observed = self.observer.0.write().unwrap();
let mut paths = Vec::new();
observed.live.retain(|weak| {
let Some(connection) = weak.upgrade().filter(|c| c.close_reason().is_none()) else {
return false;
};
for path in connection.paths().iter().filter(|path| path.is_selected()) {
paths.push(super::PeerPath {
endpoint: *connection.remote_id().as_bytes(),
route: if path.is_ip() {
"direct"
} else if path.is_relay() {
"relay"
} else {
"custom"
},
rtt_ms: path.rtt().as_millis().min(u64::MAX as u128) as u64,
});
}
true
});
super::TransportMetrics {
received_bytes: counters
.recv_data_ipv4
.get()
.saturating_add(counters.recv_data_ipv6.get())
.saturating_add(counters.recv_data_relay.get())
.saturating_add(counters.recv_data_custom.get()),
sent_bytes: counters
.send_ipv4
.get()
.saturating_add(counters.send_ipv6.get())
.saturating_add(counters.send_relay.get()),
receive_queue: 0,
paths,
paths_limited: observed.limited,
}
}
pub(super) async fn add_alpn(&self, alpn: Vec<u8>) {
let mut alpns = self.alpns.lock().await;
if alpns.insert(alpn) {
self.endpoint.set_alpns(alpns.iter().cloned().collect());
}
}
pub(super) async fn authorize_gossip(&self, alpn: Vec<u8>, peers: Vec<PeerId>) {
self.gossip_authorization
.0
.write()
.unwrap()
.insert(alpn.clone(), peers.into_iter().collect());
self.add_alpn(alpn).await;
}
pub(super) async fn network_change(&self) {
self.unreachable.lock().await.clear();
self.endpoint.network_change().await;
}
pub(super) async fn add_address_hint(&self, peer: PeerId, address: SocketAddr) -> Result<()> {
let mut addresses = self.addresses.lock().await;
if addresses.len() >= MAX_ADDRESS_HINTS && !addresses.contains_key(&peer) {
return Err(Error::TooLarge);
}
addresses.insert(peer, address);
self.unreachable.lock().await.remove(&peer);
let key = PublicKey::from_bytes(&peer).map_err(transport)?;
self.memory
.add_endpoint_info(EndpointAddr::new(key).with_ip_addr(address));
Ok(())
}
pub(super) async fn address_hint(&self, peer: PeerId) -> Option<SocketAddr> {
self.addresses.lock().await.get(&peer).copied()
}
pub(super) async fn remember_observed(&self, peer: PeerId, address: SocketAddr) {
let mut addresses = self.addresses.lock().await;
self.unreachable.lock().await.remove(&peer);
if addresses.len() < MAX_ADDRESS_HINTS || addresses.contains_key(&peer) {
addresses.insert(peer, address);
if let Ok(key) = PublicKey::from_bytes(&peer) {
self.memory
.add_endpoint_info(EndpointAddr::new(key).with_ip_addr(address));
}
}
}
pub(super) async fn connect(
&self,
peer: PeerId,
alpn: &[u8],
) -> Result<Arc<iroh::endpoint::Connection>> {
if self
.unreachable
.lock()
.await
.get(&peer)
.is_some_and(|(retry, _)| Instant::now() < *retry)
{
return Err(Error::Transport(
"peer recently unreachable; retrying later".into(),
));
}
let control = alpn == super::control::ALPN;
let budget = if control {
MAX_CONTROL_CONNECTIONS
} else {
MAX_CACHED_CONNECTIONS
};
let slot = {
let mut outgoing = self.outgoing.lock().await;
outgoing.retain(|_, cached| {
cached
.connection
.get()
.is_none_or(|c| c.close_reason().is_none())
});
let key = (peer, alpn.to_vec());
let same_plane = |key: &PeerProtocol| (key.1 == super::control::ALPN) == control;
if !outgoing.contains_key(&key)
&& outgoing.keys().filter(|key| same_plane(key)).count() >= budget
{
let idle = outgoing
.iter()
.filter(|(key, cached)| {
same_plane(key)
&& Arc::strong_count(&cached.connection) == 1
&& cached
.connection
.get()
.is_none_or(|c| Arc::strong_count(c) == 1)
})
.min_by_key(|(_, cached)| cached.last_used)
.map(|(key, _)| key.clone());
let idle = idle.ok_or(Error::Backpressure)?;
outgoing.remove(&idle);
}
let cached = outgoing.entry(key).or_insert_with(|| CachedConnection {
connection: Arc::new(OnceCell::new()),
last_used: Instant::now(),
});
cached.last_used = Instant::now();
cached.connection.clone()
};
slot.get_or_try_init(|| async { self.dial(peer, alpn).await.map(Arc::new) })
.await
.cloned()
}
async fn dial(&self, peer: PeerId, alpn: &[u8]) -> Result<iroh::endpoint::Connection> {
let key = PublicKey::from_bytes(&peer).map_err(transport)?;
let hint = if self.use_ip_hints {
self.address_hint(peer).await
} else {
None
};
let destination = match hint {
Some(address) => EndpointAddr::new(key).with_ip_addr(address),
None if self.address_lookup => EndpointAddr::new(key),
None => return Err(Error::MissingPeer),
};
let _permit = self.budget.dial(alpn)?;
let attempt = |destination: EndpointAddr, limit| async move {
match tokio::time::timeout(limit, self.endpoint.connect(destination, alpn)).await {
Ok(result) => result.map_err(transport),
Err(_) => Err(Error::Timeout("connect")),
}
};
let outcome = attempt(destination, super::TIMEOUT).await;
let mut unreachable = self.unreachable.lock().await;
match &outcome {
Ok(_) => {
unreachable.remove(&peer);
}
Err(Error::Timeout(_)) => {
let failures = unreachable
.get(&peer)
.map_or(0, |(_, failures)| *failures)
.saturating_add(1);
unreachable.insert(
peer,
(Instant::now() + unreachable_backoff(failures), failures),
);
}
Err(_) => {}
}
outcome
}
pub(super) async fn discard(&self, peer: PeerId, alpn: &[u8]) {
let cached = self.outgoing.lock().await.remove(&(peer, alpn.to_vec()));
if let Some(connection) = cached.and_then(|cached| cached.connection.get().cloned()) {
connection.close(0u8.into(), b"stalled");
}
}
pub(super) async fn nearby_peers(&self, _first_result: bool) -> Vec<PeerId> {
self.nearby.lock().await.iter().copied().take(16).collect()
}
pub(super) async fn close(&self) {
if let Some(listener) = self.nearby_listener.lock().await.take() {
listener.abort();
let _ = listener.await;
}
self.endpoint.close().await;
self.outgoing.lock().await.clear();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn reuse_is_single_flight_protocol_scoped_and_never_evicts_busy_exchanges() {
tokio::time::timeout(std::time::Duration::from_secs(15), async {
let mut alpns: Vec<_> = (0..=MAX_CACHED_CONNECTIONS)
.map(|i| format!("test-cache/{i}").into_bytes())
.collect();
alpns.push(super::super::control::ALPN.to_vec());
let peer = Endpoint::builder(presets::Minimal)
.clear_relay_transports()
.clear_ip_transports()
.bind_addr("127.0.0.1:0")
.unwrap()
.alpns(alpns.clone())
.bind()
.await
.unwrap();
let endpoint = peer.clone();
let server = tokio::spawn(async move {
let mut workers = tokio::task::JoinSet::new();
while let Some(incoming) = endpoint.accept().await {
workers.spawn(async move {
incoming.await.unwrap().closed().await;
});
}
while let Some(result) = workers.join_next().await {
result.unwrap();
}
});
let cache = Connections::bind(
"127.0.0.1:0".parse().unwrap(),
NetworkProfile::Direct,
None,
ConnectionBudget::default(),
vec![],
None,
)
.await
.unwrap();
let id = *peer.id().as_bytes();
assert!(matches!(
cache.connect(id, &alpns[0]).await,
Err(Error::MissingPeer)
));
cache
.add_address_hint(id, peer.bound_sockets()[0])
.await
.unwrap();
let mut busy =
futures_util::future::join_all((0..8).map(|_| cache.connect(id, &alpns[0])))
.await
.into_iter()
.collect::<Result<Vec<_>>>()
.unwrap();
let first = busy[0].stable_id();
assert!(
busy.iter()
.all(|connection| connection.stable_id() == first)
);
for alpn in &alpns[1..MAX_CACHED_CONNECTIONS] {
let connection = cache.connect(id, alpn).await.unwrap();
assert_eq!(connection.alpn(), alpn);
assert_ne!(connection.stable_id(), first);
busy.push(connection);
}
assert!(matches!(
cache.connect(id, &alpns[MAX_CACHED_CONNECTIONS]).await,
Err(Error::Backpressure)
));
assert!(
busy.iter()
.all(|connection| connection.close_reason().is_none())
);
let control = cache
.connect(id, super::super::control::ALPN)
.await
.unwrap();
assert_eq!(control.alpn(), super::super::control::ALPN);
drop(busy);
let extra = cache
.connect(id, &alpns[MAX_CACHED_CONNECTIONS])
.await
.unwrap();
assert_eq!(extra.alpn(), alpns[MAX_CACHED_CONNECTIONS]);
assert_eq!(
cache.outgoing.lock().await.len(),
MAX_CACHED_CONNECTIONS + 1
);
let fresh = cache.connect(id, &alpns[0]).await.unwrap();
assert_ne!(fresh.stable_id(), first);
cache.close().await;
assert!(cache.outgoing.lock().await.is_empty());
peer.close().await;
server.await.unwrap();
})
.await
.unwrap();
}
#[tokio::test]
async fn an_unreachable_peer_backs_off_without_holding_dial_capacity() {
tokio::time::timeout(std::time::Duration::from_secs(60), async {
let alpn = super::super::control::ALPN.to_vec();
let cache = Connections::bind(
"127.0.0.1:0".parse().unwrap(),
NetworkProfile::Direct,
None,
ConnectionBudget::default(),
vec![],
None,
)
.await
.unwrap();
let dead = *iroh::SecretKey::generate().public().as_bytes();
let nowhere = std::net::UdpSocket::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap();
cache.add_address_hint(dead, nowhere).await.unwrap();
assert!(cache.connect(dead, &alpn).await.is_err());
let refused = cache.connect(dead, &alpn).await.unwrap_err();
assert!(
refused.to_string().contains("recently unreachable"),
"{refused}"
);
assert_eq!(cache.budget.control_dials_available(), 16);
cache.add_address_hint(dead, nowhere).await.unwrap();
let redialed = cache.connect(dead, &alpn).await.unwrap_err();
assert!(
!redialed.to_string().contains("recently unreachable"),
"{redialed}"
);
cache.close().await;
})
.await
.unwrap();
}
#[test]
fn unreachable_backoff_doubles_from_five_seconds_to_a_two_minute_cap() {
let seconds = |failures| unreachable_backoff(failures).as_secs();
assert_eq!(
[seconds(1), seconds(2), seconds(3), seconds(4), seconds(5)],
[5, 10, 20, 40, 80]
);
assert_eq!(seconds(6), 120);
assert_eq!(seconds(60), 120);
assert_eq!(seconds(u32::MAX), 120);
}
#[tokio::test]
async fn a_network_change_ends_every_backoff() {
tokio::time::timeout(std::time::Duration::from_secs(30), async {
let alpn = super::super::control::ALPN.to_vec();
let cache = Connections::bind(
"127.0.0.1:0".parse().unwrap(),
NetworkProfile::Direct,
None,
ConnectionBudget::default(),
vec![],
None,
)
.await
.unwrap();
let dead = *iroh::SecretKey::generate().public().as_bytes();
let nowhere = std::net::UdpSocket::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap();
cache.add_address_hint(dead, nowhere).await.unwrap();
assert!(cache.connect(dead, &alpn).await.is_err());
assert!(
cache
.connect(dead, &alpn)
.await
.unwrap_err()
.to_string()
.contains("recently unreachable")
);
cache.network_change().await;
let redialed = cache.connect(dead, &alpn).await.unwrap_err();
assert!(
!redialed.to_string().contains("recently unreachable"),
"{redialed}"
);
cache.close().await;
})
.await
.unwrap();
}
#[tokio::test]
async fn a_discarded_connection_is_replaced_on_the_next_dial() {
tokio::time::timeout(std::time::Duration::from_secs(20), async {
let alpn = super::super::control::ALPN.to_vec();
let peer = Endpoint::builder(presets::Minimal)
.clear_relay_transports()
.bind_addr("127.0.0.1:0")
.unwrap()
.alpns(vec![alpn.clone()])
.bind()
.await
.unwrap();
let endpoint = peer.clone();
let server = tokio::spawn(async move {
while let Some(incoming) = endpoint.accept().await {
tokio::spawn(async move {
if let Ok(connection) = incoming.await {
connection.closed().await;
}
});
}
});
let cache = Connections::bind(
"127.0.0.1:0".parse().unwrap(),
NetworkProfile::Direct,
None,
ConnectionBudget::default(),
vec![],
None,
)
.await
.unwrap();
let id = *peer.id().as_bytes();
cache
.add_address_hint(id, peer.bound_sockets()[0])
.await
.unwrap();
let first = cache.connect(id, &alpn).await.unwrap();
let again = cache.connect(id, &alpn).await.unwrap();
assert_eq!(
again.stable_id(),
first.stable_id(),
"a healthy connection is reused"
);
cache.discard(id, &alpn).await;
assert!(
first.close_reason().is_some(),
"the discarded connection is closed"
);
let fresh = cache.connect(id, &alpn).await.unwrap();
assert_ne!(
fresh.stable_id(),
first.stable_id(),
"the next dial is a new connection"
);
cache.close().await;
peer.close().await;
server.abort();
})
.await
.unwrap();
}
#[tokio::test]
async fn a_stale_address_hint_falls_back_to_address_lookup() {
tokio::time::timeout(std::time::Duration::from_secs(30), async {
let alpn = super::super::control::ALPN.to_vec();
let peer = Endpoint::builder(presets::Minimal)
.clear_relay_transports()
.bind_addr("127.0.0.1:0")
.unwrap()
.alpns(vec![alpn.clone()])
.bind()
.await
.unwrap();
let endpoint = peer.clone();
let server = tokio::spawn(async move {
while let Some(incoming) = endpoint.accept().await {
tokio::spawn(async move {
if let Ok(connection) = incoming.await {
connection.closed().await;
}
});
}
});
let stale = std::net::UdpSocket::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap();
let mut cache = Connections::bind(
"127.0.0.1:0".parse().unwrap(),
NetworkProfile::Direct,
None,
ConnectionBudget::default(),
vec![],
None,
)
.await
.unwrap();
cache.address_lookup = true;
let id = *peer.id().as_bytes();
cache.add_address_hint(id, stale).await.unwrap();
cache.memory.add_endpoint_info(
EndpointAddr::new(peer.id()).with_ip_addr(peer.bound_sockets()[0]),
);
let connection = cache.connect(id, &alpn).await.unwrap();
assert_eq!(connection.alpn(), alpn.as_slice());
cache.close().await;
peer.close().await;
server.abort();
})
.await
.unwrap();
}
}