dig_nat/relay.rs
1//! Relay client — the LAST-RESORT transport + the node's persistent reachability channel.
2//!
3//! Relocated + generalized from `dig-node`'s `relay.rs`. Two responsibilities:
4//!
5//! 1. **Persistent reservation** ([`run_relay_connection`]) — a DIG Node behind NAT can't accept
6//! inbound dials, so it holds a CONSTANT registered connection with a publicly-reachable relay
7//! (default [`dig_constants::DIG_RELAY_URL`], override `DIG_RELAY_URL`, opt out with
8//! `DIG_RELAY_URL=off`). This is the reachability channel other peers reach it through and the
9//! rendezvous for relay-coordinated hole-punch.
10//! 2. **Relayed transport** — when every NAT-traversal method fails, peer traffic is tunnelled
11//! THROUGH the relay (RLY-002 `relay_message`). This is the last resort in the traversal order.
12//!
13//! **Graceful-fallback guarantees (baked in):** the reservation loop NEVER blocks startup, NEVER
14//! panics/exits, and NEVER hot-loops error-spam — failures log ONCE per state change (a transition
15//! into `Disconnected`), and every retry sleeps a bounded, capped-exponential backoff. If the relay
16//! is unreachable the node keeps serving indefinitely; the task just keeps retrying in the
17//! background. State is published through [`RelayStatus`] (a cheap atomic snapshot) as one of four
18//! [`RelayState`]s and surfaced verbatim to a `control.relayStatus`-style RPC / `/health`.
19
20use std::collections::{HashMap, HashSet};
21use std::net::{IpAddr, SocketAddr};
22use std::sync::atomic::{AtomicU32, AtomicU64, AtomicU8, Ordering};
23use std::sync::{Arc, Mutex};
24use std::time::Duration;
25
26use dig_ip::{CandidateSource, DialConfig, LocalStack, PeerCandidates};
27use futures_util::{SinkExt, StreamExt};
28use tokio::net::TcpStream;
29use tokio::sync::mpsc;
30use tokio_tungstenite::tungstenite::Message;
31use tokio_tungstenite::{client_async_tls_with_config, MaybeTlsStream, WebSocketStream};
32
33use crate::wire::{RelayMessage, RelayPeerInfo};
34
35/// Default network id a node registers under (matches dig-gossip `DEFAULT_INTRODUCER_NETWORK_ID`
36/// and dig-node's `DEFAULT_NETWORK_ID`).
37pub const DEFAULT_NETWORK_ID: &str = "DIG_MAINNET";
38
39/// Relay protocol version the node advertises in `Register` (RLY-001).
40pub const RELAY_PROTOCOL_VERSION: u32 = 1;
41
42/// Base reconnect delay (dig-gossip `RelayConfig::reconnect_delay_secs` = 5).
43const BASE_BACKOFF_SECS: u64 = 5;
44/// Cap on the exponential backoff so a long outage doesn't push the retry interval to hours.
45const MAX_BACKOFF_SECS: u64 = 300;
46/// Keepalive ping period (RLY-006; dig-gossip `PING_INTERVAL_SECS` = 30).
47const PING_INTERVAL_SECS: u64 = 30;
48/// How often the held reservation re-pulls the relay peer list (RLY-005 `GetPeers`) over the SAME
49/// persistent socket, so a peer that registers AFTER this node — or one missed on the first pull —
50/// is still discovered without ever reopening the connection (the connect-leg fix).
51const DISCOVERY_INTERVAL_SECS: u64 = 60;
52
53/// Hard cap on the peers retained in the discovered set ([`RelayStatus::known_peers`]).
54///
55/// SECURITY: the relay is an UNTRUSTED intermediary. A hostile/compromised relay can stream an
56/// unbounded flood of `PeerConnected` frames — or a single oversized `Peers` frame — with distinct
57/// fabricated `peer_id`s, so an uncapped set is a memory-exhaustion DoS. 1024 is far more than any
58/// honest relay reports for one network's live reservations (the set is folded into a peer pool that
59/// itself selects a small working subset), yet small enough that the worst case is bounded, cheap
60/// memory. Beyond the cap, further distinct peers are DROPPED rather than grown.
61pub const MAX_KNOWN_PEERS: usize = 1024;
62
63/// Hard cap on the byte length of a single RLY-002 relayed-transport payload (both directions).
64///
65/// SECURITY / backpressure: the relay is UNTRUSTED and a peer reached over relayed transport is the
66/// last-resort TURN path, so an oversized frame is refused rather than buffered — an outbound `send`
67/// larger than this errors, and an inbound frame larger than this is dropped. 1 MiB comfortably
68/// holds a sealed gossip message (NC-1 ciphertext) while bounding the worst-case per-frame memory.
69pub const MAX_RELAY_PAYLOAD: usize = 1 << 20;
70
71/// Bounded inbound capacity for one open [`RelayTunnel`]. A full channel applies backpressure — the
72/// reservation loop `try_send`s inbound relayed bytes and DROPS the frame when the consumer is not
73/// keeping up, so a hostile relay flooding one tunnel cannot exhaust memory (matches the
74/// [`MAX_KNOWN_PEERS`] bounded-set philosophy). The RLY-002 `seq` lets the consumer detect the gap.
75const RELAY_TUNNEL_INBOUND_CAP: usize = 256;
76
77/// Upper bound on concurrently-registered relay tunnels (outbound dial + inbound accept combined)
78/// before the RESPONDER path refuses to create a new inbound circuit.
79///
80/// SECURITY: the relay is UNTRUSTED. When the accept path ([`RelayStatus::enable_accept`]) is on, an
81/// inbound RLY-002 frame from an unknown peer creates a server-role tunnel + surfaces an accept — so
82/// an uncapped accept lets a hostile relay flood distinct fabricated `from` ids to spawn unbounded
83/// tunnels/accept-tasks (a memory/task-exhaustion DoS). Beyond this cap the introduced circuit is
84/// DROPPED rather than accepted. 256 is far more concurrent relayed peers than the last-resort tier
85/// ever legitimately carries, yet bounds the worst case to cheap, bounded memory.
86pub const MAX_RELAY_TUNNELS: usize = 256;
87
88/// Bounded capacity of the inbound-accept channel ([`RelayStatus::enable_accept`]). A full channel
89/// means the consumer is not accepting introduced circuits fast enough; the newest is dropped
90/// (bounded backpressure), never queued unboundedly.
91const INBOUND_ACCEPT_CAP: usize = 64;
92
93/// The mTLS role a locally-registered [`RelayTunnel`] runs — the discriminator that resolves the
94/// GLARE / simultaneous-mutual-dial case (#1536). A relay circuit needs exactly ONE mTLS client + ONE
95/// mTLS server; when two NAT'd peers fall to the relay tier and dial EACH OTHER at the same time (the
96/// common two-NAT'd-peer flywheel case), both open a `Client` tunnel and both send a ClientHello —
97/// each ClientHello would route into the OTHER side's client session (double-ClientHello deadlock).
98/// Tagging the tunnel with its role lets [`route_relayed`](RelayStatus::route_relayed) detect that a
99/// ClientHello arrived on a tunnel where WE are also the client (the glare signature) and apply the
100/// deterministic tie-break instead of feeding it to our doomed client.
101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
102enum TunnelRole {
103 /// WE initiated the dial — running [`PeerSession::client`](crate::mux::PeerSession::client) over
104 /// this tunnel; inbound frames are the peer's ServerHello / server-side records.
105 Client,
106 /// WE accepted an introduced circuit — running [`PeerSession::server`](crate::mux::PeerSession::server)
107 /// over this tunnel; inbound frames are the dialing peer's client-side records.
108 Server,
109}
110
111/// A registered relayed tunnel: the inbound sink frames are routed into, the mTLS [`TunnelRole`] we
112/// run over it (for the #1536 glare tie-break), and a monotonic `id` distinguishing this registration
113/// from a later one on the SAME peer key. The id matters when the glare tie-break REPLACES our client
114/// tunnel with a server tunnel under the same `from` key: the old client [`RelayTunnel`]'s `Drop`
115/// (fired when its doomed dial fails) must not deregister the NEW server entry, so `close_tunnel` only
116/// removes when the stored id still matches.
117#[derive(Debug)]
118struct TunnelEntry {
119 sink: mpsc::Sender<Vec<u8>>,
120 role: TunnelRole,
121 id: u64,
122}
123
124/// Whether `payload` begins with a TLS handshake record whose first message is a ClientHello — a TLS
125/// record has content-type `0x16` (handshake) at byte 0 and the handshake message type at byte 5
126/// (`0x01` = ClientHello, `0x02` = ServerHello). A rustls client ships its ClientHello flight as the
127/// first `poll_write`, so the first relayed frame from a fresh dialer matches this.
128///
129/// This is the relay layer's DIRECTION discriminator, and it decides the mTLS role in both directions:
130/// a ClientHello means the remote is the circuit's client, so we are its server; anything else is not
131/// the start of an inbound circuit at all. [`RelayStatus::route_relayed`] uses it to tell a peer's
132/// GLARE ClientHello (a competing simultaneous dial) from the ServerHello / app records expected where
133/// we are the client (#1536), and [`RelayStatus::accept_introduced`] uses it to refuse to manufacture
134/// a server-role circuit out of a frame no dialer sent (#1761).
135fn is_tls_client_hello(payload: &[u8]) -> bool {
136 payload.len() >= 6 && payload[0] == 0x16 && payload[5] == 0x01
137}
138
139/// Relay error codes that invalidate THIS node's own reservation, mirroring `dig-relay`'s
140/// `errcode` catalogue. Kept as an explicit list so the two stay conformance-checkable.
141///
142/// - `1 NOT_REGISTERED` — the relay does not consider us registered, so the reservation is gone.
143/// - `4 CAPACITY`, `5 ID_IN_USE`, `6 IDENTITY_MISMATCH`, `7 RATE_LIMITED` — each accompanies a
144/// FAILING `register_ack`: the registration did not happen.
145const FATAL_RELAY_ERROR_CODES: [u32; 5] = [1, 4, 5, 6, 7];
146
147/// Whether a relay `Error` frame means the RESERVATION is invalid (end the loop, back off and
148/// re-register) rather than a single request having failed (log it; keep serving).
149///
150/// The relay reports both kinds on one channel, and the per-request kind is the COMMON one:
151/// `3 PEER_NOT_FOUND` simply means the peer we tried to reach is no longer on this relay, which
152/// happens constantly on a live network and says nothing about our own registration. Treating it as
153/// fatal cost the node its reservation on every failed dial (dig_ecosystem #1932).
154///
155/// An UNKNOWN (future) code is deliberately treated as NON-fatal. Guessing wrong in that direction
156/// costs one logged line; guessing wrong the other way would let a newly-introduced code drop every
157/// node on the network off its relay at once.
158pub fn relay_error_is_fatal(code: u32) -> bool {
159 FATAL_RELAY_ERROR_CODES.contains(&code)
160}
161
162/// Compute the next reconnect backoff: capped exponential in the number of consecutive failures.
163/// `failures == 0` → base; doubles each failure up to [`MAX_BACKOFF_SECS`]. Pure → unit-tested.
164pub fn backoff_secs(consecutive_failures: u32) -> u64 {
165 backoff_secs_with(consecutive_failures, BASE_BACKOFF_SECS, MAX_BACKOFF_SECS)
166}
167
168/// Capped-exponential backoff with an explicit base + cap. Always returns a value in `[base, cap]`
169/// — never zero — so a failing connect can never busy-loop.
170fn backoff_secs_with(consecutive_failures: u32, base: u64, cap: u64) -> u64 {
171 let shifted = base.checked_shl(consecutive_failures).unwrap_or(cap);
172 shifted.clamp(base, cap)
173}
174
175/// Backoff schedule for the reconnect loop — production defaults, or fast values for tests.
176#[derive(Debug, Clone, Copy)]
177pub struct Backoff {
178 /// First-retry delay (seconds).
179 pub base_secs: u64,
180 /// Upper bound on the delay (seconds).
181 pub cap_secs: u64,
182}
183
184impl Default for Backoff {
185 fn default() -> Self {
186 Backoff {
187 base_secs: BASE_BACKOFF_SECS,
188 cap_secs: MAX_BACKOFF_SECS,
189 }
190 }
191}
192
193/// The four observable states of the relay reservation, surfaced verbatim (lowercase) as the
194/// `state` field of a `control.relayStatus`-style RPC.
195///
196/// - `Disabled` — reservation OFF (`DIG_RELAY_URL=off`); no task runs, no attempts made.
197/// - `Connecting` — actively dialing/registering.
198/// - `Connected` — a reservation is held (`RegisterAck{success:true}` arrived); reachable to peers.
199/// - `Disconnected` — not connected; backing off + will retry. The graceful-fallback resting state.
200#[derive(Debug, Clone, Copy, PartialEq, Eq)]
201pub enum RelayState {
202 /// Reservation OFF (`DIG_RELAY_URL=off`); no task runs, no attempts made.
203 Disabled,
204 /// Actively dialing/registering (initial attempt or a reconnect in flight).
205 Connecting,
206 /// A reservation is held (`RegisterAck{success:true}` arrived); reachable to NAT'd peers.
207 Connected,
208 /// Not connected; backing off + will retry. The graceful-fallback resting state.
209 Disconnected,
210}
211
212impl RelayState {
213 /// The stable lowercase wire string for the RPC `state` field.
214 pub fn as_str(self) -> &'static str {
215 match self {
216 RelayState::Disabled => "disabled",
217 RelayState::Connecting => "connecting",
218 RelayState::Connected => "connected",
219 RelayState::Disconnected => "disconnected",
220 }
221 }
222
223 fn to_u8(self) -> u8 {
224 match self {
225 RelayState::Disabled => 0,
226 RelayState::Connecting => 1,
227 RelayState::Connected => 2,
228 RelayState::Disconnected => 3,
229 }
230 }
231
232 fn from_u8(v: u8) -> Self {
233 match v {
234 0 => RelayState::Disabled,
235 1 => RelayState::Connecting,
236 2 => RelayState::Connected,
237 _ => RelayState::Disconnected,
238 }
239 }
240}
241
242/// The peers discovered over the live reservation socket, in insertion order with O(1) dedup +
243/// membership by `peer_id`, bounded to [`MAX_KNOWN_PEERS`].
244///
245/// `order` preserves discovery order so [`RelayStatus::known_peers`] returns a stable sequence;
246/// `ids` mirrors `order`'s `peer_id`s so dedup and removal are O(1) instead of a linear scan (the
247/// old `iter().any(...)` was O(n²) over a flood). The two are kept in lockstep — every mutation
248/// touches both.
249#[derive(Debug, Default)]
250struct DiscoveredPeers {
251 order: Vec<RelayPeerInfo>,
252 ids: HashSet<String>,
253}
254
255impl DiscoveredPeers {
256 /// Insert `peer` unless already present or the set is full. Returns nothing — a full set simply
257 /// drops the newcomer (the untrusted-relay flood defense).
258 fn insert(&mut self, peer: RelayPeerInfo) {
259 if self.order.len() >= MAX_KNOWN_PEERS {
260 return;
261 }
262 if self.ids.insert(peer.peer_id.clone()) {
263 self.order.push(peer);
264 }
265 }
266
267 /// Remove the peer with this `peer_id`, if present.
268 fn remove(&mut self, peer_id: &str) {
269 if self.ids.remove(peer_id) {
270 self.order.retain(|p| p.peer_id != peer_id);
271 }
272 }
273
274 /// Replace the whole set from a `Peers` frame, deduped + truncated to the cap.
275 fn replace(&mut self, peers: Vec<RelayPeerInfo>) {
276 self.order.clear();
277 self.ids.clear();
278 for peer in peers {
279 self.insert(peer);
280 }
281 }
282
283 fn clear(&mut self) {
284 self.order.clear();
285 self.ids.clear();
286 }
287}
288
289/// Live relay-connection status, shared (via `Arc`) between the connection task and an RPC handler.
290/// Cheap atomic reads. State setters do STATE-CHANGE-ONLY logging so a long outage never hot-loops
291/// identical error lines.
292#[derive(Debug)]
293pub struct RelayStatus {
294 state: AtomicU8,
295 reconnect_attempts: AtomicU32,
296 connected_peers: AtomicU64,
297 last_error: Mutex<Option<String>>,
298 /// Peers learned over the LIVE reservation socket — the relay's `GetPeers` response (RLY-005)
299 /// plus `PeerConnected`/`PeerDisconnected` pushes. This is the discovery output of the persistent
300 /// reservation: a consumer (dig-gossip's pool/address book) reads it instead of reopening an
301 /// ephemeral socket per pass. Keyed by `peer_id` (deduped); bounded to [`MAX_KNOWN_PEERS`] so an
302 /// untrusted relay can't exhaust memory; cleared on every reconnect so a stale list is never
303 /// served across a drop.
304 known_peers: Mutex<DiscoveredPeers>,
305 /// Sink that injects an outbound [`RelayMessage`] into the LIVE reservation socket's write half.
306 /// `Some` only while a session is held (set by `connect_once`, cleared on every drop) — this is
307 /// what lets a [`RelayTunnel`] reuse the ONE persistent reservation socket for RLY-002 relayed
308 /// transport instead of opening a second connection.
309 outbound: Mutex<Option<mpsc::UnboundedSender<RelayMessage>>>,
310 /// This node's own `peer_id` (hex), stamped as `from` on every RLY-002 frame the tunnels send.
311 /// Set when a session registers; needed because a tunnel is opened from the shared status handle.
312 local_peer_id: Mutex<Option<String>>,
313 /// The network id this reservation registered under. Echoed onto an inbound accepted tunnel (the
314 /// RLY-002 frame itself does not carry it). Set alongside [`local_peer_id`] when a session registers.
315 local_network_id: Mutex<Option<String>>,
316 /// Sink that surfaces an INTRODUCED inbound circuit — a frame from a peer with NO open outbound
317 /// tunnel — as a server-role [`RelayTunnel`] for a consumer to accept + serve
318 /// ([`crate::accept::RelayAcceptor`]). `None` (default) = the original untrusted-relay behavior:
319 /// drop an unknown-peer frame. `Some` once a consumer calls [`RelayStatus::enable_accept`].
320 ///
321 /// This is the RESPONDER counterpart to [`open_tunnel`](Self::open_tunnel): a relay circuit needs
322 /// exactly ONE mTLS client + ONE mTLS server. The DIALER calls `open_tunnel` and runs
323 /// `PeerSession::client`; the reservation-HOLDER that RECEIVES the introduced circuit accepts here
324 /// and runs `PeerSession::server`. Without this path both ends acted as TLS client and the
325 /// handshake deadlocked (`got ClientHello when expecting ServerHello`, #1536).
326 inbound_accept: Mutex<Option<mpsc::Sender<RelayTunnel>>>,
327 /// Open relayed-transport tunnels, keyed by the REMOTE peer's `peer_id` (hex). An inbound RLY-002
328 /// `relay_message` from a peer is routed to its tunnel's inbound channel; a frame from a peer with
329 /// no open tunnel is dropped (the untrusted-relay default). Entries are removed on tunnel drop.
330 /// Each entry carries the mTLS [`TunnelRole`] we run over it so `route_relayed` can resolve the
331 /// #1536 simultaneous-mutual-dial glare deterministically.
332 tunnels: Mutex<HashMap<String, TunnelEntry>>,
333 /// Monotonic per-node sequence number stamped on outbound RLY-002 frames (ordering/dedup).
334 relay_seq: AtomicU64,
335 /// Monotonic id assigned to each tunnel registration so a stale [`RelayTunnel`]'s `Drop` never
336 /// deregisters a NEWER entry under the same peer key (see [`TunnelEntry::id`]; #1536 glare replace).
337 next_tunnel_id: AtomicU64,
338}
339
340impl Default for RelayStatus {
341 fn default() -> Self {
342 RelayStatus {
343 state: AtomicU8::new(RelayState::Disconnected.to_u8()),
344 reconnect_attempts: AtomicU32::new(0),
345 connected_peers: AtomicU64::new(0),
346 last_error: Mutex::new(None),
347 known_peers: Mutex::new(DiscoveredPeers::default()),
348 outbound: Mutex::new(None),
349 local_peer_id: Mutex::new(None),
350 local_network_id: Mutex::new(None),
351 inbound_accept: Mutex::new(None),
352 tunnels: Mutex::new(HashMap::new()),
353 relay_seq: AtomicU64::new(0),
354 next_tunnel_id: AtomicU64::new(0),
355 }
356 }
357}
358
359impl RelayStatus {
360 /// A fresh status (resting `Disconnected` until the task runs / the relay is reached).
361 pub fn new() -> Arc<Self> {
362 Arc::new(RelayStatus::default())
363 }
364
365 /// Read the current state.
366 pub fn state(&self) -> RelayState {
367 RelayState::from_u8(self.state.load(Ordering::Relaxed))
368 }
369
370 /// Transition to `next`, returning `true` IFF the state actually changed. Callers use the return
371 /// to log ONCE per transition (no hot-loop spam).
372 fn transition_to(&self, next: RelayState) -> bool {
373 let prev = self.state.swap(next.to_u8(), Ordering::Relaxed);
374 prev != next.to_u8()
375 }
376
377 /// Enter `Disabled` (reservation off). Idempotent; logs only on the first entry.
378 pub fn set_disabled(&self) {
379 if self.transition_to(RelayState::Disabled) {
380 tracing::info!("relay reservation disabled (DIG_RELAY_URL=off)");
381 }
382 }
383
384 /// Enter `Connecting`. Logs only on the transition (so reconnect attempts don't spam).
385 pub fn set_connecting(&self) {
386 if self.transition_to(RelayState::Connecting) {
387 tracing::debug!("relay connecting");
388 }
389 }
390
391 /// Mark `Connected` (clears the last error, resets the attempt counter). Logs recovery once.
392 pub fn set_connected(&self, connected_peers: u64) {
393 self.connected_peers
394 .store(connected_peers, Ordering::Relaxed);
395 self.reconnect_attempts.store(0, Ordering::Relaxed);
396 *self.last_error.lock().unwrap() = None;
397 if self.transition_to(RelayState::Connected) {
398 tracing::info!(connected_peers, "relay reservation established");
399 }
400 }
401
402 /// Mark `Disconnected` with an optional error and bump the attempt counter. Logs the failure
403 /// ONLY on the transition into `Disconnected` (the first drop); subsequent failed retries while
404 /// already `Disconnected` update the error/counter SILENTLY.
405 pub fn set_disconnected(&self, error: Option<String>) {
406 self.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
407 if let Some(e) = &error {
408 *self.last_error.lock().unwrap() = Some(e.clone());
409 }
410 let changed = self.transition_to(RelayState::Disconnected);
411 if changed {
412 match &error {
413 Some(e) => tracing::warn!(
414 error = %e,
415 "relay reservation lost — node still serving; retrying in background"
416 ),
417 None => tracing::info!("relay reservation closed — retrying in background"),
418 }
419 }
420 }
421
422 /// Whether a relay session is currently held.
423 pub fn is_connected(&self) -> bool {
424 self.state() == RelayState::Connected
425 }
426
427 /// The current reconnect-attempt count (for tests / RPC).
428 pub fn reconnect_attempts(&self) -> u32 {
429 self.reconnect_attempts.load(Ordering::Relaxed)
430 }
431
432 /// Snapshot of the peers discovered over the live reservation socket (RLY-005 `Peers` +
433 /// `PeerConnected` pushes, minus `PeerDisconnected`). The consumer folds these into its address
434 /// book / pool. Returns a clone so the caller holds no lock.
435 pub fn known_peers(&self) -> Vec<RelayPeerInfo> {
436 self.known_peers.lock().unwrap().order.clone()
437 }
438
439 /// Count of peers currently discovered over the live reservation socket.
440 pub fn known_peer_count(&self) -> usize {
441 self.known_peers.lock().unwrap().order.len()
442 }
443
444 /// Replace the discovered-peer set with a `GetPeers` response (RLY-005 `Peers`), deduped and
445 /// truncated to [`MAX_KNOWN_PEERS`] (an untrusted relay could send an oversized frame).
446 fn replace_known_peers(&self, peers: Vec<RelayPeerInfo>) {
447 self.known_peers.lock().unwrap().replace(peers);
448 }
449
450 /// Fold in a relay-pushed `PeerConnected` notice, deduped by `peer_id`; dropped once the set is
451 /// full ([`MAX_KNOWN_PEERS`]) so a flood can't exhaust memory.
452 fn add_known_peer(&self, peer: RelayPeerInfo) {
453 self.known_peers.lock().unwrap().insert(peer);
454 }
455
456 /// Drop a peer on a relay-pushed `PeerDisconnected` notice.
457 fn remove_known_peer(&self, peer_id: &str) {
458 self.known_peers.lock().unwrap().remove(peer_id);
459 }
460
461 /// Clear the discovered-peer set (on every reconnect — the list is per-session).
462 fn clear_known_peers(&self) {
463 self.known_peers.lock().unwrap().clear();
464 }
465
466 // -- RLY-002 relayed transport (the tier-6 TURN fallback) ------------------------------------
467 //
468 // A relayed tunnel reuses the ONE persistent reservation socket: outbound frames go through
469 // `outbound` (drained by the reservation loop's write half), inbound `relay_message` frames are
470 // routed by `from` peer_id to the matching tunnel. Available only while the reservation is held.
471
472 /// Install the live session's outbound sink + this node's `peer_id` + the registered `network_id`.
473 /// Called by `connect_once` once registered; cleared by [`clear_transport`](Self::clear_transport)
474 /// on every drop.
475 fn set_transport(
476 &self,
477 peer_id: &str,
478 network_id: &str,
479 outbound: mpsc::UnboundedSender<RelayMessage>,
480 ) {
481 *self.local_peer_id.lock().unwrap() = Some(peer_id.to_string());
482 *self.local_network_id.lock().unwrap() = Some(network_id.to_string());
483 *self.outbound.lock().unwrap() = Some(outbound);
484 }
485
486 /// Enable the RESPONDER (accept) path and return the receiver of INTRODUCED inbound circuits.
487 ///
488 /// A relayed connection needs one mTLS client + one mTLS server. The dialer opens a tunnel and
489 /// runs the client; the reservation-HOLDER calls this once at startup and, for every inbound
490 /// frame from a peer it has no open outbound tunnel to, receives a server-role [`RelayTunnel`]
491 /// here to hand to a [`crate::accept::RelayAcceptor`] (which runs `PeerSession::server`). Until a
492 /// consumer calls this, unknown-peer frames are DROPPED (the untrusted-relay default), so the
493 /// accept path is strictly opt-in. The channel is bounded ([`INBOUND_ACCEPT_CAP`]).
494 pub fn enable_accept(&self) -> mpsc::Receiver<RelayTunnel> {
495 let (tx, rx) = mpsc::channel(INBOUND_ACCEPT_CAP);
496 *self.inbound_accept.lock().unwrap() = Some(tx);
497 rx
498 }
499
500 /// Tear down the transport on session drop: drop the outbound sink (so tunnel sends fail fast)
501 /// and close every open tunnel's inbound channel (so a blocked `recv` wakes with `None`).
502 fn clear_transport(&self) {
503 *self.outbound.lock().unwrap() = None;
504 self.tunnels.lock().unwrap().clear();
505 }
506
507 /// Whether a relayed tunnel can currently be opened — a reservation is held AND its outbound sink
508 /// is live. The tier-6 [`RelayedTransport`](crate::method::relayed::RelayedTransport) gates on this.
509 pub fn relay_transport_ready(&self) -> bool {
510 self.is_connected() && self.outbound.lock().unwrap().is_some()
511 }
512
513 /// Open an RLY-002 relayed-transport tunnel to `target_peer` (hex `peer_id`) over the held
514 /// reservation socket — the traversal ladder's FINAL tier when a pair can neither direct-dial nor
515 /// hole-punch. The returned [`RelayTunnel`] sends/receives opaque payloads that the relay forwards
516 /// A→relay→B; per NC-1 the payload is END-TO-END SEALED to the recipient so the relay forwards
517 /// ciphertext only. `Err` if no reservation is held. Dropping the tunnel deregisters it.
518 pub fn open_tunnel(
519 self: &Arc<Self>,
520 target_peer: &str,
521 network_id: &str,
522 ) -> Result<RelayTunnel, String> {
523 if !self.relay_transport_ready() {
524 return Err("relay reservation not connected — cannot open relayed tunnel".into());
525 }
526 // Self / SPKI-collision guard: a relayed circuit to our OWN peer_id has no lower/higher end
527 // for the glare tie-break, so it could never converge to one-client-one-server — refuse it
528 // (#1536).
529 let local = self
530 .local_peer_id
531 .lock()
532 .unwrap()
533 .clone()
534 .unwrap_or_default();
535 if !local.is_empty() && local == target_peer {
536 return Err("refusing relayed self-dial (target == local peer_id)".into());
537 }
538 // NON-CLOBBER (#1536): if a circuit to this peer already exists — because an introduced
539 // ClientHello made us its SERVER before this dial ran (a timing-ordered glare) — do NOT open a
540 // second, conflicting circuit under the same key. The existing circuit IS the connection; a
541 // duplicate would orphan one role and leave two mTLS sessions racing to one peer. Checked +
542 // inserted under ONE lock so a concurrent `route_relayed` cannot slip a role in between.
543 let mut tunnels = self.tunnels.lock().unwrap();
544 if tunnels.contains_key(target_peer) {
545 return Err("existing relay circuit to peer — not opening a duplicate".into());
546 }
547 // A dialer runs the mTLS client over the tunnel it opens.
548 Ok(self.insert_entry(&mut tunnels, target_peer, network_id, TunnelRole::Client))
549 }
550
551 /// Register a tunnel routing entry for `target_peer` and build its [`RelayTunnel`], overwriting any
552 /// existing entry under the key. Test-only (the `open_server_tunnel` + flood-cap tests use it); the
553 /// production paths ([`open_tunnel`](Self::open_tunnel) + [`accept_introduced`](Self::accept_introduced))
554 /// go through non-clobber checks first.
555 #[cfg(test)]
556 fn register_tunnel(
557 self: &Arc<Self>,
558 target_peer: &str,
559 network_id: &str,
560 role: TunnelRole,
561 ) -> RelayTunnel {
562 let mut tunnels = self.tunnels.lock().unwrap();
563 self.insert_entry(&mut tunnels, target_peer, network_id, role)
564 }
565
566 /// Build a fresh [`RelayTunnel`] for `target_peer` with `role` and insert its entry into the
567 /// already-locked `tunnels` map (assigning a monotonic id so a stale `Drop` never evicts a newer
568 /// registration). The caller holds the lock, so the check-then-insert is atomic — the #1536
569 /// non-clobber + role-race defense.
570 fn insert_entry(
571 self: &Arc<Self>,
572 tunnels: &mut HashMap<String, TunnelEntry>,
573 target_peer: &str,
574 network_id: &str,
575 role: TunnelRole,
576 ) -> RelayTunnel {
577 let (tx, rx) = mpsc::channel(RELAY_TUNNEL_INBOUND_CAP);
578 let id = self.next_tunnel_id.fetch_add(1, Ordering::Relaxed);
579 tunnels.insert(target_peer.to_string(), TunnelEntry { sink: tx, role, id });
580 RelayTunnel {
581 target: target_peer.to_string(),
582 network_id: network_id.to_string(),
583 status: Arc::clone(self),
584 inbound: rx,
585 id,
586 }
587 }
588
589 /// Route one inbound RLY-002 `relay_message` to its tunnel by `from` peer_id. Oversized payloads
590 /// are dropped (size cap); a frame from a peer with no open tunnel becomes an introduced circuit
591 /// ONLY when it is a dialer's opening ClientHello (#1761 — see
592 /// [`accept_introduced`](Self::accept_introduced)), and is otherwise dropped, as it is when the
593 /// responder path is off or a flood cap is hit; a full inbound channel drops the frame
594 /// (backpressure). Returns silently in every drop case (untrusted relay).
595 ///
596 /// GLARE (#1536): when a ClientHello arrives on a tunnel where WE are ALSO the client — the peer
597 /// dialed us at the same time we dialed it — a deterministic tie-break makes exactly ONE side the
598 /// server: the numerically-LOWER `peer_id` becomes the server. Both ends compute the same rule, so
599 /// a crossed pair converges to one-client-one-server under ANY frame ordering with no retry loop.
600 /// The TIMING-ordered variant (a peer's ClientHello arrives BEFORE our own dial registers) cannot
601 /// produce a conflicting second circuit either: [`open_tunnel`](Self::open_tunnel) is non-clobber,
602 /// so once we serve a peer our later dial to it is refused. The per-frame role LOOKUP + any
603 /// same-frame yield (client-tunnel removal) happen under a single lock acquisition; the server
604 /// registration in [`accept_introduced`](Self::accept_introduced) re-acquires and re-checks
605 /// (non-clobber), so a dial racing between the two regions is abandoned, never a double-session.
606 fn route_relayed(self: &Arc<Self>, from: &str, payload: Vec<u8>) {
607 if payload.len() > MAX_RELAY_PAYLOAD {
608 tracing::debug!(
609 from,
610 len = payload.len(),
611 "dropping oversized relayed frame"
612 );
613 return;
614 }
615 let local = self
616 .local_peer_id
617 .lock()
618 .unwrap()
619 .clone()
620 .unwrap_or_default();
621 // Self / SPKI-collision guard: a frame stamped with our OWN id can never be a real remote peer
622 // (and the tie-break has no lower/higher end for it) — drop it rather than risk a no-server
623 // hang (#1536).
624 if !local.is_empty() && local == from {
625 tracing::debug!("dropping relayed frame stamped with our own peer_id (self/collision)");
626 return;
627 }
628
629 // Decide the action for this frame under ONE tunnels-lock so a concurrent `open_tunnel` cannot
630 // race the role assignment.
631 enum Route {
632 /// Deliver into an existing tunnel's inbound sink.
633 Deliver(mpsc::Sender<Vec<u8>>),
634 /// Drop the frame (we retain our client role, or cannot serve).
635 Ignore,
636 /// Accept as an introduced server-role circuit.
637 Accept,
638 }
639 let route = {
640 let mut tunnels = self.tunnels.lock().unwrap();
641 match tunnels.get(from).map(|e| (e.sink.clone(), e.role, e.id)) {
642 // We are the SERVER for this peer — every frame is the client's; route it.
643 Some((sink, TunnelRole::Server, _)) => Route::Deliver(sink),
644 // We are the CLIENT. A ServerHello / app record is the expected response; a ClientHello
645 // means the peer dialed us at the same time (GLARE).
646 Some((sink, TunnelRole::Client, id)) => {
647 if !is_tls_client_hello(&payload) {
648 Route::Deliver(sink)
649 } else if local.as_str() > from {
650 // Higher id → WE keep the client role; ignore the peer's competing ClientHello
651 // (the lower-id peer yields to server and answers with a ServerHello).
652 tracing::debug!(
653 from,
654 "relay glare — retaining client role (peer yields to server)"
655 );
656 Route::Ignore
657 } else if self.inbound_accept.lock().unwrap().is_none() {
658 // Lower id → should be server, but no responder path is enabled; cannot serve,
659 // so keep the (doomed) client tunnel and drop rather than tear it down.
660 tracing::debug!(
661 from,
662 "relay glare — should be server but accept path off; dropping"
663 );
664 Route::Ignore
665 } else {
666 // Lower id → yield to the server role: drop our client tunnel (only if it is
667 // still ours) and accept the peer's circuit as server below.
668 if tunnels.get(from).map(|e| e.id) == Some(id) {
669 tunnels.remove(from);
670 }
671 tracing::debug!(from, "relay glare — yielding to server role");
672 Route::Accept
673 }
674 }
675 // No circuit under this key — a candidate INTRODUCED circuit (a peer dialing us over
676 // the relay). Whether it really is one is decided by the frame's direction in
677 // `accept_introduced`, which admits only a dialer's opening ClientHello (#1761).
678 None => Route::Accept,
679 }
680 };
681
682 match route {
683 Route::Deliver(sink) => {
684 if sink.try_send(payload).is_err() {
685 tracing::debug!(from, "relayed tunnel inbound full/closed — frame dropped");
686 }
687 }
688 Route::Ignore => {}
689 Route::Accept => self.accept_introduced(from, payload),
690 }
691 }
692
693 /// Accept an INTRODUCED inbound circuit from `from` as a server-role tunnel and surface it to the
694 /// consumer's [`RelayAcceptor`](crate::accept::RelayAcceptor) — the RESPONDER path. Gated on: the
695 /// frame actually being a dialer's OPENING HANDSHAKE (below), the responder path being enabled
696 /// ([`enable_accept`](Self::enable_accept)), NON-CLOBBER (a circuit already under this key is kept,
697 /// never replaced by a racing registration — the #1536 double-session defense), and the flood cap
698 /// ([`MAX_RELAY_TUNNELS`]). The opening frame (the dialer's ClientHello) is delivered into the fresh
699 /// tunnel so the server handshake sees it; a full accept channel drops the newest circuit (bounded
700 /// backpressure).
701 ///
702 /// This is the ONE place a server-role circuit is created, so the mTLS role of a relayed circuit is
703 /// decided HERE and only here, from the circuit's own direction — never from what the accept path
704 /// happens to be handed.
705 fn accept_introduced(self: &Arc<Self>, from: &str, payload: Vec<u8>) {
706 // #1761: ONLY a client's opening handshake may open an inbound circuit. A relayed frame from a
707 // peer we hold no tunnel to is otherwise NOT the start of a circuit — it is a frame belonging to
708 // a circuit that no longer exists here, or one that was never ours. The whole class must be
709 // dropped, not just the observed instance: a peer's ServerHello or application record arriving
710 // after `fast_connect` released the per-peer tunnel on a relayed→direct promotion, a frame that
711 // outlived a timed-out or torn-down circuit, and any garbage an untrusted relay injects. Accepting
712 // any of them stands up a TLS SERVER against a peer that is itself a server, which is the live
713 // `got ServerHello when expecting ClientHello` / `UnexpectedMessage` deadlock — and it also let a
714 // single arbitrary byte cost a tunnel slot plus an accept-task.
715 if !is_tls_client_hello(&payload) {
716 // The relay-supplied `from` is deliberately NOT logged: it is an untrusted, unbounded
717 // peer-controlled string, and this line is reachable by any frame a hostile relay cares to
718 // send. The frame length is the only diagnostic that cannot carry attacker text.
719 tracing::debug!(
720 len = payload.len(),
721 "dropping a relayed frame that is not a dialer's opening handshake — no circuit is open \
722 for this peer"
723 );
724 return;
725 }
726 let Some(accept_tx) = self.inbound_accept.lock().unwrap().clone() else {
727 return;
728 };
729 let network_id = self
730 .local_network_id
731 .lock()
732 .unwrap()
733 .clone()
734 .unwrap_or_default();
735 let tunnel = {
736 let mut tunnels = self.tunnels.lock().unwrap();
737 if tunnels.contains_key(from) {
738 // A circuit to this peer already exists (a racing dial claimed the key) — do NOT
739 // clobber it into a conflicting second session.
740 return;
741 }
742 if tunnels.len() >= MAX_RELAY_TUNNELS {
743 tracing::debug!(
744 from,
745 "inbound relay accept cap reached — dropping introduced circuit"
746 );
747 return;
748 }
749 self.insert_entry(&mut tunnels, from, &network_id, TunnelRole::Server)
750 };
751 // Deliver the opening frame so the server-side handshake sees the ClientHello.
752 if let Some(sink) = self
753 .tunnels
754 .lock()
755 .unwrap()
756 .get(from)
757 .map(|e| e.sink.clone())
758 {
759 let _ = sink.try_send(payload);
760 }
761 // Hand the server-role tunnel to the consumer to run `PeerSession::server` over. A full/closed
762 // accept channel drops the tunnel here — its `Drop` deregisters the routing.
763 if accept_tx.try_send(tunnel).is_err() {
764 tracing::debug!(
765 from,
766 "inbound accept channel full/closed — dropping introduced circuit"
767 );
768 }
769 }
770
771 /// Remove a tunnel's routing entry (called on [`RelayTunnel`] drop) — but ONLY when the stored
772 /// entry is still THIS registration (`id` matches). A glare tie-break can replace our client
773 /// tunnel with a server tunnel under the same peer key (#1536); the old client tunnel's `Drop`
774 /// must not then evict the newer server entry.
775 fn close_tunnel(&self, target_peer: &str, id: u64) {
776 let mut tunnels = self.tunnels.lock().unwrap();
777 if tunnels.get(target_peer).map(|e| e.id) == Some(id) {
778 tunnels.remove(target_peer);
779 }
780 }
781
782 /// Whether a relayed tunnel to `target_peer` is currently registered — the test hook fast-connect
783 /// uses to assert the per-peer tunnel was released (dropped) after a relayed→direct promotion,
784 /// while the reservation itself stays held.
785 #[cfg(test)]
786 pub(crate) fn open_tunnel_exists(&self, target_peer: &str) -> bool {
787 self.tunnels.lock().unwrap().contains_key(target_peer)
788 }
789
790 /// Test-only: register a SERVER-role tunnel to `target_peer` directly, for tests that drive an mTLS
791 /// SERVER over a hand-wired relay tunnel (the production server path is [`enable_accept`] +
792 /// [`route_relayed`]'s accept branch). A server-role tunnel routes an incoming ClientHello straight
793 /// through instead of treating it as the #1536 glare signal, so these tests mirror a real server
794 /// receiving a dialer's ClientHello.
795 #[cfg(test)]
796 pub(crate) fn open_server_tunnel(
797 self: &Arc<Self>,
798 target_peer: &str,
799 network_id: &str,
800 ) -> RelayTunnel {
801 self.register_tunnel(target_peer, network_id, TunnelRole::Server)
802 }
803
804 /// A JSON snapshot for a `control.relayStatus`-style RPC. `state` is the canonical truth;
805 /// `connected` is a convenience boolean (== `state == connected`).
806 pub fn snapshot_json(&self, endpoint: &str, peer_id: &str) -> serde_json::Value {
807 let state = self.state();
808 serde_json::json!({
809 "state": state.as_str(),
810 "connected": state == RelayState::Connected,
811 "endpoint": endpoint,
812 "peer_id": peer_id,
813 "reconnect_attempts": self.reconnect_attempts.load(Ordering::Relaxed),
814 "connected_peers": self.connected_peers.load(Ordering::Relaxed),
815 "last_error": *self.last_error.lock().unwrap(),
816 })
817 }
818}
819
820/// A live RLY-002 relayed-transport tunnel to one peer, multiplexed over the node's persistent relay
821/// reservation socket (the tier-6 TURN fallback). Writes are framed as RLY-002 `relay_message` to the
822/// target and forwarded A→relay→B; reads are the payloads the relay forwards back from that peer.
823///
824/// Per NC-1 the payload MUST be END-TO-END SEALED to the recipient's key by the caller — the relay is
825/// an untrusted forwarder that sees only ciphertext. Dropping the tunnel deregisters its routing.
826pub struct RelayTunnel {
827 /// The remote peer's `peer_id` (hex) — the RLY-002 `to`, and the routing key for inbound frames.
828 target: String,
829 /// The network the tunnel is scoped to (echoed for the consumer; relay routes by peer_id).
830 network_id: String,
831 /// Shared status handle — provides the live outbound sink, this node's `peer_id`, and the seq.
832 status: Arc<RelayStatus>,
833 /// Inbound payloads the relay forwarded from `target`, in arrival order (bounded — see
834 /// [`RELAY_TUNNEL_INBOUND_CAP`]).
835 inbound: mpsc::Receiver<Vec<u8>>,
836 /// This registration's monotonic id — so `Drop` only deregisters when the map still holds THIS
837 /// entry (a glare replace may have superseded it under the same key; #1536).
838 id: u64,
839}
840
841impl RelayTunnel {
842 /// The remote peer this tunnel forwards to/from (hex `peer_id`).
843 pub fn target(&self) -> &str {
844 &self.target
845 }
846
847 /// The network the tunnel is scoped to.
848 pub fn network_id(&self) -> &str {
849 &self.network_id
850 }
851
852 /// Send `payload` to the target peer through the relay (RLY-002 `relay_message`). `payload` MUST
853 /// already be sealed to the recipient (NC-1). `Err` if the reservation dropped (send after the
854 /// session closed) or `payload` exceeds [`MAX_RELAY_PAYLOAD`].
855 pub fn send(&self, payload: Vec<u8>) -> Result<(), String> {
856 if payload.len() > MAX_RELAY_PAYLOAD {
857 return Err(format!(
858 "relayed payload {} exceeds cap {MAX_RELAY_PAYLOAD}",
859 payload.len()
860 ));
861 }
862 let from = self
863 .status
864 .local_peer_id
865 .lock()
866 .unwrap()
867 .clone()
868 .ok_or("relay reservation not connected — no local peer_id")?;
869 let seq = self.status.relay_seq.fetch_add(1, Ordering::Relaxed);
870 let frame = RelayMessage::RelayGossipMessage {
871 from,
872 to: self.target.clone(),
873 payload,
874 seq,
875 };
876 let guard = self.status.outbound.lock().unwrap();
877 let sink = guard
878 .as_ref()
879 .ok_or("relay reservation not connected — cannot send relayed frame")?;
880 sink.send(frame)
881 .map_err(|_| "relay reservation write half closed".to_string())
882 }
883
884 /// Await the next payload the relay forwards from the target peer. `None` once the reservation
885 /// drops (the session closed) — the caller should re-open the tunnel after the relay reconnects.
886 pub async fn recv(&mut self) -> Option<Vec<u8>> {
887 self.inbound.recv().await
888 }
889
890 /// Poll for the next inbound payload. This is the non-`async` primitive the
891 /// [`RelayTunnelStream`](crate::tunnel::RelayTunnelStream) `AsyncRead` adapter drives so an mTLS
892 /// session can run OVER the relay tunnel. `Poll::Ready(None)` once the reservation drops.
893 pub(crate) fn poll_recv(
894 &mut self,
895 cx: &mut std::task::Context<'_>,
896 ) -> std::task::Poll<Option<Vec<u8>>> {
897 self.inbound.poll_recv(cx)
898 }
899}
900
901impl Drop for RelayTunnel {
902 fn drop(&mut self) {
903 self.status.close_tunnel(&self.target, self.id);
904 }
905}
906
907/// Resolve the relay endpoint: `DIG_RELAY_URL` if set + non-empty (and not the opt-out token), else
908/// the canonical [`dig_constants::DIG_RELAY_URL`].
909pub fn relay_url_from_env() -> String {
910 std::env::var("DIG_RELAY_URL")
911 .ok()
912 .filter(|s| !s.trim().is_empty())
913 .filter(|s| !is_off_token(s))
914 .unwrap_or_else(|| dig_constants::DIG_RELAY_URL.to_string())
915}
916
917/// Whether the relay connection is enabled. Disabled when `DIG_RELAY_URL` is `off`/`disabled`/
918/// empty-after-trim — an explicit opt-out for air-gapped/standalone nodes.
919pub fn relay_enabled() -> bool {
920 match std::env::var("DIG_RELAY_URL") {
921 Ok(v) => !is_off_token(&v),
922 Err(_) => true,
923 }
924}
925
926/// `true` if `v` is the reservation opt-out token (`off`/`disabled`, case-insensitive, trimmed).
927fn is_off_token(v: &str) -> bool {
928 let v = v.trim();
929 v.eq_ignore_ascii_case("off") || v.eq_ignore_ascii_case("disabled")
930}
931
932/// Current unix time (seconds), saturating.
933fn now_secs() -> u64 {
934 std::time::SystemTime::now()
935 .duration_since(std::time::UNIX_EPOCH)
936 .map(|d| d.as_secs())
937 .unwrap_or(0)
938}
939
940/// Maintain a CONSTANT relay reservation forever: connect, register, keepalive, and on any drop
941/// reconnect with capped exponential backoff. Spawned as a background task; tolerates the relay
942/// being down (retries forever, never crashes). `peer_id` is the node's stable identity hex.
943pub async fn run_relay_connection(
944 endpoint: String,
945 peer_id: String,
946 network_id: String,
947 listen_addrs: Vec<SocketAddr>,
948 status: Arc<RelayStatus>,
949) {
950 run_relay_connection_with(
951 endpoint,
952 peer_id,
953 network_id,
954 listen_addrs,
955 status,
956 Backoff::default(),
957 )
958 .await
959}
960
961/// [`run_relay_connection`] with an explicit backoff schedule (tests pass tiny values for fast,
962/// deterministic reconnect timing; the LOGIC is identical — only the sleep durations differ).
963pub async fn run_relay_connection_with(
964 endpoint: String,
965 peer_id: String,
966 network_id: String,
967 listen_addrs: Vec<SocketAddr>,
968 status: Arc<RelayStatus>,
969 backoff: Backoff,
970) {
971 let mut consecutive_failures: u32 = 0;
972 loop {
973 status.set_connecting();
974 match connect_once(&endpoint, &peer_id, &network_id, &listen_addrs, &status).await {
975 Ok(()) => {
976 consecutive_failures = 0;
977 status.set_disconnected(None);
978 }
979 Err(e) => {
980 consecutive_failures = consecutive_failures.saturating_add(1);
981 status.set_disconnected(Some(e));
982 }
983 }
984 // ALWAYS sleep a bounded backoff before retrying — prevents a busy error loop.
985 let delay = backoff_secs_with(consecutive_failures, backoff.base_secs, backoff.cap_secs);
986 tokio::time::sleep(Duration::from_secs(delay)).await;
987 }
988}
989
990/// A relay WebSocket endpoint parsed into the pieces the happy-eyeballs dial needs: the host to
991/// resolve and the TCP port. The scheme (`ws`/`wss`) only selects the default port here — the
992/// plaintext-vs-TLS choice is re-derived from the URL by [`client_async_tls_with_config`] during the
993/// handshake, so a single code path serves both.
994#[derive(Debug, PartialEq, Eq)]
995struct RelayEndpoint {
996 host: String,
997 port: u16,
998}
999
1000/// Parse a relay endpoint URL (`ws://host[:port][/path]` / `wss://host[:port][/path]`, IPv6 hosts in
1001/// `[…]`) into its host + port. Only the authority is needed for the dial; any path/query/fragment and
1002/// userinfo are ignored (the full URL is still handed to the WS handshake for the correct `Host`/SNI).
1003fn parse_relay_endpoint(endpoint: &str) -> Result<RelayEndpoint, String> {
1004 let (scheme, rest) = endpoint
1005 .split_once("://")
1006 .ok_or_else(|| format!("relay endpoint missing scheme: {endpoint}"))?;
1007 let default_port = match scheme.to_ascii_lowercase().as_str() {
1008 "ws" => 80,
1009 "wss" => 443,
1010 other => return Err(format!("unsupported relay scheme: {other}")),
1011 };
1012 // Authority only: drop any path/query/fragment, then any `userinfo@`.
1013 let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest);
1014 let authority = authority
1015 .rsplit_once('@')
1016 .map(|(_, h)| h)
1017 .unwrap_or(authority);
1018
1019 let (host, port) = if let Some(stripped) = authority.strip_prefix('[') {
1020 // Bracketed IPv6 literal: `[addr]` or `[addr]:port`.
1021 let (h, after) = stripped
1022 .split_once(']')
1023 .ok_or_else(|| format!("malformed IPv6 authority: {authority}"))?;
1024 let port = match after.strip_prefix(':') {
1025 Some(p) => p.parse().map_err(|_| format!("bad relay port: {after}"))?,
1026 None => default_port,
1027 };
1028 (h.to_string(), port)
1029 } else if let Some((h, p)) = authority.rsplit_once(':') {
1030 (
1031 h.to_string(),
1032 p.parse().map_err(|_| format!("bad relay port: {p}"))?,
1033 )
1034 } else {
1035 (authority.to_string(), default_port)
1036 };
1037
1038 if host.is_empty() {
1039 return Err(format!("relay endpoint missing host: {endpoint}"));
1040 }
1041 Ok(RelayEndpoint { host, port })
1042}
1043
1044/// Resolve a relay host to its family-tagged dial candidates: a literal IP yields one candidate (no
1045/// DNS), a hostname is resolved to its full A + AAAA set. The candidates feed `dig_ip::connect`, which
1046/// applies the §5.2 IPv6-first preference + local∩peer family intersection, so no ordering is imposed
1047/// here — the addresses are added as resolved and tagged by family for observability.
1048async fn resolve_relay_candidates(host: &str, port: u16) -> Result<PeerCandidates, String> {
1049 let mut candidates = PeerCandidates::new();
1050 let source_for = |ip: &IpAddr| {
1051 if ip.is_ipv6() {
1052 CandidateSource::DnsAAAA
1053 } else {
1054 CandidateSource::DnsA
1055 }
1056 };
1057 if let Ok(ip) = host.parse::<IpAddr>() {
1058 candidates.add(SocketAddr::new(ip, port), source_for(&ip));
1059 } else {
1060 let resolved = tokio::net::lookup_host((host, port))
1061 .await
1062 .map_err(|e| format!("resolve {host}:{port}: {e}"))?;
1063 for addr in resolved {
1064 candidates.add(addr, source_for(&addr.ip()));
1065 }
1066 }
1067 if candidates.is_empty() {
1068 return Err(format!("no addresses resolved for {host}:{port}"));
1069 }
1070 Ok(candidates)
1071}
1072
1073/// Race the relay `candidates` IPv6-first with graceful IPv4 fallback via `dig_ip::connect` (§5.2,
1074/// RFC 8305). The transport connect stays a caller-supplied closure so the racing logic is unit-tested
1075/// with a fake dial (no real DNS/sockets) exactly as the direct-peer dialer does in `dialer.rs`; the
1076/// production caller ([`open_relay_ws`]) hands it a real [`TcpStream::connect`].
1077async fn race_relay_candidates<C, F, Fut>(
1078 local: &LocalStack,
1079 candidates: &PeerCandidates,
1080 config: DialConfig,
1081 dial_fn: F,
1082) -> Result<dig_ip::DialWinner<C>, String>
1083where
1084 F: Fn(SocketAddr) -> Fut + Sync,
1085 Fut: std::future::Future<Output = Result<C, String>> + Send,
1086 C: Send,
1087{
1088 dig_ip::connect(local, candidates, config, dial_fn)
1089 .await
1090 .map_err(|e| format!("relay happy-eyeballs dial: {e}"))
1091}
1092
1093/// Open the relay WebSocket over an IPv6-first happy-eyeballs TCP race (§5.2), matching the direct-peer
1094/// dial path in `dialer.rs`: resolve the endpoint host to its A + AAAA candidates, race the TCP connect
1095/// via `dig_ip::connect` (IPv6-first, fast IPv4 fallback), then run the WS handshake over the WINNING
1096/// socket — TLS-over-that-stream for `wss://`, plaintext for `ws://` (the mode is taken from the URL by
1097/// [`client_async_tls_with_config`]). Replaces `tokio_tungstenite::connect_async`, whose sequential,
1098/// single-family resolve-and-connect contradicted the IPv6-first reservation guarantee.
1099async fn open_relay_ws(
1100 endpoint: &str,
1101) -> Result<WebSocketStream<MaybeTlsStream<TcpStream>>, String> {
1102 let parsed = parse_relay_endpoint(endpoint)?;
1103 let candidates = resolve_relay_candidates(&parsed.host, parsed.port).await?;
1104 let local = LocalStack::cached();
1105 let winner = race_relay_candidates(
1106 &local,
1107 &candidates,
1108 DialConfig::default(),
1109 |addr| async move {
1110 TcpStream::connect(addr)
1111 .await
1112 .map_err(|e| format!("tcp connect {addr}: {e}"))
1113 },
1114 )
1115 .await?;
1116 let (ws, _resp) = client_async_tls_with_config(endpoint, winner.conn, None, None)
1117 .await
1118 .map_err(|e| format!("ws handshake: {e}"))?;
1119 Ok(ws)
1120}
1121
1122/// One connect → register → serve cycle. Returns `Ok` on a clean close, `Err(reason)` on failure.
1123async fn connect_once(
1124 endpoint: &str,
1125 peer_id: &str,
1126 network_id: &str,
1127 listen_addrs: &[SocketAddr],
1128 status: &Arc<RelayStatus>,
1129) -> Result<(), String> {
1130 // Each session's discovered-peer set + transport are independent — never carry state across a
1131 // drop. `clear_transport` also runs at the end so a dropped session's tunnels/sink never linger.
1132 status.clear_known_peers();
1133 status.clear_transport();
1134
1135 let ws = open_relay_ws(endpoint).await?;
1136 let (mut write, mut read) = ws.split();
1137
1138 // RLY-001: register immediately so the relay holds our reservation, advertising the node's gossip
1139 // listen candidates (B1) so the relay can hand other peers a dialable candidate (§5.2 IPv6-first).
1140 let register = RelayMessage::Register {
1141 peer_id: peer_id.to_string(),
1142 network_id: network_id.to_string(),
1143 protocol_version: RELAY_PROTOCOL_VERSION,
1144 listen_addrs: listen_addrs.to_vec(),
1145 };
1146 send(&mut write, ®ister).await?;
1147
1148 // Publish the outbound sink so RLY-002 relayed tunnels can reuse THIS persistent socket. Drained
1149 // in the select loop below; cleared when the session ends.
1150 let (out_tx, mut out_rx) = mpsc::unbounded_channel::<RelayMessage>();
1151 status.set_transport(peer_id, network_id, out_tx);
1152
1153 // RLY-005: pull the current peer list right away, then again periodically — all over THIS
1154 // persistent socket, so discovery never requires reopening a connection.
1155 let get_peers = RelayMessage::GetPeers {
1156 network_id: Some(network_id.to_string()),
1157 };
1158 send(&mut write, &get_peers).await?;
1159
1160 let mut ping = tokio::time::interval(Duration::from_secs(PING_INTERVAL_SECS));
1161 ping.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1162 ping.tick().await; // skip the immediate first tick
1163
1164 let mut discovery = tokio::time::interval(Duration::from_secs(DISCOVERY_INTERVAL_SECS));
1165 discovery.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1166 discovery.tick().await; // skip the immediate first tick (we already pulled once above)
1167
1168 // Run the session; whatever the outcome, tear the transport down so a dropped session never
1169 // leaves a stale outbound sink or open tunnels behind (they'd send into a closed socket).
1170 let result = serve_session(
1171 &mut write,
1172 &mut read,
1173 &mut ping,
1174 &mut discovery,
1175 &mut out_rx,
1176 network_id,
1177 status,
1178 )
1179 .await;
1180 status.clear_transport();
1181 result
1182}
1183
1184/// The connected-session select loop: keepalive pings, periodic RLY-005 discovery, draining the
1185/// outbound relayed-transport sink onto the socket, and handling inbound frames. Returns `Ok` on a
1186/// clean close, `Err(reason)` on a failure. Split out of `connect_once` so its caller can always run
1187/// transport teardown regardless of how the session ends.
1188#[allow(clippy::too_many_arguments)]
1189async fn serve_session<W, R>(
1190 write: &mut W,
1191 read: &mut R,
1192 ping: &mut tokio::time::Interval,
1193 discovery: &mut tokio::time::Interval,
1194 out_rx: &mut mpsc::UnboundedReceiver<RelayMessage>,
1195 network_id: &str,
1196 status: &Arc<RelayStatus>,
1197) -> Result<(), String>
1198where
1199 W: SinkExt<Message> + Unpin,
1200 <W as futures_util::Sink<Message>>::Error: std::fmt::Display,
1201 R: StreamExt<Item = Result<Message, tokio_tungstenite::tungstenite::Error>> + Unpin,
1202{
1203 loop {
1204 tokio::select! {
1205 _ = ping.tick() => {
1206 send(write, &RelayMessage::Ping { timestamp: now_secs() }).await?;
1207 }
1208 _ = discovery.tick() => {
1209 send(write, &RelayMessage::GetPeers {
1210 network_id: Some(network_id.to_string()),
1211 }).await?;
1212 }
1213 // A relayed tunnel queued an RLY-002 frame — forward it over THIS persistent socket.
1214 Some(frame) = out_rx.recv() => {
1215 send(write, &frame).await?;
1216 }
1217 frame = read.next() => {
1218 match frame {
1219 None => return Ok(()),
1220 Some(Err(e)) => return Err(format!("read: {e}")),
1221 Some(Ok(Message::Close(_))) => return Ok(()),
1222 Some(Ok(Message::Ping(p))) => {
1223 write.send(Message::Pong(p)).await.map_err(|e| format!("pong: {e}"))?;
1224 }
1225 Some(Ok(Message::Pong(_))) | Some(Ok(Message::Frame(_))) => {}
1226 Some(Ok(Message::Text(t))) => {
1227 handle_incoming(t.into_bytes(), write, status).await?;
1228 }
1229 Some(Ok(Message::Binary(b))) => {
1230 handle_incoming(b, write, status).await?;
1231 }
1232 }
1233 }
1234 }
1235 }
1236}
1237
1238/// Handle one decoded inbound relay frame: track RegisterAck (→ connected), answer relay Pings.
1239async fn handle_incoming<W>(
1240 bytes: Vec<u8>,
1241 write: &mut W,
1242 status: &Arc<RelayStatus>,
1243) -> Result<(), String>
1244where
1245 W: SinkExt<Message> + Unpin,
1246 <W as futures_util::Sink<Message>>::Error: std::fmt::Display,
1247{
1248 let Ok(msg) = serde_json::from_slice::<RelayMessage>(&bytes) else {
1249 return Ok(()); // ignore anything we can't parse; the relay is untrusted
1250 };
1251 match msg {
1252 RelayMessage::RegisterAck {
1253 success,
1254 message,
1255 connected_peers,
1256 } => {
1257 if success {
1258 status.set_connected(connected_peers as u64);
1259 } else {
1260 return Err(format!("register rejected: {message}"));
1261 }
1262 }
1263 RelayMessage::Ping { timestamp } => {
1264 send(write, &RelayMessage::Pong { timestamp }).await?;
1265 }
1266 // RLY-005 + push notices: fold peers discovered over the live socket into the status so the
1267 // consumer's pool/address book sees them without opening an ephemeral discovery connection.
1268 RelayMessage::Peers { peers } => status.replace_known_peers(peers),
1269 RelayMessage::PeerConnected { peer } => status.add_known_peer(peer),
1270 RelayMessage::PeerDisconnected { peer_id } => status.remove_known_peer(&peer_id),
1271 // RLY-002 relayed transport (tier-6 TURN): route a payload the relay forwarded from `from` to
1272 // that peer's open tunnel. Unknown-peer / oversized / full-channel frames are dropped inside
1273 // `route_relayed` (untrusted-relay defense). `to`/`seq` are the relay's concern; we key on
1274 // `from`. Per NC-1 `payload` is sealed ciphertext the relay could not read.
1275 RelayMessage::RelayGossipMessage { from, payload, .. } => {
1276 status.route_relayed(&from, payload)
1277 }
1278 RelayMessage::Error { code, message } => {
1279 // A relay `Error` frame is NOT automatically a reservation problem — the relay reports
1280 // per-REQUEST failures on the same channel. Treating them all as fatal made a routine
1281 // "the peer you asked for has left" (`PEER_NOT_FOUND`) tear down this node's own
1282 // reservation, de-register it, and force a full reconnect — a flap that repeatedly
1283 // pulled the node out of the relay's introductions (dig_ecosystem #1932).
1284 if relay_error_is_fatal(code) {
1285 return Err(format!("relay error {code}: {message}"));
1286 }
1287 tracing::debug!(code, %message, "relay reported a per-request error; reservation held");
1288 }
1289 other => tracing::debug!(?other, "relay message ignored by reservation loop"),
1290 }
1291 Ok(())
1292}
1293
1294/// Wire two in-memory relay reservations to forward RLY-002 frames to each OTHER — a loopback relay
1295/// with no real network. Each returned [`RelayStatus`] is `Connected` with a live outbound sink whose
1296/// frames are routed into the peer's tunnels by `from` peer_id, exactly as a real relay would forward
1297/// A→relay→B. Used to prove a full mTLS session round-trips over [`RelayTunnel`]s (see `tunnel.rs`).
1298///
1299/// `a` opens tunnels targeting `b_id`; `b` opens tunnels targeting `a_id`.
1300#[cfg(test)]
1301pub(crate) fn loopback_reservation_pair(
1302 a_id: &str,
1303 b_id: &str,
1304) -> (Arc<RelayStatus>, Arc<RelayStatus>) {
1305 let a = RelayStatus::new();
1306 let b = RelayStatus::new();
1307 a.set_connected(1);
1308 b.set_connected(1);
1309
1310 let (a_tx, mut a_rx) = mpsc::unbounded_channel::<RelayMessage>();
1311 let (b_tx, mut b_rx) = mpsc::unbounded_channel::<RelayMessage>();
1312 a.set_transport(a_id, DEFAULT_NETWORK_ID, a_tx);
1313 b.set_transport(b_id, DEFAULT_NETWORK_ID, b_tx);
1314
1315 // Drain a's outbound → forward into b (route by `from`), and symmetrically b → a. This is the
1316 // relay's forwarding role, in-process.
1317 let b_route = Arc::clone(&b);
1318 tokio::spawn(async move {
1319 while let Some(RelayMessage::RelayGossipMessage { from, payload, .. }) = a_rx.recv().await {
1320 b_route.route_relayed(&from, payload);
1321 }
1322 });
1323 let a_route = Arc::clone(&a);
1324 tokio::spawn(async move {
1325 while let Some(RelayMessage::RelayGossipMessage { from, payload, .. }) = b_rx.recv().await {
1326 a_route.route_relayed(&from, payload);
1327 }
1328 });
1329
1330 (a, b)
1331}
1332
1333/// Serialize + send one `RelayMessage` as a WebSocket text frame.
1334async fn send<W>(write: &mut W, msg: &RelayMessage) -> Result<(), String>
1335where
1336 W: SinkExt<Message> + Unpin,
1337 <W as futures_util::Sink<Message>>::Error: std::fmt::Display,
1338{
1339 let txt = serde_json::to_string(msg).map_err(|e| format!("encode: {e}"))?;
1340 write
1341 .send(Message::Text(txt))
1342 .await
1343 .map_err(|e| format!("send: {e}"))
1344}
1345
1346#[cfg(test)]
1347mod tests {
1348 use super::*;
1349 use dig_ip::Family;
1350 use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
1351 use std::sync::Mutex as StdMutex;
1352
1353 // -- #1932: a per-request relay error must not cost us the reservation -------------------
1354
1355 #[test]
1356 fn peer_not_found_is_not_fatal_because_it_is_about_another_peer() {
1357 // The exact frame that was flapping the fleet: the peer we dialled had left the relay.
1358 // Routine on a live network, and no statement at all about our own registration.
1359 assert!(!relay_error_is_fatal(3));
1360 }
1361
1362 #[test]
1363 fn a_bad_frame_we_sent_costs_that_frame_not_the_reservation() {
1364 assert!(!relay_error_is_fatal(2));
1365 }
1366
1367 #[test]
1368 fn every_code_that_means_our_registration_failed_is_fatal() {
1369 // 1 NOT_REGISTERED, and the four that accompany a failing register_ack. For these the
1370 // reservation genuinely does not exist, so ending the loop and re-registering is correct.
1371 for code in [1, 4, 5, 6, 7] {
1372 assert!(
1373 relay_error_is_fatal(code),
1374 "code {code} invalidates the reservation and must end the loop"
1375 );
1376 }
1377 }
1378
1379 #[test]
1380 fn an_unknown_future_code_keeps_the_reservation() {
1381 // Deliberate asymmetry: a wrong guess here costs one log line, whereas defaulting to fatal
1382 // would let a newly-introduced code take every node off its relay simultaneously.
1383 for code in [0, 8, 99, u32::MAX] {
1384 assert!(!relay_error_is_fatal(code), "code {code} must not be fatal");
1385 }
1386 }
1387
1388 #[test]
1389 fn parses_wss_host_and_explicit_port() {
1390 let ep = parse_relay_endpoint("wss://relay.dig.net:443").unwrap();
1391 assert_eq!(ep.host, "relay.dig.net");
1392 assert_eq!(ep.port, 443);
1393 }
1394
1395 #[test]
1396 fn parses_default_ports_by_scheme() {
1397 assert_eq!(
1398 parse_relay_endpoint("wss://relay.dig.net").unwrap().port,
1399 443
1400 );
1401 assert_eq!(parse_relay_endpoint("ws://relay.dig.net").unwrap().port, 80);
1402 }
1403
1404 #[test]
1405 fn parses_bracketed_ipv6_authority_with_and_without_port() {
1406 let with_port = parse_relay_endpoint("wss://[2001:db8::1]:8443").unwrap();
1407 assert_eq!(with_port.host, "2001:db8::1");
1408 assert_eq!(with_port.port, 8443);
1409 let no_port = parse_relay_endpoint("wss://[2001:db8::1]/ws").unwrap();
1410 assert_eq!(no_port.host, "2001:db8::1");
1411 assert_eq!(no_port.port, 443);
1412 }
1413
1414 #[test]
1415 fn ignores_path_query_and_userinfo() {
1416 let ep = parse_relay_endpoint("wss://user@relay.dig.net:9443/ws?x=1#f").unwrap();
1417 assert_eq!(ep.host, "relay.dig.net");
1418 assert_eq!(ep.port, 9443);
1419 }
1420
1421 #[test]
1422 fn rejects_malformed_endpoints() {
1423 assert!(parse_relay_endpoint("relay.dig.net:443").is_err()); // no scheme
1424 assert!(parse_relay_endpoint("http://relay.dig.net").is_err()); // wrong scheme
1425 assert!(parse_relay_endpoint("wss://relay.dig.net:notaport").is_err());
1426 }
1427
1428 #[tokio::test]
1429 async fn resolve_relay_candidates_handles_ip_literals_without_dns() {
1430 let v6 = resolve_relay_candidates("2001:db8::1", 443).await.unwrap();
1431 assert_eq!(v6.all().len(), 1);
1432 assert_eq!(v6.all()[0].family, Family::V6);
1433 assert_eq!(v6.all()[0].source, CandidateSource::DnsAAAA);
1434
1435 let v4 = resolve_relay_candidates("203.0.113.7", 443).await.unwrap();
1436 assert_eq!(v4.all()[0].family, Family::V4);
1437 assert_eq!(v4.all()[0].source, CandidateSource::DnsA);
1438 }
1439
1440 /// The relay dial races BOTH families and falls back to IPv4 when the IPv6 candidate is dead —
1441 /// the §5.2 happy-eyeballs guarantee, proven with a FAKE dial closure (no real DNS/sockets). A
1442 /// dead IPv6 candidate + a live IPv4 candidate on a dual-stack host must yield the IPv4 winner,
1443 /// and BOTH families must have been attempted.
1444 #[tokio::test]
1445 async fn relay_dial_races_both_families_and_falls_back_to_ipv4() {
1446 let mut candidates = PeerCandidates::new();
1447 let v6: SocketAddr = "[2001:db8::1]:443".parse().unwrap();
1448 let v4: SocketAddr = "203.0.113.7:443".parse().unwrap();
1449 candidates.add(v6, CandidateSource::DnsAAAA);
1450 candidates.add(v4, CandidateSource::DnsA);
1451
1452 let dual = LocalStack::from_flags(true, true);
1453 let attempted: StdMutex<Vec<SocketAddr>> = StdMutex::new(Vec::new());
1454 // Fast attempt-delay so the hedged IPv4 starts promptly once the IPv6 attempt fails.
1455 let cfg = DialConfig {
1456 per_attempt_timeout: Duration::from_secs(1),
1457 attempt_delay: Duration::from_millis(5),
1458 };
1459
1460 let winner = race_relay_candidates(&dual, &candidates, cfg, |addr| {
1461 let attempted = &attempted;
1462 async move {
1463 attempted.lock().unwrap().push(addr);
1464 if addr.is_ipv6() {
1465 Err(format!("simulated dead IPv6 {addr}"))
1466 } else {
1467 Ok(addr) // the fake "connection" is just the address that won
1468 }
1469 }
1470 })
1471 .await
1472 .expect("IPv4 fallback wins when IPv6 is dead");
1473
1474 assert_eq!(winner.conn, v4, "the live IPv4 candidate won");
1475 assert_eq!(winner.family, Family::V4);
1476 let tried = attempted.lock().unwrap();
1477 assert!(
1478 tried.contains(&v6),
1479 "the IPv6 candidate was attempted first"
1480 );
1481 assert!(
1482 tried.contains(&v4),
1483 "the IPv4 candidate was attempted as fallback"
1484 );
1485 }
1486
1487 /// IPv6 is the PREFERENCE, not merely first-attempted: with both families live on a dual-stack
1488 /// host, the IPv6 candidate wins the race (IPv4 is only a fallback).
1489 #[tokio::test]
1490 async fn relay_dial_prefers_ipv6_when_both_live() {
1491 let mut candidates = PeerCandidates::new();
1492 let v6: SocketAddr = "[2001:db8::2]:443".parse().unwrap();
1493 let v4: SocketAddr = "203.0.113.8:443".parse().unwrap();
1494 candidates.add(v6, CandidateSource::DnsAAAA);
1495 candidates.add(v4, CandidateSource::DnsA);
1496
1497 let dual = LocalStack::from_flags(true, true);
1498 let calls = AtomicUsize::new(0);
1499 let winner = race_relay_candidates(&dual, &candidates, DialConfig::default(), |addr| {
1500 calls.fetch_add(1, AtomicOrdering::Relaxed);
1501 async move { Ok::<SocketAddr, String>(addr) }
1502 })
1503 .await
1504 .unwrap();
1505
1506 assert_eq!(winner.conn, v6, "IPv6 preferred when both are viable");
1507 assert_eq!(winner.family, Family::V6);
1508 }
1509
1510 /// A minimal TLS handshake record whose first message is a ClientHello — the ONLY frame that opens
1511 /// an introduced circuit (#1761), so every test that drives the responder path must send this shape.
1512 fn client_hello_frame() -> Vec<u8> {
1513 vec![0x16, 0x03, 0x01, 0x00, 0x05, 0x01, 0, 0, 0, 0]
1514 }
1515
1516 /// Build a `Connected` status with a live (dummy) outbound sink + local identity, so
1517 /// `route_relayed`'s introduced-circuit path can run without a real relay socket.
1518 fn connected_status(local_id: &str) -> Arc<RelayStatus> {
1519 let status = RelayStatus::new();
1520 status.set_connected(1);
1521 let (out_tx, _out_rx) = mpsc::unbounded_channel::<RelayMessage>();
1522 status.set_transport(local_id, DEFAULT_NETWORK_ID, out_tx);
1523 // These tests exercise the introduced-circuit ROUTING (register/accept/drop), which never uses
1524 // the outbound sink, so the receiver may drop at end of scope — the sink stays `Some`.
1525 status
1526 }
1527
1528 /// SECURITY (accept OFF): with NO responder path enabled, an introduced RLY-002 frame from an
1529 /// unknown peer is DROPPED — no tunnel is created and nothing is surfaced. This is the
1530 /// untrusted-relay default: a node that never opted into accepting relayed circuits cannot be made
1531 /// to spawn one by a hostile relay.
1532 #[test]
1533 fn introduced_frame_dropped_when_accept_disabled() {
1534 let status = connected_status("00aa");
1535 // A ClientHello-shaped frame from a peer we hold NO tunnel to — the introduced-circuit trigger.
1536 status.route_relayed("ffbb", client_hello_frame());
1537 assert!(
1538 !status.open_tunnel_exists("ffbb"),
1539 "no tunnel surfaced for an introduced circuit while accept is off"
1540 );
1541 assert_eq!(
1542 status.tunnels.lock().unwrap().len(),
1543 0,
1544 "accept-off drops the introduced circuit entirely"
1545 );
1546 }
1547
1548 /// SECURITY (flood defense, tunnel cap): once [`MAX_RELAY_TUNNELS`] tunnels are open, a further
1549 /// introduced circuit from a new peer is DROPPED rather than registered — a hostile relay flooding
1550 /// distinct fabricated `from` ids cannot spawn unbounded server tunnels/accept-tasks.
1551 #[test]
1552 fn introduced_circuit_dropped_at_max_tunnels_cap() {
1553 let status = connected_status("00aa");
1554 let mut accept_rx = status.enable_accept();
1555 // Saturate the tunnel table at the cap (held open by the returned RelayTunnels).
1556 let mut held = Vec::new();
1557 for i in 0..MAX_RELAY_TUNNELS {
1558 held.push(status.register_tunnel(
1559 &format!("peer{i:05}"),
1560 DEFAULT_NETWORK_ID,
1561 TunnelRole::Server,
1562 ));
1563 }
1564 assert_eq!(status.tunnels.lock().unwrap().len(), MAX_RELAY_TUNNELS);
1565
1566 status.route_relayed("overflowpeer", client_hello_frame());
1567 assert!(
1568 !status.open_tunnel_exists("overflowpeer"),
1569 "an introduced circuit beyond the tunnel cap is dropped, not registered"
1570 );
1571 assert_eq!(
1572 status.tunnels.lock().unwrap().len(),
1573 MAX_RELAY_TUNNELS,
1574 "tunnel count never grows past the cap"
1575 );
1576 assert!(
1577 accept_rx.try_recv().is_err(),
1578 "the capped circuit is never surfaced to the acceptor"
1579 );
1580 drop(held);
1581 }
1582
1583 /// SECURITY (flood defense, accept channel): when the bounded inbound-accept channel
1584 /// ([`INBOUND_ACCEPT_CAP`]) is full — the consumer is not accepting fast enough — a further
1585 /// introduced circuit is DROPPED (its freshly-registered tunnel is torn down), bounded
1586 /// backpressure rather than unbounded queueing.
1587 #[test]
1588 fn introduced_circuit_dropped_when_accept_channel_full() {
1589 let status = connected_status("00aa");
1590 let mut accept_rx = status.enable_accept();
1591 // Fill the accept channel to capacity WITHOUT draining it — each surfaced circuit occupies one
1592 // slot and keeps its server tunnel registered (the RelayTunnel lives in the channel).
1593 for i in 0..INBOUND_ACCEPT_CAP {
1594 status.route_relayed(&format!("in{i:05}"), client_hello_frame());
1595 }
1596 assert_eq!(
1597 status.tunnels.lock().unwrap().len(),
1598 INBOUND_ACCEPT_CAP,
1599 "each surfaced circuit registered exactly one server tunnel"
1600 );
1601
1602 // One more: the accept channel is full → the tunnel is registered then immediately dropped,
1603 // so its routing is deregistered and nothing new is surfaced.
1604 status.route_relayed("overflow", client_hello_frame());
1605 assert!(
1606 !status.open_tunnel_exists("overflow"),
1607 "an introduced circuit is dropped when the accept channel is full"
1608 );
1609
1610 let mut surfaced = 0;
1611 while accept_rx.try_recv().is_ok() {
1612 surfaced += 1;
1613 }
1614 assert_eq!(
1615 surfaced, INBOUND_ACCEPT_CAP,
1616 "exactly the channel capacity surfaced — never the overflow circuit"
1617 );
1618 }
1619
1620 /// REGRESSION (#1536 glare, TIMING order): a peer's ClientHello arrives BEFORE our own dial to it
1621 /// registers. We accept it as a server; our later dial to the SAME peer MUST then be refused
1622 /// (non-clobber) so no conflicting second circuit / double mTLS session is created. This is the
1623 /// deeper ordering the first tie-break missed (role was decided by who-registered-first).
1624 #[test]
1625 fn clienthello_before_local_dial_does_not_double_register() {
1626 let status = connected_status("bbbb");
1627 let mut accept_rx = status.enable_accept();
1628
1629 // The peer's introduced ClientHello arrives first → we accept it as a server-role circuit.
1630 status.route_relayed("aaaa", client_hello_frame());
1631 assert!(status.open_tunnel_exists("aaaa"));
1632 let _server_tunnel = accept_rx
1633 .try_recv()
1634 .expect("introduced circuit surfaced as a server");
1635
1636 // Our own dial to that peer now must be REFUSED — the existing circuit is the connection.
1637 let dial = status.open_tunnel("aaaa", DEFAULT_NETWORK_ID);
1638 assert!(
1639 dial.is_err(),
1640 "a second dial to a peer we already serve is refused (no double-session)"
1641 );
1642 assert_eq!(
1643 status.tunnels.lock().unwrap().len(),
1644 1,
1645 "exactly one circuit per peer — never a conflicting client+server pair"
1646 );
1647 }
1648
1649 /// REGRESSION (#1536 equal-id): a relayed self-dial, or a frame stamped with our OWN peer_id
1650 /// (theoretical SPKI collision / a hostile relay reflecting our id), has no lower/higher end for
1651 /// the tie-break — it MUST be rejected outright, never producing a no-server hang.
1652 #[test]
1653 fn self_dial_and_self_stamped_frame_rejected() {
1654 let status = connected_status("cccc");
1655 assert!(
1656 status.open_tunnel("cccc", DEFAULT_NETWORK_ID).is_err(),
1657 "a relayed self-dial (target == local id) is refused"
1658 );
1659
1660 let mut accept_rx = status.enable_accept();
1661 status.route_relayed("cccc", client_hello_frame());
1662 assert!(
1663 !status.open_tunnel_exists("cccc"),
1664 "a frame stamped with our own id is dropped, never registered"
1665 );
1666 assert!(
1667 accept_rx.try_recv().is_err(),
1668 "a self-stamped frame is never surfaced as an accept"
1669 );
1670 }
1671
1672 /// SECURITY (#1536 relay-injected-ClientHello DoS): an untrusted relay can inject a bogus
1673 /// ClientHello on a lower-id node's client tunnel to force it to yield its outbound dial to a
1674 /// server accept that no real peer completes. mTLS identity is never bypassed, but the outbound
1675 /// dial MUST NOT be permanently lost — once the bogus (never-completing) server circuit is dropped,
1676 /// the peer key frees and a fresh dial is possible.
1677 #[test]
1678 fn injected_clienthello_yield_does_not_permanently_block_redial() {
1679 // local id "00aa" is numerically lower than the peer "ffff", so we are the yield-to-server side.
1680 let status = connected_status("00aa");
1681 let mut accept_rx = status.enable_accept();
1682
1683 let client_tunnel = status
1684 .open_tunnel("ffff", DEFAULT_NETWORK_ID)
1685 .expect("outbound relayed dial opens");
1686 assert!(status.open_tunnel_exists("ffff"));
1687
1688 // The relay injects a bogus ClientHello with from=ffff → glare on our client tunnel → we (the
1689 // lower id) yield: drop the client tunnel, surface a server accept.
1690 status.route_relayed("ffff", client_hello_frame());
1691 let server_tunnel = accept_rx
1692 .try_recv()
1693 .expect("the injected ClientHello yielded a server accept");
1694
1695 // The original outbound dial is cancelled; dropping its handle must NOT evict the newer server
1696 // entry (generation-id guard).
1697 drop(client_tunnel);
1698 assert!(
1699 status.open_tunnel_exists("ffff"),
1700 "the server circuit survives the cancelled client dial's drop"
1701 );
1702
1703 // The bogus circuit completes no handshake; once its tunnel is dropped the key frees...
1704 drop(server_tunnel);
1705 assert!(
1706 !status.open_tunnel_exists("ffff"),
1707 "dropping the never-completing server circuit releases the peer key"
1708 );
1709 // ...and a fresh dial is possible — no permanent lockout from the injected frame.
1710 assert!(
1711 status.open_tunnel("ffff", DEFAULT_NETWORK_ID).is_ok(),
1712 "a fresh outbound dial succeeds after the bogus injection is cleaned up"
1713 );
1714 }
1715
1716 /// REGRESSION (#1761): only a client's OPENING HANDSHAKE may create a server-role circuit.
1717 ///
1718 /// A relayed frame from a peer we hold no tunnel to used to be accepted as an introduced circuit
1719 /// WHATEVER its content, so any frame that is not a dialer's ClientHello manufactured a bogus
1720 /// server-role circuit — and the mTLS server behind it then failed with the live
1721 /// `got ServerHello when expecting ClientHello` (both ends running the TLS server role).
1722 ///
1723 /// The frames covered here are the whole CLASS of "not the start of an inbound circuit", each with
1724 /// a real production or adversarial origin: a peer's ServerHello or application record arriving
1725 /// after we released our client tunnel (a `fast_connect` relayed→direct promotion drops the
1726 /// per-peer tunnel while the peer's frames are still in flight), a truncated record too short to
1727 /// classify, and arbitrary relay garbage. The final ClientHello is the truthful CONTROL: the
1728 /// responder path still works, and dropping the stray frames never blacklists the peer.
1729 #[test]
1730 fn only_a_clienthello_creates_an_introduced_circuit() {
1731 let status = connected_status("00aa");
1732 let mut accept_rx = status.enable_accept();
1733
1734 // A TLS handshake record whose message type is ServerHello (0x02) — the live #1761 frame.
1735 let server_hello = vec![0x16, 0x03, 0x03, 0x00, 0x05, 0x02, 0, 0, 0, 0];
1736 // A TLS application-data record (0x17) — a mid-session frame from a released tunnel.
1737 let app_data = vec![0x17, 0x03, 0x03, 0x00, 0x04, 0xde, 0xad, 0xbe, 0xef];
1738 // A record header truncated before the handshake-message byte — unclassifiable, so not an
1739 // opening handshake.
1740 let truncated = vec![0x16, 0x03, 0x03, 0x00, 0x05];
1741 // Not TLS at all.
1742 let garbage = vec![0x00, 0x01, 0x02, 0x03, 0x04, 0x05];
1743
1744 for (label, frame) in [
1745 ("ServerHello", server_hello),
1746 ("application record", app_data),
1747 ("truncated record", truncated),
1748 ("garbage", garbage),
1749 ] {
1750 status.route_relayed("ffbb", frame);
1751 assert!(
1752 !status.open_tunnel_exists("ffbb"),
1753 "a {label} frame must not register a server-role circuit"
1754 );
1755 assert!(
1756 accept_rx.try_recv().is_err(),
1757 "a {label} frame must not surface an accept"
1758 );
1759 }
1760
1761 // CONTROL: the peer's genuine ClientHello still opens the circuit — the responder path is
1762 // intact and the earlier drops left no per-peer state behind.
1763 status.route_relayed("ffbb", client_hello_frame());
1764 assert!(
1765 status.open_tunnel_exists("ffbb"),
1766 "a genuine ClientHello still opens an introduced circuit"
1767 );
1768 assert!(
1769 accept_rx.try_recv().is_ok(),
1770 "a genuine ClientHello still surfaces an accept"
1771 );
1772 }
1773}