use std::collections::HashSet;
use std::future::Future;
use std::net::{IpAddr, SocketAddr};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use iroh::PublicKey;
use iroh_base::CustomAddr;
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
use tokio::net::{TcpStream, lookup_host};
use tokio::sync::mpsc;
use tokio::time::{sleep, timeout};
use tokio_rustls::client::TlsStream;
use tokio_util::sync::CancellationToken;
use tracing::warn;
use crate::error::{Result, SyncwebError};
use super::RelayConfig;
use super::certs::{self, RelayCerts};
use super::pool;
use super::protocol::{
MAX_BYTE_FIELD, Message, PROTOCOL_NAME, read_data_frame, read_message, write_data_frame, write_message,
};
use super::transport::{Backend, IncomingPacket, Session, SyncthingRelayTransport, own_addr, peer_addr};
const DEFAULT_RELAY_PORT: u16 = 22_067;
const SESSION_QUEUE: usize = 1024;
const RELAY_REFRESH: Duration = Duration::from_mins(30);
const POOL_RETRY: Duration = Duration::from_secs(30);
const CONTROL_IDLE: Duration = Duration::from_mins(1);
const JOIN_BACKOFF: Duration = Duration::from_secs(2);
const MAX_DIAL_ATTEMPTS: usize = 4;
const UNKNOWN_NODE: [u8; 32] = [0_u8; 32];
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct RelayManagerOptions {
pub config: RelayConfig,
pub data_dir: PathBuf,
}
impl RelayManagerOptions {
#[must_use]
pub const fn new(config: RelayConfig, data_dir: PathBuf) -> Self {
Self { config, data_dir }
}
}
#[derive(Clone)]
pub struct RelayManager {
backend: Arc<Backend>,
config: RelayConfig,
cancel: CancellationToken,
}
impl std::fmt::Debug for RelayManager {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RelayManager").finish_non_exhaustive()
}
}
impl RelayManager {
pub fn start(options: RelayManagerOptions, own_node: PublicKey) -> Result<Self> {
let certs = Arc::new(RelayCerts::load_or_create(&options.data_dir)?);
let connector = certs::tls_connector(&certs)?;
let backend = Arc::new(Backend::new(certs, connector, options.config.timeout));
backend.update_local_addr(own_addr(&own_node, backend.certs.relay_id()));
let cancel = CancellationToken::new();
let _spawned = tokio::spawn(supervisor(
backend.clone(),
options.config.clone(),
own_node,
cancel.clone(),
));
Ok(Self {
backend,
config: options.config,
cancel,
})
}
#[must_use]
pub fn transport(&self) -> SyncthingRelayTransport {
SyncthingRelayTransport::new(self.backend.clone())
}
#[must_use]
pub fn local_addr(&self) -> CustomAddr {
self.backend.local_addr()
}
#[must_use]
pub fn relay_id(&self) -> [u8; 32] {
*self.backend.certs.relay_id()
}
pub fn remember_node(&self, relay_id: &[u8; 32], node: PublicKey) {
self.backend.remember_node(*relay_id, *node.as_bytes());
}
pub async fn test_relay(&self, url: &str) -> Result<TestRelayReport> {
let started = Instant::now();
let (host, port, token) = parse_relay_url(url)?;
let mut stream = control_connect(&self.backend, url, &host, port).await?;
limited(
self.config.timeout,
write_message(
&mut stream,
&Message::JoinRelayRequest {
token: token.unwrap_or_default(),
},
),
)
.await?;
let reply = limited(self.config.timeout, read_message(&mut stream)).await?;
match &reply {
Message::Response {
code: Message::RESPONSE_SUCCESS | Message::RESPONSE_ALREADY_CONNECTED,
..
} => {}
Message::Response { code, message } => {
return Err(SyncwebError::RelayUnreachable {
reasons: format!("{url} rejected our registration (code {code}): {message}"),
});
}
Message::RelayFull => {
return Err(SyncwebError::RelayUnreachable {
reasons: format!("{url} is full"),
});
}
Message::Ping
| Message::Pong
| Message::JoinRelayRequest { .. }
| Message::JoinSessionRequest { .. }
| Message::ConnectRequest { .. }
| Message::SessionInvitation { .. } => {
return Err(unexpected_reply(&reply, "unexpected reply to join request"));
}
}
Ok(TestRelayReport {
url: url.to_owned(),
own_relay_id: *self.backend.certs.relay_id(),
elapsed: started.elapsed(),
})
}
#[doc(hidden)]
pub fn dial_peer_for_test(&self, peer: [u8; 32], node: PublicKey) -> Result<()> {
dial_peer(&self.backend, peer, *node.as_bytes())
}
pub async fn shutdown(&self) {
self.cancel.cancel();
self.backend.cancel.cancel();
sleep(Duration::from_millis(50)).await;
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub struct TestRelayReport {
pub url: String,
pub own_relay_id: [u8; 32],
pub elapsed: Duration,
}
impl std::fmt::Display for TestRelayReport {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"relay {} reachable in {:?} (device {})",
self.url,
self.elapsed,
device_id(self.own_relay_id)
)
}
}
fn unexpected_reply(reply: &Message, context: &str) -> SyncwebError {
SyncwebError::RelayDecode(format!("{context}: {reply:?}"))
}
pub(crate) fn start_peer_dial(backend: Arc<Backend>, peer: [u8; 32], node: [u8; 32]) {
if !backend.dials.lock().insert(peer) {
return;
}
tokio::spawn(async move {
let result = dial_peer(&backend, peer, node);
if result.is_ok() {
for _ in 0..150 {
if backend.sessions.read().contains_key(&peer) {
break;
}
sleep(Duration::from_millis(100)).await;
}
}
backend.dials.lock().remove(&peer);
if let Err(error) = result {
warn!(%error, "relay dial to peer failed");
}
});
}
fn dial_peer(backend: &Arc<Backend>, peer: [u8; 32], node: [u8; 32]) -> Result<()> {
if peer == [0_u8; 32] {
return Err(SyncwebError::RelayDecode("peer device id is empty".to_owned()));
}
backend.remember_node(peer, node);
let mut failures = Vec::new();
for _ in 0..MAX_DIAL_ATTEMPTS {
let Some(url) = backend.next_relay() else {
failures.push("no relays configured".to_owned());
break;
};
let sender = backend.control.read().get(&url).cloned();
match sender {
Some(control) => match control.send(Message::ConnectRequest { id: peer.to_vec() }) {
Ok(()) => return Ok(()),
Err(_) => failures.push(format!("{url}: registration connection closed")),
},
None => failures.push(format!("{url}: not registered")),
}
}
Err(SyncwebError::RelayUnreachable {
reasons: failures.join("; "),
})
}
struct SessionSpec {
key: Vec<u8>,
address: Vec<u8>,
port: u16,
}
async fn join_loop(backend: Arc<Backend>, url: String, cancel: CancellationToken) {
loop {
if cancel.is_cancelled() {
return;
}
let (host, port, token) = match parse_relay_url(&url) {
Ok(parts) => parts,
Err(error) => {
warn!(%url, %error, "invalid relay url");
return;
}
};
if let Err(error) = serve_registration(&backend, &url, &host, port, token).await {
warn!(%url, %error, "relay registration dropped");
}
sleep_with_cancel(cancel.clone(), JOIN_BACKOFF).await;
}
}
async fn serve_registration(
backend: &Arc<Backend>,
url: &str,
host: &str,
port: u16,
token: Option<String>,
) -> Result<()> {
let mut stream = control_connect(backend, url, host, port).await?;
limited(
backend.timeout,
write_message(
&mut stream,
&Message::JoinRelayRequest {
token: token.unwrap_or_default(),
},
),
)
.await?;
let reply = limited(backend.timeout, read_message(&mut stream)).await?;
match &reply {
Message::Response {
code: Message::RESPONSE_SUCCESS | Message::RESPONSE_ALREADY_CONNECTED,
..
} => {}
Message::Response { code, message } => {
return Err(SyncwebError::RelayUnreachable {
reasons: format!("{url} rejected registration (code {code}): {message}"),
});
}
Message::RelayFull => {
return Err(SyncwebError::RelayUnreachable {
reasons: format!("{url} is full"),
});
}
Message::Ping
| Message::Pong
| Message::JoinRelayRequest { .. }
| Message::JoinSessionRequest { .. }
| Message::ConnectRequest { .. }
| Message::SessionInvitation { .. } => {
return Err(unexpected_reply(&reply, "unexpected reply to join"));
}
}
let (mut reader, mut writer) = tokio::io::split(stream);
let (tx, mut rx) = mpsc::unbounded_channel::<Message>();
backend.control.write().insert(url.to_owned(), tx.clone());
let control_tx = tx.clone();
let writer_tx = tx;
let writer_backend = backend.clone();
let writer_url = url.to_owned();
let writer_cancel = backend.cancel.clone();
tokio::spawn(async move {
loop {
tokio::select! {
() = writer_cancel.cancelled() => break,
incoming = rx.recv() => {
let Some(message) = incoming else { break };
if write_message(&mut writer, &message).await.is_err() {
break;
}
}
}
}
let mut control = writer_backend.control.write();
if control
.get(&writer_url)
.is_some_and(|existing| existing.same_channel(&writer_tx))
{
control.remove(&writer_url);
}
});
loop {
let message = match limited(backend.timeout.max(CONTROL_IDLE), read_message(&mut reader)).await {
Err(error) => return Err(error),
Ok(message) => message,
};
match &message {
Message::Ping => {
let _ = control_tx.send(Message::Pong);
}
Message::Pong | Message::RelayFull => {}
Message::SessionInvitation {
from: inviter,
key,
address,
port: invite_port,
..
} => {
let from = try_into_32(inviter)?;
let node = backend.node_ids.read().get(&from).copied().unwrap_or(UNKNOWN_NODE);
let spec = SessionSpec {
key: key.clone(),
address: address.clone(),
port: *invite_port,
};
tokio::spawn(inbound_session(backend.clone(), url.to_owned(), from, node, spec));
}
Message::JoinRelayRequest { .. }
| Message::JoinSessionRequest { .. }
| Message::Response { .. }
| Message::ConnectRequest { .. } => {
return Err(unexpected_reply(
&message,
"unexpected message on registration connection",
));
}
}
}
}
async fn inbound_session(backend: Arc<Backend>, url: String, peer: [u8; 32], node: [u8; 32], spec: SessionSpec) {
if let Err(error) = establish_session(&backend, &url, peer, node, spec).await {
warn!(%url, %error, "inbound session failed");
}
}
async fn establish_session(
backend: &Arc<Backend>,
url: &str,
peer: [u8; 32],
node: [u8; 32],
spec: SessionSpec,
) -> Result<()> {
let addrs = session_addrs(url, spec.address.as_slice(), spec.port).await?;
let mut last_error = None;
for addr in addrs {
match join_session(backend, url, peer, node, addr, &spec.key).await {
Ok(()) => return Ok(()),
Err(error) => last_error = Some(error),
}
}
Err(last_error.unwrap_or_else(|| SyncwebError::RelayUnreachable {
reasons: format!("{url}: no session address was usable"),
}))
}
async fn join_session(
backend: &Arc<Backend>,
url: &str,
peer: [u8; 32],
node: [u8; 32],
addr: SocketAddr,
key: &[u8],
) -> Result<()> {
if key.len() > MAX_BYTE_FIELD {
return Err(SyncwebError::RelayDecode(
"session key exceeds the protocol limit".to_owned(),
));
}
let mut stream = limited_io(url, backend.timeout, TcpStream::connect(addr)).await?;
limited(
backend.timeout,
write_message(&mut stream, &Message::JoinSessionRequest { key: key.to_vec() }),
)
.await?;
let reply = limited(backend.timeout, read_message(&mut stream)).await?;
match &reply {
Message::Response {
code: Message::RESPONSE_SUCCESS,
..
} => {}
Message::Response { code, message } => {
return Err(SyncwebError::RelayUnreachable {
reasons: format!("{url}: session join rejected (code {code}): {message}"),
});
}
Message::Ping
| Message::Pong
| Message::JoinRelayRequest { .. }
| Message::JoinSessionRequest { .. }
| Message::ConnectRequest { .. }
| Message::SessionInvitation { .. }
| Message::RelayFull => {
return Err(unexpected_reply(&reply, "unexpected reply to session join"));
}
}
register_session(backend, peer, node, stream);
Ok(())
}
fn register_session(backend: &Arc<Backend>, peer: [u8; 32], node: [u8; 32], stream: TcpStream) {
let (read_half, write_half) = stream.into_split();
let (tx, rx) = mpsc::channel::<bytes::Bytes>(SESSION_QUEUE);
let cancel = backend.cancel.clone();
let session = Arc::new(Session {
peer,
peer_node: node,
queue: tx,
cancel: cancel.clone(),
});
{
let mut sessions = backend.sessions.write();
if let Some(previous) = sessions.insert(peer, session.clone()) {
previous.cancel.cancel();
}
}
tokio::spawn(run_session_reader(backend.clone(), session.clone(), read_half));
tokio::spawn(run_session_writer(backend.clone(), session, write_half, rx, cancel));
}
async fn run_session_reader(backend: Arc<Backend>, session: Arc<Session>, mut read_half: OwnedReadHalf) {
let remote = peer_addr(session.peer, session.peer_node);
let local = backend.local_addr();
while let Ok(frame) = read_data_frame(&mut read_half).await {
let Some(tx) = backend.ingress.lock().clone() else {
continue;
};
if tx
.send(IncomingPacket {
remote: remote.clone(),
local: local.clone(),
data: bytes::Bytes::from(frame),
})
.is_err()
{
break;
}
}
remove_session(&backend, session.peer, &session);
}
async fn run_session_writer(
backend: Arc<Backend>,
session: Arc<Session>,
mut write_half: OwnedWriteHalf,
mut rx: mpsc::Receiver<bytes::Bytes>,
cancel: CancellationToken,
) {
loop {
let data = tokio::select! {
() = cancel.cancelled() => break,
data = rx.recv() => match data {
Some(packet) => packet,
None => break,
},
};
if write_data_frame(&mut write_half, &data).await.is_err() {
break;
}
}
remove_session(&backend, session.peer, &session);
}
fn remove_session(backend: &Backend, peer: [u8; 32], session: &Arc<Session>) {
let mut sessions = backend.sessions.write();
if sessions
.get(&peer)
.is_some_and(|existing| Arc::ptr_eq(existing, session))
{
sessions.remove(&peer);
}
}
async fn supervisor(backend: Arc<Backend>, config: RelayConfig, own_node: PublicKey, cancel: CancellationToken) {
let mut running: HashSet<String> = HashSet::new();
let mut pool_seen = false;
loop {
if cancel.is_cancelled() {
return;
}
let urls = match relay_list(&config, pool_seen).await {
Ok(urls) => urls,
Err(error) => {
warn!(%error, "relay list unavailable");
sleep_with_cancel(cancel.clone(), POOL_RETRY).await;
continue;
}
};
pool_seen = true;
let addr = own_addr(&own_node, backend.certs.relay_id());
let stored = urls.clone();
*backend.relays.write() = stored;
backend.update_local_addr(addr);
for url in urls {
if running.insert(url.clone()) {
tokio::spawn(join_loop(backend.clone(), url, cancel.clone()));
}
}
sleep_with_cancel(cancel.clone(), RELAY_REFRESH).await;
}
}
async fn relay_list(config: &RelayConfig, pool_seen: bool) -> Result<Vec<String>> {
if !config.relay_urls.is_empty() {
return Ok(config.relay_urls.clone());
}
if !config.auto_fallback {
return Ok(Vec::new());
}
match pool::fetch_relay_pool(config.timeout.max(Duration::from_secs(5))).await {
Ok(urls) if urls.is_empty() => Err(SyncwebError::RelayUnreachable {
reasons: "relay pool returned no relays".to_owned(),
}),
Ok(urls) => Ok(urls),
Err(error) => {
if !pool_seen {
warn!(%error, "initial relay pool fetch failed");
}
Err(error)
}
}
}
async fn sleep_with_cancel(cancel: CancellationToken, duration: Duration) {
tokio::select! {
() = cancel.cancelled() => {}
() = sleep(duration) => {}
}
}
async fn control_connect(backend: &Backend, url: &str, host: &str, port: u16) -> Result<TlsStream<TcpStream>> {
let socket = resolve(host, port)
.await
.ok_or_else(|| SyncwebError::RelayUnreachable {
reasons: format!("{url}: host {host} did not resolve"),
})?;
let name = certs::server_name_for(host)?;
let tcp = limited_io(url, backend.timeout, TcpStream::connect(socket)).await?;
let tls = limited_io(url, backend.timeout, backend.connector.connect(name, tcp)).await?;
if let Some(named) = tls.get_ref().1.alpn_protocol()
&& named != PROTOCOL_NAME
{
return Err(SyncwebError::RelayUnreachable {
reasons: format!(
"{url}: relay negotiated unexpected ALPN protocol {:?}",
String::from_utf8_lossy(named)
),
});
}
Ok(tls)
}
async fn resolve(host: &str, port: u16) -> Option<SocketAddr> {
lookup_host((host, port)).await.ok()?.next()
}
async fn session_addrs(url: &str, address: &[u8], invite_port: u16) -> Result<Vec<SocketAddr>> {
let port = if invite_port == 0 {
DEFAULT_RELAY_PORT
} else {
invite_port
};
if address.is_empty() || address.iter().all(|byte| *byte == 0) {
let (host, _token, _port) = parse_relay_url(url)?;
let addresses = lookup_host((host.as_str(), port)).await?.collect::<Vec<SocketAddr>>();
if addresses.is_empty() {
return Err(SyncwebError::RelayUnreachable {
reasons: format!("{url}: session host {host} did not resolve"),
});
}
Ok(addresses)
} else {
Ok(vec![SocketAddr::new(ip_from_bytes(address)?, port)])
}
}
fn ip_from_bytes(bytes: &[u8]) -> Result<IpAddr> {
match bytes.len() {
4 => Ok(IpAddr::from(try_into_array::<4>(bytes)?)),
16 => Ok(IpAddr::from(try_into_array::<16>(bytes)?)),
length => Err(SyncwebError::RelayDecode(format!(
"invitation address has invalid length {length}"
))),
}
}
fn try_into_array<const N: usize>(bytes: &[u8]) -> Result<[u8; N]> {
bytes.try_into().map_err(|error| {
SyncwebError::RelayDecode(format!("byte field has unexpected length {}: {error}", bytes.len()))
})
}
fn try_into_32(bytes: &[u8]) -> Result<[u8; 32]> {
try_into_array(bytes)
}
fn parse_relay_url(url: &str) -> Result<(String, u16, Option<String>)> {
let parsed = url::Url::parse(url).map_err(|error| SyncwebError::RelayBadAddress(format!("{url}: {error}")))?;
match parsed.scheme() {
"relay" | "tcp" => {}
_ => return Err(SyncwebError::RelayBadScheme),
}
let host = parsed
.host_str()
.filter(|host| !host.is_empty())
.ok_or_else(|| SyncwebError::RelayBadAddress(url.to_owned()))?;
let port = parsed.port().unwrap_or(DEFAULT_RELAY_PORT);
let token = parsed
.query_pairs()
.find(|(key, _)| key == "token")
.map(|(_, value)| value.into_owned());
Ok((host.to_owned(), port, token))
}
async fn limited<T>(duration: Duration, future: impl Future<Output = Result<T>>) -> Result<T> {
timeout(duration, future).await.map_or_else(
|_| {
Err(SyncwebError::RelayUnreachable {
reasons: "relay operation timed out".to_owned(),
})
},
std::convert::identity,
)
}
async fn limited_io<T>(
context: &str,
duration: Duration,
future: impl Future<Output = std::io::Result<T>>,
) -> Result<T> {
match timeout(duration, future).await {
Err(_) => Err(SyncwebError::RelayUnreachable {
reasons: format!("{context}: operation timed out"),
}),
Ok(Err(error)) => Err(SyncwebError::RelayUnreachable {
reasons: format!("{context}: {error}"),
}),
Ok(Ok(value)) => Ok(value),
}
}
#[must_use]
fn device_id(bytes: [u8; 32]) -> String {
use std::fmt::Write as _;
let mut out = String::with_capacity(64);
for (index, byte) in bytes.iter().enumerate() {
if index == 8 || index == 16 || index == 24 {
out.push('-');
}
let _ = write!(out, "{byte:02X}");
}
out
}