use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use futures_util::stream::SplitSink;
use futures_util::stream::SplitStream;
use futures_util::{SinkExt, StreamExt};
use tokio::net::TcpStream;
use tokio::sync::Mutex;
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::WebSocketStream;
use crate::network::{NetworkProvider, PeerAddr};
use super::{
resolve_dial_addr, FramedStream, Handshake, StreamListener, StreamTransport, TransportError,
WsConfig, PROTOCOL_VERSION,
};
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
pub struct WebSocketTransport<N: NetworkProvider> {
network: Arc<N>,
config: WsConfig,
}
impl<N: NetworkProvider + 'static> WebSocketTransport<N> {
pub fn new(network: Arc<N>, config: WsConfig) -> Self {
Self { network, config }
}
fn local_handshake(&self) -> Handshake {
let identity = self.network.local_identity();
Handshake {
peer_id: identity.id,
capabilities: vec!["ws".to_string(), "binary".to_string()],
protocol_version: PROTOCOL_VERSION,
}
}
fn ws_protocol_config(&self) -> WebSocketConfig {
let mut config = WebSocketConfig::default();
config.max_message_size = Some(self.config.max_message_size);
config.max_frame_size = Some(self.config.max_message_size);
config
}
async fn client_handshake(
ws: &mut WebSocketStream<TcpStream>,
local_hs: &Handshake,
) -> Result<Handshake, TransportError> {
let hs_json = serde_json::to_string(local_hs)
.map_err(|e| TransportError::Serialize(e.to_string()))?;
ws.send(Message::Text(hs_json.into()))
.await
.map_err(|e| TransportError::HandshakeFailed(format!("send handshake: {e}")))?;
let remote_hs = receive_handshake(ws).await?;
if remote_hs.protocol_version != PROTOCOL_VERSION {
return Err(TransportError::VersionMismatch {
local: PROTOCOL_VERSION,
remote: remote_hs.protocol_version,
});
}
Ok(remote_hs)
}
}
async fn receive_handshake(
ws: &mut WebSocketStream<TcpStream>,
) -> Result<Handshake, TransportError> {
match ws.next().await {
Some(Ok(Message::Text(text))) => serde_json::from_str::<Handshake>(&text)
.map_err(|e| TransportError::HandshakeFailed(format!("parse handshake: {e}"))),
Some(Ok(other)) => Err(TransportError::HandshakeFailed(format!(
"expected text frame for handshake, got: {other:?}"
))),
Some(Err(e)) => Err(TransportError::HandshakeFailed(format!(
"receive handshake: {e}"
))),
None => Err(TransportError::HandshakeFailed(
"connection closed before handshake".to_string(),
)),
}
}
impl<N: NetworkProvider + 'static> StreamTransport for WebSocketTransport<N> {
type Stream = WsFramedStream;
async fn connect(&self, addr: &PeerAddr) -> Result<Self::Stream, TransportError> {
let dial_addr = resolve_dial_addr(addr);
tracing::debug!(addr = %dial_addr, port = self.config.port, "ws: dialing peer");
let tcp_stream = self
.network
.dial_tcp(&dial_addr, self.config.port)
.await
.map_err(|e| TransportError::ConnectFailed(format!("dial tcp: {e}")))?;
let ws_url = format!("ws://{dial_addr}:{}/ws", self.config.port);
let ws_config = self.ws_protocol_config();
let (mut ws, _response) =
tokio_tungstenite::client_async_with_config(ws_url, tcp_stream, Some(ws_config))
.await
.map_err(|e| TransportError::ConnectFailed(format!("ws upgrade: {e}")))?;
let local_hs = self.local_handshake();
let remote_hs = tokio::time::timeout(HANDSHAKE_TIMEOUT, Self::client_handshake(&mut ws, &local_hs))
.await
.map_err(|_| TransportError::Timeout("handshake timed out".to_string()))??;
tracing::info!(
remote_peer = %remote_hs.peer_id,
remote_version = remote_hs.protocol_version,
"ws: connected"
);
Ok(WsFramedStream::new(
ws,
remote_hs.peer_id,
dial_addr,
self.config.ping_interval,
self.config.pong_timeout,
))
}
async fn listen(&self) -> Result<StreamListener<Self::Stream>, TransportError> {
let port = self.config.port;
tracing::debug!(port, "ws: starting listener");
let mut tcp_listener = self
.network
.listen_tcp(port)
.await
.map_err(|e| TransportError::ListenFailed(format!("listen tcp: {e}")))?;
let (tx, rx) = tokio::sync::mpsc::channel::<WsFramedStream>(64);
let local_hs = self.local_handshake();
let ping_interval = self.config.ping_interval;
let pong_timeout = self.config.pong_timeout;
let ws_config = self.ws_protocol_config();
tokio::spawn(async move {
loop {
match tcp_listener.incoming.recv().await {
Some(incoming) => {
let tx = tx.clone();
let local_hs = local_hs.clone();
let remote_addr = incoming.remote_addr.clone();
let ws_config = ws_config;
tokio::spawn(async move {
let mut ws =
match tokio_tungstenite::accept_async_with_config(
incoming.stream,
Some(ws_config),
)
.await
{
Ok(ws) => ws,
Err(e) => {
tracing::warn!(
remote = %remote_addr,
"ws: upgrade failed: {e}"
);
return;
}
};
let remote_hs = match tokio::time::timeout(
HANDSHAKE_TIMEOUT,
server_handshake_standalone(&mut ws, &local_hs),
)
.await
{
Ok(Ok(hs)) => hs,
Ok(Err(e)) => {
tracing::warn!(
remote = %remote_addr,
"ws: handshake failed: {e}"
);
return;
}
Err(_) => {
tracing::warn!(
remote = %remote_addr,
"ws: handshake timed out"
);
return;
}
};
tracing::info!(
remote_peer = %remote_hs.peer_id,
remote_addr = %remote_addr,
"ws: accepted connection"
);
let stream = WsFramedStream::new(
ws,
remote_hs.peer_id,
remote_addr,
ping_interval,
pong_timeout,
);
if tx.send(stream).await.is_err() {
tracing::debug!("ws: listener channel closed");
}
});
}
None => {
tracing::debug!("ws: tcp listener channel closed");
break;
}
}
}
});
Ok(StreamListener::new(rx, port))
}
}
async fn server_handshake_standalone(
ws: &mut WebSocketStream<TcpStream>,
local_hs: &Handshake,
) -> Result<Handshake, TransportError> {
let remote_hs = receive_handshake(ws).await?;
if remote_hs.protocol_version != PROTOCOL_VERSION {
return Err(TransportError::VersionMismatch {
local: PROTOCOL_VERSION,
remote: remote_hs.protocol_version,
});
}
let hs_json = serde_json::to_string(local_hs)
.map_err(|e| TransportError::Serialize(e.to_string()))?;
ws.send(Message::Text(hs_json.into()))
.await
.map_err(|e| TransportError::HandshakeFailed(format!("send handshake: {e}")))?;
Ok(remote_hs)
}
pub struct WsFramedStream {
write: Arc<Mutex<SplitSink<WebSocketStream<TcpStream>, Message>>>,
read: SplitStream<WebSocketStream<TcpStream>>,
remote_peer_id: String,
remote_addr: String,
heartbeat_handle: Option<tokio::task::JoinHandle<()>>,
last_pong: Arc<AtomicU64>,
closed: Arc<AtomicBool>,
}
impl std::fmt::Debug for WsFramedStream {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WsFramedStream")
.field("remote_peer_id", &self.remote_peer_id)
.field("remote_addr", &self.remote_addr)
.field("closed", &self.closed.load(Ordering::Relaxed))
.finish_non_exhaustive()
}
}
unsafe impl Sync for WsFramedStream {}
fn epoch_millis() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}
impl WsFramedStream {
fn new(
ws: WebSocketStream<TcpStream>,
remote_peer_id: String,
remote_addr: String,
ping_interval: Duration,
pong_timeout: Duration,
) -> Self {
let (write, read) = ws.split();
let write = Arc::new(Mutex::new(write));
let last_pong = Arc::new(AtomicU64::new(epoch_millis()));
let closed = Arc::new(AtomicBool::new(false));
let hb_write = write.clone();
let hb_last_pong = last_pong.clone();
let hb_closed = closed.clone();
let hb_addr = remote_addr.clone();
let heartbeat_handle = tokio::spawn(async move {
heartbeat_loop(hb_write, hb_last_pong, hb_closed, ping_interval, pong_timeout, &hb_addr)
.await;
});
Self {
write,
read,
remote_peer_id,
remote_addr,
heartbeat_handle: Some(heartbeat_handle),
last_pong,
closed,
}
}
pub fn remote_peer_id(&self) -> &str {
&self.remote_peer_id
}
}
async fn heartbeat_loop(
write: Arc<Mutex<SplitSink<WebSocketStream<TcpStream>, Message>>>,
last_pong: Arc<AtomicU64>,
closed: Arc<AtomicBool>,
ping_interval: Duration,
pong_timeout: Duration,
remote_addr: &str,
) {
let mut interval = tokio::time::interval(ping_interval);
interval.tick().await;
loop {
interval.tick().await;
if closed.load(Ordering::Acquire) {
return;
}
let last = last_pong.load(Ordering::Acquire);
let now = epoch_millis();
let elapsed = Duration::from_millis(now.saturating_sub(last));
if elapsed > pong_timeout {
tracing::warn!(
remote = %remote_addr,
elapsed = ?elapsed,
"heartbeat: pong timeout after {pong_timeout:?}"
);
closed.store(true, Ordering::Release);
let mut w = write.lock().await;
let _ = w.close().await;
return;
}
{
let mut w = write.lock().await;
let ping_data = b"truffle-ping".to_vec();
if let Err(e) = w.send(Message::Ping(ping_data.into())).await {
tracing::debug!(remote = %remote_addr, "heartbeat: ping send failed: {e}");
closed.store(true, Ordering::Release);
return;
}
}
}
}
impl FramedStream for WsFramedStream {
async fn send(&mut self, data: &[u8]) -> Result<(), TransportError> {
if self.closed.load(Ordering::Acquire) {
return Err(TransportError::ConnectionClosed(
"connection already closed".to_string(),
));
}
let mut w = self.write.lock().await;
w.send(Message::Binary(data.to_vec().into()))
.await
.map_err(|e| TransportError::WebSocket(format!("send: {e}")))
}
async fn recv(&mut self) -> Result<Option<Vec<u8>>, TransportError> {
if self.closed.load(Ordering::Acquire) {
return Ok(None);
}
loop {
match self.read.next().await {
Some(Ok(Message::Binary(data))) => return Ok(Some(data.to_vec())),
Some(Ok(Message::Text(text))) => {
return Ok(Some(text.as_bytes().to_vec()));
}
Some(Ok(Message::Ping(_))) => {
continue;
}
Some(Ok(Message::Pong(_))) => {
self.last_pong.store(epoch_millis(), Ordering::Release);
continue;
}
Some(Ok(Message::Close(_))) => {
self.closed.store(true, Ordering::Release);
return Ok(None);
}
Some(Ok(Message::Frame(_))) => {
continue;
}
Some(Err(e)) => {
self.closed.store(true, Ordering::Release);
return Err(TransportError::WebSocket(format!("recv: {e}")));
}
None => {
self.closed.store(true, Ordering::Release);
return Ok(None);
}
}
}
}
async fn close(&mut self) -> Result<(), TransportError> {
if let Some(handle) = self.heartbeat_handle.take() {
handle.abort();
}
self.closed.store(true, Ordering::Release);
let mut w = self.write.lock().await;
w.close()
.await
.map_err(|e| TransportError::WebSocket(format!("close: {e}")))
}
fn peer_addr(&self) -> String {
self.remote_addr.clone()
}
}
impl Drop for WsFramedStream {
fn drop(&mut self) {
if let Some(handle) = self.heartbeat_handle.take() {
handle.abort();
}
}
}
#[cfg(test)]
mod unit_tests {
use super::*;
#[test]
fn resolve_dial_addr_prefers_ip() {
let addr = PeerAddr {
ip: Some("100.64.0.1".parse().unwrap()),
hostname: "peer".to_string(),
dns_name: Some("peer.tailnet.ts.net".to_string()),
};
assert_eq!(resolve_dial_addr(&addr), "100.64.0.1");
}
#[test]
fn resolve_dial_addr_falls_back_to_dns() {
let addr = PeerAddr {
ip: None,
hostname: "peer".to_string(),
dns_name: Some("peer.tailnet.ts.net".to_string()),
};
assert_eq!(resolve_dial_addr(&addr), "peer.tailnet.ts.net");
}
#[test]
fn resolve_dial_addr_falls_back_to_hostname() {
let addr = PeerAddr {
ip: None,
hostname: "peer".to_string(),
dns_name: None,
};
assert_eq!(resolve_dial_addr(&addr), "peer");
}
}