use anyhow::{Context, Result};
use bytes::Bytes;
use dashmap::DashMap;
use std::os::unix::fs::FileTypeExt;
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use tokio::net::UnixStream;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use crate::transports::transport::{
AdmissionError, AdmissionGate, HealthCheckError, SendOutcome, ShutdownState, TransportError,
TransportErrorHandler,
};
use velo_ext::{MessageType, PeerInfo, Transport, TransportAdapter, TransportKey, WorkerAddress};
use crate::transports::tcp::framing::DEFAULT_MAX_FRAME_SIZE;
use super::listener::{UdsListener, default_shrink_threshold};
use crate::transports::coalesce::{
Coalescable, WriterFailure, WriterObserver, run_coalescing_writer,
};
use crate::transports::ingress::{DialedReaderContext, run_dialed_reader};
pub struct UdsTransport {
key: TransportKey,
socket_path: PathBuf,
local_address: WorkerAddress,
peers: Arc<DashMap<crate::InstanceId, PathBuf>>,
connections: Arc<DashMap<crate::InstanceId, ConnectionHandle>>,
runtime: OnceLock<tokio::runtime::Handle>,
cancel_token: CancellationToken,
shutdown_state: OnceLock<ShutdownState>,
channel_capacity: usize,
connect_timeout: Duration,
metrics: OnceLock<std::sync::Arc<dyn velo_ext::TransportObservability>>,
shrink_threshold: usize,
dialed_ctx: OnceLock<DialedReaderContext>,
}
#[derive(Clone)]
struct ConnectionHandle {
tx: flume::Sender<SendTask>,
gate: AdmissionGate<SendTask>,
}
impl ConnectionHandle {
fn retire(&self) {
self.gate.fail_all(AdmissionError::ConnectionReplaced);
}
}
struct SendTask {
msg_type: MessageType,
header: Bytes,
payload: Bytes,
on_error: Arc<dyn TransportErrorHandler>,
}
impl SendTask {
fn on_error(self, error: impl Into<String>) {
self.on_error
.on_error(self.header, self.payload, error.into());
}
}
impl UdsTransport {
pub fn new(
socket_path: PathBuf,
key: TransportKey,
local_address: WorkerAddress,
channel_capacity: usize,
connect_timeout: Duration,
) -> Self {
Self {
key,
socket_path,
local_address,
peers: Arc::new(DashMap::new()),
connections: Arc::new(DashMap::new()),
runtime: OnceLock::new(),
cancel_token: CancellationToken::new(),
shutdown_state: OnceLock::new(),
channel_capacity,
connect_timeout,
metrics: OnceLock::new(),
shrink_threshold: default_shrink_threshold(),
dialed_ctx: OnceLock::new(),
}
}
pub fn socket_path(&self) -> &Path {
&self.socket_path
}
pub fn ensure_connected(&self, instance_id: crate::InstanceId) -> Result<()> {
self.get_or_create_connection(instance_id)?;
Ok(())
}
fn reap_stale_connection(&self, instance_id: crate::InstanceId) {
if let Some((_, stale)) = self
.connections
.remove_if(&instance_id, |_, h| h.tx.is_disconnected())
{
stale.retire();
self.update_connection_gauge();
}
}
fn get_or_create_connection(&self, instance_id: crate::InstanceId) -> Result<ConnectionHandle> {
if let Some(handle) = self.connections.get(&instance_id) {
if !handle.tx.is_disconnected() {
return Ok(handle.clone());
}
drop(handle);
self.reap_stale_connection(instance_id);
}
let rt = self.runtime.get().ok_or(TransportError::NotStarted)?;
let handle = match self.connections.entry(instance_id) {
dashmap::mapref::entry::Entry::Occupied(mut entry) => {
if !entry.get().tx.is_disconnected() {
entry.get().clone()
} else {
entry.get().retire();
let handle = self.create_connection(instance_id, rt)?;
entry.insert(handle.clone());
self.update_connection_gauge();
handle
}
}
dashmap::mapref::entry::Entry::Vacant(entry) => {
let handle = self.create_connection(instance_id, rt)?;
entry.insert(handle.clone());
self.update_connection_gauge();
handle
}
};
Ok(handle)
}
fn create_connection(
&self,
instance_id: crate::InstanceId,
rt: &tokio::runtime::Handle,
) -> Result<ConnectionHandle> {
let path = self
.peers
.get(&instance_id)
.ok_or(TransportError::PeerNotRegistered(instance_id))?
.value()
.clone();
let (tx, rx) = flume::bounded(self.channel_capacity);
let handle = ConnectionHandle {
gate: AdmissionGate::new(tx.clone(), rt.clone()),
tx,
};
debug!("Created new UDS connection to {} ({:?})", instance_id, path);
rt.spawn(connection_writer_task(
path,
instance_id,
rx,
WriterTaskContext {
connections: Arc::clone(&self.connections),
cancel_token: self.cancel_token.clone(),
connect_timeout: self.connect_timeout,
reader_ctx: self.dialed_ctx.get().cloned(),
metrics: self.metrics.get().cloned(),
},
));
Ok(handle)
}
fn update_peer_gauge(&self) {
if let Some(metrics) = self.metrics.get() {
metrics.set_registered_peers(self.peers.len());
}
}
fn update_connection_gauge(&self) {
if let Some(metrics) = self.metrics.get() {
metrics.set_active_connections(self.connections.len());
}
}
fn slow_path_send(&self, instance_id: crate::InstanceId, send_msg: SendTask) -> SendOutcome {
if self.runtime.get().is_none() {
send_msg.on_error("Transport not started");
return SendOutcome::Admitted;
}
let handle = match self.get_or_create_connection(instance_id) {
Ok(h) => h,
Err(e) => {
send_msg.on_error(format!("Failed to create connection: {}", e));
return SendOutcome::Admitted;
}
};
self.admit(&handle, send_msg)
}
fn admit(&self, handle: &ConnectionHandle, send_msg: SendTask) -> SendOutcome {
let outcome = handle.gate.send(send_msg);
if let Some(m) = self.metrics.get()
&& !outcome.is_admitted()
{
m.record_send_backpressure();
}
outcome
}
}
impl Transport for UdsTransport {
fn key(&self) -> TransportKey {
self.key.clone()
}
fn address(&self) -> WorkerAddress {
self.local_address.clone()
}
fn max_message_size(&self, _target: crate::InstanceId) -> Option<usize> {
Some(DEFAULT_MAX_FRAME_SIZE as usize)
}
fn register(&self, peer_info: PeerInfo) -> Result<(), TransportError> {
let endpoint = peer_info
.worker_address()
.get_entry(&self.key)
.map_err(|_| TransportError::NoEndpoint)?
.ok_or(TransportError::NoEndpoint)?;
let path = parse_uds_endpoint(&endpoint).map_err(|e| {
error!("Failed to parse UDS endpoint: {}", e);
TransportError::InvalidEndpoint
})?;
match std::fs::metadata(&path) {
Ok(m) if m.file_type().is_socket() => {}
Ok(_) => {
debug!(
"UDS path {:?} exists but is not a socket; rejecting UDS for peer {}",
path,
peer_info.instance_id()
);
return Err(TransportError::NoEndpoint);
}
Err(_) => {
debug!(
"UDS path {:?} not visible on this host; rejecting UDS for peer {}",
path,
peer_info.instance_id()
);
return Err(TransportError::NoEndpoint);
}
}
self.peers.insert(peer_info.instance_id(), path.clone());
self.update_peer_gauge();
debug!("Registered peer {} at {:?}", peer_info.instance_id(), path);
Ok(())
}
#[inline]
fn send_message(
&self,
instance_id: crate::InstanceId,
header: Bytes,
payload: Bytes,
message_type: MessageType,
on_error: Arc<dyn TransportErrorHandler>,
) -> SendOutcome {
let send_msg = SendTask {
msg_type: message_type,
header,
payload,
on_error,
};
if let Some(handle) = self.connections.get(&instance_id) {
let live = (!handle.tx.is_disconnected()).then(|| handle.clone());
drop(handle);
match live {
Some(handle) => return self.admit(&handle, send_msg),
None => self.reap_stale_connection(instance_id),
}
}
self.slow_path_send(instance_id, send_msg)
}
fn start(
&self,
_instance_id: crate::InstanceId,
channels: TransportAdapter,
rt: tokio::runtime::Handle,
) -> futures::future::BoxFuture<'_, anyhow::Result<()>> {
self.dialed_ctx
.set(DialedReaderContext {
adapter: channels.clone(),
error_handler: Arc::new(DialedReaderErrorHandler),
transport_key: self.key.as_str().to_string(),
shrink_threshold: self.shrink_threshold,
})
.ok();
self.runtime.set(rt.clone()).ok();
self.shutdown_state
.set(channels.shutdown_state.clone())
.ok();
let socket_path = self.socket_path.clone();
let shutdown_state = channels.shutdown_state.clone();
Box::pin(async move {
struct DefaultErrorHandler;
impl TransportErrorHandler for DefaultErrorHandler {
fn on_error(&self, _header: Bytes, _payload: Bytes, error: String) {
warn!("UDS transport error: {}", error);
}
}
if socket_path.exists() {
let is_socket = std::fs::metadata(&socket_path)
.map(|m| m.file_type().is_socket())
.unwrap_or(false);
if !is_socket {
anyhow::bail!(
"path {:?} exists and is not a Unix domain socket",
socket_path
);
}
match tokio::time::timeout(
Duration::from_millis(100),
UnixStream::connect(&socket_path),
)
.await
{
Ok(Ok(_)) => {
anyhow::bail!(
"a live UDS listener is already running at {:?}",
socket_path
);
}
_ => {
std::fs::remove_file(&socket_path).ok();
}
}
}
let uds_listener = UdsListener::builder()
.socket_path(socket_path.clone())
.adapter(channels)
.error_handler(Arc::new(DefaultErrorHandler))
.shutdown_state(shutdown_state)
.transport_key(self.key.as_str())
.metrics(self.metrics.get().cloned())
.shrink_threshold(self.shrink_threshold)
.build()?;
let bound_listener = uds_listener.bind()?;
rt.spawn(async move {
if let Err(e) = bound_listener.serve().await {
error!("UDS listener error: {}", e);
}
});
info!("UDS transport started on {:?}", socket_path);
Ok(())
})
}
fn begin_drain(&self) {
}
fn shutdown(&self) {
info!("Shutting down UDS transport");
if let Some(state) = self.shutdown_state.get() {
state.teardown_token().cancel();
}
self.cancel_token.cancel();
self.connections.clear();
self.update_connection_gauge();
}
fn set_observability(
&self,
observability: std::sync::Arc<dyn velo_ext::TransportObservability>,
) {
let _ = self.metrics.set(observability);
self.update_peer_gauge();
self.update_connection_gauge();
}
fn check_health(
&self,
instance_id: crate::InstanceId,
timeout: Duration,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = Result<(), HealthCheckError>> + Send + '_>,
> {
Box::pin(async move {
let connection_exists = self.connections.contains_key(&instance_id);
if let Some(handle) = self.connections.get(&instance_id) {
if !handle.tx.is_disconnected() {
return Ok(());
}
drop(handle);
self.reap_stale_connection(instance_id);
}
let path = self
.peers
.get(&instance_id)
.ok_or(HealthCheckError::PeerNotRegistered)?
.value()
.clone();
match tokio::time::timeout(timeout, UnixStream::connect(&path)).await {
Ok(Ok(_stream)) => {
if connection_exists {
Ok(())
} else {
Err(HealthCheckError::NeverConnected)
}
}
Ok(Err(_)) => Err(HealthCheckError::ConnectionFailed),
Err(_) => Err(HealthCheckError::Timeout),
}
})
}
}
struct WriterTaskContext {
connections: Arc<DashMap<crate::InstanceId, ConnectionHandle>>,
cancel_token: CancellationToken,
connect_timeout: Duration,
reader_ctx: Option<DialedReaderContext>,
metrics: Option<std::sync::Arc<dyn velo_ext::TransportObservability>>,
}
async fn connection_writer_task(
path: PathBuf,
instance_id: crate::InstanceId,
rx: flume::Receiver<SendTask>,
ctx: WriterTaskContext,
) -> Result<()> {
let WriterTaskContext {
connections,
cancel_token,
connect_timeout,
reader_ctx,
metrics,
} = ctx;
let result = connection_writer_inner(
&path,
instance_id,
&rx,
&cancel_token,
connect_timeout,
reader_ctx,
metrics.clone(),
)
.await;
while let Ok(msg) = rx.try_recv() {
msg.on_error("Connection closed");
}
drop(rx);
if let Some((_, stale)) = connections.remove_if(&instance_id, |_, h| h.tx.is_disconnected()) {
stale.retire();
}
if let Some(metrics) = metrics.as_ref() {
metrics.set_active_connections(connections.len());
}
debug!("UDS connection to {} ({:?}) closed", instance_id, path);
result
}
async fn connection_writer_inner(
path: &Path,
instance_id: crate::InstanceId,
rx: &flume::Receiver<SendTask>,
cancel_token: &CancellationToken,
connect_timeout: Duration,
reader_ctx: Option<DialedReaderContext>,
metrics: Option<std::sync::Arc<dyn velo_ext::TransportObservability>>,
) -> Result<()> {
debug!("Connecting to UDS {:?}", path);
let stream = tokio::select! {
_ = cancel_token.cancelled() => return Ok(()),
res = tokio::time::timeout(connect_timeout, UnixStream::connect(path)) => {
res.context("UDS connect timeout")?.context("UDS connect failed")?
},
};
let sock = socket2::SockRef::from(&stream);
if let Err(e) = sock.set_send_buffer_size(2_097_152) {
warn!("Failed to set UDS send buffer size: {}", e);
}
if let Err(e) = sock.set_recv_buffer_size(2_097_152) {
warn!("Failed to set UDS recv buffer size: {}", e);
}
debug!("Connected to UDS {:?}", path);
let (read_half, mut write_half) = stream.into_split();
let conn_cancel = cancel_token.child_token();
let reader = reader_ctx.map(|ctx| {
tokio::spawn(run_dialed_reader(
read_half,
ctx,
metrics,
conn_cancel.clone(),
format!("{} ({:?})", instance_id, path),
))
});
run_coalescing_writer(
&mut write_half,
rx,
std::convert::identity,
Some(&conn_cancel),
&UdsWriterObserver { instance_id, path },
)
.await;
if let Some(reader) = reader {
reader.abort();
}
Ok(())
}
struct DialedReaderErrorHandler;
impl TransportErrorHandler for DialedReaderErrorHandler {
fn on_error(&self, _header: Bytes, _payload: Bytes, error: String) {
warn!("UDS transport error: {}", error);
}
}
impl Coalescable for SendTask {
type FailureToken = Self;
fn msg_type(&self) -> MessageType {
self.msg_type
}
fn header(&self) -> &[u8] {
&self.header
}
fn payload(&self) -> &[u8] {
&self.payload
}
fn into_failure_token(self) -> Self {
self
}
fn fail(token: Self, reason: &str) {
token.on_error(format!("Failed to write to UDS stream: {}", reason));
}
}
struct UdsWriterObserver<'a> {
instance_id: crate::InstanceId,
path: &'a Path,
}
impl WriterObserver for UdsWriterObserver<'_> {
fn on_failure(&self, kind: WriterFailure, err: &std::io::Error, frames: usize) {
match kind {
WriterFailure::Write => error!(
"Write error to {} ({:?}): {} ({} message(s) in batch)",
self.instance_id, self.path, err, frames
),
WriterFailure::Encode => error!(
"Encode error to {} ({:?}): {}",
self.instance_id, self.path, err
),
}
}
}
fn parse_uds_endpoint(endpoint: &[u8]) -> Result<PathBuf> {
let endpoint_str = std::str::from_utf8(endpoint).context("endpoint is not valid UTF-8")?;
let path_str = endpoint_str.strip_prefix("uds://").unwrap_or(endpoint_str);
if path_str.is_empty() {
anyhow::bail!("empty UDS socket path");
}
Ok(PathBuf::from(path_str))
}
pub struct UdsTransportBuilder {
socket_path: Option<PathBuf>,
key: Option<TransportKey>,
channel_capacity: usize,
connect_timeout: Duration,
shrink_threshold: Option<usize>,
}
impl UdsTransportBuilder {
pub fn new() -> Self {
Self {
socket_path: None,
key: None,
channel_capacity: 256,
connect_timeout: Duration::from_secs(5),
shrink_threshold: None,
}
}
pub fn socket_path(mut self, path: impl Into<PathBuf>) -> Self {
self.socket_path = Some(path.into());
self
}
pub fn key(mut self, key: TransportKey) -> Self {
self.key = Some(key);
self
}
pub fn channel_capacity(mut self, capacity: usize) -> Self {
self.channel_capacity = capacity;
self
}
pub fn connect_timeout(mut self, timeout: Duration) -> Self {
self.connect_timeout = timeout;
self
}
pub fn shrink_threshold(mut self, bytes: usize) -> Self {
self.shrink_threshold = Some(bytes);
self
}
pub fn build(self) -> Result<UdsTransport> {
let socket_path = self
.socket_path
.ok_or_else(|| anyhow::anyhow!("socket_path is required"))?;
let key = self.key.unwrap_or_else(|| TransportKey::from("uds"));
let local_endpoint = format!("uds://{}", socket_path.display());
let mut addr_builder = crate::transports::address::WorkerAddressBuilder::new();
addr_builder.add_entry(key.clone(), local_endpoint.as_bytes().to_vec())?;
let local_address = addr_builder.build()?;
let mut transport = UdsTransport::new(
socket_path,
key,
local_address,
self.channel_capacity,
self.connect_timeout,
);
if let Some(t) = self.shrink_threshold {
transport.shrink_threshold = t;
}
Ok(transport)
}
}
impl Default for UdsTransportBuilder {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
#[path = "tests.rs"]
mod tests;