mod driver;
mod session;
use std::{
cell::RefCell,
collections::{HashMap, VecDeque},
hash::{BuildHasher, Hasher},
net::{IpAddr, SocketAddr},
os::fd::RawFd,
rc::Rc,
time::Duration,
};
use cares::{
ARecord, AaaaRecord, DnsRecordType, DnsSection, ProactorResolver, options::ResolverOptions,
};
use driver::{
PeerAddr, close_raw, drain_tls, handshake_seed, invalid, open_socket, operation_id, push_wire,
system_servers_csv,
};
use liburing::{
Completion, CompletionDispatcher, OperationId, OperationPayload, Proactor, SubmissionContext,
SubsystemId,
ops::{Timeout, net},
};
use nexora_proto::{
AlpnProtocol, Bytes, BytesMut, Headers, Http1ClientCodec, Method, ProtoError, TlsClientConfig,
TlsSession, Uri, WebSocketCloseCode, WebSocketCloseFrame, WebSocketCodec, WebSocketConfig,
WebSocketHandshake, WebSocketMessage, WebSocketRole,
};
use session::{DnsOutcome, Session, State};
use crate::{
error::{Error, Result},
inbox::{ClientContext, InboxCommand, SubsystemInbox, WebSocketSessionId},
};
const RX_BUFFER_SIZE: usize = 64 * 1024;
const TLS_CHUNK_SIZE: usize = 16 * 1024;
const DNS_BRIDGE_CAPACITY: usize = 8;
const MAX_WRITE_ROUNDS: usize = 64;
const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_MAX_FRAME_SIZE: usize = 1024 * 1024;
const DEFAULT_MAX_MESSAGE_SIZE: usize = 16 * 1024 * 1024;
const DNS_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
const TAG_SHIFT: u64 = 52;
const ID_MASK: u64 = (1 << TAG_SHIFT) - 1;
const TAG_CONNECT: u64 = 1;
const TAG_RECV: u64 = 2;
const TAG_SEND: u64 = 3;
const TAG_CLOSE: u64 = 4;
const TAG_CANCEL: u64 = 5;
const TAG_CONNECT_TIMEOUT: u64 = 6;
const TAG_PING_TIMEOUT: u64 = 7;
type OpenCallback = Box<dyn FnOnce(&mut ClientContext, WebSocketSessionId, Result<()>) + 'static>;
type EventCallback =
Box<dyn FnMut(&mut ClientContext, &mut WebSocketSession, WebSocketEvent) + 'static>;
#[derive(Debug, Clone)]
pub struct WebSocketClientConfig {
pub connect_timeout: Duration,
pub max_frame_size: usize,
pub max_message_size: usize,
pub tls_config: Option<TlsClientConfig>,
pub dns_servers_csv: Option<String>,
pub dns_subsystem: SubsystemId,
pub net_subsystem: SubsystemId,
pub ping_interval: Option<Duration>,
}
impl Default for WebSocketClientConfig {
fn default() -> Self {
Self {
connect_timeout: DEFAULT_CONNECT_TIMEOUT,
max_frame_size: DEFAULT_MAX_FRAME_SIZE,
max_message_size: DEFAULT_MAX_MESSAGE_SIZE,
tls_config: None,
dns_servers_csv: None,
dns_subsystem: SubsystemId::new_unchecked(1),
net_subsystem: SubsystemId::new_unchecked(2),
ping_interval: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WebSocketEvent {
Text(String),
Binary(Bytes),
Ping(Bytes),
Pong(Bytes),
Close(Option<WebSocketCloseFrame>),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WebSocketSession {
id: WebSocketSessionId,
}
impl WebSocketSession {
pub fn id(&self) -> WebSocketSessionId {
self.id
}
pub fn send_text(&mut self, ctx: &mut ClientContext, text: impl Into<String>) {
ctx.websocket_send(self.id, WebSocketMessage::Text(text.into()));
}
pub fn send_binary(&mut self, ctx: &mut ClientContext, data: impl Into<Bytes>) {
ctx.websocket_send(self.id, WebSocketMessage::Binary(data.into()));
}
pub fn ping(&mut self, ctx: &mut ClientContext, payload: impl Into<Bytes>) {
ctx.websocket_send(self.id, WebSocketMessage::Ping(payload.into()));
}
pub fn close(&mut self, ctx: &mut ClientContext, close: Option<WebSocketCloseFrame>) {
ctx.websocket_close(self.id, close);
}
}
pub struct WebSocketClient {
config: WebSocketClientConfig,
inbox: SubsystemInbox,
dns: ProactorResolver,
sessions: HashMap<WebSocketSessionId, Session>,
handlers: HashMap<WebSocketSessionId, EventCallback>,
static_hosts: HashMap<(String, u16), Vec<SocketAddr>>,
dns_waiters: HashMap<u64, WebSocketSessionId>,
dns_outcomes: Rc<RefCell<VecDeque<DnsOutcome>>>,
next_lookup: u64,
}
impl WebSocketClient {
pub fn new(proactor: &mut Proactor, config: WebSocketClientConfig) -> Result<Self> {
if config.dns_subsystem == SubsystemId::INTERNAL
|| config.net_subsystem == SubsystemId::INTERNAL
|| config.dns_subsystem == config.net_subsystem
{
return Err(invalid(
"subsystems must be distinct non-internal identifiers",
));
}
let servers = config
.dns_servers_csv
.clone()
.unwrap_or_else(system_servers_csv);
let options = ResolverOptions::builder()
.tries(1)
.timeout(DNS_QUERY_TIMEOUT)
.query_cache_ttl(Duration::ZERO)
.build()?;
let dns =
ProactorResolver::new(config.dns_subsystem, options, &servers, DNS_BRIDGE_CAPACITY)?;
let mut client = Self {
config,
inbox: SubsystemInbox::new(),
dns,
sessions: HashMap::new(),
handlers: HashMap::new(),
static_hosts: HashMap::new(),
dns_waiters: HashMap::new(),
dns_outcomes: Rc::new(RefCell::new(VecDeque::new())),
next_lookup: 1,
};
client.dns.flush(proactor)?;
Ok(client)
}
pub fn resolve_host(&mut self, host: impl Into<String>, port: u16, ips: Vec<IpAddr>) {
let addrs = ips
.into_iter()
.map(|ip| SocketAddr::new(ip, port))
.collect();
self.static_hosts.insert((host.into(), port), addrs);
}
pub fn connect<F>(
&mut self,
proactor: &mut Proactor,
url: &str,
extra: Headers,
on_open: F,
) -> Result<WebSocketSessionId>
where
F: FnOnce(&mut ClientContext, WebSocketSessionId, Result<()>) + 'static,
{
let uri = Uri::parse(url).map_err(|error| Error::InvalidUri(error.to_string()))?;
if !uri.scheme().is_websocket() {
return Err(Error::InvalidUri(String::from(
"the URL requires a ws or wss scheme",
)));
}
if uri.scheme().is_tls() && self.config.tls_config.is_none() {
return Err(invalid("a wss URL requires a TLS client configuration"));
}
let id = self.inbox.next_websocket_session_id();
let (host, port) = (uri.host().to_owned(), uri.port());
let key = WebSocketHandshake::generate_key(handshake_seed(id));
let mut session = Session::new(id, uri, &self.config, extra, key, Box::new(on_open));
let addr = match self.static_hosts.get(&(host.clone(), port)) {
Some(addrs) => addrs.first().copied(),
None => host
.parse::<IpAddr>()
.ok()
.map(|ip| SocketAddr::new(ip, port)),
};
if let Some(addr) = addr {
session.state = State::Connecting;
self.sessions.insert(id, session);
if let Err(error) = self.begin_connect(proactor, id, addr) {
drop(self.sessions.remove(&id));
return Err(error);
}
return Ok(id);
}
self.sessions.insert(id, session);
if let Err(error) = self.start_dns(proactor, id, &host, port) {
drop(self.sessions.remove(&id));
return Err(error);
}
Ok(id)
}
pub fn set_message_handler<F>(&mut self, session: WebSocketSessionId, on_event: F) -> Result<()>
where
F: FnMut(&mut ClientContext, &mut WebSocketSession, WebSocketEvent) + 'static,
{
if !self.sessions.contains_key(&session) {
return Err(invalid("unknown WebSocket session"));
}
self.handlers.insert(session, Box::new(on_event));
Ok(())
}
pub fn send_text(
&mut self,
proactor: &mut Proactor,
session: WebSocketSessionId,
text: impl Into<String>,
) -> Result<()> {
self.send_message(proactor, session, WebSocketMessage::Text(text.into()))
}
pub fn send_binary(
&mut self,
proactor: &mut Proactor,
session: WebSocketSessionId,
data: impl Into<Bytes>,
) -> Result<()> {
self.send_message(proactor, session, WebSocketMessage::Binary(data.into()))
}
pub fn ping(
&mut self,
proactor: &mut Proactor,
session: WebSocketSessionId,
payload: impl Into<Bytes>,
) -> Result<()> {
self.send_message(proactor, session, WebSocketMessage::Ping(payload.into()))
}
pub fn pong(
&mut self,
proactor: &mut Proactor,
session: WebSocketSessionId,
payload: impl Into<Bytes>,
) -> Result<()> {
self.send_message(proactor, session, WebSocketMessage::Pong(payload.into()))
}
pub fn close(
&mut self,
proactor: &mut Proactor,
session: WebSocketSessionId,
close: Option<WebSocketCloseFrame>,
) -> Result<()> {
if !self.sessions.contains_key(&session) {
return Err(invalid("unknown WebSocket session"));
}
self.request_close(session, close);
self.pump(proactor)
}
pub fn drive(&mut self, proactor: &mut Proactor) -> Result<usize> {
self.drive_with(proactor, &mut crate::dispatch::RejectForeign)
}
pub fn drive_with<D: CompletionDispatcher<Kind> + ?Sized, Kind>(
&mut self,
proactor: &mut Proactor,
peer: &mut D,
) -> Result<usize> {
let mut processed = self.drain_inbox();
processed += self.apply_dns(proactor)?;
self.pump(proactor)?;
self.dns.flush(proactor)?;
proactor.submit()?;
let mut failure: Option<Error> = None;
let harvested = proactor.wait_completions(
1,
Some(Duration::ZERO),
&mut |completion: Completion, submissions: &mut SubmissionContext<'_>| {
if !crate::dispatch::CompletionRoute::accepts(
self,
completion.operation_id.subsystem_id(),
) {
return peer.dispatch(completion, submissions);
}
if let Err(error) = self.on_completion(completion, submissions)
&& failure.is_none()
{
failure = Some(error);
}
Ok(())
},
)?;
processed += harvested;
self.pump(proactor)?;
self.dns.flush(proactor)?;
proactor.submit()?;
match failure {
Some(error) => Err(error),
None => Ok(processed),
}
}
pub fn is_open(&self, session: WebSocketSessionId) -> bool {
self.sessions
.get(&session)
.is_some_and(|owned| owned.state == State::Open && !owned.closing && !owned.teardown)
}
pub fn active_session_count(&self) -> usize {
self.sessions.len()
}
fn send_message(
&mut self,
proactor: &mut Proactor,
session: WebSocketSessionId,
message: WebSocketMessage,
) -> Result<()> {
let Some(mut owned) = self.sessions.remove(&session) else {
return Err(invalid("unknown WebSocket session"));
};
let allowed = owned.state == State::Open && !owned.closing && !owned.teardown;
let result = if allowed {
owned.queue_message(message).map_err(Error::Proto)
} else {
Err(invalid("WebSocket session is not open"))
};
self.sessions.insert(session, owned);
result?;
self.pump(proactor)
}
fn drain_inbox(&mut self) -> usize {
let mut applied = 0;
while let Some(command) = self.inbox.pop() {
applied += 1;
match command {
InboxCommand::WebSocketSend {
session_id,
message,
} => self.queue_message(session_id, message),
InboxCommand::WebSocketClose { session_id, close } => {
self.request_close(session_id, close)
}
InboxCommand::Submit(_)
| InboxCommand::Cancel(_)
| InboxCommand::SetTimer(_)
| InboxCommand::DnsResolved(_)
| InboxCommand::Respond(_) => {}
}
}
applied
}
fn queue_message(&mut self, session: WebSocketSessionId, message: WebSocketMessage) {
let Some(mut owned) = self.sessions.remove(&session) else {
return;
};
if owned.state == State::Open && !owned.closing && !owned.teardown {
let closing = matches!(message, WebSocketMessage::Close(_));
if let Err(error) = owned.queue_message(message) {
self.fail(&mut owned, Error::Proto(error));
} else if closing {
owned.closing = true;
}
}
self.sessions.insert(session, owned);
}
fn request_close(&mut self, session: WebSocketSessionId, close: Option<WebSocketCloseFrame>) {
let Some(mut owned) = self.sessions.remove(&session) else {
return;
};
if !owned.teardown && !owned.closing {
if owned.state == State::Open {
match owned.queue_message(WebSocketMessage::Close(close)) {
Ok(()) => owned.closing = true,
Err(error) => self.fail(&mut owned, Error::Proto(error)),
}
} else {
self.fail(&mut owned, Error::Cancelled);
}
}
self.sessions.insert(session, owned);
}
fn start_dns(
&mut self,
proactor: &mut Proactor,
id: WebSocketSessionId,
host: &str,
port: u16,
) -> Result<()> {
let lookup = self.next_lookup;
self.next_lookup = self.next_lookup.wrapping_add(1);
self.dns_waiters.insert(lookup, id);
if let Some(owned) = self.sessions.get_mut(&id) {
owned.lookup = Some(lookup);
owned.dns_left = 2;
}
let mut handles = Vec::with_capacity(2);
for record_type in [DnsRecordType::A, DnsRecordType::AAAA] {
let outcomes = Rc::clone(&self.dns_outcomes);
let submitted =
self.dns
.resolver
.submit_query(host, record_type, move |_ctx, outcome| {
let mut addrs = Vec::new();
if let Some(record) = outcome.record.as_ref() {
addrs.extend(
record
.records::<ARecord>(DnsSection::Answer)
.unwrap_or_default()
.into_iter()
.map(|a| SocketAddr::new(IpAddr::V4(a.address), port)),
);
addrs.extend(
record
.records::<AaaaRecord>(DnsSection::Answer)
.unwrap_or_default()
.into_iter()
.map(|a| SocketAddr::new(IpAddr::V6(a.address), port)),
);
}
let error = if outcome.status.is_success() {
None
} else {
Some(format!("{:?}", outcome.status))
};
if let Ok(mut queue) = outcomes.try_borrow_mut() {
queue.push_back((lookup, addrs, error));
}
});
match submitted {
Ok(handle) => handles.push(handle),
Err(error) => {
for handle in &handles {
drop(self.dns.resolver.cancel_query(*handle));
}
self.dns_waiters.remove(&lookup);
return Err(Error::Dns(error));
}
}
}
if let Err(error) = self.dns.flush(proactor) {
for handle in &handles {
drop(self.dns.resolver.cancel_query(*handle));
}
self.dns_waiters.remove(&lookup);
return Err(Error::Dns(error));
}
Ok(())
}
fn begin_connect(
&mut self,
proactor: &mut Proactor,
id: WebSocketSessionId,
addr: SocketAddr,
) -> Result<()> {
let Some(mut owned) = self.sessions.remove(&id) else {
return Ok(());
};
let fd = match open_socket(&addr) {
Ok(fd) => fd,
Err(error) => {
self.sessions.insert(id, owned);
return Err(error);
}
};
let peer = Box::new(PeerAddr::new(addr));
let (ptr, len) = peer.raw();
let op = operation_id(self.config.net_subsystem, TAG_CONNECT, id);
if let Err(error) = proactor.prepare_sqe_with(op, |sqe| {
unsafe { net::prep_connect(sqe, fd, ptr, len) }
}) {
close_raw(fd);
self.sessions.insert(id, owned);
return Err(Error::from(error));
}
owned.fd = fd;
owned.peer = Some(peer);
owned.connect_op = Some(op);
owned.inflight += 1;
owned.state = State::Connecting;
if let Ok(timeout) = Timeout::new(self.config.connect_timeout) {
let timer_op = operation_id(self.config.net_subsystem, TAG_CONNECT_TIMEOUT, id);
let staged = proactor.prepare_sqe_with(timer_op, |sqe| {
unsafe { timeout.prepare(sqe) }
});
if staged.is_ok() {
owned.timer = Some(timeout);
owned.timer_op = Some(timer_op);
owned.inflight += 1;
}
}
self.sessions.insert(id, owned);
Ok(())
}
}
impl CompletionDispatcher for WebSocketClient {
fn dispatch(
&mut self,
completion: Completion,
submissions: &mut SubmissionContext<'_>,
) -> liburing::Result<()> {
if !crate::dispatch::CompletionRoute::accepts(self, completion.operation_id.subsystem_id())
{
return Err(liburing::UringError::InvalidSubsystemId(
completion.operation_id.subsystem_id().raw(),
));
}
self.on_completion(completion, submissions)
.map_err(|_| liburing::UringError::InvalidConfig("WebSocket completion failure"))
}
}
impl crate::dispatch::CompletionRoute for WebSocketClient {
fn accepts(&self, subsystem: SubsystemId) -> bool {
subsystem == self.config.net_subsystem || subsystem == self.config.dns_subsystem
}
}
#[cfg(test)]
mod tests {
use super::{driver::decode_op, *};
#[test]
fn completion_payloads_round_trip_session_ids() {
let subsystem = SubsystemId::new_unchecked(2);
let largest = WebSocketSessionId::new((1_u64 << 52) - 1);
let op = operation_id(subsystem, TAG_PING_TIMEOUT, largest);
assert_eq!(op.subsystem_id(), subsystem);
assert_eq!(decode_op(op.payload()), (TAG_PING_TIMEOUT, largest.raw()));
let allocated = WebSocketSessionId::new(0x0000_00FF_FFFF_FFFF);
let recv = operation_id(subsystem, TAG_RECV, allocated);
assert_eq!(decode_op(recv.payload()).1, allocated.raw());
}
}