#![cfg_attr(
feature = "stats",
doc = "- `stats` enables [`ServerStats`] and [`Server::stats`]. Byte counters include successful socket reads and writes; `bytes_consumed` tracks bytes accepted by successful packet decoders. Received-packet counters increase after decoding; sent-packet counters increase after all encoded bytes are written to the socket."
)]
#![cfg_attr(
not(feature = "stats"),
doc = "- `stats` enables `ServerStats` and `Server::stats`. Byte counters include successful socket reads and writes; `bytes_consumed` tracks bytes accepted by successful packet decoders. Received-packet counters increase after decoding; sent-packet counters increase after all encoded bytes are written to the socket."
)]
use std::{
collections::{HashMap, VecDeque},
fmt::Display,
io::{self, Cursor, Read, Write},
net::{Shutdown, SocketAddr, TcpListener as StdTcpListener},
sync::{
Arc, Mutex,
atomic::{AtomicBool, AtomicU64, Ordering},
},
thread,
time::Duration,
};
use crossbeam_channel::{Receiver, Sender, TryRecvError, TrySendError, bounded};
use mio::{
Events, Interest, Poll, Token, Waker,
event::Event,
net::{TcpListener, TcpStream},
};
use socket2::{SockRef, TcpKeepalive};
use crate::packet::encode_bounded;
pub use crate::packet::{PacketIn, PacketOut};
#[derive(Clone, Copy, Debug, PartialEq, Eq, better_default::Default)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
#[cfg_attr(feature = "serde", serde(default))]
pub struct ServerConfig {
#[default(DEFAULT_COMMAND_QUEUE_LIMIT)]
pub command_queue_limit: usize,
#[default(DEFAULT_CONTROL_QUEUE_LIMIT)]
pub control_queue_limit: usize,
#[default(DEFAULT_EVENT_QUEUE_LIMIT)]
pub event_queue_limit: usize,
#[default(DEFAULT_MAX_CLIENTS)]
pub max_clients: usize,
#[default(true)]
pub nodelay: bool,
#[default(Some(DEFAULT_KEEPALIVE_TIME))]
pub keepalive_time: Option<Duration>,
#[default(Some(DEFAULT_IDLE_TIMEOUT))]
pub idle_timeout: Option<Duration>,
#[default(Some(DEFAULT_PACKET_ASSEMBLY_TIMEOUT))]
pub packet_assembly_timeout: Option<Duration>,
#[default(DEFAULT_INCOMING_BUFFER_INITIAL)]
pub incoming_buffer_initial: usize,
#[default(DEFAULT_INCOMING_BUFFER_LIMIT)]
pub incoming_buffer_limit: usize,
#[default(DEFAULT_OUTGOING_BUFFER_LIMIT)]
pub outgoing_buffer_limit: usize,
#[default(DEFAULT_GLOBAL_OUTGOING_BUFFER_LIMIT)]
pub global_outgoing_buffer_limit: usize,
#[default(DEFAULT_OUTGOING_PACKET_QUEUE_LIMIT)]
pub outgoing_packet_queue_limit: usize,
#[default(DEFAULT_ACCEPT_BUDGET)]
pub accept_budget: usize,
#[default(DEFAULT_CONTROL_COMMAND_BUDGET)]
pub control_command_budget: usize,
#[default(DEFAULT_PACKET_COMMAND_BUDGET)]
pub packet_command_budget: usize,
#[default(DEFAULT_PACKET_DECODE_BUDGET)]
pub packet_decode_budget: usize,
#[default(DEFAULT_READ_BYTE_BUDGET)]
pub read_byte_budget: usize,
#[default(DEFAULT_WRITE_BYTE_BUDGET)]
pub write_byte_budget: usize,
}
impl ServerConfig {
fn validate(self) -> io::Result<Self> {
let nonzero = [
(self.command_queue_limit, "command_queue_limit"),
(self.control_queue_limit, "control_queue_limit"),
(self.event_queue_limit, "event_queue_limit"),
(self.max_clients, "max_clients"),
(self.incoming_buffer_initial, "incoming_buffer_initial"),
(self.outgoing_buffer_limit, "outgoing_buffer_limit"),
(
self.global_outgoing_buffer_limit,
"global_outgoing_buffer_limit",
),
(
self.outgoing_packet_queue_limit,
"outgoing_packet_queue_limit",
),
(self.accept_budget, "accept_budget"),
(self.control_command_budget, "control_command_budget"),
(self.packet_command_budget, "packet_command_budget"),
(self.packet_decode_budget, "packet_decode_budget"),
(self.read_byte_budget, "read_byte_budget"),
(self.write_byte_budget, "write_byte_budget"),
];
if let Some((_, name)) = nonzero.into_iter().find(|(value, _)| *value == 0) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("{name} must be greater than zero"),
));
}
if self.incoming_buffer_limit < self.incoming_buffer_initial {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"incoming_buffer_limit must be at least incoming_buffer_initial",
));
}
if self.max_clients == usize::MAX {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"max_clients must be less than usize::MAX",
));
}
for (timeout, name) in [
(self.idle_timeout, "idle_timeout"),
(self.packet_assembly_timeout, "packet_assembly_timeout"),
(self.keepalive_time, "keepalive_time"),
] {
if timeout.is_some_and(|timeout| timeout.is_zero()) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("{name} must be greater than zero when enabled"),
));
}
}
Ok(self)
}
}
pub const DEFAULT_COMMAND_QUEUE_LIMIT: usize = 4096 * 2;
pub const DEFAULT_CONTROL_QUEUE_LIMIT: usize = 1024;
pub const DEFAULT_EVENT_QUEUE_LIMIT: usize = 4096;
pub const DEFAULT_MAX_CLIENTS: usize = 4096;
pub const DEFAULT_KEEPALIVE_TIME: Duration = Duration::from_secs(60);
pub const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
pub const DEFAULT_PACKET_ASSEMBLY_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_INCOMING_BUFFER_INITIAL: usize = 4096;
pub const DEFAULT_INCOMING_BUFFER_LIMIT: usize = 64 * 1024;
pub const DEFAULT_OUTGOING_BUFFER_LIMIT: usize = 1024 * 1024;
pub const DEFAULT_GLOBAL_OUTGOING_BUFFER_LIMIT: usize = 64 * 1024 * 1024;
pub const DEFAULT_OUTGOING_PACKET_QUEUE_LIMIT: usize = 1024;
pub const DEFAULT_ACCEPT_BUDGET: usize = 64;
pub const DEFAULT_CONTROL_COMMAND_BUDGET: usize = 64;
pub const DEFAULT_PACKET_COMMAND_BUDGET: usize = 256;
pub const DEFAULT_PACKET_DECODE_BUDGET: usize = 256;
pub const DEFAULT_READ_BYTE_BUDGET: usize = 1024 * 1024;
pub const DEFAULT_WRITE_BYTE_BUDGET: usize = 1024 * 1024;
const WAKER_TOKEN: Token = Token(usize::MAX - 1);
type ProtocolErrorHook<O> = Arc<dyn Fn(&io::Error) -> O + Send + Sync + 'static>;
#[derive(Clone)]
pub struct Server<I, O> {
packet_commands_sender: Sender<ThreadCommand<O>>,
control_commands_sender: Sender<ControlCommand>,
events_receiver: Receiver<ThreadEvent<I>>,
#[cfg(feature = "stats")]
stats: Arc<SharedStats>,
waker: Arc<Waker>,
shutdown_requested: Arc<AtomicBool>,
accepted_packet_commands: Arc<AtomicU64>,
worker: Arc<WorkerHandle>,
config: ServerConfig,
local_addr: SocketAddr,
}
struct WorkerHandle {
join: Mutex<Option<thread::JoinHandle<()>>>,
}
#[cfg(feature = "stats")]
#[derive(Default)]
struct SharedStats {
bytes_received: AtomicU64,
bytes_consumed: AtomicU64,
bytes_sent: AtomicU64,
packets_sent: AtomicU64,
packets_received: AtomicU64,
}
#[cfg(feature = "stats")]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ServerStats {
pub bytes_received: u64,
pub bytes_consumed: u64,
pub bytes_sent: u64,
pub packets_received: u64,
pub packets_sent: u64,
}
impl<I, O> Server<I, O>
where
I: PacketIn + Send + 'static,
O: PacketOut + Send + 'static,
{
pub fn bind_with_protocol_error(
addr: SocketAddr,
on_protocol_error: impl Fn(&io::Error) -> O + Send + Sync + 'static,
) -> io::Result<Self> {
let hook: ProtocolErrorHook<O> = Arc::new(on_protocol_error);
Self::bind_internal(addr, Some(hook))
}
pub fn bind(addr: SocketAddr) -> io::Result<Self> {
Self::bind_internal(addr, None)
}
pub fn bind_with_config(addr: SocketAddr, config: ServerConfig) -> io::Result<Self> {
Self::bind_internal_with_config(addr, None, config)
}
pub fn bind_with_config_and_protocol_error(
addr: SocketAddr,
config: ServerConfig,
on_protocol_error: impl Fn(&io::Error) -> O + Send + Sync + 'static,
) -> io::Result<Self> {
let hook: ProtocolErrorHook<O> = Arc::new(on_protocol_error);
Self::bind_internal_with_config(addr, Some(hook), config)
}
pub fn from_listener(listener: StdTcpListener) -> io::Result<Self> {
Self::from_listener_internal(listener, None, ServerConfig::default())
}
pub fn from_listener_with_protocol_error(
listener: StdTcpListener,
on_protocol_error: impl Fn(&io::Error) -> O + Send + Sync + 'static,
) -> io::Result<Self> {
let hook: ProtocolErrorHook<O> = Arc::new(on_protocol_error);
Self::from_listener_internal(listener, Some(hook), ServerConfig::default())
}
pub fn from_listener_with_config(
listener: StdTcpListener,
config: ServerConfig,
) -> io::Result<Self> {
Self::from_listener_internal(listener, None, config)
}
pub fn from_listener_with_config_and_protocol_error(
listener: StdTcpListener,
config: ServerConfig,
on_protocol_error: impl Fn(&io::Error) -> O + Send + Sync + 'static,
) -> io::Result<Self> {
let hook: ProtocolErrorHook<O> = Arc::new(on_protocol_error);
Self::from_listener_internal(listener, Some(hook), config)
}
fn bind_internal(
addr: SocketAddr,
on_protocol_error: Option<ProtocolErrorHook<O>>,
) -> io::Result<Self> {
Self::bind_internal_with_config(addr, on_protocol_error, ServerConfig::default())
}
fn bind_internal_with_config(
addr: SocketAddr,
on_protocol_error: Option<ProtocolErrorHook<O>>,
config: ServerConfig,
) -> io::Result<Self> {
let config = config.validate()?;
let listener = TcpListener::bind(addr)?;
Self::start(listener, on_protocol_error, config)
}
fn from_listener_internal(
listener: StdTcpListener,
on_protocol_error: Option<ProtocolErrorHook<O>>,
config: ServerConfig,
) -> io::Result<Self> {
let config = config.validate()?;
listener.set_nonblocking(true)?;
Self::start(TcpListener::from_std(listener), on_protocol_error, config)
}
fn start(
mut listener: TcpListener,
on_protocol_error: Option<ProtocolErrorHook<O>>,
config: ServerConfig,
) -> io::Result<Self> {
let poll = Poll::new()?;
let waker = Arc::new(Waker::new(poll.registry(), WAKER_TOKEN)?);
poll.registry()
.register(&mut listener, LISTENER_TOKEN, Interest::READABLE)?;
let local_addr = listener.local_addr()?;
let (packet_commands_sender, packet_commands_receiver) =
bounded(config.command_queue_limit);
let (control_commands_sender, control_commands_receiver) =
bounded(config.control_queue_limit);
let (events_sender, events_receiver) = bounded(config.event_queue_limit);
#[cfg(feature = "stats")]
let stats = Arc::new(SharedStats::default());
let shutdown_requested = Arc::new(AtomicBool::new(false));
let accepted_packet_commands = Arc::new(AtomicU64::new(0));
#[cfg(feature = "stats")]
let stats_for_poll = Arc::clone(&stats);
let shutdown_for_poll = Arc::clone(&shutdown_requested);
let join = thread::Builder::new()
.name("Packet Poll Thread".to_string())
.spawn(move || {
PollThread::<I, O> {
packet_commands_receiver,
control_commands_receiver,
events_sender,
listener,
poll,
client_ids: ClientIdAllocator::new(),
clients: HashMap::new(),
ready_order: VecDeque::new(),
ready_events: HashMap::new(),
listener_retry_at: None,
control_commands_deferred: false,
packet_commands_deferred: false,
packet_commands_processed: 0,
pending_disconnects: VecDeque::new(),
pending_events: VecDeque::new(),
pending_event_limit: config.max_clients + 1,
total_queued_bytes: 0,
on_protocol_error,
#[cfg(feature = "stats")]
stats: stats_for_poll,
config,
shutdown_requested: shutdown_for_poll,
}
.run();
})?;
let worker = Arc::new(WorkerHandle {
join: Mutex::new(Some(join)),
});
Ok(Self {
packet_commands_sender,
control_commands_sender,
events_receiver,
#[cfg(feature = "stats")]
stats,
waker,
shutdown_requested,
accepted_packet_commands,
worker,
config,
local_addr,
})
}
pub fn poll(&self) -> io::Result<Option<ServerEvent<I>>> {
match self.events_receiver.try_recv() {
Ok(ThreadEvent::Accept { id, peer_addr }) => Ok(Some(ServerEvent::Accept {
client: ServerClient(id),
peer_addr,
})),
Ok(ThreadEvent::Lost { id, error }) => Ok(Some(ServerEvent::Lost {
client: ServerClient(id),
error,
})),
Ok(ThreadEvent::ReceivePacket { id, packet }) => Ok(Some(ServerEvent::Packet {
client: ServerClient(id),
packet,
})),
Ok(ThreadEvent::UnrecoverableError { error }) => Err(error),
Err(TryRecvError::Empty) => Ok(None),
Err(TryRecvError::Disconnected) => Err(new_io_abort_error(
"previous error made this server unusable",
)),
}
}
pub fn send(&self, client: ServerClient, packet: O) -> io::Result<()> {
match self
.packet_commands_sender
.try_send(ThreadCommand::SendPacket {
id: client.0,
packet,
}) {
Ok(()) => {
let _ = self.accepted_packet_commands.fetch_update(
Ordering::Release,
Ordering::Relaxed,
|current| Some(current.saturating_add(1)),
);
self.waker.wake()?;
Ok(())
}
Err(TrySendError::Full(_)) => Err(io::Error::new(
io::ErrorKind::WouldBlock,
"server packet command queue is full",
)),
Err(TrySendError::Disconnected(_)) => {
Err(new_io_abort_error("server command thread is stopped"))
}
}
}
pub fn disconnect(&self, client: ServerClient) -> io::Result<()> {
let packet_command_cutoff = self.accepted_packet_commands.load(Ordering::Acquire);
self.queue_control(ControlCommand::DisconnectClient {
id: client.0,
packet_command_cutoff,
})
}
pub fn disconnect_now(&self, client: ServerClient) -> io::Result<()> {
self.queue_control(ControlCommand::DisconnectClientNow { id: client.0 })
}
pub fn shutdown(&self) -> io::Result<()> {
self.shutdown_requested.store(true, Ordering::Release);
let _ = self
.control_commands_sender
.try_send(ControlCommand::Shutdown);
self.waker.wake()
}
pub fn shutdown_and_wait(&self) -> io::Result<()> {
let wake_result = self.shutdown();
let mut join = self
.worker
.join
.lock()
.map_err(|_| new_io_abort_error("server worker join lock is poisoned"))?;
if let Some(join) = join.take() {
join.join()
.map_err(|_| new_io_abort_error("server worker thread panicked"))?;
}
wake_result
}
pub fn config(&self) -> ServerConfig {
self.config
}
pub fn local_addr(&self) -> SocketAddr {
self.local_addr
}
fn queue_control(&self, command: ControlCommand) -> io::Result<()> {
match self.control_commands_sender.try_send(command) {
Ok(()) => {
self.waker.wake()?;
Ok(())
}
Err(TrySendError::Full(_)) => Err(io::Error::new(
io::ErrorKind::WouldBlock,
"server control command queue is full",
)),
Err(TrySendError::Disconnected(_)) => {
Err(new_io_abort_error("server command thread is stopped"))
}
}
}
#[cfg(feature = "stats")]
pub fn stats(&self) -> ServerStats {
ServerStats {
bytes_received: self.stats.bytes_received.load(Ordering::Relaxed),
bytes_consumed: self.stats.bytes_consumed.load(Ordering::Relaxed),
bytes_sent: self.stats.bytes_sent.load(Ordering::Relaxed),
packets_received: self.stats.packets_received.load(Ordering::Relaxed),
packets_sent: self.stats.packets_sent.load(Ordering::Relaxed),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ServerClient(ClientId);
impl Display for ServerClient {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}:{}", self.0.token.0, self.0.generation)
}
}
impl ServerClient {
#[inline]
pub fn id(self) -> u64 {
self.0.generation
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
struct ClientId {
token: Token,
generation: u64,
}
struct ClientIdAllocator {
next_token: Token,
next_generation: u64,
free_tokens: Vec<Token>,
}
impl ClientIdAllocator {
fn new() -> Self {
Self {
next_token: CLIENT_FIRST_TOKEN,
next_generation: 1,
free_tokens: Vec::new(),
}
}
#[expect(
clippy::expect_used,
reason = "the only other option is to propagate a result type through
client acceptance, which isn't worth it for such a practically
unreachable condition"
)]
fn allocate(&mut self) -> ClientId {
let token = if let Some(t) = self.free_tokens.pop() {
t
} else {
let t = self.next_token;
self.next_token = Token(t.0.checked_add(1).expect("out of client tokens"));
t
};
let generation = self.next_generation;
self.next_generation = self.next_generation.wrapping_add(1).max(1);
ClientId { token, generation }
}
fn release(&mut self, token: Token) {
self.free_tokens.push(token);
}
}
#[derive(Debug)]
pub enum ServerEvent<I> {
Accept {
client: ServerClient,
peer_addr: SocketAddr,
},
Lost {
client: ServerClient,
error: Option<io::Error>,
},
Packet {
client: ServerClient,
packet: I,
},
}
const LISTENER_TOKEN: Token = Token(0);
const CLIENT_FIRST_TOKEN: Token = Token(1);
const POLL_TICK: Duration = Duration::from_millis(100);
const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(100);
struct PollThread<I, O> {
packet_commands_receiver: Receiver<ThreadCommand<O>>,
control_commands_receiver: Receiver<ControlCommand>,
events_sender: Sender<ThreadEvent<I>>,
listener: TcpListener,
poll: Poll,
client_ids: ClientIdAllocator,
clients: HashMap<Token, PollClient>,
ready_order: VecDeque<Token>,
ready_events: HashMap<Token, ReadyEvent>,
listener_retry_at: Option<std::time::Instant>,
control_commands_deferred: bool,
packet_commands_deferred: bool,
packet_commands_processed: u64,
pending_disconnects: VecDeque<PendingDisconnect>,
pending_events: VecDeque<ThreadEvent<I>>,
pending_event_limit: usize,
total_queued_bytes: usize,
on_protocol_error: Option<ProtocolErrorHook<O>>,
#[cfg(feature = "stats")]
stats: Arc<SharedStats>,
config: ServerConfig,
shutdown_requested: Arc<AtomicBool>,
}
struct PollClient {
stream: TcpStream,
id: ClientId,
_peer_addr: SocketAddr,
buf: Vec<u8>,
buf_cursor: usize,
state: ClientState,
outgoing: VecDeque<Vec<u8>>,
outgoing_offset: usize,
queued_bytes: usize,
registered_readable: bool,
registered_writable: bool,
last_activity: std::time::Instant,
packet_started_at: Option<std::time::Instant>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ClientState {
Open,
DisconnectAfterFlush,
PeerEofAfterFlush,
ProtocolErrorAfterFlush,
}
impl ClientState {
fn is_open(self) -> bool {
self == Self::Open
}
}
struct PendingDisconnect {
id: ClientId,
packet_command_cutoff: u64,
}
#[derive(Clone, Copy, Debug, better_default::Default)]
struct ReadyEvent {
#[default(LISTENER_TOKEN)]
token: Token,
readable: bool,
writable: bool,
read_closed: bool,
write_closed: bool,
}
impl ReadyEvent {
fn from_mio(event: &Event) -> Self {
Self {
token: event.token(),
readable: event.is_readable(),
writable: event.is_writable(),
read_closed: event.is_read_closed(),
write_closed: event.is_write_closed(),
}
}
fn writable(token: Token) -> Self {
Self {
token,
writable: true,
..Self::default()
}
}
fn merge(&mut self, other: Self) {
debug_assert_eq!(self.token, other.token);
self.readable |= other.readable;
self.writable |= other.writable;
self.read_closed |= other.read_closed;
self.write_closed |= other.write_closed;
}
fn has_readiness(self) -> bool {
self.readable || self.writable || self.read_closed || self.write_closed
}
}
struct TickBudget {
accepts: usize,
control_commands: usize,
packet_commands: usize,
decoded_packets: usize,
read_bytes: usize,
written_bytes: usize,
}
impl TickBudget {
fn new(config: ServerConfig) -> Self {
Self {
accepts: config.accept_budget,
control_commands: config.control_command_budget,
packet_commands: config.packet_command_budget,
decoded_packets: config.packet_decode_budget,
read_bytes: config.read_byte_budget,
written_bytes: config.write_byte_budget,
}
}
}
#[derive(Clone, Copy, Debug, Default)]
struct WorkOutcome {
keep_running: bool,
retry: bool,
}
impl WorkOutcome {
fn running(retry: bool) -> Self {
Self {
keep_running: true,
retry,
}
}
}
impl<I: PacketIn, O: PacketOut> PollThread<I, O> {
fn run(mut self) {
let mut events = Events::with_capacity(128);
while !self.shutdown_requested.load(Ordering::Acquire) {
match self.poll_once(&mut events) {
Ok(true) => {}
Ok(false) => return,
Err(e) => {
let mut event = ThreadEvent::UnrecoverableError { error: e };
loop {
match self.events_sender.try_send(event) {
Ok(()) | Err(TrySendError::Disconnected(_)) => break,
Err(TrySendError::Full(returned)) => {
if self.shutdown_requested.load(Ordering::Acquire) {
break;
}
event = returned;
thread::sleep(Duration::from_millis(1));
}
}
}
return;
}
}
}
self.close_all(None);
}
fn poll_once(&mut self, events: &mut Events) -> io::Result<bool> {
if self.shutdown_requested.load(Ordering::Acquire) {
self.close_all(None);
return Ok(false);
}
if !self.flush_pending_event() {
return Ok(false);
}
let mut budget = TickBudget::new(self.config);
if !self.drain_control_commands(&mut budget) {
return Ok(false);
}
self.drain_packet_commands(&mut budget);
let intake_paused = !self.pending_events.is_empty();
let has_deferred_work = self.has_actionable_ready(intake_paused)
|| self.control_commands_deferred
|| self.packet_commands_deferred;
let timeout = if has_deferred_work {
Duration::ZERO
} else {
POLL_TICK
};
self.poll.poll(events, Some(timeout))?;
for event in events.iter() {
self.queue_ready_event(ReadyEvent::from_mio(event));
}
if !self.drain_control_commands(&mut budget) {
return Ok(false);
}
self.drain_packet_commands(&mut budget);
let ready_count = self.ready_order.len();
for _ in 0..ready_count {
let Some(token) = self.ready_order.pop_front() else {
break;
};
let Some(event) = self.ready_events.remove(&token) else {
continue;
};
let intake_paused = !self.pending_events.is_empty();
if intake_paused && token == LISTENER_TOKEN {
self.ready_events.insert(token, event);
self.ready_order.push_back(token);
continue;
}
let mut process_event = event;
if intake_paused {
process_event.readable = false;
if event.readable {
process_event.read_closed = false;
process_event.write_closed = false;
}
}
let can_progress = if token == LISTENER_TOKEN {
budget.accepts > 0
} else {
process_event.write_closed
|| process_event.read_closed
|| (process_event.writable && budget.written_bytes > 0)
|| (process_event.readable
&& budget.read_bytes > 0
&& budget.decoded_packets > 0)
};
if !can_progress {
self.ready_events.insert(token, event);
let blocked_by_budget = if token == LISTENER_TOKEN {
!intake_paused && budget.accepts == 0
} else {
(process_event.writable && budget.written_bytes == 0)
|| (process_event.readable
&& (budget.read_bytes == 0 || budget.decoded_packets == 0))
};
if blocked_by_budget {
self.ready_order.push_front(token);
break;
}
self.ready_order.push_back(token);
continue;
}
let (keep_running, retry) = if token == LISTENER_TOKEN {
let outcome = self.handle_listener(&mut budget)?;
(outcome.keep_running, outcome.retry.then_some(event))
} else {
let (keep_running, mut retry) = self.handle_client(process_event, &mut budget);
if intake_paused {
retry.readable |= event.readable;
retry.read_closed |= event.read_closed;
retry.write_closed |= event.write_closed;
}
(keep_running, retry.has_readiness().then_some(retry))
};
if !keep_running {
return Ok(false);
}
if let Some(event) = retry {
self.queue_ready_event(event);
}
}
if !self.drain_control_commands(&mut budget) {
return Ok(false);
}
self.drain_packet_commands(&mut budget);
self.expire_timed_out_clients();
if self.shutdown_requested.load(Ordering::Acquire) {
self.close_all(None);
return Ok(false);
}
Ok(true)
}
fn has_actionable_ready(&self, intake_paused: bool) -> bool {
self.ready_events.values().any(|event| {
if event.token == LISTENER_TOKEN {
!intake_paused
&& self
.listener_retry_at
.is_none_or(|retry_at| std::time::Instant::now() >= retry_at)
} else {
event.writable
|| event.write_closed
|| (!intake_paused && event.readable)
|| (!event.readable && event.read_closed)
}
})
}
fn queue_ready_event(&mut self, event: ReadyEvent) {
let token = event.token;
if token == WAKER_TOKEN {
return;
}
if token != LISTENER_TOKEN && !self.clients.contains_key(&token) {
return;
}
if let Some(queued) = self.ready_events.get_mut(&token) {
queued.merge(event);
} else {
self.ready_events.insert(token, event);
self.ready_order.push_back(token);
}
}
fn handle_listener(&mut self, budget: &mut TickBudget) -> io::Result<WorkOutcome> {
if self
.listener_retry_at
.is_some_and(|retry_at| std::time::Instant::now() < retry_at)
{
return Ok(WorkOutcome::running(true));
}
while budget.accepts > 0 {
let (mut stream, peer_addr) = match self.listener.accept() {
Ok(accepted) => {
self.listener_retry_at = None;
accepted
}
Err(error) => match classify_accept_error(&error) {
AcceptErrorAction::Done => {
self.listener_retry_at = None;
return Ok(WorkOutcome::running(false));
}
AcceptErrorAction::RetryImmediately => {
budget.accepts -= 1;
continue;
}
AcceptErrorAction::RetryAfterBackoff => {
self.listener_retry_at =
Some(std::time::Instant::now() + ACCEPT_RETRY_DELAY);
return Ok(WorkOutcome::running(true));
}
AcceptErrorAction::Fatal => return Err(error),
},
};
budget.accepts -= 1;
if self.clients.len() >= self.config.max_clients {
let _ = stream.shutdown(Shutdown::Both);
continue;
}
if configure_accepted_stream(&stream, self.config).is_err() {
let _ = stream.shutdown(Shutdown::Both);
continue;
}
let id = self.client_ids.allocate();
let token = id.token;
self.poll
.registry()
.register(&mut stream, token, Interest::READABLE)?;
self.clients.insert(
token,
PollClient {
stream,
id,
_peer_addr: peer_addr,
buf: vec![0u8; self.config.incoming_buffer_initial],
buf_cursor: 0,
state: ClientState::Open,
outgoing: VecDeque::new(),
outgoing_offset: 0,
queued_bytes: 0,
registered_readable: true,
registered_writable: false,
last_activity: std::time::Instant::now(),
packet_started_at: None,
},
);
if !self.send_event(ThreadEvent::Accept { id, peer_addr }) {
return Ok(WorkOutcome::default());
}
if !self.pending_events.is_empty() {
return Ok(WorkOutcome::running(true));
}
}
Ok(WorkOutcome::running(true))
}
fn handle_client(&mut self, event: ReadyEvent, budget: &mut TickBudget) -> (bool, ReadyEvent) {
let token = event.token;
let mut retry = ReadyEvent {
token,
..ReadyEvent::default()
};
if event.writable {
match self.flush_client(token, &mut budget.written_bytes) {
Err(e) => return (self.finalize_close(token, Some(e)), retry),
Ok(outcome) if !outcome.keep_running => return (false, retry),
Ok(outcome) => retry.writable = outcome.retry,
}
}
if event.readable {
match self.handle_client_read(token, budget) {
Err(e) => return (self.finalize_close(token, Some(e)), retry),
Ok(outcome) if !outcome.keep_running => return (false, retry),
Ok(outcome) => retry.readable = outcome.retry,
}
if !self.pending_events.is_empty() {
retry.read_closed = event.read_closed;
retry.write_closed = event.write_closed;
return (true, retry);
}
}
if event.write_closed {
return (
self.finalize_close(token, Some(new_io_abort_error("peer write-closed"))),
retry,
);
}
if event.read_closed && retry.readable {
retry.read_closed = true;
} else if event.read_closed {
return (self.mark_read_eof(token), retry);
}
(true, retry)
}
fn handle_client_read(
&mut self,
token: Token,
budget: &mut TickBudget,
) -> io::Result<WorkOutcome> {
let mut saw_eof = false;
let mut receiver_connected = true;
let mut protocol_error_packet = None;
let mut retry = false;
{
let Some(client) = self.clients.get_mut(&token) else {
return Ok(WorkOutcome::running(false));
};
if !client.state.is_open() {
return Ok(WorkOutcome::running(false));
}
'read: loop {
while client.buf_cursor > 0 {
if budget.decoded_packets == 0 {
retry = true;
break 'read;
}
let mut cursor = Cursor::new(&client.buf[..client.buf_cursor]);
match I::read(&mut cursor) {
Ok(packet) => {
let consumed = cursor.position() as usize;
if consumed == 0 {
return Err(new_io_abort_error("decoder made no progress"));
}
#[cfg(feature = "stats")]
atomic_saturating_add(&self.stats.bytes_consumed, consumed as u64);
#[cfg(feature = "stats")]
atomic_saturating_add(&self.stats.packets_received, 1);
budget.decoded_packets -= 1;
let event = ThreadEvent::ReceivePacket {
id: client.id,
packet,
};
match self.events_sender.try_send(event) {
Ok(()) => {}
Err(TrySendError::Full(event)) => {
self.pending_events.push_back(event);
}
Err(TrySendError::Disconnected(_)) => {
receiver_connected = false;
}
}
client.buf.copy_within(consumed..client.buf_cursor, 0);
client.buf_cursor -= consumed;
client.packet_started_at = None;
if !self.pending_events.is_empty() || !receiver_connected {
retry = receiver_connected;
break 'read;
}
}
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => {
if client.buf_cursor == self.config.incoming_buffer_limit {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"incoming packet exceeded size limit",
));
}
if self.config.packet_assembly_timeout.is_some() {
client
.packet_started_at
.get_or_insert_with(std::time::Instant::now);
}
break;
}
Err(e) if e.kind() == io::ErrorKind::InvalidData => {
if let Some(hook) = &self.on_protocol_error {
let packet = (hook)(&e);
protocol_error_packet = Some((client.id, packet));
break 'read;
} else {
return Err(e);
}
}
Err(e) => return Err(e),
}
}
if saw_eof {
if client.buf_cursor != 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"peer closed with an incomplete packet",
));
}
break;
}
if budget.read_bytes == 0 || budget.decoded_packets == 0 {
retry = true;
break;
}
if client.buf_cursor == client.buf.len() {
let new_len = (client.buf.len() * 2).min(self.config.incoming_buffer_limit);
client.buf.resize(new_len, 0);
}
debug_assert!(client.buf_cursor < client.buf.len());
let read_end = client
.buf_cursor
.saturating_add(budget.read_bytes)
.min(client.buf.len());
match (&client.stream).read(&mut client.buf[client.buf_cursor..read_end]) {
Ok(0) => saw_eof = true,
Ok(len) => {
client.buf_cursor += len;
budget.read_bytes -= len;
client.last_activity = std::time::Instant::now();
#[cfg(feature = "stats")]
atomic_saturating_add(&self.stats.bytes_received, len as u64);
}
Err(e) if e.kind() == io::ErrorKind::WouldBlock => break,
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e),
}
}
}
if let Some((id, packet)) = protocol_error_packet {
self.handle_protocol_error_packet(id, packet);
return Ok(WorkOutcome::running(false));
}
if !receiver_connected {
return Ok(WorkOutcome::default());
}
if saw_eof {
return Ok(WorkOutcome {
keep_running: self.mark_read_eof(token),
retry: false,
});
}
Ok(WorkOutcome::running(retry))
}
fn finalize_close(&mut self, token: Token, error: Option<io::Error>) -> bool {
let Some(client) = self.clients.remove(&token) else {
return true;
};
self.ready_events.remove(&token);
self.client_ids.release(token);
self.total_queued_bytes = self.total_queued_bytes.saturating_sub(client.queued_bytes);
let mut stream = client.stream;
let _ = stream.shutdown(Shutdown::Both);
let _ = self.poll.registry().deregister(&mut stream);
self.send_event(ThreadEvent::Lost {
id: client.id,
error,
})
}
fn mark_read_eof(&mut self, token: Token) -> bool {
if let Some(client) = self.clients.get_mut(&token) {
if client.state.is_open() {
client.state = ClientState::PeerEofAfterFlush;
}
if client.outgoing.is_empty() {
return self.finalize_close(token, None);
}
if let Err(e) = self.update_client_interest(token) {
return self.finalize_close(token, Some(e));
}
}
true
}
fn drain_control_commands(&mut self, budget: &mut TickBudget) -> bool {
while budget.control_commands > 0 {
match self.control_commands_receiver.try_recv() {
Ok(ControlCommand::DisconnectClient {
id,
packet_command_cutoff,
}) => {
budget.control_commands -= 1;
self.pending_disconnects.push_back(PendingDisconnect {
id,
packet_command_cutoff,
});
self.finish_pending_disconnects();
}
Ok(ControlCommand::DisconnectClientNow { id }) => {
budget.control_commands -= 1;
self.close_current(id, None);
}
Ok(ControlCommand::Shutdown) => {
budget.control_commands -= 1;
self.shutdown_requested.store(true, Ordering::Release);
self.close_all(None);
return false;
}
Err(TryRecvError::Empty) => {
self.control_commands_deferred = false;
return true;
}
Err(TryRecvError::Disconnected) => return false,
}
}
self.control_commands_deferred = true;
true
}
fn drain_packet_commands(&mut self, budget: &mut TickBudget) {
while budget.packet_commands > 0 {
match self.packet_commands_receiver.try_recv() {
Ok(ThreadCommand::SendPacket { id, packet }) => {
budget.packet_commands -= 1;
self.handle_client_send(id, packet);
self.packet_commands_processed =
self.packet_commands_processed.saturating_add(1);
self.finish_pending_disconnects();
}
Err(TryRecvError::Empty) => {
self.packet_commands_deferred = false;
return;
}
Err(TryRecvError::Disconnected) => return,
}
}
self.packet_commands_deferred = true;
}
fn finish_pending_disconnects(&mut self) {
let pending_count = self.pending_disconnects.len();
for _ in 0..pending_count {
let Some(pending) = self.pending_disconnects.pop_front() else {
break;
};
if pending.packet_command_cutoff <= self.packet_commands_processed {
self.handle_client_disconnect(pending.id);
} else {
self.pending_disconnects.push_back(pending);
}
}
}
fn handle_protocol_error_packet(&mut self, id: ClientId, packet: O) {
let Some(client) = self.current_mut(id) else {
return;
};
if !client.state.is_open() {
return;
}
client.state = ClientState::ProtocolErrorAfterFlush;
client.buf_cursor = 0;
self.queue_client_packet(id, packet);
}
fn handle_client_disconnect(&mut self, id: ClientId) {
let Some(client) = self.current_mut(id) else {
return;
};
let transitioned = client.state.is_open();
if client.state.is_open() {
client.state = ClientState::DisconnectAfterFlush;
}
if client.outgoing.is_empty() {
let _ = self.close_current(id, None);
} else if transitioned && let Err(error) = self.update_client_interest(id.token) {
self.close_current(id, Some(error));
}
}
fn handle_client_send(&mut self, id: ClientId, packet: O) {
let Some(client) = self.current_mut(id) else {
return;
};
if !client.state.is_open() {
return;
}
self.queue_client_packet(id, packet);
}
fn queue_client_packet(&mut self, id: ClientId, packet: O) {
let encoded = match encode_bounded(&packet, self.config.outgoing_buffer_limit) {
Ok(encoded) => encoded,
Err(e) => {
self.close_current(id, Some(e));
return;
}
};
let Some(limit) = self.config.outgoing_buffer_limit.checked_sub(encoded.len()) else {
self.close_current(
id,
Some(io::Error::new(
io::ErrorKind::InvalidData,
"outgoing packet exceeded size limit",
)),
);
return;
};
let packet_queue_limit = self.config.outgoing_packet_queue_limit;
let Some(global_queued_bytes) = self.total_queued_bytes.checked_add(encoded.len()) else {
self.close_current(
id,
Some(io::Error::new(
io::ErrorKind::OutOfMemory,
"global outgoing queue size overflow",
)),
);
return;
};
if global_queued_bytes > self.config.global_outgoing_buffer_limit {
self.close_current(
id,
Some(io::Error::new(
io::ErrorKind::OutOfMemory,
"global outgoing queue limit exceeded",
)),
);
return;
}
let Some(client) = self.current_mut(id) else {
return;
};
if client.queued_bytes > limit || client.outgoing.len() >= packet_queue_limit {
self.close_current(
id,
Some(io::Error::new(
io::ErrorKind::OutOfMemory,
"client outgoing queue limit exceeded",
)),
);
return;
}
client.queued_bytes += encoded.len();
client.outgoing.push_back(encoded);
self.total_queued_bytes = global_queued_bytes;
if let Err(e) = self.update_client_interest(id.token) {
self.close_current(id, Some(e));
} else {
self.queue_ready_event(ReadyEvent::writable(id.token));
}
}
fn flush_client(&mut self, token: Token, budget: &mut usize) -> io::Result<WorkOutcome> {
let Some(client) = self.clients.get_mut(&token) else {
return Ok(WorkOutcome::running(false));
};
let mut retry = false;
while let Some(front) = client.outgoing.front() {
if *budget == 0 {
retry = true;
break;
}
let write_end = client
.outgoing_offset
.saturating_add(*budget)
.min(front.len());
match client
.stream
.write(&front[client.outgoing_offset..write_end])
{
Ok(0) => break,
Ok(wrote) => {
*budget -= wrote;
client.queued_bytes = client.queued_bytes.saturating_sub(wrote);
self.total_queued_bytes = self.total_queued_bytes.saturating_sub(wrote);
client.outgoing_offset += wrote;
client.last_activity = std::time::Instant::now();
#[cfg(feature = "stats")]
atomic_saturating_add(&self.stats.bytes_sent, wrote as u64);
if client.outgoing_offset == front.len() {
client.outgoing.pop_front();
client.outgoing_offset = 0;
#[cfg(feature = "stats")]
atomic_saturating_add(&self.stats.packets_sent, 1);
}
}
Err(e) if e.kind() == io::ErrorKind::WouldBlock => break,
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e),
}
}
let Some(client) = self.clients.get(&token) else {
return Ok(WorkOutcome::running(false));
};
if client.outgoing.is_empty() && !client.state.is_open() {
return Ok(WorkOutcome {
keep_running: self.finalize_close(token, None),
retry: false,
});
}
self.update_client_interest(token)?;
Ok(WorkOutcome::running(retry))
}
fn update_client_interest(&mut self, token: Token) -> io::Result<()> {
let Some(client) = self.clients.get_mut(&token) else {
return Ok(());
};
let wants_writable = !client.outgoing.is_empty();
let wants_readable = client.state.is_open();
if client.registered_readable == wants_readable
&& client.registered_writable == wants_writable
{
return Ok(());
}
client.registered_readable = wants_readable;
client.registered_writable = wants_writable;
let interest = if !wants_readable {
Interest::WRITABLE
} else if wants_writable {
Interest::READABLE | Interest::WRITABLE
} else {
Interest::READABLE
};
self.poll
.registry()
.reregister(&mut client.stream, token, interest)
}
fn current_mut(&mut self, id: ClientId) -> Option<&mut PollClient> {
self.clients
.get_mut(&id.token)
.filter(|client| client.id == id)
}
fn close_current(&mut self, id: ClientId, error: Option<io::Error>) -> bool {
self.clients
.get(&id.token)
.is_none_or(|client| client.id != id)
|| self.finalize_close(id.token, error)
}
fn send_event(&mut self, event: ThreadEvent<I>) -> bool {
if !self.pending_events.is_empty() {
if self.pending_events.len() >= self.pending_event_limit {
return false;
}
self.pending_events.push_back(event);
return true;
}
match self.events_sender.try_send(event) {
Ok(()) => true,
Err(TrySendError::Full(event)) => {
debug_assert!(self.pending_event_limit > 0);
self.pending_events.push_back(event);
true
}
Err(TrySendError::Disconnected(_)) => false,
}
}
fn flush_pending_event(&mut self) -> bool {
while let Some(event) = self.pending_events.pop_front() {
match self.events_sender.try_send(event) {
Ok(()) => {}
Err(TrySendError::Full(event)) => {
self.pending_events.push_front(event);
return true;
}
Err(TrySendError::Disconnected(_)) => return false,
}
}
true
}
fn expire_timed_out_clients(&mut self) {
let now = std::time::Instant::now();
let expired: Vec<(Token, &'static str)> = self
.clients
.iter()
.filter_map(|(&token, client)| {
if client.state.is_open()
&& self.config.packet_assembly_timeout.is_some_and(|timeout| {
client
.packet_started_at
.is_some_and(|started_at| now.duration_since(started_at) >= timeout)
})
{
Some((token, "incoming packet assembly timeout elapsed"))
} else if self
.config
.idle_timeout
.is_some_and(|timeout| now.duration_since(client.last_activity) >= timeout)
{
Some((token, "client idle timeout elapsed"))
} else {
None
}
})
.collect();
for (token, message) in expired {
self.finalize_close(
token,
Some(io::Error::new(io::ErrorKind::TimedOut, message)),
);
}
}
fn close_all(&mut self, error: Option<io::Error>) {
let tokens: Vec<Token> = self.clients.keys().copied().collect();
for token in tokens {
let err = error
.as_ref()
.map(|e| io::Error::new(e.kind(), e.to_string()));
let _ = self.finalize_close(token, err);
}
}
}
enum ThreadCommand<O> {
SendPacket { id: ClientId, packet: O },
}
enum ControlCommand {
DisconnectClient {
id: ClientId,
packet_command_cutoff: u64,
},
DisconnectClientNow {
id: ClientId,
},
Shutdown,
}
enum ThreadEvent<I> {
Accept {
id: ClientId,
peer_addr: SocketAddr,
},
Lost {
id: ClientId,
error: Option<io::Error>,
},
ReceivePacket {
id: ClientId,
packet: I,
},
UnrecoverableError {
error: io::Error,
},
}
fn new_io_abort_error(message: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::ConnectionAborted, message)
}
fn configure_accepted_stream(stream: &TcpStream, config: ServerConfig) -> io::Result<()> {
if let Err(error) = stream.set_nodelay(config.nodelay)
&& !is_best_effort_socket_option_error(&error)
{
return Err(error);
}
let socket = SockRef::from(stream);
let keepalive_result = if let Some(time) = config.keepalive_time {
socket.set_tcp_keepalive(&TcpKeepalive::new().with_time(time))
} else {
socket.set_keepalive(false)
};
if let Err(error) = keepalive_result
&& !is_best_effort_socket_option_error(&error)
{
return Err(error);
}
Ok(())
}
fn is_best_effort_socket_option_error(error: &io::Error) -> bool {
matches!(
error.kind(),
io::ErrorKind::PermissionDenied | io::ErrorKind::Unsupported
)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum AcceptErrorAction {
Done,
RetryImmediately,
RetryAfterBackoff,
Fatal,
}
fn classify_accept_error(error: &io::Error) -> AcceptErrorAction {
if error.kind() == io::ErrorKind::WouldBlock {
return AcceptErrorAction::Done;
}
if matches!(
error.kind(),
io::ErrorKind::Interrupted | io::ErrorKind::ConnectionAborted
) {
return AcceptErrorAction::RetryImmediately;
}
if error.kind() == io::ErrorKind::OutOfMemory {
return AcceptErrorAction::RetryAfterBackoff;
}
if matches!(
error.raw_os_error(),
Some(12 | 23 | 24 | 55 | 105 | 10024 | 10055)
) {
AcceptErrorAction::RetryAfterBackoff
} else {
AcceptErrorAction::Fatal
}
}
#[cfg(feature = "stats")]
fn atomic_saturating_add(counter: &AtomicU64, value: u64) {
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
Some(current.saturating_add(value))
});
}
#[cfg(test)]
mod tests {
use super::{
AcceptErrorAction, ClientIdAllocator, ServerClient, ServerConfig, classify_accept_error,
configure_accepted_stream,
};
use mio::net::TcpStream as MioTcpStream;
use socket2::SockRef;
use std::{io, net::TcpListener, net::TcpStream, time::Duration};
#[cfg(feature = "stats")]
use super::atomic_saturating_add;
#[cfg(feature = "stats")]
use std::sync::atomic::{AtomicU64, Ordering};
#[test]
fn client_id_is_generation_not_token_packed_value() {
let mut ids = ClientIdAllocator::new();
let first = ids.allocate();
ids.release(first.token);
let second = ids.allocate();
assert_eq!(first.token, second.token);
assert_eq!(ServerClient(first).id(), 1);
assert_eq!(ServerClient(second).id(), 2);
}
#[test]
#[ignore = "socket option introspection is platform dependent"]
fn accepted_socket_options_are_applied() -> io::Result<()> {
let listener = TcpListener::bind("127.0.0.1:0")?;
let client = TcpStream::connect(listener.local_addr()?)?;
let (server, _) = listener.accept()?;
server.set_nonblocking(true)?;
let server = MioTcpStream::from_std(server);
configure_accepted_stream(
&server,
ServerConfig {
nodelay: true,
keepalive_time: Some(Duration::from_secs(60)),
..ServerConfig::default()
},
)?;
if let Ok(nodelay) = server.nodelay() {
assert!(nodelay);
}
if let Ok(keepalive) = SockRef::from(&server).keepalive() {
assert!(keepalive);
}
configure_accepted_stream(
&server,
ServerConfig {
nodelay: false,
keepalive_time: None,
..ServerConfig::default()
},
)?;
if let Ok(nodelay) = server.nodelay() {
assert!(!nodelay);
}
if let Ok(keepalive) = SockRef::from(&server).keepalive() {
assert!(!keepalive);
}
drop(client);
Ok(())
}
#[test]
fn accept_errors_are_classified_for_retry() {
assert_eq!(
classify_accept_error(&io::Error::from(io::ErrorKind::WouldBlock)),
AcceptErrorAction::Done
);
assert_eq!(
classify_accept_error(&io::Error::from(io::ErrorKind::Interrupted)),
AcceptErrorAction::RetryImmediately
);
assert_eq!(
classify_accept_error(&io::Error::from(io::ErrorKind::ConnectionAborted)),
AcceptErrorAction::RetryImmediately
);
assert_eq!(
classify_accept_error(&io::Error::from_raw_os_error(24)),
AcceptErrorAction::RetryAfterBackoff
);
assert_eq!(
classify_accept_error(&io::Error::from(io::ErrorKind::PermissionDenied)),
AcceptErrorAction::Fatal
);
}
#[cfg(feature = "stats")]
#[test]
fn stats_counters_saturate() {
let counter = AtomicU64::new(u64::MAX - 1);
atomic_saturating_add(&counter, 2);
assert_eq!(counter.load(Ordering::Relaxed), u64::MAX);
}
}