use std::any::TypeId;
use std::collections::VecDeque;
use std::io::IoSlice;
use std::net::{SocketAddr, TcpStream};
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, AtomicUsize, Ordering, fence};
use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError, Weak};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
use super::source::SourceTable;
use super::{BlockingHandler, DoorState, DrainReport, ListenerSpec, Server, Sessions, Stats};
use crate::accept::{Accepted, Acceptor};
use crate::addr::PrefixKey;
use crate::buf::{Buffer, BufferPool};
use crate::completer::{Completer, CompleterShared, CompleterTarget};
use crate::control::{ControlLine, ControlState, ControlView, NoControl};
use crate::driver::DriverKind;
use crate::error::{CloseReason, KillReason, NetError, OsErrorClass, RefusalCause, TimeoutKind};
use crate::events::{CoalescedKind, Coalescer, Event, EventSink, NoopSink};
use crate::gate::{AdmitStage, AllowAll, ErasedGate, Gate, GateCell, Refusal, guarded};
use crate::handoff::ListenerHandle;
use crate::id::{ConnectionId, IdBlock, ListenerId, WorkerId};
#[cfg(all(unix, feature = "local"))]
use crate::info::Endpoint;
use crate::info::{ConnectionInfo, Lane, Origin, Transport};
#[cfg(any(feature = "proxy", feature = "tls"))]
use crate::limits::SizeClass;
use crate::limits::{Effective, Limits};
use crate::link::{ClockRef, Got, Link, WakeSource};
#[cfg(all(unix, feature = "local"))]
use crate::local::UnixListener;
use crate::opts::{ListenOptions, TcpOptions};
use crate::proxy::{ProxyFacts, ProxyMode, ProxyVersions};
use crate::session::{Input, Output, Phase, Session, SessionContext, SessionFactory, Step};
use crate::sys::{self, AsSock, Interest, KillSlot, Wake};
use crate::tcp::{ListenerInfo, TcpConnection, TcpListener};
use crate::time::Clock;
use crate::tls::ErasedAcceptor;
use crate::trusted::TrustedProxies;
#[cfg(feature = "reactor")]
use super::reactor::Backend;
#[cfg(feature = "reactor")]
use crate::driver::DriverWaker;
#[cfg(feature = "reactor")]
use std::collections::HashMap;
const WORKER_STACK: usize = 256 * 1024;
const DRAIN_KILL: KillReason = KillReason(0);
const CAUSE_USER: u8 = 0;
pub(crate) const CAUSE_DRAIN: u8 = 1;
pub(crate) const CAUSE_STOP: u8 = 2;
#[cfg(feature = "reactor")]
pub(crate) const CMD_SWEEP: u32 = 1;
#[cfg(feature = "reactor")]
pub(crate) const CMD_SHED: u32 = 2;
#[cfg(feature = "reactor")]
pub(crate) const CMD_KILL_ALL: u32 = 4;
#[cfg(feature = "reactor")]
pub(crate) const CMD_LISTENER_SWEEP: u32 = 8;
#[cfg(feature = "reactor")]
pub(crate) const CMD_LISTENER_KILL: u32 = 16;
#[cfg(feature = "reactor")]
pub(crate) const CMD_PRESSURE: u32 = 32;
#[cfg(feature = "reactor")]
const INBOX_MAX: usize = 4_096;
#[cfg(feature = "reactor")]
const CONN_SHARDS: usize = 16;
const IDLE_DRAIN: u8 = 0;
const IDLE_SHED: u8 = 1;
const NO_LISTENER: u32 = u32::MAX;
const DRAIN_PASS: Duration = Duration::from_millis(10);
const DRAIN_GRACE: Duration = Duration::from_secs(2);
const JOIN_SLACK: Duration = Duration::from_millis(100);
const SKIPS_BEFORE_PAUSE: u32 = 64;
#[inline]
fn take_slot(counter: &AtomicU32, cap: u32) -> bool {
let mut n = counter.load(Ordering::Acquire);
while n < cap {
match counter.compare_exchange_weak(n, n + 1, Ordering::AcqRel, Ordering::Acquire) {
Ok(_) => return true,
Err(seen) => n = seen,
}
}
false
}
pub(crate) struct Config<W, G, S> {
pub tls: Option<Arc<dyn ErasedAcceptor>>,
pub trusted: Option<Arc<TrustedProxies>>,
pub listeners: Vec<ListenerSpec>,
pub driver: DriverKind,
pub workers: u16,
pub limits: Limits,
pub work: W,
pub gate: G,
pub sink: S,
pub control: Option<Arc<dyn ControlLine>>,
pub wipe: bool,
pub clock: Option<Arc<dyn Clock>>,
pub placement: super::Placement,
#[cfg(feature = "udp")]
pub udp: Option<Arc<dyn crate::udp::UdpSessionFactory>>,
}
#[derive(Default)]
pub(crate) struct WorkerStats {
seq: AtomicU64,
closed: AtomicU64,
bytes_in: AtomicU64,
bytes_out: AtomicU64,
egress_denied: AtomicU64,
killed: AtomicU64,
slow_peers: AtomicU64,
timeouts: AtomicU64,
stale_completions: AtomicU64,
session_panics: AtomicU64,
closed_by: [AtomicU64; CloseReason::COUNT],
}
impl WorkerStats {
#[inline]
fn begin_write(&self) {
let s = self.seq.load(Ordering::Relaxed);
self.seq.store(s.wrapping_add(1), Ordering::Relaxed);
fence(Ordering::Release);
}
#[inline]
fn end_write(&self) {
let s = self.seq.load(Ordering::Relaxed);
self.seq.store(s.wrapping_add(1), Ordering::Release);
}
#[cfg(feature = "reactor")]
pub(crate) fn count_stale(&self) {
self.begin_write();
self.stale_completions.fetch_add(1, Ordering::Relaxed);
self.end_write();
}
fn read_into(&self, s: &mut Stats, closed_by: &mut [u64; CloseReason::COUNT]) {
loop {
let before = self.seq.load(Ordering::Acquire);
if before & 1 == 1 {
std::hint::spin_loop();
continue;
}
let snap = [
self.closed.load(Ordering::Relaxed),
self.bytes_in.load(Ordering::Relaxed),
self.bytes_out.load(Ordering::Relaxed),
self.egress_denied.load(Ordering::Relaxed),
self.killed.load(Ordering::Relaxed),
self.slow_peers.load(Ordering::Relaxed),
self.timeouts.load(Ordering::Relaxed),
self.stale_completions.load(Ordering::Relaxed),
self.session_panics.load(Ordering::Relaxed),
];
let mut by = [0u64; CloseReason::COUNT];
for (b, c) in by.iter_mut().zip(&self.closed_by) {
*b = c.load(Ordering::Relaxed);
}
fence(Ordering::Acquire);
if self.seq.load(Ordering::Relaxed) != before {
continue;
}
s.closed += snap[0];
s.bytes_in += snap[1];
s.bytes_out += snap[2];
s.egress_denied += snap[3];
s.killed += snap[4];
s.slow_peers += snap[5];
s.timeouts += snap[6];
s.stale_completions += snap[7];
s.session_panics += snap[8];
for (total, b) in closed_by.iter_mut().zip(by) {
*total += b;
}
return;
}
}
}
#[repr(align(128))]
pub(crate) struct WorkerShared {
slot: Arc<KillSlot>,
wake: Arc<Wake>,
current: AtomicU64,
pub(crate) listener: AtomicU32,
pub(crate) lane: AtomicU8,
pub(crate) kill_cause: AtomicU8,
idle_cause: AtomicU8,
pub(crate) completer: Arc<CompleterShared>,
pub(crate) stats: WorkerStats,
#[cfg(feature = "reactor")]
pub(crate) cmd: AtomicU32,
#[cfg(feature = "reactor")]
pub(crate) load: AtomicU32,
#[cfg(feature = "reactor")]
pub(crate) waker: OnceLock<DriverWaker>,
#[cfg(feature = "reactor")]
pub(crate) dead: AtomicBool,
}
#[cfg(feature = "reactor")]
#[repr(align(128))]
struct Inbox<T>(Mutex<VecDeque<T>>);
#[cfg(feature = "reactor")]
pub(crate) struct RegisteredConn {
pub(crate) slot: Arc<KillSlot>,
pub(crate) generation: u32,
pub(crate) worker: u32,
pub(crate) index: u32,
}
struct WorkerTarget {
shared: Weak<Shared>,
idx: usize,
}
impl CompleterTarget for WorkerTarget {
fn owns(&self, id: ConnectionId) -> bool {
self.shared.upgrade().is_some_and(|s| {
#[cfg(feature = "reactor")]
if s.reactor {
return s.conn_worker(id) == Some(self.idx as u32);
}
s.workers[self.idx].current.load(Ordering::Acquire) == id.get()
})
}
fn wake(&self) {
if let Some(s) = self.shared.upgrade() {
s.wake_worker(self.idx);
}
}
fn stopped(&self) -> bool {
self.shared
.upgrade()
.is_none_or(|s| s.stopping.load(Ordering::Acquire))
}
}
#[derive(Clone, Debug)]
struct Layout {
workers: u16,
placement: super::Placement,
}
impl Layout {
fn cpu_of(&self, i: usize) -> Option<usize> {
match &self.placement {
super::Placement::Off => None,
super::Placement::PinPerCore => Some(i),
super::Placement::Explicit(cpus) => Some(cpus[i % cpus.len()]),
}
}
}
pub(crate) struct Shared {
layout: Layout,
pressure: Option<crate::limits::Pressure>,
pressured: AtomicBool,
#[cfg(feature = "udp")]
udp: Option<Arc<dyn crate::udp::UdpSessionFactory>>,
#[cfg(feature = "reactor")]
pub(crate) reactor: bool,
#[cfg(feature = "reactor")]
next_worker: AtomicUsize,
#[cfg(feature = "reactor")]
conns: Box<[Mutex<HashMap<u64, RegisteredConn>>]>,
pid: u32,
door: AtomicBool,
draining: AtomicBool,
stopping: AtomicBool,
stop: Mutex<Option<Option<Instant>>>,
stop_cv: Condvar,
progress: Mutex<()>,
progress_cv: Condvar,
pub(crate) workers: Vec<WorkerShared>,
listeners: Mutex<Vec<ListenerEntry>>,
next_listener: AtomicU32,
spawn_accept: OnceLock<SpawnAccept>,
adopt: OnceLock<AdoptFn>,
driver: DriverKind,
#[cfg_attr(not(feature = "reactor"), allow(dead_code))]
max_open: u32,
#[cfg_attr(not(feature = "reactor"), allow(dead_code))]
shared_workers: usize,
pub(crate) soft_closes: Arc<crate::link::SoftCloses>,
pub(crate) base_limits: Limits,
pub(crate) trusted: Option<Arc<TrustedProxies>>,
queue_notify: OnceLock<Box<dyn Fn() + Send + Sync>>,
queue_len: AtomicUsize,
admitting: AtomicUsize,
#[cfg(feature = "reactor")]
pub(crate) listener_epoch: AtomicU64,
#[cfg(feature = "reactor")]
control_drain: AtomicBool,
control: Arc<dyn ControlLine>,
sink: Option<Arc<dyn EventSink>>,
open: AtomicU32,
busy: AtomicU32,
accepted: AtomicU64,
refused: AtomicU64,
refused_by: [AtomicU64; RefusalCause::COUNT],
descriptor_pressure: AtomicU64,
drained_graceful: AtomicU32,
drained_killed: AtomicU32,
#[cfg_attr(not(feature = "reactor"), allow(dead_code))]
workers_lost: AtomicU32,
workers_detached: AtomicU32,
running: AtomicUsize,
pub(crate) drain_timeout: Duration,
#[cfg_attr(not(feature = "tls"), allow(dead_code))]
pub(crate) tls: Option<Arc<dyn ErasedAcceptor>>,
}
type AdoptFn = Box<
dyn Fn(
TcpStream,
SocketAddr,
SocketAddr,
&Arc<ListenerConf>,
Option<Lane>,
) -> Result<ConnectionId, NetError>
+ Send
+ Sync,
>;
type SpawnAccept = Box<
dyn Fn(Arc<ListenerConf>, Arc<ListenerRuntime>, usize) -> std::io::Result<JoinHandle<()>>
+ Send
+ Sync,
>;
impl Shared {
pub(crate) fn event(&self, e: Event) {
if let Some(s) = &self.sink {
let _ = guarded(|| s.event(&e));
}
}
#[cfg_attr(not(feature = "reactor"), allow(dead_code))]
pub(crate) fn is_stopping(&self) -> bool {
self.stopping.load(Ordering::Acquire)
}
#[cfg_attr(not(feature = "reactor"), allow(dead_code))]
pub(crate) fn is_draining(&self) -> bool {
self.draining.load(Ordering::Acquire)
}
#[cfg(feature = "reactor")]
pub(crate) fn control_line(&self) -> &dyn ControlLine {
self.control.as_ref()
}
pub(crate) fn occupancy(&self) -> f32 {
let open = self.open.load(Ordering::Relaxed) as f32;
(open / self.max_open.max(1) as f32).min(1.0)
}
#[cfg(feature = "reactor")]
fn conn_shard(&self, id: u64) -> std::sync::MutexGuard<'_, HashMap<u64, RegisteredConn>> {
self.conns[(id as usize) % CONN_SHARDS]
.lock()
.unwrap_or_else(PoisonError::into_inner)
}
#[cfg(feature = "reactor")]
pub(crate) fn register_conn(&self, id: ConnectionId, conn: RegisteredConn) {
self.conn_shard(id.get()).insert(id.get(), conn);
}
#[cfg(feature = "reactor")]
pub(crate) fn unregister_conn(&self, id: ConnectionId) {
self.conn_shard(id.get()).remove(&id.get());
}
#[cfg(feature = "reactor")]
fn conn_worker(&self, id: ConnectionId) -> Option<u32> {
self.conn_shard(id.get()).get(&id.get()).map(|c| c.worker)
}
#[cfg(feature = "reactor")]
pub(crate) fn conn_index(&self, id: ConnectionId, worker: u32) -> Option<u32> {
self.conn_shard(id.get())
.get(&id.get())
.filter(|c| c.worker == worker)
.map(|c| c.index)
}
fn wake_worker(&self, idx: usize) {
#[cfg(feature = "reactor")]
if self.reactor {
if let Some(w) = self.workers[idx].waker.get() {
w.wake();
}
return;
}
self.workers[idx].wake.wake();
}
#[cfg(feature = "reactor")]
fn command(&self, bit: u32) {
for (i, w) in self.workers.iter().enumerate() {
w.cmd.fetch_or(bit, Ordering::AcqRel);
self.wake_worker(i);
}
}
fn forked(&self) -> bool {
std::process::id() != self.pid
}
pub(crate) fn open_door(&self) -> Result<(), NetError> {
if self.forked() {
return Err(NetError::ForkedChild);
}
if self.draining.load(Ordering::Acquire) || self.stopping.load(Ordering::Acquire) {
return Err(NetError::ShuttingDown);
}
let opened = !self.door.swap(true, Ordering::AcqRel);
let mut workers = false;
for e in self.lock_listeners().iter() {
if !e.conf.door.swap(true, Ordering::AcqRel) {
e.conf.wake.wake();
workers |= e.conf.on_workers;
}
}
if workers {
self.wake_workers();
}
if opened {
self.event(Event::DoorOpened);
}
Ok(())
}
pub(crate) fn close_door(&self) {
let closed = self.door.swap(false, Ordering::SeqCst);
for e in self.lock_listeners().iter() {
if e.conf.door.swap(false, Ordering::SeqCst) {
e.conf.wake.wake();
}
}
if closed {
self.event(Event::DoorClosed);
}
}
fn lock_listeners(&self) -> std::sync::MutexGuard<'_, Vec<ListenerEntry>> {
self.listeners
.lock()
.unwrap_or_else(PoisonError::into_inner)
}
fn spawn_listener_threads(&self) -> std::io::Result<()> {
let Some(spawn) = self.spawn_accept.get() else {
return Ok(());
};
for e in self.lock_listeners().iter_mut() {
if e.thread.is_none() && !e.conf.on_workers {
e.thread = Some(spawn(e.conf.clone(), e.runtime.clone(), 0)?);
for (i, s) in e.siblings.iter_mut().enumerate() {
s.thread = Some(spawn(e.conf.clone(), s.runtime.clone(), i + 1)?);
}
}
}
Ok(())
}
pub(crate) fn join_listeners(&self) {
let entries = std::mem::take(&mut *self.lock_listeners());
for e in &entries {
e.conf.removed.store(true, Ordering::Release);
e.conf.wake.wake();
}
#[cfg(feature = "reactor")]
for e in entries.iter().filter(|e| e.conf.on_workers) {
let mut all = vec![&e.runtime];
all.extend(e.siblings.iter().map(|s| &s.runtime));
self.release_from_workers(&all);
}
for mut e in entries {
if let Some(t) = e.thread.take() {
let _ = t.join();
}
for s in &mut e.siblings {
if let Some(t) = s.thread.take() {
let _ = t.join();
}
}
}
}
pub(crate) fn adopt(
&self,
stream: TcpStream,
peer: SocketAddr,
local: SocketAddr,
listener: &str,
lane: Option<Lane>,
) -> Result<ConnectionId, NetError> {
let conf = self
.lock_listeners()
.iter()
.find(|e| e.runtime.info().name.as_deref() == Some(listener))
.map(|e| e.conf.clone())
.ok_or(NetError::InvalidConfig {
field: "listener",
reason: crate::error::ConfigReason::Missing,
})?;
match self.adopt.get() {
Some(f) => f(stream, peer, local, &conf, lane),
None => Err(NetError::ShuttingDown),
}
}
pub(crate) fn release_listeners(&self) -> Result<Vec<ListenerHandle>, NetError> {
if self.forked() {
return Err(NetError::ForkedChild);
}
let list = self.lock_listeners();
let mut out = Vec::with_capacity(list.len());
for e in list.iter() {
if !e.siblings.is_empty() {
return Err(NetError::Unsupported {
feature: "handing off a per-worker listener",
platform: std::env::consts::OS,
});
}
let info = e.runtime.info();
let sock = e
.runtime
.duplicate()
.map_err(|err| NetError::from_os("dup", err))?;
out.push(ListenerHandle::new(
info.transport,
info.name.clone(),
info.local.clone(),
sock,
));
}
Ok(out)
}
pub(crate) fn add_listener(&self, spec: ListenerSpec) -> Result<ListenerInfo, NetError> {
if self.forked() {
return Err(NetError::ForkedChild);
}
if self.draining.load(Ordering::Acquire) || self.stopping.load(Ordering::Acquire) {
return Err(NetError::ShuttingDown);
}
let n = self.next_listener.fetch_add(1, Ordering::AcqRel);
let id = ListenerId(u16::try_from(n).map_err(|_| NetError::InvalidConfig {
field: "listener",
reason: crate::error::ConfigReason::TooLarge {
max: u64::from(u16::MAX),
},
})?);
let (mut entry, info) =
bind_listener(spec, id, &self.base_limits, self.driver, &self.layout, None)?;
let Some(spawn) = self.spawn_accept.get() else {
return Err(NetError::ShuttingDown);
};
let setup = |e| NetError::DriverSetup {
driver: DriverKind::Blocking,
source: e,
};
let on_workers = entry.conf.on_workers;
if !on_workers {
entry.thread =
Some(spawn(entry.conf.clone(), entry.runtime.clone(), 0).map_err(setup)?);
for (i, s) in entry.siblings.iter_mut().enumerate() {
s.thread =
Some(spawn(entry.conf.clone(), s.runtime.clone(), i + 1).map_err(setup)?);
}
}
self.lock_listeners().push(entry);
#[cfg(feature = "reactor")]
if on_workers {
self.listener_epoch.fetch_add(1, Ordering::AcqRel);
self.wake_workers();
}
self.event(Event::ListenerOpened {
listener: id,
local: info.local.socket_addr(),
backlog: info.backlog,
});
Ok(info)
}
pub(crate) fn remove_listener(
&self,
id: ListenerId,
deadline: Instant,
) -> Result<DrainReport, NetError> {
if self.forked() {
return Err(NetError::ForkedChild);
}
let entry = {
let mut list = self.lock_listeners();
let Some(i) = list.iter().position(|e| e.conf.id == id) else {
return Err(NetError::InvalidConfig {
field: "listener",
reason: crate::error::ConfigReason::Missing,
});
};
list.remove(i)
};
let ListenerEntry {
conf,
runtime,
thread,
siblings,
} = entry;
conf.door.store(false, Ordering::Release);
conf.removed.store(true, Ordering::Release);
conf.wake.wake();
#[cfg(feature = "reactor")]
if conf.on_workers {
let mut all = vec![&runtime];
all.extend(siblings.iter().map(|s| &s.runtime));
self.release_from_workers(&all);
}
if let Some(t) = thread {
let _ = t.join();
}
for s in siblings {
if let Some(t) = s.thread {
let _ = t.join();
}
}
drop(runtime);
self.event(Event::ListenerClosed { listener: id });
Ok(self.drain_listener(&conf, deadline))
}
#[cfg(feature = "reactor")]
fn drain_listener_reactor(&self, conf: &ListenerConf, deadline: Instant) -> DrainReport {
let start = Instant::now();
let id = u32::from(conf.id.0);
let initial = conf.count.load(Ordering::Acquire);
for w in &self.workers {
w.listener.store(id, Ordering::Release);
}
let mut killed_at_deadline = false;
let mut guard = self.progress.lock().unwrap_or_else(PoisonError::into_inner);
while conf.count.load(Ordering::Acquire) > 0 {
self.command(CMD_LISTENER_SWEEP);
let now = Instant::now();
if now >= deadline && !killed_at_deadline {
killed_at_deadline = true;
self.command(CMD_LISTENER_KILL);
}
if killed_at_deadline && now >= deadline + DRAIN_GRACE {
break;
}
let wait = if killed_at_deadline {
Duration::from_millis(50)
} else {
deadline
.saturating_duration_since(now)
.clamp(Duration::from_millis(1), DRAIN_PASS)
};
guard = self
.progress_cv
.wait_timeout(guard, wait)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
drop(guard);
let killed = conf.killed.load(Ordering::Acquire);
DrainReport {
graceful: initial.saturating_sub(killed),
killed,
took: start.elapsed(),
}
}
fn drain_listener(&self, conf: &ListenerConf, deadline: Instant) -> DrainReport {
let start = Instant::now();
let id = u32::from(conf.id.0);
let initial = conf.count.load(Ordering::Acquire);
let mut killed = 0u32;
let mut killed_at_deadline = false;
#[cfg(feature = "reactor")]
if self.reactor {
return self.drain_listener_reactor(conf, deadline);
}
let mut guard = self.progress.lock().unwrap_or_else(PoisonError::into_inner);
while conf.count.load(Ordering::Acquire) > 0 {
let mine = |w: &&WorkerShared| {
w.listener.load(Ordering::Acquire) == id && w.current.load(Ordering::Acquire) != 0
};
for w in &self.workers {
let generation = w.slot.generation();
if mine(&w) && !w.slot.drained() {
w.idle_cause.store(IDLE_DRAIN, Ordering::Release);
w.slot.drain_idle(generation);
}
}
let now = Instant::now();
if now >= deadline && !killed_at_deadline {
killed_at_deadline = true;
for w in &self.workers {
let generation = w.slot.generation();
if !mine(&w) {
continue;
}
let claimed = w
.kill_cause
.compare_exchange(
CAUSE_USER,
CAUSE_DRAIN,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok();
if w.slot.kill(generation, DRAIN_KILL) {
killed += 1;
} else if claimed {
w.kill_cause.store(CAUSE_USER, Ordering::Release);
}
}
}
if killed_at_deadline && now >= deadline + DRAIN_GRACE {
break;
}
let wait = if killed_at_deadline {
Duration::from_millis(50)
} else {
deadline
.saturating_duration_since(now)
.clamp(Duration::from_millis(1), DRAIN_PASS)
};
guard = self
.progress_cv
.wait_timeout(guard, wait)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
drop(guard);
DrainReport {
graceful: initial.saturating_sub(killed),
killed,
took: start.elapsed(),
}
}
pub(crate) fn door(&self) -> DoorState {
if self.door.load(Ordering::Acquire) {
DoorState::Open
} else {
DoorState::Closed
}
}
pub(crate) fn request_stop(&self, deadline: Option<Instant>) {
let mut stop = self.stop.lock().unwrap_or_else(PoisonError::into_inner);
if stop.is_none() {
*stop = Some(deadline);
}
self.stop_cv.notify_all();
}
pub(crate) fn wait_for_stop_request(&self) -> Instant {
let mut stop = self.stop.lock().unwrap_or_else(PoisonError::into_inner);
while stop.is_none() {
stop = self
.stop_cv
.wait(stop)
.unwrap_or_else(PoisonError::into_inner);
}
stop.flatten()
.unwrap_or_else(|| Instant::now() + self.drain_timeout)
}
pub(crate) fn worker_count(&self) -> usize {
self.workers.len()
}
pub(crate) fn driver(&self) -> DriverKind {
self.driver
}
fn wake_workers(&self) {
for i in 0..self.workers.len() {
self.wake_worker(i);
}
}
#[cfg(feature = "reactor")]
fn release_from_workers(&self, runtimes: &[&Arc<ListenerRuntime>]) {
self.listener_epoch.fetch_add(1, Ordering::AcqRel);
while runtimes.iter().any(|r| Arc::strong_count(r) > 1) {
self.wake_workers();
std::thread::sleep(Duration::from_micros(200));
}
}
#[cfg(feature = "reactor")]
pub(crate) fn worker_listeners(
&self,
idx: usize,
) -> Vec<(Arc<ListenerConf>, Arc<ListenerRuntime>)> {
let dead = |i: usize| {
self.workers
.get(i)
.is_none_or(|w| w.dead.load(Ordering::Acquire))
};
let mut out = Vec::new();
if dead(idx) {
return out;
}
for e in self
.lock_listeners()
.iter()
.filter(|e| e.conf.on_workers && !e.conf.removed.load(Ordering::Acquire))
{
let places = e.siblings.len() + 1;
for place in 0..places {
let heir = (0..places)
.map(|k| (place + k) % places)
.find(|&i| !dead(i));
if heir != Some(idx) {
continue;
}
let rt = match place {
0 => &e.runtime,
i => &e.siblings[i - 1].runtime,
};
out.push((e.conf.clone(), rt.clone()));
}
}
out
}
#[cfg(feature = "reactor")]
pub(crate) fn worker_lost(&self) {
self.workers_lost.fetch_add(1, Ordering::Relaxed);
self.listener_epoch.fetch_add(1, Ordering::AcqRel);
self.wake_workers();
self.notify_progress();
}
pub(crate) fn wake_all(&self) {
for e in self.lock_listeners().iter() {
e.conf.wake.wake();
}
for i in 0..self.workers.len() {
self.wake_worker(i);
}
}
pub(crate) fn kill(&self, id: ConnectionId, reason: KillReason) -> bool {
self.with_slot(id, |slot, generation| slot.kill(generation, reason))
}
pub(crate) fn close_after_flush(
&self,
id: ConnectionId,
reason: KillReason,
within: Duration,
) -> bool {
let within = within.min(crate::time::LONGEST);
self.with_slot(id, |slot, generation| {
slot.kill_soft(generation, reason, within)
})
}
fn with_slot(&self, id: ConnectionId, f: impl FnOnce(&KillSlot, u32) -> bool) -> bool {
#[cfg(feature = "reactor")]
if self.reactor {
let found = self
.conn_shard(id.get())
.get(&id.get())
.map(|c| (c.slot.clone(), c.generation));
return found.is_some_and(|(slot, generation)| f(&slot, generation));
}
for w in &self.workers {
if w.current.load(Ordering::Acquire) != id.get() {
continue;
}
let generation = w.slot.generation();
if w.current.load(Ordering::Acquire) != id.get() {
return false;
}
return f(&w.slot, generation);
}
false
}
pub(crate) fn completer(&self, worker: WorkerId) -> Option<Completer> {
self.workers.get(usize::from(worker.0)).map(|w| Completer {
shared: w.completer.clone(),
})
}
pub(crate) fn stats(&self) -> Stats {
let mut s = Stats {
open: u64::from(self.open.load(Ordering::Relaxed)),
accepted: self.accepted.load(Ordering::Relaxed),
refused: self.refused.load(Ordering::Relaxed),
descriptor_pressure: self.descriptor_pressure.load(Ordering::Relaxed),
workers_lost: u64::from(self.workers_lost.load(Ordering::Relaxed)),
workers_detached: u64::from(self.workers_detached.load(Ordering::Relaxed)),
..Stats::default()
};
let mut closed_by = [0u64; CloseReason::COUNT];
for w in &self.workers {
w.stats.read_into(&mut s, &mut closed_by);
}
let mut refused_by = [0u64; RefusalCause::COUNT];
for (i, c) in self.refused_by.iter().enumerate() {
refused_by[i] = c.load(Ordering::Relaxed);
}
Stats::from_parts(s, refused_by, closed_by)
}
pub(crate) fn notify_progress(&self) {
let _g = self.progress.lock().unwrap_or_else(PoisonError::into_inner);
self.progress_cv.notify_all();
}
fn idle(&self) -> bool {
if self.admitting.load(Ordering::SeqCst) != 0 {
return false;
}
#[cfg(feature = "reactor")]
if self.reactor {
return self.open.load(Ordering::Acquire) == 0;
}
self.busy.load(Ordering::Acquire) == 0 && self.queue_len.load(Ordering::Acquire) == 0
}
pub(crate) fn drain(&self, deadline: Instant) -> DrainReport {
let start = Instant::now();
self.close_door();
let first = !self.draining.swap(true, Ordering::AcqRel);
if first {
self.drained_graceful.store(0, Ordering::Relaxed);
self.drained_killed.store(0, Ordering::Relaxed);
self.event(Event::DrainStarted {
open: self.open.load(Ordering::Relaxed),
});
}
let mut guard = self.progress.lock().unwrap_or_else(PoisonError::into_inner);
let mut killed_at: Option<Instant> = None;
while !self.idle() {
self.drain_idle_pass();
let now = Instant::now();
let stop = self.stopping.load(Ordering::Acquire)
|| self.control.state() == ControlState::Stopping;
if stop {
self.kill_all(CAUSE_STOP);
let _ = killed_at.get_or_insert(now);
} else if now >= deadline {
let _ = killed_at.get_or_insert(now);
self.kill_all(CAUSE_DRAIN);
}
if killed_at.is_some_and(|at| now >= at + DRAIN_GRACE) {
break;
}
let killed_at_deadline = killed_at.is_some();
let wait = if killed_at_deadline {
Duration::from_millis(50)
} else {
deadline
.saturating_duration_since(now)
.clamp(Duration::from_millis(1), DRAIN_PASS)
};
guard = self
.progress_cv
.wait_timeout(guard, wait)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
drop(guard);
let report = DrainReport {
graceful: self.drained_graceful.load(Ordering::Relaxed),
killed: self.drained_killed.load(Ordering::Relaxed),
took: start.elapsed(),
};
if first {
self.event(Event::DrainFinished {
graceful: report.graceful,
killed: report.killed,
});
}
report
}
fn drain_idle_pass(&self) {
#[cfg(feature = "reactor")]
if self.reactor {
return self.command(CMD_SWEEP);
}
for w in &self.workers {
if w.current.load(Ordering::Acquire) != 0 && !w.slot.drained() {
w.idle_cause.store(IDLE_DRAIN, Ordering::Release);
w.slot.drain_idle(w.slot.generation());
}
}
}
fn kill_all(&self, cause: u8) {
#[cfg(feature = "reactor")]
if self.reactor {
for w in &self.workers {
w.kill_cause.store(cause, Ordering::Release);
}
return self.command(CMD_KILL_ALL);
}
for w in &self.workers {
let generation = w.slot.generation();
if w.current.load(Ordering::Acquire) != 0 {
let claimed = w
.kill_cause
.compare_exchange(CAUSE_USER, cause, Ordering::AcqRel, Ordering::Acquire)
.is_ok();
if !w.slot.kill(generation, DRAIN_KILL) && claimed {
w.kill_cause.store(CAUSE_USER, Ordering::Release);
}
}
}
}
fn shed_idle(&self, from_lane: Lane) {
#[cfg(feature = "reactor")]
if self.reactor {
for w in &self.workers {
w.lane.store(from_lane.0, Ordering::Release);
}
return self.command(CMD_SHED);
}
for w in &self.workers {
let generation = w.slot.generation();
if w.current.load(Ordering::Acquire) != 0
&& w.lane.load(Ordering::Acquire) >= from_lane.0
&& !w.slot.drained()
{
w.idle_cause.store(IDLE_SHED, Ordering::Release);
if !w.slot.shed_idle(generation) {
w.idle_cause.store(IDLE_DRAIN, Ordering::Release);
}
}
}
}
pub(crate) fn join_workers(&self, threads: Vec<JoinHandle<()>>) {
let bound = Instant::now() + if self.idle() { DRAIN_GRACE } else { JOIN_SLACK };
let mut guard = self.progress.lock().unwrap_or_else(PoisonError::into_inner);
while self.running.load(Ordering::Acquire) > 0 {
let now = Instant::now();
if now >= bound {
break;
}
guard = self
.progress_cv
.wait_timeout(guard, bound - now)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
drop(guard);
let left = self.running.load(Ordering::Acquire);
for t in threads {
if left == 0 || t.is_finished() {
let _ = t.join();
}
}
if left > 0 {
let left = u32::try_from(left).unwrap_or(u32::MAX);
self.workers_detached.fetch_add(left, Ordering::Relaxed);
}
}
pub(crate) fn stop_threads(&self) {
self.stopping.store(true, Ordering::Release);
self.request_stop(None);
if let Some(n) = self.queue_notify.get() {
n();
}
self.wake_all();
}
fn update_pressure(&self) {
let Some(p) = self.pressure else { return };
let fill = self.occupancy();
let on = self.pressured.load(Ordering::Acquire);
if on != (fill >= p.high) && (!on || fill <= p.low) {
self.pressured.store(!on, Ordering::Release);
#[cfg(feature = "reactor")]
if self.reactor {
self.command(CMD_PRESSURE);
}
}
}
pub(crate) fn deadline_scale(&self) -> f32 {
match self.pressure {
Some(p) if self.pressured.load(Ordering::Acquire) => p.factor,
_ => 1.0,
}
}
pub(crate) fn release(&self, listener: &ListenerConf, source: Option<PrefixKey>) {
self.open.fetch_sub(1, Ordering::AcqRel);
self.update_pressure();
listener.count.fetch_sub(1, Ordering::AcqRel);
if let Some(k) = source {
listener.sources.release(k);
}
}
}
#[derive(Clone)]
#[cfg_attr(not(feature = "proxy"), allow(dead_code))]
pub(crate) struct ListenerProxy {
pub(crate) versions: ProxyVersions,
trusted: Option<Arc<TrustedProxies>>,
}
impl ListenerProxy {
#[cfg_attr(not(any(feature = "reactor", feature = "proxy")), allow(dead_code))]
pub(crate) fn is_trusted(&self, info: &ConnectionInfo) -> bool {
match (&self.trusted, info.peer.socket_addr()) {
(Some(list), Some(peer)) => list.contains(peer.ip()),
(None, None) => info.transport == Transport::Unix,
_ => false,
}
}
}
pub(crate) struct ListenerConf {
pub(crate) id: ListenerId,
pub(crate) limits: Arc<Effective>,
tcp: TcpOptions,
#[cfg_attr(not(feature = "proxy"), allow(dead_code))]
pub(crate) proxy: Option<ListenerProxy>,
lockdown_allowed: bool,
#[cfg_attr(not(feature = "tls"), allow(dead_code))]
pub(crate) tls: bool,
#[cfg(feature = "reactor")]
killed: AtomicU32,
count: AtomicU32,
sources: SourceTable,
door: AtomicBool,
pub(crate) removed: AtomicBool,
wake: Wakes,
#[cfg_attr(not(feature = "reactor"), allow(dead_code))]
pub(crate) workers: Option<(usize, usize)>,
pub(crate) on_workers: bool,
}
#[cfg(feature = "reactor")]
impl ListenerConf {
pub(crate) fn note_killed(&self) {
self.killed.fetch_add(1, Ordering::AcqRel);
}
}
struct ListenerEntry {
conf: Arc<ListenerConf>,
runtime: Arc<ListenerRuntime>,
thread: Option<JoinHandle<()>>,
siblings: Vec<Sibling>,
}
struct Sibling {
runtime: Arc<ListenerRuntime>,
thread: Option<JoinHandle<()>>,
}
struct Wakes(Vec<Arc<Wake>>);
impl Wakes {
fn wake(&self) {
for w in &self.0 {
w.wake();
}
}
}
thread_local! {
static FEEDS: std::cell::Cell<Option<(usize, bool)>> = const { std::cell::Cell::new(None) };
}
struct Running(Arc<Shared>);
impl Drop for Running {
fn drop(&mut self) {
self.0.running.fetch_sub(1, Ordering::AcqRel);
self.0.notify_progress();
}
}
struct Admitting<'a>(&'a Shared);
impl<'a> Admitting<'a> {
fn enter(shared: &'a Shared) -> Admitting<'a> {
shared.admitting.fetch_add(1, Ordering::SeqCst);
Admitting(shared)
}
}
impl Drop for Admitting<'_> {
fn drop(&mut self) {
self.0.admitting.fetch_sub(1, Ordering::SeqCst);
if self.0.draining.load(Ordering::Acquire) {
self.0.notify_progress();
}
}
}
fn bind_listener(
spec: ListenerSpec,
id: ListenerId,
base: &Limits,
driver: DriverKind,
layout: &Layout,
dedicated: Option<(usize, usize)>,
) -> Result<(ListenerEntry, ListenerInfo), NetError> {
let per_worker = spec.options.is_some_and(|o| o.per_worker);
if let Some(n) = spec.workers {
let reason = if n == 0 {
crate::error::ConfigReason::Zero
} else {
crate::error::ConfigReason::Inconsistent
};
#[cfg(feature = "udp")]
let udp = spec.udp.is_some();
#[cfg(not(feature = "udp"))]
let udp = false;
if n == 0
|| per_worker
|| udp
|| driver.resolve() != DriverKind::Reactor
|| dedicated.is_none()
{
return Err(NetError::InvalidConfig {
field: "workers",
reason,
});
}
}
if per_worker && !cfg!(target_os = "linux") {
return Err(NetError::Unsupported {
feature: "per-worker listeners",
platform: std::env::consts::OS,
});
}
if per_worker && driver.resolve() != DriverKind::Reactor {
return Err(NetError::InvalidConfig {
field: "per_worker",
reason: crate::error::ConfigReason::Inconsistent,
});
}
let lim = base.overlay(&spec.limits).effective(driver);
let proxy = match spec.proxy {
ProxyMode::Required { versions } => Some(ListenerProxy {
versions,
trusted: spec.trusted.clone(),
}),
_ => None,
};
let group_options = spec.options;
let (runtime, info, tcp) = 'bound: {
if let Some(handle) = spec.adopt {
let name = spec.name.clone();
match handle.kind() {
Transport::Tcp => {
let std = std::net::TcpListener::from(handle.into_sock());
let mut listener = TcpListener::from_std(std)?;
listener.set_identity(id, name);
listener.set_proxy_mode(spec.proxy);
listener.set_tls(spec.tls);
let info = listener.info().clone();
let tcp = *listener.tcp_options();
break 'bound (ListenerRuntime::Tcp(listener), info, tcp);
}
#[cfg(all(unix, feature = "local"))]
Transport::Unix => {
let mut listener = UnixListener::adopt(handle.into_sock())?;
listener.set_identity(id, name);
listener.set_proxy_mode(spec.proxy);
listener.set_tls(spec.tls);
let info = listener.info().clone();
break 'bound (ListenerRuntime::Unix(listener), info, TcpOptions::default());
}
_ => {
return Err(NetError::InvalidConfig {
field: "socket",
reason: crate::error::ConfigReason::NotListening,
});
}
}
}
#[cfg(feature = "udp")]
if let Some(options) = spec.udp {
let addrs = spec.addrs.clone().map_err(|text| NetError::Os {
op: "resolve",
source: std::io::Error::other(text),
})?;
let socket = crate::udp::UdpSocket::bind_with(&addrs[..], options)?;
socket.set_nonblocking()?;
let local = socket.local_addr()?;
let info = ListenerInfo {
id,
name: spec.name.clone(),
local: crate::info::Endpoint::Ip(local),
backlog: None,
transport: Transport::Udp,
proxy: ProxyMode::Off,
tls: false,
};
break 'bound (
ListenerRuntime::Udp(Mutex::new(socket), info.clone()),
info,
TcpOptions::default(),
);
}
#[cfg(all(windows, feature = "local"))]
if let Some((path, options)) = spec.unix {
let mut listener = crate::local_windows::UnixListener::bind_with(&path, options)?;
listener.set_identity(id, spec.name.clone());
listener.set_proxy_mode(spec.proxy);
listener.set_tls(spec.tls);
let info = listener.info().clone();
break 'bound (
ListenerRuntime::AfUnix(listener),
info,
TcpOptions::default(),
);
}
#[cfg(all(windows, feature = "local"))]
if let Some((name, options)) = spec.pipe {
let mut listener = crate::pipe::PipeListener::create_with(&name, options)?;
listener.set_identity(id, spec.name.clone());
listener.set_proxy_mode(spec.proxy);
listener.set_tls(spec.tls);
let info = listener.info().clone();
break 'bound (ListenerRuntime::Pipe(listener), info, TcpOptions::default());
}
#[cfg(all(unix, feature = "local"))]
if let Some((path, options)) = spec.unix {
let mut listener = UnixListener::bind_with(&path, options)?;
listener.set_identity(id, spec.name.clone());
listener.set_proxy_mode(spec.proxy);
listener.set_tls(spec.tls);
let info = listener.info().clone();
break 'bound (ListenerRuntime::Unix(listener), info, TcpOptions::default());
}
let addrs = match spec.addrs {
Ok(a) if !a.is_empty() => a,
Ok(_) => return Err(NetError::NoAddresses),
Err(text) => {
return Err(NetError::Os {
op: "resolve",
source: std::io::Error::other(text),
});
}
};
let options = spec
.options
.unwrap_or_else(|| ListenOptions::new().backlog(lim.backlog));
let mut bound = None;
let mut last = NetError::NoAddresses;
for a in addrs {
match sys::tcp_listener(a, &options.params()) {
Ok(l) => {
bound = Some(TcpListener::finish(l, options, id, spec.name.clone())?);
break;
}
Err(e) => last = crate::tcp::bind_error(a, e),
}
}
let mut listener = bound.ok_or(last)?;
listener.set_proxy_mode(spec.proxy);
listener.set_tls(spec.tls);
let info = listener.info().clone();
let tcp = *listener.tcp_options();
(ListenerRuntime::Tcp(listener), info, tcp)
};
let threads = if per_worker {
usize::from(layout.workers)
} else {
1
};
let mut siblings = Vec::with_capacity(threads.saturating_sub(1));
if let (true, Some(options), Some(local)) =
(per_worker, group_options, info.local.socket_addr())
{
for i in 1..threads {
let cpu = layout.cpu_of(i).and_then(|c| u32::try_from(c).ok());
let params = options.incoming_cpu(cpu).params();
let socket =
sys::tcp_listener(local, ¶ms).map_err(|e| crate::tcp::bind_error(local, e))?;
let mut l = TcpListener::finish(socket, options, id, spec.name.clone())?;
l.set_proxy_mode(spec.proxy);
l.set_tls(spec.tls);
siblings.push(Sibling {
runtime: Arc::new(ListenerRuntime::Tcp(l)),
thread: None,
});
}
}
let wakes = (0..threads)
.map(|_| Wake::new().map(Arc::new))
.collect::<std::io::Result<Vec<_>>>()
.map_err(|e| NetError::DriverSetup {
driver: DriverKind::Blocking,
source: e,
})?;
let wake = Wakes(wakes);
let on_workers =
per_worker && cfg!(feature = "reactor") && matches!(runtime, ListenerRuntime::Tcp(_));
let conf = Arc::new(ListenerConf {
id,
limits: Arc::new(lim),
tcp,
proxy,
lockdown_allowed: spec.lockdown_allowed,
tls: spec.tls,
#[cfg(feature = "reactor")]
killed: AtomicU32::new(0),
count: AtomicU32::new(0),
sources: SourceTable::new(lim.max_per_listener, lim.max_per_source),
door: AtomicBool::new(false),
removed: AtomicBool::new(false),
wake,
workers: dedicated,
on_workers,
});
Ok((
ListenerEntry {
conf,
runtime: Arc::new(runtime),
thread: None,
siblings,
},
info,
))
}
pub struct Job<T> {
stream: sys::Stream,
info: ConnectionInfo,
ticket: T,
listener: Arc<ListenerConf>,
source: Option<PrefixKey>,
}
#[cfg(feature = "reactor")]
pub(crate) type JobParts<T> = (
sys::Stream,
ConnectionInfo,
T,
Arc<ListenerConf>,
Option<PrefixKey>,
);
#[cfg(feature = "reactor")]
impl<T> Job<T> {
pub(crate) fn into_parts(self) -> JobParts<T> {
(
self.stream,
self.info,
self.ticket,
self.listener,
self.source,
)
}
}
#[allow(clippy::large_enum_variant)]
pub(crate) enum ListenerRuntime {
Tcp(TcpListener),
#[cfg(feature = "udp")]
Udp(Mutex<crate::udp::UdpSocket>, ListenerInfo),
#[cfg(all(unix, feature = "local"))]
Unix(UnixListener),
#[cfg(all(windows, feature = "local"))]
AfUnix(crate::local_windows::UnixListener),
#[cfg(all(windows, feature = "local"))]
Pipe(crate::pipe::PipeListener),
}
impl ListenerRuntime {
#[cfg(feature = "reactor")]
pub(crate) fn tcp_sock(&self) -> Option<sys::Sock<'_>> {
match self {
ListenerRuntime::Tcp(l) => Some(l.std().sock()),
#[allow(unreachable_patterns)]
_ => None,
}
}
fn wait(&self, wake: &sys::Wake) {
let sock = match self {
ListenerRuntime::Tcp(l) => l.std().sock(),
#[cfg(all(unix, feature = "local"))]
ListenerRuntime::Unix(l) => std::os::fd::AsFd::as_fd(l.std()),
#[cfg(all(windows, feature = "local"))]
ListenerRuntime::AfUnix(l) => l.std().sock(),
#[cfg(all(windows, feature = "local"))]
ListenerRuntime::Pipe(_) => return,
#[cfg(feature = "udp")]
ListenerRuntime::Udp(..) => return,
};
let _ = sys::wait(sock, Interest::Read, Some(wake), None);
}
fn info(&self) -> &ListenerInfo {
match self {
ListenerRuntime::Tcp(l) => l.info(),
#[cfg(all(unix, feature = "local"))]
ListenerRuntime::Unix(l) => l.info(),
#[cfg(all(windows, feature = "local"))]
ListenerRuntime::AfUnix(l) => l.info(),
#[cfg(all(windows, feature = "local"))]
ListenerRuntime::Pipe(l) => l.info(),
#[cfg(feature = "udp")]
ListenerRuntime::Udp(_, info) => info,
}
}
fn duplicate(&self) -> std::io::Result<sys::OwnedSock> {
match self {
ListenerRuntime::Tcp(l) => l.std().try_clone().map(sys::OwnedSock::from),
#[cfg(all(unix, feature = "local"))]
ListenerRuntime::Unix(l) => {
let dup = l.std().try_clone().map(sys::OwnedSock::from)?;
l.keep_file();
Ok(dup)
}
#[cfg(all(windows, feature = "local"))]
ListenerRuntime::AfUnix(_) | ListenerRuntime::Pipe(_) => {
Err(std::io::Error::from(std::io::ErrorKind::Unsupported))
}
#[cfg(feature = "udp")]
ListenerRuntime::Udp(..) => Err(std::io::Error::from(std::io::ErrorKind::Unsupported)),
}
}
}
pub(crate) struct Engine<W, G: Gate> {
pub(crate) shared: Arc<Shared>,
pub(crate) work: W,
pub(crate) gate: Arc<G>,
pub(crate) gated: bool,
queue: Mutex<VecDeque<Job<G::Ticket>>>,
queue_cv: Condvar,
pub(crate) clock: ClockRef,
pub(crate) wipe: bool,
pub(crate) budget: Arc<AtomicU64>,
pub(crate) pool_keep: u64,
pub(crate) limits: Effective,
#[cfg(feature = "reactor")]
inboxes: Vec<Inbox<Job<G::Ticket>>>,
#[cfg(feature = "reactor")]
drivers: Mutex<Vec<Option<Backend>>>,
}
#[cfg(feature = "reactor")]
impl<W, G: Gate> Engine<W, G> {
pub(crate) fn take_driver(&self, idx: usize) -> Option<Backend> {
self.drivers
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get_mut(idx)
.and_then(Option::take)
}
pub(crate) fn next_inbox(&self, idx: usize) -> Option<Job<G::Ticket>> {
self.inboxes[idx]
.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.pop_front()
}
pub(crate) fn retire(&self, idx: usize) -> VecDeque<Job<G::Ticket>> {
let mut inbox = self.inboxes[idx]
.0
.lock()
.unwrap_or_else(PoisonError::into_inner);
self.shared.workers[idx].dead.store(true, Ordering::Release);
std::mem::take(&mut *inbox)
}
pub(crate) fn inbox_empty(&self, idx: usize) -> bool {
self.inboxes[idx]
.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.is_empty()
}
#[allow(clippy::result_large_err)]
fn hand_to_worker(&self, job: Job<G::Ticket>) -> Result<(), Job<G::Ticket>> {
let shared = &self.shared;
let (base, len) = job.listener.workers.unwrap_or((0, shared.shared_workers));
let feed = FEEDS
.with(std::cell::Cell::get)
.filter(|&(i, _)| !shared.workers[i].dead.load(Ordering::Relaxed));
let idx = feed.map_or_else(
|| base + least_loaded(&shared.workers[base..base + len], &shared.next_worker),
|(i, _)| i,
);
{
let mut inbox = self.inboxes[idx]
.0
.lock()
.unwrap_or_else(PoisonError::into_inner);
if inbox.len() >= INBOX_MAX || shared.workers[idx].dead.load(Ordering::Acquire) {
return Err(job);
}
shared.workers[idx].load.fetch_add(1, Ordering::Relaxed);
inbox.push_back(job);
}
if !feed.is_some_and(|(_, own)| own) {
shared.wake_worker(idx);
}
Ok(())
}
}
#[cfg(feature = "reactor")]
fn least_loaded(workers: &[WorkerShared], next: &AtomicUsize) -> usize {
let n = workers.len();
if n <= 1 {
return 0;
}
let start = next.fetch_add(1, Ordering::Relaxed) % n;
let mut best = start;
let mut least = u32::MAX;
for k in 0..n {
let i = (start + k) % n;
if workers[i].dead.load(Ordering::Relaxed) {
continue;
}
let load = workers[i].load.load(Ordering::Relaxed);
if load < least {
least = load;
best = i;
if load == 0 {
break;
}
}
}
best
}
pub(crate) fn start<W, G, S>(cfg: Config<W, G, S>) -> Result<Server, NetError>
where
W: super::Workload,
G: Gate,
S: EventSink,
{
let driver = cfg.driver;
let reactor = driver == DriverKind::Reactor;
let limits = cfg.limits.effective(driver);
let shared_workers = usize::from(cfg.workers);
let mut ranges = Vec::with_capacity(cfg.listeners.len());
let mut total = shared_workers;
for spec in &cfg.listeners {
ranges.push(spec.workers.filter(|&n| n > 0).map(|n| {
let range = (total, usize::from(n));
total += usize::from(n);
range
}));
}
let Ok(all_workers) = u16::try_from(total) else {
return Err(NetError::InvalidConfig {
field: "workers",
reason: crate::error::ConfigReason::TooLarge {
max: u64::from(u16::MAX),
},
});
};
let layout = Layout {
workers: all_workers,
placement: cfg.placement.clone(),
};
let mut entries = Vec::with_capacity(cfg.listeners.len());
let mut infos = Vec::with_capacity(cfg.listeners.len());
for (i, (spec, range)) in cfg.listeners.into_iter().zip(ranges).enumerate() {
let id = ListenerId(u16::try_from(i).unwrap_or(u16::MAX));
let (entry, info) = bind_listener(spec, id, &cfg.limits, driver, &layout, range)?;
entries.push(entry);
infos.push(info);
}
let sink: Option<Arc<dyn EventSink>> = if TypeId::of::<S>() == TypeId::of::<NoopSink>() {
None
} else {
Some(Arc::new(cfg.sink))
};
let mut worker_wakes = Vec::with_capacity(total);
for _ in 0..all_workers {
worker_wakes.push(
Wake::new()
.map(Arc::new)
.map_err(|e| NetError::DriverSetup {
driver: DriverKind::Blocking,
source: e,
})?,
);
}
#[cfg(feature = "reactor")]
let mut backends = Vec::with_capacity(total);
#[cfg(feature = "reactor")]
if reactor {
for _ in 0..all_workers {
backends.push(Some(Backend::new(1024)?));
}
}
#[cfg(not(feature = "reactor"))]
let _ = reactor;
let queue_capacity = limits.completion_queue as usize;
let next_listener = u32::try_from(infos.len()).unwrap_or(u32::MAX);
let shared = Arc::new_cyclic(|weak: &Weak<Shared>| Shared {
layout: layout.clone(),
pressure: limits.pressure,
pressured: AtomicBool::new(false),
#[cfg(feature = "udp")]
udp: cfg.udp,
pid: std::process::id(),
door: AtomicBool::new(false),
draining: AtomicBool::new(false),
stopping: AtomicBool::new(false),
stop: Mutex::new(None),
stop_cv: Condvar::new(),
progress: Mutex::new(()),
progress_cv: Condvar::new(),
workers: worker_wakes
.into_iter()
.enumerate()
.map(|(i, wake)| WorkerShared {
slot: Arc::new(KillSlot::new()),
wake,
current: AtomicU64::new(0),
listener: AtomicU32::new(NO_LISTENER),
lane: AtomicU8::new(0),
kill_cause: AtomicU8::new(CAUSE_USER),
idle_cause: AtomicU8::new(IDLE_DRAIN),
completer: Arc::new(CompleterShared::new(
queue_capacity,
Arc::new(WorkerTarget {
shared: weak.clone(),
idx: i,
}),
WorkerId(u16::try_from(i).unwrap_or(u16::MAX)),
)),
stats: WorkerStats::default(),
#[cfg(feature = "reactor")]
cmd: AtomicU32::new(0),
#[cfg(feature = "reactor")]
load: AtomicU32::new(0),
#[cfg(feature = "reactor")]
waker: OnceLock::new(),
#[cfg(feature = "reactor")]
dead: AtomicBool::new(false),
})
.collect(),
#[cfg(feature = "reactor")]
reactor,
#[cfg(feature = "reactor")]
next_worker: AtomicUsize::new(0),
#[cfg(feature = "reactor")]
conns: (0..CONN_SHARDS)
.map(|_| Mutex::new(HashMap::new()))
.collect(),
driver,
max_open: limits.max_connections,
soft_closes: crate::link::SoftCloses::new(limits.denial_linger_max),
shared_workers,
listeners: Mutex::new(entries),
next_listener: AtomicU32::new(next_listener),
spawn_accept: OnceLock::new(),
adopt: OnceLock::new(),
base_limits: cfg.limits.clone(),
trusted: cfg.trusted.clone(),
queue_notify: OnceLock::new(),
queue_len: AtomicUsize::new(0),
admitting: AtomicUsize::new(0),
#[cfg(feature = "reactor")]
listener_epoch: AtomicU64::new(0),
#[cfg(feature = "reactor")]
control_drain: AtomicBool::new(false),
control: cfg.control.unwrap_or_else(|| Arc::new(NoControl)),
sink,
open: AtomicU32::new(0),
busy: AtomicU32::new(0),
accepted: AtomicU64::new(0),
refused: AtomicU64::new(0),
refused_by: Default::default(),
descriptor_pressure: AtomicU64::new(0),
drained_graceful: AtomicU32::new(0),
drained_killed: AtomicU32::new(0),
workers_lost: AtomicU32::new(0),
workers_detached: AtomicU32::new(0),
running: AtomicUsize::new(0),
drain_timeout: limits.drain_timeout,
tls: cfg.tls,
});
for info in &infos {
shared.event(Event::ListenerOpened {
listener: info.id,
local: info.local.socket_addr(),
backlog: info.backlog,
});
}
#[cfg(feature = "reactor")]
for (i, b) in backends.iter().enumerate() {
if let Some(b) = b {
let _ = shared.workers[i].waker.set(crate::driver::Driver::waker(b));
}
}
let engine = Arc::new(Engine {
shared: shared.clone(),
#[cfg(feature = "reactor")]
inboxes: (0..backends.len())
.map(|_| Inbox(Mutex::new(VecDeque::new())))
.collect(),
#[cfg(feature = "reactor")]
drivers: Mutex::new(backends),
work: cfg.work,
gated: TypeId::of::<G>() != TypeId::of::<AllowAll>()
&& TypeId::of::<G>() != TypeId::of::<super::NoGate>(),
gate: Arc::new(cfg.gate),
queue: Mutex::new(VecDeque::with_capacity(total)),
queue_cv: Condvar::new(),
clock: cfg.clock.map_or(ClockRef::Monotonic, ClockRef::Custom),
wipe: cfg.wipe,
budget: Arc::new(AtomicU64::new(limits.buffer_budget)),
pool_keep: limits.buffer_budget / u64::from(cfg.workers.max(1)),
limits,
});
let weak = Arc::downgrade(&engine);
let _ = shared.queue_notify.set(Box::new(move || {
if let Some(e) = weak.upgrade() {
let _g = e.queue.lock().unwrap_or_else(PoisonError::into_inner);
e.queue_cv.notify_all();
}
}));
let weak = Arc::downgrade(&engine);
let _ = shared.adopt.set(Box::new(
move |stream, peer, local, conf, lane| match weak.upgrade() {
Some(e) => e.adopt(stream, peer, local, conf, lane),
None => Err(NetError::ShuttingDown),
},
));
let weak = Arc::downgrade(&engine);
let _ = shared.spawn_accept.set(Box::new(move |conf, rt, slot| {
let Some(e) = weak.upgrade() else {
return Err(std::io::Error::other("the server stopped"));
};
std::thread::Builder::new()
.name(format!("net-io-accept-{}", conf.id.0))
.spawn(move || accept_loop(&e, &conf, &rt, slot))
}));
let mut threads = Vec::with_capacity(total);
let spawn_error = |e: std::io::Error| NetError::DriverSetup {
driver: DriverKind::Blocking,
source: e,
};
let mut failure = None;
for i in 0..total {
let e = engine.clone();
let cpu = layout.cpu_of(i);
shared.running.fetch_add(1, Ordering::AcqRel);
let running = Running(shared.clone());
match std::thread::Builder::new()
.name(format!("net-io-worker-{i}"))
.stack_size(WORKER_STACK)
.spawn(move || {
let _running = running;
if let Some(cpu) = cpu {
let _ = sys::pin_current_thread(cpu);
}
W::run_worker(&e, i);
}) {
Ok(t) => threads.push(t),
Err(err) => {
failure = Some(spawn_error(err));
break;
}
}
}
if failure.is_none() {
if let Err(err) = shared.spawn_listener_threads() {
failure = Some(spawn_error(err));
}
}
drop(engine);
if let Some(err) = failure {
shared.stop_threads();
for t in threads {
let _ = t.join();
}
shared.join_listeners();
return Err(err);
}
Ok(Server {
shared,
threads,
listeners: infos,
stopped: false,
})
}
#[cfg(feature = "udp")]
const UDP_TURN: usize = 64;
#[cfg(feature = "udp")]
fn udp_loop(
shared: &Arc<Shared>,
conf: &Arc<ListenerConf>,
socket: &Mutex<crate::udp::UdpSocket>,
info: &ListenerInfo,
) {
let Some(factory) = shared.udp.as_ref() else {
return;
};
let mut session = factory.open(info);
let mut socket = socket.lock().unwrap_or_else(PoisonError::into_inner);
let wake = conf.wake.0[0].clone();
let mut buf = vec![0u8; 65_536];
loop {
if shared.stopping.load(Ordering::Acquire)
|| shared.draining.load(Ordering::Acquire)
|| conf.removed.load(Ordering::Acquire)
{
return;
}
if !shared.door.load(Ordering::Acquire) || !conf.door.load(Ordering::Acquire) {
let _ = sys::wait_wake(&wake, None);
continue;
}
let _ = sys::wait(socket.raw_sock(), Interest::Read, Some(&wake), None);
for _ in 0..UDP_TURN {
let Ok(d) = socket.recv_from(&mut buf) else {
break;
};
let mut cx = crate::udp::UdpContext {
socket: &mut socket,
};
let bytes = &buf[..d.len.min(65_536)];
let called = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
session.on_datagram(&mut cx, d.from, bytes);
}));
let _ = called;
}
}
}
fn accept_loop<W: super::Workload, G: Gate>(
engine: &Arc<Engine<W, G>>,
conf: &Arc<ListenerConf>,
rt: &ListenerRuntime,
slot: usize,
) {
let shared = &engine.shared;
#[cfg(feature = "udp")]
if let ListenerRuntime::Udp(socket, info) = rt {
return udp_loop(shared, conf, socket, info);
}
let wake = conf.wake.0[slot].clone();
if conf.wake.0.len() > 1 {
FEEDS.with(|f| f.set(Some((slot, false))));
}
let mut st = AcceptState::new();
loop {
if shared.stopping.load(Ordering::Acquire) || conf.removed.load(Ordering::Acquire) {
return;
}
if st.view.refresh(shared.control.as_ref()) {
shared.event(Event::ControlReacted {
state: st.view.state(),
});
match st.view.state() {
ControlState::Draining => {
shared.close_door();
shared.request_stop(st.view.deadline());
let deadline = st
.view
.deadline()
.unwrap_or_else(|| Instant::now() + shared.drain_timeout);
shared.drain(deadline);
}
ControlState::Stopping => {
shared.close_door();
shared.request_stop(Some(Instant::now()));
shared.kill_all(CAUSE_STOP);
}
ControlState::Shed { from_lane } => shared.shed_idle(from_lane),
_ => {}
}
}
let now = engine.clock.now();
if !accepting(shared, conf) {
let _ = sys::wait_wake(&wake, None);
continue;
}
if let Some(until) = st.pause_until {
if now < until {
let _ = sys::wait_wake(&wake, Some(until - now));
continue;
}
st.pause_until = None;
}
match engine.accept_some(conf, rt, &mut st, &wake) {
BatchEnd::More => {}
BatchEnd::Again => rt.wait(&wake),
BatchEnd::Failed => return,
}
}
}
fn accepting(shared: &Shared, conf: &ListenerConf) -> bool {
shared.door.load(Ordering::Acquire)
&& conf.door.load(Ordering::Acquire)
&& !shared.draining.load(Ordering::Acquire)
}
pub(crate) struct AcceptState {
acceptor: Acceptor,
ids: IdBlock,
view: ControlView,
coalescer: Coalescer,
pause_until: Option<Instant>,
skipped: u32,
}
impl AcceptState {
pub(crate) fn new() -> AcceptState {
AcceptState {
acceptor: Acceptor::new(),
ids: IdBlock::new(),
view: ControlView::new(),
coalescer: Coalescer::new(),
pause_until: None,
skipped: 0,
}
}
}
enum BatchEnd {
More,
Again,
Failed,
}
#[cfg(feature = "reactor")]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum WorkerAccept {
Armed,
Kick,
Park,
Paused(Instant),
Gone,
}
impl<W: super::Workload, G: Gate> Engine<W, G> {
fn accept_some(
&self,
conf: &Arc<ListenerConf>,
rt: &ListenerRuntime,
st: &mut AcceptState,
wake: &Wake,
) -> BatchEnd {
let shared = &self.shared;
let _ = wake;
for _ in 0..conf.limits.accept_batch {
let admitting = Admitting::enter(shared);
if !shared.door.load(Ordering::SeqCst)
|| !conf.door.load(Ordering::SeqCst)
|| shared.draining.load(Ordering::Acquire)
|| shared.stopping.load(Ordering::Acquire)
|| st.view.moved(shared.control.as_ref())
{
break;
}
let outcome = match rt {
ListenerRuntime::Tcp(l) => match st.acceptor.accept(l.std()) {
Accepted::Conn((stream, peer)) => {
st.skipped = 0;
self.admit_tcp(
stream,
peer,
conf,
&mut st.ids,
&st.view,
&mut st.coalescer,
);
continue;
}
other => other.without_conn(),
},
#[cfg(all(unix, feature = "local"))]
ListenerRuntime::Unix(l) => match st.acceptor.accept_unix(l.std()) {
Accepted::Conn(stream) => {
st.skipped = 0;
self.admit_unix(l, stream, conf, &mut st.ids, &st.view, &mut st.coalescer);
continue;
}
other => other.without_conn(),
},
#[cfg(feature = "udp")]
ListenerRuntime::Udp(..) => return BatchEnd::Failed,
#[cfg(all(windows, feature = "local"))]
ListenerRuntime::AfUnix(l) => match st.acceptor.accept_af_unix(l.std()) {
Accepted::Conn(stream) => {
st.skipped = 0;
self.admit_af_unix(
l,
stream,
conf,
&mut st.ids,
&st.view,
&mut st.coalescer,
);
continue;
}
other => other.without_conn(),
},
#[cfg(all(windows, feature = "local"))]
ListenerRuntime::Pipe(l) => match l.accept_or_wake(wake) {
Ok(Some(stream)) => {
st.skipped = 0;
self.admit_pipe(l, stream, conf, &mut st.ids, &st.view, &mut st.coalescer);
continue;
}
Ok(None) => break,
Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => Accepted::Failed(e),
Err(_) => st.acceptor.pause(),
},
};
drop(admitting);
match outcome {
Accepted::Conn(()) => {}
Accepted::Skipped(class) => {
shared.event(Event::AcceptError {
listener: conf.id,
class,
});
st.skipped += 1;
if st.skipped >= SKIPS_BEFORE_PAUSE {
st.skipped = 0;
if let Accepted::Pause(backoff) = st.acceptor.pause() {
st.pause_until = Some(self.clock.now() + backoff);
break;
}
}
}
Accepted::Again => {
st.skipped = 0;
if let Some(sink) = &shared.sink {
st.coalescer.flush(&**sink);
}
return BatchEnd::Again;
}
Accepted::Pause(backoff) => {
shared.descriptor_pressure.fetch_add(1, Ordering::Relaxed);
shared.event(Event::DescriptorPressure {
listener: conf.id,
backoff,
});
st.pause_until = Some(self.clock.now() + backoff);
break;
}
Accepted::Failed(e) => {
shared.event(Event::ListenerFailed {
listener: conf.id,
class: OsErrorClass::from_io(&e),
});
return BatchEnd::Failed;
}
}
}
BatchEnd::More
}
#[cfg(feature = "reactor")]
pub(crate) fn worker_accept(
&self,
conf: &Arc<ListenerConf>,
rt: &ListenerRuntime,
idx: usize,
st: &mut AcceptState,
) -> WorkerAccept {
let shared = &self.shared;
if shared.stopping.load(Ordering::Acquire) || conf.removed.load(Ordering::Acquire) {
return WorkerAccept::Gone;
}
if st.view.refresh(shared.control.as_ref()) {
shared.event(Event::ControlReacted {
state: st.view.state(),
});
match st.view.state() {
ControlState::Draining => {
shared.close_door();
shared.request_stop(st.view.deadline());
if !shared.control_drain.swap(true, Ordering::AcqRel) {
let deadline = st
.view
.deadline()
.unwrap_or_else(|| Instant::now() + shared.drain_timeout);
let s = shared.clone();
let _ = std::thread::Builder::new()
.name(String::from("net-io-drain"))
.spawn(move || {
let _ = s.drain(deadline);
});
}
}
ControlState::Stopping => {
shared.close_door();
shared.request_stop(Some(Instant::now()));
shared.kill_all(CAUSE_STOP);
}
ControlState::Shed { from_lane } => shared.shed_idle(from_lane),
_ => {}
}
}
if !accepting(shared, conf) {
return WorkerAccept::Park;
}
if let Some(until) = st.pause_until {
if self.clock.now() < until {
return WorkerAccept::Paused(until);
}
st.pause_until = None;
}
FEEDS.with(|f| f.set(Some((idx, true))));
let end = self.accept_some(conf, rt, st, &conf.wake.0[0]);
FEEDS.with(|f| f.set(None));
match end {
BatchEnd::Again => WorkerAccept::Armed,
BatchEnd::Failed => WorkerAccept::Gone,
BatchEnd::More => match st.pause_until {
Some(until) => WorkerAccept::Paused(until),
None if !accepting(shared, conf) => WorkerAccept::Park,
None => WorkerAccept::Kick,
},
}
}
#[cfg(feature = "reactor")]
pub(crate) fn accepting(&self, conf: &ListenerConf) -> bool {
accepting(&self.shared, conf)
}
}
impl<W, G: Gate> Engine<W, G> {
fn refuse(
&self,
stream: sys::Stream,
peer: Option<SocketAddr>,
cause: RefusalCause,
bytes: Option<&[u8]>,
co: &mut Coalescer,
) {
let shared = &self.shared;
shared.refused.fetch_add(1, Ordering::Relaxed);
shared.refused_by[cause.index()].fetch_add(1, Ordering::Relaxed);
let mut sent = false;
if let Some(b) = bytes {
let max = usize::from(self.limits.refusal_bytes_max);
if b.len() <= max {
sent = sys::send(&stream, b).is_ok_and(|n| n > 0);
}
}
if sent && !self.limits.denial_linger.is_zero() {
let _ = stream.shutdown(std::net::Shutdown::Write);
} else {
let _ = sys::set_linger(stream.sock(), Some(Duration::ZERO));
}
drop(stream);
if let Some(sink) = &shared.sink {
co.emit(
sink.as_ref(),
self.clock.now(),
CoalescedKind::Refused(cause),
Event::Refused { peer, cause },
);
}
}
fn admit_tcp(
&self,
stream: TcpStream,
peer: SocketAddr,
conf: &Arc<ListenerConf>,
ids: &mut IdBlock,
view: &ControlView,
co: &mut Coalescer,
) {
let source = PrefixKey::of(peer.ip(), conf.limits.source_policy);
let id = conf.id;
let _ = self.admit_one(
sys::Stream::Tcp(stream),
Some(peer),
Some(source),
conf,
&mut || ids.issue(),
view,
co,
AdmitStage::Accept,
|stream, cid, at| {
let local = stream.tcp()?.local_addr().ok()?;
Some(ConnectionInfo::new_tcp(
cid,
Some(id),
WorkerId(0),
Origin::Accepted,
local,
peer,
at,
))
},
);
}
#[cfg(all(unix, feature = "local"))]
fn admit_unix(
&self,
listener: &UnixListener,
stream: std::os::unix::net::UnixStream,
conf: &Arc<ListenerConf>,
ids: &mut IdBlock,
view: &ControlView,
co: &mut Coalescer,
) {
let credentials = crate::local::credentials_of(&stream);
let stream = sys::Stream::Unix(stream);
if !listener.permits(credentials) {
return self.refuse(stream, None, RefusalCause::Credentials, None, co);
}
let path = listener.shared_path().clone();
let id = conf.id;
let next = &mut || ids.issue();
let stage = AdmitStage::Accept;
let _ = self.admit_one(
stream,
None,
None,
conf,
next,
view,
co,
stage,
|_, cid, at| {
Some(ConnectionInfo::new_unix(
cid,
Some(id),
Origin::Accepted,
Endpoint::Unix(path),
Endpoint::Unnamed,
credentials,
at,
))
},
);
}
#[cfg(all(windows, feature = "local"))]
fn admit_af_unix(
&self,
listener: &crate::local_windows::UnixListener,
stream: TcpStream,
conf: &Arc<ListenerConf>,
ids: &mut IdBlock,
view: &ControlView,
co: &mut Coalescer,
) {
let next = &mut || ids.issue();
let stage = AdmitStage::Accept;
let _ = self.admit_one(
sys::Stream::Tcp(stream),
None,
None,
conf,
next,
view,
co,
stage,
|stream, cid, at| stream.tcp().map(|s| listener.info_of(s, cid, at)),
);
}
#[cfg(all(windows, feature = "local"))]
fn admit_pipe(
&self,
listener: &crate::pipe::PipeListener,
stream: sys::PipeStream,
conf: &Arc<ListenerConf>,
ids: &mut IdBlock,
view: &ControlView,
co: &mut Coalescer,
) {
let stream = sys::Stream::Pipe(stream);
let next = &mut || ids.issue();
let stage = AdmitStage::Accept;
let _ = self.admit_one(
stream,
None,
None,
conf,
next,
view,
co,
stage,
|stream, cid, at| match stream {
sys::Stream::Pipe(p) => Some(listener.info_of(p, cid, at)),
_ => None,
},
);
}
#[allow(clippy::too_many_arguments)]
fn admit_one(
&self,
stream: sys::Stream,
peer: Option<SocketAddr>,
source: Option<PrefixKey>,
conf: &Arc<ListenerConf>,
next_id: &mut dyn FnMut() -> ConnectionId,
view: &ControlView,
co: &mut Coalescer,
stage: AdmitStage,
make_info: impl FnOnce(&sys::Stream, ConnectionId, Instant) -> Option<ConnectionInfo>,
) -> Result<ConnectionId, Option<RefusalCause>> {
let shared = &self.shared;
let lim = &conf.limits;
let count = &conf.count;
if !take_slot(&shared.open, shared.max_open) {
{
self.refuse(stream, peer, RefusalCause::GlobalCap, None, co);
return Err(Some(RefusalCause::GlobalCap));
}
}
if !take_slot(count, lim.max_per_listener) {
shared.open.fetch_sub(1, Ordering::AcqRel);
{
self.refuse(stream, peer, RefusalCause::ListenerCap, None, co);
return Err(Some(RefusalCause::ListenerCap));
}
}
if let Some(source) = source {
if !conf.sources.acquire(source) {
shared.open.fetch_sub(1, Ordering::AcqRel);
count.fetch_sub(1, Ordering::AcqRel);
{
self.refuse(stream, peer, RefusalCause::SourceCap, None, co);
return Err(Some(RefusalCause::SourceCap));
}
}
}
shared.update_pressure();
let release = |s: &Self| s.shared.release(conf, source);
if matches!(view.state(), ControlState::Lockdown) && !conf.lockdown_allowed {
release(self);
{
self.refuse(stream, peer, RefusalCause::Lockdown, None, co);
return Err(Some(RefusalCause::Lockdown));
}
}
let id = next_id();
let Some(mut info) = make_info(&stream, id, self.clock.now()) else {
release(self);
drop(stream);
return Err(None);
};
let admitted = match guarded(|| self.gate.admit(&info, stage)) {
Some(Ok(a)) => a,
Some(Err(refusal)) => {
release(self);
let (code, bytes) = match refusal {
Refusal::Reset { code } => (code, None),
Refusal::WithBytes { code, bytes } => (code, Some(bytes)),
#[allow(unreachable_patterns)]
_ => (refusal.code(), None),
};
{
self.refuse(stream, peer, RefusalCause::Gate(code), bytes, co);
return Err(Some(RefusalCause::Gate(code)));
}
}
None => {
release(self);
{
self.refuse(stream, peer, RefusalCause::Gate(0), None, co);
return Err(Some(RefusalCause::Gate(0)));
}
}
};
info.lane = admitted.lane;
info.gate = admitted.tag;
let ticket = admitted.ticket;
let reserved = lim.reserved_above(info.lane);
if reserved > 0
&& shared.open.load(Ordering::Acquire).saturating_add(reserved) > shared.max_open
{
let _ = guarded(|| {
self.gate.closed(
ticket,
&info,
CloseReason::Refused(RefusalCause::LaneReserve),
)
});
release(self);
self.refuse(stream, peer, RefusalCause::LaneReserve, None, co);
return Err(Some(RefusalCause::LaneReserve));
}
if let ControlState::Shed { from_lane } = view.state() {
if info.lane >= from_lane {
let _ = guarded(|| {
self.gate
.closed(ticket, &info, CloseReason::Refused(RefusalCause::Shed))
});
release(self);
{
self.refuse(stream, peer, RefusalCause::Shed, None, co);
return Err(Some(RefusalCause::Shed));
}
}
}
let job = Job {
stream,
info,
ticket,
listener: conf.clone(),
source,
};
let (event_id, event_lane) = (job.info.id, job.info.lane);
if let Err(job) = self.hand_over(job) {
let Job {
stream,
info,
ticket,
..
} = job;
let _ = guarded(|| {
self.gate
.closed(ticket, &info, CloseReason::Refused(RefusalCause::Busy))
});
release(self);
self.refuse(stream, peer, RefusalCause::Busy, None, co);
return Err(Some(RefusalCause::Busy));
}
shared.accepted.fetch_add(1, Ordering::Relaxed);
let _ = (peer, event_lane);
Ok(event_id)
}
#[allow(clippy::result_large_err)]
fn hand_over(&self, job: Job<G::Ticket>) -> Result<(), Job<G::Ticket>> {
#[cfg(feature = "reactor")]
if self.shared.reactor {
return self.hand_to_worker(job);
}
let shared = &self.shared;
let mut queue = self.queue.lock().unwrap_or_else(PoisonError::into_inner);
let busy = shared.busy.load(Ordering::Acquire) as usize;
if queue.len() + busy >= shared.workers.len() {
return Err(job);
}
queue.push_back(job);
shared.queue_len.store(queue.len(), Ordering::Release);
drop(queue);
self.queue_cv.notify_one();
Ok(())
}
fn adopt(
&self,
stream: TcpStream,
peer: SocketAddr,
local: SocketAddr,
conf: &Arc<ListenerConf>,
lane: Option<Lane>,
) -> Result<ConnectionId, NetError> {
let shared = &self.shared;
if shared.forked() {
return Err(NetError::ForkedChild);
}
if shared.draining.load(Ordering::Acquire) || shared.stopping.load(Ordering::Acquire) {
return Err(NetError::ShuttingDown);
}
let mut view = ControlView::new();
view.refresh(shared.control.as_ref());
let mut co = Coalescer::new();
let peer = crate::addr::normalize_socket(peer);
let source = PrefixKey::of(peer.ip(), conf.limits.source_policy);
let listener = conf.id;
let result = self.admit_one(
sys::Stream::Tcp(stream),
Some(peer),
Some(source),
conf,
&mut crate::id::issue_single,
&view,
&mut co,
AdmitStage::Adopt,
|_, cid, at| {
let mut info = ConnectionInfo::new_tcp(
cid,
Some(listener),
WorkerId(0),
Origin::Adopted,
local,
peer,
at,
);
if let Some(l) = lane {
info.lane = l;
}
Some(info)
},
);
match result {
Ok(id) => Ok(id),
Err(Some(cause)) => Err(NetError::Refused { cause }),
Err(None) => Err(NetError::Closed),
}
}
fn next_job(&self) -> Option<Job<G::Ticket>> {
let mut queue = self.queue.lock().unwrap_or_else(PoisonError::into_inner);
loop {
if self.shared.stopping.load(Ordering::Acquire) && queue.is_empty() {
return None;
}
if let Some(job) = queue.pop_front() {
self.shared.busy.fetch_add(1, Ordering::AcqRel);
self.shared.queue_len.store(queue.len(), Ordering::Release);
return Some(job);
}
queue = self
.queue_cv
.wait(queue)
.unwrap_or_else(PoisonError::into_inner);
}
}
}
pub trait Serve: Send + Sync + Sized + 'static {
const BLOCKING_ONLY: bool;
const UDP_ONLY: bool = false;
fn serve<G: Gate>(&self, w: &mut Worker<G>, job: Job<G::Ticket>);
#[allow(private_interfaces)]
fn run_worker<G: Gate>(engine: &Arc<Engine<Self, G>>, idx: usize) {
worker_loop(engine, idx);
}
}
pub struct Worker<G: Gate> {
shared: Arc<Shared>,
idx: usize,
gated: bool,
gate: Arc<G>,
gate_cell: Option<Box<GateCell<G>>>,
pool: BufferPool,
output: Output,
input_buf: Option<Buffer>,
completer: Completer,
view: ControlView,
clock: ClockRef,
facts_pool: Vec<Arc<ProxyFacts>>,
holding: bool,
}
fn worker_loop<W: Serve, G: Gate>(engine: &Arc<Engine<W, G>>, idx: usize) {
let shared = engine.shared.clone();
let ws = &shared.workers[idx];
ws.wake.bind_thread(std::thread::current());
ws.slot.set_wake(Some(ws.wake.clone()));
let mut w = Worker {
shared: shared.clone(),
idx,
gated: engine.gated,
gate: engine.gate.clone(),
gate_cell: Some(Box::new(GateCell {
gate: engine.gate.clone(),
ticket: None,
})),
pool: BufferPool::new(engine.budget.clone(), engine.wipe, engine.pool_keep),
output: Output::new(
engine.budget.clone(),
engine.limits.output_budget,
engine.wipe,
),
input_buf: None,
completer: Completer {
shared: ws.completer.clone(),
},
view: ControlView::new(),
clock: engine.clock.clone(),
facts_pool: Vec::new(),
holding: false,
};
while let Some(job) = engine.next_job() {
let (listener, source, info) = (job.listener.clone(), job.source, job.info.clone());
w.holding = true;
let served = guarded(|| engine.work.serve(&mut w, job)).is_some();
if w.holding {
let reason = if served {
CloseReason::SessionError
} else {
CloseReason::SessionPanicked
};
w.abandon(&info, &listener, source, reason);
}
w.holding = false;
}
}
impl<G: Gate> Worker<G> {
fn recycle(&mut self, info: ConnectionInfo) {
if let Some(mut facts) = info.proxy {
if Arc::get_mut(&mut facts).is_some() && self.facts_pool.len() < 4 {
self.facts_pool.push(facts);
}
}
}
#[cfg(feature = "tls")]
fn tls_handshake(&mut self, link: &mut Link) -> Result<(), CloseReason> {
let Some(acceptor) = self.shared.tls.clone() else {
return Err(CloseReason::TlsFailed);
};
let session = match acceptor.session_boxed(&link.info) {
Ok(s) => s,
Err(e) if e.kind == crate::tls::TlsFailure::HelloRefused => {
return Err(CloseReason::HelloRefused);
}
Err(_) => return Err(CloseReason::TlsFailed),
};
let lim = *link.limits;
let Some(cin) = self.pool.take(lim.read_class_max) else {
return Err(CloseReason::Limit(crate::error::LimitKind::BufferBudget));
};
let Some(cout) = self.pool.take(SizeClass::LARGE) else {
self.pool.put(cin);
return Err(CloseReason::Limit(crate::error::LimitKind::BufferBudget));
};
if let Some(ahead) = link.attach_tls(session, cin, cout) {
self.pool.put(ahead);
}
link.set_deadline_scale(self.shared.deadline_scale());
link.begin_handshake(link.info.accepted_at + link.scaled(lim.handshake_timeout));
let r = link.tls_handshake();
let _ = link.end_handshake(None);
match r {
Ok(()) => Ok(()),
Err(_) => Err(link.reason_or(CloseReason::TlsFailed)),
}
}
#[cfg(feature = "proxy")]
fn proxy_handshake(
&mut self,
link: &mut Link,
proxy: &ListenerProxy,
listener: &ListenerConf,
source: &mut Option<PrefixKey>,
) -> Result<(), CloseReason> {
use crate::proxy_parse::{ProxyAddress, ProxyCommand, ProxyParser, ProxyProgress};
if !proxy.is_trusted(&link.info) {
return Err(CloseReason::ProxyUntrusted);
}
let lim = *link.limits;
let need = 16 + u32::from(lim.proxy_header_max);
let class = if need <= SizeClass::SMALL.bytes() {
SizeClass::SMALL
} else if need <= SizeClass::MEDIUM.bytes() {
SizeClass::MEDIUM
} else {
SizeClass::LARGE
};
let Some(mut buf) = self.pool.take(class) else {
return Err(CloseReason::Limit(crate::error::LimitKind::BufferBudget));
};
let body_max = u16::try_from(buf.capacity().saturating_sub(16))
.unwrap_or(u16::MAX)
.min(lim.proxy_header_max);
let mut parser = ProxyParser::new(proxy.versions, body_max);
link.set_deadline_scale(self.shared.deadline_scale());
link.begin_handshake(link.info.accepted_at + link.scaled(lim.handshake_timeout));
let outcome = loop {
match parser.feed(buf.as_slice()) {
Ok(ProxyProgress::Done { header, consumed }) => {
let real = match (header.command, header.source) {
(ProxyCommand::Proxy, Some(ProxyAddress::Ip(a))) => {
Some(crate::addr::normalize_socket(a))
}
_ => None,
};
break Ok((header.to_facts(), real, consumed));
}
Ok(ProxyProgress::NeedMore { .. }) => {}
Err(crate::proxy::ProxyError::NotProxy) => break Err(CloseReason::ProxyMissing),
Err(_) => break Err(CloseReason::ProxyInvalid),
}
if buf.spare().is_empty() {
break Err(CloseReason::ProxyInvalid);
}
match link.read(buf.spare()) {
Ok(Got::Data(n)) => buf.set_len(buf.len() + n),
Ok(Got::Eof) => break Err(CloseReason::ProxyMissing),
Ok(Got::Woken) => {}
Err(_) => break Err(link.reason_or(CloseReason::Io(OsErrorClass::Other(0)))),
}
};
let (facts, real, consumed) = match outcome {
Ok(v) => v,
Err(reason) => {
if let Some(b) = link.end_handshake(Some((buf, usize::MAX))) {
self.pool.put(b);
}
return Err(reason);
}
};
if let Some(b) = link.end_handshake(Some((buf, consumed))) {
self.pool.put(b);
}
proxy_resolved(
&self.shared,
link,
listener,
source,
facts,
real,
&mut self.facts_pool,
)
}
#[allow(clippy::type_complexity)]
fn begin(
&mut self,
job: Job<G::Ticket>,
) -> Option<(Link, Arc<ListenerConf>, Option<PrefixKey>)> {
let Job {
stream,
mut info,
ticket,
listener,
source,
} = job;
let shared = self.shared.clone();
let limits = listener.limits.clone();
info.worker = WorkerId(self.idx as u16);
announce(&shared, &info, &listener);
let mut cell = self.gate_cell.take().unwrap_or_else(|| {
Box::new(GateCell {
gate: self.gate.clone(),
ticket: None,
})
});
cell.ticket = Some(ticket);
let mut failed = None;
if !cfg!(any(target_os = "linux", windows)) {
let tcp = listener.tcp;
let v6 = info.local.socket_addr().is_some_and(|a| a.is_ipv6());
if let (false, Some(stream)) = (tcp.is_empty(), stream.tcp()) {
if let Err(e) = tcp.apply(stream, v6) {
failed = Some(match e.os_error_code() {
Some(code) => CloseReason::Io(OsErrorClass::Other(code)),
None => CloseReason::Io(OsErrorClass::Other(0)),
});
}
}
}
let ws = &shared.workers[self.idx];
ws.kill_cause.store(CAUSE_USER, Ordering::Release);
ws.idle_cause.store(IDLE_DRAIN, Ordering::Release);
ws.lane.store(info.lane.0, Ordering::Release);
ws.listener
.store(u32::from(listener.id.0), Ordering::Release);
let mut link = Link::new(
stream,
info,
limits,
self.clock.clone(),
ws.slot.clone(),
WakeSource::Fixed(ws.wake.clone()),
Phase::Idle,
);
link.set_soft_closes(&shared.soft_closes);
ws.current.store(link.info.id.get(), Ordering::Release);
if self.gated {
link.gate = Some(cell as Box<dyn ErasedGate>);
} else {
self.gate_cell = Some(cell);
}
let failed = if listener.removed.load(Ordering::Acquire) {
Some(CloseReason::Drained)
} else if self.shared.stopping.load(Ordering::Acquire) {
Some(CloseReason::Shutdown)
} else {
failed
};
if let Some(reason) = failed {
self.end(link, reason, listener, source);
return None;
}
#[cfg(feature = "proxy")]
let mut source = source;
#[cfg(feature = "proxy")]
if let Some(proxy) = listener.proxy.clone() {
if let Err(reason) = self.proxy_handshake(&mut link, &proxy, &listener, &mut source) {
self.end(link, reason, listener, source);
return None;
}
}
#[cfg(feature = "tls")]
if listener.tls {
if let Err(reason) = self.tls_handshake(&mut link) {
self.end(link, reason, listener, source);
return None;
}
}
Some((link, listener, source))
}
fn end(
&mut self,
link: Link,
reason: CloseReason,
listener: Arc<ListenerConf>,
source: Option<PrefixKey>,
) -> (ConnectionInfo, CloseReason) {
let shared = self.shared.clone();
let ws = &shared.workers[self.idx];
let mut link = link;
if let Some(b) = link.take_pending() {
self.pool.put(b);
}
let mut reason = reason;
if let Some(k) = link.killed() {
reason = match (ws.kill_cause.load(Ordering::Acquire), k == DRAIN_KILL) {
(CAUSE_DRAIN, true) => CloseReason::DrainDeadline,
(CAUSE_STOP, true) => CloseReason::Shutdown,
_ => CloseReason::Killed(k),
};
} else if reason == CloseReason::Drained
&& ws.idle_cause.load(Ordering::Acquire) == IDLE_SHED
{
reason = CloseReason::Shed;
} else if shared.draining.load(Ordering::Acquire) && !reason.is_abortive() {
reason = CloseReason::Drained;
}
let info = link.info.clone();
let (bytes_in, bytes_out) = (link.bytes_in, link.bytes_out);
let erased = link.gate.take();
let lifetime = self.clock.now().saturating_duration_since(info.accepted_at);
if !reason.is_abortive() {
link.tls_close_notify();
}
if let Some((cin, cout)) = link.take_tls_buffers() {
self.pool.put(cin);
self.pool.put(cout);
}
let own =
|k: KillReason| k == DRAIN_KILL && ws.kill_cause.load(Ordering::Acquire) != CAUSE_USER;
let reason = link.finish_with(reason, own);
match erased {
Some(mut g) => {
g.close(&info, reason);
if let Ok(cell) = g.into_any().downcast::<GateCell<G>>() {
self.gate_cell = Some(cell);
}
}
None => {
if let Some(cell) = self.gate_cell.as_mut() {
cell.close(&info, reason);
}
}
}
self.output.reset();
ws.current.store(0, Ordering::Release);
ws.listener.store(NO_LISTENER, Ordering::Release);
count_drained(&shared, reason);
self.holding = false;
shared.release(&listener, source);
record_close(&shared, ws, &info, reason, bytes_in, bytes_out, lifetime);
shared.busy.fetch_sub(1, Ordering::AcqRel);
shared.notify_progress();
(info, reason)
}
fn draining(&self) -> bool {
self.shared.draining.load(Ordering::Acquire)
}
fn abandon(
&mut self,
info: &ConnectionInfo,
listener: &ListenerConf,
source: Option<PrefixKey>,
reason: CloseReason,
) {
let shared = self.shared.clone();
let ws = &shared.workers[self.idx];
self.output.reset();
self.input_buf = None;
if self.gate_cell.is_none() {
self.gate_cell = Some(Box::new(GateCell {
gate: self.gate.clone(),
ticket: None,
}));
}
ws.current.store(0, Ordering::Release);
ws.listener.store(NO_LISTENER, Ordering::Release);
count_drained(&shared, reason);
shared.release(listener, source);
let lifetime = self.clock.now().saturating_duration_since(info.accepted_at);
record_close(&shared, ws, info, reason, 0, 0, lifetime);
shared.busy.fetch_sub(1, Ordering::AcqRel);
shared.notify_progress();
}
fn control(&mut self) -> Option<CloseReason> {
if self.shared.stopping.load(Ordering::Acquire) {
return Some(CloseReason::Shutdown);
}
if self.view.refresh(self.shared.control.as_ref()) {
self.shared.event(Event::ControlReacted {
state: self.view.state(),
});
}
match self.view.state() {
ControlState::Stopping => Some(CloseReason::Shutdown),
_ => None,
}
}
fn shed(&self, lane: Lane) -> bool {
matches!(self.view.state(), ControlState::Shed { from_lane } if lane >= from_lane)
}
fn pressure(&self) -> f32 {
if matches!(self.view.state(), ControlState::Shed { .. }) {
return 1.0;
}
let busy = self.shared.busy.load(Ordering::Relaxed) as f32;
let total = self.shared.workers.len().max(1) as f32;
(busy / total).min(1.0)
}
}
impl<H> Serve for BlockingHandler<H>
where
H: Fn(&mut TcpConnection) -> Result<(), NetError> + Send + Sync + 'static,
{
const BLOCKING_ONLY: bool = true;
fn serve<G: Gate>(&self, w: &mut Worker<G>, job: Job<G::Ticket>) {
let Some((link, listener, source)) = w.begin(job) else {
return;
};
let mut conn = TcpConnection::from_link(link);
let result = guarded(|| (self.handler)(&mut conn));
let Some(link) = conn.take_link() else { return };
drop(conn);
let reason = match result {
None => CloseReason::SessionPanicked,
Some(Ok(())) if link.peer_closed => link.reason_or(CloseReason::PeerClosed),
Some(Ok(())) => link.reason_or(CloseReason::SessionClosed),
Some(Err(_)) => link.reason_or(CloseReason::SessionError),
};
let (info, _) = w.end(link, reason, listener, source);
w.recycle(info);
}
}
#[cfg(feature = "udp")]
impl Serve for super::UdpOnly {
const BLOCKING_ONLY: bool = true;
const UDP_ONLY: bool = true;
fn serve<G: Gate>(&self, _w: &mut Worker<G>, _job: Job<G::Ticket>) {}
}
impl<F: SessionFactory> Serve for Sessions<F> {
const BLOCKING_ONLY: bool = false;
#[cfg(feature = "reactor")]
#[allow(private_interfaces)]
fn run_worker<G: Gate>(engine: &Arc<Engine<Self, G>>, idx: usize) {
if engine.shared.reactor {
super::reactor::run(engine, idx);
} else {
worker_loop(engine, idx);
}
}
fn serve<G: Gate>(&self, w: &mut Worker<G>, job: Job<G::Ticket>) {
let Some((mut link, listener, source)) = w.begin(job) else {
return;
};
let mut session = match guarded(|| self.factory.open(&link.info)) {
Some(Ok(s)) => s,
Some(Err(cause)) => {
w.end(link, CloseReason::Refused(cause), listener, source);
return;
}
None => {
w.end(link, CloseReason::SessionPanicked, listener, source);
return;
}
};
let class = link.limits.read_class;
let buf = match w.input_buf.take() {
Some(b) if b.class() == class => Some(b),
Some(other) => {
w.pool.put(other);
w.pool.take(class)
}
None => w.pool.take(class),
};
let reason = match buf {
None => CloseReason::Limit(crate::error::LimitKind::BufferBudget),
Some(buf) => {
let mut input = Input::new(buf);
let reason = drive(w, &mut link, &mut session, &mut input);
let mut buf = input.buf;
buf.set_len(buf.capacity());
w.pool.put(buf);
w.input_buf = w.pool.take(class);
reason
}
};
let (info, reason) = w.end(link, reason, listener, source);
let _ = guarded(|| session.on_close(&info, reason));
drop(session);
w.recycle(info);
}
}
pub(crate) fn announce(shared: &Shared, info: &ConnectionInfo, listener: &ListenerConf) {
shared.event(Event::Accepted {
id: info.id,
peer: info.peer.socket_addr(),
listener: listener.id,
});
shared.event(Event::Admitted {
id: info.id,
lane: info.lane,
});
}
pub(crate) fn count_drained(shared: &Shared, reason: CloseReason) {
if shared.draining.load(Ordering::Acquire) {
if reason == CloseReason::DrainDeadline {
shared.drained_killed.fetch_add(1, Ordering::Relaxed);
} else {
shared.drained_graceful.fetch_add(1, Ordering::Relaxed);
}
}
}
pub(crate) fn record_close(
shared: &Shared,
ws: &WorkerShared,
info: &ConnectionInfo,
reason: CloseReason,
bytes_in: u64,
bytes_out: u64,
lifetime: Duration,
) {
let st = &ws.stats;
let bump = |c: &AtomicU64, n: u64| c.store(c.load(Ordering::Relaxed) + n, Ordering::Relaxed);
st.begin_write();
bump(&st.closed, 1);
bump(&st.bytes_in, bytes_in);
bump(&st.bytes_out, bytes_out);
bump(&st.closed_by[reason.index()], 1);
match reason {
CloseReason::EgressDenied(_) => bump(&st.egress_denied, 1),
CloseReason::Killed(_) => bump(&st.killed, 1),
CloseReason::SlowPeer { .. } => bump(&st.slow_peers, 1),
CloseReason::Idle
| CloseReason::ReadTimeout
| CloseReason::WriteTimeout
| CloseReason::ExchangeDeadline
| CloseReason::ParkDeadline
| CloseReason::HandshakeTimeout => bump(&st.timeouts, 1),
CloseReason::SessionPanicked => bump(&st.session_panics, 1),
_ => {}
}
st.end_write();
match reason {
CloseReason::EgressDenied(code) => shared.event(Event::EgressDenied { id: info.id, code }),
CloseReason::Killed(k) => shared.event(Event::Killed {
id: info.id,
reason: k,
}),
CloseReason::SessionPanicked => shared.event(Event::SessionPanicked { id: info.id }),
_ => {}
}
shared.event(Event::Closed {
id: info.id,
reason,
bytes_in,
bytes_out,
lifetime,
});
}
#[cfg(feature = "proxy")]
pub(crate) fn proxy_resolved(
shared: &Shared,
link: &mut Link,
listener: &ListenerConf,
source: &mut Option<PrefixKey>,
facts: ProxyFacts,
real: Option<SocketAddr>,
facts_pool: &mut Vec<Arc<ProxyFacts>>,
) -> Result<(), CloseReason> {
let lim = *link.limits;
if let Some(real) = real {
let key = PrefixKey::of(real.ip(), lim.source_policy);
if Some(key) != *source {
let table = &listener.sources;
if !table.acquire(key) {
return Err(CloseReason::Refused(RefusalCause::SourceCap));
}
if let Some(old) = source.replace(key) {
table.release(old);
}
}
link.info.real_peer = Some(real);
}
let mut arc = facts_pool
.pop()
.unwrap_or_else(|| Arc::new(ProxyFacts::default()));
if let Some(f) = Arc::get_mut(&mut arc) {
*f = facts;
} else {
arc = Arc::new(facts);
}
link.info.proxy = Some(arc);
if let Some(real) = link.info.real_peer {
let readmitted = match link.gate.as_mut() {
Some(g) => g.readmit(&link.info),
None => Ok(()),
};
if let Err(refusal) = readmitted {
if let Refusal::WithBytes { bytes, .. } = refusal {
let max = usize::from(lim.refusal_bytes_max);
if bytes.len() <= max {
link.send_denial_once(bytes);
}
}
return Err(CloseReason::Refused(RefusalCause::Gate(refusal.code())));
}
shared.event(Event::ProxyResolved {
id: link.info.id,
real_peer: real,
});
}
Ok(())
}
fn call<G: Gate, S: Session, R>(
w: &mut Worker<G>,
link: &mut Link,
session: &mut S,
f: impl FnOnce(&mut S, &mut SessionContext<'_>) -> R,
) -> (Option<R>, Option<crate::limits::SizeClass>) {
let draining = w.draining();
let pressure = w.pressure();
let worker = WorkerId(w.idx as u16);
let mut cx = SessionContext {
link,
output: &mut w.output,
pool: &mut w.pool,
completer: &w.completer,
draining,
pressure,
worker,
grow_to: None,
};
let r = guarded(|| f(session, &mut cx));
let grow = cx.grow_to;
(r, grow)
}
fn drive<G: Gate, S: Session>(
w: &mut Worker<G>,
link: &mut Link,
s: &mut S,
input: &mut Input,
) -> CloseReason {
link.hear_drains();
let Some(mut step) = call(w, link, s, |s, cx| s.on_open(cx)).0 else {
return CloseReason::SessionPanicked;
};
loop {
if let Some(reason) = w.control() {
return reason;
}
match step {
Step::Read => {
if input.changed && !input.bytes().is_empty() {
input.changed = false;
match on_data(w, link, s, input) {
Some(next) => step = next,
None => return CloseReason::SessionPanicked,
}
continue;
}
input.changed = false;
link.settle_phase(input.bytes().is_empty());
link.set_deadline_scale(w.shared.deadline_scale());
link.set_rescales();
let idle = link.phase() == Phase::Idle && input.bytes().is_empty();
if w.draining()
&& idle
&& !link.drain_kept()
&& !link.input_waiting()
&& !link.awaits_first_request()
{
match drain_reached(w, link, s) {
Ok(next) => step = next,
Err(reason) => return reason,
}
continue;
}
if idle && w.shed(link.info.lane) {
return CloseReason::Shed;
}
if input.is_full() {
return CloseReason::Limit(crate::error::LimitKind::ReadBuffer);
}
let got = link.read(input.spare_for_read());
match got {
Ok(Got::Data(n)) => {
input.filled(n);
input.changed = false;
match on_data(w, link, s, input) {
Some(next) => step = next,
None => return CloseReason::SessionPanicked,
}
}
Ok(Got::Eof) => match call(w, link, s, |s, cx| s.on_peer_closed(cx)).0 {
Some(next) => step = next,
None => return CloseReason::SessionPanicked,
},
Ok(Got::Woken) => {
if link.take_drain_reached() {
match drain_reached(w, link, s) {
Ok(next) => step = next,
Err(reason) => return reason,
}
} else if let Some((token, value)) = take_post(w, link) {
match call(w, link, s, |s, cx| s.on_wake(cx, token, value)).0 {
Some(next) => step = next,
None => return CloseReason::SessionPanicked,
}
}
}
Err(NetError::TimedOut { kind }) => match timeout(w, link, s, kind) {
Ok(next) => step = next,
Err(reason) => return reason,
},
Err(_) => return link.reason_or(CloseReason::Io(OsErrorClass::Other(0))),
}
}
Step::Write => match flush(w, link, s) {
Ok(()) => match call(w, link, s, |s, cx| s.on_writable(cx)).0 {
Some(next) => step = next,
None => return CloseReason::SessionPanicked,
},
Err(reason) => return reason,
},
Step::Park { until } => match park(w, link, until) {
Parked::Wake(token, value) => {
match call(w, link, s, |s, cx| s.on_wake(cx, token, value)).0 {
Some(next) => step = next,
None => return CloseReason::SessionPanicked,
}
}
Parked::Timeout => match timeout(w, link, s, TimeoutKind::Park) {
Ok(next) => step = next,
Err(reason) => return reason,
},
Parked::Closed(reason) => return reason,
},
Step::Close => {
if let Err(reason) = flush(w, link, s) {
return reason;
}
return if w.draining() {
CloseReason::Drained
} else if link.peer_closed {
CloseReason::PeerClosed
} else {
CloseReason::SessionClosed
};
}
_ => return link.reason_or(CloseReason::SessionError),
}
}
}
fn drain_reached<G: Gate, S: Session>(
w: &mut Worker<G>,
link: &mut Link,
s: &mut S,
) -> Result<Step, CloseReason> {
match call(w, link, s, |s, cx| s.on_drain(cx)).0 {
None => Err(CloseReason::SessionPanicked),
Some(Step::Close) => {
flush(w, link, s)?;
Err(CloseReason::Drained)
}
Some(next) => {
link.keep_through_drain();
Ok(next)
}
}
}
fn on_data<G: Gate, S: Session>(
w: &mut Worker<G>,
link: &mut Link,
s: &mut S,
input: &mut Input,
) -> Option<Step> {
link.note_heard();
let (step, grow) = call(w, link, s, |s, cx| s.on_data(cx, input));
if let Some(class) = grow {
if class > input.buf.class() {
if let Some(mut bigger) = w.pool.take(class) {
let bytes = input.bytes();
bigger.spare()[..bytes.len()].copy_from_slice(bytes);
bigger.set_len(bytes.len());
let old = std::mem::replace(&mut input.buf, bigger);
input.start = 0;
w.pool.put(old);
}
}
}
step
}
fn timeout<G: Gate, S: Session>(
w: &mut Worker<G>,
link: &mut Link,
s: &mut S,
kind: TimeoutKind,
) -> Result<Step, CloseReason> {
match call(w, link, s, |s, cx| s.on_timeout(cx, kind)).0 {
None => Err(CloseReason::SessionPanicked),
Some(Step::Close | Step::Abort) => Err(CloseReason::for_timeout(kind)),
Some(next) => {
link.rearm();
Ok(next)
}
}
}
fn flush<G: Gate, S: Session>(
w: &mut Worker<G>,
link: &mut Link,
s: &mut S,
) -> Result<(), CloseReason> {
if w.output.take_refused() && w.output.is_empty() {
return Err(CloseReason::Limit(crate::error::LimitKind::BufferBudget));
}
while !w.output.is_empty() {
link.response_started = if w.output.response_open {
w.output.bytes_since_begin > 0
} else {
link.bytes_out > 0
};
link.response_tag = w.output.response_tag;
let result = if let Some((file, offset, left)) = w.output.front_file() {
link.send_file(&file, offset, left).map(|n| n as usize)
} else {
let mut slices = [IoSlice::new(&[]); 64];
let mut k = 0;
for chunk in w
.output
.chunks
.iter()
.take(slices.len())
.take_while(|c| !c.is_file())
{
slices[k] = IoSlice::new(chunk.remaining());
k += 1;
}
link.write_vectored(&slices[..k])
};
match result {
Ok(n) => {
w.output.advance(n);
w.output.return_spares(&mut w.pool);
w.output.bytes_since_begin += n as u64;
}
Err(NetError::EgressDenied { code }) => {
let started = link.response_started;
w.output.wipe_all();
if !started {
let out = &mut w.output;
if guarded(|| s.on_denied(code, out)).is_some() {
let mut budget = usize::from(link.limits.refusal_bytes_max);
for chunk in &w.output.chunks {
let bytes = chunk.remaining();
let take = bytes.len().min(budget);
if take == 0 {
break;
}
link.send_denial_once(&bytes[..take]);
budget -= take;
}
}
w.output.wipe_all();
}
return Err(CloseReason::EgressDenied(code));
}
Err(_) => return Err(link.reason_or(CloseReason::Io(OsErrorClass::Other(0)))),
}
}
link.output_done();
Ok(())
}
enum Parked {
Wake(u32, u64),
Timeout,
Closed(CloseReason),
}
fn take_post<G: Gate>(w: &Worker<G>, link: &Link) -> Option<(u32, u64)> {
let shared = &w.shared;
let ws = &shared.workers[w.idx];
ws.completer.wake_pending.swap(false, Ordering::AcqRel);
while let Some(p) = ws.completer.ring.pop() {
if p.connection == link.info.id {
return Some((p.token, p.value));
}
ws.stats.begin_write();
ws.stats.stale_completions.fetch_add(1, Ordering::Relaxed);
ws.stats.end_write();
shared.event(Event::StaleCompletion {
worker: WorkerId(w.idx as u16),
});
}
None
}
fn park<G: Gate>(w: &mut Worker<G>, link: &mut Link, until: Option<Instant>) -> Parked {
let deadline = until.unwrap_or_else(|| link.now() + link.limits.park_timeout);
loop {
if let Some((token, value)) = take_post(w, link) {
return Parked::Wake(token, value);
}
if let Some(k) = link.killed() {
return Parked::Closed(CloseReason::Killed(k));
}
if let Some(reason) = w.control() {
return Parked::Closed(reason);
}
let now = link.now();
if now >= deadline {
return Parked::Timeout;
}
std::thread::park_timeout(deadline - now);
}
}