net_io/driver/mod.rs
1//! Drivers: what moves the bytes.
2//!
3//! The blocking pool ships in `0.3.0`, the reactor (epoll, kqueue, IOCP) in
4//! `0.5.0`, io_uring after `1.0.0`. The [`Driver`] trait is public from `0.5.0`
5//! (Tier 3, unstable until `1.0.0`) so an executor adapter and later drivers can
6//! build on it.
7//!
8//! The interface is completion-shaped (decision D-005): an operation owns its
9//! buffers until its completion hands them back, which is what IOCP and io_uring
10//! need; epoll and kqueue adapt by making the non-blocking call when readiness
11//! arrives. Cancellation always ends in exactly one completion.
12
13// Without the reactor only the vocabulary is left, for the public interface.
14#![cfg_attr(not(feature = "reactor"), allow(dead_code, unused_imports))]
15
16#[cfg(all(feature = "reactor", windows))]
17pub(crate) mod iocp;
18#[cfg(all(feature = "reactor", unix))]
19pub(crate) mod ready;
20
21use std::collections::VecDeque;
22use std::fs::File;
23use std::io::IoSlice;
24use std::net::SocketAddr;
25use std::sync::Arc;
26use std::time::Duration;
27
28use crate::buf::Buffer;
29use crate::error::{NetError, OsErrorClass};
30use crate::session::{Chunk, Output};
31
32/// Which driver moves the bytes.
33///
34/// `Auto` picks the reactor where it is built and available, otherwise the
35/// blocking pool.
36///
37/// # Examples
38///
39/// ```
40/// assert_eq!(net_io::DriverKind::default(), net_io::DriverKind::Auto);
41/// ```
42#[non_exhaustive]
43#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
44pub enum DriverKind {
45 /// A bounded pool of threads, one connection per thread.
46 Blocking,
47 /// One event loop per worker (epoll, kqueue, IOCP); from `0.5.0`.
48 Reactor,
49 /// io_uring on Linux; after `1.0.0`.
50 Uring,
51 /// The reactor where available, otherwise the blocking pool.
52 #[default]
53 Auto,
54}
55
56impl DriverKind {
57 /// The driver `Auto` resolves to in this build.
58 pub(crate) fn resolve(self) -> DriverKind {
59 match self {
60 DriverKind::Auto if DriverKind::reactor_built() && !cfg!(net_io_force_blocking) => {
61 DriverKind::Reactor
62 }
63 DriverKind::Auto => DriverKind::Blocking,
64 other => other,
65 }
66 }
67
68 /// Whether this build has a reactor for this OS.
69 pub(crate) const fn reactor_built() -> bool {
70 cfg!(feature = "reactor")
71 }
72}
73
74/// A socket as a [`Driver`] registers it: shared, so the driver keeps it open
75/// until [`Driver::deregister`] whatever the caller drops. Any `Arc` of a socket
76/// type converts (`Arc<TcpStream>`, `Arc<TcpListener>`, `Arc<OwnedFd>`); keep a
77/// clone to accept from a listener or to read the socket's options.
78///
79/// The driver works on the descriptor `as_fd` returns at registration, so the
80/// value must keep returning the same one while registered, as every `std`
81/// socket type does.
82///
83/// ```
84/// use std::sync::Arc;
85/// let listener = Arc::new(std::net::TcpListener::bind("127.0.0.1:0").unwrap());
86/// let shared: net_io::SharedSocket = listener.clone();
87/// # let _ = shared;
88/// ```
89#[cfg(unix)]
90pub type SharedSocket = Arc<dyn std::os::fd::AsFd + Send + Sync>;
91/// A socket as a [`Driver`] registers it: shared, so the driver keeps it open
92/// until [`Driver::deregister`] whatever the caller drops. Any `Arc` of a socket
93/// type converts (`Arc<TcpStream>`, `Arc<TcpListener>`, `Arc<OwnedSocket>`); keep
94/// a clone to read the socket's options.
95///
96/// The driver works on the handle `as_socket` returns at registration, so the
97/// value must keep returning the same one while registered, as every `std`
98/// socket type does.
99///
100/// ```
101/// use std::sync::Arc;
102/// let stream = Arc::new(std::net::UdpSocket::bind("127.0.0.1:0").unwrap());
103/// let shared: net_io::SharedSocket = stream.clone();
104/// # let _ = shared;
105/// ```
106#[cfg(windows)]
107pub type SharedSocket = Arc<dyn std::os::windows::io::AsSocket + Send + Sync>;
108
109/// A borrowed socket, as the server's own reactor registers it.
110#[cfg(unix)]
111pub(crate) type BorrowedSocket<'a> = std::os::fd::BorrowedFd<'a>;
112/// A borrowed socket, as the server's own reactor registers it.
113#[cfg(windows)]
114pub(crate) type BorrowedSocket<'a> = std::os::windows::io::BorrowedSocket<'a>;
115
116/// Names a registered socket. The low 32 bits index a dense table inside the
117/// driver, so keep them small; the high bits are free for a generation.
118///
119/// ```
120/// let t = net_io::Token(7);
121/// assert_eq!(t, net_io::Token(7));
122/// ```
123#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
124pub struct Token(pub u64);
125
126impl Token {
127 pub(crate) fn index(self) -> usize {
128 (self.0 & 0xFFFF_FFFF) as usize
129 }
130}
131
132/// What to do on a registered socket. The operation owns its buffers until its
133/// completion returns them.
134#[non_exhaustive]
135#[derive(Debug)]
136pub enum Operation {
137 /// Completes when a connection is waiting (accept it from the listener).
138 /// Refused on Windows, where a listener is never registered (D-016).
139 Accept {
140 /// The listener.
141 listener: Token,
142 },
143 /// Receives into `buffer`'s spare room (at most once); `Ok(0)` is end of stream.
144 Receive {
145 /// The socket.
146 token: Token,
147 /// Where the bytes land; returned by the completion.
148 buffer: Buffer,
149 },
150 /// Completes when bytes can be read, without holding a buffer: idle
151 /// connections wait this way.
152 ReceiveZero {
153 /// The socket.
154 token: Token,
155 },
156 /// Sends what `chunks` holds; the result says how many bytes went out, which
157 /// can be fewer than all of them. The chunks come back with the completion.
158 Send {
159 /// The socket.
160 token: Token,
161 /// The bytes.
162 chunks: SendChunks,
163 },
164 /// Sends up to `len` bytes of `file` from `offset`, zero copy where the OS can. The
165 /// result says how many went out; `Ok(0)` means the file has nothing there.
166 SendFile {
167 /// The socket.
168 token: Token,
169 /// The file; held until the completion.
170 file: Arc<File>,
171 /// Where to start.
172 offset: u64,
173 /// The most to send.
174 len: u32,
175 },
176 /// Connects a registered, non-blocking socket.
177 Connect {
178 /// The socket.
179 token: Token,
180 /// The peer.
181 addr: SocketAddr,
182 },
183 /// Completes once every other operation on the socket has; the caller then
184 /// closes it. Abortive when asked (linger 0).
185 Close {
186 /// The socket.
187 token: Token,
188 /// Reset instead of FIN.
189 abortive: bool,
190 },
191 /// Completes at once (a turn of the loop).
192 Nop {
193 /// Echoed in the completion.
194 token: Token,
195 },
196}
197
198impl Operation {
199 fn token(&self) -> Token {
200 match self {
201 Operation::Accept { listener } => *listener,
202 Operation::Receive { token, .. }
203 | Operation::ReceiveZero { token }
204 | Operation::Send { token, .. }
205 | Operation::SendFile { token, .. }
206 | Operation::Connect { token, .. }
207 | Operation::Close { token, .. }
208 | Operation::Nop { token } => *token,
209 }
210 }
211}
212
213/// Which operation a completion finishes.
214///
215/// ```
216/// assert_ne!(net_io::OperationKind::Receive, net_io::OperationKind::Send);
217/// ```
218#[non_exhaustive]
219#[derive(Clone, Copy, Debug, PartialEq, Eq)]
220pub enum OperationKind {
221 /// [`Operation::Accept`].
222 Accept,
223 /// [`Operation::Receive`] and [`Operation::ReceiveZero`].
224 Receive,
225 /// [`Operation::Send`].
226 Send,
227 /// [`Operation::SendFile`].
228 SendFile,
229 /// [`Operation::Connect`].
230 Connect,
231 /// [`Operation::Close`].
232 Close,
233 /// [`Operation::Nop`].
234 Nop,
235}
236
237/// A finished operation.
238#[non_exhaustive]
239#[derive(Debug)]
240pub struct Completion {
241 /// The socket.
242 pub token: Token,
243 /// Which operation finished.
244 pub kind: OperationKind,
245 /// Bytes moved, or the error class. A cancelled operation is `Aborted`.
246 pub result: Result<usize, OsErrorClass>,
247 /// The receive buffer, returned with its new length.
248 pub buffer: Option<Buffer>,
249 /// The send chunks, returned as they were.
250 pub chunks: Option<SendChunks>,
251}
252
253impl Completion {
254 pub(crate) fn new(
255 token: Token,
256 kind: OperationKind,
257 result: Result<usize, OsErrorClass>,
258 ) -> Completion {
259 Completion {
260 token,
261 kind,
262 result,
263 buffer: None,
264 chunks: None,
265 }
266 }
267}
268
269/// How many completions one wait returns.
270pub(crate) const COMPLETIONS: usize = 256;
271
272/// What a wait fills; fixed capacity (256), so a wait never allocates.
273#[derive(Debug)]
274pub struct Completions {
275 items: Vec<Completion>,
276}
277
278impl Default for Completions {
279 fn default() -> Completions {
280 Completions::new()
281 }
282}
283
284impl Completions {
285 /// Room for 256 completions.
286 #[must_use]
287 pub fn new() -> Completions {
288 Completions {
289 items: Vec::with_capacity(COMPLETIONS),
290 }
291 }
292
293 /// How many completions are held.
294 #[must_use]
295 pub fn len(&self) -> usize {
296 self.items.len()
297 }
298
299 /// Whether none are held.
300 #[must_use]
301 pub fn is_empty(&self) -> bool {
302 self.items.is_empty()
303 }
304
305 /// Takes the completions out, oldest first.
306 pub fn drain(&mut self) -> impl Iterator<Item = Completion> + '_ {
307 self.items.drain(..)
308 }
309
310 #[cfg(feature = "reactor")]
311 pub(crate) fn push(&mut self, c: Completion) {
312 self.items.push(c);
313 }
314
315 #[cfg(feature = "reactor")]
316 pub(crate) fn has_room(&self) -> bool {
317 self.items.len() < COMPLETIONS
318 }
319}
320
321/// The bytes of one send: up to 64 chunks, taken from an [`Output`] or built from
322/// a pool buffer.
323#[derive(Debug, Default)]
324pub struct SendChunks {
325 pub(crate) chunks: VecDeque<Chunk>,
326}
327
328impl SendChunks {
329 /// One pool buffer.
330 #[must_use]
331 pub fn from_buffer(buffer: Buffer) -> SendChunks {
332 let mut chunks = VecDeque::with_capacity(1);
333 chunks.push_back(Chunk::Pooled(buffer, 0));
334 SendChunks { chunks }
335 }
336
337 /// Takes everything queued in `output`; give it back with [`SendChunks::restore`].
338 pub(crate) fn take(output: &mut Output) -> SendChunks {
339 SendChunks {
340 chunks: std::mem::take(&mut output.chunks),
341 }
342 }
343
344 /// Puts the chunks back into `output`, keeping the queue's allocation.
345 pub(crate) fn restore(self, output: &mut Output) {
346 output.chunks = self.chunks;
347 }
348
349 /// Bytes held.
350 #[must_use]
351 pub fn pending(&self) -> usize {
352 self.chunks.iter().map(Chunk::len).sum()
353 }
354
355 /// Marks `n` bytes of the front chunks sent; `true` when bytes remain.
356 #[cfg(feature = "reactor")]
357 pub(crate) fn advance(&mut self, mut n: usize) -> bool {
358 while let Some(front) = self.chunks.front_mut() {
359 let rem = front.len();
360 if n < rem {
361 front.advance(n);
362 return true;
363 }
364 if self.chunks.len() == 1 {
365 return false;
366 }
367 n -= rem;
368 self.chunks.pop_front();
369 }
370 false
371 }
372
373 /// The first buffer back, for a send built with [`SendChunks::from_buffer`].
374 pub(crate) fn into_buffer(mut self) -> Option<Buffer> {
375 match self.chunks.pop_front() {
376 Some(Chunk::Pooled(b, _)) => Some(b),
377 _ => None,
378 }
379 }
380
381 /// Fills `out` with the chunks' bytes; returns how many slices.
382 #[cfg(feature = "reactor")]
383 pub(crate) fn slices<'a>(&'a self, out: &mut [IoSlice<'a>]) -> usize {
384 let mut n = 0;
385 // A file goes by its own operation: the slices stop before it.
386 for chunk in self
387 .chunks
388 .iter()
389 .take(out.len())
390 .take_while(|c| !c.is_file())
391 {
392 out[n] = IoSlice::new(chunk.remaining());
393 n += 1;
394 }
395 n
396 }
397}
398
399/// Why [`Driver::submit`] refused an operation; it comes back with its buffers.
400#[derive(Debug)]
401pub struct Rejected {
402 /// The operation.
403 pub operation: Operation,
404}
405
406/// Wakes a driver's [`Driver::wait`] from any thread.
407#[derive(Clone, Debug)]
408pub struct DriverWaker {
409 pub(crate) inner: Arc<WakerInner>,
410}
411
412#[cfg(feature = "reactor")]
413pub(crate) type WakerInner = crate::sys::PollWaker;
414#[cfg(not(feature = "reactor"))]
415#[derive(Debug)]
416pub(crate) struct WakerInner;
417
418impl DriverWaker {
419 /// Ends the current or next wait. Cheap to call repeatedly.
420 pub fn wake(&self) {
421 #[cfg(feature = "reactor")]
422 self.inner.wake();
423 }
424}
425
426/// The reactor driver for this OS: epoll, kqueue, or an I/O completion port.
427/// `capacity` sizes its tables for that many sockets; they grow past it.
428///
429/// One per worker. A token's low 32 bits index a table the driver keeps, so keep
430/// them small and dense, and put a generation in the high 32 bits to catch a
431/// completion for a slot that moved on. On Windows a listener cannot be registered
432/// and `Operation::Accept` is refused (accept from a thread of your own, D-016).
433///
434/// ```
435/// # fn main() -> Result<(), net_io::NetError> {
436/// use std::time::Duration;
437/// use net_io::{Completions, Operation, Token};
438/// let mut driver = net_io::new_reactor_driver(16)?;
439/// driver.submit(Operation::Nop { token: Token(1) }).expect("a no-op is always taken");
440/// let mut done = Completions::new();
441/// assert_eq!(driver.wait(&mut done, Some(Duration::from_secs(1)))?, 1);
442/// # Ok(())
443/// # }
444/// ```
445///
446/// # Errors
447///
448/// `DriverSetup` when the OS refuses the queue.
449#[cfg(feature = "reactor")]
450pub fn new_reactor_driver(capacity: usize) -> Result<Box<dyn Driver>, NetError> {
451 #[cfg(unix)]
452 let driver = ready::ReadyDriver::new(capacity)?;
453 #[cfg(windows)]
454 let driver = iocp::IocpDriver::new(capacity)?;
455 Ok(Box::new(driver))
456}
457
458/// A driver: one per worker, used from one thread. See the module docs.
459///
460/// Unstable until `1.0.0`. The driver holds each registered socket (a
461/// [`SharedSocket`]) from [`Driver::register`] until [`Driver::deregister`], so the
462/// descriptor it does I/O on can never close and be reused by another connection
463/// while an operation or a cached readiness refers to it.
464pub trait Driver: Send {
465 /// Which driver this is.
466 fn kind(&self) -> DriverKind;
467
468 /// Adds a non-blocking socket under `token`. The driver keeps `socket` until
469 /// [`Driver::deregister`]: dropping the caller's own clones before then does
470 /// not close it.
471 ///
472 /// # Errors
473 ///
474 /// `Os` when the OS refuses.
475 fn register(&mut self, socket: SharedSocket, token: Token) -> Result<(), NetError>;
476
477 /// Starts an operation; its completion arrives through [`Driver::wait`].
478 ///
479 /// # Errors
480 ///
481 /// [`Rejected`] gives the operation and its buffers back: the token is not
482 /// registered, or the same kind of operation is already pending on it.
483 fn submit(&mut self, operation: Operation) -> Result<(), Rejected>;
484
485 /// Fills `out` with finished operations, waiting up to `timeout` for the
486 /// first (`None`: until one finishes or the waker fires). Returns how many.
487 ///
488 /// # Errors
489 ///
490 /// `Os` when the wait itself fails.
491 fn wait(&mut self, out: &mut Completions, timeout: Option<Duration>)
492 -> Result<usize, NetError>;
493
494 /// Cancels the pending operation of kind `which`; it still completes exactly
495 /// once (with `Aborted`, or with its result if it already finished).
496 ///
497 /// # Errors
498 ///
499 /// `Os` when the OS refuses.
500 fn cancel(&mut self, token: Token, which: OperationKind) -> Result<(), NetError>;
501
502 /// Removes the socket; no completion arrives for it afterwards. The driver
503 /// drops its clone of the socket, which closes it if no other clone is left.
504 ///
505 /// # Errors
506 ///
507 /// `Os` when the OS refuses.
508 fn deregister(&mut self, token: Token) -> Result<(), NetError>;
509
510 /// A waker for this driver.
511 fn waker(&self) -> DriverWaker;
512}
513
514pub(crate) fn operation_token(op: &Operation) -> Token {
515 op.token()
516}