net_io/driver/mod.rs
1//! Drivers: what moves the bytes.
2//!
3//! The blocking pool ships in `0.3.0`, the reactor (epoll, kqueue, IOCP) in
4//! `0.5.0`, io_uring after `1.0.0`. The [`Driver`] trait is public from `0.5.0`
5//! (Tier 3, unstable until `1.0.0`) so an executor adapter and later drivers can
6//! build on it.
7//!
8//! The interface is completion-shaped (decision D-005): an operation owns its
9//! buffers until its completion hands them back, which is what IOCP and io_uring
10//! need; epoll and kqueue adapt by making the non-blocking call when readiness
11//! arrives. Cancellation always ends in exactly one completion.
12
13// Without the reactor only the vocabulary is left, for the public interface.
14#![cfg_attr(not(feature = "reactor"), allow(dead_code, unused_imports))]
15
16#[cfg(all(feature = "reactor", windows))]
17pub(crate) mod iocp;
18#[cfg(all(feature = "reactor", unix))]
19pub(crate) mod ready;
20
21use std::collections::VecDeque;
22use std::fs::File;
23use std::io::IoSlice;
24use std::net::SocketAddr;
25use std::sync::Arc;
26use std::time::Duration;
27
28use crate::buf::Buffer;
29use crate::error::{NetError, OsErrorClass};
30use crate::session::{Chunk, Output};
31
32/// Which driver moves the bytes.
33///
34/// `Auto` picks the reactor where it is built and available, otherwise the
35/// blocking pool.
36///
37/// # Examples
38///
39/// ```
40/// assert_eq!(net_io::DriverKind::default(), net_io::DriverKind::Auto);
41/// ```
42#[non_exhaustive]
43#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
44pub enum DriverKind {
45 /// A bounded pool of threads, one connection per thread.
46 Blocking,
47 /// One event loop per worker (epoll, kqueue, IOCP); from `0.5.0`.
48 Reactor,
49 /// io_uring on Linux; after `1.0.0`.
50 Uring,
51 /// The reactor where available, otherwise the blocking pool.
52 #[default]
53 Auto,
54}
55
56impl DriverKind {
57 /// The driver `Auto` resolves to in this build.
58 pub(crate) fn resolve(self) -> DriverKind {
59 match self {
60 DriverKind::Auto if DriverKind::reactor_built() && !cfg!(net_io_force_blocking) => {
61 DriverKind::Reactor
62 }
63 DriverKind::Auto => DriverKind::Blocking,
64 other => other,
65 }
66 }
67
68 /// Whether this build has a reactor for this OS.
69 pub(crate) const fn reactor_built() -> bool {
70 cfg!(feature = "reactor")
71 }
72}
73
74/// A socket as a [`Driver`] registers it: shared, so the driver keeps it open
75/// until [`Driver::deregister`] whatever the caller drops. Any `Arc` of a socket
76/// type converts (`Arc<TcpStream>`, `Arc<TcpListener>`, `Arc<OwnedFd>`); keep a
77/// clone to accept from a listener or to read the socket's options.
78///
79/// The driver works on the descriptor `as_fd` returns at registration, so the
80/// value must keep returning the same one while registered, as every `std`
81/// socket type does.
82///
83/// ```
84/// use std::sync::Arc;
85/// let listener = Arc::new(std::net::TcpListener::bind("127.0.0.1:0").unwrap());
86/// let shared: net_io::SharedSocket = listener.clone();
87/// # let _ = shared;
88/// ```
89#[cfg(unix)]
90pub type SharedSocket = Arc<dyn std::os::fd::AsFd + Send + Sync>;
91/// A socket as a [`Driver`] registers it: shared, so the driver keeps it open
92/// until [`Driver::deregister`] whatever the caller drops. Any `Arc` of a socket
93/// type converts (`Arc<TcpStream>`, `Arc<TcpListener>`, `Arc<OwnedSocket>`); keep
94/// a clone to read the socket's options.
95///
96/// The driver works on the handle `as_socket` returns at registration, so the
97/// value must keep returning the same one while registered, as every `std`
98/// socket type does.
99///
100/// ```
101/// use std::sync::Arc;
102/// let stream = Arc::new(std::net::UdpSocket::bind("127.0.0.1:0").unwrap());
103/// let shared: net_io::SharedSocket = stream.clone();
104/// # let _ = shared;
105/// ```
106#[cfg(windows)]
107pub type SharedSocket = Arc<dyn std::os::windows::io::AsSocket + Send + Sync>;
108
109/// A borrowed socket, as the server's own reactor registers it.
110#[cfg(unix)]
111pub(crate) type BorrowedSocket<'a> = std::os::fd::BorrowedFd<'a>;
112/// A borrowed socket, as the server's own reactor registers it.
113#[cfg(windows)]
114pub(crate) type BorrowedSocket<'a> = std::os::windows::io::BorrowedSocket<'a>;
115
116/// Names a registered socket. The low 32 bits index a dense table inside the
117/// driver, so keep them small; the high bits are free for a generation.
118///
119/// ```
120/// let t = net_io::Token(7);
121/// assert_eq!(t, net_io::Token(7));
122/// ```
123#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
124pub struct Token(pub u64);
125
126impl Token {
127 pub(crate) fn index(self) -> usize {
128 (self.0 & 0xFFFF_FFFF) as usize
129 }
130}
131
132/// What to do on a registered socket. The operation owns its buffers until its
133/// completion returns them.
134#[non_exhaustive]
135#[derive(Debug)]
136pub enum Operation {
137 /// Completes when a connection is waiting (accept it from the listener).
138 /// Refused on Windows, where a listener is never registered (D-016).
139 Accept {
140 /// The listener.
141 listener: Token,
142 },
143 /// Receives into `buffer`'s spare room (at most once); `Ok(0)` is end of stream.
144 Receive {
145 /// The socket.
146 token: Token,
147 /// Where the bytes land; returned by the completion.
148 buffer: Buffer,
149 },
150 /// Completes when bytes can be read, without holding a buffer: idle
151 /// connections wait this way.
152 ReceiveZero {
153 /// The socket.
154 token: Token,
155 },
156 /// Sends what `chunks` holds; the result says how many bytes went out, which
157 /// can be fewer than all of them. The chunks come back with the completion.
158 Send {
159 /// The socket.
160 token: Token,
161 /// The bytes.
162 chunks: SendChunks,
163 },
164 /// Sends up to `len` bytes of `file` from `offset`, zero copy where the OS can. The
165 /// result says how many went out; `Ok(0)` means the file has nothing there. A `len`
166 /// of 0 completes at once with `Ok(0)` and sends nothing.
167 SendFile {
168 /// The socket.
169 token: Token,
170 /// The file; held until the completion.
171 file: Arc<File>,
172 /// Where to start.
173 offset: u64,
174 /// The most to send.
175 len: u32,
176 },
177 /// Connects a registered, non-blocking socket.
178 Connect {
179 /// The socket.
180 token: Token,
181 /// The peer.
182 addr: SocketAddr,
183 },
184 /// Completes once every other operation on the socket has; the caller then
185 /// closes it. Abortive when asked (linger 0).
186 Close {
187 /// The socket.
188 token: Token,
189 /// Reset instead of FIN.
190 abortive: bool,
191 },
192 /// Completes at once (a turn of the loop).
193 Nop {
194 /// Echoed in the completion.
195 token: Token,
196 },
197}
198
199impl Operation {
200 fn token(&self) -> Token {
201 match self {
202 Operation::Accept { listener } => *listener,
203 Operation::Receive { token, .. }
204 | Operation::ReceiveZero { token }
205 | Operation::Send { token, .. }
206 | Operation::SendFile { token, .. }
207 | Operation::Connect { token, .. }
208 | Operation::Close { token, .. }
209 | Operation::Nop { token } => *token,
210 }
211 }
212}
213
214/// Which operation a completion finishes.
215///
216/// ```
217/// assert_ne!(net_io::OperationKind::Receive, net_io::OperationKind::Send);
218/// ```
219#[non_exhaustive]
220#[derive(Clone, Copy, Debug, PartialEq, Eq)]
221pub enum OperationKind {
222 /// [`Operation::Accept`].
223 Accept,
224 /// [`Operation::Receive`] and [`Operation::ReceiveZero`].
225 Receive,
226 /// [`Operation::Send`].
227 Send,
228 /// [`Operation::SendFile`].
229 SendFile,
230 /// [`Operation::Connect`].
231 Connect,
232 /// [`Operation::Close`].
233 Close,
234 /// [`Operation::Nop`].
235 Nop,
236}
237
238/// A finished operation.
239#[non_exhaustive]
240#[derive(Debug)]
241pub struct Completion {
242 /// The socket.
243 pub token: Token,
244 /// Which operation finished.
245 pub kind: OperationKind,
246 /// Bytes moved, or the error class. A cancelled operation is `Aborted`.
247 pub result: Result<usize, OsErrorClass>,
248 /// The receive buffer, returned with its new length.
249 pub buffer: Option<Buffer>,
250 /// The send chunks, returned as they were.
251 pub chunks: Option<SendChunks>,
252}
253
254impl Completion {
255 pub(crate) fn new(
256 token: Token,
257 kind: OperationKind,
258 result: Result<usize, OsErrorClass>,
259 ) -> Completion {
260 Completion {
261 token,
262 kind,
263 result,
264 buffer: None,
265 chunks: None,
266 }
267 }
268}
269
270/// How many completions one wait returns.
271pub(crate) const COMPLETIONS: usize = 256;
272
273/// What a wait fills; fixed capacity (256), so a wait never allocates.
274#[derive(Debug)]
275pub struct Completions {
276 items: Vec<Completion>,
277}
278
279impl Default for Completions {
280 fn default() -> Completions {
281 Completions::new()
282 }
283}
284
285impl Completions {
286 /// Room for 256 completions.
287 #[must_use]
288 pub fn new() -> Completions {
289 Completions {
290 items: Vec::with_capacity(COMPLETIONS),
291 }
292 }
293
294 /// How many completions are held.
295 #[must_use]
296 pub fn len(&self) -> usize {
297 self.items.len()
298 }
299
300 /// Whether none are held.
301 #[must_use]
302 pub fn is_empty(&self) -> bool {
303 self.items.is_empty()
304 }
305
306 /// Takes the completions out, oldest first.
307 pub fn drain(&mut self) -> impl Iterator<Item = Completion> + '_ {
308 self.items.drain(..)
309 }
310
311 #[cfg(feature = "reactor")]
312 pub(crate) fn push(&mut self, c: Completion) {
313 self.items.push(c);
314 }
315
316 #[cfg(feature = "reactor")]
317 pub(crate) fn has_room(&self) -> bool {
318 self.items.len() < COMPLETIONS
319 }
320}
321
322/// The bytes of one send: up to 64 chunks, taken from an [`Output`] or built from
323/// a pool buffer.
324#[derive(Debug, Default)]
325pub struct SendChunks {
326 pub(crate) chunks: VecDeque<Chunk>,
327}
328
329impl SendChunks {
330 /// One pool buffer.
331 #[must_use]
332 pub fn from_buffer(buffer: Buffer) -> SendChunks {
333 let mut chunks = VecDeque::with_capacity(1);
334 chunks.push_back(Chunk::Pooled(buffer, 0));
335 SendChunks { chunks }
336 }
337
338 /// Takes everything queued in `output`; give it back with [`SendChunks::restore`].
339 pub(crate) fn take(output: &mut Output) -> SendChunks {
340 SendChunks {
341 chunks: std::mem::take(&mut output.chunks),
342 }
343 }
344
345 /// Puts the chunks back into `output`, keeping the queue's allocation.
346 pub(crate) fn restore(self, output: &mut Output) {
347 output.chunks = self.chunks;
348 }
349
350 /// Bytes held.
351 #[must_use]
352 pub fn pending(&self) -> usize {
353 self.chunks.iter().map(Chunk::len).sum()
354 }
355
356 /// Marks `n` bytes of the front chunks sent; `true` when bytes remain.
357 #[cfg(feature = "reactor")]
358 pub(crate) fn advance(&mut self, mut n: usize) -> bool {
359 while let Some(front) = self.chunks.front_mut() {
360 let rem = front.len();
361 if n < rem {
362 front.advance(n);
363 return true;
364 }
365 if self.chunks.len() == 1 {
366 return false;
367 }
368 n -= rem;
369 self.chunks.pop_front();
370 }
371 false
372 }
373
374 /// The first buffer back, for a send built with [`SendChunks::from_buffer`].
375 pub(crate) fn into_buffer(mut self) -> Option<Buffer> {
376 match self.chunks.pop_front() {
377 Some(Chunk::Pooled(b, _)) => Some(b),
378 _ => None,
379 }
380 }
381
382 /// Fills `out` with the chunks' bytes; returns how many slices.
383 #[cfg(feature = "reactor")]
384 pub(crate) fn slices<'a>(&'a self, out: &mut [IoSlice<'a>]) -> usize {
385 let mut n = 0;
386 // A file goes by its own operation: the slices stop before it.
387 for chunk in self
388 .chunks
389 .iter()
390 .take(out.len())
391 .take_while(|c| !c.is_file())
392 {
393 out[n] = IoSlice::new(chunk.remaining());
394 n += 1;
395 }
396 n
397 }
398}
399
400/// Why [`Driver::submit`] refused an operation; it comes back with its buffers.
401#[derive(Debug)]
402pub struct Rejected {
403 /// The operation.
404 pub operation: Operation,
405}
406
407/// Wakes a driver's [`Driver::wait`] from any thread.
408#[derive(Clone, Debug)]
409pub struct DriverWaker {
410 pub(crate) inner: Arc<WakerInner>,
411}
412
413#[cfg(feature = "reactor")]
414pub(crate) type WakerInner = crate::sys::PollWaker;
415#[cfg(not(feature = "reactor"))]
416#[derive(Debug)]
417pub(crate) struct WakerInner;
418
419impl DriverWaker {
420 /// Ends the current or next wait. Cheap to call repeatedly.
421 pub fn wake(&self) {
422 #[cfg(feature = "reactor")]
423 self.inner.wake();
424 }
425}
426
427/// The reactor driver for this OS: epoll, kqueue, or an I/O completion port.
428/// `capacity` sizes its tables for that many sockets; they grow past it.
429///
430/// One per worker. A token's low 32 bits index a table the driver keeps, so keep
431/// them small and dense, and put a generation in the high 32 bits to catch a
432/// completion for a slot that moved on. On Windows a listener cannot be registered
433/// and `Operation::Accept` is refused (accept from a thread of your own, D-016).
434///
435/// ```
436/// # fn main() -> Result<(), net_io::NetError> {
437/// use std::time::Duration;
438/// use net_io::{Completions, Operation, Token};
439/// let mut driver = net_io::new_reactor_driver(16)?;
440/// driver.submit(Operation::Nop { token: Token(1) }).expect("a no-op is always taken");
441/// let mut done = Completions::new();
442/// assert_eq!(driver.wait(&mut done, Some(Duration::from_secs(1)))?, 1);
443/// # Ok(())
444/// # }
445/// ```
446///
447/// # Errors
448///
449/// `DriverSetup` when the OS refuses the queue.
450#[cfg(feature = "reactor")]
451pub fn new_reactor_driver(capacity: usize) -> Result<Box<dyn Driver>, NetError> {
452 #[cfg(unix)]
453 let driver = ready::ReadyDriver::new(capacity)?;
454 #[cfg(windows)]
455 let driver = iocp::IocpDriver::new(capacity)?;
456 Ok(Box::new(driver))
457}
458
459/// A driver: one per worker, used from one thread. See the module docs.
460///
461/// Unstable until `1.0.0`. The driver holds each registered socket (a
462/// [`SharedSocket`]) from [`Driver::register`] until [`Driver::deregister`], so the
463/// descriptor it does I/O on can never close and be reused by another connection
464/// while an operation or a cached readiness refers to it.
465pub trait Driver: Send {
466 /// Which driver this is.
467 fn kind(&self) -> DriverKind;
468
469 /// Adds a non-blocking socket under `token`. The driver keeps `socket` until
470 /// [`Driver::deregister`]: dropping the caller's own clones before then does
471 /// not close it.
472 ///
473 /// # Errors
474 ///
475 /// `Os` when the OS refuses.
476 fn register(&mut self, socket: SharedSocket, token: Token) -> Result<(), NetError>;
477
478 /// Starts an operation; its completion arrives through [`Driver::wait`].
479 ///
480 /// # Errors
481 ///
482 /// [`Rejected`] gives the operation and its buffers back: the token is not
483 /// registered, or the same kind of operation is already pending on it.
484 fn submit(&mut self, operation: Operation) -> Result<(), Rejected>;
485
486 /// Fills `out` with finished operations, waiting up to `timeout` for the
487 /// first (`None`: until one finishes or the waker fires). Returns how many.
488 ///
489 /// # Errors
490 ///
491 /// `Os` when the wait itself fails.
492 fn wait(&mut self, out: &mut Completions, timeout: Option<Duration>)
493 -> Result<usize, NetError>;
494
495 /// Cancels the pending operation of kind `which`; it still completes exactly
496 /// once (with `Aborted`, or with its result if it already finished).
497 ///
498 /// # Errors
499 ///
500 /// `Os` when the OS refuses.
501 fn cancel(&mut self, token: Token, which: OperationKind) -> Result<(), NetError>;
502
503 /// Removes the socket; no completion arrives for it afterwards. The driver
504 /// drops its clone of the socket, which closes it if no other clone is left.
505 ///
506 /// # Errors
507 ///
508 /// `Os` when the OS refuses.
509 fn deregister(&mut self, token: Token) -> Result<(), NetError>;
510
511 /// A waker for this driver.
512 fn waker(&self) -> DriverWaker;
513}
514
515pub(crate) fn operation_token(op: &Operation) -> Token {
516 op.token()
517}