Skip to main content

net_io/driver/
mod.rs

1//! Drivers: what moves the bytes.
2//!
3//! The blocking pool ships in `0.3.0`, the reactor (epoll, kqueue, IOCP) in
4//! `0.5.0`, io_uring after `1.0.0`. The [`Driver`] trait is public from `0.5.0`
5//! (Tier 3, unstable until `1.0.0`) so an executor adapter and later drivers can
6//! build on it.
7//!
8//! The interface is completion-shaped (decision D-005): an operation owns its
9//! buffers until its completion hands them back, which is what IOCP and io_uring
10//! need; epoll and kqueue adapt by making the non-blocking call when readiness
11//! arrives. Cancellation always ends in exactly one completion.
12
13// Without the reactor only the vocabulary is left, for the public interface.
14#![cfg_attr(not(feature = "reactor"), allow(dead_code, unused_imports))]
15
16#[cfg(all(feature = "reactor", windows))]
17pub(crate) mod iocp;
18#[cfg(all(feature = "reactor", unix))]
19pub(crate) mod ready;
20
21use std::collections::VecDeque;
22use std::fs::File;
23use std::io::IoSlice;
24use std::net::SocketAddr;
25use std::sync::Arc;
26use std::time::Duration;
27
28use crate::buf::Buffer;
29use crate::error::{NetError, OsErrorClass};
30use crate::session::{Chunk, Output};
31
32/// Which driver moves the bytes.
33///
34/// `Auto` picks the reactor where it is built and available, otherwise the
35/// blocking pool.
36///
37/// # Examples
38///
39/// ```
40/// assert_eq!(net_io::DriverKind::default(), net_io::DriverKind::Auto);
41/// ```
42#[non_exhaustive]
43#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
44pub enum DriverKind {
45    /// A bounded pool of threads, one connection per thread.
46    Blocking,
47    /// One event loop per worker (epoll, kqueue, IOCP); from `0.5.0`.
48    Reactor,
49    /// io_uring on Linux; after `1.0.0`.
50    Uring,
51    /// The reactor where available, otherwise the blocking pool.
52    #[default]
53    Auto,
54}
55
56impl DriverKind {
57    /// The driver `Auto` resolves to in this build.
58    pub(crate) fn resolve(self) -> DriverKind {
59        match self {
60            DriverKind::Auto if DriverKind::reactor_built() && !cfg!(net_io_force_blocking) => {
61                DriverKind::Reactor
62            }
63            DriverKind::Auto => DriverKind::Blocking,
64            other => other,
65        }
66    }
67
68    /// Whether this build has a reactor for this OS.
69    pub(crate) const fn reactor_built() -> bool {
70        cfg!(feature = "reactor")
71    }
72}
73
74/// A socket as a [`Driver`] registers it: shared, so the driver keeps it open
75/// until [`Driver::deregister`] whatever the caller drops. Any `Arc` of a socket
76/// type converts (`Arc<TcpStream>`, `Arc<TcpListener>`, `Arc<OwnedFd>`); keep a
77/// clone to accept from a listener or to read the socket's options.
78///
79/// The driver works on the descriptor `as_fd` returns at registration, so the
80/// value must keep returning the same one while registered, as every `std`
81/// socket type does.
82///
83/// ```
84/// use std::sync::Arc;
85/// let listener = Arc::new(std::net::TcpListener::bind("127.0.0.1:0").unwrap());
86/// let shared: net_io::SharedSocket = listener.clone();
87/// # let _ = shared;
88/// ```
89#[cfg(unix)]
90pub type SharedSocket = Arc<dyn std::os::fd::AsFd + Send + Sync>;
91/// A socket as a [`Driver`] registers it: shared, so the driver keeps it open
92/// until [`Driver::deregister`] whatever the caller drops. Any `Arc` of a socket
93/// type converts (`Arc<TcpStream>`, `Arc<TcpListener>`, `Arc<OwnedSocket>`); keep
94/// a clone to read the socket's options.
95///
96/// The driver works on the handle `as_socket` returns at registration, so the
97/// value must keep returning the same one while registered, as every `std`
98/// socket type does.
99///
100/// ```
101/// use std::sync::Arc;
102/// let stream = Arc::new(std::net::UdpSocket::bind("127.0.0.1:0").unwrap());
103/// let shared: net_io::SharedSocket = stream.clone();
104/// # let _ = shared;
105/// ```
106#[cfg(windows)]
107pub type SharedSocket = Arc<dyn std::os::windows::io::AsSocket + Send + Sync>;
108
109/// A borrowed socket, as the server's own reactor registers it.
110#[cfg(unix)]
111pub(crate) type BorrowedSocket<'a> = std::os::fd::BorrowedFd<'a>;
112/// A borrowed socket, as the server's own reactor registers it.
113#[cfg(windows)]
114pub(crate) type BorrowedSocket<'a> = std::os::windows::io::BorrowedSocket<'a>;
115
116/// Names a registered socket. The low 32 bits index a dense table inside the
117/// driver, so keep them small; the high bits are free for a generation.
118///
119/// ```
120/// let t = net_io::Token(7);
121/// assert_eq!(t, net_io::Token(7));
122/// ```
123#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
124pub struct Token(pub u64);
125
126impl Token {
127    pub(crate) fn index(self) -> usize {
128        (self.0 & 0xFFFF_FFFF) as usize
129    }
130}
131
132/// What to do on a registered socket. The operation owns its buffers until its
133/// completion returns them.
134#[non_exhaustive]
135#[derive(Debug)]
136pub enum Operation {
137    /// Completes when a connection is waiting (accept it from the listener).
138    /// Refused on Windows, where a listener is never registered (D-016).
139    Accept {
140        /// The listener.
141        listener: Token,
142    },
143    /// Receives into `buffer`'s spare room (at most once); `Ok(0)` is end of stream.
144    Receive {
145        /// The socket.
146        token: Token,
147        /// Where the bytes land; returned by the completion.
148        buffer: Buffer,
149    },
150    /// Completes when bytes can be read, without holding a buffer: idle
151    /// connections wait this way.
152    ReceiveZero {
153        /// The socket.
154        token: Token,
155    },
156    /// Sends what `chunks` holds; the result says how many bytes went out, which
157    /// can be fewer than all of them. The chunks come back with the completion.
158    Send {
159        /// The socket.
160        token: Token,
161        /// The bytes.
162        chunks: SendChunks,
163    },
164    /// Sends up to `len` bytes of `file` from `offset`, zero copy where the OS can. The
165    /// result says how many went out; `Ok(0)` means the file has nothing there. A `len`
166    /// of 0 completes at once with `Ok(0)` and sends nothing.
167    SendFile {
168        /// The socket.
169        token: Token,
170        /// The file; held until the completion.
171        file: Arc<File>,
172        /// Where to start.
173        offset: u64,
174        /// The most to send.
175        len: u32,
176    },
177    /// Connects a registered, non-blocking socket.
178    Connect {
179        /// The socket.
180        token: Token,
181        /// The peer.
182        addr: SocketAddr,
183    },
184    /// Completes once every other operation on the socket has; the caller then
185    /// closes it. Abortive when asked (linger 0).
186    Close {
187        /// The socket.
188        token: Token,
189        /// Reset instead of FIN.
190        abortive: bool,
191    },
192    /// Completes at once (a turn of the loop).
193    Nop {
194        /// Echoed in the completion.
195        token: Token,
196    },
197}
198
199impl Operation {
200    fn token(&self) -> Token {
201        match self {
202            Operation::Accept { listener } => *listener,
203            Operation::Receive { token, .. }
204            | Operation::ReceiveZero { token }
205            | Operation::Send { token, .. }
206            | Operation::SendFile { token, .. }
207            | Operation::Connect { token, .. }
208            | Operation::Close { token, .. }
209            | Operation::Nop { token } => *token,
210        }
211    }
212}
213
214/// Which operation a completion finishes.
215///
216/// ```
217/// assert_ne!(net_io::OperationKind::Receive, net_io::OperationKind::Send);
218/// ```
219#[non_exhaustive]
220#[derive(Clone, Copy, Debug, PartialEq, Eq)]
221pub enum OperationKind {
222    /// [`Operation::Accept`].
223    Accept,
224    /// [`Operation::Receive`] and [`Operation::ReceiveZero`].
225    Receive,
226    /// [`Operation::Send`].
227    Send,
228    /// [`Operation::SendFile`].
229    SendFile,
230    /// [`Operation::Connect`].
231    Connect,
232    /// [`Operation::Close`].
233    Close,
234    /// [`Operation::Nop`].
235    Nop,
236}
237
238/// A finished operation.
239#[non_exhaustive]
240#[derive(Debug)]
241pub struct Completion {
242    /// The socket.
243    pub token: Token,
244    /// Which operation finished.
245    pub kind: OperationKind,
246    /// Bytes moved, or the error class. A cancelled operation is `Aborted`.
247    pub result: Result<usize, OsErrorClass>,
248    /// The receive buffer, returned with its new length.
249    pub buffer: Option<Buffer>,
250    /// The send chunks, returned as they were.
251    pub chunks: Option<SendChunks>,
252}
253
254impl Completion {
255    pub(crate) fn new(
256        token: Token,
257        kind: OperationKind,
258        result: Result<usize, OsErrorClass>,
259    ) -> Completion {
260        Completion {
261            token,
262            kind,
263            result,
264            buffer: None,
265            chunks: None,
266        }
267    }
268}
269
270/// How many completions one wait returns.
271pub(crate) const COMPLETIONS: usize = 256;
272
273/// What a wait fills; fixed capacity (256), so a wait never allocates.
274#[derive(Debug)]
275pub struct Completions {
276    items: Vec<Completion>,
277}
278
279impl Default for Completions {
280    fn default() -> Completions {
281        Completions::new()
282    }
283}
284
285impl Completions {
286    /// Room for 256 completions.
287    #[must_use]
288    pub fn new() -> Completions {
289        Completions {
290            items: Vec::with_capacity(COMPLETIONS),
291        }
292    }
293
294    /// How many completions are held.
295    #[must_use]
296    pub fn len(&self) -> usize {
297        self.items.len()
298    }
299
300    /// Whether none are held.
301    #[must_use]
302    pub fn is_empty(&self) -> bool {
303        self.items.is_empty()
304    }
305
306    /// Takes the completions out, oldest first.
307    pub fn drain(&mut self) -> impl Iterator<Item = Completion> + '_ {
308        self.items.drain(..)
309    }
310
311    #[cfg(feature = "reactor")]
312    pub(crate) fn push(&mut self, c: Completion) {
313        self.items.push(c);
314    }
315
316    #[cfg(feature = "reactor")]
317    pub(crate) fn has_room(&self) -> bool {
318        self.items.len() < COMPLETIONS
319    }
320}
321
322/// The bytes of one send: up to 64 chunks, taken from an [`Output`] or built from
323/// a pool buffer.
324#[derive(Debug, Default)]
325pub struct SendChunks {
326    pub(crate) chunks: VecDeque<Chunk>,
327}
328
329impl SendChunks {
330    /// One pool buffer.
331    #[must_use]
332    pub fn from_buffer(buffer: Buffer) -> SendChunks {
333        let mut chunks = VecDeque::with_capacity(1);
334        chunks.push_back(Chunk::Pooled(buffer, 0));
335        SendChunks { chunks }
336    }
337
338    /// Takes everything queued in `output`; give it back with [`SendChunks::restore`].
339    pub(crate) fn take(output: &mut Output) -> SendChunks {
340        SendChunks {
341            chunks: std::mem::take(&mut output.chunks),
342        }
343    }
344
345    /// Puts the chunks back into `output`, keeping the queue's allocation.
346    pub(crate) fn restore(self, output: &mut Output) {
347        output.chunks = self.chunks;
348    }
349
350    /// Bytes held.
351    #[must_use]
352    pub fn pending(&self) -> usize {
353        self.chunks.iter().map(Chunk::len).sum()
354    }
355
356    /// Marks `n` bytes of the front chunks sent; `true` when bytes remain.
357    #[cfg(feature = "reactor")]
358    pub(crate) fn advance(&mut self, mut n: usize) -> bool {
359        while let Some(front) = self.chunks.front_mut() {
360            let rem = front.len();
361            if n < rem {
362                front.advance(n);
363                return true;
364            }
365            if self.chunks.len() == 1 {
366                return false;
367            }
368            n -= rem;
369            self.chunks.pop_front();
370        }
371        false
372    }
373
374    /// The first buffer back, for a send built with [`SendChunks::from_buffer`].
375    pub(crate) fn into_buffer(mut self) -> Option<Buffer> {
376        match self.chunks.pop_front() {
377            Some(Chunk::Pooled(b, _)) => Some(b),
378            _ => None,
379        }
380    }
381
382    /// Fills `out` with the chunks' bytes; returns how many slices.
383    #[cfg(feature = "reactor")]
384    pub(crate) fn slices<'a>(&'a self, out: &mut [IoSlice<'a>]) -> usize {
385        let mut n = 0;
386        // A file goes by its own operation: the slices stop before it.
387        for chunk in self
388            .chunks
389            .iter()
390            .take(out.len())
391            .take_while(|c| !c.is_file())
392        {
393            out[n] = IoSlice::new(chunk.remaining());
394            n += 1;
395        }
396        n
397    }
398}
399
400/// Why [`Driver::submit`] refused an operation; it comes back with its buffers.
401#[derive(Debug)]
402pub struct Rejected {
403    /// The operation.
404    pub operation: Operation,
405}
406
407/// Wakes a driver's [`Driver::wait`] from any thread.
408#[derive(Clone, Debug)]
409pub struct DriverWaker {
410    pub(crate) inner: Arc<WakerInner>,
411}
412
413#[cfg(feature = "reactor")]
414pub(crate) type WakerInner = crate::sys::PollWaker;
415#[cfg(not(feature = "reactor"))]
416#[derive(Debug)]
417pub(crate) struct WakerInner;
418
419impl DriverWaker {
420    /// Ends the current or next wait. Cheap to call repeatedly.
421    pub fn wake(&self) {
422        #[cfg(feature = "reactor")]
423        self.inner.wake();
424    }
425}
426
427/// The reactor driver for this OS: epoll, kqueue, or an I/O completion port.
428/// `capacity` sizes its tables for that many sockets; they grow past it.
429///
430/// One per worker. A token's low 32 bits index a table the driver keeps, so keep
431/// them small and dense, and put a generation in the high 32 bits to catch a
432/// completion for a slot that moved on. On Windows a listener cannot be registered
433/// and `Operation::Accept` is refused (accept from a thread of your own, D-016).
434///
435/// ```
436/// # fn main() -> Result<(), net_io::NetError> {
437/// use std::time::Duration;
438/// use net_io::{Completions, Operation, Token};
439/// let mut driver = net_io::new_reactor_driver(16)?;
440/// driver.submit(Operation::Nop { token: Token(1) }).expect("a no-op is always taken");
441/// let mut done = Completions::new();
442/// assert_eq!(driver.wait(&mut done, Some(Duration::from_secs(1)))?, 1);
443/// # Ok(())
444/// # }
445/// ```
446///
447/// # Errors
448///
449/// `DriverSetup` when the OS refuses the queue.
450#[cfg(feature = "reactor")]
451pub fn new_reactor_driver(capacity: usize) -> Result<Box<dyn Driver>, NetError> {
452    #[cfg(unix)]
453    let driver = ready::ReadyDriver::new(capacity)?;
454    #[cfg(windows)]
455    let driver = iocp::IocpDriver::new(capacity)?;
456    Ok(Box::new(driver))
457}
458
459/// A driver: one per worker, used from one thread. See the module docs.
460///
461/// Unstable until `1.0.0`. The driver holds each registered socket (a
462/// [`SharedSocket`]) from [`Driver::register`] until [`Driver::deregister`], so the
463/// descriptor it does I/O on can never close and be reused by another connection
464/// while an operation or a cached readiness refers to it.
465pub trait Driver: Send {
466    /// Which driver this is.
467    fn kind(&self) -> DriverKind;
468
469    /// Adds a non-blocking socket under `token`. The driver keeps `socket` until
470    /// [`Driver::deregister`]: dropping the caller's own clones before then does
471    /// not close it.
472    ///
473    /// # Errors
474    ///
475    /// `Os` when the OS refuses.
476    fn register(&mut self, socket: SharedSocket, token: Token) -> Result<(), NetError>;
477
478    /// Starts an operation; its completion arrives through [`Driver::wait`].
479    ///
480    /// # Errors
481    ///
482    /// [`Rejected`] gives the operation and its buffers back: the token is not
483    /// registered, or the same kind of operation is already pending on it.
484    fn submit(&mut self, operation: Operation) -> Result<(), Rejected>;
485
486    /// Fills `out` with finished operations, waiting up to `timeout` for the
487    /// first (`None`: until one finishes or the waker fires). Returns how many.
488    ///
489    /// # Errors
490    ///
491    /// `Os` when the wait itself fails.
492    fn wait(&mut self, out: &mut Completions, timeout: Option<Duration>)
493    -> Result<usize, NetError>;
494
495    /// Cancels the pending operation of kind `which`; it still completes exactly
496    /// once (with `Aborted`, or with its result if it already finished).
497    ///
498    /// # Errors
499    ///
500    /// `Os` when the OS refuses.
501    fn cancel(&mut self, token: Token, which: OperationKind) -> Result<(), NetError>;
502
503    /// Removes the socket; no completion arrives for it afterwards. The driver
504    /// drops its clone of the socket, which closes it if no other clone is left.
505    ///
506    /// # Errors
507    ///
508    /// `Os` when the OS refuses.
509    fn deregister(&mut self, token: Token) -> Result<(), NetError>;
510
511    /// A waker for this driver.
512    fn waker(&self) -> DriverWaker;
513}
514
515pub(crate) fn operation_token(op: &Operation) -> Token {
516    op.token()
517}