Skip to main content

net_io/driver/
mod.rs

1//! Drivers: what moves the bytes.
2//!
3//! The blocking pool ships in `0.3.0`, the reactor (epoll, kqueue, IOCP) in
4//! `0.5.0`, io_uring after `1.0.0`. The [`Driver`] trait is public from `0.5.0`
5//! (Tier 3, unstable until `1.0.0`) so an executor adapter and later drivers can
6//! build on it.
7//!
8//! The interface is completion-shaped (decision D-005): an operation owns its
9//! buffers until its completion hands them back, which is what IOCP and io_uring
10//! need; epoll and kqueue adapt by making the non-blocking call when readiness
11//! arrives. Cancellation always ends in exactly one completion.
12
13// Without the reactor only the vocabulary is left, for the public interface.
14#![cfg_attr(not(feature = "reactor"), allow(dead_code, unused_imports))]
15
16#[cfg(all(feature = "reactor", windows))]
17pub(crate) mod iocp;
18#[cfg(all(feature = "reactor", unix))]
19pub(crate) mod ready;
20
21use std::collections::VecDeque;
22use std::fs::File;
23use std::io::IoSlice;
24use std::net::SocketAddr;
25use std::sync::Arc;
26use std::time::Duration;
27
28use crate::buf::Buffer;
29use crate::error::{NetError, OsErrorClass};
30use crate::session::{Chunk, Output};
31
32/// Which driver moves the bytes.
33///
34/// `Auto` picks the reactor where it is built and available, otherwise the
35/// blocking pool.
36///
37/// # Examples
38///
39/// ```
40/// assert_eq!(net_io::DriverKind::default(), net_io::DriverKind::Auto);
41/// ```
42#[non_exhaustive]
43#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
44pub enum DriverKind {
45    /// A bounded pool of threads, one connection per thread.
46    Blocking,
47    /// One event loop per worker (epoll, kqueue, IOCP); from `0.5.0`.
48    Reactor,
49    /// io_uring on Linux; after `1.0.0`.
50    Uring,
51    /// The reactor where available, otherwise the blocking pool.
52    #[default]
53    Auto,
54}
55
56impl DriverKind {
57    /// The driver `Auto` resolves to in this build.
58    pub(crate) fn resolve(self) -> DriverKind {
59        match self {
60            DriverKind::Auto if DriverKind::reactor_built() && !cfg!(net_io_force_blocking) => {
61                DriverKind::Reactor
62            }
63            DriverKind::Auto => DriverKind::Blocking,
64            other => other,
65        }
66    }
67
68    /// Whether this build has a reactor for this OS.
69    pub(crate) const fn reactor_built() -> bool {
70        cfg!(feature = "reactor")
71    }
72}
73
74/// A socket as a [`Driver`] registers it: shared, so the driver keeps it open
75/// until [`Driver::deregister`] whatever the caller drops. Any `Arc` of a socket
76/// type converts (`Arc<TcpStream>`, `Arc<TcpListener>`, `Arc<OwnedFd>`); keep a
77/// clone to accept from a listener or to read the socket's options.
78///
79/// The driver works on the descriptor `as_fd` returns at registration, so the
80/// value must keep returning the same one while registered, as every `std`
81/// socket type does.
82///
83/// ```
84/// use std::sync::Arc;
85/// let listener = Arc::new(std::net::TcpListener::bind("127.0.0.1:0").unwrap());
86/// let shared: net_io::SharedSocket = listener.clone();
87/// # let _ = shared;
88/// ```
89#[cfg(unix)]
90pub type SharedSocket = Arc<dyn std::os::fd::AsFd + Send + Sync>;
91/// A socket as a [`Driver`] registers it: shared, so the driver keeps it open
92/// until [`Driver::deregister`] whatever the caller drops. Any `Arc` of a socket
93/// type converts (`Arc<TcpStream>`, `Arc<TcpListener>`, `Arc<OwnedSocket>`); keep
94/// a clone to read the socket's options.
95///
96/// The driver works on the handle `as_socket` returns at registration, so the
97/// value must keep returning the same one while registered, as every `std`
98/// socket type does.
99///
100/// ```
101/// use std::sync::Arc;
102/// let stream = Arc::new(std::net::UdpSocket::bind("127.0.0.1:0").unwrap());
103/// let shared: net_io::SharedSocket = stream.clone();
104/// # let _ = shared;
105/// ```
106#[cfg(windows)]
107pub type SharedSocket = Arc<dyn std::os::windows::io::AsSocket + Send + Sync>;
108
109/// A borrowed socket, as the server's own reactor registers it.
110#[cfg(unix)]
111pub(crate) type BorrowedSocket<'a> = std::os::fd::BorrowedFd<'a>;
112/// A borrowed socket, as the server's own reactor registers it.
113#[cfg(windows)]
114pub(crate) type BorrowedSocket<'a> = std::os::windows::io::BorrowedSocket<'a>;
115
116/// Names a registered socket. The low 32 bits index a dense table inside the
117/// driver, so keep them small; the high bits are free for a generation.
118///
119/// ```
120/// let t = net_io::Token(7);
121/// assert_eq!(t, net_io::Token(7));
122/// ```
123#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
124pub struct Token(pub u64);
125
126impl Token {
127    pub(crate) fn index(self) -> usize {
128        (self.0 & 0xFFFF_FFFF) as usize
129    }
130}
131
132/// What to do on a registered socket. The operation owns its buffers until its
133/// completion returns them.
134#[non_exhaustive]
135#[derive(Debug)]
136pub enum Operation {
137    /// Completes when a connection is waiting (accept it from the listener).
138    /// Refused on Windows, where a listener is never registered (D-016).
139    Accept {
140        /// The listener.
141        listener: Token,
142    },
143    /// Receives into `buffer`'s spare room (at most once); `Ok(0)` is end of stream.
144    Receive {
145        /// The socket.
146        token: Token,
147        /// Where the bytes land; returned by the completion.
148        buffer: Buffer,
149    },
150    /// Completes when bytes can be read, without holding a buffer: idle
151    /// connections wait this way.
152    ReceiveZero {
153        /// The socket.
154        token: Token,
155    },
156    /// Sends what `chunks` holds; the result says how many bytes went out, which
157    /// can be fewer than all of them. The chunks come back with the completion.
158    Send {
159        /// The socket.
160        token: Token,
161        /// The bytes.
162        chunks: SendChunks,
163    },
164    /// Sends up to `len` bytes of `file` from `offset`, zero copy where the OS can. The
165    /// result says how many went out; `Ok(0)` means the file has nothing there.
166    SendFile {
167        /// The socket.
168        token: Token,
169        /// The file; held until the completion.
170        file: Arc<File>,
171        /// Where to start.
172        offset: u64,
173        /// The most to send.
174        len: u32,
175    },
176    /// Connects a registered, non-blocking socket.
177    Connect {
178        /// The socket.
179        token: Token,
180        /// The peer.
181        addr: SocketAddr,
182    },
183    /// Completes once every other operation on the socket has; the caller then
184    /// closes it. Abortive when asked (linger 0).
185    Close {
186        /// The socket.
187        token: Token,
188        /// Reset instead of FIN.
189        abortive: bool,
190    },
191    /// Completes at once (a turn of the loop).
192    Nop {
193        /// Echoed in the completion.
194        token: Token,
195    },
196}
197
198impl Operation {
199    fn token(&self) -> Token {
200        match self {
201            Operation::Accept { listener } => *listener,
202            Operation::Receive { token, .. }
203            | Operation::ReceiveZero { token }
204            | Operation::Send { token, .. }
205            | Operation::SendFile { token, .. }
206            | Operation::Connect { token, .. }
207            | Operation::Close { token, .. }
208            | Operation::Nop { token } => *token,
209        }
210    }
211}
212
213/// Which operation a completion finishes.
214///
215/// ```
216/// assert_ne!(net_io::OperationKind::Receive, net_io::OperationKind::Send);
217/// ```
218#[non_exhaustive]
219#[derive(Clone, Copy, Debug, PartialEq, Eq)]
220pub enum OperationKind {
221    /// [`Operation::Accept`].
222    Accept,
223    /// [`Operation::Receive`] and [`Operation::ReceiveZero`].
224    Receive,
225    /// [`Operation::Send`].
226    Send,
227    /// [`Operation::SendFile`].
228    SendFile,
229    /// [`Operation::Connect`].
230    Connect,
231    /// [`Operation::Close`].
232    Close,
233    /// [`Operation::Nop`].
234    Nop,
235}
236
237/// A finished operation.
238#[non_exhaustive]
239#[derive(Debug)]
240pub struct Completion {
241    /// The socket.
242    pub token: Token,
243    /// Which operation finished.
244    pub kind: OperationKind,
245    /// Bytes moved, or the error class. A cancelled operation is `Aborted`.
246    pub result: Result<usize, OsErrorClass>,
247    /// The receive buffer, returned with its new length.
248    pub buffer: Option<Buffer>,
249    /// The send chunks, returned as they were.
250    pub chunks: Option<SendChunks>,
251}
252
253impl Completion {
254    pub(crate) fn new(
255        token: Token,
256        kind: OperationKind,
257        result: Result<usize, OsErrorClass>,
258    ) -> Completion {
259        Completion {
260            token,
261            kind,
262            result,
263            buffer: None,
264            chunks: None,
265        }
266    }
267}
268
269/// How many completions one wait returns.
270pub(crate) const COMPLETIONS: usize = 256;
271
272/// What a wait fills; fixed capacity (256), so a wait never allocates.
273#[derive(Debug)]
274pub struct Completions {
275    items: Vec<Completion>,
276}
277
278impl Default for Completions {
279    fn default() -> Completions {
280        Completions::new()
281    }
282}
283
284impl Completions {
285    /// Room for 256 completions.
286    #[must_use]
287    pub fn new() -> Completions {
288        Completions {
289            items: Vec::with_capacity(COMPLETIONS),
290        }
291    }
292
293    /// How many completions are held.
294    #[must_use]
295    pub fn len(&self) -> usize {
296        self.items.len()
297    }
298
299    /// Whether none are held.
300    #[must_use]
301    pub fn is_empty(&self) -> bool {
302        self.items.is_empty()
303    }
304
305    /// Takes the completions out, oldest first.
306    pub fn drain(&mut self) -> impl Iterator<Item = Completion> + '_ {
307        self.items.drain(..)
308    }
309
310    #[cfg(feature = "reactor")]
311    pub(crate) fn push(&mut self, c: Completion) {
312        self.items.push(c);
313    }
314
315    #[cfg(feature = "reactor")]
316    pub(crate) fn has_room(&self) -> bool {
317        self.items.len() < COMPLETIONS
318    }
319}
320
321/// The bytes of one send: up to 64 chunks, taken from an [`Output`] or built from
322/// a pool buffer.
323#[derive(Debug, Default)]
324pub struct SendChunks {
325    pub(crate) chunks: VecDeque<Chunk>,
326}
327
328impl SendChunks {
329    /// One pool buffer.
330    #[must_use]
331    pub fn from_buffer(buffer: Buffer) -> SendChunks {
332        let mut chunks = VecDeque::with_capacity(1);
333        chunks.push_back(Chunk::Pooled(buffer, 0));
334        SendChunks { chunks }
335    }
336
337    /// Takes everything queued in `output`; give it back with [`SendChunks::restore`].
338    pub(crate) fn take(output: &mut Output) -> SendChunks {
339        SendChunks {
340            chunks: std::mem::take(&mut output.chunks),
341        }
342    }
343
344    /// Puts the chunks back into `output`, keeping the queue's allocation.
345    pub(crate) fn restore(self, output: &mut Output) {
346        output.chunks = self.chunks;
347    }
348
349    /// Bytes held.
350    #[must_use]
351    pub fn pending(&self) -> usize {
352        self.chunks.iter().map(Chunk::len).sum()
353    }
354
355    /// Marks `n` bytes of the front chunks sent; `true` when bytes remain.
356    #[cfg(feature = "reactor")]
357    pub(crate) fn advance(&mut self, mut n: usize) -> bool {
358        while let Some(front) = self.chunks.front_mut() {
359            let rem = front.len();
360            if n < rem {
361                front.advance(n);
362                return true;
363            }
364            if self.chunks.len() == 1 {
365                return false;
366            }
367            n -= rem;
368            self.chunks.pop_front();
369        }
370        false
371    }
372
373    /// The first buffer back, for a send built with [`SendChunks::from_buffer`].
374    pub(crate) fn into_buffer(mut self) -> Option<Buffer> {
375        match self.chunks.pop_front() {
376            Some(Chunk::Pooled(b, _)) => Some(b),
377            _ => None,
378        }
379    }
380
381    /// Fills `out` with the chunks' bytes; returns how many slices.
382    #[cfg(feature = "reactor")]
383    pub(crate) fn slices<'a>(&'a self, out: &mut [IoSlice<'a>]) -> usize {
384        let mut n = 0;
385        // A file goes by its own operation: the slices stop before it.
386        for chunk in self
387            .chunks
388            .iter()
389            .take(out.len())
390            .take_while(|c| !c.is_file())
391        {
392            out[n] = IoSlice::new(chunk.remaining());
393            n += 1;
394        }
395        n
396    }
397}
398
399/// Why [`Driver::submit`] refused an operation; it comes back with its buffers.
400#[derive(Debug)]
401pub struct Rejected {
402    /// The operation.
403    pub operation: Operation,
404}
405
406/// Wakes a driver's [`Driver::wait`] from any thread.
407#[derive(Clone, Debug)]
408pub struct DriverWaker {
409    pub(crate) inner: Arc<WakerInner>,
410}
411
412#[cfg(feature = "reactor")]
413pub(crate) type WakerInner = crate::sys::PollWaker;
414#[cfg(not(feature = "reactor"))]
415#[derive(Debug)]
416pub(crate) struct WakerInner;
417
418impl DriverWaker {
419    /// Ends the current or next wait. Cheap to call repeatedly.
420    pub fn wake(&self) {
421        #[cfg(feature = "reactor")]
422        self.inner.wake();
423    }
424}
425
426/// The reactor driver for this OS: epoll, kqueue, or an I/O completion port.
427/// `capacity` sizes its tables for that many sockets; they grow past it.
428///
429/// One per worker. A token's low 32 bits index a table the driver keeps, so keep
430/// them small and dense, and put a generation in the high 32 bits to catch a
431/// completion for a slot that moved on. On Windows a listener cannot be registered
432/// and `Operation::Accept` is refused (accept from a thread of your own, D-016).
433///
434/// ```
435/// # fn main() -> Result<(), net_io::NetError> {
436/// use std::time::Duration;
437/// use net_io::{Completions, Operation, Token};
438/// let mut driver = net_io::new_reactor_driver(16)?;
439/// driver.submit(Operation::Nop { token: Token(1) }).expect("a no-op is always taken");
440/// let mut done = Completions::new();
441/// assert_eq!(driver.wait(&mut done, Some(Duration::from_secs(1)))?, 1);
442/// # Ok(())
443/// # }
444/// ```
445///
446/// # Errors
447///
448/// `DriverSetup` when the OS refuses the queue.
449#[cfg(feature = "reactor")]
450pub fn new_reactor_driver(capacity: usize) -> Result<Box<dyn Driver>, NetError> {
451    #[cfg(unix)]
452    let driver = ready::ReadyDriver::new(capacity)?;
453    #[cfg(windows)]
454    let driver = iocp::IocpDriver::new(capacity)?;
455    Ok(Box::new(driver))
456}
457
458/// A driver: one per worker, used from one thread. See the module docs.
459///
460/// Unstable until `1.0.0`. The driver holds each registered socket (a
461/// [`SharedSocket`]) from [`Driver::register`] until [`Driver::deregister`], so the
462/// descriptor it does I/O on can never close and be reused by another connection
463/// while an operation or a cached readiness refers to it.
464pub trait Driver: Send {
465    /// Which driver this is.
466    fn kind(&self) -> DriverKind;
467
468    /// Adds a non-blocking socket under `token`. The driver keeps `socket` until
469    /// [`Driver::deregister`]: dropping the caller's own clones before then does
470    /// not close it.
471    ///
472    /// # Errors
473    ///
474    /// `Os` when the OS refuses.
475    fn register(&mut self, socket: SharedSocket, token: Token) -> Result<(), NetError>;
476
477    /// Starts an operation; its completion arrives through [`Driver::wait`].
478    ///
479    /// # Errors
480    ///
481    /// [`Rejected`] gives the operation and its buffers back: the token is not
482    /// registered, or the same kind of operation is already pending on it.
483    fn submit(&mut self, operation: Operation) -> Result<(), Rejected>;
484
485    /// Fills `out` with finished operations, waiting up to `timeout` for the
486    /// first (`None`: until one finishes or the waker fires). Returns how many.
487    ///
488    /// # Errors
489    ///
490    /// `Os` when the wait itself fails.
491    fn wait(&mut self, out: &mut Completions, timeout: Option<Duration>)
492    -> Result<usize, NetError>;
493
494    /// Cancels the pending operation of kind `which`; it still completes exactly
495    /// once (with `Aborted`, or with its result if it already finished).
496    ///
497    /// # Errors
498    ///
499    /// `Os` when the OS refuses.
500    fn cancel(&mut self, token: Token, which: OperationKind) -> Result<(), NetError>;
501
502    /// Removes the socket; no completion arrives for it afterwards. The driver
503    /// drops its clone of the socket, which closes it if no other clone is left.
504    ///
505    /// # Errors
506    ///
507    /// `Os` when the OS refuses.
508    fn deregister(&mut self, token: Token) -> Result<(), NetError>;
509
510    /// A waker for this driver.
511    fn waker(&self) -> DriverWaker;
512}
513
514pub(crate) fn operation_token(op: &Operation) -> Token {
515    op.token()
516}