Skip to main content

net_io/
buf.rs

1//! Pooled buffers with a total budget, wiped on release.
2
3use std::fmt;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicU64, Ordering};
6
7use crate::limits::SizeClass;
8
9/// A byte buffer from a pool: a fixed capacity (its size class) and a filled length.
10///
11/// # Examples
12///
13/// ```
14/// use net_io::{BufferPool, SizeClass};
15/// let mut pool = BufferPool::standalone(1 << 20);
16/// let mut buf = pool.take(SizeClass::SMALL).unwrap();
17/// buf.spare()[..5].copy_from_slice(b"hello");
18/// buf.set_len(5);
19/// assert_eq!(buf.as_slice(), b"hello");
20/// pool.put(buf);
21/// ```
22pub struct Buffer {
23    data: Box<[u8]>,
24    len: usize,
25    /// The longest `len` since the last wipe: `clear`, `compact`, and a consumed
26    /// input shrink `len` but leave the bytes, so the wipe covers this much. A `u32`
27    /// (buffers are at most 1 MiB), so it packs beside the class.
28    dirty: u32,
29    class: SizeClass,
30}
31
32impl Buffer {
33    /// A buffer not drawn from any pool's free list (the caller already charged the budget).
34    pub(crate) fn detached(class: SizeClass) -> Buffer {
35        Buffer::new(class)
36    }
37
38    /// An empty stand-in that owns no memory, left where a buffer travels with a
39    /// driver operation.
40    #[cfg(feature = "reactor")]
41    pub(crate) fn placeholder() -> Buffer {
42        Buffer {
43            data: Box::default(),
44            len: 0,
45            dirty: 0,
46            class: SizeClass::SMALL,
47        }
48    }
49
50    fn new(class: SizeClass) -> Buffer {
51        Buffer {
52            data: vec![0u8; class.bytes() as usize].into_boxed_slice(),
53            len: 0,
54            dirty: 0,
55            class,
56        }
57    }
58
59    /// The filled part.
60    #[must_use]
61    pub fn as_slice(&self) -> &[u8] {
62        &self.data[..self.len]
63    }
64
65    /// The filled part, mutable.
66    pub fn as_mut_slice(&mut self) -> &mut [u8] {
67        &mut self.data[..self.len]
68    }
69
70    /// The unfilled tail.
71    pub fn spare(&mut self) -> &mut [u8] {
72        &mut self.data[self.len..]
73    }
74
75    /// The filled length.
76    #[must_use]
77    pub fn len(&self) -> usize {
78        self.len
79    }
80
81    /// Whether nothing is filled.
82    #[must_use]
83    pub fn is_empty(&self) -> bool {
84        self.len == 0
85    }
86
87    /// Sets the filled length; saturates at the capacity.
88    pub fn set_len(&mut self, n: usize) {
89        self.len = n.min(self.data.len());
90        self.dirty = self.dirty.max(u32::try_from(self.len).unwrap_or(u32::MAX));
91    }
92
93    /// The capacity in bytes.
94    #[must_use]
95    pub fn capacity(&self) -> usize {
96        self.data.len()
97    }
98
99    /// The size class.
100    #[must_use]
101    pub fn class(&self) -> SizeClass {
102        self.class
103    }
104
105    /// Empties the buffer (does not wipe; the pool wipes on release).
106    pub fn clear(&mut self) {
107        self.len = 0;
108    }
109
110    /// Overwrites everything filled since the last wipe with zeroes and empties the
111    /// buffer.
112    pub(crate) fn wipe(&mut self) {
113        let held = (self.dirty as usize).max(self.len).min(self.data.len());
114        let used = &mut self.data[..held];
115        used.fill(0);
116        // The buffer stays reachable and is read again by the next owner, so the
117        // stores cannot be removed as dead; black_box makes that explicit.
118        std::hint::black_box(&*used);
119        self.len = 0;
120        self.dirty = 0;
121    }
122
123    /// Moves the bytes `from..` to the front.
124    pub(crate) fn compact(&mut self, from: usize) {
125        let from = from.min(self.len);
126        self.data.copy_within(from..self.len, 0);
127        self.len -= from;
128    }
129}
130
131impl fmt::Debug for Buffer {
132    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
133        f.debug_struct("Buffer")
134            .field("len", &self.len)
135            .field("capacity", &self.data.len())
136            .finish()
137    }
138}
139
140/// Counts of one pool.
141///
142/// ```
143/// let pool = net_io::BufferPool::standalone(1 << 20);
144/// assert_eq!(pool.stats().allocated, 0);
145/// ```
146#[non_exhaustive]
147#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
148pub struct PoolStats {
149    /// Buffers this pool allocated.
150    pub allocated: u64,
151    /// Buffers in its free lists.
152    pub free: u64,
153    /// Bytes left in the shared budget.
154    pub budget_left: u64,
155    /// Takes refused because the budget was spent.
156    pub budget_refusals: u64,
157}
158
159/// A worker's buffer pool: free lists per size class, drawing new buffers from a
160/// budget shared by every worker of a server.
161pub struct BufferPool {
162    lists: Vec<(SizeClass, Vec<Buffer>)>,
163    budget: Arc<AtomicU64>,
164    wipe: bool,
165    allocated: u64,
166    refusals: u64,
167}
168
169impl BufferPool {
170    pub(crate) fn new(budget: Arc<AtomicU64>, wipe: bool) -> BufferPool {
171        BufferPool {
172            lists: Vec::with_capacity(4),
173            budget,
174            wipe,
175            allocated: 0,
176            refusals: 0,
177        }
178    }
179
180    /// A pool with its own budget of `bytes`, for use outside a server.
181    ///
182    /// ```
183    /// let mut pool = net_io::BufferPool::standalone(64 * 1024);
184    /// assert!(pool.take(net_io::SizeClass::LARGE).is_some());
185    /// assert!(pool.take(net_io::SizeClass::LARGE).is_none()); // budget spent
186    /// ```
187    #[must_use]
188    pub fn standalone(bytes: u64) -> BufferPool {
189        BufferPool::new(Arc::new(AtomicU64::new(bytes)), true)
190    }
191
192    /// A buffer of `class`, or `None` when the free list is empty and the budget is spent.
193    pub fn take(&mut self, class: SizeClass) -> Option<Buffer> {
194        if let Some(buf) = self.list(class).pop() {
195            return Some(buf);
196        }
197        let bytes = u64::from(class.bytes());
198        let mut left = self.budget.load(Ordering::Relaxed);
199        loop {
200            if left < bytes {
201                self.refusals += 1;
202                return None;
203            }
204            match self.budget.compare_exchange_weak(
205                left,
206                left - bytes,
207                Ordering::AcqRel,
208                Ordering::Relaxed,
209            ) {
210                Ok(_) => break,
211                Err(actual) => left = actual,
212            }
213        }
214        self.allocated += 1;
215        Some(Buffer::new(class))
216    }
217
218    /// A buffer of `class` this pool already holds; never allocates or charges the
219    /// budget.
220    pub(crate) fn take_cached(&mut self, class: SizeClass) -> Option<Buffer> {
221        self.list(class).pop()
222    }
223
224    /// Returns a buffer; its used part is wiped first unless wiping is off.
225    ///
226    /// A buffer from another worker's pool is accepted (it is just memory).
227    pub fn put(&mut self, mut buf: Buffer) {
228        if buf.data.is_empty() {
229            // A stand-in owns no memory and was never charged to the budget.
230            return;
231        }
232        if self.wipe {
233            buf.wipe();
234        } else {
235            buf.clear();
236        }
237        let class = buf.class;
238        self.list(class).push(buf);
239    }
240
241    /// Counts of this pool.
242    #[must_use]
243    pub fn stats(&self) -> PoolStats {
244        PoolStats {
245            allocated: self.allocated,
246            free: self.lists.iter().map(|(_, l)| l.len() as u64).sum(),
247            budget_left: self.budget.load(Ordering::Relaxed),
248            budget_refusals: self.refusals,
249        }
250    }
251
252    fn list(&mut self, class: SizeClass) -> &mut Vec<Buffer> {
253        let pos = match self.lists.iter().position(|(c, _)| *c == class) {
254            Some(p) => p,
255            None => {
256                self.lists.push((class, Vec::new()));
257                self.lists.len() - 1
258            }
259        };
260        &mut self.lists[pos].1
261    }
262}
263
264impl fmt::Debug for BufferPool {
265    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
266        f.debug_struct("BufferPool")
267            .field("stats", &self.stats())
268            .finish()
269    }
270}
271
272#[cfg(test)]
273mod tests {
274    use super::*;
275
276    #[test]
277    fn test_pool_reuses_and_wipes() {
278        let mut pool = BufferPool::standalone(1 << 20);
279        let mut b = pool.take(SizeClass::SMALL).unwrap();
280        b.spare()[..6].copy_from_slice(b"secret");
281        b.set_len(6);
282        pool.put(b);
283        let mut b = pool.take(SizeClass::SMALL).unwrap();
284        assert_eq!(pool.stats().allocated, 1);
285        assert!(b.is_empty());
286        b.set_len(6);
287        assert_eq!(b.as_slice(), &[0u8; 6]);
288    }
289
290    /// Bytes left behind a shorter `len` are wiped too: a cleared buffer, a
291    /// compacted one, and a consumed input (review finding: only `[..len]` was).
292    #[test]
293    fn test_pool_wipes_what_a_shorter_len_left_behind() {
294        let mut pool = BufferPool::standalone(1 << 20);
295        let mut b = pool.take(SizeClass::SMALL).unwrap();
296        b.spare()[..12].copy_from_slice(b"token=secret");
297        b.set_len(12);
298        b.clear();
299        pool.put(b);
300        let mut b = pool.take(SizeClass::SMALL).unwrap();
301        b.set_len(b.capacity());
302        assert!(
303            b.as_slice().iter().all(|&x| x == 0),
304            "cleared bytes survived"
305        );
306        b.set_len(0);
307        b.spare()[..12].copy_from_slice(b"token=secret");
308        b.set_len(12);
309        b.compact(6);
310        assert_eq!(b.as_slice(), b"secret");
311        pool.put(b);
312        let mut b = pool.take(SizeClass::SMALL).unwrap();
313        b.set_len(b.capacity());
314        assert!(
315            b.as_slice().iter().all(|&x| x == 0),
316            "compacted tail survived"
317        );
318        let mut input = crate::session::Input::new({
319            b.set_len(0);
320            b
321        });
322        input.buf.spare()[..6].copy_from_slice(b"secret");
323        input.filled(6);
324        input.consume(6);
325        pool.put(input.buf);
326        let mut b = pool.take(SizeClass::SMALL).unwrap();
327        b.set_len(b.capacity());
328        assert!(
329            b.as_slice().iter().all(|&x| x == 0),
330            "consumed input survived"
331        );
332    }
333
334    #[test]
335    fn test_pool_budget_bounds_allocation() {
336        let mut pool = BufferPool::standalone(u64::from(SizeClass::SMALL.bytes()) * 2);
337        let a = pool.take(SizeClass::SMALL).unwrap();
338        let _b = pool.take(SizeClass::SMALL).unwrap();
339        assert!(pool.take(SizeClass::SMALL).is_none());
340        assert_eq!(pool.stats().budget_refusals, 1);
341        pool.put(a);
342        assert!(pool.take(SizeClass::SMALL).is_some());
343    }
344
345    #[test]
346    fn test_set_len_saturates_and_compact_moves_tail() {
347        let mut pool = BufferPool::standalone(1 << 20);
348        let mut b = pool.take(SizeClass::SMALL).unwrap();
349        b.set_len(usize::MAX);
350        assert_eq!(b.len(), b.capacity());
351        b.set_len(0);
352        b.spare()[..4].copy_from_slice(b"abcd");
353        b.set_len(4);
354        b.compact(2);
355        assert_eq!(b.as_slice(), b"cd");
356        b.compact(99);
357        assert!(b.is_empty());
358    }
359
360    #[test]
361    fn test_pool_without_wipe_keeps_bytes_but_empties() {
362        let mut pool = BufferPool::new(Arc::new(AtomicU64::new(1 << 20)), false);
363        let mut b = pool.take(SizeClass::SMALL).unwrap();
364        b.spare()[0] = 9;
365        b.set_len(1);
366        pool.put(b);
367        let b = pool.take(SizeClass::SMALL).unwrap();
368        assert!(b.is_empty());
369    }
370
371    /// The cost of T-16: `cargo test --release --lib bench_ -- --ignored --nocapture`.
372    #[test]
373    #[ignore = "a benchmark"]
374    fn bench_wipe() {
375        for (name, used) in [("2 KiB", 2048), ("16 KiB", 16 * 1024)] {
376            let mut b = Buffer::new(SizeClass::MEDIUM);
377            let rounds = 1_000_000u32;
378            let started = std::time::Instant::now();
379            for _ in 0..rounds {
380                b.set_len(used);
381                b.wipe();
382            }
383            let ns = started.elapsed().as_nanos() as f64 / f64::from(rounds);
384            println!("Wipe {name}: {ns:.1} ns");
385        }
386    }
387}