1use std::fmt;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicU64, Ordering};
6
7use crate::limits::SizeClass;
8
9pub struct Buffer {
23 data: Box<[u8]>,
24 len: usize,
25 dirty: u32,
29 class: SizeClass,
30}
31
32impl Buffer {
33 pub(crate) fn detached(class: SizeClass) -> Buffer {
35 Buffer::new(class)
36 }
37
38 #[cfg(feature = "reactor")]
41 pub(crate) fn placeholder() -> Buffer {
42 Buffer {
43 data: Box::default(),
44 len: 0,
45 dirty: 0,
46 class: SizeClass::SMALL,
47 }
48 }
49
50 fn new(class: SizeClass) -> Buffer {
51 Buffer {
52 data: vec![0u8; class.bytes() as usize].into_boxed_slice(),
53 len: 0,
54 dirty: 0,
55 class,
56 }
57 }
58
59 #[must_use]
61 pub fn as_slice(&self) -> &[u8] {
62 &self.data[..self.len]
63 }
64
65 pub fn as_mut_slice(&mut self) -> &mut [u8] {
67 &mut self.data[..self.len]
68 }
69
70 pub fn spare(&mut self) -> &mut [u8] {
72 &mut self.data[self.len..]
73 }
74
75 #[must_use]
77 pub fn len(&self) -> usize {
78 self.len
79 }
80
81 #[must_use]
83 pub fn is_empty(&self) -> bool {
84 self.len == 0
85 }
86
87 pub fn set_len(&mut self, n: usize) {
89 self.len = n.min(self.data.len());
90 self.dirty = self.dirty.max(u32::try_from(self.len).unwrap_or(u32::MAX));
91 }
92
93 #[must_use]
95 pub fn capacity(&self) -> usize {
96 self.data.len()
97 }
98
99 #[must_use]
101 pub fn class(&self) -> SizeClass {
102 self.class
103 }
104
105 pub fn clear(&mut self) {
107 self.len = 0;
108 }
109
110 pub(crate) fn wipe(&mut self) {
113 let held = (self.dirty as usize).max(self.len).min(self.data.len());
114 let used = &mut self.data[..held];
115 used.fill(0);
116 std::hint::black_box(&*used);
119 self.len = 0;
120 self.dirty = 0;
121 }
122
123 pub(crate) fn compact(&mut self, from: usize) {
125 let from = from.min(self.len);
126 self.data.copy_within(from..self.len, 0);
127 self.len -= from;
128 }
129}
130
131impl fmt::Debug for Buffer {
132 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
133 f.debug_struct("Buffer")
134 .field("len", &self.len)
135 .field("capacity", &self.data.len())
136 .finish()
137 }
138}
139
140#[non_exhaustive]
147#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
148pub struct PoolStats {
149 pub allocated: u64,
151 pub free: u64,
153 pub budget_left: u64,
155 pub budget_refusals: u64,
157}
158
159pub struct BufferPool {
162 lists: Vec<(SizeClass, Vec<Buffer>)>,
163 budget: Arc<AtomicU64>,
164 wipe: bool,
165 allocated: u64,
166 refusals: u64,
167}
168
169impl BufferPool {
170 pub(crate) fn new(budget: Arc<AtomicU64>, wipe: bool) -> BufferPool {
171 BufferPool {
172 lists: Vec::with_capacity(4),
173 budget,
174 wipe,
175 allocated: 0,
176 refusals: 0,
177 }
178 }
179
180 #[must_use]
188 pub fn standalone(bytes: u64) -> BufferPool {
189 BufferPool::new(Arc::new(AtomicU64::new(bytes)), true)
190 }
191
192 pub fn take(&mut self, class: SizeClass) -> Option<Buffer> {
194 if let Some(buf) = self.list(class).pop() {
195 return Some(buf);
196 }
197 let bytes = u64::from(class.bytes());
198 let mut left = self.budget.load(Ordering::Relaxed);
199 loop {
200 if left < bytes {
201 self.refusals += 1;
202 return None;
203 }
204 match self.budget.compare_exchange_weak(
205 left,
206 left - bytes,
207 Ordering::AcqRel,
208 Ordering::Relaxed,
209 ) {
210 Ok(_) => break,
211 Err(actual) => left = actual,
212 }
213 }
214 self.allocated += 1;
215 Some(Buffer::new(class))
216 }
217
218 pub(crate) fn take_cached(&mut self, class: SizeClass) -> Option<Buffer> {
221 self.list(class).pop()
222 }
223
224 pub fn put(&mut self, mut buf: Buffer) {
228 if buf.data.is_empty() {
229 return;
231 }
232 if self.wipe {
233 buf.wipe();
234 } else {
235 buf.clear();
236 }
237 let class = buf.class;
238 self.list(class).push(buf);
239 }
240
241 #[must_use]
243 pub fn stats(&self) -> PoolStats {
244 PoolStats {
245 allocated: self.allocated,
246 free: self.lists.iter().map(|(_, l)| l.len() as u64).sum(),
247 budget_left: self.budget.load(Ordering::Relaxed),
248 budget_refusals: self.refusals,
249 }
250 }
251
252 fn list(&mut self, class: SizeClass) -> &mut Vec<Buffer> {
253 let pos = match self.lists.iter().position(|(c, _)| *c == class) {
254 Some(p) => p,
255 None => {
256 self.lists.push((class, Vec::new()));
257 self.lists.len() - 1
258 }
259 };
260 &mut self.lists[pos].1
261 }
262}
263
264impl fmt::Debug for BufferPool {
265 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
266 f.debug_struct("BufferPool")
267 .field("stats", &self.stats())
268 .finish()
269 }
270}
271
272#[cfg(test)]
273mod tests {
274 use super::*;
275
276 #[test]
277 fn test_pool_reuses_and_wipes() {
278 let mut pool = BufferPool::standalone(1 << 20);
279 let mut b = pool.take(SizeClass::SMALL).unwrap();
280 b.spare()[..6].copy_from_slice(b"secret");
281 b.set_len(6);
282 pool.put(b);
283 let mut b = pool.take(SizeClass::SMALL).unwrap();
284 assert_eq!(pool.stats().allocated, 1);
285 assert!(b.is_empty());
286 b.set_len(6);
287 assert_eq!(b.as_slice(), &[0u8; 6]);
288 }
289
290 #[test]
293 fn test_pool_wipes_what_a_shorter_len_left_behind() {
294 let mut pool = BufferPool::standalone(1 << 20);
295 let mut b = pool.take(SizeClass::SMALL).unwrap();
296 b.spare()[..12].copy_from_slice(b"token=secret");
297 b.set_len(12);
298 b.clear();
299 pool.put(b);
300 let mut b = pool.take(SizeClass::SMALL).unwrap();
301 b.set_len(b.capacity());
302 assert!(
303 b.as_slice().iter().all(|&x| x == 0),
304 "cleared bytes survived"
305 );
306 b.set_len(0);
307 b.spare()[..12].copy_from_slice(b"token=secret");
308 b.set_len(12);
309 b.compact(6);
310 assert_eq!(b.as_slice(), b"secret");
311 pool.put(b);
312 let mut b = pool.take(SizeClass::SMALL).unwrap();
313 b.set_len(b.capacity());
314 assert!(
315 b.as_slice().iter().all(|&x| x == 0),
316 "compacted tail survived"
317 );
318 let mut input = crate::session::Input::new({
319 b.set_len(0);
320 b
321 });
322 input.buf.spare()[..6].copy_from_slice(b"secret");
323 input.filled(6);
324 input.consume(6);
325 pool.put(input.buf);
326 let mut b = pool.take(SizeClass::SMALL).unwrap();
327 b.set_len(b.capacity());
328 assert!(
329 b.as_slice().iter().all(|&x| x == 0),
330 "consumed input survived"
331 );
332 }
333
334 #[test]
335 fn test_pool_budget_bounds_allocation() {
336 let mut pool = BufferPool::standalone(u64::from(SizeClass::SMALL.bytes()) * 2);
337 let a = pool.take(SizeClass::SMALL).unwrap();
338 let _b = pool.take(SizeClass::SMALL).unwrap();
339 assert!(pool.take(SizeClass::SMALL).is_none());
340 assert_eq!(pool.stats().budget_refusals, 1);
341 pool.put(a);
342 assert!(pool.take(SizeClass::SMALL).is_some());
343 }
344
345 #[test]
346 fn test_set_len_saturates_and_compact_moves_tail() {
347 let mut pool = BufferPool::standalone(1 << 20);
348 let mut b = pool.take(SizeClass::SMALL).unwrap();
349 b.set_len(usize::MAX);
350 assert_eq!(b.len(), b.capacity());
351 b.set_len(0);
352 b.spare()[..4].copy_from_slice(b"abcd");
353 b.set_len(4);
354 b.compact(2);
355 assert_eq!(b.as_slice(), b"cd");
356 b.compact(99);
357 assert!(b.is_empty());
358 }
359
360 #[test]
361 fn test_pool_without_wipe_keeps_bytes_but_empties() {
362 let mut pool = BufferPool::new(Arc::new(AtomicU64::new(1 << 20)), false);
363 let mut b = pool.take(SizeClass::SMALL).unwrap();
364 b.spare()[0] = 9;
365 b.set_len(1);
366 pool.put(b);
367 let b = pool.take(SizeClass::SMALL).unwrap();
368 assert!(b.is_empty());
369 }
370
371 #[test]
373 #[ignore = "a benchmark"]
374 fn bench_wipe() {
375 for (name, used) in [("2 KiB", 2048), ("16 KiB", 16 * 1024)] {
376 let mut b = Buffer::new(SizeClass::MEDIUM);
377 let rounds = 1_000_000u32;
378 let started = std::time::Instant::now();
379 for _ in 0..rounds {
380 b.set_len(used);
381 b.wipe();
382 }
383 let ns = started.elapsed().as_nanos() as f64 / f64::from(rounds);
384 println!("Wipe {name}: {ns:.1} ns");
385 }
386 }
387}