ntex_bytes/
cache.rs

1//! Per-thread page cache and its statistics.
2use std::cell::Cell;
3use std::sync::{Arc, atomic::AtomicU64, atomic::Ordering::Relaxed};
4
5use crate::stvec::{Header, StorageVec};
6use crate::{BytePageSize, size::PAGE_SIZES};
7
8/// Number of page sizes, `BytePageSize::Unset` excluded.
9const CLASSES: usize = PAGE_SIZES.len();
10const _: () = assert!(CLASSES == BytePageSize::Unset as usize);
11
12/// Default number of cached pages per page size, fewer pages are cached
13/// for larger sizes. Reads start at 4 KiB pages and adapt up to 64 KiB
14/// pages under load, 16 KiB is the default page size of writes.
15const DEFAULT_PAGES_CACHE: [usize; CLASSES] = [128, 64, 64, 32, 16, 8, 16, 2, 1];
16
17thread_local! {
18    static CACHE: Cell<Option<Box<Cache>>> = Cell::new(Some(Box::default()));
19}
20
21struct Cache {
22    limits: [usize; CLASSES],
23    pages: [Vec<StorageVec>; CLASSES],
24    metrics: Arc<PageCacheMetrics>,
25}
26
27impl Default for Cache {
28    fn default() -> Self {
29        Self {
30            limits: DEFAULT_PAGES_CACHE,
31            pages: Default::default(),
32            metrics: PageCacheMetrics::new(),
33        }
34    }
35}
36
37impl Drop for Cache {
38    fn drop(&mut self) {
39        for (class, pages) in self.pages.iter().enumerate() {
40            self.metrics.remove_pages(class, pages.len());
41        }
42    }
43}
44
45impl Cache {
46    /// Sets the limit of `class`, the pages above the limit are released.
47    fn set_limit(&mut self, class: usize, limit: usize) {
48        self.limits[class] = limit;
49        let pages = &mut self.pages[class];
50        if pages.len() > limit {
51            self.metrics.remove_pages(class, pages.len() - limit);
52            // the cache is taken out of the thread-local, the released pages
53            // are freed instead of being cached again
54            pages.truncate(limit);
55        }
56    }
57}
58
59/// Runs `f` with the cache of the current thread.
60///
61/// Returns `None` if the cache is in use or the thread-local is being
62/// destroyed.
63#[inline]
64fn with_cache<R>(f: impl FnOnce(&mut Cache) -> R) -> Option<R> {
65    CACHE
66        .try_with(|c| {
67            let mut cst = c.take()?;
68            let res = f(&mut cst);
69            c.set(Some(cst));
70            Some(res)
71        })
72        .ok()
73        .flatten()
74}
75
76/// Takes a page of size `size` from the cache.
77#[inline]
78pub(crate) fn take_page(size: BytePageSize) -> Option<StorageVec> {
79    let class = size as usize;
80    with_cache(|cst| {
81        let page = cst.pages[class].pop();
82        if page.is_some() {
83            cst.metrics.hit(class);
84        } else {
85            cst.metrics.miss(class);
86        }
87        page
88    })
89    .flatten()
90}
91
92/// Takes a page for a new buffer of page size `size` from the cache.
93///
94/// If no page of `size` is cached, a cached page of the next smaller size
95/// is taken instead. Growing buffers use [`take_page`], they need the exact
96/// size.
97#[inline]
98pub(crate) fn take_page_or_prev(size: BytePageSize) -> Option<StorageVec> {
99    let class = size as usize;
100    with_cache(|cst| {
101        if let Some(page) = cst.pages[class].pop() {
102            cst.metrics.hit(class);
103            return Some(page);
104        }
105        cst.metrics.miss(class);
106
107        let prev = size.prev() as usize;
108        if prev != class
109            && let Some(page) = cst.pages[prev].pop()
110        {
111            cst.metrics.hit(prev);
112            cst.metrics.fallback(class);
113            return Some(page);
114        }
115        None
116    })
117    .flatten()
118}
119
120/// Puts a released page of size `size` to the cache.
121///
122/// Returns `false` if the page is not cached and must be freed.
123///
124/// # Safety
125///
126/// `ptr` must point to a live `Header` of page size `size` without
127/// references.
128#[inline]
129pub(crate) unsafe fn put_page(ptr: *mut Header, size: BytePageSize) -> bool {
130    let class = size as usize;
131    with_cache(|cst| {
132        if cst.pages[class].len() < cst.limits[class] {
133            cst.pages[class].push(unsafe { StorageVec::recycle(ptr) });
134            cst.metrics.cached(class);
135            true
136        } else {
137            cst.metrics.freed(class);
138            false
139        }
140    })
141    .unwrap_or(false)
142}
143
144/// Sets the cache limit of every page size, see [`crate::set_pages_cache`].
145pub(crate) fn set_pages_cache(count: usize) {
146    with_cache(|cst| {
147        for class in 0..CLASSES {
148            cst.set_limit(class, count);
149        }
150    });
151}
152
153/// Sets the cache limit of page size `size`, see [`crate::set_page_cache_size`].
154pub(crate) fn set_page_cache_size(size: BytePageSize, count: usize) {
155    if size != BytePageSize::Unset {
156        with_cache(|cst| cst.set_limit(size as usize, count));
157    }
158}
159
160#[cfg(test)]
161/// Replaces the cache of the current thread with an empty one.
162///
163/// The old cache is taken out first, so its pages are freed rather than
164/// returned to the new cache.
165pub(crate) fn reset() {
166    CACHE.with(|c| {
167        drop(c.take());
168        c.set(Some(Box::default()));
169    });
170}
171
172#[cfg(test)]
173/// Returns the number of cached pages of size `size`.
174pub(crate) fn cached_pages(size: BytePageSize) -> usize {
175    with_cache(|cst| cst.pages[size as usize].len()).unwrap()
176}
177
178/// Page cache statistics.
179///
180/// Each thread has its own page cache, see [`crate::set_page_cache_size`].
181/// The cache reports to the metrics that are current on its thread, see
182/// [`PageCacheMetrics::current`]. Statistics are updated with relaxed atomics,
183/// a snapshot is not a consistent cut across all fields.
184///
185/// Updates are a plain load and store, not an atomic read-modify-write, so
186/// metrics must be updated by one thread at a time. If the same metrics are
187/// installed on several threads with [`PageCacheMetrics::set_current`],
188/// concurrent updates can be lost, counters undercount and the `pages` and
189/// `bytes` gauges drift permanently.
190///
191/// A page released on another thread goes to the cache of that thread, so
192/// allocations and releases of the same page can be reported to different
193/// metrics.
194#[derive(Debug, Default)]
195pub struct PageCacheMetrics {
196    classes: [Class; CLASSES],
197}
198
199#[derive(Debug, Default)]
200struct Class {
201    hits: AtomicU64,
202    misses: AtomicU64,
203    fallbacks: AtomicU64,
204    cached: AtomicU64,
205    freed: AtomicU64,
206    // pages that entered or left the cache other than by `cached` and `hits`,
207    // the pages gauge is derived, so the hot paths update one counter each
208    added: AtomicU64,
209    removed: AtomicU64,
210}
211
212/// A snapshot of [`PageCacheMetrics`].
213#[derive(Copy, Clone, Debug, PartialEq, Eq)]
214pub struct PageCacheSnapshot {
215    sizes: [PageCacheStats; CLASSES],
216}
217
218/// Page cache statistics of one page size.
219#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
220#[non_exhaustive]
221pub struct PageCacheStats {
222    /// Page size, [`BytePageSize::Unset`] for the totals of all page sizes.
223    pub size: BytePageSize,
224    /// Number of pages of this size taken from the cache, including pages
225    /// taken in place of the next larger size, see `fallbacks`.
226    pub hits: u64,
227    /// Number of requests for this page size that found no cached page of
228    /// this size.
229    ///
230    /// `misses - fallbacks` pages were allocated.
231    pub misses: u64,
232    /// Number of misses of a new buffer served with a cached page of the
233    /// next smaller size, which is counted as a hit of that size.
234    pub fallbacks: u64,
235    /// Number of released pages returned to the cache.
236    pub cached: u64,
237    /// Number of released pages freed because the cache was full.
238    pub freed: u64,
239    /// Number of pages in the cache.
240    pub pages: usize,
241    /// Memory held by the pages in the cache, in bytes.
242    pub bytes: usize,
243}
244
245impl PageCacheMetrics {
246    /// Creates new metrics.
247    pub fn new() -> Arc<Self> {
248        Arc::default()
249    }
250
251    /// Returns the metrics of the current thread's page cache.
252    ///
253    /// Each thread has its own metrics, created with the cache, unless other
254    /// metrics were installed with [`PageCacheMetrics::set_current`].
255    pub fn current() -> Arc<Self> {
256        with_cache(|cst| cst.metrics.clone()).unwrap_or_default()
257    }
258
259    /// Installs `metrics` as the metrics of the current thread's page cache.
260    ///
261    /// The pages held by the cache are moved from the previous metrics to
262    /// `metrics`. Metrics must not be current on more than one thread at a
263    /// time, updates are not atomic read-modify-writes.
264    pub fn set_current(metrics: Arc<Self>) {
265        with_cache(|cst| {
266            for (class, pages) in cst.pages.iter().enumerate() {
267                cst.metrics.remove_pages(class, pages.len());
268                metrics.add_pages(class, pages.len());
269            }
270            cst.metrics = metrics;
271        });
272    }
273
274    /// Returns a snapshot of the statistics.
275    pub fn snapshot(&self) -> PageCacheSnapshot {
276        let mut snapshot = PageCacheSnapshot::default();
277        for (stats, class) in snapshot.sizes.iter_mut().zip(&self.classes) {
278            stats.hits = class.hits.load(Relaxed);
279            stats.misses = class.misses.load(Relaxed);
280            stats.fallbacks = class.fallbacks.load(Relaxed);
281            stats.cached = class.cached.load(Relaxed);
282            stats.freed = class.freed.load(Relaxed);
283            let removed = class.removed.load(Relaxed);
284            let hits = stats.hits;
285            // fields are read one by one, a racing update can make it negative
286            stats.pages =
287                (stats.cached + class.added.load(Relaxed)).saturating_sub(hits + removed) as usize;
288            stats.bytes = stats.pages * stats.size.alloc_size();
289        }
290        snapshot
291    }
292
293    pub(crate) fn hit(&self, class: usize) {
294        inc(&self.classes[class].hits, 1);
295    }
296
297    pub(crate) fn miss(&self, class: usize) {
298        inc(&self.classes[class].misses, 1);
299    }
300
301    pub(crate) fn fallback(&self, class: usize) {
302        inc(&self.classes[class].fallbacks, 1);
303    }
304
305    pub(crate) fn cached(&self, class: usize) {
306        inc(&self.classes[class].cached, 1);
307    }
308
309    pub(crate) fn freed(&self, class: usize) {
310        inc(&self.classes[class].freed, 1);
311    }
312
313    /// Pages leave the cache without being taken, e.g. the cache is dropped.
314    pub(crate) fn remove_pages(&self, class: usize, pages: usize) {
315        if pages != 0 {
316            inc(&self.classes[class].removed, pages as u64);
317        }
318    }
319
320    pub(crate) fn add_pages(&self, class: usize, pages: usize) {
321        if pages != 0 {
322            inc(&self.classes[class].added, pages as u64);
323        }
324    }
325}
326
327// single writer, cheaper than `fetch_add`
328#[inline]
329fn inc(counter: &AtomicU64, n: u64) {
330    counter.store(counter.load(Relaxed).wrapping_add(n), Relaxed);
331}
332
333impl PageCacheSnapshot {
334    /// Returns the statistics of page size `size`, `None` for
335    /// [`BytePageSize::Unset`].
336    pub fn get(&self, size: BytePageSize) -> Option<&PageCacheStats> {
337        self.sizes.get(size as usize)
338    }
339
340    /// Returns the statistics of every page size, in increasing order of size.
341    pub fn iter(&self) -> impl Iterator<Item = &PageCacheStats> {
342        self.sizes.iter()
343    }
344
345    /// Returns the totals of all page sizes.
346    pub fn total(&self) -> PageCacheStats {
347        self.sizes.iter().fold(
348            PageCacheStats {
349                size: BytePageSize::Unset,
350                ..PageCacheStats::default()
351            },
352            |mut total, s| {
353                total.hits += s.hits;
354                total.misses += s.misses;
355                total.fallbacks += s.fallbacks;
356                total.cached += s.cached;
357                total.freed += s.freed;
358                total.pages += s.pages;
359                total.bytes += s.bytes;
360                total
361            },
362        )
363    }
364}
365
366impl Default for PageCacheSnapshot {
367    fn default() -> Self {
368        Self {
369            sizes: PAGE_SIZES.map(|size| PageCacheStats {
370                size,
371                ..PageCacheStats::default()
372            }),
373        }
374    }
375}
376
377#[cfg(test)]
378mod tests {
379    use super::*;
380    use crate::*;
381
382    /// STATEMENT: Dropped page storage is reused by the next allocation in the same size class.
383    ///
384    /// HOW:
385    /// - Reset the cache, write to a `Size8` page, drop it, and check the next `Size8` allocation has its address.
386    #[test]
387    fn cached() {
388        reset();
389
390        let mut st = StorageVec::sized(BytePageSize::Size8);
391        assert_eq!(st.page_size(), BytePageSize::Size8);
392
393        st.put_u8(b'h');
394        let addr = st.0;
395        drop(st);
396
397        let st = StorageVec::sized(BytePageSize::Size8);
398        assert_eq!(addr, st.0);
399    }
400
401    /// STATEMENT: Cache metrics distinguish hits, misses, cached pages, and full-cache frees by size class.
402    ///
403    /// HOW:
404    /// - Allocate, cache, reuse, and free a `Size8` page and check counters and byte totals.
405    /// - Cache `Size4` and non-page buffers, verify totals exclude non-pages, then reset and check old resident counts.
406    #[test]
407    fn cache_metrics() {
408        reset();
409        let metrics = PageCacheMetrics::current();
410        assert_eq!(metrics.snapshot(), PageCacheSnapshot::default());
411
412        let size = BytePageSize::Size8;
413        let st = StorageVec::sized(size);
414        let s = *metrics.snapshot().get(size).unwrap();
415        assert_eq!(
416            (s.size, s.hits, s.misses, s.cached, s.pages),
417            (size, 0, 1, 0, 0)
418        );
419
420        drop(st);
421        let s = *metrics.snapshot().get(size).unwrap();
422        assert_eq!(
423            (s.hits, s.misses, s.cached, s.freed, s.pages),
424            (0, 1, 1, 0, 1)
425        );
426        assert_eq!(s.bytes, 8 * 1024);
427
428        let st = StorageVec::sized(size);
429        let s = *metrics.snapshot().get(size).unwrap();
430        assert_eq!(
431            (s.hits, s.misses, s.cached, s.pages, s.bytes),
432            (1, 1, 1, 0, 0)
433        );
434
435        crate::set_page_cache_size(size, 0);
436        drop(st);
437        let s = *metrics.snapshot().get(size).unwrap();
438        assert_eq!((s.cached, s.freed, s.pages), (1, 1, 0));
439
440        // other page sizes are separate, non-page buffers are not counted
441        drop(StorageVec::sized(BytePageSize::Size4));
442        drop(StorageVec::with_capacity(100));
443        let snapshot = metrics.snapshot();
444        let s = *snapshot.get(BytePageSize::Size4).unwrap();
445        assert_eq!((s.misses, s.cached, s.pages), (1, 1, 1));
446        assert!(snapshot.get(BytePageSize::Unset).is_none());
447        assert_eq!(snapshot.iter().count(), 9);
448        let total = snapshot.total();
449        assert_eq!(total.size, BytePageSize::Unset);
450        assert_eq!(
451            (total.hits, total.misses, total.cached, total.freed),
452            (1, 2, 2, 1)
453        );
454        assert_eq!((total.pages, total.bytes), (1, 4 * 1024));
455
456        // a replaced cache leaves the metrics
457        reset();
458        assert_eq!(metrics.snapshot().total().pages, 0);
459        assert!(!Arc::ptr_eq(&metrics, &PageCacheMetrics::current()));
460    }
461
462    /// STATEMENT: Replacing current metrics transfers resident accounting without sharing metrics across threads.
463    ///
464    /// HOW:
465    /// - Cache a `Size16` page, install metrics, and verify resident pages transfer and only new reuse counters change.
466    /// - Compare metrics identity with another thread and confirm its metrics are distinct.
467    #[test]
468    fn cache_metrics_set_current() {
469        reset();
470        let size = BytePageSize::Size16;
471        let old = PageCacheMetrics::current();
472        drop(StorageVec::sized(size));
473        assert_eq!(old.snapshot().get(size).unwrap().pages, 1);
474
475        let metrics = PageCacheMetrics::new();
476        PageCacheMetrics::set_current(metrics.clone());
477        assert!(Arc::ptr_eq(&metrics, &PageCacheMetrics::current()));
478        assert_eq!(old.snapshot().get(size).unwrap().pages, 0);
479        assert_eq!(metrics.snapshot().get(size).unwrap().pages, 1);
480
481        drop(StorageVec::sized(size));
482        let s = *metrics.snapshot().get(size).unwrap();
483        assert_eq!((s.hits, s.misses, s.cached, s.pages), (1, 0, 1, 1));
484        assert_eq!(old.snapshot().get(size).unwrap().hits, 0);
485
486        // other threads have their own metrics
487        let other = std::thread::spawn(PageCacheMetrics::current)
488            .join()
489            .unwrap();
490        assert!(!Arc::ptr_eq(&metrics, &other));
491        reset();
492    }
493
494    /// STATEMENT: Each page size retains no more than its default cache limit.
495    ///
496    /// HOW:
497    /// - Check default limits, allocate four excess pages per size class, drop them all, and verify each cached count equals its limit.
498    #[test]
499    fn default_cache_limit_per_page_size() {
500        assert_eq!(DEFAULT_PAGES_CACHE, [128, 64, 64, 32, 16, 8, 16, 2, 1]);
501        for size in crate::size::PAGE_SIZES {
502            reset();
503
504            let limit = DEFAULT_PAGES_CACHE[size as usize];
505            let pages: Vec<_> = (0..limit + 4).map(|_| StorageVec::sized(size)).collect();
506            drop(pages);
507            assert_eq!(cached_pages(size), limit, "{size:?}");
508        }
509        reset();
510    }
511
512    /// STATEMENT: Global cache limits control retained pages and ignore reentrant updates.
513    ///
514    /// HOW:
515    /// - Set limits of one and zero, drop pages, and check cached counts.
516    /// - Take `CACHE`, drop a page and attempt a limit update, then verify no page was cached and limits stayed unchanged.
517    #[test]
518    #[allow(deprecated)]
519    fn pages_cache_size() {
520        reset();
521
522        crate::set_pages_cache(1);
523        drop((
524            StorageVec::sized(BytePageSize::Size4),
525            StorageVec::sized(BytePageSize::Size4),
526        ));
527        assert_eq!(cached_pages(BytePageSize::Size4), 1);
528
529        crate::set_pages_cache(0);
530        drop(StorageVec::sized(BytePageSize::Size8));
531        assert_eq!(cached_pages(BytePageSize::Size8), 0);
532
533        // the cache is in use, the page is freed
534        crate::set_pages_cache(16);
535        let st = StorageVec::sized(BytePageSize::Size16);
536        let cache = CACHE.with(Cell::take);
537        drop(st);
538        CACHE.with(|c| c.set(cache));
539        assert_eq!(cached_pages(BytePageSize::Size16), 0);
540
541        // the setting is ignored while the cache is in use
542        let cache = CACHE.with(Cell::take);
543        crate::set_pages_cache(3);
544        CACHE.with(|c| c.set(cache));
545        assert_eq!(cache_limits(), [16; CLASSES]);
546    }
547
548    fn cache_limits() -> [usize; CLASSES] {
549        CACHE.with(|c| {
550            let cst = c.take().unwrap();
551            let limits = cst.limits;
552            c.set(Some(cst));
553            limits
554        })
555    }
556
557    /// STATEMENT: Per-size limits leave other classes unchanged and ignore invalid or reentrant updates.
558    ///
559    /// HOW:
560    /// - Limit `Size64` to one, configure `Unset`, and check limits and retained `Size64` and `Size48` pages.
561    /// - Verify an update while `CACHE` is taken is ignored, then check a global limit applies to all classes.
562    #[test]
563    #[allow(deprecated)]
564    fn page_cache_size() {
565        reset();
566
567        crate::set_page_cache_size(BytePageSize::Size64, 1);
568        crate::set_page_cache_size(BytePageSize::Unset, 5);
569        let mut expected = DEFAULT_PAGES_CACHE;
570        expected[BytePageSize::Size64 as usize] = 1;
571        assert_eq!(cache_limits(), expected);
572
573        drop((
574            StorageVec::sized(BytePageSize::Size64),
575            StorageVec::sized(BytePageSize::Size64),
576            StorageVec::sized(BytePageSize::Size48),
577            StorageVec::sized(BytePageSize::Size48),
578        ));
579        assert_eq!(cached_pages(BytePageSize::Size64), 1);
580        assert_eq!(cached_pages(BytePageSize::Size48), 2);
581
582        // the setting is ignored while the cache is in use
583        let cache = CACHE.with(Cell::take);
584        crate::set_page_cache_size(BytePageSize::Size48, 3);
585        CACHE.with(|c| c.set(cache));
586        assert_eq!(cache_limits(), expected);
587
588        crate::set_pages_cache(2);
589        assert_eq!(cache_limits(), [2; CLASSES]);
590        reset();
591    }
592
593    /// STATEMENT: Lowering cache limits releases excess pages without counting them as full-cache frees.
594    ///
595    /// HOW:
596    /// - Cache four `Size4` and four `Size64` pages, lower per-class and global limits, and check resident counts and counters.
597    /// - Raise the remaining class's limit and verify the retained page stays cached.
598    #[test]
599    #[allow(deprecated)]
600    fn lowered_limit_releases_pages() {
601        reset();
602        let metrics = PageCacheMetrics::current();
603        let pages = |size| metrics.snapshot().get(size).unwrap().pages;
604
605        let (s4, s64) = (BytePageSize::Size4, BytePageSize::Size64);
606        drop([0; 4].map(|_| StorageVec::sized(s64)));
607        drop([0; 4].map(|_| StorageVec::sized(s4)));
608        assert_eq!((cached_pages(s64), pages(s64)), (4, 4));
609
610        crate::set_page_cache_size(s64, 1);
611        assert_eq!((cached_pages(s64), pages(s64)), (1, 1));
612        assert_eq!((cached_pages(s4), pages(s4)), (4, 4));
613        crate::set_page_cache_size(s64, 0);
614        assert_eq!((cached_pages(s64), pages(s64)), (0, 0));
615
616        // released pages are not counted as freed by a full cache
617        crate::set_pages_cache(1);
618        assert_eq!((cached_pages(s4), pages(s4)), (1, 1));
619        let total = metrics.snapshot().total();
620        assert_eq!((total.cached, total.freed, total.pages), (8, 0, 1));
621
622        // a raised limit keeps the cached pages
623        crate::set_page_cache_size(s4, 8);
624        assert_eq!((cached_pages(s4), pages(s4)), (1, 1));
625        reset();
626    }
627
628    /// STATEMENT: Page-sized `BytesMut` storage is cached only after its last shared view drops.
629    ///
630    /// HOW:
631    /// - Split a `Size8` buffer, drop both handles in turn, and check cache counts and pointer reuse.
632    /// - Verify `Unset` selects `Size64` while ordinary-capacity and copied buffers remain uncached.
633    #[test]
634    fn bytes_mut_with_page_size() {
635        reset();
636
637        let mut buf = BytesMut::with_page_size(BytePageSize::Size8);
638        assert_eq!(buf.page_size(), BytePageSize::Size8);
639        assert_eq!(buf.capacity(), BytePageSize::Size8.capacity());
640        let ptr = buf.as_ptr();
641
642        // split off data keeps the page until the last reference is dropped
643        buf.extend_from_slice(&[1; 1000]);
644        let head = buf.split_to(500);
645        drop(buf);
646        assert_eq!(cached_pages(BytePageSize::Size8), 0);
647        drop(head);
648        assert_eq!(cached_pages(BytePageSize::Size8), 1);
649
650        let buf = BytesMut::with_page_size(BytePageSize::Size8);
651        assert_eq!(buf.as_ptr(), ptr);
652        assert_eq!(buf.capacity(), BytePageSize::Size8.capacity());
653        assert_eq!(cached_pages(BytePageSize::Size8), 0);
654
655        // `Unset` creates a `Size64` page
656        let buf = BytesMut::with_page_size(BytePageSize::Unset);
657        assert_eq!(buf.page_size(), BytePageSize::Size64);
658        assert_eq!(buf.capacity(), BytePageSize::Size64.capacity());
659        drop(buf);
660        assert_eq!(cached_pages(BytePageSize::Size64), 1);
661
662        // buffers without a page size are not cached
663        let buf = BytesMut::with_capacity(BytePageSize::Size64.capacity() + 1);
664        assert_eq!(buf.page_size(), BytePageSize::Unset);
665        assert_eq!(BytesMut::new().page_size(), BytePageSize::Unset);
666        assert_eq!(BytesMut::with_capacity(64).page_size(), BytePageSize::Unset);
667        assert_eq!(
668            BytesMut::copy_from_slice(b"hello").page_size(),
669            BytePageSize::Unset
670        );
671        drop(buf);
672        for size in crate::size::PAGE_SIZES {
673            let expected = usize::from(size == BytePageSize::Size64);
674            assert_eq!(cached_pages(size), expected);
675        }
676    }
677
678    /// STATEMENT: A new page request with no cached page of its size takes a cached page one size smaller, and the metrics record the fallback.
679    ///
680    /// HOW:
681    /// - Cache a `Size8` page, request `Size16`, and check the same `Size8` page is returned, counted as a `Size16` miss and fallback and a `Size8` hit.
682    /// - Request `Size16` again with an empty cache and check a `Size16` page is allocated, counted as a miss only.
683    /// - Check the totals include the fallback.
684    #[test]
685    fn new_page_falls_back_to_smaller_size() {
686        reset();
687        let metrics = PageCacheMetrics::current();
688        let (s8, s16) = (BytePageSize::Size8, BytePageSize::Size16);
689
690        let st = StorageVec::sized(s8);
691        let addr = st.0;
692        drop(st);
693
694        let st = StorageVec::sized(s16);
695        assert_eq!(st.0, addr);
696        assert_eq!(st.page_size(), s8);
697        assert_eq!(cached_pages(s8), 0);
698        let snapshot = metrics.snapshot();
699        let s = *snapshot.get(s16).unwrap();
700        assert_eq!((s.hits, s.misses, s.fallbacks, s.pages), (0, 1, 1, 0));
701        let s = *snapshot.get(s8).unwrap();
702        assert_eq!(
703            (s.hits, s.misses, s.fallbacks, s.cached, s.pages),
704            (1, 1, 0, 1, 0)
705        );
706
707        let st2 = StorageVec::sized(s16);
708        assert_eq!(st2.page_size(), s16);
709        let snapshot = metrics.snapshot();
710        let s = *snapshot.get(s16).unwrap();
711        assert_eq!((s.hits, s.misses, s.fallbacks), (0, 2, 1));
712        let total = snapshot.total();
713        assert_eq!((total.hits, total.misses, total.fallbacks), (1, 3, 1));
714        drop((st, st2));
715        reset();
716    }
717
718    /// STATEMENT: The fallback goes exactly one page size down and only when the requested size has no cached page.
719    ///
720    /// HOW:
721    /// - Cache a `Size4` page and check a `Size16` request allocates `Size16` and leaves the `Size4` page cached.
722    /// - Cache `Size8` and `Size16` pages and check a `Size16` request takes the `Size16` page and leaves the `Size8` page cached.
723    #[test]
724    fn new_page_falls_back_one_size_only() {
725        reset();
726        let (s4, s8, s16) = (
727            BytePageSize::Size4,
728            BytePageSize::Size8,
729            BytePageSize::Size16,
730        );
731
732        drop(StorageVec::sized_exact(s4));
733        let st = StorageVec::sized(s16);
734        assert_eq!(st.page_size(), s16);
735        assert_eq!(cached_pages(s4), 1);
736
737        let p8 = StorageVec::sized_exact(s8);
738        drop(st);
739        drop(p8);
740        let st = StorageVec::sized(s16);
741        assert_eq!(st.page_size(), s16);
742        assert_eq!((cached_pages(s8), cached_pages(s16)), (1, 0));
743        drop(st);
744        reset();
745    }
746
747    /// STATEMENT: New `BytesMut` buffers and new `BytePages` pages take a cached page one size smaller, `Unset` falls back from `Size64` to `Size48`.
748    ///
749    /// HOW:
750    /// - Cache a `Size8` page and check `BytesMut::with_page_size(Size16)` returns it with `Size8` capacity.
751    /// - Cache a `Size48` page and check `BytesMut::with_page_size(Unset)` returns it.
752    /// - With the `tracker` feature, check `BytesMut::with_page_size_in` falls back too.
753    /// - Cache a `Size8` page, write to a `Size16` `BytePages`, and check the page left the cache as a `Size16` fallback.
754    #[test]
755    fn new_buffers_fall_back_to_smaller_page() {
756        reset();
757        let metrics = PageCacheMetrics::current();
758        let (s8, s16) = (BytePageSize::Size8, BytePageSize::Size16);
759
760        drop(StorageVec::sized_exact(s8));
761        let buf = BytesMut::with_page_size(s16);
762        assert_eq!(buf.page_size(), s8);
763        assert_eq!(buf.capacity(), s8.capacity());
764        drop(buf);
765
766        drop(StorageVec::sized_exact(BytePageSize::Size48));
767        let buf = BytesMut::with_page_size(BytePageSize::Unset);
768        assert_eq!(buf.page_size(), BytePageSize::Size48);
769        drop(buf);
770
771        #[cfg(feature = "tracker")]
772        {
773            let tracker = BytesTracker::new();
774            let buf = BytesMut::with_page_size_in(s16, &tracker);
775            assert_eq!(buf.page_size(), s8);
776            assert_eq!(tracker.snapshot().live_bytes, s8.alloc_size());
777        }
778
779        assert_eq!(cached_pages(s8), 1);
780        let mut pages = BytePages::new(s16);
781        pages.put_slice(b"hello");
782        assert_eq!(cached_pages(s8), 0);
783        let expected = if cfg!(feature = "tracker") { 3 } else { 2 };
784        assert_eq!(metrics.snapshot().get(s16).unwrap().fallbacks, expected);
785        drop(pages);
786        reset();
787    }
788}
ntex_bytes/cache.rs

ntex_bytes/
cache.rs
