ntex_bytes/
cache.rs

1//! Per-thread page cache and its statistics.
2use std::cell::Cell;
3use std::sync::{Arc, atomic::AtomicU64, atomic::Ordering::Relaxed};
4
5use crate::stvec::{Header, StorageVec};
6use crate::{BytePageSize, size::PAGE_SIZES};
7
8/// Number of page sizes, `BytePageSize::Unset` excluded.
9const CLASSES: usize = PAGE_SIZES.len();
10const _: () = assert!(CLASSES == BytePageSize::Unset as usize);
11
12/// Default number of cached pages per page size, fewer pages are cached
13/// for larger sizes. Reads start at 4 KiB pages and adapt up to 64 KiB
14/// pages under load, 16 KiB is the default page size of writes.
15const DEFAULT_PAGES_CACHE: [usize; CLASSES] = [128, 64, 64, 32, 16, 8, 16, 2, 1];
16
17thread_local! {
18    static CACHE: Cell<Option<Box<Cache>>> = Cell::new(Some(Box::default()));
19}
20
21struct Cache {
22    limits: [usize; CLASSES],
23    pages: [Vec<StorageVec>; CLASSES],
24    metrics: Arc<PageCacheMetrics>,
25}
26
27impl Default for Cache {
28    fn default() -> Self {
29        Self {
30            limits: DEFAULT_PAGES_CACHE,
31            pages: Default::default(),
32            metrics: PageCacheMetrics::new(),
33        }
34    }
35}
36
37impl Drop for Cache {
38    fn drop(&mut self) {
39        for (class, pages) in self.pages.iter().enumerate() {
40            self.metrics.remove_pages(class, pages.len());
41        }
42    }
43}
44
45impl Cache {
46    /// Sets the limit of `class`, the pages above the limit are released.
47    fn set_limit(&mut self, class: usize, limit: usize) {
48        self.limits[class] = limit;
49        let pages = &mut self.pages[class];
50        if pages.len() > limit {
51            self.metrics.remove_pages(class, pages.len() - limit);
52            // the cache is taken out of the thread-local, the released pages
53            // are freed instead of being cached again
54            pages.truncate(limit);
55        }
56    }
57}
58
59/// Runs `f` with the cache of the current thread.
60///
61/// Returns `None` if the cache is in use or the thread-local is being
62/// destroyed.
63#[inline]
64fn with_cache<R>(f: impl FnOnce(&mut Cache) -> R) -> Option<R> {
65    CACHE
66        .try_with(|c| {
67            let mut cst = c.take()?;
68            let res = f(&mut cst);
69            c.set(Some(cst));
70            Some(res)
71        })
72        .ok()
73        .flatten()
74}
75
76/// Takes a page of size `size` from the cache.
77#[inline]
78pub(crate) fn take_page(size: BytePageSize) -> Option<StorageVec> {
79    let class = size as usize;
80    with_cache(|cst| {
81        let page = cst.pages[class].pop();
82        if page.is_some() {
83            cst.metrics.hit(class);
84        } else {
85            cst.metrics.miss(class);
86        }
87        page
88    })
89    .flatten()
90}
91
92/// Takes a page for a new buffer of page size `size` from the cache.
93///
94/// If no page of `size` is cached, a cached page of the next smaller size
95/// is taken instead. Growing buffers use [`take_page`], they need the exact
96/// size.
97#[inline]
98pub(crate) fn take_page_or_prev(size: BytePageSize) -> Option<StorageVec> {
99    let class = size as usize;
100    with_cache(|cst| {
101        if let Some(page) = cst.pages[class].pop() {
102            cst.metrics.hit(class);
103            return Some(page);
104        }
105        cst.metrics.miss(class);
106
107        let prev = size.prev() as usize;
108        if prev != class
109            && let Some(page) = cst.pages[prev].pop()
110        {
111            cst.metrics.hit(prev);
112            cst.metrics.fallback(class);
113            return Some(page);
114        }
115        None
116    })
117    .flatten()
118}
119
120/// Puts a released page of size `size` to the cache.
121///
122/// Returns `false` if the page is not cached and must be freed.
123///
124/// # Safety
125///
126/// `ptr` must point to a live `Header` of page size `size` without
127/// references.
128#[inline]
129pub(crate) unsafe fn put_page(ptr: *mut Header, size: BytePageSize) -> bool {
130    let class = size as usize;
131    with_cache(|cst| {
132        if cst.pages[class].len() < cst.limits[class] {
133            // SAFETY: The caller transfers a live page with no references; the cache takes
134            // exclusive ownership.
135            cst.pages[class].push(unsafe { StorageVec::recycle(ptr) });
136            cst.metrics.cached(class);
137            true
138        } else {
139            cst.metrics.freed(class);
140            false
141        }
142    })
143    .unwrap_or(false)
144}
145
146/// Sets the cache limit of every page size, see [`crate::set_pages_cache`].
147pub(crate) fn set_pages_cache(count: usize) {
148    with_cache(|cst| {
149        for class in 0..CLASSES {
150            cst.set_limit(class, count);
151        }
152    });
153}
154
155/// Sets the cache limit of page size `size`, see [`crate::set_page_cache_size`].
156pub(crate) fn set_page_cache_size(size: BytePageSize, count: usize) {
157    if size != BytePageSize::Unset {
158        with_cache(|cst| cst.set_limit(size as usize, count));
159    }
160}
161
162#[cfg(test)]
163/// Replaces the cache of the current thread with an empty one.
164///
165/// The old cache is taken out first, so its pages are freed rather than
166/// returned to the new cache.
167pub(crate) fn reset() {
168    CACHE.with(|c| {
169        drop(c.take());
170        c.set(Some(Box::default()));
171    });
172}
173
174#[cfg(test)]
175/// Returns the number of cached pages of size `size`.
176pub(crate) fn cached_pages(size: BytePageSize) -> usize {
177    with_cache(|cst| cst.pages[size as usize].len()).unwrap()
178}
179
180/// Page cache statistics.
181///
182/// Each thread has its own page cache, see [`crate::set_page_cache_size`].
183/// The cache reports to the metrics that are current on its thread, see
184/// [`PageCacheMetrics::current`]. Statistics are updated with relaxed atomics,
185/// a snapshot is not a consistent cut across all fields.
186///
187/// Updates are a plain load and store, not an atomic read-modify-write, so
188/// metrics must be updated by one thread at a time. If the same metrics are
189/// installed on several threads with [`PageCacheMetrics::set_current`],
190/// concurrent updates can be lost, counters undercount and the `pages` and
191/// `bytes` gauges drift permanently.
192///
193/// A page released on another thread goes to the cache of that thread, so
194/// allocations and releases of the same page can be reported to different
195/// metrics.
196#[derive(Debug, Default)]
197pub struct PageCacheMetrics {
198    classes: [Class; CLASSES],
199}
200
201#[derive(Debug, Default)]
202struct Class {
203    hits: AtomicU64,
204    misses: AtomicU64,
205    fallbacks: AtomicU64,
206    cached: AtomicU64,
207    freed: AtomicU64,
208    // pages that entered or left the cache other than by `cached` and `hits`,
209    // the pages gauge is derived, so the hot paths update one counter each
210    added: AtomicU64,
211    removed: AtomicU64,
212}
213
214/// A snapshot of [`PageCacheMetrics`].
215#[derive(Copy, Clone, Debug, PartialEq, Eq)]
216pub struct PageCacheSnapshot {
217    sizes: [PageCacheStats; CLASSES],
218}
219
220/// Page cache statistics of one page size.
221#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
222#[non_exhaustive]
223pub struct PageCacheStats {
224    /// Page size, [`BytePageSize::Unset`] for the totals of all page sizes.
225    pub size: BytePageSize,
226    /// Number of pages of this size taken from the cache, including pages
227    /// taken in place of the next larger size, see `fallbacks`.
228    pub hits: u64,
229    /// Number of requests for this page size that found no cached page of
230    /// this size.
231    ///
232    /// `misses - fallbacks` pages were allocated.
233    pub misses: u64,
234    /// Number of misses of a new buffer served with a cached page of the
235    /// next smaller size, which is counted as a hit of that size.
236    pub fallbacks: u64,
237    /// Number of released pages returned to the cache.
238    pub cached: u64,
239    /// Number of released pages freed because the cache was full.
240    pub freed: u64,
241    /// Number of pages in the cache.
242    pub pages: usize,
243    /// Memory held by the pages in the cache, in bytes.
244    pub bytes: usize,
245}
246
247impl PageCacheMetrics {
248    /// Creates new metrics.
249    pub fn new() -> Arc<Self> {
250        Arc::default()
251    }
252
253    /// Returns the metrics of the current thread's page cache.
254    ///
255    /// Each thread has its own metrics, created with the cache, unless other
256    /// metrics were installed with [`PageCacheMetrics::set_current`].
257    pub fn current() -> Arc<Self> {
258        with_cache(|cst| cst.metrics.clone()).unwrap_or_default()
259    }
260
261    /// Installs `metrics` as the metrics of the current thread's page cache.
262    ///
263    /// The pages held by the cache are moved from the previous metrics to
264    /// `metrics`. Metrics must not be current on more than one thread at a
265    /// time, updates are not atomic read-modify-writes.
266    pub fn set_current(metrics: Arc<Self>) {
267        with_cache(|cst| {
268            for (class, pages) in cst.pages.iter().enumerate() {
269                cst.metrics.remove_pages(class, pages.len());
270                metrics.add_pages(class, pages.len());
271            }
272            cst.metrics = metrics;
273        });
274    }
275
276    /// Returns a snapshot of the statistics.
277    pub fn snapshot(&self) -> PageCacheSnapshot {
278        let mut snapshot = PageCacheSnapshot::default();
279        for (stats, class) in snapshot.sizes.iter_mut().zip(&self.classes) {
280            stats.hits = class.hits.load(Relaxed);
281            stats.misses = class.misses.load(Relaxed);
282            stats.fallbacks = class.fallbacks.load(Relaxed);
283            stats.cached = class.cached.load(Relaxed);
284            stats.freed = class.freed.load(Relaxed);
285            let removed = class.removed.load(Relaxed);
286            let hits = stats.hits;
287            // fields are read one by one, a racing update can make it negative
288            stats.pages =
289                (stats.cached + class.added.load(Relaxed)).saturating_sub(hits + removed) as usize;
290            stats.bytes = stats.pages * stats.size.alloc_size();
291        }
292        snapshot
293    }
294
295    pub(crate) fn hit(&self, class: usize) {
296        inc(&self.classes[class].hits, 1);
297    }
298
299    pub(crate) fn miss(&self, class: usize) {
300        inc(&self.classes[class].misses, 1);
301    }
302
303    pub(crate) fn fallback(&self, class: usize) {
304        inc(&self.classes[class].fallbacks, 1);
305    }
306
307    pub(crate) fn cached(&self, class: usize) {
308        inc(&self.classes[class].cached, 1);
309    }
310
311    pub(crate) fn freed(&self, class: usize) {
312        inc(&self.classes[class].freed, 1);
313    }
314
315    /// Pages leave the cache without being taken, e.g. the cache is dropped.
316    pub(crate) fn remove_pages(&self, class: usize, pages: usize) {
317        if pages != 0 {
318            inc(&self.classes[class].removed, pages as u64);
319        }
320    }
321
322    pub(crate) fn add_pages(&self, class: usize, pages: usize) {
323        if pages != 0 {
324            inc(&self.classes[class].added, pages as u64);
325        }
326    }
327}
328
329// single writer, cheaper than `fetch_add`
330#[inline]
331fn inc(counter: &AtomicU64, n: u64) {
332    counter.store(counter.load(Relaxed).wrapping_add(n), Relaxed);
333}
334
335impl PageCacheSnapshot {
336    /// Returns the statistics of page size `size`, `None` for
337    /// [`BytePageSize::Unset`].
338    pub fn get(&self, size: BytePageSize) -> Option<&PageCacheStats> {
339        self.sizes.get(size as usize)
340    }
341
342    /// Returns the statistics of every page size, in increasing order of size.
343    pub fn iter(&self) -> impl Iterator<Item = &PageCacheStats> {
344        self.sizes.iter()
345    }
346
347    /// Returns the totals of all page sizes.
348    pub fn total(&self) -> PageCacheStats {
349        self.sizes.iter().fold(
350            PageCacheStats {
351                size: BytePageSize::Unset,
352                ..PageCacheStats::default()
353            },
354            |mut total, s| {
355                total.hits += s.hits;
356                total.misses += s.misses;
357                total.fallbacks += s.fallbacks;
358                total.cached += s.cached;
359                total.freed += s.freed;
360                total.pages += s.pages;
361                total.bytes += s.bytes;
362                total
363            },
364        )
365    }
366}
367
368impl Default for PageCacheSnapshot {
369    fn default() -> Self {
370        Self {
371            sizes: PAGE_SIZES.map(|size| PageCacheStats {
372                size,
373                ..PageCacheStats::default()
374            }),
375        }
376    }
377}
378
379#[cfg(test)]
380mod tests {
381    use super::*;
382    use crate::*;
383
384    /// STATEMENT: Dropped page storage is reused by the next allocation in the same size class.
385    ///
386    /// HOW:
387    /// - Reset the cache, write to a `Size8` page, drop it, and check the next `Size8` allocation has its address.
388    #[test]
389    fn cached() {
390        reset();
391
392        let mut st = StorageVec::sized(BytePageSize::Size8);
393        assert_eq!(st.page_size(), BytePageSize::Size8);
394
395        st.put_u8(b'h');
396        let addr = st.0;
397        drop(st);
398
399        let st = StorageVec::sized(BytePageSize::Size8);
400        assert_eq!(addr, st.0);
401    }
402
403    /// STATEMENT: Cache metrics distinguish hits, misses, cached pages, and full-cache frees by size class.
404    ///
405    /// HOW:
406    /// - Allocate, cache, reuse, and free a `Size8` page and check counters and byte totals.
407    /// - Cache `Size4` and non-page buffers, verify totals exclude non-pages, then reset and check old resident counts.
408    #[test]
409    fn cache_metrics() {
410        reset();
411        let metrics = PageCacheMetrics::current();
412        assert_eq!(metrics.snapshot(), PageCacheSnapshot::default());
413
414        let size = BytePageSize::Size8;
415        let st = StorageVec::sized(size);
416        let s = *metrics.snapshot().get(size).unwrap();
417        assert_eq!(
418            (s.size, s.hits, s.misses, s.cached, s.pages),
419            (size, 0, 1, 0, 0)
420        );
421
422        drop(st);
423        let s = *metrics.snapshot().get(size).unwrap();
424        assert_eq!(
425            (s.hits, s.misses, s.cached, s.freed, s.pages),
426            (0, 1, 1, 0, 1)
427        );
428        assert_eq!(s.bytes, 8 * 1024);
429
430        let st = StorageVec::sized(size);
431        let s = *metrics.snapshot().get(size).unwrap();
432        assert_eq!(
433            (s.hits, s.misses, s.cached, s.pages, s.bytes),
434            (1, 1, 1, 0, 0)
435        );
436
437        crate::set_page_cache_size(size, 0);
438        drop(st);
439        let s = *metrics.snapshot().get(size).unwrap();
440        assert_eq!((s.cached, s.freed, s.pages), (1, 1, 0));
441
442        // other page sizes are separate, non-page buffers are not counted
443        drop(StorageVec::sized(BytePageSize::Size4));
444        drop(StorageVec::with_capacity(100));
445        let snapshot = metrics.snapshot();
446        let s = *snapshot.get(BytePageSize::Size4).unwrap();
447        assert_eq!((s.misses, s.cached, s.pages), (1, 1, 1));
448        assert!(snapshot.get(BytePageSize::Unset).is_none());
449        assert_eq!(snapshot.iter().count(), 9);
450        let total = snapshot.total();
451        assert_eq!(total.size, BytePageSize::Unset);
452        assert_eq!(
453            (total.hits, total.misses, total.cached, total.freed),
454            (1, 2, 2, 1)
455        );
456        assert_eq!((total.pages, total.bytes), (1, 4 * 1024));
457
458        // a replaced cache leaves the metrics
459        reset();
460        assert_eq!(metrics.snapshot().total().pages, 0);
461        assert!(!Arc::ptr_eq(&metrics, &PageCacheMetrics::current()));
462    }
463
464    /// STATEMENT: Replacing current metrics transfers resident accounting without sharing metrics across threads.
465    ///
466    /// HOW:
467    /// - Cache a `Size16` page, install metrics, and verify resident pages transfer and only new reuse counters change.
468    /// - Compare metrics identity with another thread and confirm its metrics are distinct.
469    #[test]
470    fn cache_metrics_set_current() {
471        reset();
472        let size = BytePageSize::Size16;
473        let old = PageCacheMetrics::current();
474        drop(StorageVec::sized(size));
475        assert_eq!(old.snapshot().get(size).unwrap().pages, 1);
476
477        let metrics = PageCacheMetrics::new();
478        PageCacheMetrics::set_current(metrics.clone());
479        assert!(Arc::ptr_eq(&metrics, &PageCacheMetrics::current()));
480        assert_eq!(old.snapshot().get(size).unwrap().pages, 0);
481        assert_eq!(metrics.snapshot().get(size).unwrap().pages, 1);
482
483        drop(StorageVec::sized(size));
484        let s = *metrics.snapshot().get(size).unwrap();
485        assert_eq!((s.hits, s.misses, s.cached, s.pages), (1, 0, 1, 1));
486        assert_eq!(old.snapshot().get(size).unwrap().hits, 0);
487
488        // other threads have their own metrics
489        let other = std::thread::spawn(PageCacheMetrics::current)
490            .join()
491            .unwrap();
492        assert!(!Arc::ptr_eq(&metrics, &other));
493        reset();
494    }
495
496    /// STATEMENT: Each page size retains no more than its default cache limit.
497    ///
498    /// HOW:
499    /// - Check default limits, allocate four excess pages per size class, drop them all, and verify each cached count equals its limit.
500    #[test]
501    fn default_cache_limit_per_page_size() {
502        assert_eq!(DEFAULT_PAGES_CACHE, [128, 64, 64, 32, 16, 8, 16, 2, 1]);
503        for size in crate::size::PAGE_SIZES {
504            reset();
505
506            let limit = DEFAULT_PAGES_CACHE[size as usize];
507            let pages: Vec<_> = (0..limit + 4).map(|_| StorageVec::sized(size)).collect();
508            drop(pages);
509            assert_eq!(cached_pages(size), limit, "{size:?}");
510        }
511        reset();
512    }
513
514    /// STATEMENT: Global cache limits control retained pages and ignore reentrant updates.
515    ///
516    /// HOW:
517    /// - Set limits of one and zero, drop pages, and check cached counts.
518    /// - Take `CACHE`, drop a page and attempt a limit update, then verify no page was cached and limits stayed unchanged.
519    #[test]
520    #[allow(deprecated)]
521    fn pages_cache_size() {
522        reset();
523
524        crate::set_pages_cache(1);
525        drop((
526            StorageVec::sized(BytePageSize::Size4),
527            StorageVec::sized(BytePageSize::Size4),
528        ));
529        assert_eq!(cached_pages(BytePageSize::Size4), 1);
530
531        crate::set_pages_cache(0);
532        drop(StorageVec::sized(BytePageSize::Size8));
533        assert_eq!(cached_pages(BytePageSize::Size8), 0);
534
535        // the cache is in use, the page is freed
536        crate::set_pages_cache(16);
537        let st = StorageVec::sized(BytePageSize::Size16);
538        let cache = CACHE.with(Cell::take);
539        drop(st);
540        CACHE.with(|c| c.set(cache));
541        assert_eq!(cached_pages(BytePageSize::Size16), 0);
542
543        // the setting is ignored while the cache is in use
544        let cache = CACHE.with(Cell::take);
545        crate::set_pages_cache(3);
546        CACHE.with(|c| c.set(cache));
547        assert_eq!(cache_limits(), [16; CLASSES]);
548    }
549
550    fn cache_limits() -> [usize; CLASSES] {
551        CACHE.with(|c| {
552            let cst = c.take().unwrap();
553            let limits = cst.limits;
554            c.set(Some(cst));
555            limits
556        })
557    }
558
559    /// STATEMENT: Per-size limits leave other classes unchanged and ignore invalid or reentrant updates.
560    ///
561    /// HOW:
562    /// - Limit `Size64` to one, configure `Unset`, and check limits and retained `Size64` and `Size48` pages.
563    /// - Verify an update while `CACHE` is taken is ignored, then check a global limit applies to all classes.
564    #[test]
565    #[allow(deprecated)]
566    fn page_cache_size() {
567        reset();
568
569        crate::set_page_cache_size(BytePageSize::Size64, 1);
570        crate::set_page_cache_size(BytePageSize::Unset, 5);
571        let mut expected = DEFAULT_PAGES_CACHE;
572        expected[BytePageSize::Size64 as usize] = 1;
573        assert_eq!(cache_limits(), expected);
574
575        drop((
576            StorageVec::sized(BytePageSize::Size64),
577            StorageVec::sized(BytePageSize::Size64),
578            StorageVec::sized(BytePageSize::Size48),
579            StorageVec::sized(BytePageSize::Size48),
580        ));
581        assert_eq!(cached_pages(BytePageSize::Size64), 1);
582        assert_eq!(cached_pages(BytePageSize::Size48), 2);
583
584        // the setting is ignored while the cache is in use
585        let cache = CACHE.with(Cell::take);
586        crate::set_page_cache_size(BytePageSize::Size48, 3);
587        CACHE.with(|c| c.set(cache));
588        assert_eq!(cache_limits(), expected);
589
590        crate::set_pages_cache(2);
591        assert_eq!(cache_limits(), [2; CLASSES]);
592        reset();
593    }
594
595    /// STATEMENT: Lowering cache limits releases excess pages without counting them as full-cache frees.
596    ///
597    /// HOW:
598    /// - Cache four `Size4` and four `Size64` pages, lower per-class and global limits, and check resident counts and counters.
599    /// - Raise the remaining class's limit and verify the retained page stays cached.
600    #[test]
601    #[allow(deprecated)]
602    fn lowered_limit_releases_pages() {
603        reset();
604        let metrics = PageCacheMetrics::current();
605        let pages = |size| metrics.snapshot().get(size).unwrap().pages;
606
607        let (s4, s64) = (BytePageSize::Size4, BytePageSize::Size64);
608        drop([0; 4].map(|_| StorageVec::sized(s64)));
609        drop([0; 4].map(|_| StorageVec::sized(s4)));
610        assert_eq!((cached_pages(s64), pages(s64)), (4, 4));
611
612        crate::set_page_cache_size(s64, 1);
613        assert_eq!((cached_pages(s64), pages(s64)), (1, 1));
614        assert_eq!((cached_pages(s4), pages(s4)), (4, 4));
615        crate::set_page_cache_size(s64, 0);
616        assert_eq!((cached_pages(s64), pages(s64)), (0, 0));
617
618        // released pages are not counted as freed by a full cache
619        crate::set_pages_cache(1);
620        assert_eq!((cached_pages(s4), pages(s4)), (1, 1));
621        let total = metrics.snapshot().total();
622        assert_eq!((total.cached, total.freed, total.pages), (8, 0, 1));
623
624        // a raised limit keeps the cached pages
625        crate::set_page_cache_size(s4, 8);
626        assert_eq!((cached_pages(s4), pages(s4)), (1, 1));
627        reset();
628    }
629
630    /// STATEMENT: Page-sized `BytesMut` storage is cached only after its last shared view drops.
631    ///
632    /// HOW:
633    /// - Split a `Size8` buffer, drop both handles in turn, and check cache counts and pointer reuse.
634    /// - Verify `Unset` selects `Size64` while ordinary-capacity and copied buffers remain uncached.
635    #[test]
636    fn bytes_mut_with_page_size() {
637        reset();
638
639        let mut buf = BytesMut::with_page_size(BytePageSize::Size8);
640        assert_eq!(buf.page_size(), BytePageSize::Size8);
641        assert_eq!(buf.capacity(), BytePageSize::Size8.capacity());
642        let ptr = buf.as_ptr();
643
644        // split off data keeps the page until the last reference is dropped
645        buf.extend_from_slice(&[1; 1000]);
646        let head = buf.split_to(500);
647        drop(buf);
648        assert_eq!(cached_pages(BytePageSize::Size8), 0);
649        drop(head);
650        assert_eq!(cached_pages(BytePageSize::Size8), 1);
651
652        let buf = BytesMut::with_page_size(BytePageSize::Size8);
653        assert_eq!(buf.as_ptr(), ptr);
654        assert_eq!(buf.capacity(), BytePageSize::Size8.capacity());
655        assert_eq!(cached_pages(BytePageSize::Size8), 0);
656
657        // `Unset` creates a `Size64` page
658        let buf = BytesMut::with_page_size(BytePageSize::Unset);
659        assert_eq!(buf.page_size(), BytePageSize::Size64);
660        assert_eq!(buf.capacity(), BytePageSize::Size64.capacity());
661        drop(buf);
662        assert_eq!(cached_pages(BytePageSize::Size64), 1);
663
664        // buffers without a page size are not cached
665        let buf = BytesMut::with_capacity(BytePageSize::Size64.capacity() + 1);
666        assert_eq!(buf.page_size(), BytePageSize::Unset);
667        assert_eq!(BytesMut::new().page_size(), BytePageSize::Unset);
668        assert_eq!(BytesMut::with_capacity(64).page_size(), BytePageSize::Unset);
669        assert_eq!(
670            BytesMut::copy_from_slice(b"hello").page_size(),
671            BytePageSize::Unset
672        );
673        drop(buf);
674        for size in crate::size::PAGE_SIZES {
675            let expected = usize::from(size == BytePageSize::Size64);
676            assert_eq!(cached_pages(size), expected);
677        }
678    }
679
680    /// STATEMENT: A new page request with no cached page of its size takes a cached page one size smaller, and the metrics record the fallback.
681    ///
682    /// HOW:
683    /// - Cache a `Size8` page, request `Size16`, and check the same `Size8` page is returned, counted as a `Size16` miss and fallback and a `Size8` hit.
684    /// - Request `Size16` again with an empty cache and check a `Size16` page is allocated, counted as a miss only.
685    /// - Check the totals include the fallback.
686    #[test]
687    fn new_page_falls_back_to_smaller_size() {
688        reset();
689        let metrics = PageCacheMetrics::current();
690        let (s8, s16) = (BytePageSize::Size8, BytePageSize::Size16);
691
692        let st = StorageVec::sized(s8);
693        let addr = st.0;
694        drop(st);
695
696        let st = StorageVec::sized(s16);
697        assert_eq!(st.0, addr);
698        assert_eq!(st.page_size(), s8);
699        assert_eq!(cached_pages(s8), 0);
700        let snapshot = metrics.snapshot();
701        let s = *snapshot.get(s16).unwrap();
702        assert_eq!((s.hits, s.misses, s.fallbacks, s.pages), (0, 1, 1, 0));
703        let s = *snapshot.get(s8).unwrap();
704        assert_eq!(
705            (s.hits, s.misses, s.fallbacks, s.cached, s.pages),
706            (1, 1, 0, 1, 0)
707        );
708
709        let st2 = StorageVec::sized(s16);
710        assert_eq!(st2.page_size(), s16);
711        let snapshot = metrics.snapshot();
712        let s = *snapshot.get(s16).unwrap();
713        assert_eq!((s.hits, s.misses, s.fallbacks), (0, 2, 1));
714        let total = snapshot.total();
715        assert_eq!((total.hits, total.misses, total.fallbacks), (1, 3, 1));
716        drop((st, st2));
717        reset();
718    }
719
720    /// STATEMENT: The fallback goes exactly one page size down and only when the requested size has no cached page.
721    ///
722    /// HOW:
723    /// - Cache a `Size4` page and check a `Size16` request allocates `Size16` and leaves the `Size4` page cached.
724    /// - Cache `Size8` and `Size16` pages and check a `Size16` request takes the `Size16` page and leaves the `Size8` page cached.
725    #[test]
726    fn new_page_falls_back_one_size_only() {
727        reset();
728        let (s4, s8, s16) = (
729            BytePageSize::Size4,
730            BytePageSize::Size8,
731            BytePageSize::Size16,
732        );
733
734        drop(StorageVec::sized_exact(s4));
735        let st = StorageVec::sized(s16);
736        assert_eq!(st.page_size(), s16);
737        assert_eq!(cached_pages(s4), 1);
738
739        let p8 = StorageVec::sized_exact(s8);
740        drop(st);
741        drop(p8);
742        let st = StorageVec::sized(s16);
743        assert_eq!(st.page_size(), s16);
744        assert_eq!((cached_pages(s8), cached_pages(s16)), (1, 0));
745        drop(st);
746        reset();
747    }
748
749    /// STATEMENT: New `BytesMut` buffers and new `BytePages` pages take a cached page one size smaller, `Unset` falls back from `Size64` to `Size48`.
750    ///
751    /// HOW:
752    /// - Cache a `Size8` page and check `BytesMut::with_page_size(Size16)` returns it with `Size8` capacity.
753    /// - Cache a `Size48` page and check `BytesMut::with_page_size(Unset)` returns it.
754    /// - With the `tracker` feature, check `BytesMut::with_page_size_in` falls back too.
755    /// - Cache a `Size8` page, write to a `Size16` `BytePages`, and check the page left the cache as a `Size16` fallback.
756    #[test]
757    fn new_buffers_fall_back_to_smaller_page() {
758        reset();
759        let metrics = PageCacheMetrics::current();
760        let (s8, s16) = (BytePageSize::Size8, BytePageSize::Size16);
761
762        drop(StorageVec::sized_exact(s8));
763        let buf = BytesMut::with_page_size(s16);
764        assert_eq!(buf.page_size(), s8);
765        assert_eq!(buf.capacity(), s8.capacity());
766        drop(buf);
767
768        drop(StorageVec::sized_exact(BytePageSize::Size48));
769        let buf = BytesMut::with_page_size(BytePageSize::Unset);
770        assert_eq!(buf.page_size(), BytePageSize::Size48);
771        drop(buf);
772
773        #[cfg(feature = "tracker")]
774        {
775            let tracker = BytesTracker::new();
776            let buf = BytesMut::with_page_size_in(s16, &tracker);
777            assert_eq!(buf.page_size(), s8);
778            assert_eq!(tracker.snapshot().live_bytes, s8.alloc_size());
779        }
780
781        assert_eq!(cached_pages(s8), 1);
782        let mut pages = BytePages::new(s16);
783        pages.put_slice(b"hello");
784        assert_eq!(cached_pages(s8), 0);
785        let expected = if cfg!(feature = "tracker") { 3 } else { 2 };
786        assert_eq!(metrics.snapshot().get(s16).unwrap().fallbacks, expected);
787        drop(pages);
788        reset();
789    }
790}
ntex_bytes/cache.rs

ntex_bytes/
cache.rs
