kevy_alloc/
reclaim.rs

1//! The reclaim sweep — where pages actually go back.
2//!
3//! Split from `heap.rs` for the file-size rule at the seam that makes
4//! sense: everything here runs on the shard tick, nothing on the
5//! allocation fast path. `reclaim` walks the segments; whole spans with
6//! nothing live are unassigned and discarded (v1 behaviour, with the
7//! per-sweep hysteresis), and spans that still hold live slots get the
8//! v2 treatment: every page no live slot overlaps is handed back
9//! individually, which is the structure M3 forced (RFC §5.1).
10
11use core::ptr::NonNull;
12
13use crate::class;
14use crate::class::SPAN_BYTES;
15use crate::heap::{EMPTY_SPAN_HYSTERESIS, Heap};
16use crate::os;
17use crate::segment::{FIRST_DATA_SPAN, NO_CLASS, SPANS_PER_SEGMENT, Segment};
18
19impl Heap {
20    /// Return free pages to the OS — whole spans where nothing is live,
21    /// and *individual pages* inside spans that still are. The second
22    /// half is v2 (RFC §5.1): M3 measured the whole-span rule returning
23    /// 3 % because a span only empties when all its slots die together,
24    /// while glibc works at page granularity. Now so do we.
25    ///
26    /// Drains foreign frees first: slots parked by other shards pin
27    /// their pages exactly as live slots do, so sweeping before
28    /// draining under-returns for no reason.
29    ///
30    /// The retained count is per sweep rather than cumulative. A running
31    /// counter looked equivalent and was not: it only ever grew, so the
32    /// second sweep found it already past the threshold and returned
33    /// everything, which made the hysteresis vanish after one call.
34    pub fn reclaim(&mut self) {
35        // Claimed-word bits pin their pages exactly as live slots do;
36        // write them back first so the sweep sees true occupancy.
37        self.flush_claims();
38        // Retained large mappings go back each tick: retention beyond a
39        // tick requires sustained traffic to re-earn, and idle memory
40        // stays bounded by the tick length rather than the pool size.
41        crate::large::pool_drain();
42        // Ship pending foreign frees home before sweeping: they pin
43        // pages on OTHER heaps' segments, and the tick is the latency
44        // bound on how long a batch may sit.
45        if !self.outbound.is_empty() {
46            self.outbound.flush();
47        }
48        self.drain_foreign();
49        let mut kept: u16 = 0;
50        let mut seg = self.segments;
51        while !seg.is_null() {
52            // SAFETY: live header from our own list.
53            let s = unsafe { &mut *seg };
54            for ix in FIRST_DATA_SPAN..SPANS_PER_SEGMENT {
55                if s.spans[ix].class == NO_CLASS {
56                    continue;
57                }
58                if s.spans[ix].live != 0 {
59                    Self::discard_free_pages(s, ix);
60                    continue;
61                }
62                if kept < EMPTY_SPAN_HYSTERESIS {
63                    kept += 1;
64                    continue;
65                }
66                let c = s.spans[ix].class as usize;
67                if self.partial[c] == Some((unsafe { NonNull::new_unchecked(seg) }, ix as u8)) {
68                    self.partial[c] = None;
69                }
70                self.spans_in_class[c] -= 1;
71                s.spans[ix].reset(crate::pagemap::NO_CLASS);
72                let base = s.span_base(ix);
73                // SAFETY: nothing is live in this span, and the range is
74                // page-aligned and inside a live mapping.
75                unsafe {
76                    os::discard(NonNull::new_unchecked(base), SPAN_BYTES);
77                }
78            }
79            seg = s.next;
80        }
81    }
82
83    /// Hand back every page of a *live* span that no live slot overlaps.
84    ///
85    /// The page rule, exactly: below the high-water byte (never-touched
86    /// pages are already non-resident — discarding them is a wasted
87    /// syscall), not already discarded, and no live slot overlapping.
88    /// Contiguous runs go to the OS in one call.
89    fn discard_free_pages(s: &mut Segment, ix: usize) {
90        use crate::pagemap::{PAGES_PER_SPAN, slots_of_page};
91        let meta = &mut s.spans[ix];
92        let slot = class::size_of(meta.class as usize);
93        let cap = meta.capacity();
94        let hw_bytes = meta.high_water as usize * slot;
95        let base = s.span_base(ix);
96        let mut run: Option<usize> = None;
97        for p in 0..PAGES_PER_SPAN {
98            let meta = &mut s.spans[ix];
99            let fresh = meta.discarded & (1u16 << p) == 0
100                && p * os::PAGE < hw_bytes
101                && {
102                    let (a, b) = slots_of_page(p, slot, cap);
103                    !meta.range_has_live(a, b)
104                };
105            if fresh {
106                meta.discarded |= 1u16 << p;
107                run.get_or_insert(p);
108            } else if let Some(r0) = run.take() {
109                // SAFETY: pages r0..p hold no live slot and no metadata
110                // (the bitmap lives in the header — the whole point).
111                unsafe {
112                    os::discard(
113                        NonNull::new_unchecked(base.wrapping_add(r0 * os::PAGE)),
114                        (p - r0) * os::PAGE,
115                    );
116                }
117            }
118        }
119        if let Some(r0) = run {
120            // SAFETY: as above, through the end of the span.
121            unsafe {
122                os::discard(
123                    NonNull::new_unchecked(base.wrapping_add(r0 * os::PAGE)),
124                    (PAGES_PER_SPAN - r0) * os::PAGE,
125                );
126            }
127        }
128    }
129}
kevy_alloc/reclaim.rs

kevy_alloc/
reclaim.rs
