aligned_vmem/reservation.rs
1use core::ptr::NonNull;
2#[cfg(feature = "bench-internals")]
3use core::sync::atomic::Ordering;
4
5#[cfg(feature = "lazy-commit")]
6use crate::api::{commit_range, try_commit_range};
7use crate::api::{decommit, decommit_lazy, dispatch_try_decommit, recommit, try_recommit};
8#[cfg(feature = "bench-internals")]
9use crate::bench_internals::HUGE_DECOMMIT_ATTEMPTS;
10use crate::decommit_outcome::DecommitOutcome;
11use crate::error::VmemError;
12use crate::os::release_reservation;
13use crate::page::PAGE;
14use crate::page_size::{page_size_or_poison, PAGE_SIZE_QUERY_FAILED};
15use crate::reservation_full_parts::ReservationFullParts;
16use crate::reservation_parts::ReservationParts;
17
18/// An owning handle to one aligned span of anonymous virtual memory.
19///
20/// `as_ptr()` is non-null, aligned to the `align` requested at reservation, and
21/// valid for `len()` bytes for the lifetime of this handle **with the following
22/// exceptions**:
23///
24/// - **Decommitted ranges**: Ranges that the caller has decommitted (via the
25/// free functions or the safe methods) and not yet recommitted have
26/// platform-specific behavior:
27/// - **Windows**: pages are unmapped until `recommit`; access before `recommit`
28/// crashes with `STATUS_ACCESS_VIOLATION`.
29/// - **Linux (eager `decommit`)**: pages are zeroed on next access via `MADV_DONTNEED`.
30/// - **Linux (lazy `decommit_lazy`)**: pages keep old contents until kernel
31/// reclaims them under pressure; writes before reclamation cancel the free.
32/// - **Darwin/BSD**: pages keep old contents; `MADV_DONTNEED` is advisory-only
33/// and does not reliably zero.
34/// - **Huge reservations, `decommit_lazy` (both layers), and `decommit`/
35/// `try_decommit` on Windows or on a non-huge-page-aligned range on
36/// Linux/Android**: old contents remain. The safe methods
37/// [`Reservation::decommit`]/[`Reservation::decommit_lazy`] skip the
38/// backend call outright in this case (they can consult `is_huge()` and,
39/// for `decommit`, the requested range); the free functions cannot
40/// consult `is_huge()`, so they still issue the syscall — which the OS
41/// then refuses or ignores. Same observable outcome, different mechanism;
42/// do not read "no-op" as "no syscall" for the free functions.
43/// - **Huge reservations, eager `decommit`/`try_decommit`, Linux/Android
44/// kernel >= 5.18, range aligned to the huge page size (2 MiB) at both
45/// endpoints (task #1140)**: this is the ONE huge-page case where decommit
46/// actually works — pages ARE zeroed on next access via `MADV_DONTNEED`,
47/// same as the ordinary eager-Linux case above. Both the safe method and
48/// the free function issue the real syscall here; they agree. See
49/// [`Reservation::decommit`]'s own doc for the exact eligibility rule.
50///
51/// - **Lazy reservations on Windows (feature `lazy-commit`)**: When created via
52/// `reserve_aligned_lazy`, only the `initial_commit` prefix is committed at
53/// reservation time. The tail `[initial_commit, len())` must be committed via
54/// `commit_range` before it becomes writable. Writing to the uncommitted tail
55/// results in an access violation.
56///
57/// The span is **not** initialised. Dropping the handle returns the whole
58/// underlying OS reservation to the OS exactly once.
59///
60/// For a self-hosted allocator that records `(reservation, reservation_len)` in
61/// its own metadata rather than keeping a `Vec<Reservation>`, use
62/// [`into_parts`](Self::into_parts) to take the raw reservation (suppressing the
63/// `Drop`) and release it later with [`release`](crate::api::release).
64///
65/// `Reservation` is `Send` (the span is owned exclusively) but not `Sync`
66/// (writes through the raw pointer are unsynchronised — that is the caller's
67/// concern).
68pub struct Reservation {
69 pub(crate) base: NonNull<u8>,
70 pub(crate) len: usize,
71 pub(crate) reservation: NonNull<u8>,
72 pub(crate) reservation_len: usize,
73 /// The alignment requested at reservation time. Carried so the `Drop` /
74 /// [`release`](crate::api::release) path can reconstruct the exact `Layout` under miri (the
75 /// native `munmap` / `VirtualFree` paths ignore it). See [`into_parts`].
76 pub(crate) align: usize,
77 /// Whether OS large/huge pages were actually granted for this reservation.
78 /// True if `reserve_aligned_huge` succeeded in obtaining large pages on
79 /// Linux (`MAP_HUGETLB`) or Windows (`MEM_LARGE_PAGES` when the OS grants
80 /// the request). False if the request fell back to ordinary pages.
81 ///
82 /// This flag is the "best-effort" observable: a caller can detect whether
83 /// the huge-page feature actually engaged, rather than receiving only an
84 /// indistinguishable `Ok(Reservation)` on every fallback path.
85 ///
86 /// **Windows limitation (task #848 single-call fast path):** on Windows,
87 /// this flag is `true` only when ALL of the following hold:
88 /// 1. The fast-path condition `align <= GetLargePageMinimum()` is satisfied
89 /// (typically `align <= 2 MiB` on x86_64)
90 /// 2. `size` is a multiple of the system's large-page minimum
91 /// 3. The calling process has `SeLockMemoryPrivilege` granted AND has
92 /// **enabled** it via `AdjustTokenPrivileges` (the crate does not do
93 /// this for you — a process with the privilege granted but not
94 /// enabled fails exactly like an unprivileged one and silently falls
95 /// back to ordinary pages)
96 ///
97 /// NOTE: The widened fast-path condition (II-3, 2026-08-16 audit finding) expanded
98 /// the single-call ATTEMPT window from `align <= 64 KiB` to `align <= GetLargePageMinimum()`,
99 /// but on an unprivileged host the actual paths that SUCCEED (pass the post-call alignment
100 /// check) are typically still limited. When large pages are NOT granted (unprivileged),
101 /// `VirtualAlloc`'s alignment guarantee is only 64 KiB; in practice it typically does NOT
102 /// happen to land on the requested alignment, so the post-call check fails and the fast
103 /// path falls through to the two-call path. Practically, this means `is_huge == true` only
104 /// for shapes where large pages are actually granted, which requires all three conditions
105 /// above to hold.
106 ///
107 /// If any of these conditions fail, the function falls back to ordinary
108 /// pages and this flag is `false`. On Windows, large pages (`MEM_LARGE_PAGES`)
109 /// are only ever requested and possibly granted via the single-call fast path;
110 /// the two-call path never requests large pages, so
111 /// `granted_huge` is always `false` for a reservation that takes it.
112 pub(crate) granted_huge: bool,
113}
114
115impl core::fmt::Debug for Reservation {
116 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
117 f.debug_struct("Reservation")
118 .field("base", &self.base.as_ptr())
119 .field("len", &self.len)
120 .field("reservation", &self.reservation.as_ptr())
121 .field("reservation_len", &self.reservation_len)
122 .field("align", &self.align)
123 .field("granted_huge", &self.granted_huge)
124 .finish()
125 }
126}
127
128impl Reservation {
129 /// The aligned usable base of this span. Non-null, aligned to the `align`
130 /// requested at reservation.
131 ///
132 /// **Validity scope:** Valid for [`len()`](Self::len) bytes, with the
133 /// following exceptions:
134 ///
135 /// - **Decommitted ranges:** Ranges decommitted via the free functions or
136 /// safe methods and not yet recommitted have platform-specific behavior:
137 /// - **Windows**: pages are unmapped until `recommit`; access before
138 /// `recommit` crashes with `STATUS_ACCESS_VIOLATION`.
139 /// - **Linux (eager `decommit`)**: pages are zeroed on next access via
140 /// `MADV_DONTNEED`.
141 /// - **Linux (lazy `decommit_lazy`)**: pages keep old contents until kernel
142 /// reclaims them under pressure; writes before reclamation cancel the free.
143 /// - **Darwin/BSD**: pages keep old contents; `MADV_DONTNEED` is
144 /// advisory-only and does not reliably zero.
145 /// - **Huge reservations, `decommit_lazy` (both layers), and `decommit`/
146 /// `try_decommit` on Windows or on a non-huge-page-aligned range on
147 /// Linux/Android**: old contents remain. The safe
148 /// methods [`Self::decommit`]/[`Self::decommit_lazy`] skip the backend
149 /// call outright in this case (they can consult [`Self::is_huge`] and,
150 /// for `decommit`, the requested range); the free
151 /// functions cannot consult [`Self::is_huge`], so they still issue the
152 /// syscall — which the OS then refuses or ignores. Same observable
153 /// outcome, different mechanism; do not read "no-op" as "no syscall"
154 /// for the free functions.
155 /// - **Huge reservations, eager `decommit`/`try_decommit`, Linux/Android
156 /// kernel >= 5.18, range aligned to the huge page size (2 MiB) at both
157 /// endpoints (task #1140):** the one huge-page case where decommit
158 /// actually works — pages ARE zeroed on next access via
159 /// `MADV_DONTNEED`. Both layers issue the real syscall here and agree.
160 /// See [`Self::decommit`]'s own doc for the exact eligibility rule.
161 ///
162 /// - **Lazy reservations on Windows (feature `lazy-commit`):** When created
163 /// via `reserve_aligned_lazy`, only the `initial_commit` prefix is
164 /// committed at reservation time. The tail `[initial_commit, len())` must
165 /// be committed via `commit_range` before it becomes writable. Writing
166 /// to the uncommitted tail results in an access violation.
167 ///
168 /// Returns `*mut u8` (rather than the std convention of `*const T` from
169 /// `&self`) because a raw pointer carries no borrow obligation in this
170 /// crate's model, and the span is exclusively owned by this `Reservation`
171 /// handle. The mutability reflects ownership, not mutability of the
172 /// borrow itself.
173 #[must_use]
174 #[inline]
175 pub fn as_ptr(&self) -> *mut u8 {
176 self.base.as_ptr()
177 }
178
179 /// The number of usable bytes at [`as_ptr`](Self::as_ptr).
180 #[must_use]
181 #[inline]
182 #[allow(clippy::len_without_is_empty)]
183 pub const fn len(&self) -> usize {
184 self.len
185 }
186
187 /// The start of the underlying OS reservation (may sit below
188 /// [`as_ptr`](Self::as_ptr) because the reservation is over-reserved
189 /// to achieve alignment and the full mapping is kept).
190 #[must_use]
191 #[inline]
192 pub fn reservation_ptr(&self) -> *mut u8 {
193 self.reservation.as_ptr()
194 }
195
196 /// The **requested/logical** span length of this reservation.
197 ///
198 /// **This value is NOT necessarily the actual OS reservation size** — at least
199 /// three paths under-report the true VA span the OS mapped:
200 ///
201 /// - **Windows single-call fast path** (`align <= 64 KiB`): this returns
202 /// `commit_len` (which equals `size`), not the rounded-up VA reservation
203 /// size. Windows rounds VA reservations up to the 64 KiB allocation
204 /// granularity internally, so `reserve_aligned(4096, 4096)` reports
205 /// `reservation_len() == 4096` while actually consuming 64 KiB of address
206 /// space.
207 /// - **Windows two-call path's fast-reserve sub-path** (`align <= 64 KiB`
208 /// via `reserve_aligned_lazy`): when the candidate `VirtualAlloc(NULL,
209 /// size, MEM_RESERVE)` happens to be aligned, this returns `size` directly,
210 /// not the rounded-up 64 KiB granularity. The underlying reservation still
211 /// consumes a 64 KiB-granular region.
212 /// - **Any page-rounding `mmap` where the OS page size exceeds the requested
213 /// granularity** — e.g. Apple-Silicon macOS's 16 KiB pages, or 64 KiB on
214 /// some Linux configurations (see [`MIN_PAGE`](crate::min_page::MIN_PAGE)'s doc above): `mmap` rounds
215 /// `length` up to the page size, so `reserve_aligned(PAGE, PAGE)` on a 16
216 /// KiB-page host actually maps a full 16 KiB page while this returns
217 /// `4096`.
218 ///
219 /// Both cases are harmless for correctness (`VirtualFree(base, 0,
220 /// MEM_RELEASE)` ignores the length argument; `munmap` rounds its length
221 /// argument up to the page size the same way `mmap` did, so `release`
222 /// still unmaps the whole underlying mapping) — but the return value is
223 /// not a portable measure of the true reservation size.
224 #[must_use]
225 #[inline]
226 pub const fn reservation_len(&self) -> usize {
227 self.reservation_len
228 }
229
230 // Historical note (task #848, #921): the Windows single-call fast path
231 // (align <= WIN_ALLOCATION_GRANULARITY, typically 64 KiB; widens to
232 // GetLargePageMinimum() when requesting large pages) and the two-call
233 // path's fast-reserve sub-path (align <= WIN_ALLOCATION_GRANULARITY
234 // via reserve_aligned_lazy) are the primary under-report cases for
235 // this method; the page-rounding mmap case is the third. These are
236 // documented in the method's rustdoc above without task-number references.
237
238 /// The alignment requested at reservation time.
239 #[must_use]
240 #[inline]
241 pub const fn align(&self) -> usize {
242 self.align
243 }
244
245 /// Whether OS large/huge pages were actually granted for this reservation.
246 ///
247 /// Returns `true` if the reservation successfully obtained large/huge pages
248 /// from the OS (Linux `MAP_HUGETLB` or Windows `MEM_LARGE_PAGES`), and `false`
249 /// if it fell back to ordinary pages or was not a huge-page request.
250 ///
251 /// This is the "best-effort" observable: a caller using `reserve_aligned_huge`
252 /// can now detect whether the huge-page feature actually engaged, rather than
253 /// receiving only an indistinguishable `Ok(Reservation)` on every fallback.
254 ///
255 /// **Windows limitation (task #848 single-call fast path):** on Windows,
256 /// this returns `true` only when ALL of the following hold:
257 /// 1. The fast-path condition `align <= GetLargePageMinimum()` is satisfied
258 /// (typically `align <= 2 MiB` on x86_64)
259 /// 2. `size` is a multiple of the system's large-page minimum
260 /// 3. The calling process has `SeLockMemoryPrivilege` granted AND has
261 /// **enabled** it via `AdjustTokenPrivileges` (the crate does not do
262 /// this for you — a process with the privilege granted but not
263 /// enabled fails exactly like an unprivileged one and silently falls
264 /// back to ordinary pages)
265 ///
266 /// NOTE: The widened fast-path condition (II-3, 2026-08-16 audit finding) expanded
267 /// the single-call ATTEMPT window from `align <= 64 KiB` to `align <= GetLargePageMinimum()`,
268 /// but on an unprivileged host the actual paths that SUCCEED (pass the post-call alignment
269 /// check) are typically still limited. When large pages are NOT granted (unprivileged),
270 /// `VirtualAlloc`'s alignment guarantee is only 64 KiB; in practice it typically does NOT
271 /// happen to land on the requested alignment, so the post-call check fails and the fast
272 /// path falls through to the two-call path. Practically, this means `is_huge() == true` only
273 /// for shapes where large pages are actually granted, which requires all three conditions
274 /// above to hold.
275 ///
276 /// If any of these conditions fail, the function falls back to ordinary pages
277 /// and this flag is `false`. On Windows, large pages (`MEM_LARGE_PAGES`)
278 /// are only ever requested and possibly granted via the single-call fast path;
279 /// the two-call path never requests large pages, so
280 /// `is_huge()` is always `false` for a reservation that takes it. See
281 /// [`reserve_aligned_huge`](crate::api::reserve_aligned_huge)'s rustdoc for details.
282 ///
283 /// **Note:** reservations adopted via [`from_raw_parts`](Self::from_raw_parts)
284 /// report whatever `granted_huge` value the caller passed to that constructor,
285 /// which the caller is responsible for getting right (see that constructor's
286 /// `# Safety` section).
287 ///
288 /// **This method has no `huge-pages` feature gate — [`Self::decommit`]'s
289 /// eligible-forward behavior does (task #1156, finding F10).** `is_huge()`
290 /// reports `true`/`false` identically regardless of which features are
291 /// enabled; whether a `true` result also gets you a real Linux/Android
292 /// kernel >= 5.18 decommit forward instead of a guaranteed skip depends
293 /// on the `huge-pages` feature being enabled too. See [`Self::decommit`]'s
294 /// doc for the full explanation — this asymmetry matters most for
295 /// [`from_raw_parts`](Self::from_raw_parts) callers, since that
296 /// constructor is also unconditionally compiled.
297 #[must_use]
298 #[inline]
299 pub const fn is_huge(&self) -> bool {
300 self.granted_huge
301 }
302
303 /// Returns `true` if the current platform's **ordinary native backend** guarantees
304 /// that eager [`Self::decommit`] returns physical backing to the OS and zero-fills
305 /// on next access, `false` otherwise.
306 ///
307 /// **Scope:** this is a platform-level query about the ordinary native backend's
308 /// contract. It does NOT apply to:
309 /// - **huge-page reservations** (those with [`Self::is_huge`] == `true`) — eligibility
310 /// there depends on the platform, the requested range, and (on Linux/Android) the
311 /// running kernel, not on a single platform-wide answer: on Windows decommit is a
312 /// guaranteed no-op; on Linux/Android with kernel >= 5.18, a huge-page-size-aligned
313 /// range CAN actually decommit (see the free [`decommit`] function's "Huge-page
314 /// granularity" rustdoc section for the exact split, and
315 /// [`Self::can_decommit_reclaim_and_zero`]'s own huge-page bullet for the
316 /// conservative instance-level answer this platform-level query cannot give).
317 /// - **miri** — under miri, the backend is a no-op that doesn't model RSS or reclaim.
318 /// - **the `aligned_vmem_mock` cfg** (`RUSTFLAGS="--cfg aligned_vmem_mock"`) — the
319 /// recording mock backend's decommit logs the call WITHOUT touching the OS, so it
320 /// reclaims nothing and zeroes nothing (task #1066). Excluded for the same reason
321 /// the sibling capability query `lazy_commit_is_honored()` (feature `lazy-commit`)
322 /// excludes it: this family answers for the backend actually linked into the
323 /// compilation, and the miri bullet above is already that same substituted-backend
324 /// category rather than a platform property.
325 ///
326 /// For an instance-level query that accounts for huge pages, use
327 /// [`Self::can_decommit_reclaim_and_zero`].
328 ///
329 /// Platform behavior (ordinary native backend only, eager decommit path):
330 /// - **Linux (all targets)**: returns `true`. `MADV_DONTNEED` unmaps physical pages
331 /// and re-faults fresh zero pages on next access.
332 /// - **Windows**: returns `true`. `MEM_DECOMMIT` unmaps physical pages and
333 /// re-faults fresh zero pages on next access.
334 /// - **Darwin family (macOS/iOS/tvOS/watchOS)**: returns `false`. `MADV_DONTNEED`
335 /// is advisory-only for anonymous memory and does not reliably unmap/zero pages.
336 /// A decommit+recommit roundtrip can observe old data still resident.
337 /// - **BSD family (FreeBSD/DragonFly/NetBSD/OpenBSD)**: returns `false`. Same
338 /// advisory-only caveat as Darwin for eager decommit. (Note: lazy decommit
339 /// via [`decommit_lazy`] DOES reclaim on BSD via `MADV_FREE`, even though
340 /// eager decommit does not.)
341 ///
342 /// This is a compile-time constant per platform: the return value is the same
343 /// for all calls within a single compilation unit, determined by the target
344 /// OS triple, whether miri is active, and whether the `aligned_vmem_mock` recording
345 /// backend is compiled in. It provides programmatic access to the
346 /// platform-specific guarantee that [`Self::decommit`]'s rustdoc describes in prose.
347 #[must_use]
348 #[inline]
349 pub const fn decommit_reclaims_and_zeroes() -> bool {
350 cfg!(not(any(
351 target_os = "macos",
352 target_os = "ios",
353 target_os = "tvos",
354 target_os = "watchos",
355 target_os = "freebsd",
356 target_os = "dragonfly",
357 target_os = "netbsd",
358 target_os = "openbsd",
359 miri,
360 aligned_vmem_mock
361 )))
362 }
363
364 /// Returns `true` if eager [`Self::decommit`] on **this specific reservation**
365 /// guarantees reclaim+zero-fill semantics, `false` otherwise.
366 ///
367 /// This is an **advisory** capability query. It is computed from
368 /// compile-time platform capability and the reservation's huge-page status only,
369 /// and does **not** issue any runtime syscall or observe whether a prior `decommit`
370 /// call actually succeeded. Specifically:
371 ///
372 /// - On Linux/Windows (native — not miri, not the `aligned_vmem_mock` cfg), `true` means the platform guarantees that
373 /// `decommit` will return physical backing and zero-fill on next access via
374 /// `MADV_DONTNEED` / `MEM_DECOMMIT`. Backend syscall failures (e.g. rare kernel
375 /// failures) are silently discarded and not reflected in this query's return value.
376 /// - On Darwin/BSDs, under miri, or under the `aligned_vmem_mock` cfg, `false` means
377 /// decommit is advisory-only (Darwin/BSDs) or a recorded no-op (miri, mock) with no
378 /// reclaim or zero-fill guarantee.
379 /// - On huge-page reservations, `false` — **this bool is CONSERVATIVE and is NOT
380 /// range-aware** (task #1140): on Windows it is unconditionally correct (large-page
381 /// decommit never works there). On Linux/Android with kernel >= 5.18, it
382 /// UNDER-reports: [`Self::decommit`]/[`Self::try_decommit`] DO issue a real
383 /// `MADV_DONTNEED` for a `[start, end)` range that is itself huge-page-size-aligned
384 /// (2 MiB) at both endpoints — see those methods' own doc comments — but this
385 /// instance-level query has no `start`/`end` parameters to judge that per-call, so it
386 /// answers `false` for EVERY range on a huge reservation, including the ranges that
387 /// actually do work. Call [`Self::try_decommit`] directly and judge by its
388 /// [`DecommitOutcome`] return value (task #1180: `Skipped` vs `Advised` vs
389 /// `Refused` — `Self::decommit` itself stays `()`/infallible and carries no
390 /// such signal) / the `bench-internals`
391 /// `huge_decommit_attempts` counter (not an intra-doc link: `bench-internals` is excluded from the published docs.rs feature set)
392 /// if you need to distinguish "this exact range worked" from "this bool said no."
393 ///
394 /// A `true` return is therefore a statement about the **platform and reservation type**,
395 /// not a guarantee that a specific `decommit` call actually released memory or zeroed
396 /// pages — OS errors in that path are unobservable through this API by design
397 /// (the same contract as the infallible `decommit` method itself). A `false` return is
398 /// similarly not a guarantee that no range on this reservation can ever be decommitted
399 /// (see the huge-page bullet above).
400 ///
401 /// This query combines:
402 /// - the platform-level guarantee (see [`Self::decommit_reclaims_and_zeroes`]), and
403 /// - the reservation's huge-page status (via [`Self::is_huge`]).
404 ///
405 /// Returns `false` if EITHER condition fails:
406 /// - the platform doesn't guarantee reclaim+zero-fill (Darwin/BSDs, miri, or the
407 /// `aligned_vmem_mock` cfg), or
408 /// - this reservation uses huge pages — **conservatively**: on Windows this is
409 /// always correct (huge-page decommit is a genuine no-op there), but on
410 /// Linux/Android >= 5.18 a huge-page-size-aligned range CAN actually
411 /// decommit (see [`Self::decommit`]'s doc and the bullet on this fact
412 /// above); this bool has no range to judge, so it answers `false`
413 /// unconditionally for a huge reservation regardless of platform.
414 ///
415 /// Use this when you have an actual `Reservation` and need to know whether decommit
416 /// will work on it **for an ordinary (non-huge) reservation, or to conservatively rule
417 /// out a huge one**. Use the associated function [`Self::decommit_reclaims_and_zeroes`]
418 /// when you only care about platform capability without a reservation instance. For a
419 /// huge reservation on Linux/Android, this bool cannot tell you whether a SPECIFIC
420 /// `[start, end)` will work — call [`Self::try_decommit`] and judge by its
421 /// [`DecommitOutcome`] instead (see the huge-page bullet above).
422 ///
423 /// # Example
424 ///
425 /// Ordinary reservation: decommit works on Linux/Windows (except miri):
426 /// ```text
427 /// let ordinary = reserve_aligned(1024 * 1024, 4096).expect("reserve");
428 /// // On Linux/Windows (native): ordinary.can_decommit_reclaim_and_zero() == true
429 /// // On Darwin/BSD, under miri, or under `aligned_vmem_mock`:
430 /// // ordinary.can_decommit_reclaim_and_zero() == false
431 /// ```
432 ///
433 /// Huge-page reservation: this bool is always false, but on Linux/Android
434 /// >= 5.18 that does NOT mean `decommit` itself is a no-op for every range:
435 /// ```text
436 /// let huge = reserve_aligned_huge(2 * 1024 * 1024, 2 * 1024 * 1024);
437 /// if let Some(ref reservation) = huge {
438 /// if reservation.is_huge() {
439 /// // The bool is always false, regardless of platform — conservative,
440 /// // not "decommit never works" (see the doc above this example).
441 /// assert!(!reservation.can_decommit_reclaim_and_zero());
442 /// }
443 /// }
444 /// ```
445 /// NOTE: On Linux/Android with the `huge-pages` feature enabled, the
446 /// arguments must be multiples of the huge page size (2 MiB); the example
447 /// above uses 2 MiB for both size and align to avoid rejection. On other
448 /// platforms, the function is a best-effort no-op and any size/align will
449 /// succeed (falling back to ordinary pages).
450 ///
451 /// See the tests in `tests/decommit_capability.rs` for runnable coverage of both cases.
452 #[must_use]
453 #[inline]
454 pub fn can_decommit_reclaim_and_zero(&self) -> bool {
455 Self::decommit_reclaims_and_zeroes() && !self.is_huge()
456 }
457
458 /// Consume the handle WITHOUT releasing the OS reservation, returning the
459 /// `(reservation_ptr, reservation_len, align)` the caller must later hand to
460 /// [`release`](crate::api::release) exactly once. Use this when your allocator records the
461 /// reservation in its own self-hosted metadata instead of relying on
462 /// `Drop`.
463 ///
464 /// `align` is the alignment originally requested; the native release paths
465 /// ignore it, but it is required for the miri fallback to reconstruct the
466 /// exact `Layout`. A self-hosting allocator that always uses one alignment
467 /// can pass that constant to [`release`](crate::api::release) instead of storing this value.
468 ///
469 /// **Warning:** This method returns a raw tuple. Consider using
470 /// [`into_reservation_parts`](Self::into_reservation_parts) instead, which
471 /// returns a named struct that prevents accidentally swapping `len` and `align`.
472 #[must_use]
473 pub fn into_parts(self) -> (*mut u8, usize, usize) {
474 let parts = (self.reservation.as_ptr(), self.reservation_len, self.align);
475 core::mem::forget(self);
476 parts
477 }
478
479 /// Consume the handle WITHOUT releasing the OS reservation, returning the
480 /// [`ReservationParts`] struct the caller must later hand to [`release_parts`](crate::api::release_parts)
481 /// exactly once. Use this when your allocator records the reservation in its
482 /// own self-hosted metadata instead of relying on `Drop`.
483 ///
484 /// This method is the typed, named alternative to [`into_parts`](Self::into_parts);
485 /// it prevents the footgun of accidentally swapping `len` and `align`, which
486 /// would be undefined behavior on the native backend and cause leaks or crashes
487 /// on the Unix backend.
488 ///
489 /// **WARNING:** This method discards `base`, `len`, and `granted_huge`. To
490 /// reconstruct a full `Reservation` via [`from_raw_parts`](Self::from_raw_parts),
491 /// you MUST preserve these three fields separately alongside the returned
492 /// `ReservationParts`. If you omit `granted_huge`, the reconstructed reservation
493 /// will incorrectly report `is_huge() == false` even if the original used huge
494 /// pages, which can lead to incorrect decommit-availability decisions.
495 ///
496 /// For backwards compatibility with code that already uses the tuple form,
497 /// you can call [`ReservationParts::as_tuple`] to get a raw tuple.
498 ///
499 /// No message-less `#[must_use]` on this function itself (task
500 /// #1213/L3): the return type [`ReservationParts`] now carries its own
501 /// `#[must_use]` with a leak-specific message (dropping it leaks the
502 /// reservation), which already fires for every caller of every function
503 /// returning it, this one included — clippy's `double_must_use` lint
504 /// flags a redundant message-less attribute stacked on top of that.
505 pub fn into_reservation_parts(self) -> ReservationParts {
506 let parts = ReservationParts {
507 ptr: self.reservation.as_ptr(),
508 len: self.reservation_len,
509 align: self.align,
510 };
511 // Same suppression as `into_parts` -- without this, `self` would run
512 // its normal `Drop` (which now also releases the OS reservation) at
513 // the end of this function, and the returned `ReservationParts`
514 // would describe already-freed memory: a guaranteed double-free the
515 // moment the caller follows this method's own contract and passes
516 // it to `release_parts`.
517 core::mem::forget(self);
518 parts
519 }
520
521 /// Consume the handle WITHOUT releasing the OS reservation, returning a
522 /// full [`ReservationFullParts`] struct containing all six fields needed to
523 /// reconstruct the original `Reservation` via [`from_raw_parts`](Self::from_raw_parts).
524 ///
525 /// This is the lossless round-trip alternative to [`into_reservation_parts`](Self::into_reservation_parts):
526 /// it preserves `base`, `len`, and `granted_huge` in addition to the underlying
527 /// reservation metadata, eliminating the risk of silent huge-page status loss
528 /// or usable-span information loss.
529 ///
530 /// Use this when you need to temporarily extract all reservation state for
531 /// later reconstruction, such as in a custom allocator that hands off
532 /// reservations between components within the same process.
533 ///
534 /// **IMPORTANT:** `ReservationFullParts` is a plain struct with no `Drop`
535 /// implementation — dropping or forgetting it does NOT release the underlying
536 /// OS reservation. The reservation will leak until you reconstruct it via
537 /// `into_reservation()` and drop the resulting `Reservation`, or release it
538 /// manually via [`release`](crate::api::release) (using the `reservation`, `reservation_len`, and
539 /// `align` fields from `ReservationFullParts`). If you only need manual
540 /// release and don't require preserving `base`, `len`, and `granted_huge`,
541 /// prefer [`into_reservation_parts`](Self::into_reservation_parts) instead,
542 /// which provides the `release_parts` function.
543 ///
544 /// No message-less `#[must_use]` on this function itself (task
545 /// #1213/L3): the return type [`ReservationFullParts`] now carries its
546 /// own `#[must_use]` with a leak-specific message, for the same reason
547 /// as [`into_reservation_parts`](Self::into_reservation_parts) above.
548 pub fn into_full_parts(self) -> ReservationFullParts {
549 let parts = ReservationFullParts {
550 base: self.base.as_ptr(),
551 len: self.len,
552 reservation: self.reservation.as_ptr(),
553 reservation_len: self.reservation_len,
554 align: self.align,
555 granted_huge: self.granted_huge,
556 };
557 core::mem::forget(self);
558 parts
559 }
560
561 /// Decommit pages `[start, end)` within this reservation.
562 ///
563 /// This is the safe, bounds-checked alternative to the free [`decommit`]
564 /// function for callers already holding a `Reservation`. It delegates to
565 /// the underlying implementation with `self.as_ptr()` as base and
566 /// automatically ensures `[start, end)` is within the reservation's usable span.
567 ///
568 /// Takes `&mut self` (task #1113): OS-state mutation requires exclusive
569 /// access, so a shared `&Reservation` can reach none of the seven state
570 /// mutators. This is what structurally seals the `LazyReservation`
571 /// watermark (finding H1, task #1104): a leaked `&Reservation` is now
572 /// read-only by construction, not by policing.
573 ///
574 /// **Programmatically check platform guarantees:** use
575 /// [`Self::decommit_reclaims_and_zeroes`] to query whether the current
576 /// platform guarantees reclaim+zero-fill semantics.
577 ///
578 /// Hint the OS to return the physical backing of `[start, end)` while keeping the
579 /// address-space reservation alive. On Linux and Windows this is guaranteed to
580 /// return physical backing and zero-fill on next access (Linux `MADV_DONTNEED`,
581 /// Windows `MEM_DECOMMIT`). On the Darwin family (macOS/iOS/tvOS/watchOS) and the
582 /// four BSDs (FreeBSD/DragonFly/NetBSD/OpenBSD), this is a best-effort hint with no
583 /// zero-fill or reclaim guarantee — the physical pages may remain resident and
584 /// old data may be observed after a decommit+recommit roundtrip.
585 ///
586 /// `start` and `end` must be multiples of the runtime page size ([`page_size()`](crate::page_size::page_size)).
587 /// A no-op if the range is out of bounds (`end > self.len()`); an empty
588 /// range is a no-op only when page-ALIGNED — an empty MISALIGNED range
589 /// (`start == end`, endpoints not page multiples, e.g. `decommit(1, 1)`)
590 /// is a contract violation like any other, with the SAME profile split
591 /// as every other violated range: a silent no-op in a RELEASE build
592 /// (the forwarded free function returns at `start >= end` once the
593 /// `debug_assert!` is compiled out) and a tripwire panic in a DEBUG
594 /// build — EXCEPT on a huge-page reservation, where a NON-huge-aligned
595 /// (or inverted) range never reaches the forward at all, so it is a
596 /// silent no-op on EVERY profile there and the debug tripwire never
597 /// fires (see `# Panics`; task #1084/M2 wrote the split into `# Panics`,
598 /// task #1097/L4 qualified this summary line to match, task #1108 added
599 /// the huge exception that the paragraph below and `# Panics` both
600 /// already stated but this sentence did not; task #1140 narrowed the
601 /// huge exception to "non-huge-aligned or inverted" — see below).
602 ///
603 /// **Contract violations, by build profile (task #1051, narrowed task
604 /// #1140):** this method forwards to the free [`decommit`] function
605 /// UNFILTERED whenever it forwards at all, so a violated range
606 /// (`start > end`, or an endpoint not a multiple of
607 /// [`page_size()`](crate::page_size::page_size)) follows that function's
608 /// documented profile split exactly on a NON-huge reservation — a silent
609 /// no-op in a RELEASE build (no OS call, nothing recorded), a tripwire
610 /// panic in a DEBUG build. On a HUGE-page reservation
611 /// ([`Self::is_huge`] == `true`), whether this method forwards at all
612 /// now depends on the range (task #1140, Linux/Android kernel >= 5.18
613 /// only): a WELL-FORMED range that is ALSO aligned to the huge page size
614 /// (2 MiB) at both endpoints forwards to the real backend exactly like a
615 /// non-huge reservation would (and can therefore reach that same debug
616 /// tripwire, only for a range that manages to be simultaneously
617 /// huge-aligned AND page-size-misaligned — impossible in practice since
618 /// 2 MiB is already page-size-aligned on every supported page size, so
619 /// this case cannot actually occur); every OTHER range on a huge
620 /// reservation (not huge-aligned, or `start > end`) never reaches the
621 /// forward and is a silent no-op on every profile (see `# Panics`).
622 /// [`Self::try_decommit`] is the fallible form: it reports a violated
623 /// range as `Err` on every profile — including huge reservations (task
624 /// #1084/M3) — and never trips the tripwire.
625 ///
626 /// See [`decommit`] for platform divergence notes (Windows crashes on write
627 /// before recommit, Linux and Android do not), huge-page incompatibility,
628 /// and Darwin zero-fill caveats. Under the `bench-internals` feature, the
629 /// `huge_decommit_attempts` counter (not an intra-doc link: `bench-internals` is excluded from the published docs.rs feature set) is incremented when decommit
630 /// is called on a huge-page reservation with a range that is NOT eligible
631 /// for the Linux/Android >= 5.18 huge-aligned real-call path (i.e. the
632 /// counter tracks calls that hit the silent-no-op path, not every call on
633 /// a huge reservation — task #1140 narrowed this from "every huge-reservation
634 /// call" to "every huge-reservation call that is actually skipped").
635 ///
636 /// **The Linux/Android kernel >= 5.18 eligible-forward path itself
637 /// requires the `huge-pages` feature (task #1156, finding F10) —
638 /// [`Self::is_huge`] does NOT.** The eligibility check this method
639 /// consults (`linux_huge_range_is_madvise_eligible`) is compiled only
640 /// under `#[cfg(all(not(miri), not(aligned_vmem_mock),
641 /// any(target_os = "linux", target_os = "android"),
642 /// feature = "huge-pages"))]`; without that
643 /// feature enabled, EVERY range on a huge-flagged reservation takes the
644 /// silent-no-op early-exit above, unconditionally, on every platform —
645 /// there is no huge-aligned-range exception without the feature.
646 /// **[`Self::from_raw_parts`] no longer creates a mismatch here (task
647 /// #1172/M1-hybrid, closing finding M2):** that constructor now
648 /// `assert!`s that `granted_huge: true` requires the `huge-pages` feature
649 /// to be enabled in THIS crate, so a reservation with `is_huge() == true`
650 /// cannot exist without `huge-pages` — the scenario this paragraph used
651 /// to describe (an adopted `MAP_HUGETLB` reservation reporting
652 /// `is_huge() == true` while `decommit`/`try_decommit` silently and
653 /// permanently skip the backend regardless of range or kernel version,
654 /// because the CONSUMER's Cargo feature set diverged from what the flag
655 /// promised) can no longer arise: without `huge-pages`, adopting such a
656 /// reservation panics at construction instead of silently under-serving
657 /// it later. **[`decommit_lazy`] is NOT a
658 /// workaround** (task #1172, correcting the advice this paragraph used
659 /// to give): [`Self::decommit_lazy`] skips its backend call
660 /// UNCONDITIONALLY for every huge-flagged reservation, on every
661 /// platform, regardless of feature flags or kernel version — it has no
662 /// Linux >= 5.18 huge-aligned carve-out at all (see its own doc). Routing
663 /// around a `huge-pages`-gated no-op into a permanent no-op is not a
664 /// substitute. If you cannot enable `huge-pages`, either accept the
665 /// no-op (RSS will not drop for this reservation) or track the huge-page
666 /// state yourself and avoid relying on either decommit path for it.
667 ///
668 /// # Panics
669 ///
670 /// DEBUG builds only, and only for a contract-violating range (`start >
671 /// end`, or an endpoint not a multiple of the runtime
672 /// [`page_size()`](crate::page_size::page_size)) that actually reaches the
673 /// forwarded free [`decommit`]: unconditionally true on a NON-huge
674 /// reservation, or — since task #1140 — on a huge reservation whenever the
675 /// range happens to be huge-page-size-aligned at both endpoints (in
676 /// practice this can only be a WELL-FORMED range, since a huge-page-size
677 /// multiple is always also a `page_size()` multiple, so the tripwire is
678 /// not actually reachable through the huge-aligned path — this bullet
679 /// exists to be precise about the forwarding rule, not because a real
680 /// input triggers it). That includes an EMPTY MISALIGNED range such as
681 /// `decommit(1, 1)` — emptiness is NOT a pre-check (task #1084, finding
682 /// M2, rewrote this section, which previously claimed "empty and
683 /// out-of-bounds ranges are checked by this method first and never
684 /// panic"; only the out-of-bounds half of that sentence was true). The
685 /// two classes that never panic on any profile: out-of-bounds
686 /// (`end > self.len()`), the one range class this method itself
687 /// pre-checks, and an empty PAGE-ALIGNED range (`start == end`, both
688 /// endpoints multiples of `page_size()`), which forwards as
689 /// well-formed. On a huge-page reservation ([`Self::is_huge`] == `true`)
690 /// a range that is NOT huge-page-size-aligned at both endpoints (or is
691 /// inverted, `start > end`) never reaches the tripwire: it is a silent
692 /// no-op there on every profile, same as before task #1140. RELEASE
693 /// builds silently skip a violated range regardless of huge-page status.
694 /// This is the free function's own documented panic surface reached
695 /// through the safe method, not a new one (task #1079 added this
696 /// `# Panics` section to a doc that previously promised "the same
697 /// silent-skip behavior as the free `decommit` function" with no profile
698 /// qualifier; task #1084 corrected its empty-range claim; task #1140
699 /// narrowed the huge-page exception). **A poisoned page-size query is
700 /// NOT a panic source, on any profile** (task #1173/L1) — see the free
701 /// [`decommit`]'s own "Contract violations, by build profile" section
702 /// for why that state is a silent no-op unconditionally, unlike the
703 /// range-contract tripwire this section describes.
704 pub fn decommit(&mut self, start: usize, end: usize) {
705 // Bounds check: the range must be within the reservation's usable span.
706 if end > self.len() {
707 return;
708 }
709 // Huge-page reservations (finding R6-7, revised task #1140): on Windows,
710 // decommit NEVER works — `VirtualFree(MEM_DECOMMIT)` unconditionally fails
711 // on a large-page region, full stop, so the backend call is skipped
712 // unconditionally there. On Linux/Android, that used to be believed true
713 // unconditionally too, but it is not: Linux 5.18+ added `MADV_DONTNEED`
714 // support for HugeTLB mappings, gated on the address/length both being
715 // aligned to the mapping's huge page size (`man 2 madvise`). This
716 // reservation's `base` is always huge-page-aligned by construction
717 // whenever `is_huge()` (see `linux_huge_range_is_madvise_eligible`'s own
718 // doc), so only `[start, end)` needs checking. See `try_decommit`'s doc
719 // for why an eligible-but-malformed range is still handled by validation,
720 // not by this eligibility check.
721 //
722 // The `if` itself is UNCONDITIONAL and only the diagnostic increment is
723 // feature-gated. Putting the whole block (and therefore the `return`)
724 // behind `#[cfg(feature = "bench-internals")]` would confine the
725 // optimisation to diagnostic builds and leave the useless syscall in
726 // every production build — the exact inverse of the point — while also
727 // making an observable behaviour (syscall issued or not) depend on a
728 // feature flag. Caught at review of task #1040's delegated diff, which
729 // had exactly that shape.
730 if self.is_huge() {
731 #[cfg(all(
732 not(miri),
733 not(aligned_vmem_mock),
734 any(target_os = "linux", target_os = "android"),
735 feature = "huge-pages"
736 ))]
737 if crate::os::linux_huge_range_is_madvise_eligible(start, end) {
738 // SAFETY: `self.as_ptr()` is a valid reservation base, and
739 // we've just verified `[start, end)` is within `self.len()`;
740 // the free function's own contract is validated inside it.
741 // The huge-decommit-eligibility check above is this method's
742 // own addition on top of that contract.
743 unsafe { decommit(self.as_ptr(), start, end) };
744 return;
745 }
746 // Counts calls that hit this early-exit path; the increment is a
747 // single relaxed fetch_add and compiles out when the feature is off.
748 #[cfg(feature = "bench-internals")]
749 HUGE_DECOMMIT_ATTEMPTS.fetch_add(1, Ordering::Relaxed);
750 return;
751 }
752 // SAFETY: `self.as_ptr()` is a valid reservation base, and we've just
753 // verified `[start, end)` is within `self.len()`. The free function's
754 // own contract (multiples of page_size(), etc.) is validated inside it.
755 unsafe { decommit(self.as_ptr(), start, end) };
756 }
757
758 /// Fallible [`Self::decommit`]: `Ok(DecommitOutcome)` on a well-formed
759 /// range, `Err(VmemError::invalid_argument())` if the offsets violated
760 /// the contract (misaligned, `start > end`, or `end > self.len()`) — on
761 /// EVERY reservation kind, huge included (task #1084/M3: the huge-page
762 /// skip used to sit ahead of validation and answer `Ok(())` for a
763 /// malformed range on a huge reservation, disagreeing with both this
764 /// promise and the free [`try_decommit`](crate::try_decommit)'s validate-first order — that
765 /// ordering is unchanged by task #1180, only the `Ok` payload is new).
766 /// Never panics on any build profile: the violation is rejected here,
767 /// before the eager path's tripwire can see it.
768 ///
769 /// **`Ok` payload, task #1180 (PUB-R2 phase 2):** before this task the
770 /// `Ok` case was a bare `Ok(())`, unable to distinguish "the range was
771 /// empty", "this is a huge-page reservation and the backend call was
772 /// skipped", "the backend was called and the OS refused it", and "the
773 /// backend was called and the OS accepted it" — all four collapsed into
774 /// the same signal. [`DecommitOutcome`] now names each case:
775 /// - [`DecommitOutcome::Skipped`] — an empty page-aligned range
776 /// (`start == end`), OR a well-formed non-empty range on a huge-page
777 /// reservation that does not reach the real backend (see the
778 /// "huge-page reservations" paragraph below for exactly which ranges
779 /// those are). No syscall was issued either way.
780 /// - [`DecommitOutcome::Advised`] — the SELECTED BACKEND accepted the
781 /// call — see that variant's own doc for the native-vs-mock-vs-miri
782 /// split. **Never a claim that physical pages were actually
783 /// reclaimed**, even on the native backend. Task #1174 (closed) added
784 /// `ci_hugetlb_real_pool_decommit_actually_zeroes_memory_on_reaccess`
785 /// (`tests/decommit_capability.rs`), hard-enabled in the
786 /// `aligned-vmem-hugetlb-real` CI job: it writes a non-zero pattern,
787 /// decommits an eligible huge-aligned range under a real `MAP_HUGETLB`
788 /// grant, and hard-asserts EVERY byte reads back zero — proving
789 /// zero-fill-on-readback for that one case. **What #1174 did NOT
790 /// prove and does not claim to: physical reclaim to the OS/hugetlb
791 /// pool.** That same CI job's own comments are explicit that
792 /// `HugePages_Free` (the kernel's pool-page-count) is logged only as
793 /// an OBSERVATION around the test, never a pass/fail gate, because it
794 /// is a kernel-global counter shared with the job's other concurrent
795 /// reservations and cannot be safely attributed to one test's own
796 /// `decommit()` call. So: zero-fill on readback is proven for the
797 /// real-HugeTLB/eligible-range case, on a Linux runner — the code
798 /// path itself is gated on Linux **and Android** as a pair (as every
799 /// huge-page mechanism in this crate is), so the Android half is
800 /// inherited from that shared `cfg`, not separately executed by any
801 /// CI job; physical page return to
802 /// the pool remains unmeasured. Do not conflate the two when reading
803 /// `Advised`.
804 /// - [`DecommitOutcome::Refused`] — the backend call was made and the
805 /// OS/kernel refused it (carries the captured [`VmemError`]).
806 ///
807 /// This is the safe, bounds-checked alternative to the free [`try_decommit`](crate::try_decommit)
808 /// function for callers already holding a `Reservation` — and the form to
809 /// reach for when [`Self::decommit`]'s DEBUG-build tripwire is itself
810 /// unwelcome. Until task #1079 this was the one fallible pair with no
811 /// safe-method twin: `recommit`/`try_recommit` and `commit_range`/
812 /// `try_commit_range` already existed at both layers, and
813 /// [`Self::decommit`]'s forwarded tripwire message ("Use try_decommit
814 /// for the fallible form") pointed safe-API callers straight at an
815 /// `unsafe fn` with a raw-pointer signature.
816 ///
817 /// **Huge-page reservations** — FOR A WELL-FORMED RANGE: on Windows, or
818 /// on a Linux/Android range that is NOT huge-page-size-aligned at both
819 /// endpoints, this method skips the backend call entirely, same as
820 /// [`Self::decommit`], incrementing the same `bench-internals`
821 /// `huge_decommit_attempts` counter (not an intra-doc link: `bench-internals` is excluded from the published docs.rs feature set) and returning
822 /// `Ok(DecommitOutcome::Skipped)`. On Linux/Android kernel >= 5.18 (task
823 /// #1140), a well-formed range that IS huge-page-size-aligned at both
824 /// endpoints instead forwards to the real backend (same as a non-huge
825 /// reservation) and returns whatever that call reports —
826 /// [`DecommitOutcome::Advised`] or [`DecommitOutcome::Refused`], never
827 /// `Err`, per the "best-effort" note below, but now backed by a real
828 /// attempt rather than a guaranteed skip. A malformed range is `Err` even
829 /// on a huge reservation: validation runs before the skip/forward
830 /// decision, so neither the counter nor the real backend ever sees a
831 /// malformed range (task #1084/M3).
832 ///
833 /// Decommit is best-effort by nature; use
834 /// [`Self::decommit_reclaims_and_zeroes`] to learn what the platform
835 /// actually does. `Refused` is reported through the `Ok` payload, not as
836 /// an `Err` of the outer `Result` — see the free [`try_decommit`](crate::try_decommit)'s own
837 /// `# Errors` section for why the outer `Result` stays reserved for
838 /// caller-contract validity.
839 ///
840 /// **The Linux/Android >= 5.18 eligible-forward path requires the
841 /// `huge-pages` feature (task #1156, finding F10); [`Self::is_huge`]
842 /// does not — it is a pure query, always compiled.** Without
843 /// `huge-pages` enabled, this method always takes the
844 /// skip-and-return-`Ok(DecommitOutcome::Skipped)` path on a huge-flagged
845 /// reservation, on every platform, regardless of range or kernel
846 /// version. **A huge-flagged reservation cannot even be constructed
847 /// without `huge-pages`, as of task #1172/M1-hybrid** —
848 /// [`Self::from_raw_parts`] now requires the feature to accept
849 /// `granted_huge: true` — see [`Self::decommit`]'s doc and
850 /// [`Self::from_raw_parts`]'s "Correctness contract" section for the
851 /// full explanation.
852 pub fn try_decommit(&mut self, start: usize, end: usize) -> Result<DecommitOutcome, VmemError> {
853 // Bounds check: the range must be within the reservation's usable span.
854 if end > self.len() {
855 return Err(VmemError::invalid_argument());
856 }
857 // Range-contract validation BEFORE the huge-page skip (task #1084,
858 // finding M3). The huge early-return below used to sit ahead of ALL
859 // validation, so on a reservation with `is_huge() == true` a
860 // malformed range — the exact input a caller uses the fallible form
861 // to detect — was answered `Ok(())`, contradicting both this
862 // method's own `Err` contract and the free `try_decommit`'s
863 // validate-first order. The three conditions mirror the free
864 // function's private `decommit_range_is_well_formed`
865 // (`api/decommit.rs`) — this is now the ONLY place in this method's
866 // call chain that reads `page_size_or_poison()` (task #1180/P2: the
867 // free `try_decommit` used to re-validate the same range a second
868 // time on the forwarded path; `dispatch_try_decommit` below takes an
869 // already-validated non-empty range and does no page-size read of
870 // its own) — so the two layers cannot drift apart silently —
871 // `method_try_decommit_reports_malformed_range_on_huge_flagged_
872 // reservation` and `method_try_decommit_reports_violations_and_
873 // never_panics` (tests/reservation_decommit_contract.rs) pin the
874 // agreement, on huge-flagged and ordinary reservations
875 // respectively.
876 let ps = page_size_or_poison();
877 // Failed OS page-size query: fail closed with the OS-side no-code
878 // error, BEFORE the huge skip and the range validation — mirroring
879 // the free `try_decommit` (NOT `invalid_argument`; the caller's
880 // arguments are not at fault). See `page_size`'s "If the one-time
881 // OS query fails" paragraph.
882 if ps == PAGE_SIZE_QUERY_FAILED {
883 return Err(VmemError::os_refusal_unknown_code());
884 }
885 if start > end || !start.is_multiple_of(ps) || !end.is_multiple_of(ps) {
886 return Err(VmemError::invalid_argument());
887 }
888 if start == end {
889 // Well-formed empty range: a deliberate no-op, on every
890 // reservation kind including huge — no backend call, no huge
891 // eligibility check, no counter increment.
892 return Ok(DecommitOutcome::Skipped);
893 }
894 // Huge-page reservations (finding R6-7, revised task #1140): a
895 // WELL-FORMED range that reaches this point (validated above) is still
896 // not guaranteed to actually decommit anything — see `Self::decommit`'s
897 // doc comment for the Windows-vs-Linux/Android split this mirrors.
898 // Whether the backend call is skipped (no-op) or actually issued
899 // (Linux/Android 5.18+, huge-aligned range), this method's `Err`
900 // contract is "the range was well-formed", not "the OS actually
901 // reclaimed anything" — the free `try_decommit` deliberately does not
902 // report OS refusal/ignore as an `Err` either, and this decision
903 // applies equally to "the OS was never even asked" (Windows, or a
904 // page-size-but-not-huge-size-granular range on Linux/Android) and
905 // "the OS was asked and may have declined" (any ordinary
906 // reservation) — both now distinguishable through the `Ok` payload
907 // instead of being silently folded together.
908 if self.is_huge() {
909 #[cfg(all(
910 not(miri),
911 not(aligned_vmem_mock),
912 any(target_os = "linux", target_os = "android"),
913 feature = "huge-pages"
914 ))]
915 if crate::os::linux_huge_range_is_madvise_eligible(start, end) {
916 // SAFETY: `self.as_ptr()` is a valid reservation base, and
917 // `[start, end)` was just validated as well-formed, non-empty,
918 // and in-span.
919 return Ok(unsafe { dispatch_try_decommit(self.as_ptr(), start, end) });
920 }
921 // Same reasoning and same cfg placement rule as `Self::decommit`
922 // above — the `if`/`return` are unconditional, only the counter is
923 // gated.
924 #[cfg(feature = "bench-internals")]
925 HUGE_DECOMMIT_ATTEMPTS.fetch_add(1, Ordering::Relaxed);
926 return Ok(DecommitOutcome::Skipped);
927 }
928 // SAFETY: `self.as_ptr()` is a valid reservation base, and we've just
929 // verified `[start, end)` is within `self.len()`, well-formed, and
930 // non-empty.
931 Ok(unsafe { dispatch_try_decommit(self.as_ptr(), start, end) })
932 }
933
934 /// Lazy decommit variant: hint the OS it MAY reclaim `[start, end)` under memory
935 /// pressure, cheaper than [`Self::decommit`] (Linux `MADV_FREE`, macOS/iOS
936 /// `MADV_FREE_REUSABLE`, FreeBSD/DragonFly `MADV_FREE`, NetBSD/OpenBSD
937 /// `MADV_FREE`, other Unix (including tvOS/watchOS) falls back to `MADV_DONTNEED`;
938 /// Windows falls back to the eager [`Self::decommit`] path, which has no lazy equivalent).
939 ///
940 /// This is the safe, bounds-checked alternative to the free [`decommit_lazy`]
941 /// function for callers already holding a `Reservation`. It delegates to the
942 /// underlying implementation with `self.as_ptr()` as base and automatically
943 /// ensures `[start, end)` is within the reservation's usable span.
944 ///
945 /// `start` and `end` must be multiples of the runtime page size
946 /// ([`page_size()`](crate::page_size::page_size)); an empty or
947 /// out-of-bounds (`end > self.len()`) range is a no-op, and a VIOLATED
948 /// range (`start > end`, or a misaligned endpoint) is a silent no-op on
949 /// EVERY build profile — the deliberate eager/lazy asymmetry settled by
950 /// task #1072: the eager [`Self::decommit`] trips a debug-build
951 /// tripwire, this lazy variant has none on any profile.
952 ///
953 /// See [`decommit_lazy`] for the platform-specific cost inversion on macOS/iOS
954 /// (this variant actually drops RSS immediately there, unlike the eager path)
955 /// and other caveats. Under the `bench-internals` feature, the
956 /// `huge_decommit_attempts` counter (not an intra-doc link: `bench-internals` is excluded from the published docs.rs feature set) is incremented when decommit is called
957 /// on a huge-page reservation (same logic as `Self::decommit`).
958 pub fn decommit_lazy(&mut self, start: usize, end: usize) {
959 // Bounds check: the range must be within the reservation's usable span.
960 if end > self.len() {
961 return;
962 }
963 // Huge-page reservations: skip the backend call entirely (finding R6-7).
964 // Same reasoning, same cfg placement rule as `Self::decommit` above —
965 // the `if`/`return` are unconditional, only the counter is gated.
966 //
967 // Deliberately NOT extended to match `Self::decommit`'s task #1140
968 // Linux-5.18+ carve-out: `man 2 madvise` documents `MADV_DONTNEED`
969 // gaining HugeTLB support in 5.18, but says nothing of the kind for
970 // `MADV_FREE` (the backend `decommit_lazy` uses) — the lazy advice
971 // family is a different kernel code path with its own support
972 // history, and this crate does not assume one advice value's support
973 // change implies another's without a citation. Windows large pages
974 // remain a `MEM_DECOMMIT` no-op unconditionally either way (there is
975 // no lazy/eager split on Windows — see this method's own rustdoc).
976 if self.is_huge() {
977 #[cfg(feature = "bench-internals")]
978 HUGE_DECOMMIT_ATTEMPTS.fetch_add(1, Ordering::Relaxed);
979 return;
980 }
981 // SAFETY: same safety argument as `decommit` above.
982 unsafe { decommit_lazy(self.as_ptr(), start, end) };
983 }
984
985 /// Recommit pages `[start, end)` previously passed to [`Self::decommit`].
986 ///
987 /// This is the safe, bounds-checked alternative to the free [`recommit`]
988 /// function for callers already holding a `Reservation`. It delegates to
989 /// the underlying implementation with `self.as_ptr()` as base and automatically
990 /// ensures `[start, end)` is within the reservation's usable span.
991 ///
992 /// Returns `true` if the range is now committed (or the call was a well-formed
993 /// no-op — an empty PAGE-ALIGNED range, `start == end`), and `false` if the
994 /// OS refused to
995 /// commit the pages (commit-charge exhaustion / true OOM) OR the offsets
996 /// violated the contract below. On `false` the caller MUST NOT write into
997 /// `[start, end)`. Never panics. For the cause use [`Self::try_recommit`].
998 ///
999 /// `start` and `end` must be multiples of the runtime page size ([`page_size()`](crate::page_size::page_size)).
1000 /// A well-formed no-op (an empty PAGE-ALIGNED range, `start == end`)
1001 /// returns `true`; any other contract violation (misaligned, or
1002 /// `start > end`, or `end > self.len()`) returns `false`.
1003 #[must_use]
1004 pub fn recommit(&mut self, start: usize, end: usize) -> bool {
1005 // Bounds check: the range must be within the reservation's usable span.
1006 if end > self.len() {
1007 return false;
1008 }
1009 // SAFETY: `self.as_ptr()` is a valid reservation base, and we've just
1010 // verified `[start, end)` is within `self.len()`. The free function's
1011 // own contract (multiples of page_size(), etc.) is validated inside it.
1012 unsafe { recommit(self.as_ptr(), start, end) }
1013 }
1014
1015 /// Fallible [`Self::recommit`]: `Ok(())` if the range is now committed
1016 /// (or was a well-formed no-op), `Err(VmemError::invalid_argument())` if the
1017 /// offsets violated the contract (misaligned, or `start > end`, or `end > self.len()`),
1018 /// `Err(VmemError)` carrying the OS cause on genuine commit failure.
1019 ///
1020 /// This is the safe, bounds-checked alternative to the free [`try_recommit`]
1021 /// function for callers already holding a `Reservation`.
1022 pub fn try_recommit(&mut self, start: usize, end: usize) -> Result<(), VmemError> {
1023 // Bounds check: the range must be within the reservation's usable span.
1024 if end > self.len() {
1025 return Err(VmemError::invalid_argument());
1026 }
1027 // SAFETY: same safety argument as `recommit` above.
1028 unsafe { try_recommit(self.as_ptr(), start, end) }
1029 }
1030
1031 /// Commit pages `[start, end)` within this reservation.
1032 ///
1033 /// This is the safe, bounds-checked alternative to the free [`commit_range`]
1034 /// function for callers already holding a `Reservation`. It delegates to
1035 /// the underlying implementation with `self.as_ptr()` as base and automatically
1036 /// ensures `[start, end)` is within the reservation's usable span.
1037 ///
1038 /// After a [`reserve_aligned_lazy`](crate::api::reserve_aligned_lazy) call that left some pages reserved-but-uncommitted,
1039 /// `commit_range` commits exactly the requested sub-range so it becomes writable.
1040 ///
1041 /// Returns `true` if the range is now committed, `false` if the OS refused
1042 /// (commit-charge exhaustion / true OOM) OR the offsets violated the contract
1043 /// above. On `false` the caller MUST NOT write into the range. Never panics.
1044 /// For the cause use [`Self::try_commit_range`].
1045 ///
1046 /// `start` and `end` must be multiples of the runtime page size ([`page_size()`](crate::page_size::page_size)).
1047 /// A well-formed no-op (an empty PAGE-ALIGNED range, `start == end`)
1048 /// returns `true`; any other contract violation (misaligned, or
1049 /// `start > end`, or `end > self.len()`) returns `false`.
1050 #[must_use]
1051 #[cfg(feature = "lazy-commit")]
1052 #[cfg_attr(docsrs, doc(cfg(feature = "lazy-commit")))]
1053 pub fn commit_range(&mut self, start: usize, end: usize) -> bool {
1054 // Bounds check: the range must be within the reservation's usable span.
1055 if end > self.len() {
1056 return false;
1057 }
1058 // SAFETY: same safety argument as `recommit` above.
1059 unsafe { commit_range(self.as_ptr(), start, end) }
1060 }
1061
1062 /// Fallible [`Self::commit_range`]: `Ok(())` on success (or was a well-formed no-op),
1063 /// `Err(VmemError::invalid_argument())` if the offsets violated the contract
1064 /// (misaligned, or `start > end`, or `end > self.len()`), `Err(VmemError)` carrying
1065 /// the OS cause on genuine commit failure.
1066 ///
1067 /// This is the safe, bounds-checked alternative to the free [`try_commit_range`]
1068 /// function for callers already holding a `Reservation`.
1069 #[cfg(feature = "lazy-commit")]
1070 #[cfg_attr(docsrs, doc(cfg(feature = "lazy-commit")))]
1071 pub fn try_commit_range(&mut self, start: usize, end: usize) -> Result<(), VmemError> {
1072 // Bounds check: the range must be within the reservation's usable span.
1073 if end > self.len() {
1074 return Err(VmemError::invalid_argument());
1075 }
1076 // SAFETY: same safety argument as `recommit` above.
1077 unsafe { try_commit_range(self.as_ptr(), start, end) }
1078 }
1079
1080 /// Wrap a pre-existing OS reservation (e.g. one obtained from
1081 /// `VirtualAllocExNuma` or another platform-specific allocator that
1082 /// `reserve_aligned` does not call directly) in a [`Reservation`] handle.
1083 ///
1084 /// The handle then participates in the normal RAII lifecycle: on `Drop`
1085 /// (or via [`release`](crate::api::release)) the underlying reservation is returned to the OS
1086 /// using the platform-appropriate release routine
1087 /// (`VirtualFree(MEM_RELEASE)` on Windows, `munmap` on Unix,
1088 /// `std::alloc::dealloc` on miri).
1089 ///
1090 /// This is **not** the inverse of [`into_parts`](Self::into_parts): that
1091 /// method returns only 3 of the 6 fields this constructor requires
1092 /// (`reservation_ptr, reservation_len, align`), discarding `base`, `len`,
1093 /// and `granted_huge` entirely. [`into_parts`](Self::into_parts)'s true structural complement
1094 /// is [`release`](crate::api::release), whose signature is exactly the 3-tuple `into_parts`
1095 /// returns — that is the intended matched pair for "take ownership out of
1096 /// RAII, then give it back to the OS manually". `from_raw_parts` is a
1097 /// separate, more general constructor for the cross-crate handoff pattern:
1098 /// a sibling crate (`numa-shim` on Windows) issues a platform-specific
1099 /// reservation call that `aligned-vmem` itself does not wrap, then adopts
1100 /// the result via this constructor — it needs `base`/`len` too because the
1101 /// adopted reservation's usable span need not start at the OS reservation's
1102 /// own base (this crate over-reserves `size + align` and keeps the full
1103 /// mapping whenever the exact-size fast path misses, or on Windows when
1104 /// `align > 64 KiB`, which is exactly that shape).
1105 ///
1106 /// # Safety
1107 ///
1108 /// This section covers ONLY memory-safety preconditions: liveness,
1109 /// exclusive ownership, pointer provenance, and exact-once release. A
1110 /// violation here is undefined behaviour. Functional/behavioral
1111 /// requirements — whether `granted_huge` accurately describes the
1112 /// mapping, and Windows commit-state compatibility — are NOT memory-
1113 /// safety preconditions and live in the "Correctness contract" section
1114 /// below instead (task #1172/M3: this section used to mix both kinds
1115 /// together, which made it impossible to state honestly that this
1116 /// crate's own integration tests deliberately violate some of the
1117 /// mixed-in conditions while remaining sound — see that section's
1118 /// opening paragraph for why that is not a contradiction).
1119 ///
1120 /// All six values must describe a **live, exclusively-owned OS
1121 /// reservation** compatible with `aligned-vmem`'s release path:
1122 ///
1123 /// - `base` is the *aligned usable* start; non-null, valid for `len` bytes,
1124 /// aligned to `align`. For correct `decommit`/`decommit_lazy` behavior,
1125 /// `base` must also be aligned to the runtime [`page_size()`](crate::page_size::page_size) (not just
1126 /// the compile-time [`PAGE`]). On systems with non-4 KiB pages (e.g., 16 KiB on
1127 /// Apple Silicon), passing a 4 KiB-aligned `base` will cause `decommit`,
1128 /// `decommit_lazy`, or `munmap` calls to fail silently or return `EINVAL`.
1129 /// **This alignment to page_size() is NOT checked by the constructor's
1130 /// `assert!`** — it is the caller's responsibility to ensure it.
1131 /// - `len` is the usable span size, a non-zero multiple of [`PAGE`].
1132 /// - `reservation` is the *underlying OS reservation* start (often equal
1133 /// to `base`, but may be lower because the reservation is over-reserved
1134 /// to achieve alignment and the full mapping is kept). For correct OS
1135 /// release behavior, it must be aligned to the runtime [`page_size()`](crate::page_size::page_size).
1136 /// **This alignment to page_size() is NOT checked by the constructor's
1137 /// `assert!`** — it is the caller's responsibility to ensure it.
1138 /// - Under miri specifically, `reservation` — NOT `base` — MUST be the exact
1139 /// pointer returned by a `std::alloc::alloc` call, and that call's `Layout`
1140 /// must equal `Layout::from_size_align(reservation_len, align)`. The miri
1141 /// `release_reservation` reconstructs precisely that `Layout` and hands
1142 /// `reservation` to `std::alloc::dealloc`, which requires the pointer to be
1143 /// the one `alloc` returned and the layout to match exactly; anything else
1144 /// is undefined behaviour, not a leak.
1145 ///
1146 /// The distinction between `reservation` and `base` is load-bearing here
1147 /// and is why this bullet names one and not the other: they are SEPARATE
1148 /// parameters, and `base` MAY sit at a non-zero offset inside the region
1149 /// `reservation` points at whenever the caller obtained that region with
1150 /// extra slack to satisfy alignment. Satisfying the provenance
1151 /// requirement at `base` while `reservation` points somewhere else is
1152 /// exactly the mistake this wording exists to prevent. (This crate's own
1153 /// miri backend returns `base == reservation`, so the distinction never
1154 /// bites internally — which is what makes it easy to get wrong for a
1155 /// caller-supplied pair.)
1156 ///
1157 /// This requirement is specific to the miri backend; the Windows and Unix
1158 /// backends release by address and do not track allocator provenance.
1159 /// It complements — and does not restate — the `reservation_len`
1160 /// precision rule below: that one governs the SIZE, this one governs
1161 /// WHICH POINTER and WHERE THE MEMORY CAME FROM.
1162 /// - `reservation_len` must cover the underlying OS mapping/allocation.
1163 /// The required PRECISION differs per backend, and is spelled out here
1164 /// because the two halves of this rule used to contradict each other
1165 /// (task #1035, finding F9: this bullet said an undersized value "leaks
1166 /// memory (Unix)", while the "Important" note below said under-reporting
1167 /// on a large-page host is "harmless for correctness" — both about Unix):
1168 /// - **Native Unix, ORDINARY (non-huge) mapping:** `release` passes this
1169 /// value straight to `munmap`, which ROUNDS THE LENGTH UP to a whole
1170 /// page. A value short of the true mapping by less than one runtime
1171 /// page therefore still unmaps the whole mapping and is harmless —
1172 /// that is exactly the case the "Important" note below describes, and
1173 /// it is the case this crate itself produces on a host whose page
1174 /// size exceeds [`PAGE`]. What DOES leak is a value short by a whole
1175 /// page or more: those trailing pages stay mapped for the life of the
1176 /// process.
1177 /// - **Native Unix, `granted_huge == true` (task #1172/M1, HugeTLB
1178 /// exception to the paragraph above):** the "rounds up, so a
1179 /// less-than-one-page shortfall is harmless" reasoning does NOT
1180 /// transfer to a HugeTLB mapping. Linux's — and Android's, which
1181 /// shares that kernel interface and is covered by the same `#[cfg]`
1182 /// gate on the assert below — `mmap(2)` "Huge TLB
1183 /// mappings" section requires BOTH `munmap(2)`'s `addr` and `length`
1184 /// to be multiples of the huge page size — an undersized
1185 /// `reservation_len` that is not huge-page-size-aligned gets `EINVAL`
1186 /// from `munmap`, not a rounded-up unmap, and the ENTIRE mapping
1187 /// (plus its pinned physical huge pages) leaks for the life of the
1188 /// process. This crate's own `reserve_aligned_huge` path already
1189 /// documents and upholds this requirement (see
1190 /// `crates/aligned-vmem/src/os/unix.rs`'s huge-page-alignment
1191 /// comments); `from_raw_parts` did not previously carry the same
1192 /// requirement for an ADOPTED huge mapping. See the "2 MiB-multiple
1193 /// requirement" bullet in "Correctness contract" below for the
1194 /// checked form of this requirement.
1195 /// - **miri:** `release` reconstructs a `Layout` from
1196 /// `reservation_len`/`align` and hands it to `std::alloc::dealloc`,
1197 /// which requires the EXACT size the allocation was made with — no
1198 /// rounding, and a mismatch is undefined behaviour rather than a leak.
1199 /// The rounding case above cannot arise here: under `cfg(miri)`
1200 /// `query_os_page_size()` returns [`PAGE`] unconditionally, so
1201 /// `page_size() == PAGE` and there is no larger runtime page to round
1202 /// up to. The exact-size requirement is unqualified under miri.
1203 /// - **Windows:** `VirtualFree(MEM_RELEASE)` ignores the length
1204 /// entirely, so the value is advisory — reporting whatever
1205 /// `Reservation::reservation_len` would report for an equivalent
1206 /// reservation is sufficient.
1207 ///
1208 /// **Important:** On hosts where the OS page size exceeds [`PAGE`]
1209 /// (e.g., 16 KiB on Apple Silicon macOS, 64 KiB on some Linux
1210 /// configurations), `reservation_len` may under-report the actual OS
1211 /// mapping size — `mmap` rounds its length argument up to the page size,
1212 /// so `reserve_aligned(PAGE, PAGE)` actually maps a full 16 KiB page
1213 /// while `reservation_len()` returns `4096`. This is harmless for
1214 /// correctness on an ORDINARY mapping (`munmap` rounds its length
1215 /// argument up the same way; `VirtualFree(MEM_RELEASE)` ignores the
1216 /// length on Windows) — it does NOT apply to a `granted_huge == true`
1217 /// mapping on Linux or Android, per the HugeTLB bullet above — but it
1218 /// means
1219 /// `reservation_len` is a **logical** length, not a measure of the true
1220 /// OS reservation size. It must be a non-zero multiple of [`PAGE`] with
1221 /// `reservation_len >= len + (base - reservation)`.
1222 /// - `align` is a power of two `>= PAGE` and matches the alignment the OS
1223 /// reservation was created with.
1224 /// - `granted_huge` itself carries NO memory-safety precondition: it is
1225 /// stored and read back verbatim by every unsafe operation this
1226 /// constructor, `Drop`, and `release_reservation` perform, and branched
1227 /// on by neither. Its accuracy requirement, its interaction with
1228 /// `reservation_len`'s HugeTLB exception above, and Windows commit-state
1229 /// compatibility are all functional requirements — see "Correctness
1230 /// contract" below.
1231 ///
1232 /// The reservation must be released **exactly once** — by dropping this
1233 /// handle, or by extracting via `into_parts` and calling [`release`](crate::api::release)
1234 /// manually. Constructing two `Reservation` handles over the same OS
1235 /// reservation is undefined behaviour (double release).
1236 ///
1237 /// # Correctness contract
1238 ///
1239 /// These requirements are NOT memory-safety preconditions — violating one
1240 /// changes observable *behavior* (a query result, a dispatch decision, or
1241 /// an OS-level no-op/leak) but never causes undefined behavior by itself.
1242 /// This crate's own integration tests deliberately violate the
1243 /// `granted_huge`-accuracy requirement below (see
1244 /// `tests/reservation_decommit_contract.rs`'s
1245 /// `method_try_decommit_reports_malformed_range_on_huge_flagged_reservation`
1246 /// and `tests/decommit_capability.rs`'s
1247 /// `simulated_huge_flag_drives_the_same_branch_dispatch_on_any_host`) to
1248 /// exercise huge-page branch dispatch without a real hugetlb-configured
1249 /// host — both tests' own SAFETY comments enumerate every reader of the
1250 /// flag and confirm none is memory-safety-relevant. That is a deliberate,
1251 /// reviewed use of this contract's slack, not a bug in either the tests
1252 /// or this documentation; production callers must still pass a truthful
1253 /// `granted_huge`, because the CONSEQUENCE of getting it wrong (below) is
1254 /// real even though it is not UB.
1255 ///
1256 /// - **`granted_huge` accuracy.** MUST accurately reflect whether the OS
1257 /// actually granted huge pages for this reservation. Pass `true` only
1258 /// if the reservation was obtained via a huge-page allocation (e.g.
1259 /// `reserve_aligned_huge`) and the OS confirmed the grant (via
1260 /// `Reservation::is_huge()` or equivalent platform-specific detection).
1261 /// **Consequence of a wrong value:** `Reservation::is_huge()` reports
1262 /// the wrong value, and any decommit-availability decision made from
1263 /// that wrong result is wrong (on huge pages, `decommit` is a silent
1264 /// no-op — RSS does not drop and reads return the old data). This
1265 /// changes DISPATCH and query results, never memory safety. If you
1266 /// cannot determine whether the OS granted huge pages, you MUST pass
1267 /// `false` and use `reserve_aligned` instead. If you KNOW the
1268 /// mapping is a HugeTLB mapping whose granularity is not 2 MiB
1269 /// (e.g. 1 GiB on Linux or Android), NEITHER flag value is legal:
1270 /// `true` violates the 2 MiB-multiple contract below, and `false`
1271 /// does not make the kernel's `munmap` alignment requirement go
1272 /// away — it only misreports `is_huge()` (violating this accuracy
1273 /// bullet) and routes release through ordinary-munmap assumptions,
1274 /// where `munmap(2)` on a `MAP_HUGETLB` mapping still requires
1275 /// `addr` and `length` to be multiples of THAT mapping's huge-page
1276 /// size, so a release whose shape satisfies 2 MiB but not the
1277 /// mapping's real granularity can fail `EINVAL` and leak the entire
1278 /// mapping, including its pinned pages from the (bounded) hugetlb
1279 /// pool. Do not construct a `Reservation` over such a mapping at
1280 /// all. "Can leak", not "will leak": this consequence is reasoned
1281 /// from `man 2 munmap` and this crate's own `os/unix.rs` contract
1282 /// docs (`unix_reserve`'s task-#714 note), not executed in CI — no
1283 /// CI host configures a hugetlb pool larger than 2 MiB, and this
1284 /// crate never creates such a mapping itself, always requesting
1285 /// `MAP_HUGE_2MB` (`crates/aligned-vmem/src/os/unix.rs`).
1286 ///
1287 /// - **2 MiB-multiple requirement, Linux/Android, `granted_huge == true`
1288 /// (task #1172/M1-hybrid).** On `target_os = "linux"` or `"android"`,
1289 /// when `granted_huge` is `true`, this constructor additionally
1290 /// `assert!`s that `len`, `reservation_len`, `reservation`, `base`,
1291 /// and the offset `base - reservation` are all multiples of 2 MiB
1292 /// (this crate's one supported HugeTLB granularity, `MAP_HUGE_2MB`; see
1293 /// `crates/aligned-vmem/src/os/unix.rs`'s `LINUX_HUGE_PAGE_SIZE`). Five
1294 /// names are listed, but only FOUR are independent: `reservation` and
1295 /// `base` both being 2-MiB multiples already implies their difference
1296 /// `base - reservation` is too, so the offset conjunct can never be
1297 /// the one that fails (task #1196/OX6-L1). It stays in the assert
1298 /// anyway — for the panic message's diagnostics, and because it would
1299 /// become load-bearing again if either address conjunct were ever
1300 /// dropped.
1301 /// **Consequence of a violation:** an immediate, loud, attributable
1302 /// panic at the call site — not deferred to `Drop`, and not a silent
1303 /// leak — because a non-2-MiB-aligned `reservation`/`reservation_len`
1304 /// on a real HugeTLB mapping would otherwise make the eventual
1305 /// `munmap` fail `EINVAL` and leak the entire mapping (see the
1306 /// `reservation_len` bullet in `# Safety` above). This assert narrows
1307 /// what `granted_huge == true` is allowed to MEAN through this
1308 /// constructor to "the mapping is in this crate's own 2 MiB HugeTLB
1309 /// format" — it does not by itself prove the memory is really
1310 /// `MAP_HUGETLB`-backed (that remains a `# Safety` precondition the
1311 /// assert cannot check), only that its shape is consistent with being
1312 /// so. **Owner decision (2026-08-20, task #1190): NO.** A HugeTLB
1313 /// mapping whose page granularity is not 2 MiB (e.g. 1 GiB) is NOT
1314 /// supported for adoption through this constructor — this assert is
1315 /// the crate's CURRENT contract, not a temporary narrowing pending a
1316 /// wider one (the decision is recorded in
1317 /// <https://github.com/PHPCraftdream/sefer-alloc/blob/main/docs/CORRECTNESS_OPEN_ITEMS.md>
1318 /// item 90's OPEN QUESTION block). A
1319 /// future "yes", if it ever comes, would arrive as an ADDITIVE new
1320 /// constructor carrying typed huge-granularity metadata — not as a
1321 /// relaxation of this assert presented as a bugfix. That widening
1322 /// would be additive rather than semver-breaking precisely because
1323 /// task #1172 already narrowed what `granted_huge == true` MEANS
1324 /// here to "the mapping is in this crate's own 2 MiB HugeTLB
1325 /// format", so this `bool` stays truthful forever for the one case
1326 /// it admits, and because the adoption surface is structurally
1327 /// extensible where it is public: `Reservation`'s fields are
1328 /// private, and `ReservationParts`, `ReservationFullParts`, and
1329 /// every `mock::Call` variant are `#[non_exhaustive]`.
1330 ///
1331 /// - **`huge-pages` feature required to pass `granted_huge: true` (task
1332 /// #1172/M1-hybrid, closing finding M2 as a consequence).** Passing
1333 /// `granted_huge: true` when this crate is built WITHOUT the
1334 /// `huge-pages` feature is itself a contract violation and `assert!`s
1335 /// immediately, for the same "loud at the call site, not silently
1336 /// divergent later" reason as the 2 MiB bullet above. Before this
1337 /// requirement, a caller who adopted a `MAP_HUGETLB` mapping through
1338 /// their own crate but did not separately enable `huge-pages` in THIS
1339 /// crate's `Cargo.toml` got `is_huge() == true` (accurately reflecting
1340 /// what they passed) while `decommit`/`try_decommit` silently,
1341 /// unconditionally skipped the backend call regardless of range or
1342 /// kernel version (finding M2: the SAME live mapping served or skipped
1343 /// decommit depending on the CONSUMER's Cargo feature set, invisible at
1344 /// the call site). Requiring the feature to accept the flag at all
1345 /// means there is no longer a huge-flagged ADOPTED reservation without
1346 /// `huge-pages` enabled, so the divergent-behavior scenario cannot
1347 /// arise — see [`Self::decommit`]'s doc for the full eligible-forward
1348 /// explanation this closes the gap in.
1349 ///
1350 /// - **Windows commit state.** On Windows, the reservation's commit state
1351 /// (which pages are committed vs. reserved-only) must be compatible
1352 /// with the `granted_huge` value:
1353 ///
1354 /// - If `granted_huge == false`, the reservation may be in any valid
1355 /// commit state: fully committed (created via `reserve_aligned` or
1356 /// the single-call Windows fast path), partially committed (created
1357 /// via the two-call `reserve_aligned_lazy` path), or reserved-only
1358 /// (not a common pattern but valid).
1359 ///
1360 /// - If `granted_huge == true`, the reservation MUST have been created
1361 /// with `MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES` in a single call
1362 /// (the only way Windows grants large pages). The crate itself only
1363 /// produces such reservations via its `reserve_aligned_huge`
1364 /// single-call fast path. The crate's own two-call
1365 /// `reserve_aligned_lazy` path (which issues
1366 /// `VirtualAlloc(MEM_RESERVE)` followed by `VirtualAlloc(MEM_COMMIT)`)
1367 /// is incompatible with `granted_huge == true`, because `MEM_COMMIT`
1368 /// cannot be combined with `MEM_LARGE_PAGES` on a pre-reserved region
1369 /// — MSDN requires all three flags in a single call.
1370 ///
1371 /// **Consequence of a violation:** `Reservation::is_huge()` reports a
1372 /// value inconsistent with the reservation's actual commit state —
1373 /// the same DISPATCH/query-result consequence as the accuracy bullet
1374 /// above, not a new failure mode. If you adopted a reservation from
1375 /// another source and cannot determine whether it was created with the
1376 /// one-call large-page path, you MUST pass `granted_huge == false`.
1377 #[must_use]
1378 pub unsafe fn from_raw_parts(
1379 base: *mut u8,
1380 len: usize,
1381 reservation: *mut u8,
1382 reservation_len: usize,
1383 align: usize,
1384 granted_huge: bool,
1385 ) -> Self {
1386 // Historical notes (task #719, #776, #916):
1387 //
1388 // - task #719: validate the documented `align`/`reservation_len` contract
1389 // HERE, at the unsafe call site, rather than leaving it to surface later
1390 // as a panic inside `Drop::drop` (via the miri backend's
1391 // `Layout::from_size_align(reservation_len, align).expect(...)` in
1392 // `release_reservation`) -- a panic reachable from `Drop` is far more
1393 // dangerous than one at construction time: if this `Reservation` is ever
1394 // dropped while ANOTHER panic is already unwinding the stack, Rust
1395 // aborts the whole process on the second panic. Every other construction
1396 // path in this crate already produces a valid `(align, reservation_len)`
1397 // pair by construction (validated at each public entry point), so this
1398 // check is specific to the caller-supplied values `from_raw_parts`
1399 // accepts. Violating the documented contract is already undefined
1400 // behaviour per this function's own `# Safety` section; panicking
1401 // immediately here converts a silently-deferred hazard into a loud,
1402 // attributable failure at the actual point of misuse.
1403 //
1404 // - task #776 (F2 revision -- round-closing review finding F7): the
1405 // original check validated only `align`, but `Layout::from_size_align`
1406 // also fails when `reservation_len` overflows `isize::MAX` once rounded
1407 // up to `align` -- an `align`-only check left that half of the SAME
1408 // Drop-reachable-panic hazard open (e.g. `from_raw_parts(b, PAGE, r,
1409 // usize::MAX, PAGE)` still constructed successfully and still panicked
1410 // inside `Drop` under miri). The explicit `reservation_len != 0 &&
1411 // reservation_len.is_multiple_of(PAGE)` checks enforce the documented
1412 // nonzero/page-multiple invariants, while `Layout::from_size_align(...).
1413 // is_ok()` catches overflow cases.
1414 //
1415 // - task #916 (H2C3): the comment above previously claimed these checks
1416 // "cover all documented contract violations immediately at the call
1417 // site" -- this was false. Four documented invariants were uncheckable
1418 // from the arguments alone (pointer validity, liveness, exclusivity,
1419 // and exact-once release), but three MORE were cheaply checkable and
1420 // were NOT checked:
1421 // - `len` must be a non-zero multiple of `PAGE` (documented, not checked)
1422 // - `base` must be aligned to `align` (documented, not checked)
1423 // - `reservation <= base` (documented, now checked below via `base_addr >= res_addr`)
1424 // - `reservation_len >= len + (base - reservation)` (documented, not checked)
1425 // All four are now checked explicitly below, leaving only the genuinely
1426 // uncheckable invariants (pointer validity, liveness, exclusivity) as
1427 // unchecked caller responsibilities.
1428 let base_nn = NonNull::new(base).expect("from_raw_parts: base must be non-null");
1429 let res_nn =
1430 NonNull::new(reservation).expect("from_raw_parts: reservation must be non-null");
1431 let base_addr = base.addr();
1432 let res_addr = reservation.addr();
1433 assert!(
1434 align.is_power_of_two()
1435 && align >= PAGE
1436 && reservation_len != 0
1437 && reservation_len.is_multiple_of(PAGE)
1438 && len != 0
1439 && len.is_multiple_of(PAGE)
1440 && base_addr >= res_addr
1441 && base_addr.is_multiple_of(align)
1442 && len
1443 .checked_add(base_addr - res_addr)
1444 .is_some_and(|required| reservation_len >= required)
1445 && std::alloc::Layout::from_size_align(reservation_len, align).is_ok(),
1446 "Reservation::from_raw_parts: \
1447 align must be a power of two >= PAGE; \
1448 reservation_len must be non-zero and a multiple of PAGE; \
1449 len must be non-zero and a multiple of PAGE; \
1450 base must be >= reservation; \
1451 base must be aligned to align; \
1452 reservation_len must be >= len + (base - reservation); \
1453 (reservation_len, align) must form a valid Layout; \
1454 NOTE: alignment to runtime page_size() is NOT checked — \
1455 caller must ensure base/reservation are page_size()-aligned; \
1456 got align={align}, reservation_len={reservation_len}, len={len}, \
1457 base={base:?}, reservation={reservation:?}"
1458 );
1459 // Task #1172 (M1-hybrid, item 90 of docs/CORRECTNESS_OPEN_ITEMS.md):
1460 // narrow what `granted_huge == true` is allowed to MEAN through this
1461 // constructor. Two independent checks, both loud-at-the-call-site
1462 // for the same task #719 reason as the block above (a Drop-reachable
1463 // panic is far more dangerous than one here):
1464 //
1465 // 1. `huge-pages` feature required to accept `granted_huge: true` at
1466 // all (closes finding M2 as a consequence: with no feature there
1467 // are no huge-flagged ADOPTED reservations, so the "same live
1468 // mapping served or skipped depending on the CONSUMER's feature
1469 // set" divergence cannot arise).
1470 #[cfg(not(feature = "huge-pages"))]
1471 assert!(
1472 !granted_huge,
1473 "Reservation::from_raw_parts: granted_huge=true requires this crate's \
1474 `huge-pages` feature to be enabled. Without it, this constructor would \
1475 accept an adopted huge-flagged reservation whose eligible-forward \
1476 decommit path is compiled out, making `is_huge()` report true while \
1477 decommit/try_decommit silently skip on every call regardless of range \
1478 or kernel version -- see Reservation::decommit's rustdoc. Enable \
1479 `huge-pages`, or pass granted_huge: false."
1480 );
1481 // 2. On Linux/Android, when `granted_huge` is true, `len`,
1482 // `reservation_len`, `reservation`, `base`, and the offset
1483 // `base - reservation` must all be 2 MiB multiples (this crate's
1484 // one supported HugeTLB granularity). Linux's `mmap(2)`/`munmap(2)`
1485 // require both the address and the length of a `MAP_HUGETLB`
1486 // mapping's release call to be huge-page-size-aligned; violating
1487 // this on a REAL HugeTLB mapping would make the eventual `munmap`
1488 // fail EINVAL and leak the entire mapping (see the
1489 // `reservation_len` HugeTLB bullet in `# Safety` above). This
1490 // assert cannot by itself prove the memory really is
1491 // `MAP_HUGETLB`-backed (a `# Safety` precondition it has no way
1492 // to check) -- only that its shape is consistent with being so.
1493 // Five names are listed below, but only FOUR are independent
1494 // checks (task #1196/OX6-L1): if `res_addr` and `base_addr` are
1495 // both 2-MiB multiples, their difference is necessarily a 2-MiB
1496 // multiple too, so the offset conjunct can never be the one that
1497 // fails -- it is implied by the two address conjuncts, not a
1498 // fifth independent requirement.
1499 #[cfg(any(target_os = "linux", target_os = "android"))]
1500 if granted_huge {
1501 // 2 MiB: this crate's one supported HugeTLB granularity
1502 // (`MAP_HUGE_2MB`), matching `os::unix::LINUX_HUGE_PAGE_SIZE`.
1503 // Not imported directly: that constant is private to a module
1504 // compiled only under `#[cfg(all(unix, not(miri)))]` plus
1505 // `huge-pages`, narrower than this assert's own gate (Linux/
1506 // Android, any feature set, so the `huge-pages`-off panic above
1507 // fires first and with a clearer message on that combination).
1508 const HUGE_PAGE_SIZE_2MIB: usize = 2 * 1024 * 1024;
1509 assert!(
1510 len.is_multiple_of(HUGE_PAGE_SIZE_2MIB)
1511 && reservation_len.is_multiple_of(HUGE_PAGE_SIZE_2MIB)
1512 && res_addr.is_multiple_of(HUGE_PAGE_SIZE_2MIB)
1513 && base_addr.is_multiple_of(HUGE_PAGE_SIZE_2MIB)
1514 // Implied by the two conjuncts directly above (res_addr
1515 // and base_addr both 2-MiB-aligned => their difference
1516 // is too), so this can never be the failing conjunct
1517 // today. Kept for the panic message's diagnostics and
1518 // as an explicit statement of intent: if either address
1519 // conjunct above is ever weakened or removed, this one
1520 // becomes load-bearing again.
1521 && (base_addr - res_addr).is_multiple_of(HUGE_PAGE_SIZE_2MIB),
1522 "Reservation::from_raw_parts: granted_huge=true on Linux/Android \
1523 requires len, reservation_len, reservation, base, and the offset \
1524 (base - reservation) to ALL be multiples of 2 MiB (this crate's \
1525 supported HugeTLB granularity) -- a non-2-MiB-aligned reservation \
1526 or reservation_len would make munmap(2) fail EINVAL and leak the \
1527 entire mapping on release; \
1528 got len={len}, reservation_len={reservation_len}, \
1529 base={base:?}, reservation={reservation:?}, \
1530 offset={}",
1531 base_addr - res_addr
1532 );
1533 }
1534 Self {
1535 base: base_nn,
1536 len,
1537 reservation: res_nn,
1538 reservation_len,
1539 align,
1540 granted_huge,
1541 }
1542 }
1543}
1544
1545impl Drop for Reservation {
1546 fn drop(&mut self) {
1547 // Record the release for mock observers (RAII path visibility).
1548 #[cfg(aligned_vmem_mock)]
1549 crate::mock::record(crate::mock::Call::Release {
1550 reservation: self.reservation.as_ptr().addr(),
1551 reservation_len: self.reservation_len,
1552 });
1553 // SAFETY: every safe constructor of `Self` (`reserve_aligned` and
1554 // friends) upholds `from_raw_parts`'s `# Safety` contract by
1555 // construction, so that contract covers `self.reservation` /
1556 // `self.reservation_len` / `self.align` regardless of which
1557 // constructor built this handle: `self.reservation` describes a
1558 // live OS reservation valid for `self.reservation_len` bytes,
1559 // release-compatible with the platform backend below (an exact
1560 // `std::alloc::alloc` provenance/`Layout` match under miri; a live
1561 // `mmap`'d region on Unix; a `VirtualAlloc(MEM_RESERVE)` region on
1562 // Windows), and this handle owns it exclusively (no aliasing —
1563 // `Reservation` is `Send` but not `Sync`). Dropping returns the
1564 // entire reservation to the OS exactly once.
1565 unsafe { release_reservation(self.reservation, self.reservation_len, self.align) };
1566 }
1567}
1568
1569// SAFETY (Send): a `Reservation` owns its OS reservation exclusively; moving it
1570// to another thread moves ownership of every byte, leaving no aliasing on the
1571// origin thread. The memory is plain uninitialised bytes (no `Rc`/`Cell`/TLS
1572// affinity).
1573unsafe impl Send for Reservation {}