1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
//! The classified free path: block, page, and segment bookkeeping.
use super::cold::thread_free_cold;
use super::internal::do_local_free_internal_policy;
use crate::free_helpers::{commit_in_place_free, free_large_or_huge_raw, resolve_owner_slot};
use crate::{LocalAllocatorSelector, poison_freed_bytes};
use core::ptr::NonNull;
use mnemosyne_arena::HasSegmentPool;
use mnemosyne_core::constants::PAGE_SIZE;
use mnemosyne_core::policy::AllocPolicy;
use mnemosyne_core::types::{Block, Page, Segment, locate_page, locate_segment};
#[inline(always)]
pub(super) unsafe fn thread_free_classified<
P: AllocPolicy + crate::tls_slot::PolicySlotSelection<B>,
B: HasSegmentPool + LocalAllocatorSelector<B>,
const LAYOUT_PROVES_SMALL: bool,
>(
ptr: *mut u8,
) {
if ptr.is_null() {
return;
}
let ptr_val = ptr as usize;
// SAFETY: `ptr` was previously returned by this allocator, satisfying
// `locate_segment`'s contract; it recovers the live segment header and the
// bounded page index.
let (segment, page_index) = unsafe { locate_segment(ptr) };
// SAFETY: `segment` and `page_index` were validated by `locate_segment`.
// `locate_page` avoids retaining a reference-derived metadata tag across
// the alloc/free boundary of the shared metadata-and-payload mapping.
let page_ptr = unsafe { locate_page(segment, page_index) };
if mnemosyne_prof::is_active() {
unsafe { record_free_profile(ptr, page_ptr, page_index) };
}
// `page_index == 0` short-circuits before the metadata read: page 0 is
// never allocated from, so a zero index means `ptr` is segment-aligned and
// the address `locate_segment` masked to is payload rather than a header.
// Reading `block_size` from it would interpret user bytes as page metadata.
// See `usable_size` for the full argument.
if !LAYOUT_PROVES_SMALL && (page_index == 0 || unsafe { (*page_ptr).block_size } == 0) {
// SAFETY: huge-allocation metadata layout — segment pointer stored one
// slot before the payload; `free_large_or_huge_raw` reads and releases it.
unsafe {
free_large_or_huge_raw::<B>(ptr, P::ENABLE_POISONING, P::POISON_FREE_BYTE);
}
#[cfg(feature = "dealloc-probe")]
crate::dealloc_counters::record(crate::dealloc_counters::DeallocPath::HugeClassifier);
return;
}
// SAFETY: `page_ptr` is the live page metadata `locate_page` recovered for
// `ptr`'s segment and index; `block_size` is written once at page
// initialization and only read here.
debug_assert_eq!(
(ptr_val & (PAGE_SIZE - 1)) % unsafe { (*page_ptr).block_size as usize },
0,
"small free ptr must be aligned to the page's block stride"
);
// Record per-size-class dealloc telemetry before poison overwrites the block.
// `size_to_class_nonzero` is a single table lookup (O(1)); Relaxed atomics
// mean no memory-fence overhead on the hot path.
{
use mnemosyne_core::size_class::size_to_class_nonzero;
let block_size = unsafe { (*page_ptr).block_size as usize };
if let Some(class) = size_to_class_nonzero(block_size) {
crate::bin_stats::record_dealloc(class);
}
}
// SAFETY: `ptr` is the block being freed — `block_size` bytes this free owns
// exclusively until the block re-enters a free list — so the poison write
// stays inside the block.
if P::ENABLE_POISONING {
unsafe { poison_freed_bytes::<P>(ptr, (*page_ptr).block_size as usize) };
}
let block = ptr as *mut Block;
// SAFETY: `segment` is the live mapping `locate_segment` recovered for `ptr`;
// `owner` reads its ownership token, which is immutable while the segment
// is mapped.
let owner = unsafe { Segment::owner(segment) };
let encrypted = unsafe { Segment::free_list_encrypted(segment) };
let randomized =
(P::RANDOMIZE_ALLOCATION && encrypted) || unsafe { (*page_ptr).secondary_free.is_some() };
// Resolve whether the caller owns this segment's allocator, using the shared
// platform-specific logic (SSOT for the Windows same-thread cross-policy case).
let caller_allocator = B::get_allocator_ptr_raw_for_policy::<P>();
// SAFETY: `segment` is the live header from `locate_segment`; the helper
// reads only ownership metadata through raw-pointer projections.
let (is_owner, owner_allocator) =
unsafe { resolve_owner_slot(segment, owner, caller_allocator) };
if is_owner && !owner_allocator.is_null() {
// SAFETY: `page_ptr` is live (above) and this thread owns the segment, so no
// other thread writes `alloc_count` while it is read.
let page_alloc_count = unsafe { (*page_ptr).alloc_count };
if page_alloc_count == 0 {
std::process::abort();
}
// SAFETY: `block` is a user pointer previously returned by the
// allocator; non-nullness is the allocator invariant. Equality
// with `page.free` is the double-free guard.
if Some(unsafe { NonNull::new_unchecked(block) }) == unsafe { (*page_ptr).free }
|| (randomized
&& Some(unsafe { NonNull::new_unchecked(block) })
== unsafe { (*page_ptr).secondary_free })
{
std::process::abort();
}
// `owner_allocator` is the owner token, which by the slot's offset-0
// invariant is also this thread's slot address — so the re-entrancy
// gate is reachable from it without borrowing the allocator.
// SAFETY: the surrounding `is_owner && !owner_allocator.is_null()` was
// just confirmed against `segment.owner`, so this is the current
// thread's own live slot and no concurrent accessors exist.
let is_allocating =
unsafe { crate::tls_slot::LocalAllocatorSlot::<B>::is_allocating(owner_allocator) };
let page_free = unsafe { (*page_ptr).free };
// SAFETY: `segment`/`page_index` locate this page's parent header and its
// key slot, satisfying `cookie_for`'s contract.
let cookie = unsafe { Segment::cookie_for_dynamic(segment, encrypted, page_index) };
if unsafe { (*page_ptr).list_state } != 2 {
// Page is active
// Ask the segment, not the allocator. `is_current` is the owner's
// own mirror of `current_segment`, maintained by
// `set_current_segment`, and this path runs even while the gate is
// raised — reading it through the allocator would need a borrow the
// gate exists to forbid. Same form as the `is_current` reads in
// `occupancy` and the cold path below.
// SAFETY: `is_owner` was confirmed above, so `segment` is this
// thread's live, owned header.
if page_alloc_count > 1 || unsafe { Segment::is_current(segment) } {
// Free in-place (either remains active, or is current segment).
// SAFETY: `block` is non-null by the alloc_count / page.free
// corruption guards above, and `page_alloc_count == page.free`'s
// owning count; the shared commit stays inside this owned page.
unsafe {
commit_in_place_free(
block,
page_ptr,
page_free,
cookie,
encrypted,
page_alloc_count as usize,
randomized,
)
};
#[cfg(feature = "dealloc-probe")]
crate::dealloc_counters::record(crate::dealloc_counters::DeallocPath::InPlaceSmall);
return;
} else if !is_allocating {
// Page is not the current segment and this free empties it. The
// free-list head set, the segment-aware decrement, and the
// active→empty page-list transition are the shared commit in
// `do_local_free_internal`; the caller adds only the re-entrancy
// guard and the sweep-cadence bump around it.
// SAFETY: `owner_allocator` is non-null and belongs to this thread (the
// `is_owner` branch), so flipping its TLS re-entrancy flag is a same-thread
// write on a slot that outlives this call.
unsafe {
crate::tls_slot::LocalAllocatorSlot::<B>::set_allocating(owner_allocator, true)
};
// Borrow only now: the gate was false and is now raised, so no
// other `&mut` to this cache is live.
// SAFETY: this thread's own slot (offset-0 invariant), gate
// checked and raised.
let alloc = unsafe {
crate::tls_slot::LocalAllocatorSlot::<B>::allocator_mut(owner_allocator)
};
// SAFETY: `block`/`page`/`segment`/`page_index` are the validated
// free-path inputs (guards above) with `alloc` the owning
// allocator — exactly `do_local_free_internal`'s contract.
let _became_empty = unsafe {
do_local_free_internal_policy::<P, B>(
alloc, block, page_ptr, segment, page_index,
)
};
// SAFETY: `alloc` is the exclusively-borrowed owning allocator
// with `is_allocating` raised, the precondition of the cold sweep.
unsafe { alloc.record_defrag_operation(true) };
unsafe {
crate::tls_slot::LocalAllocatorSlot::<B>::set_allocating(owner_allocator, false)
};
#[cfg(feature = "dealloc-probe")]
crate::dealloc_counters::record(
crate::dealloc_counters::DeallocPath::ActiveFreeLastBlock,
);
return;
}
} else if !is_allocating {
// Page is full, transitions to active (count > 1 is guaranteed since
// max_blocks >= 8, so it never empties directly). This is the
// full→active branch of the shared `do_local_free_internal` commit.
// SAFETY: this thread's own slot (offset-0 invariant), and the gate
// read false, so no other `&mut` to this cache is live.
let alloc =
unsafe { crate::tls_slot::LocalAllocatorSlot::<B>::allocator_mut(owner_allocator) };
// SAFETY: as above — validated free-path inputs and the owning
// `alloc`, satisfying `do_local_free_internal`'s contract.
let _became_empty = unsafe {
do_local_free_internal_policy::<P, B>(alloc, block, page_ptr, segment, page_index)
};
#[cfg(feature = "dealloc-probe")]
crate::dealloc_counters::record(crate::dealloc_counters::DeallocPath::FullToActive);
return;
}
}
// SAFETY: `ptr`/`page_ptr`/`block` are the function's validated
// contract inputs from the embodiment of `thread_free`'s `// # Safety`
// rustdoc; the `#[cold]` helper handles the cross-thread / re-entrant
// push path.
unsafe { thread_free_cold::<B>(ptr, page_ptr, block) };
}
#[cold]
#[inline(never)]
unsafe fn record_free_profile(ptr: *mut u8, page: *const Page, page_index: usize) {
let block_size = unsafe { (*page).block_size };
let size = if page_index == 0 || block_size == 0 {
// Large/huge allocation: recover the size from the shared metadata-slot
// accessor.
// SAFETY: `page_index == 0 || block_size == 0` identifies a large/huge
// allocation whose metadata slot precedes `ptr`, satisfying
// `huge_allocation_size`'s precondition.
unsafe { crate::usable_size::huge_allocation_size(ptr) }
} else {
block_size as usize
};
mnemosyne_prof::on_free(ptr, size);
}