use core::ffi::c_void;
use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
use super::{Alloc, MemConfig, PrimError, TlsDtor, align_up};
const FERR: PrimError = 0xF13D;
pub const FERR_TOO_SMALL: PrimError = 0xF13E;
pub const FERR_GEOMETRY: PrimError = 0xF13F;
pub const FERR_REGISTERED: PrimError = 0xF140;
pub const MIN_REGION: usize = crate::types::SEGMENT_SIZE + FIXED_PAGE;
const FIXED_PAGE: usize = 4096;
const MAX_EXTENTS: usize = if cfg!(ra_max_extents = "8") {
8
} else if cfg!(ra_max_extents = "16") {
16
} else if cfg!(ra_max_extents = "64") {
64
} else {
32
};
static REGION_BASE: AtomicUsize = AtomicUsize::new(0);
static REGION_LEN: AtomicUsize = AtomicUsize::new(0);
static EXT_BASE: [AtomicUsize; MAX_EXTENTS] = [const { AtomicUsize::new(0) }; MAX_EXTENTS];
static EXT_LEN: [AtomicUsize; MAX_EXTENTS] = [const { AtomicUsize::new(0) }; MAX_EXTENTS];
static EXT_COUNT: AtomicUsize = AtomicUsize::new(0);
static LOCK: AtomicBool = AtomicBool::new(false);
struct Guard(&'static AtomicBool);
impl Guard {
fn acquire(lock: &'static AtomicBool) -> Self {
while lock
.compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
#[cfg(ra_single_threaded)]
if lock.load(Ordering::Relaxed) {
reentered();
}
core::hint::spin_loop();
}
Self(lock)
}
}
#[cfg(ra_single_threaded)]
#[cold]
#[inline(never)]
fn reentered() -> ! {
panic!(
"rusty_alloc: the allocator was re-entered. On a target built with \
--cfg ra_single_threaded nothing else can hold this lock, so this is \
almost certainly an interrupt handler that allocated while the main \
context was inside the allocator. prim::fixed's lock is NOT \
reentrant: do not allocate in an ISR. Note that ra_single_threaded \
means single CONTEXT, and an interrupt handler is a second context on \
one core."
)
}
impl Drop for Guard {
fn drop(&mut self) {
self.0.store(false, Ordering::Release);
}
}
#[must_use]
pub const fn usable_bytes(base: usize, len: usize) -> usize {
let seg = crate::types::SEGMENT_SIZE;
let Some(end) = base.checked_add(len) else {
return 0;
};
let Some(run_up) = base.checked_add(seg - 1) else {
return 0;
};
let first = run_up & !(seg - 1);
if first >= end {
return 0;
}
let avail = end - first;
if avail <= FIXED_PAGE {
return 0;
}
((avail - FIXED_PAGE) / seg) * seg
}
#[must_use]
pub const fn good_region_size(budget: usize) -> usize {
let seg = crate::types::SEGMENT_SIZE;
if budget < MIN_REGION {
return 0;
}
((budget - FIXED_PAGE) / seg) * seg + FIXED_PAGE
}
#[must_use]
pub const fn region_for(usable: usize) -> usize {
let seg = crate::types::SEGMENT_SIZE;
let segments = if usable == 0 { 1 } else { usable.div_ceil(seg) };
segments * seg + FIXED_PAGE
}
pub fn init_region(region: &'static mut [u8]) -> Result<(), PrimError> {
let len = region.len();
if len < FIXED_PAGE {
return Err(FERR_TOO_SMALL);
}
let base = region.as_mut_ptr().expose_provenance();
if usable_bytes(base, len) == 0 {
return Err(FERR_GEOMETRY);
}
let _g = Guard::acquire(&LOCK);
if REGION_LEN.load(Ordering::Relaxed) != 0 {
return Err(FERR_REGISTERED);
}
install_region(base, len);
Ok(())
}
fn install_region(base: usize, len: usize) {
REGION_BASE.store(base, Ordering::Relaxed);
REGION_LEN.store(len, Ordering::Relaxed);
EXT_BASE[0].store(base, Ordering::Relaxed);
EXT_LEN[0].store(len, Ordering::Relaxed);
EXT_COUNT.store(1, Ordering::Relaxed);
}
#[must_use]
pub fn region_contains(addr: usize) -> bool {
let base = REGION_BASE.load(Ordering::Relaxed);
let len = REGION_LEN.load(Ordering::Relaxed);
len != 0 && addr >= base && addr - base < len
}
#[must_use]
pub fn region_stats() -> (usize, usize, usize) {
let _g = Guard::acquire(&LOCK);
let total = REGION_LEN.load(Ordering::Relaxed);
let free: usize = (0..EXT_COUNT.load(Ordering::Relaxed))
.map(|i| EXT_LEN[i].load(Ordering::Relaxed))
.sum();
(total - free, free, total)
}
fn remove_at(idx: usize) {
let n = EXT_COUNT.load(Ordering::Relaxed);
for i in idx..n - 1 {
EXT_BASE[i].store(EXT_BASE[i + 1].load(Ordering::Relaxed), Ordering::Relaxed);
EXT_LEN[i].store(EXT_LEN[i + 1].load(Ordering::Relaxed), Ordering::Relaxed);
}
EXT_COUNT.store(n - 1, Ordering::Relaxed);
}
fn insert_at(idx: usize, base: usize, len: usize) {
let n = EXT_COUNT.load(Ordering::Relaxed);
let mut i = n;
while i > idx {
EXT_BASE[i].store(EXT_BASE[i - 1].load(Ordering::Relaxed), Ordering::Relaxed);
EXT_LEN[i].store(EXT_LEN[i - 1].load(Ordering::Relaxed), Ordering::Relaxed);
i -= 1;
}
EXT_BASE[idx].store(base, Ordering::Relaxed);
EXT_LEN[idx].store(len, Ordering::Relaxed);
EXT_COUNT.store(n + 1, Ordering::Relaxed);
}
const _: () = assert!(
crate::types::SEGMENT_SLICE_SIZE >= FIXED_PAGE,
"SEGMENT_SLICE_SIZE must be >= FIXED_PAGE or good_size over-promises"
);
pub(super) fn mem_init() -> MemConfig {
MemConfig {
page_size: FIXED_PAGE,
alloc_granularity: FIXED_PAGE,
large_page_size: 0,
has_overcommit: false,
has_partial_free: true,
}
}
fn place(base: usize, len: usize, size: usize, align: usize, from_top: bool) -> Option<usize> {
if size > len {
return None;
}
let at = if from_top {
(base + len - size) & !(align - 1)
} else {
align_up(base, align)
};
if at < base || at.saturating_add(size) > base + len {
return None;
}
Some(at)
}
pub(super) unsafe fn alloc(
size: usize,
try_alignment: usize,
_commit: bool,
_allow_large: bool,
) -> Result<Alloc, PrimError> {
if size == 0 {
return Err(FERR);
}
let align = try_alignment.max(FIXED_PAGE);
let size = align_up(size, FIXED_PAGE);
let _g = Guard::acquire(&LOCK);
if REGION_LEN.load(Ordering::Relaxed) == 0 {
return Err(FERR);
}
let from_top = align == FIXED_PAGE;
let n = EXT_COUNT.load(Ordering::Relaxed);
for k in 0..n {
let i = if from_top { n - 1 - k } else { k };
let base = EXT_BASE[i].load(Ordering::Relaxed);
let len = EXT_LEN[i].load(Ordering::Relaxed);
let Some(aligned) = place(base, len, size, align, from_top) else {
continue;
};
let head = aligned - base;
let tail = (base + len) - (aligned + size);
if head > 0 && tail > 0 && n + 1 > MAX_EXTENTS {
return Err(FERR);
}
remove_at(i);
let mut at = i;
if head > 0 {
insert_at(at, base, head);
at += 1;
}
if tail > 0 {
insert_at(at, aligned + size, tail);
}
return Ok(Alloc {
ptr: core::ptr::with_exposed_provenance_mut(aligned),
is_large: false,
is_zero: false,
});
}
Err(FERR)
}
pub(super) unsafe fn free(ptr: *mut u8, size: usize) -> Result<(), PrimError> {
if size == 0 {
return Ok(());
}
let base = ptr.expose_provenance();
let size = align_up(size, FIXED_PAGE);
let _g = Guard::acquire(&LOCK);
let rbase = REGION_BASE.load(Ordering::Relaxed);
let rlen = REGION_LEN.load(Ordering::Relaxed);
if rlen == 0 || base < rbase || base + size > rbase + rlen {
return Err(FERR);
}
let n = EXT_COUNT.load(Ordering::Relaxed);
let idx = EXT_BASE[..n]
.iter()
.position(|e| e.load(Ordering::Relaxed) > base)
.unwrap_or(n);
let prev_touches = idx > 0 && {
let pb = EXT_BASE[idx - 1].load(Ordering::Relaxed);
pb + EXT_LEN[idx - 1].load(Ordering::Relaxed) == base
};
let next_touches = idx < n && EXT_BASE[idx].load(Ordering::Relaxed) == base + size;
match (prev_touches, next_touches) {
(true, true) => {
let grown = EXT_LEN[idx - 1].load(Ordering::Relaxed)
+ size
+ EXT_LEN[idx].load(Ordering::Relaxed);
EXT_LEN[idx - 1].store(grown, Ordering::Relaxed);
remove_at(idx);
}
(true, false) => {
let grown = EXT_LEN[idx - 1].load(Ordering::Relaxed) + size;
EXT_LEN[idx - 1].store(grown, Ordering::Relaxed);
}
(false, true) => {
EXT_BASE[idx].store(base, Ordering::Relaxed);
let grown = EXT_LEN[idx].load(Ordering::Relaxed) + size;
EXT_LEN[idx].store(grown, Ordering::Relaxed);
}
(false, false) => {
if n >= MAX_EXTENTS {
return Err(FERR);
}
insert_at(idx, base, size);
}
}
Ok(())
}
#[allow(
clippy::unnecessary_wraps,
reason = "the prim backends share one signature; a no-op backend still returns the contract's Result"
)]
pub(super) unsafe fn commit(_ptr: *mut u8, _size: usize) -> Result<bool, PrimError> {
Ok(false)
}
#[allow(
clippy::unnecessary_wraps,
reason = "the prim backends share one signature; a no-op backend still returns the contract's Result"
)]
pub(super) unsafe fn decommit(_ptr: *mut u8, _size: usize) -> Result<bool, PrimError> {
Ok(false)
}
#[allow(
clippy::unnecessary_wraps,
reason = "the prim backends share one signature; a no-op backend still returns the contract's Result"
)]
pub(super) unsafe fn reset(_ptr: *mut u8, _size: usize) -> Result<(), PrimError> {
Ok(())
}
pub(super) unsafe fn protect(_ptr: *mut u8, _size: usize, _on: bool) -> Result<(), PrimError> {
Err(FERR)
}
pub(super) fn numa_node_count() -> usize {
1
}
#[inline]
pub(super) fn thread_id() -> usize {
1
}
static CLOCK_LOCK: AtomicBool = AtomicBool::new(false);
static TICK_LO: AtomicU32 = AtomicU32::new(0);
static TICK_HI: AtomicU32 = AtomicU32::new(0);
pub(super) fn clock_now() -> u64 {
let _g = Guard::acquire(&CLOCK_LOCK);
let (lo, carry) = TICK_LO.load(Ordering::Relaxed).overflowing_add(1);
TICK_LO.store(lo, Ordering::Relaxed);
let hi = if carry {
let h = TICK_HI.load(Ordering::Relaxed).wrapping_add(1);
TICK_HI.store(h, Ordering::Relaxed);
h
} else {
TICK_HI.load(Ordering::Relaxed)
};
(u64::from(hi) << 32) | u64::from(lo)
}
const MAX_TLS: usize = 8;
static TLS_VALUES: [AtomicUsize; MAX_TLS] = [const { AtomicUsize::new(0) }; MAX_TLS];
static NEXT_SLOT: AtomicUsize = AtomicUsize::new(0);
pub(super) struct TlsSlotImpl(usize);
pub(super) fn tls_new(_dtor: Option<TlsDtor>) -> Option<TlsSlotImpl> {
let idx = NEXT_SLOT.fetch_add(1, Ordering::Relaxed);
if idx < MAX_TLS {
Some(TlsSlotImpl(idx))
} else {
None
}
}
pub(super) fn tls_get(slot: &TlsSlotImpl) -> *mut c_void {
core::ptr::with_exposed_provenance_mut(TLS_VALUES[slot.0].load(Ordering::Relaxed))
}
pub(super) fn tls_set(slot: &TlsSlotImpl, value: *mut c_void) {
TLS_VALUES[slot.0].store(value.expose_provenance(), Ordering::Relaxed);
}
pub(super) fn tls_raw(slot: &TlsSlotImpl) -> usize {
slot.0
}
pub(super) fn tls_from_raw(raw: usize) -> TlsSlotImpl {
TlsSlotImpl(raw)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::SEGMENT_SIZE;
const _: () = assert!(super::super::FREE_RETURNS_MEMORY);
fn extents() -> Vec<(usize, usize)> {
let _g = Guard::acquire(&LOCK);
let rbase = REGION_BASE.load(Ordering::Relaxed);
(0..EXT_COUNT.load(Ordering::Relaxed))
.map(|i| {
(
EXT_BASE[i].load(Ordering::Relaxed) - rbase,
EXT_LEN[i].load(Ordering::Relaxed),
)
})
.collect()
}
fn free_total() -> usize {
extents().iter().map(|e| e.1).sum()
}
const REGION_ALIGN: usize = 64 * 1024;
const N: usize = 512 * 1024 + FIXED_PAGE;
static mut BACKING: [u8; N + REGION_ALIGN] = [0; N + REGION_ALIGN];
static mut OTHER: [u8; FIXED_PAGE] = [0; FIXED_PAGE];
#[test]
fn serves_and_recycles_a_static_region() {
let bp = (&raw mut BACKING).cast::<u8>();
let skip = align_up(bp.expose_provenance(), REGION_ALIGN) - bp.expose_provenance();
let rp = unsafe { bp.add(skip) };
let region: &'static mut [u8] = unsafe { core::slice::from_raw_parts_mut(rp, N) };
let tiny: &'static mut [u8] = &mut [];
assert_eq!(
init_region(tiny),
Err(FERR_TOO_SMALL),
"a region below one page is refused, and says which"
);
if usable_bytes(rp.expose_provenance(), N) == 0 {
assert_eq!(
init_region(region),
Err(FERR_GEOMETRY),
"a region that cannot hold one segment is refused BEFORE the board"
);
install_region(rp.expose_provenance(), N);
} else {
init_region(region).expect("this geometry's segment fits in N");
}
assert_eq!(free_total(), N, "the whole region starts free");
assert_eq!(extents().len(), 1, "as one extent");
let op = &raw mut OTHER;
let other: &'static mut [u8] =
unsafe { core::slice::from_raw_parts_mut(op.cast::<u8>(), FIXED_PAGE) };
let second = init_region(other);
assert!(second.is_err(), "no second region");
let (a, b, c) = unsafe {
(
alloc(64 * 1024, FIXED_PAGE, true, false).expect("a"),
alloc(128 * 1024, FIXED_PAGE, true, false).expect("b"),
alloc(64 * 1024, FIXED_PAGE, true, false).expect("c"),
)
};
assert_eq!(free_total(), N - 256 * 1024, "three blocks are out");
assert!(!a.is_zero, "recycled memory is never claimed zero");
let base = REGION_BASE.load(Ordering::Relaxed);
for (p, len) in [(a.ptr, 64 * 1024), (b.ptr, 128 * 1024), (c.ptr, 64 * 1024)] {
let off = p.expose_provenance() - base;
assert!(off + len <= N, "block lies inside the region");
}
assert_ne!(a.ptr, b.ptr);
assert_ne!(b.ptr, c.ptr);
for (p, len, tag) in [(a.ptr, 64 * 1024, 0xA5u8), (b.ptr, 128 * 1024, 0x5Au8)] {
unsafe {
core::ptr::write_bytes(p, tag, len);
assert_eq!(*p, tag);
assert_eq!(*p.add(len - 1), tag);
}
}
let holes = extents().len();
unsafe { free(b.ptr, 128 * 1024).expect("free b") };
assert_eq!(free_total(), N - 128 * 1024);
assert_eq!(extents().len(), holes + 1, "an isolated hole");
unsafe {
free(a.ptr, 64 * 1024).expect("free a");
free(c.ptr, 64 * 1024).expect("free c");
}
assert_eq!(free_total(), N, "the whole region is back");
assert_eq!(extents().len(), 1, "coalesced into one extent");
let d = unsafe { alloc(256 * 1024, FIXED_PAGE, true, false).expect("d") };
assert_eq!(free_total(), N - 256 * 1024);
unsafe { free(d.ptr, 256 * 1024).expect("free d") };
assert_eq!(free_total(), N);
assert_eq!(free_total(), N, "the whole region is free before this");
let seg = unsafe { alloc(SEGMENT_SIZE, SEGMENT_SIZE, true, false) };
if SEGMENT_SIZE > N {
assert!(
seg.is_err(),
"a {SEGMENT_SIZE}-byte segment cannot come out of a {N}-byte region"
);
assert_eq!(
free_total(),
N,
"a refused request leaves the list untouched"
);
let base = REGION_BASE.load(Ordering::Relaxed);
let boundary = align_up(base, SEGMENT_SIZE);
let straddles = boundary + FIXED_PAGE <= base + N;
let al = unsafe { alloc(FIXED_PAGE, SEGMENT_SIZE, true, false) };
if straddles {
let al = al.expect("the boundary is inside the region, so a page at it fits");
assert_eq!(
al.ptr.expose_provenance(),
boundary,
"served AT the one SEGMENT_SIZE-aligned address the region has"
);
unsafe { free(al.ptr, FIXED_PAGE).expect("free the aligned page") };
assert_eq!(free_total(), N, "and the list is whole again");
} else {
assert!(
al.is_err(),
"no SEGMENT_SIZE-aligned address lies inside this region"
);
assert_eq!(
free_total(),
N,
"a refused request leaves the list untouched"
);
}
} else {
let a = seg.expect("a segment must fit once the geometry allows it");
assert_eq!(
a.ptr.expose_provenance() % SEGMENT_SIZE,
0,
"a segment must be SEGMENT_SIZE-aligned โ `segment_of` masks on it"
);
assert_eq!(free_total(), N.saturating_sub(SEGMENT_SIZE));
unsafe { free(a.ptr, SEGMENT_SIZE).expect("free the segment") };
}
assert_eq!(free_total(), N);
let top = unsafe { alloc(FIXED_PAGE, FIXED_PAGE, true, false).expect("top") };
assert_eq!(
top.ptr.expose_provenance() - REGION_BASE.load(Ordering::Relaxed),
N - FIXED_PAGE,
"a page-aligned request is placed at the top of the region"
);
assert_eq!(
extents(),
vec![(0, N - FIXED_PAGE)],
"and leaves the low end as ONE contiguous extent"
);
unsafe { free(top.ptr, FIXED_PAGE).expect("free top") };
assert_eq!(free_total(), N);
let clean = greedy_segments();
assert_eq!(free_total(), N, "counting segments leaves the region whole");
let hdr = unsafe { alloc(FIXED_PAGE, FIXED_PAGE, true, false).expect("hdr") };
let with_hdr = greedy_segments();
assert_eq!(
with_hdr, clean,
"a page-sized block must not cost a whole segment of reach"
);
unsafe { free(hdr.ptr, FIXED_PAGE).expect("free hdr") };
assert_eq!(free_total(), N, "and the region ends whole");
}
fn greedy_segments() -> usize {
let mut held = Vec::new();
while let Ok(a) = unsafe { alloc(SEGMENT_SIZE, SEGMENT_SIZE, true, false) } {
held.push(a.ptr);
}
let n = held.len();
for p in held {
unsafe { free(p, SEGMENT_SIZE).expect("free a counted segment") };
}
n
}
#[test]
fn good_region_size_strands_nothing() {
use crate::types::SEGMENT_SIZE as SEG;
let budget = 220 * 1024;
let good = good_region_size(budget);
assert!(good <= budget, "a budget is a ceiling");
if budget >= MIN_REGION {
assert_eq!(
good,
3 * SEG + FIXED_PAGE,
"three segments plus the page at this geometry"
);
assert_eq!(
usable_bytes(0, good),
good - FIXED_PAGE,
"every byte but the page is a segment"
);
assert_eq!(
usable_bytes(0, budget),
usable_bytes(0, good),
"the good size serves as much as the budget did"
);
assert_eq!(
budget - good,
24_576,
"and that is what the budget was stranding"
);
} else {
assert_eq!(
good, 0,
"no zero-waste region fits a budget below the floor"
);
}
assert_eq!(good_region_size(0), 0);
assert_eq!(good_region_size(MIN_REGION - 1), 0);
assert_eq!(good_region_size(MIN_REGION), MIN_REGION);
let mut b = MIN_REGION;
while b < 40 * SEG {
let g = good_region_size(b);
assert!(g <= b && g >= MIN_REGION);
assert_eq!((g - FIXED_PAGE) % SEG, 0, "k * SEGMENT_SIZE + FIXED_PAGE");
assert_eq!(usable_bytes(0, g), g - FIXED_PAGE);
assert!(g + SEG > b, "not the largest: {g} for budget {b}");
b += 4093; }
let need: usize = 192 * 1024;
let k = need.div_ceil(SEG);
assert_eq!(region_for(need), k * SEG + FIXED_PAGE);
assert_eq!(usable_bytes(0, region_for(need)), k * SEG);
assert_eq!(
region_for(1),
SEG + FIXED_PAGE,
"one byte still costs a segment"
);
assert_eq!(
region_for(0),
MIN_REGION,
"and so does zero โ a region must serve something"
);
assert_eq!(
region_for(SEG + 1),
2 * SEG + FIXED_PAGE,
"a byte over rounds up"
);
let mut u = 1;
while u < 40 * SEG {
let r = region_for(u);
assert!(
usable_bytes(0, r) >= u,
"region_for({u}) = {r} serves too little"
);
assert!(
usable_bytes(0, r - SEG) < u || r - SEG < MIN_REGION,
"region_for({u}) = {r} is not the smallest"
);
assert_eq!(
good_region_size(r),
r,
"a region_for answer is already a good size"
);
u += 4093;
}
}
#[test]
fn usable_bytes_answers_the_question_a_firmware_asks() {
let seg = SEGMENT_SIZE;
assert_eq!(
usable_bytes(0, MIN_REGION),
seg,
"MIN_REGION buys a segment"
);
assert_eq!(
usable_bytes(0, MIN_REGION - 1),
0,
"one byte short buys none"
);
assert_eq!(
usable_bytes(0, seg),
0,
"a segment with no page for the heap is unusable"
);
assert_eq!(
usable_bytes(FIXED_PAGE, MIN_REGION),
0,
"unaligned base eats the segment"
);
assert_eq!(
usable_bytes(FIXED_PAGE, MIN_REGION + seg),
seg,
"one more segment of slack absorbs the misalignment"
);
let three_and_a_bit = 3 * seg + FIXED_PAGE + seg / 2;
assert_eq!(
usable_bytes(0, three_and_a_bit),
3 * seg,
"a ragged region yields whole segments and strands the remainder"
);
let stranded = three_and_a_bit - usable_bytes(0, three_and_a_bit) - FIXED_PAGE;
assert_eq!(
stranded,
seg / 2,
"and the strand is exactly the ragged part"
);
}
#[cfg(ra_single_threaded)]
#[test]
#[should_panic(expected = "re-entered")]
fn a_reentrant_acquire_is_diagnosed_not_hung() {
static LOCK2: AtomicBool = AtomicBool::new(false);
let _outer = Guard::acquire(&LOCK2);
let _inner = Guard::acquire(&LOCK2);
}
#[test]
fn no_mmu_semantics_are_explicit() {
let cfg = mem_init();
assert_eq!(cfg.page_size, FIXED_PAGE);
assert_eq!(cfg.large_page_size, 0, "no large pages without an MMU");
assert!(!cfg.has_overcommit, "nothing to overcommit");
assert!(cfg.has_partial_free, "any extent can be returned");
assert_ne!(thread_id(), 0, "zero is the abandoned-segment sentinel");
assert_eq!(numa_node_count(), 1);
assert!(clock_now() < clock_now());
let p = unsafe { protect(core::ptr::null_mut(), FIXED_PAGE, true) };
assert!(
p.is_err(),
"a guard page that cannot trap must not report success"
);
}
}