use core::ptr::NonNull;
use std::alloc::{Layout, alloc as std_alloc, dealloc as std_dealloc};
use crate::numa_topology::numa_topology;
const PAGE_SIZE: usize = 4096;
pub const NUMA_NODE_LOCAL: u32 = u32::MAX;
#[inline]
fn round_alloc_size(bytes: usize) -> usize {
let page_rounded = (bytes + PAGE_SIZE - 1) & !(PAGE_SIZE - 1);
if page_rounded <= PAGE_SIZE {
PAGE_SIZE
} else {
page_rounded.next_power_of_two()
}
}
#[derive(Copy, Clone, Debug)]
pub struct NumaAlloc;
impl NumaAlloc {
pub fn alloc_bytes(node: u32, bytes: usize) -> Option<NonNull<u8>> {
if bytes == 0 {
return None;
}
let size = round_alloc_size(bytes);
let node = resolve_node(node);
#[cfg(target_os = "windows")]
{
unsafe { windows_alloc(size, node) }
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
{
let _ = node;
unsafe { linux_mmap_alloc(size) }
}
#[cfg(not(any(
target_os = "windows",
all(target_os = "linux", target_arch = "x86_64"),
)))]
{
let _ = node;
unsafe { std_alloc_fallback(size) }
}
}
pub unsafe fn dealloc_bytes(ptr: NonNull<u8>, bytes: usize) {
if bytes == 0 {
return;
}
let size = round_alloc_size(bytes);
#[cfg(target_os = "windows")]
{
let _ = size;
unsafe { windows_dealloc(ptr) }
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
{
unsafe { linux_munmap(ptr, size) }
}
#[cfg(not(any(
target_os = "windows",
all(target_os = "linux", target_arch = "x86_64"),
)))]
{
unsafe { std_dealloc_fallback(ptr, size) }
}
}
}
fn resolve_node(node: u32) -> u32 {
if node == NUMA_NODE_LOCAL {
return current_thread_node();
}
let t = numa_topology();
if node < t.num_nodes { node } else { 0 }
}
fn current_thread_node() -> u32 {
let cpu = current_thread_cpu();
let t = numa_topology();
t.node_of_cpu.get(cpu).copied().unwrap_or(0)
}
#[cfg(target_os = "windows")]
fn current_thread_cpu() -> usize {
#[link(name = "kernel32")]
unsafe extern "system" {
fn GetCurrentProcessorNumber() -> u32;
}
unsafe { GetCurrentProcessorNumber() as usize }
}
#[cfg(target_os = "linux")]
fn current_thread_cpu() -> usize {
#[link(name = "c")]
unsafe extern "C" {
fn sched_getcpu() -> core::ffi::c_int;
}
let cpu = unsafe { sched_getcpu() };
if cpu < 0 { 0 } else { cpu as usize }
}
#[cfg(not(any(target_os = "windows", target_os = "linux")))]
fn current_thread_cpu() -> usize {
0
}
#[cfg(target_os = "windows")]
#[allow(non_camel_case_types, non_snake_case, clippy::upper_case_acronyms)]
mod win {
pub type HANDLE = *mut core::ffi::c_void;
pub type LPVOID = *mut core::ffi::c_void;
pub type SIZE_T = usize;
pub type DWORD = u32;
pub const MEM_COMMIT: DWORD = 0x1000;
pub const MEM_RESERVE: DWORD = 0x2000;
pub const MEM_RELEASE: DWORD = 0x8000;
pub const PAGE_READWRITE: DWORD = 0x04;
#[link(name = "kernel32")]
unsafe extern "system" {
pub fn GetCurrentProcess() -> HANDLE;
pub fn VirtualAllocExNuma(
hProcess: HANDLE,
lpAddress: LPVOID,
dwSize: SIZE_T,
flAllocationType: DWORD,
flProtect: DWORD,
nndPreferred: DWORD,
) -> LPVOID;
pub fn VirtualFree(
lpAddress: LPVOID,
dwSize: SIZE_T,
dwFreeType: DWORD,
) -> i32;
}
}
#[cfg(target_os = "windows")]
unsafe fn windows_alloc(size: usize, node: u32) -> Option<core::ptr::NonNull<u8>> {
use win::*;
let h = unsafe { GetCurrentProcess() };
let p = unsafe {
VirtualAllocExNuma(
h,
core::ptr::null_mut(),
size,
MEM_COMMIT | MEM_RESERVE,
PAGE_READWRITE,
node,
)
};
core::ptr::NonNull::new(p as *mut u8)
}
#[cfg(target_os = "windows")]
unsafe fn windows_dealloc(ptr: core::ptr::NonNull<u8>) {
unsafe {
let _ = win::VirtualFree(ptr.as_ptr() as win::LPVOID, 0, win::MEM_RELEASE);
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
#[allow(non_camel_case_types, non_snake_case)]
mod lnx {
use core::ffi::{c_int, c_void};
pub const PROT_READ: c_int = 0x1;
pub const PROT_WRITE: c_int = 0x2;
pub const MAP_PRIVATE: c_int = 0x2;
pub const MAP_ANONYMOUS: c_int = 0x20;
#[link(name = "c")]
unsafe extern "C" {
pub fn mmap(
addr: *mut c_void,
length: usize,
prot: c_int,
flags: c_int,
fd: c_int,
offset: i64,
) -> *mut c_void;
pub fn munmap(addr: *mut c_void, length: usize) -> c_int;
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
unsafe fn linux_mmap_alloc(size: usize) -> Option<core::ptr::NonNull<u8>> {
use lnx::*;
let p = unsafe {
mmap(
core::ptr::null_mut(),
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1,
0,
)
};
if p == (usize::MAX as *mut core::ffi::c_void) {
return None;
}
core::ptr::NonNull::new(p as *mut u8)
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
unsafe fn linux_munmap(ptr: core::ptr::NonNull<u8>, size: usize) {
unsafe {
let _ = lnx::munmap(ptr.as_ptr() as *mut core::ffi::c_void, size);
}
}
#[cfg(not(any(
target_os = "windows",
all(target_os = "linux", target_arch = "x86_64"),
)))]
unsafe fn std_alloc_fallback(size: usize) -> Option<core::ptr::NonNull<u8>> {
let layout = Layout::from_size_align(size, PAGE_SIZE).ok()?;
let p = unsafe { std_alloc(layout) };
core::ptr::NonNull::new(p)
}
#[cfg(not(any(
target_os = "windows",
all(target_os = "linux", target_arch = "x86_64"),
)))]
unsafe fn std_dealloc_fallback(ptr: core::ptr::NonNull<u8>, size: usize) {
if let Ok(layout) = Layout::from_size_align(size, PAGE_SIZE) {
unsafe { std_dealloc(ptr.as_ptr(), layout) };
}
}
#[cfg(any(
target_os = "windows",
all(target_os = "linux", target_arch = "x86_64"),
))]
const _: fn() = || {
let _ = std_alloc;
let _ = std_dealloc;
let _ = Layout::from_size_align(0, 0);
};
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn round_alloc_size_is_at_least_one_page() {
assert_eq!(round_alloc_size(1), PAGE_SIZE);
assert_eq!(round_alloc_size(0), PAGE_SIZE);
assert_eq!(round_alloc_size(PAGE_SIZE - 1), PAGE_SIZE);
assert_eq!(round_alloc_size(PAGE_SIZE), PAGE_SIZE);
}
#[test]
fn round_alloc_size_is_pow2_above_one_page() {
assert_eq!(round_alloc_size(PAGE_SIZE + 1), (PAGE_SIZE << 1));
assert_eq!(round_alloc_size(3 * PAGE_SIZE), (PAGE_SIZE << 2));
assert_eq!(round_alloc_size(5 * PAGE_SIZE), (PAGE_SIZE << 3));
assert_eq!(round_alloc_size(17 * PAGE_SIZE), (PAGE_SIZE << 5));
}
#[test]
fn alloc_zero_bytes_returns_none() {
assert!(NumaAlloc::alloc_bytes(0, 0).is_none());
}
#[test]
fn alloc_dealloc_roundtrip_small() {
let bytes = 1024;
let p = NumaAlloc::alloc_bytes(0, bytes).expect("alloc must succeed");
unsafe {
core::ptr::write_bytes(p.as_ptr(), 0xAB, 1);
core::ptr::write_bytes(p.as_ptr().add(PAGE_SIZE - 1), 0xCD, 1);
assert_eq!(*p.as_ptr(), 0xAB);
assert_eq!(*p.as_ptr().add(PAGE_SIZE - 1), 0xCD);
NumaAlloc::dealloc_bytes(p, bytes);
}
}
#[test]
fn alloc_dealloc_roundtrip_multi_page() {
let bytes = 100 * 1024;
let p = NumaAlloc::alloc_bytes(0, bytes).expect("alloc must succeed");
unsafe {
let n = round_alloc_size(bytes);
core::ptr::write(p.as_ptr(), 0x11u8);
core::ptr::write(p.as_ptr().add(n >> 1), 0x22u8);
core::ptr::write(p.as_ptr().add(n - 1), 0x33u8);
assert_eq!(*p.as_ptr(), 0x11);
assert_eq!(*p.as_ptr().add(n >> 1), 0x22);
assert_eq!(*p.as_ptr().add(n - 1), 0x33);
NumaAlloc::dealloc_bytes(p, bytes);
}
}
#[test]
fn alloc_returns_page_aligned() {
let p = NumaAlloc::alloc_bytes(0, 1024).expect("alloc must succeed");
let addr = p.as_ptr() as usize;
assert_eq!(addr & (PAGE_SIZE - 1), 0,
"alloc must return page-aligned: got 0x{addr:x}");
unsafe { NumaAlloc::dealloc_bytes(p, 1024) };
}
#[test]
fn resolve_node_clamps_out_of_range() {
assert_eq!(resolve_node(99), 0);
assert_eq!(resolve_node(0), 0);
}
#[test]
fn resolve_node_local_returns_current_thread_node() {
let n = resolve_node(NUMA_NODE_LOCAL);
let t = numa_topology();
assert!(n < t.num_nodes,
"current_thread_node {n} out of range (num_nodes {})",
t.num_nodes);
}
#[test]
fn current_thread_cpu_is_in_range() {
let cpu = current_thread_cpu();
let t = numa_topology();
let _ = (cpu, t);
}
#[test]
fn alloc_local_sentinel_does_not_crash() {
let p = NumaAlloc::alloc_bytes(NUMA_NODE_LOCAL, 8192)
.expect("local alloc must succeed");
unsafe {
for i in 0..(8192 / PAGE_SIZE) {
core::ptr::write(p.as_ptr().add(i * PAGE_SIZE), 0xFFu8);
}
NumaAlloc::dealloc_bytes(p, 8192);
}
}
}