use std::alloc::Layout;
use std::collections::LinkedList;
use std::ptr::null_mut;
use crate::constants::DEFAULT_GLOBAL_MB_ALLOCATION;
use crate::mem::alloc::{direct_alloc, direct_dealloc, DirectAllocator, DIRECT_ALLOCATOR};
use crate::rumtk_layout;
pub const MIN_SLOT_SIZE: usize = 4;
pub type ChunkList = LinkedList<Chunk, &'static DirectAllocator>;
pub type FreeList = Vec<FreeSlot, &'static DirectAllocator>;
#[derive(Debug, Copy, Clone)]
pub struct FreeSlot {
ptr: *mut u8,
size: usize,
}
#[derive(Debug)]
pub struct Chunk {
base: *mut u8,
capacity: usize,
cursor: usize,
free_slots: FreeList,
}
impl Chunk {
pub fn new(capacity: usize) -> Option<Self> {
let base = unsafe { direct_alloc(rumtk_layout!(capacity)) };
if base.is_null() {
return None;
}
Some(Self {
base,
capacity,
cursor: 0,
free_slots: FreeList::with_capacity_in(1024, &DIRECT_ALLOCATOR),
})
}
#[inline(always)]
pub fn capacity(&self) -> usize {
self.capacity
}
#[inline(always)]
pub fn remaining(&self) -> usize {
self.capacity - self.cursor
}
#[inline(always)]
pub fn contains(&self, ptr: *const u8) -> bool {
let addr = ptr as usize;
let base = self.base as usize;
addr >= base && addr < base + self.capacity
}
#[inline(always)]
pub fn can_allocate(&self, size: usize, align: usize) -> bool {
self.can_bump(size, align) || self.has_free_slot(size, align)
}
#[inline]
pub fn defragment(&mut self) {
let mut i = 0;
while i < self.free_slots.len() {
let (ptr, size) = (self.free_slots[i].ptr, self.free_slots[i].size);
let merge_size = match self.free_slots.get(i + 1) {
Some(next) if ptr.wrapping_add(size) == next.ptr => Some(next.size),
None => break,
_ => None,
};
match merge_size {
Some(extra) => {
self.free_slots.remove(i + 1);
self.free_slots[i].size += extra;
}
None => {
i += 1;
},
}
}
}
#[inline]
pub fn allocate(&mut self, size: usize, align: usize) -> Option<*mut u8> {
match self.bump(size, align) {
Some(ptr) => Some(ptr),
None => {
self.defragment();
match self.reclaim(size, align) {
Some(ptr) => Some(ptr),
None => self.bump(size, align),
}
}
}
}
pub fn deallocate(&mut self, ptr: *mut u8, size: usize) {
self.release(ptr, size);
}
#[inline(always)]
fn aligned(ptr: *const u8, align: usize) -> bool {
(ptr as usize) & (align - 1) == 0
}
#[inline(always)]
fn padding(ptr: *const u8, align: usize) -> usize {
(align - ((ptr as usize) & (align - 1))) & (align - 1)
}
#[inline(always)]
fn has_free_slot(&self, size: usize, align: usize) -> bool {
self.free_slots
.iter()
.any(|slot| slot.size >= size && Self::aligned(slot.ptr, align))
}
#[inline(always)]
fn can_bump(&self, size: usize, align: usize) -> bool {
let addr = unsafe { self.base.add(self.cursor) };
match Self::padding(addr, align).checked_add(size) {
Some(needed) => self.remaining() >= needed,
None => false,
}
}
#[inline]
fn reclaim(&mut self, size: usize, align: usize) -> Option<*mut u8> {
for i in 0.. self.free_slots.len() {
let slot = &mut self.free_slots[i];
if slot.size >= size && Self::aligned(slot.ptr, align) {
let ptr = slot.ptr;
slot.ptr = unsafe { slot.ptr.add(size) };
slot.size -= size;
if slot.size == 0 {
self.free_slots.remove(i);
}
return Some(ptr);
}
}
None
}
#[inline]
fn bump(&mut self, size: usize, align: usize) -> Option<*mut u8> {
if !self.can_bump(size, align) {
return None;
}
let addr = unsafe { self.base.add(self.cursor) };
let pad = Self::padding(addr, align);
let needed = pad.checked_add(size)?;
if self.remaining() < needed {
return None;
}
if pad > 0 {
self.release(addr, pad);
}
self.cursor += needed;
Some(unsafe { addr.add(pad) })
}
#[inline]
fn release(&mut self, ptr: *mut u8, size: usize) {
for i in 0.. self.free_slots.len() {
let slot = &mut self.free_slots[i];
if slot.ptr < ptr {
continue;
}
self.free_slots.insert(i, FreeSlot { ptr, size });
}
}
}
impl Drop for Chunk {
fn drop(&mut self) {
unsafe { direct_dealloc(self.base, rumtk_layout!(self.capacity)) };
}
}
#[derive(Debug)]
pub struct MemoryPool {
chunks: ChunkList,
chunk_size: usize,
}
impl MemoryPool {
pub const fn new() -> Self {
Self::with_chunk_size(DEFAULT_GLOBAL_MB_ALLOCATION)
}
pub const fn with_chunk_size(chunk_size: usize) -> Self {
Self {
chunks: ChunkList::new_in(&DIRECT_ALLOCATOR),
chunk_size,
}
}
#[inline]
pub fn slot_size(layout: &Layout) -> Option<usize> {
let requested = layout.size();
if requested <= MIN_SLOT_SIZE {
Some(MIN_SLOT_SIZE)
} else {
requested.checked_next_power_of_two()
}
}
#[inline(always)]
pub fn chunk_size(&self) -> usize {
self.chunk_size
}
#[inline(always)]
pub fn chunk_count(&self) -> usize {
self.chunks.len()
}
#[inline]
pub fn allocate_on_available(&mut self, size: usize, align: usize) -> Option<*mut u8> {
for chunk in self.chunks.iter_mut().rev() {
match chunk.reclaim(size, align) {
Some(ptr) => return Some(ptr),
None => match chunk.allocate(size, align) {
Some(ptr) => return Some(ptr),
None => continue,
},
}
}
None
}
#[inline]
pub fn allocate(&mut self, layout: Layout) -> *mut u8 {
let size = match Self::slot_size(&layout) {
Some(size) => size,
None => return null_mut(),
};
let align = layout.align();
match self.allocate_on_available(size, align) {
Some(ptr) => ptr,
None => {
self.grow(size, align);
match self.chunks.back_mut() {
Some(chunk) => chunk.allocate(size, align).unwrap_or(null_mut()),
None => null_mut(),
}
}
}
}
#[inline]
pub unsafe fn deallocate(&mut self, ptr: *mut u8, layout: Layout) {
if ptr.is_null() {
return;
}
let size = match Self::slot_size(&layout) {
Some(size) => size,
None => return,
};
if let Some(chunk) = self.chunks.iter_mut().rev().find(|chunk| chunk.contains(ptr)) {
chunk.deallocate(ptr, size);
}
}
#[inline]
fn grow(&mut self, size: usize, align: usize) {
let needed = size.saturating_add(align);
let capacity = if needed > self.chunk_size {
needed
} else {
self.chunk_size
};
if let Some(chunk) = Chunk::new(capacity) {
self.chunks.push_back(chunk);
}
}
}
impl Default for MemoryPool {
fn default() -> Self {
Self::new()
}
}
unsafe impl Send for MemoryPool {}
#[cfg(test)]
mod tests {
use super::*;
fn layout(size: usize, align: usize) -> Layout {
Layout::from_size_align(size, align).unwrap()
}
#[test]
fn test_mempool_slot_size_is_power_of_two() {
assert_eq!(MemoryPool::slot_size(&layout(1, 1)), Some(MIN_SLOT_SIZE));
assert_eq!(MemoryPool::slot_size(&layout(16, 1)), Some(16));
assert_eq!(MemoryPool::slot_size(&layout(17, 1)), Some(32));
assert_eq!(MemoryPool::slot_size(&layout(1000, 1)), Some(1024));
assert_eq!(MemoryPool::slot_size(&layout(1024, 1)), Some(1024));
}
#[test]
fn test_mempool_allocates_usable_memory() {
let mut pool = MemoryPool::with_chunk_size(1024);
let ptr = pool.allocate(layout(100, 1));
assert!(!ptr.is_null(), "Pool failed to allocate a slot!");
assert_eq!(pool.chunk_count(), 1, "Pool did not allocate a chunk!");
unsafe { std::ptr::write_bytes(ptr, 0xAB, 100) };
let slice = unsafe { std::slice::from_raw_parts(ptr, 100) };
assert!(slice.iter().all(|byte| *byte == 0xAB), "Slot memory is not usable!");
}
#[test]
fn test_mempool_respects_alignment() {
let mut pool = MemoryPool::with_chunk_size(1024);
let ptr = pool.allocate(layout(24, 64));
assert!(!ptr.is_null(), "Pool failed to allocate an aligned slot!");
assert_eq!(ptr as usize % 64, 0, "Slot is not aligned to the requested alignment!");
}
#[test]
fn test_mempool_allocates_new_chunk_when_full() {
let mut pool = MemoryPool::with_chunk_size(64);
let l = layout(64, 1);
let first = pool.allocate(l);
let second = pool.allocate(l);
assert!(!first.is_null(), "Pool failed to allocate the first slot!");
assert!(!second.is_null(), "Pool failed to allocate the second slot!");
assert_ne!(first, second, "Pool handed out the same slot twice!");
assert_eq!(pool.chunk_count(), 2, "Pool did not allocate a new chunk once full!");
}
#[test]
fn test_mempool_allocates_dedicated_chunk_for_big_requests() {
let mut pool = MemoryPool::with_chunk_size(64);
let ptr = pool.allocate(layout(256, 1));
assert!(!ptr.is_null(), "Pool failed to allocate a slot bigger than the chunk size!");
unsafe { std::ptr::write_bytes(ptr, 0xCD, 256) };
}
}