use core::mem::ManuallyDrop;
use core::ptr::NonNull;
use alloc_crate::vec::Vec;
use crate::error::Result;
pub(crate) const UNIT_SIZE: u32 = 12;
pub(crate) const NUM_INDEXES: usize = 38;
const fn unit_tables() -> ([u8; NUM_INDEXES], [u8; 128]) {
let mut index2units = [0u8; NUM_INDEXES];
let mut units2index = [0u8; 128];
let mut k = 0usize;
let mut i = 0usize;
while i < NUM_INDEXES {
let step = if i >= 12 { 4 } else { (i >> 2) + 1 };
let mut s = 0;
while s < step {
units2index[k] = i as u8;
k += 1;
s += 1;
}
index2units[i] = k as u8;
i += 1;
}
(index2units, units2index)
}
const TABLES: ([u8; NUM_INDEXES], [u8; 128]) = unit_tables();
pub(crate) static INDEX2UNITS: [u8; NUM_INDEXES] = TABLES.0;
pub(crate) static UNITS2INDEX: [u8; 128] = TABLES.1;
#[inline(always)]
pub(crate) fn i2u(index: u32) -> u32 {
INDEX2UNITS[index as usize] as u32
}
#[inline(always)]
pub(crate) fn u2i(nu: u32) -> u32 {
UNITS2INDEX[nu as usize - 1] as u32
}
const NODE_STAMP: u32 = 0;
const NODE_NU: u32 = 2;
const NODE_NEXT: u32 = 4;
const EMPTY_NODE: u16 = 0;
pub(crate) struct Arena {
base: NonNull<u8>,
len: usize,
capacity: usize,
size: u32,
align_offset: u32,
pub(crate) lo_unit: u32,
pub(crate) hi_unit: u32,
pub(crate) text: u32,
pub(crate) units_start: u32,
glue_count: u32,
free_list: [u32; NUM_INDEXES],
}
unsafe impl Send for Arena {}
unsafe impl Sync for Arena {}
impl Drop for Arena {
fn drop(&mut self) {
drop(unsafe { Vec::from_raw_parts(self.base.as_ptr(), self.len, self.capacity) });
}
}
impl Arena {
pub(crate) const fn arena_bytes(size: u32) -> usize {
(4u32.wrapping_sub(size) & 3) as usize + size as usize
}
pub(crate) const fn reusable(capacity: usize, needed: usize) -> bool {
capacity >= needed && capacity / 2 <= needed
}
pub(crate) fn try_new(size: u32, buf: Vec<u8>) -> Result<Self> {
debug_assert!(size >= crate::PPMD7_MIN_MEM_SIZE);
let len = Self::arena_bytes(size);
let buf = Self::fit(buf, len)?;
Ok(Self::from_vec(buf, size))
}
#[cfg(test)]
pub(crate) fn new(size: u32) -> Self {
match Self::try_new(size, Vec::new()) {
Ok(arena) => arena,
Err(e) => panic!("test arena: {e}"),
}
}
pub(crate) fn rebuild(&mut self, size: u32) -> Result<()> {
let len = Self::arena_bytes(size);
if Self::reusable(self.capacity, len) {
let buf = core::mem::replace(self, Self::from_vec(Vec::new(), 0)).into_vec();
let buf = Self::fit(buf, len)?;
*self = Self::from_vec(buf, size);
} else {
let buf = crate::arena::try_zeroed(len as u64)?;
*self = Self::from_vec(buf, size);
}
Ok(())
}
pub(crate) fn into_vec(self) -> Vec<u8> {
let this = ManuallyDrop::new(self);
unsafe { Vec::from_raw_parts(this.base.as_ptr(), this.len, this.capacity) }
}
pub(crate) fn capacity(&self) -> usize {
self.capacity
}
fn fit(mut buf: Vec<u8>, len: usize) -> Result<Vec<u8>> {
if !Self::reusable(buf.capacity(), len) {
buf = Vec::new();
}
crate::arena::fit(&mut buf, len)?;
Ok(buf)
}
fn from_vec(buf: Vec<u8>, size: u32) -> Self {
let mut buf = ManuallyDrop::new(buf);
let len = buf.len();
let capacity = buf.capacity();
let base = NonNull::new(buf.as_mut_ptr()).unwrap_or(NonNull::dangling());
let mut arena = Self {
base,
len,
capacity,
size,
align_offset: 4u32.wrapping_sub(size) & 3,
lo_unit: 0,
hi_unit: 0,
text: 0,
units_start: 0,
glue_count: 0,
free_list: [0; NUM_INDEXES],
};
if len != 0 {
arena.reset();
}
arena
}
#[inline]
pub(crate) fn size(&self) -> u32 {
self.size
}
#[cfg(any(test, feature = "internals"))]
pub(crate) fn arena_addr(&self) -> usize {
self.base.as_ptr() as usize
}
pub(crate) fn reset(&mut self) {
self.free_list = [0; NUM_INDEXES];
self.text = self.align_offset;
self.hi_unit = self.text + self.size;
self.units_start = self.hi_unit - self.size / 8 / UNIT_SIZE * 7 * UNIT_SIZE;
self.lo_unit = self.units_start;
self.glue_count = 0;
}
#[inline(always)]
fn at(&self, off: u32, len: usize) -> *mut u8 {
debug_assert!(
off as usize + len <= self.len,
"arena access {off}+{len} past {}",
self.len
);
unsafe { self.base.as_ptr().add(off as usize) }
}
#[inline(always)]
pub(crate) fn u8(&self, off: u32) -> u8 {
unsafe { *self.at(off, 1) }
}
#[inline(always)]
pub(crate) fn byte(&self, off: usize) -> u8 {
debug_assert!(off < self.len, "arena access {off} past {}", self.len);
unsafe { *self.base.as_ptr().add(off) }
}
#[cfg(any(
all(target_arch = "aarch64", target_endian = "little", not(miri)),
all(target_arch = "x86_64", not(miri))
))]
#[inline(always)]
pub(crate) fn ptr(&self, off: usize, len: usize) -> *const u8 {
debug_assert!(off + len <= self.len);
unsafe { self.base.as_ptr().add(off) }
}
#[inline(always)]
pub(crate) fn set_u8(&mut self, off: u32, v: u8) {
unsafe { *self.at(off, 1) = v }
}
#[inline(always)]
pub(crate) fn u16(&self, off: u32) -> u16 {
unsafe { self.at(off, 2).cast::<u16>().read_unaligned() }
}
#[inline(always)]
pub(crate) fn set_u16(&mut self, off: u32, v: u16) {
unsafe { self.at(off, 2).cast::<u16>().write_unaligned(v) }
}
#[inline(always)]
pub(crate) fn u32(&self, off: u32) -> u32 {
unsafe { self.at(off, 4).cast::<u32>().read_unaligned() }
}
#[inline(always)]
pub(crate) fn set_u32(&mut self, off: u32, v: u32) {
unsafe { self.at(off, 4).cast::<u32>().write_unaligned(v) }
}
#[inline]
pub(crate) fn copy(&mut self, src: u32, dst: u32, len: u32) {
let s = self.at(src, len as usize);
let d = self.at(dst, len as usize);
unsafe { core::ptr::copy(s, d, len as usize) }
}
#[inline(always)]
pub(crate) fn swap6(&mut self, a: u32, b: u32) {
let pa = self.at(a, 6);
let pb = self.at(b, 6);
unsafe { core::ptr::swap(pa.cast::<[u8; 6]>(), pb.cast::<[u8; 6]>()) }
}
#[inline(always)]
pub(crate) fn read6(&self, off: u32) -> [u8; 6] {
unsafe { self.at(off, 6).cast::<[u8; 6]>().read() }
}
#[inline(always)]
pub(crate) fn write6(&mut self, off: u32, v: [u8; 6]) {
unsafe { self.at(off, 6).cast::<[u8; 6]>().write(v) }
}
#[inline(always)]
pub(crate) fn insert_node(&mut self, node: u32, index: u32) {
let head = self.free_list[index as usize];
self.set_u32(node, head);
self.free_list[index as usize] = node;
}
#[inline(always)]
pub(crate) fn remove_node(&mut self, index: u32) -> u32 {
let node = self.free_list[index as usize];
debug_assert_ne!(node, 0);
self.free_list[index as usize] = self.u32(node);
node
}
#[inline(always)]
pub(crate) fn has_free(&self, index: u32) -> bool {
self.free_list[index as usize] != 0
}
pub(crate) fn split_block(&mut self, ptr: u32, old_index: u32, new_index: u32) {
let nu = i2u(old_index) - i2u(new_index);
let ptr = ptr + i2u(new_index) * UNIT_SIZE;
let mut i = u2i(nu);
if i2u(i) != nu {
i -= 1;
let k = i2u(i);
self.insert_node(ptr + k * UNIT_SIZE, nu - k - 1);
}
self.insert_node(ptr, i);
}
fn glue_free_blocks(&mut self) {
let mut n = 0u32;
self.glue_count = 255;
if self.lo_unit != self.hi_unit {
self.set_u16(self.lo_unit + NODE_STAMP, 1);
}
for (i, &units) in INDEX2UNITS.iter().enumerate() {
let nu = units as u16;
let mut next = self.free_list[i];
self.free_list[i] = 0;
while next != 0 {
let node = next;
next = self.u32(node);
self.set_u16(node + NODE_STAMP, EMPTY_NODE);
self.set_u16(node + NODE_NU, nu);
self.set_u32(node + NODE_NEXT, n);
n = node;
}
}
let head = self.glue_blocks(n);
self.fill_list(head);
}
fn glue_blocks(&mut self, first: u32) -> u32 {
let mut head = first;
let mut prev: Option<u32> = None;
let mut n = first;
while n != 0 {
let node = n;
let mut nu = self.u16(node + NODE_NU) as u32;
n = self.u32(node + NODE_NEXT);
if nu == 0 {
match prev {
None => head = n,
Some(p) => self.set_u32(p + NODE_NEXT, n),
}
} else {
prev = Some(node);
loop {
let node2 = node + nu * UNIT_SIZE;
nu += self.u16(node2 + NODE_NU) as u32;
if self.u16(node2 + NODE_STAMP) != EMPTY_NODE || nu >= 0x10000 {
break;
}
self.set_u16(node + NODE_NU, nu as u16);
self.set_u16(node2 + NODE_NU, 0);
}
}
}
head
}
fn fill_list(&mut self, head: u32) {
let mut n = head;
while n != 0 {
let mut node = n;
let mut nu = self.u16(node + NODE_NU) as u32;
n = self.u32(node + NODE_NEXT);
if nu == 0 {
continue;
}
while nu > 128 {
self.insert_node(node, NUM_INDEXES as u32 - 1);
nu -= 128;
node += 128 * UNIT_SIZE;
}
let mut index = u2i(nu);
if i2u(index) != nu {
index -= 1;
let k = i2u(index);
self.insert_node(node + k * UNIT_SIZE, nu - k - 1);
}
self.insert_node(node, index);
}
}
#[inline(never)]
pub(crate) fn alloc_units_rare(&mut self, index: u32) -> Option<u32> {
if self.glue_count == 0 {
self.glue_free_blocks();
if self.has_free(index) {
return Some(self.remove_node(index));
}
}
let mut i = index;
loop {
i += 1;
if i == NUM_INDEXES as u32 {
let num_bytes = i2u(index) * UNIT_SIZE;
let us = self.units_start;
self.glue_count -= 1;
return if us - self.text > num_bytes {
self.units_start = us - num_bytes;
Some(self.units_start)
} else {
None
};
}
if self.has_free(i) {
break;
}
}
let block = self.remove_node(i);
self.split_block(block, i, index);
Some(block)
}
#[inline(always)]
pub(crate) fn alloc_units(&mut self, index: u32) -> Option<u32> {
if self.has_free(index) {
return Some(self.remove_node(index));
}
let num_bytes = i2u(index) * UNIT_SIZE;
let lo = self.lo_unit;
if self.hi_unit - lo >= num_bytes {
self.lo_unit = lo + num_bytes;
return Some(lo);
}
self.alloc_units_rare(index)
}
#[inline(always)]
pub(crate) fn alloc_context(&mut self) -> Option<u32> {
if self.hi_unit != self.lo_unit {
self.hi_unit -= UNIT_SIZE;
Some(self.hi_unit)
} else if self.has_free(0) {
Some(self.remove_node(0))
} else {
self.alloc_units_rare(0)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
const SIZE: u32 = 1 << 16;
#[test]
fn tables_match_7zip() {
assert_eq!(
INDEX2UNITS,
[
1, 2, 3, 4, 6, 8, 10, 12, 15, 18, 21, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64,
68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128
]
);
for nu in 1..=128u32 {
let i = u2i(nu);
assert!(i2u(i) >= nu);
assert!(i == 0 || i2u(i - 1) < nu);
}
}
#[test]
fn layout_matches_ppmd7_restart_model() {
for size in [2048u32, 2049, 2050, 2051, 1 << 20, (1 << 20) + 7] {
let a = Arena::new(size);
let align = (4 - size % 4) % 4;
assert_eq!(a.text, align);
assert_eq!((a.text + size) % 4, 0);
assert_eq!(a.hi_unit, align + size);
assert_eq!(a.units_start, align + size - size / 8 / 12 * 7 * 12);
assert_eq!(a.lo_unit, a.units_start);
}
}
#[test]
fn freed_blocks_are_reused_last_in_first_out() {
let mut a = Arena::new(SIZE);
let x = a.alloc_units(1).unwrap();
let y = a.alloc_units(1).unwrap();
assert_eq!(y, x + 2 * UNIT_SIZE);
a.insert_node(x, 1);
a.insert_node(y, 1);
assert_eq!(a.alloc_units(1), Some(y));
assert_eq!(a.alloc_units(1), Some(x));
}
#[test]
fn split_frees_the_tail_in_list_sizes() {
let mut a = Arena::new(SIZE);
let block = a.alloc_units(NUM_INDEXES as u32 - 1).unwrap(); a.split_block(block, NUM_INDEXES as u32 - 1, 0);
assert!(a.has_free(u2i(124)));
assert!(a.has_free(u2i(3)));
}
#[test]
fn glue_merges_adjacent_free_blocks() {
let mut a = Arena::new(SIZE);
let first = a.alloc_units(0).unwrap();
let second = a.alloc_units(0).unwrap();
let gap = (a.hi_unit - a.lo_unit) / UNIT_SIZE;
for _ in 0..gap {
let c = a.alloc_context().unwrap();
a.set_u16(c, 1);
}
assert_eq!(a.lo_unit, a.hi_unit);
a.insert_node(first, 0);
a.insert_node(second, 0);
a.glue_count = 0;
assert_eq!(a.alloc_units(1), Some(first));
}
#[test]
fn rare_allocation_steals_from_the_text_region_then_fails() {
let mut a = Arena::new(SIZE);
a.lo_unit = a.hi_unit;
a.glue_count = 1;
let us = a.units_start;
assert_eq!(a.alloc_units_rare(0), Some(us - UNIT_SIZE));
assert_eq!(a.units_start, us - UNIT_SIZE);
a.text = a.units_start - UNIT_SIZE;
a.glue_count = 1;
assert_eq!(a.alloc_units_rare(0), None);
}
#[test]
fn reset_keeps_the_arena_and_its_bytes() {
let mut a = Arena::new(SIZE);
let addr = a.arena_addr();
a.set_u8(a.text + 100, 0xA5);
a.alloc_units(3).unwrap();
a.reset();
assert_eq!(a.arena_addr(), addr);
assert_eq!(a.u8(a.text + 100), 0xA5);
assert_eq!(a.lo_unit, a.units_start);
}
}