use std::alloc::Layout;
use std::alloc::alloc;
use std::alloc::dealloc;
use std::error;
use std::fmt;
use std::io;
use std::ptr::NonNull;
use std::slice;
use std::sync::Arc;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
pub const BUFFER_ALIGNMENT: usize = 4096;
pub const CACHE_THREAD_STACK_BYTES: usize = 512 * 1024;
pub struct ManagedMemoryLimits {
pub memory_limit_bytes: usize,
pub reserved_memory_bytes: usize,
}
#[derive(Debug)]
pub enum ManagedMemoryError {
Invalid(&'static str),
Allocation,
}
impl fmt::Display for ManagedMemoryError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Invalid(message) => formatter.write_str(message),
Self::Allocation => formatter.write_str("managed memory cannot be allocated"),
}
}
}
impl error::Error for ManagedMemoryError {}
impl From<ManagedMemoryError> for io::Error {
fn from(error: ManagedMemoryError) -> Self {
let kind = match error {
ManagedMemoryError::Invalid(_) => io::ErrorKind::InvalidInput,
ManagedMemoryError::Allocation => io::ErrorKind::OutOfMemory,
};
Self::new(kind, error)
}
}
pub struct ManagedMemory {
memory: Arc<MemoryTracker>,
}
pub struct MemoryReservation {
memory: Arc<MemoryTracker>,
bytes: usize,
}
impl Drop for MemoryReservation {
fn drop(&mut self) {
self.memory.release(self.bytes);
}
}
impl ManagedMemory {
pub fn try_new(limits: ManagedMemoryLimits) -> Result<Self, ManagedMemoryError> {
if limits.reserved_memory_bytes > limits.memory_limit_bytes {
return Err(ManagedMemoryError::Invalid(
"memory limit cannot hold the cache's reserved memory",
));
}
let memory = Arc::new(MemoryTracker::new(
limits.memory_limit_bytes,
limits.reserved_memory_bytes,
));
Ok(Self { memory })
}
pub fn reserve(&self, bytes: usize) -> Result<MemoryReservation, ManagedMemoryError> {
if !self.memory.try_reserve(bytes) {
return Err(ManagedMemoryError::Allocation);
}
Ok(MemoryReservation {
memory: Arc::clone(&self.memory),
bytes,
})
}
pub fn try_read_buffer(&self, length: usize) -> Option<BufferLease> {
let capacity = length.checked_next_multiple_of(BUFFER_ALIGNMENT)?;
if capacity == 0 || capacity > isize::MAX as usize {
return None;
}
let reservation = self.reserve(capacity).ok()?;
BufferLease::try_new(capacity, Some(reservation))
}
pub fn snapshot(&self) -> ManagedMemorySnapshot {
let current_bytes = self.memory.current.load(Ordering::Relaxed);
ManagedMemorySnapshot {
limit_bytes: self.memory.limit,
current_bytes,
peak_bytes: self.memory.peak.load(Ordering::Relaxed).max(current_bytes),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ManagedMemorySnapshot {
pub limit_bytes: usize,
pub current_bytes: usize,
pub peak_bytes: usize,
}
pub struct BufferLease {
ptr: NonNull<u8>,
capacity: usize,
initialized: usize,
_reservation: Option<MemoryReservation>,
}
unsafe impl Send for BufferLease {}
impl BufferLease {
pub fn try_fixed(length: usize) -> Result<Self, ManagedMemoryError> {
if length == 0 || !length.is_multiple_of(BUFFER_ALIGNMENT) || length > isize::MAX as usize {
return Err(ManagedMemoryError::Invalid(
"fixed buffer size must be a non-zero 4096-byte multiple",
));
}
let mut buffer = Self::try_new(length, None).ok_or(ManagedMemoryError::Allocation)?;
unsafe { buffer.ptr.as_ptr().write_bytes(0, length) };
buffer.initialized = length;
Ok(buffer)
}
fn try_new(capacity: usize, reservation: Option<MemoryReservation>) -> Option<Self> {
if capacity == 0 {
return None;
}
let layout = Layout::from_size_align(capacity, BUFFER_ALIGNMENT).ok()?;
let ptr = NonNull::new(unsafe { alloc(layout) })?;
Some(Self {
ptr,
capacity,
initialized: 0,
_reservation: reservation,
})
}
pub fn prepared(&self, length: usize) -> Option<&[u8]> {
if length > self.initialized {
return None;
}
Some(unsafe { slice::from_raw_parts(self.ptr.as_ptr(), length) })
}
pub fn prepared_mut(&mut self, length: usize) -> Option<&mut [u8]> {
if length > self.initialized {
return None;
}
Some(unsafe { slice::from_raw_parts_mut(self.ptr.as_ptr(), length) })
}
pub fn has_capacity(&self, length: usize) -> bool {
length <= self.capacity
}
pub const fn as_mut_ptr(&self) -> *mut u8 {
self.ptr.as_ptr()
}
pub fn mark_initialized(&mut self, length: usize) -> bool {
if length > self.capacity {
return false;
}
self.initialized = self.initialized.max(length);
true
}
}
impl Drop for BufferLease {
fn drop(&mut self) {
let layout = Layout::from_size_align(self.capacity, BUFFER_ALIGNMENT)
.expect("stored aligned-buffer layout is valid");
unsafe { dealloc(self.ptr.as_ptr(), layout) };
}
}
struct MemoryTracker {
limit: usize,
current: AtomicUsize,
peak: AtomicUsize,
}
impl MemoryTracker {
fn new(limit: usize, reserved: usize) -> Self {
Self {
limit,
current: AtomicUsize::new(reserved),
peak: AtomicUsize::new(reserved),
}
}
fn try_reserve(&self, bytes: usize) -> bool {
let mut current = self.current.load(Ordering::Relaxed);
for _ in 0..MAX_ATOMIC_UPDATE_ATTEMPTS {
let Some(next) = current.checked_add(bytes) else {
return false;
};
if next > self.limit {
return false;
}
match self
.current
.compare_exchange(current, next, Ordering::Relaxed, Ordering::Relaxed)
{
Ok(_) => {
update_peak(&self.peak, next);
return true;
}
Err(observed) => current = observed,
}
}
false
}
fn release(&self, bytes: usize) {
if bytes != 0 {
self.current.fetch_sub(bytes, Ordering::Relaxed);
}
}
}
fn update_peak(peak: &AtomicUsize, candidate: usize) {
if candidate > peak.load(Ordering::Relaxed) {
peak.fetch_max(candidate, Ordering::Relaxed);
}
}
const MAX_ATOMIC_UPDATE_ATTEMPTS: usize = 8;
#[cfg(test)]
mod tests {
use super::*;
fn limits() -> ManagedMemoryLimits {
ManagedMemoryLimits {
memory_limit_bytes: 128 * 1024,
reserved_memory_bytes: 16 * 1024,
}
}
#[test]
fn transient_read_buffers_charge_exact_aligned_bytes_and_release_them() {
let managed_memory = ManagedMemory::try_new(limits()).unwrap();
let first = managed_memory.try_read_buffer(5000).unwrap();
let second = managed_memory.try_read_buffer(9000).unwrap();
assert_eq!(first.as_mut_ptr() as usize % BUFFER_ALIGNMENT, 0);
assert_eq!(second.as_mut_ptr() as usize % BUFFER_ALIGNMENT, 0);
assert!(first.prepared(5000).is_none());
assert!(second.prepared(9000).is_none());
let snapshot = managed_memory.snapshot();
assert_eq!(
snapshot.current_bytes,
16 * 1024 + 2 * BUFFER_ALIGNMENT + 3 * BUFFER_ALIGNMENT
);
assert!(snapshot.peak_bytes <= snapshot.limit_bytes);
drop(first);
assert_eq!(
managed_memory.snapshot().current_bytes,
16 * 1024 + 3 * BUFFER_ALIGNMENT
);
drop(second);
assert_eq!(managed_memory.snapshot().current_bytes, 16 * 1024);
}
#[test]
fn read_buffers_expose_only_the_initialized_prefix() {
let managed_memory = ManagedMemory::try_new(limits()).unwrap();
let mut buffer = managed_memory.try_read_buffer(5000).unwrap();
let bytes = b"completed read prefix";
let target = buffer.as_mut_ptr();
unsafe { target.copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()) };
assert!(buffer.prepared(bytes.len()).is_none());
assert!(buffer.prepared_mut(bytes.len()).is_none());
assert!(buffer.mark_initialized(bytes.len()));
assert_eq!(buffer.prepared(bytes.len()).unwrap(), bytes);
assert!(buffer.prepared(bytes.len() + 1).is_none());
assert!(buffer.prepared_mut(bytes.len() + 1).is_none());
assert!(buffer.mark_initialized(1));
assert_eq!(buffer.prepared(bytes.len()).unwrap(), bytes);
assert!(buffer.has_capacity(2 * BUFFER_ALIGNMENT));
assert!(!buffer.has_capacity(2 * BUFFER_ALIGNMENT + 1));
assert!(!buffer.mark_initialized(2 * BUFFER_ALIGNMENT + 1));
assert!(buffer.prepared(bytes.len() + 1).is_none());
}
#[test]
fn fixed_buffers_are_zeroed_and_reject_out_of_bounds_access() {
let mut buffer = BufferLease::try_fixed(BUFFER_ALIGNMENT).unwrap();
assert!(
buffer
.prepared(BUFFER_ALIGNMENT)
.unwrap()
.iter()
.all(|byte| *byte == 0)
);
buffer.prepared_mut(BUFFER_ALIGNMENT).unwrap().fill(0xa5);
assert!(!buffer.has_capacity(BUFFER_ALIGNMENT + 1));
assert!(buffer.prepared(BUFFER_ALIGNMENT + 1).is_none());
assert!(buffer.prepared_mut(BUFFER_ALIGNMENT + 1).is_none());
assert!(
buffer
.prepared(BUFFER_ALIGNMENT)
.unwrap()
.iter()
.all(|byte| *byte == 0xa5)
);
}
#[test]
fn invalid_read_buffer_sizes_do_not_consume_memory() {
let managed_memory = ManagedMemory::try_new(limits()).unwrap();
let before = managed_memory.snapshot();
for length in [0, isize::MAX as usize, usize::MAX] {
assert!(managed_memory.try_read_buffer(length).is_none());
assert_eq!(managed_memory.snapshot(), before);
}
}
#[test]
fn transient_read_buffers_stay_within_the_memory_limit() {
let mut configured = limits();
configured.memory_limit_bytes = configured.reserved_memory_bytes + 3 * BUFFER_ALIGNMENT;
let managed_memory = ManagedMemory::try_new(configured).unwrap();
let first = managed_memory.try_read_buffer(BUFFER_ALIGNMENT).unwrap();
let second = managed_memory
.try_read_buffer(2 * BUFFER_ALIGNMENT)
.unwrap();
assert!(managed_memory.try_read_buffer(1).is_none());
let memory = managed_memory.snapshot();
assert_eq!(memory.current_bytes, memory.limit_bytes);
assert!(memory.peak_bytes <= memory.limit_bytes);
drop((first, second));
assert!(
managed_memory
.try_read_buffer(3 * BUFFER_ALIGNMENT)
.is_some()
);
}
#[test]
fn reserved_memory_over_limit_is_rejected_before_allocating_buffers() {
let mut too_small = limits();
too_small.memory_limit_bytes = too_small.reserved_memory_bytes - 1;
assert!(matches!(
ManagedMemory::try_new(too_small),
Err(ManagedMemoryError::Invalid(_))
));
}
}