buffer_sv2 4.0.0

// # Buffer Pool
//
// A memory-efficient buffer pool that minimizes allocations and deallocations for high-throughput
// message frame processing in Sv2 roles.
//
// Provides primitives for reusing memory buffers, clearing old memory chunks, and switching
// between different modes (back, front, and alloc) to manage memory effectively and reduce
// expensive heap allocations. It uses atomic operations and shared state tracking to safely manage
// memory across multiple threads.
//
// Supports different allocation modes to optimize memory usage:
//
// - **Back Mode**: Allocates from the back of the buffer pool (default).
// - **Front Mode**: Allocates from the front when the back is full and the front has space.
// - **Alloc Mode**: Falls back to heap allocation when the buffer pool cannot fulfill requests
//   (with reduced performance).
//
// ## Usage
//
// When an incoming Sv2 message is received, it needs to be buffered for processing. The pool first
// checks the back part (`PoolBack`) to see if there is enough memory available. If the back is
// full, the pool attempts to clear used memory chunks. If clearing fails, it switches to the front
// (`PoolFront`) and tries again, this time allocating memory from the front of the `BufferPool`.
// If both back and front are full and no memory can be cleared, the pool falls back to allocating
// fresh memory from the heap (`PoolMode::Alloc`) at a performance reduction. After processing the
// message, the memory can be cleared, and the buffer pool resets, making the memory available
// for future messages.

// **Note**: A slice keeps its slot, and the pool's memory, in use until it is dropped, so avoid
// holding onto slices longer than necessary. Cloning a slice copies its bytes into memory the
// clone owns, so the slot is freed as soon as the original is dropped.

use alloc::vec::Vec;

#[cfg(test)]
use crate::buffer::TestBufferFromMemory;
use crate::{
    buffer::BufferFromSystemMemory,
    slice::{SharedState, Slice},
    Buffer,
};
#[cfg(feature = "debug")]
use std::time::SystemTime;

mod pool_back;
pub use pool_back::PoolBack;
#[cfg(test)]
mod test;

// Maximum number of memory slices the buffer pool can concurrently manage.
//
// This value limits the number of slices the `BufferPool` can track and manage at once. Once the
// pool reaches its capacity of `8` slices, it may need to free memory or switch modes (e.g., to
// system memory). The choice of `8` ensures a balance between performance and memory management.
// The use of `usize` allows for compatibility with platform-dependent memory operations.
pub const POOL_CAPACITY: usize = 8;

// Manages the "front" section of the `BufferPool`.
//
// Handles the allocation of memory slices at the front of the buffer pool. It tracks the number of
// slices in use and attempts to free unused slices when necessary to maximize available memory.
// The front of the buffer pool is used if the back of the buffer pool is filled up.
#[derive(Debug)]
pub struct PoolFront {
    // Starting index of the front section of the buffer pool.
    back_start: usize,

    // Maximum number of bytes that can be allocated in the front section of the buffer pool.
    //
    // Helps manage how much memory can be used before triggering memory clearing or switching
    // `PoolMode`.
    byte_capacity: usize,

    // Number of allocated slices in the front section of the buffer pool.
    len: usize,
}

impl PoolFront {
    // Initializes a new `PoolFront` with the specified byte capacity and back start position.
    #[inline(always)]
    fn new(byte_capacity: usize, back_start: usize) -> Self {
        Self {
            byte_capacity,
            back_start,
            len: 0,
        }
    }

    // Attempts to clear unused memory slices at the tail of the front section.
    //
    // Returns `true` if slices were successfully cleared, otherwise `false` if no slices could be
    // cleared or memory conditions prevent clearing.
    #[inline(always)]
    fn try_clear_tail(&mut self, memory: &mut InnerMemory, mut shared_state: u8) -> bool {
        #[cfg(feature = "fuzz")]
        assert!(self.len > 0);
        // 1001_1100   bs=3 len=1
        // 0000_0001   >> 7
        shared_state >>= POOL_CAPACITY - self.len;

        let element_to_drop = shared_state.trailing_zeros();

        match element_to_drop {
            0 => false,
            8 => {
                self.len = 0;

                memory.len = self.len;
                let raw_offset = memory.raw_offset();
                memory.move_raw_at_offset(raw_offset);

                true
            }
            _ => {
                #[cfg(feature = "fuzz")]
                assert!(
                    element_to_drop <= self.back_start as u32 && element_to_drop <= self.len as u32
                );

                self.len -= element_to_drop as usize;

                memory.len = self.len;
                let raw_offset = memory.raw_offset();
                memory.move_raw_at_offset(raw_offset);

                true
            }
        }
    }

    // Clears the front memory slices if conditions allow and checks if the memory pool has
    // capacity to allocate `len` bytes in the buffer.
    //
    // Returns `Ok` if memory was successfully cleared and there is sufficient capacity, otherwise
    // an `Err(PoolMode::Back)` if the memory cannot be cleared or lacks capacity. This error
    // indicates the `BufferPool` should attempt a transition to use the back of the buffer pool.
    #[inline(always)]
    fn clear(
        &mut self,
        memory: &mut InnerMemory,
        shared_state: u8,
        len: usize,
    ) -> Result<(), PoolMode> {
        if self.len > 0 && self.try_clear_tail(memory, shared_state) {
            if memory.has_capacity_until_offset(len, self.byte_capacity) {
                Ok(())
            } else {
                Err(PoolMode::Back)
            }
        } else {
            Err(PoolMode::Back)
        }
    }

    // Attempts to allocate a writable memory region in the front section of the buffer pool,
    // returning a writable slice if successful, or transitioning to a new pool mode if necessary.
    //
    // Returns a pointer to the writable memory (`Ok(*mut u8)`) if successful, otherwise an
    // `Err(PoolMode::Back)` if the memory cannot be cleared or lacks capacity. This error
    // indicates the `BufferPool` should attempt a transition to use the back of the buffer pool.
    #[inline(always)]
    fn reserve(
        &mut self,
        len: usize,
        memory: &mut InnerMemory,
        shared_state: u8,
    ) -> Result<*mut u8, PoolMode> {
        let pool_has_slice_capacity = self.len < self.back_start;
        let pool_has_head_capacity = memory.has_capacity_until_offset(len, self.byte_capacity);

        if pool_has_slice_capacity && pool_has_head_capacity {
            return Ok(memory.reserve_raw(len));
        };

        self.clear(memory, shared_state, len)
            .map(|_| memory.reserve_raw(len))
    }
}

/// Current mode of operation for the `BufferPool`.
///
/// The pool operates in three modes based on memory availability: it first allocates from the
/// back, then from the front if the back is full, and finally from system memory (with reduced
/// performance) if both sections are exhausted.
#[derive(Debug)]
pub enum PoolMode {
    /// The buffer pool is operating in "back" mode, where memory is allocated from the back of the
    /// buffer pool.
    Back,

    /// The buffer pool is operating in "front" mode, where memory is allocated from the front of
    /// the buffer pool. Used when the back is full.
    Front(PoolFront),

    /// The pool has exhausted its internal memory, and it is now allocating directly from the
    /// system memory (heap).
    Alloc,
}

// Internal memory management for the `BufferPool`.
//
// Handles allocating, tracking, and managing memory slices for manipulating memory offsets,
// copying data, and managing capacity. It uses a contiguous block of memory, tracking
// its usage through offsets (`raw_offset`, `raw_length`), and manages slice allocations through
// `slots`. Used by `BufferPool` to optimize memory reused and minimize heap allocations.
//
// The memory is only ever reached through raw pointers narrowed to the range being accessed, so
// that no reference covers a range a live slice points into.
pub struct InnerMemory {
    // Underlying contiguous block of memory to be managed, shared with the slices pointing into it
    // along with the bits that track them.
    memory: SharedState,

    // Current offset into the contiguous block of memory where the next write will occur.
    pub(crate) raw_offset: usize,

    // Length of the valid data within the contiguous block of memory, starting from `raw_offset`.
    pub(crate) raw_len: usize,

    // Tracks individual chunks of memory being used in the buffer pool by tracking the start and
    // length of each allocated memory slice.
    slots: [(usize, usize); POOL_CAPACITY],

    // A pointer to the current slot. Represents how many slots are currently occupied.
    len: usize,
}

// Formats the bookkeeping only: the bytes may belong to live slices written on other threads.
impl core::fmt::Debug for InnerMemory {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        f.debug_struct("InnerMemory")
            .field("memory", &self.memory)
            .field("raw_offset", &self.raw_offset)
            .field("raw_len", &self.raw_len)
            .field("slots", &self.slots)
            .field("len", &self.len)
            .finish()
    }
}

impl InnerMemory {
    // Initializes a new `InnerMemory` with a specified size of the internal memory buffer
    // (`capacity`), in bytes.
    fn new(capacity: usize) -> Self {
        Self {
            memory: SharedState::new(capacity),
            raw_offset: 0,
            raw_len: 0,
            slots: [(0_usize, 0_usize); POOL_CAPACITY],
            len: 0,
        }
    }

    // Returns the size of the underlying memory, in bytes.
    #[inline(always)]
    fn capacity(&self) -> usize {
        self.memory.capacity()
    }

    // Checks that `len` bytes starting at `offset` lie within the underlying memory.
    #[inline(always)]
    fn in_bounds(&self, offset: usize, len: usize) -> bool {
        offset
            .checked_add(len)
            .is_some_and(|end| end <= self.capacity())
    }

    // Returns the data written so far, from `raw_offset` to `raw_offset + raw_len`.
    #[inline(always)]
    fn raw_data(&self) -> &[u8] {
        assert!(self.in_bounds(self.raw_offset, self.raw_len));
        unsafe {
            core::slice::from_raw_parts(self.memory.bytes().add(self.raw_offset), self.raw_len)
        }
    }

    // Returns the data written so far, from `raw_offset` to `raw_offset + raw_len`.
    #[inline(always)]
    fn raw_data_mut(&mut self) -> &mut [u8] {
        assert!(self.in_bounds(self.raw_offset, self.raw_len));
        unsafe {
            core::slice::from_raw_parts_mut(self.memory.bytes().add(self.raw_offset), self.raw_len)
        }
    }

    // Copies `len` bytes from offset `src` to offset `dst`. The two ranges may overlap.
    #[inline(always)]
    fn copy_within(&mut self, src: usize, len: usize, dst: usize) {
        assert!(self.in_bounds(src, len) && self.in_bounds(dst, len));
        let bytes = self.memory.bytes();
        unsafe { core::ptr::copy(bytes.add(src), bytes.add(dst), len) }
    }

    // Resets the internal memory pool, clearing all used memory and resetting the slot tracking.
    #[inline(always)]
    fn reset(&mut self) {
        self.raw_len = 0;
        self.raw_offset = 0;
        self.slots = [(0_usize, 0_usize); POOL_CAPACITY];
    }

    // Resets only the raw memory state, without affecting the slot tracking.
    #[inline(always)]
    fn reset_raw(&mut self) {
        self.raw_len = 0;
        self.raw_offset = 0;
    }

    // Returns the capacity of the front portion of the buffer.
    //
    // Used to determine how much space is available in the front of the buffer pool, based on the
    // `back_start` position.
    #[inline(always)]
    fn get_front_capacity(&self, back_start: usize) -> usize {
        #[cfg(feature = "fuzz")]
        assert!(
            (1..POOL_CAPACITY).contains(&back_start)
                && self.slots[back_start].0 != 0_usize
                && self.slots[back_start].1 != 0_usize
                && self.slots[back_start].0 + self.slots[back_start].1 <= self.capacity()
        );

        self.slots[back_start].0
    }

    // Calculates the offset for the next writable section of memory. Returns the offset based on
    // the current memory length and slot usage.
    #[inline(always)]
    fn raw_offset(&self) -> usize {
        self.raw_offset_from_len(self.len)
    }

    // Calculates the offset for a specific length of memory. Returns the offset based on the
    // provided length and the current state of the memory slots.
    #[inline(always)]
    fn raw_offset_from_len(&self, len: usize) -> usize {
        match len {
            0 => 0,
            _ => {
                let index = len - 1;
                #[cfg(feature = "fuzz")]
                assert!(
                    index < POOL_CAPACITY
                        && self.slots[index].1 != 0_usize
                        && self.slots[index].0 + self.slots[index].1 <= self.capacity()
                );

                let (index, len) = self.slots[index];

                index + len
            }
        }
    }

    // Moves the raw data to the front of the memory pool, once no live slice comes before it.
    #[inline(always)]
    fn move_raw_at_front(&mut self) {
        self.move_raw_at_offset(0);
    }

    // Returns the offset right after the highest front slot still live in `shared_state`, or `0`
    // if none is. Freed front slots are skipped, since a front slot reused for a larger slice can
    // leave a later, freed entry pointing inside it.
    #[inline(always)]
    fn front_end(&self, shared_state: u8, back_start: usize) -> usize {
        if back_start == 0 {
            return 0;
        }
        match shared_state >> (POOL_CAPACITY - back_start) {
            0 => 0,
            front => {
                let highest = back_start - 1 - front.trailing_zeros() as usize;
                let (offset, len) = self.slots[highest];
                offset + len
            }
        }
    }

    // Tries to update the length of the memory pool and to move the raw data to `raw_offset` if
    // there is enough capacity to accommodate new memory. Returns `true` if successful, otherwise
    // false.
    #[inline(always)]
    fn try_change_len(&mut self, slot_len: usize, raw_offset: usize, raw_len: usize) -> bool {
        let end = raw_offset + self.raw_len;
        if self.in_bounds(end, raw_len) {
            self.len = slot_len;
            self.move_raw_at_offset(raw_offset);
            true
        } else {
            false
        }
    }

    // Moves the raw data to a specific offset within the memory pool to avoid fragmentation, if
    // necessary.
    //
    // Panics if the data would not fit in the memory at `offset`.
    #[inline(always)]
    fn move_raw_at_offset(&mut self, offset: usize) {
        match self.raw_len {
            0 => self.raw_offset = offset,
            _ => {
                self.copy_within(self.raw_offset, self.raw_len, offset);
                self.raw_offset = offset;
            }
        }
    }

    // Inserts raw data at the front of the memory pool, adjusting the raw offset and length.
    #[inline(never)]
    fn prepend_raw_data(&mut self, raw_data: &[u8]) {
        assert!(self.in_bounds(0, raw_data.len()));
        let dest = self.memory.bytes();
        unsafe { core::ptr::copy_nonoverlapping(raw_data.as_ptr(), dest, raw_data.len()) };

        self.raw_offset = 0;
        self.raw_len = raw_data.len();
    }

    // Copies the internal raw memory into another buffer. Used when transitioning memory between
    // different pool modes.
    #[inline(never)]
    fn copy_into_buffer(&mut self, buffer: &mut impl Buffer) {
        buffer
            .reserve(self.raw_len)
            .copy_from_slice(self.raw_data());
        buffer.commit(self.raw_len);
    }

    // Checks if there is enough capacity at the tail of the memory pool to accommodate `len`
    // bytes.
    #[inline(always)]
    fn has_tail_capacity(&self, len: usize) -> bool {
        self.in_bounds(self.raw_offset + self.raw_len, len)
    }

    // Checks if there is enough capacity in the memory pool up to the specified offset to
    // accommodate `len` bytes.
    #[inline(always)]
    fn has_capacity_until_offset(&self, len: usize, offset: usize) -> bool {
        (self.raw_offset + self.raw_len)
            .checked_add(len)
            .is_some_and(|end| end <= offset)
    }

    // Returns a raw pointer to `len` bytes right after the data written so far, without counting
    // them as written.
    #[inline(always)]
    fn reserve_raw(&mut self, len: usize) -> *mut u8 {
        let writable_offset = self.raw_offset + self.raw_len;
        assert!(self.in_bounds(writable_offset, len));
        unsafe { self.memory.bytes().add(writable_offset) }
    }

    // Counts `len` more bytes, right after the data written so far, as written.
    #[inline(always)]
    fn commit_raw(&mut self, len: usize) {
        assert!(self.in_bounds(self.raw_offset + self.raw_len, len));
        self.raw_len += len;
    }

    /// Provides access to the raw memory slice containing the data written into the buffer,
    /// returning it as a [`Slice`]. This method is the primary mechanism for making processed data
    /// available from the buffer to the rest of the system.
    ///
    /// After this call, the buffer advances internally to a new slot, allowing new data to be
    /// written into an unused portion of memory. This approach avoids memory duplication and
    /// ensures efficient reuse of the buffer without transferring ownership of the memory.
    ///
    /// This method is typically used when the data in the buffer is ready for processing, sending,
    /// or further manipulation. The returned `Slice` contains the data, while the buffer itself
    /// remains ready to handle new incoming data by pointing to a fresh memory region.
    #[inline(always)]
    fn get_data_owned(&mut self, #[cfg(feature = "debug")] mode: u8) -> Slice {
        if self.raw_len == 0 {
            return Slice::from(Vec::new());
        }

        let offset = unsafe { self.memory.bytes().add(self.raw_offset) };

        self.slots[self.len] = (self.raw_offset, self.raw_len);
        self.len += 1;

        let slice = Slice::pooled(
            self.memory.clone(),
            offset,
            self.raw_len,
            self.len as u8,
            #[cfg(feature = "debug")]
            mode,
        );

        self.raw_offset += self.raw_len;
        self.raw_len = 0;

        slice
    }
}

/// A pool of reusable memory buffers to optimize memory allocation for Sv2 message frames.
///
/// Manages memory slices across three pool modes: back (default), front, and system memory. It
/// reuses preallocated memory slices, minimizing the need for frequent memory allocation. The pool
/// is thread-safe, using atomic state tracking.
///
/// Type `T` implements the [`Buffer`] trait, which defines how memory is handled in the buffer
/// pool. [`BufferFromSystemMemory`] is used as the default system memory allocator.
#[derive(Debug)]
pub struct BufferPool<T: Buffer> {
    // Manages memory allocation from the back section of the buffer pool.
    pool_back: PoolBack,

    // Tracks the current mode of memory allocation (back, front, or system).
    mode: PoolMode,

    // Core memory area from which slices are allocated and reused. Manages the actual memory
    // buffer used by the buffer pool, and tracks the usage state of memory slices using atomic
    // operations, ensuring memory is not prematurely reused.
    inner_memory: InnerMemory,

    // Allocates memory directly from system memory when the buffer pool is full, acting as a
    // fallback when preallocated memory cannot satisfy buffer requests.
    system_memory: T,

    // Length of the last reservation, `0` once it is committed.
    reserved: usize,
}

impl BufferPool<BufferFromSystemMemory> {
    /// Creates a new [`BufferPool`] with the specified memory capacity, in bytes.
    ///
    /// Initializes the buffer pool with pre-allocated memory (`inner_memory`) and sets the pool
    /// mode to the back. The buffer pool uses [`BufferFromSystemMemory`] as a fallback when the
    /// buffer sections are full.
    pub fn new(capacity: usize) -> Self {
        Self {
            pool_back: PoolBack::new(),
            mode: PoolMode::Back,
            inner_memory: InnerMemory::new(capacity),
            system_memory: BufferFromSystemMemory::default(),
            reserved: 0,
        }
    }
}

#[cfg(test)]
impl BufferPool<TestBufferFromMemory> {
    pub fn new_fail_system_memory(capacity: usize) -> Self {
        Self {
            pool_back: PoolBack::new(),
            mode: PoolMode::Back,
            inner_memory: InnerMemory::new(capacity),
            system_memory: TestBufferFromMemory(Vec::new()),
            reserved: 0,
        }
    }
}

// Defines methods specific to internal behavior and management of the `BufferPool`.
impl<T: Buffer> BufferPool<T> {
    /// Checks if the buffer pool is operating in the front mode.
    ///
    /// This mode indicates that the back of the buffer pool has been filled and the system is now
    /// using the front section for memory allocation. Returns `true` if the pool is in front mode,
    /// otherwise `false`.
    pub fn is_front_mode(&self) -> bool {
        matches!(self.mode, PoolMode::Front(_))
    }

    /// Checks if the buffer pool is operating in the back mode.
    ///
    /// The back mode is the default state, where the buffer pool first tries to allocate memory.
    /// Returns `true` if the pool is in back mode, otherwise `false`.
    pub fn is_back_mode(&self) -> bool {
        matches!(self.mode, PoolMode::Back)
    }

    /// Checks if the buffer pool is operating in the system memory allocation mode.
    ///
    /// This mode is used when both the back and front sections of the buffer pool are full,
    /// leading the system to allocate memory from the heap, which has performance trade-offs.
    /// Returns `true` if the pool is in alloc mode, otherwise `false`.
    pub fn is_alloc_mode(&self) -> bool {
        matches!(self.mode, PoolMode::Alloc)
    }

    // Resets the buffer pool based on its current mode when the shared state indicates all slices
    // have been dropped, preparing it for reuse.
    //
    // - In back or front mode, the internal memory is moved to the front, and the back is reset.
    // - In alloc mode, system memory is checked and, if smaller than pool capacity, transferred
    //   back into the pool, switching the mode to `Back`.
    #[inline(always)]
    fn reset(&mut self) {
        #[cfg(feature = "debug")]
        println!("RESET");

        self.inner_memory.len = 0;

        match self.mode {
            PoolMode::Back => {
                self.inner_memory.move_raw_at_front();
                self.pool_back.reset();
            }
            PoolMode::Front(_) => {
                self.inner_memory.move_raw_at_front();
                self.mode = PoolMode::Back;
                self.pool_back.reset();
            }
            PoolMode::Alloc => {
                if self.system_memory.len() < self.inner_memory.capacity() {
                    let raw_len = self.system_memory.len();
                    if raw_len > 0 {
                        self.inner_memory
                            .prepend_raw_data(self.system_memory.frame());
                        self.system_memory.get_data_owned();
                    } else {
                        self.inner_memory.reset();
                    }
                    self.mode = PoolMode::Back;
                    self.pool_back.reset();
                }
            }
        }
    }

    // Allocates writable memory of the specified `len` from the heap when the buffer pool cannot
    // fulfill the request.
    //
    // Determines whether to allocate memory from system memory or try to clear memory from the
    // back section of the buffer pool. If clearing is unsuccessful, it may switch to alloc mode or
    // remain in back or front pool modes. When `without_check` is `true`, the function bypasses
    // memory checks and allocates directly from the heap.
    #[inline(never)]
    fn reserve_from_system_memory(
        &mut self,
        len: usize,
        shared_state: u8,
        without_check: bool,
    ) -> &mut [u8] {
        if self.system_memory.len() > 0
            || without_check
            || self.pool_back.len() == 0
            || !self.pool_back.tail_is_clearable(shared_state)
        {
            self.system_memory.reserve(len)
        } else {
            #[cfg(feature = "fuzz")]
            assert!(self.inner_memory.raw_len == 0 && self.inner_memory.raw_offset == 0);

            match self
                .pool_back
                .clear_unchecked(&mut self.inner_memory, shared_state, len)
            {
                Ok(_) => {
                    self.change_mode(PoolMode::Back, len, shared_state);
                    self.reserve_(len, shared_state, false)
                }
                Err(PoolMode::Front(f)) => {
                    self.change_mode(PoolMode::Front(f), len, shared_state);
                    self.reserve_(len, shared_state, false)
                }
                Err(PoolMode::Alloc) => {
                    self.inner_memory.reset_raw();
                    self.system_memory.reserve(len)
                }
                Err(PoolMode::Back) => unreachable!("clearing the back never asks for the back"),
            }
        }
    }

    // Returns ownership of the heap-allocated buffer data by converting it into a `Slice` for
    // further use or processing.
    #[inline(never)]
    fn get_data_owned_from_system_memory(&mut self) -> Slice {
        self.system_memory.get_data_owned().into()
    }

    // Switches the buffer pool to a different mode of operation, adjusting based on the required
    // memory size (`len`).
    //
    // Depending on the current and target modes (back, front, or alloc), this method adjusts the
    // internal buffer pool's state and memory to ensure a smooth transition while allocating the
    // necessary buffer space (`len`), ensuring no data is lost.
    #[inline(always)]
    fn change_mode(&mut self, mode: PoolMode, len: usize, shared_state: u8) {
        match (&mut self.mode, &mode) {
            (PoolMode::Back, PoolMode::Alloc) => {
                #[cfg(feature = "debug")]
                println!(
                    "BACK => ALLOC {} {:?}",
                    self.pool_back.len(),
                    SystemTime::now()
                );

                self.inner_memory.copy_into_buffer(&mut self.system_memory);
                self.inner_memory.reset_raw();
                self.mode = mode;
            }
            (PoolMode::Back, PoolMode::Front(_)) => {
                #[cfg(feature = "fuzz")]
                assert!(shared_state.leading_zeros() > 0);
                #[cfg(feature = "debug")]
                println!("BACK => FRONT");

                self.inner_memory.len = 0;
                self.inner_memory.move_raw_at_front();
                self.mode = mode;
            }
            (PoolMode::Front(_), PoolMode::Back) => {
                #[cfg(feature = "debug")]
                println!("FRONT => BACK");

                if !self.pool_back.tail_is_clearable(shared_state) {
                    self.inner_memory.copy_into_buffer(&mut self.system_memory);
                    self.inner_memory.reset_raw();
                    self.mode = PoolMode::Alloc;
                } else if self.pool_back.try_clear_tail_unchecked(
                    &mut self.inner_memory,
                    shared_state,
                    len,
                ) {
                    self.mode = mode;
                } else {
                    #[cfg(feature = "debug")]
                    println!("ALLOC 2");

                    self.inner_memory.copy_into_buffer(&mut self.system_memory);
                    self.inner_memory.reset_raw();
                    self.mode = PoolMode::Alloc;
                }
            }
            (PoolMode::Alloc, PoolMode::Back) => {
                #[cfg(feature = "debug")]
                println!("ALLOC => BACK {:?}", SystemTime::now());

                self.inner_memory.len = self.pool_back.len() + self.pool_back.back_start();
                self.mode = mode;
            }
            (PoolMode::Alloc, PoolMode::Front(_)) => {
                #[cfg(feature = "fuzz")]
                assert!(shared_state.leading_zeros() > 0);
                #[cfg(feature = "debug")]
                println!("ALLOC => FRONT {:?} {:b}", SystemTime::now(), shared_state);

                self.inner_memory.reset_raw();
                self.inner_memory.len = 0;
                self.mode = mode;
            }
            (PoolMode::Front(_), PoolMode::Alloc) => {
                unreachable!("front mode only ever gives way to the back")
            }
            (PoolMode::Back, PoolMode::Back)
            | (PoolMode::Front(_), PoolMode::Front(_))
            | (PoolMode::Alloc, PoolMode::Alloc) => {
                unreachable!("a mode change always leaves the current mode")
            }
        }
    }

    // Recursively attempts to allocate writable memory of the specified `len`, switching modes if
    // necessary.
    //
    // First tries to allocate memory from the current mode (back, front, or alloc), and if
    // unsuccessful, switches modes and retries, starting with the memory pool before resorting to
    // system memory.
    #[inline(always)]
    fn reserve_(&mut self, len: usize, shared_state: u8, without_check: bool) -> &mut [u8] {
        let writable = match &mut self.mode {
            PoolMode::Back => self
                .pool_back
                .reserve(len, &mut self.inner_memory, shared_state),
            PoolMode::Front(front) => front.reserve(len, &mut self.inner_memory, shared_state),
            PoolMode::Alloc => {
                return self.reserve_from_system_memory(len, shared_state, without_check)
            }
        };

        match writable {
            Ok(offset) => {
                let writable: &mut [u8];
                unsafe {
                    writable = core::slice::from_raw_parts_mut(offset, len);
                }
                writable
            }
            Err(mode) => {
                self.change_mode(mode, len, shared_state);
                let without_check = self.is_alloc_mode();
                self.reserve_(len, shared_state, without_check)
            }
        }
    }
}

impl<T: Buffer> Buffer for BufferPool<T> {
    type Slice = Slice;

    // Provides a mutable slice of length `len` for writing data into the buffer pool.
    //
    // Checks the current `shared_state` to determine if the buffer pool can be reset. If all
    // slices have been dropped and there are allocated slices in `pool_back`, it resets the buffer
    // pool to free up memory. It then attempts to allocate writable memory, which may switch
    // between different modes as needed.
    #[inline(always)]
    fn reserve(&mut self, len: usize) -> &mut [u8] {
        let shared_state = self.inner_memory.memory.load();

        // If all the slices have been dropped, reset the pool to free up memory. A pool in alloc
        // mode with nothing in the back has nothing to clear, so it resets as soon as the frame
        // it was writing to system memory is handed out.
        let alloc_mode_is_done = self.is_alloc_mode() && self.system_memory.len() == 0;
        if shared_state == 0 && (self.pool_back.len() != 0 || alloc_mode_is_done) {
            self.reset();
        }

        self.reserved = len;

        // Attempt to reserve writable memory, potentially switching pool modes
        self.reserve_(len, shared_state, false)
    }

    // Counts the first `len` bytes of the last reservation as written, in whichever memory the
    // reservation was made.
    #[inline(always)]
    fn commit(&mut self, len: usize) {
        assert!(len <= self.reserved, "commit exceeds the last reservation");
        self.reserved = 0;
        match self.mode {
            PoolMode::Alloc => self.system_memory.commit(len),
            _ => self.inner_memory.commit_raw(len),
        }
    }

    // Transfers ownership of the written data as a `Slice`, handling different pool modes.
    //
    // Depending on the current mode (back, front, or alloc), it retrieves the data from the
    // appropriate memory source. In `Back` or `Front` modes, it updates internal state
    // accordingly. In alloc mode, it retrieves data from the heap-allocated system memory.
    #[inline(always)]
    fn get_data_owned(&mut self) -> Self::Slice {
        self.reserved = 0;

        #[cfg(feature = "debug")]
        let mode: u8 = match self.mode {
            PoolMode::Back => 0,
            PoolMode::Front(_) => 1,
            PoolMode::Alloc => 8,
        };

        match &mut self.mode {
            PoolMode::Back => {
                #[cfg(feature = "debug")]
                println!(
                    "{} {} {}",
                    self.inner_memory.raw_offset, self.inner_memory.raw_len, self.inner_memory.len
                );

                let res = self.inner_memory.get_data_owned(
                    #[cfg(feature = "debug")]
                    mode,
                );
                self.pool_back
                    .set_len_from_inner_memory(self.inner_memory.len);

                #[cfg(feature = "debug")]
                {
                    println!(
                        "{} {} {}",
                        self.inner_memory.raw_offset,
                        self.inner_memory.raw_len,
                        self.inner_memory.len
                    );
                    println!("GET DATA BACK {:?}", self.inner_memory.slots);
                }

                res
            }
            PoolMode::Front(f) => {
                let res = self.inner_memory.get_data_owned(
                    #[cfg(feature = "debug")]
                    mode,
                );
                f.len = self.inner_memory.len;

                #[cfg(feature = "debug")]
                println!("GET DATA FRONT {:?}", self.inner_memory.slots);

                res
            }
            PoolMode::Alloc => self.get_data_owned_from_system_memory(),
        }
    }

    // Returns the committed bytes of the frame being written:
    // - In back or front modes, from the internal memory buffer.
    // - In alloc mode, from the system memory buffer.
    fn frame(&self) -> &[u8] {
        match self.mode {
            PoolMode::Alloc => self.system_memory.frame(),
            _ => self.inner_memory.raw_data(),
        }
    }

    // Returns the committed bytes of the frame being written, mutably, from the same memory as
    // `frame`.
    fn frame_mut(&mut self) -> &mut [u8] {
        match self.mode {
            PoolMode::Alloc => self.system_memory.frame_mut(),
            _ => self.inner_memory.raw_data_mut(),
        }
    }

    // Returns the length of the written data in the buffer.
    //
    // The implementation checks the current pool mode to determine where to retrieve the length
    // from:
    // - In back or front modes, it returns the length from `inner_memory.raw_len`.
    // - In alloc mode, it returns the length from the system memory buffer.
    fn len(&self) -> usize {
        match self.mode {
            PoolMode::Alloc => self.system_memory.len(),
            _ => self.inner_memory.raw_len,
        }
    }

    // Drops the committed bytes past `len`, in whichever memory holds the frame, if there are any.
    fn truncate(&mut self, len: usize) {
        self.reserved = 0;
        match self.mode {
            PoolMode::Alloc => Buffer::truncate(&mut self.system_memory, len),
            _ => self.inner_memory.raw_len = self.inner_memory.raw_len.min(len),
        }
    }
}

#[cfg(test)]
impl<T: Buffer> BufferPool<T> {
    // Returns the bits of the slots whose slices are still alive.
    pub(crate) fn live_slots(&self) -> u8 {
        self.inner_memory.memory.load()
    }
}