takoude 0.2.1

Lock-free SPSC ring buffer with per-slot sequence flags, tuned for low latency.
Documentation
use crate::slot::Slot;
use core::sync::atomic::{AtomicU64, Ordering};

/// A cache-padded `AtomicU64` on its own 128-byte region so the consumer's
/// progress counter and the producer's final-position marker never share a
/// line with anything else (128 rather than 64 defeats Intel's adjacent-line
/// prefetcher).
#[repr(align(128))]
pub(crate) struct PaddedCounter(pub(crate) AtomicU64);

/// Fixed-size ring storage: `N` cache-line-aligned slots in one contiguous
/// allocation, plus two padded counters that stay off every hot path.
///
/// `progress` is the consumer's position, published on each pop. It lives on
/// a consumer-owned line; the producer reads it only when its cached credit
/// runs out (at most once per lap), so it never ping-pongs. `producer_final`
/// is written once, when the producer handle drops, so `Drop` can locate the
/// live `[progress, producer_final)` range.
///
/// Payloads start uninitialized; only the sequence flags are initialized.
/// There are no head/tail atomics on the hot paths; positions are plain
/// (non-atomic) locals inside `Producer` and `Consumer`.
pub(crate) struct Ring<T, const N: usize> {
    slots: Box<[Slot<T>]>,
    pub(crate) progress: PaddedCounter,
    pub(crate) producer_final: PaddedCounter,
}

// `UnsafeCell<MaybeUninit<T>>` suppresses the auto-impls; access is
// serialized by the publish-flag/credit protocol, so sharing is sound for
// `T: Send`.
unsafe impl<T: Send, const N: usize> Send for Ring<T, N> {}
unsafe impl<T: Send, const N: usize> Sync for Ring<T, N> {}

impl<T, const N: usize> Ring<T, N> {
    pub(crate) const MASK: u64 = N as u64 - 1;

    /// `base` is the initial position of both producer and consumer. It must
    /// be a multiple of `N` so that slot `i` first expects position
    /// `base + i`. (Non-zero bases exist to let tests start right below
    /// `u64::MAX` and exercise wraparound.)
    pub(crate) fn new(base: u64) -> Self {
        const {
            assert!(N >= 2, "ring capacity must be at least 2");
            assert!(N.is_power_of_two(), "ring capacity must be a power of two");
        }
        debug_assert!(base & Self::MASK == 0);
        let slots = (0..N as u64)
            .map(|i| Slot::empty(base.wrapping_add(i)))
            .collect();
        Ring {
            slots,
            progress: PaddedCounter(AtomicU64::new(base)),
            producer_final: PaddedCounter(AtomicU64::new(base)),
        }
    }

    #[inline(always)]
    pub(crate) fn slot(&self, pos: u64) -> &Slot<T> {
        // Power-of-two capacity: bitwise mask instead of modulo. The mask
        // keeps the index in bounds by construction; skip the bounds check
        // on the hot path.
        unsafe { self.slots.get_unchecked((pos & Self::MASK) as usize) }
    }

    /// Base pointer of the slot array. Stable for the ring's lifetime (the
    /// boxed slice never moves), so the handles cache it to avoid a
    /// serially-dependent load of the `Box` data pointer on every push/pop.
    pub(crate) fn slots_ptr(&self) -> core::ptr::NonNull<Slot<T>> {
        unsafe { core::ptr::NonNull::new_unchecked(self.slots.as_ptr().cast_mut()) }
    }
}

impl<T, const N: usize> Drop for Ring<T, N> {
    fn drop(&mut self) {
        // The ring is only dropped once both handles are gone; the Arc
        // refcount decrement provides the necessary happens-before edges.
        //
        // `progress` is exact (published on every pop) and the producer
        // stored its final position when it dropped, so the live values are
        // exactly the positions in `[progress, producer_final)`.
        if core::mem::needs_drop::<T>() {
            let consumed = self.progress.0.load(Ordering::Acquire);
            let produced = self.producer_final.0.load(Ordering::Acquire);
            let mut pos = consumed;
            while pos != produced {
                unsafe { (*self.slot(pos).payload.get()).assume_init_drop() };
                pos = pos.wrapping_add(1);
            }
        }
    }
}