#![deny(missing_docs)]
#![cfg_attr(docsrs, feature(doc_cfg))]
mod ring;
mod slot;
use core::sync::atomic::Ordering::{Acquire, Release};
use ring::Ring;
use std::sync::Arc;
#[inline(always)]
fn prefetcht0_line<T>(_slot: &slot::Slot<T>) {
#[cfg(all(feature = "readahead", target_arch = "x86_64"))]
unsafe {
core::arch::asm!(
"prefetcht0 [{p}]",
p = in(reg) core::ptr::from_ref(_slot),
options(nostack, preserves_flags, readonly),
);
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct Full<T>(pub T);
impl<T> Full<T> {
pub fn into_inner(self) -> T {
self.0
}
}
impl<T> core::fmt::Display for Full<T> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.write_str("ring buffer is full")
}
}
impl<T: core::fmt::Debug> std::error::Error for Full<T> {}
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
pub unsafe trait AtomicValue: Copy {}
#[cfg(feature = "packed")]
macro_rules! impl_atomic_value {
($($ty:ty),+ $(,)?) => {
$(
unsafe impl AtomicValue for $ty {}
)+
};
}
#[cfg(feature = "packed")]
impl_atomic_value!(
(),
bool,
char,
u8,
i8,
u16,
i16,
u32,
i32,
u64,
i64,
usize,
isize,
f32,
f64
);
#[cfg(feature = "packed")]
unsafe impl<T: AtomicValue, const M: usize> AtomicValue for [T; M] {}
pub fn spsc_channel<T: Send, const N: usize>() -> (Producer<T, N>, Consumer<T, N>) {
spsc_channel_at(0)
}
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
pub fn spsc_packed_channel<T: Send + AtomicValue, const N: usize>()
-> (PackedProducer<T, N>, PackedConsumer<T, N>) {
spsc_packed_channel_at(0)
}
#[cfg(feature = "packed")]
#[doc(hidden)]
pub fn spsc_packed_channel_at<T: Send + AtomicValue, const N: usize>(
base: u64,
) -> (PackedProducer<T, N>, PackedConsumer<T, N>) {
const {
assert!(
size_of::<T>() <= 4,
"packed channel requires size_of::<T>() <= 4"
);
assert!(
align_of::<T>() <= 4,
"packed channel requires align_of::<T>() <= 4"
);
assert!((N as u128) < 1 << 32, "packed channel requires N < 2^32");
}
let (producer, consumer) = spsc_channel_at(base);
(
PackedProducer { inner: producer },
PackedConsumer { inner: consumer },
)
}
#[doc(hidden)]
pub fn spsc_channel_at<T: Send, const N: usize>(base: u64) -> (Producer<T, N>, Consumer<T, N>) {
assert!(
base & (N as u64 - 1) == 0,
"base must be a multiple of the capacity"
);
let ring = Arc::new(Ring::<T, N>::new(base));
let slots = ring.slots_ptr();
(
Producer {
slots,
ring: Arc::clone(&ring),
pos: base,
limit: base.wrapping_add(N as u64),
},
Consumer {
slots,
ring,
pos: base,
},
)
}
pub struct Producer<T: Send, const N: usize> {
slots: core::ptr::NonNull<slot::Slot<T>>,
pos: u64,
limit: u64,
ring: Arc<Ring<T, N>>,
}
unsafe impl<T: Send, const N: usize> Send for Producer<T, N> {}
unsafe impl<T: Send, const N: usize> Sync for Producer<T, N> {}
impl<T: Send, const N: usize> Producer<T, N> {
#[inline(always)]
fn slot(&self, pos: u64) -> &slot::Slot<T> {
unsafe { &*self.slots.as_ptr().add((pos & Ring::<T, N>::MASK) as usize) }
}
}
impl<T: Send, const N: usize> Drop for Producer<T, N> {
fn drop(&mut self) {
self.ring.producer_final.0.store(self.pos, Release);
}
}
pub struct Consumer<T: Send, const N: usize> {
slots: core::ptr::NonNull<slot::Slot<T>>,
pos: u64,
ring: Arc<Ring<T, N>>,
}
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
pub struct PackedProducer<T: Send + AtomicValue, const N: usize> {
inner: Producer<T, N>,
}
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
pub struct PackedConsumer<T: Send + AtomicValue, const N: usize> {
inner: Consumer<T, N>,
}
unsafe impl<T: Send, const N: usize> Send for Consumer<T, N> {}
unsafe impl<T: Send, const N: usize> Sync for Consumer<T, N> {}
impl<T: Send, const N: usize> Consumer<T, N> {
#[inline(always)]
fn slot(&self, pos: u64) -> &slot::Slot<T> {
unsafe { &*self.slots.as_ptr().add((pos & Ring::<T, N>::MASK) as usize) }
}
}
impl<T: Send, const N: usize> Producer<T, N> {
#[cold]
fn refresh_limit(&mut self) -> bool {
self.limit = self.ring.progress.0.load(Acquire).wrapping_add(N as u64);
self.pos != self.limit
}
#[inline(always)]
pub fn try_push(&mut self, value: T) -> Result<(), Full<T>> {
if self.pos == self.limit && !self.refresh_limit() {
return Err(Full(value));
}
let slot = self.slot(self.pos);
unsafe { slot.payload.get().cast::<T>().write(value) };
slot.seq.store(self.pos.wrapping_add(1), Release);
self.pos = self.pos.wrapping_add(1);
Ok(())
}
#[inline(always)]
pub fn try_push_with<F: FnOnce() -> T>(&mut self, f: F) -> Result<(), F> {
if self.pos == self.limit && !self.refresh_limit() {
return Err(f);
}
let slot = self.slot(self.pos);
unsafe { slot.payload.get().cast::<T>().write(f()) };
slot.seq.store(self.pos.wrapping_add(1), Release);
self.pos = self.pos.wrapping_add(1);
Ok(())
}
#[inline]
pub fn push(&mut self, value: T) {
while self.pos == self.limit && !self.refresh_limit() {
core::hint::spin_loop();
}
let slot = self.slot(self.pos);
unsafe { slot.payload.get().cast::<T>().write(value) };
slot.seq.store(self.pos.wrapping_add(1), Release);
self.pos = self.pos.wrapping_add(1);
}
#[inline]
pub fn is_full(&self) -> bool {
self.pos == self.limit
&& self.pos == self.ring.progress.0.load(Acquire).wrapping_add(N as u64)
}
#[inline]
pub const fn capacity(&self) -> usize {
N
}
#[inline]
pub fn is_disconnected(&self) -> bool {
Arc::strong_count(&self.ring) == 1
}
}
#[cfg(feature = "packed")]
impl<T: Send + AtomicValue, const N: usize> Producer<T, N> {
#[inline(always)]
fn try_push_packed(&mut self, value: T) -> Result<(), Full<T>> {
const {
assert!(
size_of::<T>() <= 4,
"packed push requires size_of::<T>() <= 4"
);
assert!(
align_of::<T>() <= 4,
"packed push requires align_of::<T>() <= 4"
);
assert!((N as u128) < 1 << 32, "packed paths require N < 2^32");
}
if self.pos == self.limit && !self.refresh_limit() {
return Err(Full(value));
}
let slot = self.slot(self.pos);
let mut bits = 0u32;
unsafe {
core::ptr::copy_nonoverlapping(
(&raw const value).cast::<u8>(),
(&raw mut bits).cast::<u8>(),
size_of::<T>(),
)
};
let flag = self.pos.wrapping_add(1) as u32 as u64;
slot.seq.store((bits as u64) << 32 | flag, Release);
self.pos = self.pos.wrapping_add(1);
Ok(())
}
#[inline]
fn push_packed(&mut self, value: T) {
while self.pos == self.limit && !self.refresh_limit() {
core::hint::spin_loop();
}
let res = self.try_push_packed(value);
debug_assert!(res.is_ok(), "push_packed: slot vanished after spin");
let _ = res;
}
}
#[cfg(feature = "packed")]
impl<T: Send + AtomicValue, const N: usize> Consumer<T, N> {
#[inline(always)]
fn try_pop_packed(&mut self) -> Option<T> {
const {
assert!(
size_of::<T>() <= 4,
"packed pop requires size_of::<T>() <= 4"
);
assert!(
align_of::<T>() <= 4,
"packed pop requires align_of::<T>() <= 4"
);
assert!((N as u128) < 1 << 32, "packed paths require N < 2^32");
}
let slot = self.slot(self.pos);
let word = slot.seq.load(Acquire);
if word as u32 != self.pos.wrapping_add(1) as u32 {
return None;
}
let bits = (word >> 32) as u32;
let value = unsafe { core::ptr::read((&raw const bits).cast::<T>()) };
self.pos = self.pos.wrapping_add(1);
self.ring.progress.0.store(self.pos, Release);
Some(value)
}
#[inline]
fn pop_packed(&mut self) -> T {
loop {
if let Some(value) = self.try_pop_packed() {
return value;
}
core::hint::spin_loop();
}
}
}
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
impl<T: Send + AtomicValue, const N: usize> PackedProducer<T, N> {
#[inline(always)]
pub fn try_push(&mut self, value: T) -> Result<(), Full<T>> {
self.inner.try_push_packed(value)
}
#[inline]
pub fn push(&mut self, value: T) {
self.inner.push_packed(value);
}
#[inline]
pub fn is_full(&self) -> bool {
self.inner.is_full()
}
#[inline]
pub const fn capacity(&self) -> usize {
N
}
#[inline]
pub fn is_disconnected(&self) -> bool {
self.inner.is_disconnected()
}
}
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
impl<T: Send + AtomicValue, const N: usize> PackedConsumer<T, N> {
#[inline(always)]
pub fn try_pop(&mut self) -> Option<T> {
self.inner.try_pop_packed()
}
#[inline]
pub fn pop(&mut self) -> T {
self.inner.pop_packed()
}
#[inline]
pub fn is_empty(&self) -> bool {
let slot = self.inner.slot(self.inner.pos);
slot.seq.load(Acquire) as u32 != self.inner.pos.wrapping_add(1) as u32
}
#[inline]
pub const fn capacity(&self) -> usize {
N
}
#[inline]
pub fn is_disconnected(&self) -> bool {
self.inner.is_disconnected()
}
}
impl<T: Send, const N: usize> Consumer<T, N> {
#[inline(always)]
pub fn try_pop(&mut self) -> Option<T> {
let slot = self.slot(self.pos);
if slot.seq.load(Acquire) != self.pos.wrapping_add(1) {
return None;
}
let value = unsafe { slot.payload.get().cast::<T>().read() };
self.pos = self.pos.wrapping_add(1);
self.ring.progress.0.store(self.pos, Release);
prefetcht0_line(self.slot(self.pos.wrapping_add(31)));
Some(value)
}
#[inline(always)]
pub fn try_pop_with<R, F: FnOnce(&mut T) -> R>(&mut self, f: F) -> Option<R> {
let slot = self.slot(self.pos);
if slot.seq.load(Acquire) != self.pos.wrapping_add(1) {
return None;
}
let payload = unsafe { &mut *slot.payload.get().cast::<T>() };
let result = f(payload);
struct Commit<'a> {
pos: &'a mut u64,
progress: &'a core::sync::atomic::AtomicU64,
}
impl Drop for Commit<'_> {
#[inline]
fn drop(&mut self) {
*self.pos = self.pos.wrapping_add(1);
self.progress.store(*self.pos, Release);
}
}
let commit = Commit {
pos: &mut self.pos,
progress: &self.ring.progress.0,
};
unsafe { core::ptr::drop_in_place(payload) };
drop(commit);
prefetcht0_line(self.slot(self.pos.wrapping_add(31)));
Some(result)
}
#[inline]
pub fn pop(&mut self) -> T {
let slot = self.slot(self.pos);
let full_seq = self.pos.wrapping_add(1);
while slot.seq.load(Acquire) != full_seq {
core::hint::spin_loop();
}
let value = unsafe { slot.payload.get().cast::<T>().read() };
self.pos = self.pos.wrapping_add(1);
self.ring.progress.0.store(self.pos, Release);
prefetcht0_line(self.slot(self.pos.wrapping_add(31)));
value
}
#[inline]
pub fn is_empty(&self) -> bool {
self.slot(self.pos).seq.load(Acquire) != self.pos.wrapping_add(1)
}
#[inline]
pub const fn capacity(&self) -> usize {
N
}
#[inline]
pub fn is_disconnected(&self) -> bool {
Arc::strong_count(&self.ring) == 1
}
}