#![doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md"))]
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(not(loom), no_builtins)]
#![warn(
clippy::all,
clippy::nursery,
clippy::pedantic,
clippy::cargo,
)]
#![allow(
clippy::cargo_common_metadata,
clippy::module_name_repetitions,
clippy::multiple_crate_versions
)]
#[cfg(feature = "alloc")]
extern crate alloc;
#[macro_use]
mod macros;
pub mod consts;
pub mod futures;
pub mod array_queue;
pub mod backoff;
pub mod pad;
pub mod waiter;
use core::fmt::{self, Formatter, Debug, Display};
pub(crate) use waiter::Waiter;
use array_queue::ArrayQueue;
use futures::{FutureRef, FutureExclusiveRef, FutureBlockedRefs};
use pad::CacheLine;
use backoff::Backoff;
use futures::MaybeFuture;
#[macro_export]
macro_rules! maybe_await {
($maybe_fut:expr) => {{
match $maybe_fut {
$crate::futures::MaybeFuture::Future(__fut) => __fut.await,
$crate::futures::MaybeFuture::Ready(__res) => __res
}
}};
}
pub struct RefCount<const MAX_WAITING: usize = 32> {
count: CacheLine<atomic!(AtomicU32, ty)>,
#[cfg(feature = "alloc")]
waiter_queue: alloc::boxed::Box<ArrayQueue<Waiter, MAX_WAITING>>,
#[cfg(not(feature = "alloc"))]
waiter_queue: ArrayQueue<Waiter, MAX_WAITING>,
}
impl<const MAX_WAITING: usize> Default for RefCount<MAX_WAITING> {
#[inline]
fn default() -> Self {
Self::new()
}
}
impl<const MAX_WAITING: usize> RefCount<MAX_WAITING> {
#[cfg(not(feature = "alloc"))]
#[must_use] pub fn new() -> Self {
Self {
count: CacheLine::new(atomic!(u32)),
waiter_queue: ArrayQueue::new()
}
}
#[cfg(feature = "alloc")]
#[must_use] pub fn new() -> Self {
Self {
count: CacheLine::new(atomic!(u32)),
waiter_queue: alloc::boxed::Box::new(ArrayQueue::new())
}
}
#[inline] #[must_use]
pub fn ref_count(&self) -> u32 {
let state = self.count.load(ordering!(Acquire));
standard_ref_count!(state) + block_state!(state)
}
#[inline] #[must_use]
pub fn no_refs(&self) -> bool {
self.ref_count() == 0
}
#[inline] #[must_use]
pub fn standard_ref_count(&self) -> u32 {
standard_ref_count!(self.count.load(ordering!(Acquire)))
}
#[inline] #[must_use]
pub fn no_standard_refs(&self) -> bool {
self.standard_ref_count() == 0
}
#[inline] #[must_use]
pub fn raw_state(&self) -> u32 {
block_state!(self.count.load(ordering!(Acquire)))
}
#[inline] #[must_use]
pub fn is_blocked(&self) -> bool {
self.raw_state() == 1
}
#[inline] #[must_use]
pub fn is_allowed(&self) -> bool {
self.raw_state() == 0
}
#[inline] #[must_use]
pub fn state(&self) -> State {
State::from_raw(self.raw_state())
}
#[inline]
#[must_use = "Ignoring the output brings the opportunity for race conditions and deadlocks."]
pub unsafe fn try_block_refs(&self) -> bool {
let mut ref_count = self.count.load(ordering!(Relaxed)) & state_const!(REF_COUNT_MASK);
let mut next_ref_count = ref_count | state_const!(REF_BLOCKED);
weak_try_update_ref_count!(block, self.count, ref_count, next_ref_count, {return true});
weak_try_update_ref_count!(block, self.count, ref_count, next_ref_count, {return true});
weak_try_update_ref_count!(block, self.count, ref_count, next_ref_count, {return true});
weak_try_update_ref_count!(self.count, ref_count, next_ref_count, {return true}, |_s| {});
false
}
#[inline]
pub unsafe fn block_refs(&self) -> MaybeFuture<(), FutureBlockedRefs<MAX_WAITING>> {
if !self.try_block_refs() {
return MaybeFuture::Future(FutureBlockedRefs::new(self))
}
MaybeFuture::Ready(())
}
#[inline]
pub unsafe fn unblock_refs(&self) {
self.count.store(state_const!(REF_ALLOWED), ordering!(Release));
self.waiter_queue.pop()
.map_or_else(|| slight_spin!(0), Waiter::wake);
while let Some(waker) = self.waiter_queue.pop() {
waker.wake();
}
}
pub fn get_exclusive_ref(&self) -> MaybeFuture<ExclusiveRef<MAX_WAITING>, FutureExclusiveRef<MAX_WAITING>> {
if unsafe { self.try_block_refs() } {
if self.no_standard_refs() {
MaybeFuture::Ready(unsafe { ExclusiveRef::new(self) })
} else {
MaybeFuture::Future(FutureExclusiveRef::new(
self,
None
))
}
} else {
MaybeFuture::Future(FutureExclusiveRef::new(
self,
Some(FutureBlockedRefs::new(self))
))
}
}
pub fn try_get_exclusive_ref(&self) -> Option<ExclusiveRef<MAX_WAITING>> {
if unsafe { self.try_block_refs() } {
for i in 0..5 {
if self.no_standard_refs() {
return Some(unsafe { ExclusiveRef::new(self) })
}
slight_spin!(i);
}
unsafe { self.unblock_refs() };
}
None
}
pub fn blocking_get_exclusive_ref(&self) -> ExclusiveRef<MAX_WAITING> {
let backoff = Backoff::new();
while unsafe { !self.try_block_refs() } {
backoff.snooze();
}
while !self.no_standard_refs() {
backoff.snooze();
}
unsafe { ExclusiveRef::new(self) }
}
#[inline]
pub fn get_ref(&self) -> MaybeFuture<Ref<MAX_WAITING>, FutureRef<MAX_WAITING>> {
let mut ref_count = self.count.load(ordering!(Relaxed)) & state_const!(REF_COUNT_MASK);
let mut next_ref_count = ref_count + state_const!(REF_INCR);
many_weak_try_incr_refs!(self.count, ref_count, next_ref_count, {
return MaybeFuture::Ready(unsafe { Ref::new(self) })
});
return MaybeFuture::Future(FutureRef::new(self))
}
#[inline]
pub fn try_get_ref(&self) -> Option<Ref<MAX_WAITING>> {
let mut ref_count = self.count.load(ordering!(Relaxed)) & state_const!(REF_COUNT_MASK);
let mut next_ref_count = ref_count + state_const!(REF_INCR);
many_weak_try_incr_refs!(self.count, ref_count, next_ref_count, {
return Some(unsafe { Ref::new(self) })
});
None
}
pub fn blocking_get_ref(&self) -> Ref<MAX_WAITING> {
let backoff = Backoff::new();
let mut ref_count = self.count.load(ordering!(Relaxed)) & state_const!(REF_COUNT_MASK);
let mut next_ref_count = ref_count + state_const!(REF_INCR);
loop {
many_weak_try_incr_refs!(self.count, ref_count, next_ref_count, {
return unsafe { Ref::new(self) }
});
backoff.snooze();
}
}
}
impl<const MAX_WAITING: usize> Debug for RefCount<MAX_WAITING> {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
let state = self.count.load(ordering!(Relaxed));
f.debug_struct("RefCount")
.field("state", &State::from_raw(state))
.field("references", &(standard_ref_count!(state) + block_state!(state)))
.field("waiter_queue", &self.waiter_queue)
.finish()
}
}
#[repr(transparent)]
pub struct Ref<'count, const MAX_WAITING: usize> {
src: &'count RefCount<MAX_WAITING>
}
impl<'count, const MAX_WAITING: usize> Ref<'count, MAX_WAITING> {
#[inline]
pub const unsafe fn new(counter: &'count RefCount<MAX_WAITING>) -> Self {
Self {
src: counter
}
}
}
impl<'count, const MAX_WAITING: usize> Drop for Ref<'count, MAX_WAITING> {
#[inline]
fn drop(&mut self) {
self.src.count.fetch_sub(state_const!(REF_INCR), ordering!(Release));
}
}
#[repr(transparent)]
pub struct ExclusiveRef<'count, const MAX_WAITING: usize> {
src: &'count RefCount<MAX_WAITING>
}
impl<'count, const MAX_WAITING: usize> ExclusiveRef<'count, MAX_WAITING> {
#[inline]
pub const unsafe fn new(ref_count: &'count RefCount<MAX_WAITING>) -> Self {
Self { src: ref_count }
}
}
impl<'count, const MAX_WAITING: usize> Drop for ExclusiveRef<'count, MAX_WAITING> {
#[inline]
fn drop(&mut self) {
unsafe {
self.src.unblock_refs();
}
}
}
#[repr(u32)]
pub enum State {
Blocked = 1,
Allowed = 0
}
impl State {
#[inline] #[must_use]
pub const fn from_raw(state: u32) -> Self {
unsafe { core::mem::transmute(block_state!(state)) }
}
#[inline] #[must_use]
pub const fn is_allowed(&self) -> bool {
matches!(self, Self::Allowed)
}
#[inline] #[must_use]
pub const fn is_blocked(&self) -> bool {
matches!(self, Self::Blocked)
}
}
macro_rules! __state_format {
($state:ident, $formatter:ident) => {
match $state {
Self::Allowed => $formatter.write_str("allowed"),
Self::Blocked => $formatter.write_str("blocked")
}
};
}
impl Display for State {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
__state_format!(self, f)
}
}
impl Debug for State {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
__state_format!(self, f)
}
}
#[cfg(test)]
mod tests {
use crate::{MaybeFuture, RefCount, State};
use core::task::{Poll, RawWaker, RawWakerVTable, Context, Waker};
use core::pin::pin;
use core::future::Future;
static VTABLE: RawWakerVTable = RawWakerVTable::new(
|_ptr| RawWaker::new(&() as *const _ as *const (), &VTABLE),
|_ptr| (),
|_ptr| (),
|_ptr| (),
);
macro_rules! poll {
($pinned_fut:expr) => {{
$pinned_fut.as_mut().poll(&mut Context::from_waker(
&unsafe { Waker::from_raw(RawWaker::new(&() as *const _ as *const (), &VTABLE))}
))
}};
}
#[test]
fn exclusive_vs_ref() {
let ref_counter = RefCount::<30>::new();
let r = match ref_counter.get_ref() {
MaybeFuture::Ready(r) => r,
MaybeFuture::Future(_) => panic!("Went to future without contention")
};
let deadlock_fut = match ref_counter.get_exclusive_ref() {
MaybeFuture::Ready(_) => panic!("should have deadlocked but was ready"),
MaybeFuture::Future(fut) => fut
};
let not_yet = match ref_counter.get_ref() {
MaybeFuture::Ready(_) => panic!("Got new ref after exclusive ref made request"),
MaybeFuture::Future(f) => f,
};
let mut not_yet_fut = pin!(not_yet);
let mut fut = pin!(deadlock_fut);
for _ in 0..5000 {
assert!(matches!(poll!(fut), Poll::Pending));
}
drop(r);
assert!(matches!(poll!(fut), Poll::Ready(_)));
assert!(matches!(poll!(not_yet_fut), Poll::Ready(_)));
}
#[test]
fn state_enum_from_raw() {
let e = State::from_raw(1);
assert!(e.is_blocked());
let e = State::from_raw(0);
assert!(e.is_allowed());
}
}