#[cfg(not(loom))]
mod sys {
pub use std::{hint, sync};
pub mod cell {
#[derive(Debug)]
pub struct UnsafeCell<T>(std::cell::UnsafeCell<T>);
impl<T> UnsafeCell<T> {
pub fn new(data: T) -> UnsafeCell<T> {
UnsafeCell(std::cell::UnsafeCell::new(data))
}
pub fn with_mut<F, U>(&self, f: F) -> U
where
F: FnOnce(*mut T) -> U,
{
f(self.0.get())
}
}
}
}
#[cfg(loom)]
mod sys {
pub use loom::{cell, sync};
pub mod hint {
pub use loom::hint::*;
pub fn cold_path() {}
}
}
mod cache_padded;
mod slots;
mod try_lock;
use std::{
cmp,
mem::MaybeUninit,
ops::{Deref, DerefMut, Not},
pin::Pin,
ptr,
task::{Context, Poll, Waker},
};
use self::{
slots::{SlotIndex, Slots},
sys::{
cell::UnsafeCell,
hint,
sync::{
Arc,
mpsc::{Receiver, Sender, channel},
},
},
try_lock::TryLock,
};
#[derive(Debug, PartialEq)]
pub enum AcquireError {
Closed,
AllSlotsOccupied,
}
#[derive(Debug, PartialEq)]
enum ReleaseError {
InvalidPosition,
}
pub struct Pool<T> {
inner: Arc<PoolInner<T>>,
size: usize,
}
struct PoolInner<T> {
blocks: Box<[Slots]>,
objects: Box<[UnsafeCell<Option<T>>]>,
builder: Box<dyn Fn() -> T>,
get_wakers_sender: Sender<Waker>,
get_wakers_receiver: Receiver<Waker>,
closing_state: TryLock<ClosingState>,
}
#[must_use = "`Object` will immediately return in the pool if unused"]
pub struct Object<T> {
position: Position,
value: MaybeUninit<T>,
pool_inner: Arc<PoolInner<T>>,
}
impl<T> Pool<T> {
pub fn new<B>(desired_size: usize, builder: B) -> Self
where
B: Fn() -> T + 'static,
{
let size = compute_size(desired_size);
let (waker_sender, waker_receiver) = channel();
Self {
size,
inner: Arc::new(PoolInner {
blocks: {
let number_of_slots = size >> Slots::SLOTS_SIZE.ilog2();
let mut keepers = Vec::with_capacity(number_of_slots);
for _ in 0..number_of_slots {
keepers.push(Slots::new());
}
keepers.into_boxed_slice()
},
objects: {
let mut objects = Vec::with_capacity(size);
for _ in 0..size {
objects.push(UnsafeCell::new(None));
}
objects.into_boxed_slice()
},
builder: Box::new(builder),
get_wakers_sender: waker_sender,
get_wakers_receiver: waker_receiver,
closing_state: TryLock::new(ClosingState::Opened),
}),
}
}
pub fn size(&self) -> usize {
self.size
}
pub fn is_closed(&self) -> bool {
self.inner.is_closed()
}
pub fn close(self) -> Close<T> {
Close::new(self)
}
pub fn reopen(&self) {
self.inner.reopen();
}
pub fn try_get(&self) -> Result<Object<T>, AcquireError> {
let (position, object) = self.inner.try_get()?;
Ok(Object::new(position, object, self.inner.clone()))
}
pub fn get(&self) -> Get<T> {
Get::new(&self.inner)
}
}
#[inline]
fn compute_size(desired_size: usize) -> usize {
cmp::max(Slots::SLOTS_SIZE as _, desired_size).next_power_of_two()
}
impl<T> PoolInner<T> {
fn is_closed(&self) -> bool {
loop {
if let Some(closing_state) = self.closing_state.try_read_lock() {
return matches!(*closing_state, ClosingState::Opened).not();
}
hint::cold_path();
hint::spin_loop();
}
}
fn close(&self) {
loop {
if let Some(mut closing_state) = self.closing_state.try_write_lock()
{
*closing_state = ClosingState::Closing(None);
break;
}
hint::cold_path();
hint::spin_loop();
}
self.wake_pending_get_futures();
}
fn reopen(&self) {
loop {
if let Some(mut closing_state) = self.closing_state.try_write_lock()
{
*closing_state = ClosingState::Opened;
break;
}
hint::cold_path();
hint::spin_loop();
}
}
fn try_get(&self) -> Result<(Position, T), AcquireError> {
if self.is_closed() {
return Err(AcquireError::Closed);
}
for (block_index, slots) in self.blocks.iter().enumerate() {
let Ok(slot_index) = slots.try_acquire() else {
continue;
};
let object_position = Position { block_index, slot_index };
let object_index = object_position.as_object_index();
let object_value = {
assert!(object_index < self.objects.len());
let cell = &self.objects[object_index];
cell.with_mut(|cell_ptr| {
let object_place = unsafe {
&mut *cell_ptr
};
object_place.take().unwrap_or_else(|| (self.builder)())
})
};
return Ok((object_position, object_value));
}
Err(AcquireError::AllSlotsOccupied)
}
fn try_release(
&self,
object_position: &Position,
object_value: T,
) -> Result<(), ReleaseError> {
let Some(block) = self.blocks.get(object_position.block_index) else {
return Err(ReleaseError::InvalidPosition);
};
let object_index = object_position.as_object_index();
{
assert!(object_index < self.objects.len());
let cell = &self.objects[object_index];
cell.with_mut(|cell_ptr| {
unsafe {
ptr::replace(cell_ptr, Some(object_value));
}
});
};
block.release(object_position.slot_index);
self.wake_pending_get_futures();
self.wake_pending_close_future();
Ok(())
}
fn wake_pending_get_futures(&self) {
let wakers =
TryIter::new(&self.get_wakers_receiver).collect::<Vec<_>>();
for waker in wakers {
waker.wake()
}
}
fn wake_pending_close_future(&self) {
loop {
if let Some(closing_state) = self.closing_state.try_read_lock() {
if let ClosingState::Closing(maybe_waker) = &*closing_state
&& let Some(waker) = maybe_waker.as_ref()
{
waker.wake_by_ref();
}
break;
}
hint::cold_path();
hint::spin_loop();
}
}
}
unsafe impl<T> Send for PoolInner<T> where T: Send {}
unsafe impl<T> Sync for PoolInner<T> where T: Sync {}
impl<T> Object<T> {
fn new(
position: Position,
value: T,
pool_inner: Arc<PoolInner<T>>,
) -> Self {
Self { position, value: MaybeUninit::new(value), pool_inner }
}
}
impl<T> Drop for Object<T> {
fn drop(&mut self) {
let value = unsafe { self.value.assume_init_read() };
unsafe {
self.value.assume_init_drop();
}
if self.pool_inner.try_release(&self.position, value).is_err() {
hint::cold_path();
unreachable!(
"`Object` has an invalid position, API has been misused"
);
}
}
}
impl<T> Deref for Object<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
unsafe { self.value.assume_init_ref() }
}
}
impl<T> DerefMut for Object<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { self.value.assume_init_mut() }
}
}
struct Position {
block_index: usize,
slot_index: SlotIndex,
}
impl Position {
fn as_object_index(&self) -> usize {
self.block_index * Slots::SLOTS_SIZE as usize + self.slot_index as usize
}
}
#[must_use = "futures do nothing unless you `.await` or poll them"]
pub struct Get<T> {
pool_inner: Option<Arc<PoolInner<T>>>,
}
impl<T> Get<T> {
fn new(inner: &Arc<PoolInner<T>>) -> Self {
Self { pool_inner: Some(inner.clone()) }
}
}
impl<T> Future for Get<T> {
type Output = Option<Object<T>>;
fn poll(
mut self: Pin<&mut Self>,
context: &mut Context<'_>,
) -> Poll<Self::Output> {
if self.pool_inner.is_none() {
hint::cold_path();
unreachable!(
"The `Get` future has already been resolved: it is violating the contract of `Future`"
);
}
unsafe {
match self.pool_inner.as_ref().unwrap_unchecked().try_get() {
Ok((position, object)) => Poll::Ready({
let pool_inner = self.pool_inner.take().unwrap_unchecked();
Some(Object::new(position, object, pool_inner))
}),
Err(AcquireError::Closed) => Poll::Ready(None),
Err(AcquireError::AllSlotsOccupied) => {
let waker = context.waker().clone();
let _ = self
.pool_inner
.as_ref()
.unwrap_unchecked()
.get_wakers_sender
.send(waker);
Poll::Pending
}
}
}
}
}
#[must_use = "futures do nothing unless you `.await` or poll them"]
pub struct Close<T> {
pool: Option<Pool<T>>,
}
impl<T> Close<T> {
fn new(pool: Pool<T>) -> Self {
pool.inner.close();
Self { pool: Some(pool) }
}
}
impl<T> Future for Close<T> {
type Output = Pool<T>;
fn poll(
mut self: Pin<&mut Self>,
context: &mut Context<'_>,
) -> Poll<Self::Output> {
if self.pool.is_none() {
hint::cold_path();
unreachable!(
"The `Close` future has already been resolved: it is violating the contract of `Future`"
);
}
unsafe {
let mut closing_state = loop {
if let Some(closing_state) = self
.pool
.as_ref()
.unwrap_unchecked()
.inner
.closing_state
.try_write_lock()
{
break closing_state;
}
hint::cold_path();
hint::spin_loop();
};
let all_vacants = self
.pool
.as_ref()
.unwrap_unchecked()
.inner
.blocks
.iter()
.all(|slot| slot.is_all_vacants());
if all_vacants {
*closing_state = ClosingState::Closed;
drop(closing_state);
Poll::Ready(self.pool.take().unwrap_unchecked())
} else {
let waker = context.waker().clone();
*closing_state = ClosingState::Closing(Some(waker));
drop(closing_state);
Poll::Pending
}
}
}
}
enum ClosingState {
Opened,
Closing(Option<Waker>),
Closed,
}
#[derive(Debug)]
struct TryIter<'a, T: 'a> {
receiver: &'a Receiver<T>,
}
impl<'a, T> TryIter<'a, T> {
fn new(receiver: &'a Receiver<T>) -> Self {
Self { receiver }
}
}
impl<'a, T> Iterator for TryIter<'a, T> {
type Item = T;
fn next(&mut self) -> Option<T> {
self.receiver.try_recv().ok()
}
}
#[cfg(test)]
mod tests_helper {
#[cfg(not(loom))]
pub fn run(test: impl Fn()) {
test()
}
#[cfg(loom)]
pub use loom::model as run;
}
#[cfg(test)]
mod tests {
#[cfg(not(loom))]
mod sys {
pub use std::thread;
pub use futures_executor::block_on;
}
#[cfg(loom)]
mod sys {
pub use loom::future::block_on;
pub mod thread {
pub use std::thread::sleep;
pub use loom::thread::spawn;
}
}
use std::{ops::Not, time::Duration};
use self::sys::{block_on, thread};
use super::{AcquireError, Pool, compute_size, tests_helper::run};
struct T {
x: u8,
}
fn thread_wait() {
thread::sleep(Duration::from_millis(1));
#[cfg(loom)]
loom::thread::yield_now();
}
#[test]
fn test_compute_size() {
for desired_size in 0..=8 {
assert_eq!(compute_size(desired_size), 8);
}
for desired_size in 9..=16 {
assert_eq!(compute_size(desired_size), 16);
}
for desired_size in 17..=32 {
assert_eq!(compute_size(desired_size), 32);
}
}
#[test]
fn test_try_get() {
run(|| {
let pool = Pool::new(8, || T { x: 0 });
let _object_0 = pool.try_get().unwrap();
let _object_1 = pool.try_get().unwrap();
let _object_2 = pool.try_get().unwrap();
let _object_3 = pool.try_get().unwrap();
let _object_4 = pool.try_get().unwrap();
let _object_5 = pool.try_get().unwrap();
let _object_6 = pool.try_get().unwrap();
let _object_7 = pool.try_get().unwrap();
assert!(pool.try_get().is_err());
})
}
#[test]
fn test_get() {
run(|| {
let pool = Pool::new(8, || T { x: 0 });
block_on(async {
let mut object_0 = pool.get().await.unwrap();
object_0.x = 7;
});
block_on(async {
assert_eq!(pool.get().await.unwrap().x, 7);
});
let _object_0 = pool.try_get().unwrap();
let _object_1 = pool.try_get().unwrap();
let _object_2 = pool.try_get().unwrap();
let _object_3 = pool.try_get().unwrap();
let mut object_4 = pool.try_get().unwrap();
let _object_5 = pool.try_get().unwrap();
let _object_6 = pool.try_get().unwrap();
let _object_7 = pool.try_get().unwrap();
assert!(pool.try_get().is_err());
let thread_handle = thread::spawn({
move || {
block_on(async {
assert_eq!(pool.get().await.unwrap().x, 42);
})
}
});
object_4.x = 42;
thread_wait();
drop(object_4);
thread_handle.join().unwrap();
})
}
#[test]
#[cfg(not(loom))]
fn test_get_returns_none() {
run(|| {
let pool = Pool::new(8, || T { x: 0 });
let object_0 = pool.try_get().unwrap();
let object_1 = pool.try_get().unwrap();
let object_2 = pool.try_get().unwrap();
let object_3 = pool.try_get().unwrap();
let object_4 = pool.try_get().unwrap();
let object_5 = pool.try_get().unwrap();
let object_6 = pool.try_get().unwrap();
let object_7 = pool.try_get().unwrap();
assert!(pool.try_get().is_err());
let get_thread_handle = thread::spawn({
let get_future = pool.get();
move || {
block_on(async {
let maybe_object = get_future.await;
assert!(maybe_object.is_none());
})
}
});
thread::spawn(move || {
thread::sleep(Duration::from_millis(100));
drop(object_0);
drop(object_1);
drop(object_2);
drop(object_3);
drop(object_4);
drop(object_5);
drop(object_6);
drop(object_7);
});
thread_wait();
let pool = block_on(pool.close());
#[cfg(not(loom))]
assert!(get_thread_handle.is_finished());
assert!(pool.is_closed());
})
}
#[test]
fn test_object_drop_releases_the_value() {
run(|| {
let pool = Pool::new(8, || T { x: 0 });
let object_0 = pool.try_get().unwrap();
let _object_1 = pool.try_get().unwrap();
let _object_2 = pool.try_get().unwrap();
let _object_3 = pool.try_get().unwrap();
let object_4 = pool.try_get().unwrap();
let _object_5 = pool.try_get().unwrap();
let _object_6 = pool.try_get().unwrap();
let _object_7 = pool.try_get().unwrap();
assert!(pool.try_get().is_err());
drop(object_0);
let _object_0 = pool.try_get().unwrap();
assert!(pool.try_get().is_err());
drop(object_4);
let _object_4 = pool.try_get().unwrap();
assert!(pool.try_get().is_err());
})
}
#[test]
fn test_object_derefs() {
run(|| {
let pool = Pool::new(8, || T { x: 0 });
{
let object = pool.try_get().unwrap();
assert_eq!(object.x, 0);
}
{
let mut object = pool.try_get().unwrap();
object.x = 42;
assert_eq!(object.x, 42);
}
{
let object0 = pool.try_get().unwrap();
assert_eq!(object0.x, 42);
let object1 = pool.try_get().unwrap();
assert_eq!(object1.x, 0);
}
})
}
#[test]
fn test_close_and_reopen() {
run(|| {
let pool = Pool::new(8, || T { x: 0 });
let mut object_0 = pool.try_get().unwrap();
let object_1 = pool.try_get().unwrap();
let thread_handle =
thread::spawn(move || block_on(async { pool.close().await }));
thread_wait();
object_0.x = 42;
drop(object_0);
thread_wait();
#[cfg(not(loom))]
assert!(thread_handle.is_finished().not());
drop(object_1);
thread_wait();
let pool = thread_handle.join().unwrap();
assert!(pool.is_closed());
assert!(matches!(pool.try_get(), Err(AcquireError::Closed)));
pool.reopen();
assert!(pool.is_closed().not());
assert_eq!(pool.try_get().unwrap().x, 42);
})
}
#[test]
fn test_close_when_all_objects_are_wandering() {
run(|| {
let pool = Pool::new(8, || T { x: 0 });
let object_0 = pool.try_get().unwrap();
let object_1 = pool.try_get().unwrap();
let object_2 = pool.try_get().unwrap();
let object_3 = pool.try_get().unwrap();
let object_4 = pool.try_get().unwrap();
let object_5 = pool.try_get().unwrap();
let object_6 = pool.try_get().unwrap();
let object_7 = pool.try_get().unwrap();
let thread_handle =
thread::spawn(|| block_on(async { pool.close().await }));
thread_wait();
drop(object_0);
drop(object_1);
drop(object_2);
drop(object_3);
drop(object_4);
drop(object_5);
drop(object_6);
drop(object_7);
let pool = thread_handle.join().unwrap();
assert!(pool.is_closed());
})
}
#[test]
fn test_reopen_on_non_closed_pool() {
run(|| {
let pool = Pool::new(8, || T { x: 0 });
pool.reopen();
})
}
}