use std::collections::BTreeMap;
use std::future::Future;
use std::pin::Pin;
use std::sync::{Arc, Mutex, MutexGuard, Weak};
use std::task::{Context, Poll, Wake, Waker};
use rustev_core::seams::{BoxFuture, CancelSignal};
use crate::wait::CatchUnwind;
pub(crate) type Shared<T> = Result<T, String>;
fn lock<S>(m: &Mutex<S>) -> MutexGuard<'_, S> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
pub(crate) struct Registry<K: Ord + Clone + Send + Sync + 'static, T: Clone + Send + 'static> {
inner: Arc<RegistryInner<K, T>>,
}
struct RegistryInner<K, T> {
max_slots: usize,
slots: Mutex<BTreeMap<K, Arc<Slot<K, T>>>>,
}
pub(crate) enum Entered<K: Ord + Clone + Send + Sync + 'static, T: Clone + Send + 'static> {
Led(Waiter<K, T>),
Joined(Waiter<K, T>),
Ineligible(Ineligible),
Full,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Ineligible {
Deadline,
Waiters,
}
impl<K: Ord + Clone + Send + Sync + 'static, T: Clone + Send + 'static> Registry<K, T> {
pub(crate) fn new(max_slots: usize) -> Self {
Registry {
inner: Arc::new(RegistryInner {
max_slots,
slots: Mutex::new(BTreeMap::new()),
}),
}
}
pub(crate) fn len(&self) -> usize {
lock(&self.inner.slots).len()
}
pub(crate) fn enter(
&self,
key: &K,
deadline: u64,
max_waiters: usize,
label: &str,
lead: impl FnOnce(CancelSignal) -> BoxFuture<'static, T>,
) -> Entered<K, T> {
let mut slots = lock(&self.inner.slots);
if let Some(slot) = slots.get(key) {
let mut st = lock(&slot.state);
if st.result.is_none() && st.attempt.is_some() {
if slot.deadline < deadline {
return Entered::Ineligible(Ineligible::Deadline);
}
if st.waiters.len() >= max_waiters {
return Entered::Ineligible(Ineligible::Waiters);
}
let id = st.join();
drop(st);
return Entered::Joined(Waiter::new(slot.clone(), id));
}
}
if max_waiters == 0 || (slots.len() >= self.inner.max_slots && !slots.contains_key(key)) {
return Entered::Full;
}
let signal = CancelSignal::new();
let attempt = lead(signal.clone());
let slot = Arc::new(Slot {
key: key.clone(),
label: label.to_string(),
deadline,
signal,
registry: Arc::downgrade(&self.inner),
state: Mutex::new(State {
attempt: Some(Box::pin(CatchUnwind::new(attempt))),
polling: false,
result: None,
waiters: BTreeMap::new(),
next: 0,
owner: 0,
handoffs: vec![],
}),
});
let id = lock(&slot.state).join();
slots.insert(key.clone(), slot.clone());
Entered::Led(Waiter::new(slot, id))
}
}
type Attempt<T> = Pin<Box<CatchUnwind<BoxFuture<'static, T>>>>;
struct Slot<K, T> {
key: K,
label: String,
deadline: u64,
signal: CancelSignal,
registry: Weak<RegistryInner<K, T>>,
state: Mutex<State<T>>,
}
struct State<T> {
attempt: Option<Attempt<T>>,
polling: bool,
result: Option<Shared<T>>,
waiters: BTreeMap<u64, Option<Waker>>,
next: u64,
owner: u64,
handoffs: Vec<(u64, u64)>,
}
impl<T> State<T> {
fn join(&mut self) -> u64 {
let id = self.next;
self.next += 1;
if self.waiters.is_empty() {
self.owner = id;
}
self.waiters.insert(id, None);
id
}
fn remove(&mut self, id: u64) -> Option<u64> {
self.waiters.remove(&id);
if self.owner != id {
return None;
}
let next = *self.waiters.keys().next()?;
self.owner = next;
self.handoffs.push((id, next));
Some(next)
}
}
impl<K: Ord + Clone, T> Slot<K, T> {
fn retire(self: &Arc<Self>) {
if let Some(r) = self.registry.upgrade() {
let mut slots = lock(&r.slots);
if slots.get(&self.key).is_some_and(|s| Arc::ptr_eq(s, self)) {
slots.remove(&self.key);
}
}
}
}
struct Broadcast<K, T> {
slot: Weak<Slot<K, T>>,
}
impl<K: Send + Sync + 'static, T: Send + 'static> Wake for Broadcast<K, T> {
fn wake(self: Arc<Self>) {
self.wake_by_ref();
}
fn wake_by_ref(self: &Arc<Self>) {
if let Some(slot) = self.slot.upgrade() {
let wakers: Vec<Waker> = lock(&slot.state)
.waiters
.values()
.flatten()
.cloned()
.collect();
for w in wakers {
w.wake();
}
}
}
}
#[derive(Debug)]
pub(crate) enum Left<T> {
Remained { handed_off_to: Option<u64> },
Completed(Shared<T>),
Answered(Shared<T>),
Abandoned,
}
#[derive(Debug)]
pub(crate) enum Dropped<T> {
Remained,
Completed { result: Shared<T>, owner: bool },
Last,
}
pub(crate) struct Waiter<K: Ord + Clone + Send + Sync + 'static, T: Clone + Send + 'static> {
slot: Arc<Slot<K, T>>,
id: u64,
done: bool,
}
impl<K: Ord + Clone + Send + Sync + 'static, T: Clone + Send + 'static> Waiter<K, T> {
fn new(slot: Arc<Slot<K, T>>, id: u64) -> Self {
Waiter {
slot,
id,
done: false,
}
}
pub(crate) fn id(&self) -> u64 {
self.id
}
pub(crate) fn label(&self) -> &str {
&self.slot.label
}
pub(crate) fn is_owner(&self) -> bool {
lock(&self.slot.state).owner == self.id
}
pub(crate) fn received_from(&self) -> Option<u64> {
lock(&self.slot.state)
.handoffs
.iter()
.find(|(_, to)| *to == self.id)
.map(|(from, _)| *from)
}
pub(crate) fn poll_attempt(&mut self, cx: &mut Context<'_>) -> Poll<Shared<T>> {
assert!(!self.done, "a waiter that finished is not polled again");
let slot = self.slot.clone();
let mut st = lock(&slot.state);
if let Some(r) = &st.result {
self.done = true;
return Poll::Ready(r.clone());
}
st.waiters.insert(self.id, Some(cx.waker().clone()));
if st.polling {
return Poll::Pending;
}
let Some(mut attempt) = st.attempt.take() else {
unreachable!("a registered waiter's slot keeps its attempt until it completes")
};
st.polling = true;
drop(st);
let waker = Waker::from(Arc::new(Broadcast {
slot: Arc::downgrade(&slot),
}));
let polled = attempt.as_mut().poll(&mut Context::from_waker(&waker));
let mut st = lock(&slot.state);
st.polling = false;
match polled {
Poll::Pending => {
st.attempt = Some(attempt);
Poll::Pending
}
Poll::Ready(r) => {
st.result = Some(r.clone());
let wakers: Vec<Waker> = st.waiters.values().flatten().cloned().collect();
drop(st);
drop(attempt);
slot.retire();
for w in wakers {
w.wake();
}
self.done = true;
Poll::Ready(r)
}
}
}
pub(crate) fn leave(mut self, cx: &mut Context<'_>) -> Left<T> {
self.done = true;
let slot = self.slot.clone();
let mut st = lock(&slot.state);
if let Some(r) = st.result.clone() {
st.waiters.remove(&self.id);
return Left::Completed(r);
}
let handed_off_to = st.remove(self.id);
if !st.waiters.is_empty() {
return Left::Remained { handed_off_to };
}
let mut attempt = st.attempt.take();
drop(st);
slot.retire();
slot.signal.raise();
let answered = attempt.as_mut().and_then(|a| match a.as_mut().poll(cx) {
Poll::Ready(r) => Some(r),
Poll::Pending => None,
});
drop(attempt);
match answered {
Some(r) => Left::Answered(r),
None => Left::Abandoned,
}
}
}
impl<K: Ord + Clone + Send + Sync + 'static, T: Clone + Send + 'static> Waiter<K, T> {
pub(crate) fn abandon(mut self) -> Dropped<T> {
self.release().unwrap_or(Dropped::Remained)
}
fn release(&mut self) -> Option<Dropped<T>> {
if self.done {
return None;
}
self.done = true;
let mut st = lock(&self.slot.state);
if let Some(result) = st.result.clone() {
let owner = st.owner == self.id;
st.waiters.remove(&self.id);
return Some(Dropped::Completed { result, owner });
}
st.remove(self.id);
if !st.waiters.is_empty() {
return Some(Dropped::Remained);
}
let attempt = st.attempt.take();
drop(st);
drop(attempt);
self.slot.retire();
Some(Dropped::Last)
}
}
impl<K: Ord + Clone + Send + Sync + 'static, T: Clone + Send + 'static> std::fmt::Debug
for Registry<K, T>
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Registry")
.field("max_slots", &self.inner.max_slots)
.field("slots", &self.len())
.finish()
}
}
impl<K: Ord + Clone + Send + Sync + 'static, T: Clone + Send + 'static> Drop for Waiter<K, T> {
fn drop(&mut self) {
let _ = self.release();
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Default)]
struct Remote {
answer: Mutex<Option<u32>>,
waker: Mutex<Option<Waker>>,
polls: AtomicUsize,
dropped: AtomicUsize,
signal: Mutex<Option<CancelSignal>>,
panic: Mutex<bool>,
}
impl Remote {
fn answer(&self, v: u32) {
*lock(&self.answer) = Some(v);
if let Some(w) = lock(&self.waker).take() {
w.wake();
}
}
fn polls(&self) -> usize {
self.polls.load(Ordering::SeqCst)
}
fn raised(&self) -> bool {
lock(&self.signal).as_ref().is_some_and(|s| s.is_raised())
}
}
struct Work(Arc<Remote>);
impl Drop for Work {
fn drop(&mut self) {
self.0.dropped.fetch_add(1, Ordering::SeqCst);
}
}
fn lead(remote: &Arc<Remote>) -> impl FnOnce(CancelSignal) -> BoxFuture<'static, u32> {
let remote = remote.clone();
move |signal| {
*lock(&remote.signal) = Some(signal);
let work = Work(remote);
Box::pin(std::future::poll_fn(move |cx| {
let r = &work.0;
r.polls.fetch_add(1, Ordering::SeqCst);
if *lock(&r.panic) {
panic!("SYNTHETIC attempt panic");
}
match *lock(&r.answer) {
Some(v) => Poll::Ready(v),
None => {
*lock(&r.waker) = Some(cx.waker().clone());
Poll::Pending
}
}
}))
}
}
#[derive(Default)]
struct Count(AtomicUsize);
impl Wake for Count {
fn wake(self: Arc<Self>) {
self.0.fetch_add(1, Ordering::SeqCst);
}
}
fn counter() -> (Arc<Count>, Waker) {
let c = Arc::new(Count::default());
(c.clone(), Waker::from(c))
}
fn wakes(c: &Count) -> usize {
c.0.load(Ordering::SeqCst)
}
type W = Waiter<&'static str, u32>;
fn led(e: Entered<&'static str, u32>) -> W {
match e {
Entered::Led(w) => w,
_ => panic!("expected to lead"),
}
}
fn joined(e: Entered<&'static str, u32>) -> W {
match e {
Entered::Joined(w) => w,
_ => panic!("expected to join"),
}
}
#[test]
fn every_waiter_receives_one_attempts_result_and_the_slot_is_removed() {
let reg = Registry::new(4);
let remote = Arc::new(Remote::default());
let mut a = led(reg.enter(&"k", 100, 3, "a", lead(&remote)));
let mut b = joined(reg.enter(&"k", 100, 3, "a", lead(&Arc::default())));
let (ca, wa) = counter();
let (cb, wb) = counter();
assert!(a.poll_attempt(&mut Context::from_waker(&wa)).is_pending());
assert!(b.poll_attempt(&mut Context::from_waker(&wb)).is_pending());
assert_eq!(remote.polls(), 2);
remote.answer(7);
assert_eq!((wakes(&ca), wakes(&cb)), (1, 1));
assert_eq!(
b.poll_attempt(&mut Context::from_waker(&wb)),
Poll::Ready(Ok(7))
);
assert_eq!(reg.len(), 0);
assert_eq!(
a.poll_attempt(&mut Context::from_waker(&wa)),
Poll::Ready(Ok(7))
);
assert!(a.is_owner() && !b.is_owner());
assert_eq!(remote.polls(), 3);
drop((a, b));
assert_eq!(remote.dropped.load(Ordering::SeqCst), 1);
assert!(!remote.raised());
assert!(matches!(
reg.enter(&"k", 100, 3, "a", lead(&Arc::default())),
Entered::Led(_)
));
}
#[test]
fn a_waiter_finding_the_attempt_being_polled_registers_and_waits() {
let reg = Registry::<&str, u32>::new(1);
let remote = Arc::new(Remote::default());
let a = led(reg.enter(&"k", 100, 2, "a", lead(&remote)));
let mut b = joined(reg.enter(&"k", 100, 2, "a", lead(&Arc::default())));
lock(&a.slot.state).polling = true;
let (_, wb) = counter();
assert!(b.poll_attempt(&mut Context::from_waker(&wb)).is_pending());
assert_eq!(remote.polls(), 0);
assert!(lock(&a.slot.state).waiters[&b.id()].is_some());
lock(&a.slot.state).polling = false;
}
#[test]
fn joining_needs_time_room_and_an_attempt_in_flight() {
let reg = Registry::new(1);
let remote = Arc::new(Remote::default());
let _a = led(reg.enter(&"k", 100, 2, "a", lead(&remote)));
assert!(matches!(
reg.enter(&"k", 101, 2, "a", lead(&Arc::default())),
Entered::Ineligible(Ineligible::Deadline)
));
let _b = joined(reg.enter(&"k", 99, 2, "a", lead(&Arc::default())));
assert!(matches!(
reg.enter(&"k", 50, 2, "a", lead(&Arc::default())),
Entered::Ineligible(Ineligible::Waiters)
));
assert!(matches!(
reg.enter(&"other", 100, 2, "a", lead(&Arc::default())),
Entered::Full
));
assert!(matches!(
Registry::<&str, u32>::new(1).enter(&"k", 100, 0, "a", lead(&Arc::default())),
Entered::Full
));
}
#[test]
fn a_leaving_owner_returns_at_once_and_hands_off_to_the_earliest_waiter() {
let reg = Registry::new(1);
let remote = Arc::new(Remote::default());
let mut a = led(reg.enter(&"k", 100, 4, "a", lead(&remote)));
let mut b = joined(reg.enter(&"k", 100, 4, "a", lead(&Arc::default())));
let c = joined(reg.enter(&"k", 100, 4, "a", lead(&Arc::default())));
let (ca, wa) = counter();
let (_, wc) = counter();
assert!(a.poll_attempt(&mut Context::from_waker(&wa)).is_pending());
let (b_id, a_id) = (b.id(), a.id());
match a.leave(&mut Context::from_waker(&wa)) {
Left::Remained { handed_off_to } => assert_eq!(handed_off_to, Some(b_id)),
other => panic!("{other:?}"),
}
assert!(!remote.raised());
assert!(b.is_owner());
assert_eq!(b.received_from(), Some(a_id));
match c.leave(&mut Context::from_waker(&wc)) {
Left::Remained { handed_off_to } => assert_eq!(handed_off_to, None),
other => panic!("{other:?}"),
}
remote.answer(3);
assert_eq!(wakes(&ca), 0);
let (_, wb) = counter();
assert_eq!(
b.poll_attempt(&mut Context::from_waker(&wb)),
Poll::Ready(Ok(3))
);
assert!(b.is_owner());
assert_eq!(reg.len(), 0);
}
#[test]
fn the_last_waiter_raises_the_signal_and_polls_once() {
let reg = Registry::new(1);
let remote = Arc::new(Remote::default());
let mut a = led(reg.enter(&"k", 100, 2, "a", lead(&remote)));
let (_, w) = counter();
assert!(a.poll_attempt(&mut Context::from_waker(&w)).is_pending());
assert!(matches!(
a.leave(&mut Context::from_waker(&w)),
Left::Abandoned
));
assert!(remote.raised());
assert_eq!(remote.polls(), 2);
assert_eq!(remote.dropped.load(Ordering::SeqCst), 1);
assert_eq!(reg.len(), 0);
let remote = Arc::new(Remote::default());
let mut a = led(reg.enter(&"k", 100, 2, "a", lead(&remote)));
let b = joined(reg.enter(&"k", 100, 2, "a", lead(&Arc::default())));
assert!(a.poll_attempt(&mut Context::from_waker(&w)).is_pending());
drop(b);
*lock(&remote.answer) = Some(9);
assert!(matches!(
a.leave(&mut Context::from_waker(&w)),
Left::Answered(Ok(9))
));
assert!(remote.raised());
}
#[test]
fn a_dropped_waiter_hands_off_and_the_last_drops_the_attempt_unsignalled() {
let reg = Registry::new(1);
let remote = Arc::new(Remote::default());
let mut a = led(reg.enter(&"k", 100, 2, "a", lead(&remote)));
let b = joined(reg.enter(&"k", 100, 2, "a", lead(&Arc::default())));
let (_, w) = counter();
assert!(a.poll_attempt(&mut Context::from_waker(&w)).is_pending());
let a_id = a.id();
drop(a);
assert!(b.is_owner());
assert_eq!(b.received_from(), Some(a_id));
assert_eq!(remote.dropped.load(Ordering::SeqCst), 0);
drop(b);
assert_eq!(remote.dropped.load(Ordering::SeqCst), 1);
assert!(!remote.raised());
assert_eq!(reg.len(), 0);
}
#[test]
fn a_panic_in_the_attempt_is_every_waiters_outcome() {
let reg = Registry::new(1);
let remote = Arc::new(Remote::default());
*lock(&remote.panic) = true;
let mut a = led(reg.enter(&"k", 100, 2, "a", lead(&remote)));
let mut b = joined(reg.enter(&"k", 100, 2, "a", lead(&Arc::default())));
let (_, w) = counter();
let detail = Poll::Ready(Err("SYNTHETIC attempt panic".to_string()));
assert_eq!(b.poll_attempt(&mut Context::from_waker(&w)), detail);
assert_eq!(a.poll_attempt(&mut Context::from_waker(&w)), detail);
assert_eq!(reg.len(), 0);
}
#[test]
fn abandoning_reports_what_the_waiter_left_behind() {
let reg = Registry::new(1);
let remote = Arc::new(Remote::default());
let a = led(reg.enter(&"k", 100, 3, "a", lead(&remote)));
let b = joined(reg.enter(&"k", 100, 3, "a", lead(&Arc::default())));
let mut c = joined(reg.enter(&"k", 100, 3, "a", lead(&Arc::default())));
assert!(matches!(a.abandon(), Dropped::Remained));
remote.answer(4);
let (_, w) = counter();
assert_eq!(
c.poll_attempt(&mut Context::from_waker(&w)),
Poll::Ready(Ok(4))
);
assert!(matches!(
b.abandon(),
Dropped::Completed {
result: Ok(4),
owner: true
}
));
let remote = Arc::new(Remote::default());
let d = led(reg.enter(&"k", 100, 3, "a", lead(&remote)));
assert!(matches!(d.abandon(), Dropped::Last));
assert!(!remote.raised());
assert_eq!(remote.dropped.load(Ordering::SeqCst), 1);
}
#[test]
fn concurrent_abandons_leave_exactly_one_last_waiter() {
for _ in 0..2_000 {
let reg = Registry::new(1);
let remote = Arc::new(Remote::default());
let a = led(reg.enter(&"k", 100, 2, "a", lead(&remote)));
let b = joined(reg.enter(&"k", 100, 2, "a", lead(&Arc::default())));
let barrier = Arc::new(std::sync::Barrier::new(2));
let run = |w: W, barrier: Arc<std::sync::Barrier>| {
std::thread::spawn(move || {
barrier.wait();
w.abandon()
})
};
let (x, y) = (run(a, barrier.clone()), run(b, barrier));
let outcomes = [x.join().unwrap(), y.join().unwrap()];
let last = outcomes
.iter()
.filter(|d| matches!(d, Dropped::Last))
.count();
assert_eq!(last, 1, "{outcomes:?}");
assert_eq!(remote.dropped.load(Ordering::SeqCst), 1);
assert_eq!(reg.len(), 0);
}
}
#[test]
fn leaving_after_completion_takes_the_result() {
let reg = Registry::new(1);
let remote = Arc::new(Remote::default());
let a = led(reg.enter(&"k", 100, 2, "a", lead(&remote)));
let mut b = joined(reg.enter(&"k", 100, 2, "a", lead(&Arc::default())));
remote.answer(5);
let (_, w) = counter();
assert_eq!(
b.poll_attempt(&mut Context::from_waker(&w)),
Poll::Ready(Ok(5))
);
assert!(matches!(
a.leave(&mut Context::from_waker(&w)),
Left::Completed(Ok(5))
));
assert!(!remote.raised());
}
}