use core::cell::Cell;
use core::sync::atomic::{AtomicPtr, AtomicU64, Ordering};
use std::sync::Arc;
use crate::sched::hazard::HazardDomain;
#[derive(Debug, Clone)]
pub struct Pass {
pub closure_id: u32,
pub args: Vec<u8>,
}
pub type PassResult = Result<Vec<u8>, PassError>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PassError {
UnknownClosureId(u32),
ExecutionError(String),
}
impl std::fmt::Display for PassError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PassError::UnknownClosureId(id) => write!(f, "no handler for closure id {id}"),
PassError::ExecutionError(msg) => write!(f, "handler failed: {msg}"),
}
}
}
impl std::error::Error for PassError {}
pub type PassHandler = Arc<dyn Fn(&[u8]) -> PassResult + Send + Sync + 'static>;
const SLOTS: usize = 256;
const MASK: usize = SLOTS - 1;
const READERS: usize = 256;
const RETIRED: usize = 256;
const LIVE: u64 = 1 << 40;
const RESERVED: u64 = 1 << 41;
const KEY_MASK: u64 = !(LIVE | RESERVED);
struct Slot {
state: AtomicU64,
handler: AtomicPtr<PassHandler>,
}
static TABLE: [Slot; SLOTS] = [const {
Slot {
state: AtomicU64::new(0),
handler: AtomicPtr::new(core::ptr::null_mut()),
}
}; SLOTS];
static DOMAIN: HazardDomain<PassHandler, READERS, RETIRED> = HazardDomain::new();
fn key_of(id: u32) -> u64 {
id as u64 + 1
}
fn reader_slot() -> usize {
thread_local! {
static READER: Cell<usize> = const { Cell::new(usize::MAX) };
}
READER.with(|cell| {
let held = cell.get();
if held != usize::MAX {
return held;
}
let fresh = DOMAIN.claim_reader();
cell.set(fresh);
fresh
})
}
fn find_live(id: u32) -> Option<&'static Slot> {
let key = key_of(id);
let mut idx = (id as usize) & MASK;
for _ in 0..SLOTS {
let state = TABLE[idx].state.load(Ordering::Acquire);
if state == 0 {
return None;
}
if state == key | LIVE {
return Some(&TABLE[idx]);
}
idx = (idx + 1) & MASK;
}
None
}
fn install(id: u32, fresh: *mut PassHandler) -> Option<PassHandler> {
let key = key_of(id);
let slot = claim(id, key);
let previous = slot.handler.swap(fresh, Ordering::SeqCst);
slot.state.store(key | LIVE, Ordering::Release);
displace(previous)
}
fn claim(id: u32, key: u64) -> &'static Slot {
let mut idx = (id as usize) & MASK;
for _ in 0..(SLOTS * 2) {
let slot = &TABLE[idx];
let state = slot.state.load(Ordering::Acquire);
if state & KEY_MASK == key && state & (LIVE | RESERVED) != 0 {
return slot;
}
if state == 0 || state & (LIVE | RESERVED) == 0 {
if slot
.state
.compare_exchange(state, key | RESERVED, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return slot;
}
continue;
}
idx = (idx + 1) & MASK;
}
panic!("pass registry full at {SLOTS} live handlers; id {id} could not be registered");
}
fn displace(previous: *mut PassHandler) -> Option<PassHandler> {
if previous.is_null() {
return None;
}
let handed_back = unsafe { &*previous }.clone();
unsafe { DOMAIN.retire(previous) };
Some(handed_back)
}
pub fn register<F>(id: u32, handler: F) -> Option<PassHandler>
where
F: Fn(&[u8]) -> PassResult + Send + Sync + 'static,
{
let shared: PassHandler = Arc::new(handler);
install(id, Box::into_raw(Box::new(shared)))
}
pub fn unregister(id: u32) -> Option<PassHandler> {
let key = key_of(id);
let mut idx = (id as usize) & MASK;
for _ in 0..SLOTS {
let slot = &TABLE[idx];
let state = slot.state.load(Ordering::Acquire);
if state == 0 {
return None;
}
if state == key | LIVE
&& slot
.state
.compare_exchange(key | LIVE, key, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
let removed = slot.handler.swap(core::ptr::null_mut(), Ordering::SeqCst);
return displace(removed);
}
idx = (idx + 1) & MASK;
}
None
}
pub fn is_registered(id: u32) -> bool {
find_live(id).is_some()
}
pub fn registered_count() -> usize {
TABLE
.iter()
.filter(|slot| slot.state.load(Ordering::Acquire) & LIVE != 0)
.count()
}
pub fn execute(pass: &Pass) -> PassResult {
let id = pass.closure_id;
let key = key_of(id);
let reader = reader_slot();
let mut idx = (id as usize) & MASK;
for _ in 0..SLOTS {
let slot = &TABLE[idx];
let state = slot.state.load(Ordering::Acquire);
if state == 0 {
break;
}
if state == key | LIVE {
let guard = DOMAIN.protect(reader, &slot.handler);
if slot.state.load(Ordering::Acquire) != key | LIVE {
break;
}
if let Some(handler) = guard.get() {
return (**handler)(&pass.args);
}
break;
}
idx = (idx + 1) & MASK;
}
Err(PassError::UnknownClosureId(id))
}
pub fn hash_name(name: &str) -> u32 {
let mut hash: u32 = 0x811C_9DC5;
for byte in name.as_bytes() {
hash ^= *byte as u32;
hash = hash.wrapping_mul(0x0100_0193);
}
hash
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn register_then_execute_round_trips() {
let id = hash_name("flynnel_test_doubler");
register(id, |args| {
assert_eq!(args.len(), 8);
let mut arr = [0u8; 8];
arr.copy_from_slice(args);
let n = u64::from_le_bytes(arr);
Ok((n * 2).to_le_bytes().to_vec())
});
let pass = Pass {
closure_id: id,
args: 21u64.to_le_bytes().to_vec(),
};
let bytes = execute(&pass).expect("execute");
let mut arr = [0u8; 8];
arr.copy_from_slice(&bytes);
assert_eq!(u64::from_le_bytes(arr), 42);
unregister(id);
}
#[test]
fn unknown_id_returns_specific_error() {
let pass = Pass {
closure_id: 0xDEAD_BEEF,
args: vec![],
};
match execute(&pass) {
Err(PassError::UnknownClosureId(id)) => assert_eq!(id, 0xDEAD_BEEF),
other => panic!("expected UnknownClosureId, got {other:?}"),
}
}
#[test]
fn hash_name_is_deterministic() {
let a = hash_name("flynnel.kernels.add_one");
let b = hash_name("flynnel.kernels.add_one");
assert_eq!(a, b);
let c = hash_name("flynnel.kernels.add_two");
assert_ne!(a, c, "different names must hash to different ids");
}
#[test]
fn re_register_returns_previous_handler() {
let id = hash_name("flynnel_test_rereg");
assert!(register(id, |_| Ok(vec![1])).is_none());
let prev = register(id, |_| Ok(vec![2]));
assert!(prev.is_some(), "second register must surface previous");
unregister(id);
}
}