#![allow(clippy::missing_errors_doc)]
use std::cell::RefCell;
use std::sync::atomic::{AtomicPtr, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use wasmtime::{Engine, Instance, Module, Store, ValRaw, ValType};
use crate::backend::{
Backend, BackendCapabilities, BackendError, DispatchBackend, KernelArg, KernelHandle,
};
use crate::sched::notify_ring::{NotifyHub, NotifySender};
type WorkItem = Box<dyn FnOnce() + Send + 'static>;
struct KernelEntry {
shared: Mutex<SharedInstance>,
module: Module,
export: String,
}
struct SharedInstance {
store: Store<()>,
func: wasmtime::Func,
params: Box<[Option<Scalar>]>,
results: usize,
}
impl SharedInstance {
fn new(store: Store<()>, func: wasmtime::Func) -> Self {
let ty = func.ty(&store);
let params = ty.params().map(|p| Scalar::of(&p)).collect();
let results = ty.results().len();
Self {
store,
func,
params,
results,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Scalar {
I32,
I64,
F32,
F64,
}
impl Scalar {
fn of(ty: &ValType) -> Option<Self> {
match ty {
ValType::I32 => Some(Self::I32),
ValType::I64 => Some(Self::I64),
ValType::F32 => Some(Self::F32),
ValType::F64 => Some(Self::F64),
_ => None,
}
}
fn raw(arg: &KernelArg<'_>) -> Option<(Self, ValRaw)> {
Some(match arg {
KernelArg::I32(v) => (Self::I32, ValRaw::i32(*v)),
KernelArg::I64(v) => (Self::I64, ValRaw::i64(*v)),
KernelArg::U32(v) => (Self::I32, ValRaw::i32(*v as i32)),
KernelArg::U64(v) => (Self::I64, ValRaw::i64(*v as i64)),
KernelArg::F32(v) => (Self::F32, ValRaw::f32(v.to_bits())),
KernelArg::F64(v) => (Self::F64, ValRaw::f64(v.to_bits())),
KernelArg::DevicePtr(p) => (Self::I32, ValRaw::i32(*p as i32)),
KernelArg::HostSlice(_) | KernelArg::Buffer(_) => return None,
})
}
}
const INLINE_SLOTS: usize = 16;
static NEXT_BACKEND_ID: AtomicU64 = AtomicU64::new(1);
thread_local! {
static LOCAL_INSTANCES: RefCell<Vec<Option<(u64, SharedInstance)>>> =
const { RefCell::new(Vec::new()) };
}
fn with_local_instance<R>(
owner: u64,
handle: u64,
build: impl FnOnce() -> Result<SharedInstance, BackendError>,
call: impl FnOnce(&mut SharedInstance) -> Result<R, BackendError>,
) -> Result<R, BackendError> {
let index = (handle - 1) as usize;
LOCAL_INSTANCES.with(|cell| {
let mut mine = cell.borrow_mut();
if mine.len() <= index {
mine.resize_with(index + 1, || None);
}
let mismatched = match &mine[index] {
Some((held, _)) => *held != owner,
None => true,
};
if mismatched {
mine[index] = Some((owner, build()?));
}
let (_, instance) = mine[index]
.as_mut()
.expect("the slot was filled above or the call returned");
call(instance)
})
}
#[cfg(test)]
fn local_slot_owner(handle: u64) -> Option<u64> {
let index = (handle - 1) as usize;
LOCAL_INSTANCES.with(|cell| {
let mine = cell.borrow();
mine.get(index)
.and_then(|slot| slot.as_ref())
.map(|(owner, _)| *owner)
})
}
const KERNEL_BLOCK: usize = 64;
struct KernelTable {
head: Block,
}
struct Block {
entries: [AtomicPtr<Arc<KernelEntry>>; KERNEL_BLOCK],
next: AtomicPtr<Block>,
}
impl Block {
fn new() -> Self {
Self {
entries: [const { AtomicPtr::new(core::ptr::null_mut()) }; KERNEL_BLOCK],
next: AtomicPtr::new(core::ptr::null_mut()),
}
}
}
impl KernelTable {
fn new() -> Self {
Self { head: Block::new() }
}
fn block_for(&self, index: usize) -> &Block {
let mut block = &self.head;
for _ in 0..(index / KERNEL_BLOCK) {
let mut next = block.next.load(Ordering::Acquire);
if next.is_null() {
let fresh = Box::into_raw(Box::new(Block::new()));
match block.next.compare_exchange(
core::ptr::null_mut(),
fresh,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => next = fresh,
Err(theirs) => {
drop(unsafe { Box::from_raw(fresh) });
next = theirs;
}
}
}
block = unsafe { &*next };
}
block
}
fn insert(&self, handle: u64, entry: Arc<KernelEntry>) {
let index = (handle - 1) as usize;
let block = self.block_for(index);
let slot = &block.entries[index % KERNEL_BLOCK];
slot.store(Box::into_raw(Box::new(entry)), Ordering::Release);
}
fn get(&self, handle: u64) -> Option<Arc<KernelEntry>> {
if handle == 0 {
return None;
}
let index = (handle - 1) as usize;
let mut block = &self.head;
for _ in 0..(index / KERNEL_BLOCK) {
let next = block.next.load(Ordering::Acquire);
if next.is_null() {
return None;
}
block = unsafe { &*next };
}
let held = block.entries[index % KERNEL_BLOCK].load(Ordering::Acquire);
if held.is_null() {
return None;
}
Some(unsafe { (*held).clone() })
}
}
pub struct WasmBackend {
device_id: u32,
engine: Engine,
caps: BackendCapabilities,
backend_id: u64,
next_handle: AtomicU64,
kernels: KernelTable,
worker_hub: NotifyHub<WorkItem>,
worker_tx: NotifySender<WorkItem>,
worker_handle: Option<JoinHandle<()>>,
}
impl std::fmt::Debug for WasmBackend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WasmBackend")
.field("device_id", &self.device_id)
.finish()
}
}
impl WasmBackend {
pub fn new() -> Result<Self, BackendError> {
Self::with_device(0)
}
pub fn with_device(device_id: u32) -> Result<Self, BackendError> {
let engine = Engine::default();
const WASM_WORKER_RING_CAPACITY: usize = 1024;
let worker_hub = NotifyHub::<WorkItem>::new(WASM_WORKER_RING_CAPACITY, 1);
let worker_tx = worker_hub.sender();
let hub_for_worker = worker_hub.clone();
let worker_handle = std::thread::Builder::new()
.name(format!("flynnel-wasm-{device_id}"))
.spawn(move || {
let rx = hub_for_worker.register_consumer();
while let Some(work) = rx.recv() {
work();
}
})
.map_err(|_| BackendError::DeviceUnavailable(Backend::Wasm { device_id }))?;
Ok(Self {
device_id,
engine,
caps: probe_capabilities(),
backend_id: NEXT_BACKEND_ID.fetch_add(1, Ordering::Relaxed),
next_handle: AtomicU64::new(1),
kernels: KernelTable::new(),
worker_hub,
worker_tx,
worker_handle: Some(worker_handle),
})
}
}
fn instantiate(entry: &KernelEntry) -> Result<SharedInstance, BackendError> {
let mut store: Store<()> = Store::new(entry.module.engine(), ());
let instance = Instance::new(&mut store, &entry.module, &[]).map_err(|e| {
BackendError::Launch(format!(
"wasm instance create failed for kernel `{}`: {e}",
entry.export
))
})?;
let func = instance
.get_func(&mut store, &entry.export)
.ok_or_else(|| {
BackendError::Launch(format!(
"wasm export `{}` not found in module",
entry.export
))
})?;
Ok(SharedInstance::new(store, func))
}
fn call_instance(
instance: &mut SharedInstance,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
let expected_params = instance.params.len();
if expected_params != args.len() {
return Err(BackendError::Launch(format!(
"wasm kernel arg count mismatch: function expects {} params, caller provided {}",
expected_params,
args.len(),
)));
}
let slots = args.len().max(instance.results);
let mut inline = [ValRaw::i32(0); INLINE_SLOTS];
let mut spilled: Vec<ValRaw> = Vec::new();
let buf: &mut [ValRaw] = if slots <= INLINE_SLOTS {
&mut inline[..slots]
} else {
spilled.resize(slots, ValRaw::i32(0));
&mut spilled[..]
};
for (i, (arg, want)) in args.iter().zip(instance.params.iter()).enumerate() {
let (got, raw) = Scalar::raw(arg).ok_or(BackendError::NotSupported)?;
if Some(got) != *want {
return Err(BackendError::Launch(match want {
Some(want) => format!(
"wasm kernel call failed: argument type mismatch: argument {i} is {got:?}, parameter {i} is {want:?}"
),
None => format!(
"wasm kernel call failed: argument type mismatch: parameter {i} is a type no argument can carry"
),
}));
}
buf[i] = raw;
}
let SharedInstance {
ref mut store,
ref func,
..
} = *instance;
unsafe { func.call_unchecked(store, buf as *mut [ValRaw]) }
.map_err(|e| BackendError::Launch(format!("wasm kernel call failed: {e}")))?;
Ok(())
}
fn probe_capabilities() -> BackendCapabilities {
let threads = std::thread::available_parallelism()
.map(std::num::NonZeroUsize::get)
.unwrap_or(1) as u32;
BackendCapabilities {
simt_width: 1,
max_threads_in_flight: threads,
launch_latency_ns: 5_000,
h2d_bw_bytes_per_sec: 0,
}
}
impl DispatchBackend for WasmBackend {
fn id(&self) -> Backend {
Backend::Wasm {
device_id: self.device_id,
}
}
fn capabilities(&self) -> BackendCapabilities {
self.caps
}
fn dispatch_parallel_for(&self, count: u32, work: &(dyn Fn(u32) + Send + Sync)) {
let plan = crate::sched::JobPlan::new(0, count);
let mut indices: Vec<u32> = (0..count).collect();
crate::sched::par_iter::for_each_chunk(
&plan,
indices.as_mut_slice(),
move |slice: &mut [u32]| {
for i in slice.iter() {
work(*i);
}
},
);
}
fn dispatch_one(&self, work: Box<dyn FnOnce() + Send>) {
drop(self.worker_tx.send(work));
}
fn register_kernel(&self, name: &str, source: &[u8]) -> Result<KernelHandle, BackendError> {
let module = Module::new(&self.engine, source)
.map_err(|e| BackendError::KernelCompile(format!("wasm module compile failed: {e}")))?;
let mut store: Store<()> = Store::new(&self.engine, ());
let instance = Instance::new(&mut store, &module, &[]).map_err(|e| {
BackendError::KernelCompile(format!(
"wasm instance create failed for kernel `{name}`: {e}"
))
})?;
let func = instance.get_func(&mut store, name).ok_or_else(|| {
BackendError::KernelCompile(format!("wasm export `{name}` not found in module"))
})?;
let handle_id = self.next_handle.fetch_add(1, Ordering::SeqCst);
let entry = KernelEntry {
shared: Mutex::new(SharedInstance::new(store, func)),
module,
export: name.to_string(),
};
self.kernels.insert(handle_id, Arc::new(entry));
Ok(KernelHandle(handle_id))
}
fn dispatch_kernel(
&self,
handle: KernelHandle,
_count: u32,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
if args.iter().any(|a| Scalar::raw(a).is_none()) {
return Err(BackendError::NotSupported);
}
let entry_arc = self.kernels.get(handle.0).ok_or_else(|| {
BackendError::Launch(format!("wasm kernel handle {} not registered", handle.0))
})?;
if crate::sched::levers::wasm_local_store() {
return with_local_instance(
self.backend_id,
handle.0,
|| instantiate(&entry_arc),
|instance| call_instance(instance, args),
);
}
let mut shared = entry_arc
.shared
.lock()
.map_err(|_| BackendError::Launch("wasm kernel store mutex poisoned".into()))?;
call_instance(&mut shared, args)
}
}
impl Drop for WasmBackend {
fn drop(&mut self) {
self.worker_hub.shutdown();
if let Some(handle) = self.worker_handle.take() {
match handle.join() {
Ok(()) => {}
Err(panicked) => drop(panicked),
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn table_entry(engine: &Engine) -> Arc<KernelEntry> {
let module = Module::new(engine, ADD_WASM).expect("the add module builds");
let mut store: Store<()> = Store::new(engine, ());
let instance = Instance::new(&mut store, &module, &[]).expect("it instantiates");
let func = instance
.get_func(&mut store, "add")
.expect("add is exported");
Arc::new(KernelEntry {
shared: Mutex::new(SharedInstance::new(store, func)),
module,
export: "add".to_string(),
})
}
#[test]
fn a_second_backend_does_not_inherit_the_first_ones_slot() {
let engine = Engine::default();
let entry = table_entry(&engine);
let build = || instantiate(&entry);
with_local_instance(101, 1, build, |_| Ok(())).expect("the first backend builds its slot");
assert_eq!(
local_slot_owner(1),
Some(101),
"the slot belongs to whichever backend built it"
);
with_local_instance(202, 1, build, |_| Ok(())).expect("the second backend builds its own");
assert_eq!(
local_slot_owner(1),
Some(202),
"a slot another backend holds is rebuilt rather than reused"
);
with_local_instance(101, 1, build, |_| Ok(())).expect("the first backend builds again");
assert_eq!(
local_slot_owner(1),
Some(101),
"and back again, because the slot follows whoever asked last"
);
}
#[test]
fn the_kernel_table_holds_every_handle_across_blocks() {
let engine = Engine::default();
let table = KernelTable::new();
let count = (KERNEL_BLOCK * 2 + 3) as u64;
for handle in 1..=count {
table.insert(handle, table_entry(&engine));
}
for handle in 1..=count {
assert!(
table.get(handle).is_some(),
"handle {handle} was registered and must be found"
);
}
assert!(table.get(0).is_none(), "zero is never a handle");
assert!(
table.get(count + 1).is_none(),
"a handle nothing registered under reads as absent"
);
assert!(
table.get(count + KERNEL_BLOCK as u64 * 4).is_none(),
"so does one whose block was never appended"
);
}
#[test]
fn two_registrations_racing_on_one_missing_block_both_land() {
let engine = Engine::default();
let table = Arc::new(KernelTable::new());
let first = KERNEL_BLOCK as u64 * 3 + 1;
let second = first + 1;
let barrier = Arc::new(std::sync::Barrier::new(2));
let mut threads = Vec::new();
for handle in [first, second] {
let table = Arc::clone(&table);
let barrier = Arc::clone(&barrier);
let engine = engine.clone();
threads.push(std::thread::spawn(move || {
let entry = table_entry(&engine);
barrier.wait();
table.insert(handle, entry);
}));
}
for t in threads {
t.join().expect("a registering thread must not panic");
}
assert!(table.get(first).is_some(), "the first handle is present");
assert!(table.get(second).is_some(), "and so is the second");
}
const ADD_WASM: &[u8] = &[
0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x07, 0x01, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f,
0x03, 0x02, 0x01, 0x00, 0x07, 0x07, 0x01, 0x03, 0x61, 0x64, 0x64, 0x00, 0x00, 0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x20, 0x01, 0x6a, 0x0b,
];
#[test]
fn wasm_backend_constructs() {
WasmBackend::new().expect("wasm backend should init");
}
#[test]
fn wasm_backend_reports_correct_id_and_caps() {
let b = WasmBackend::with_device(7).expect("init");
assert_eq!(b.id(), Backend::Wasm { device_id: 7 });
let caps = b.capabilities();
assert_eq!(caps.simt_width, 1);
assert!(caps.max_threads_in_flight >= 1);
}
#[test]
fn wasm_backend_dispatches_kernel_with_add_module() {
let b = WasmBackend::new().expect("init");
let handle = b
.register_kernel("add", ADD_WASM)
.expect("register `add` export");
b.dispatch_kernel(handle, 1, &[KernelArg::I32(3), KernelArg::I32(4)])
.expect("dispatch_kernel should succeed on valid args");
}
#[test]
fn wasm_backend_rejects_arity_mismatch() {
let b = WasmBackend::new().expect("init");
let handle = b.register_kernel("add", ADD_WASM).expect("register");
let err = b
.dispatch_kernel(handle, 1, &[KernelArg::I32(3)])
.expect_err("arity mismatch must error");
assert!(matches!(err, BackendError::Launch(_)));
}
const F64_ADD_WASM: &[u8] = &[
0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x07, 0x01, 0x60, 0x02, 0x7c, 0x7c, 0x01, 0x7c, 0x03, 0x02, 0x01, 0x00, 0x07, 0x07, 0x01, 0x03, 0x61, 0x64, 0x64, 0x00, 0x00, 0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x20, 0x01, 0xa0, 0x0b, ];
const SEVEN_WASM: &[u8] = &[
0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x05, 0x01, 0x60, 0x00, 0x01, 0x7f, 0x03, 0x02, 0x01, 0x00, 0x07, 0x09, 0x01, 0x05, 0x73, 0x65, 0x76, 0x65, 0x6e, 0x00, 0x00, 0x0a, 0x06, 0x01, 0x04, 0x00, 0x41, 0x07, 0x0b, ];
#[test]
fn wasm_backend_rejects_an_argument_of_the_wrong_type() {
let b = WasmBackend::new().expect("init");
let handle = b.register_kernel("add", ADD_WASM).expect("register");
let err = b
.dispatch_kernel(handle, 1, &[KernelArg::I64(3), KernelArg::I32(4)])
.expect_err("a mistyped argument must error");
assert!(matches!(err, BackendError::Launch(_)), "{err:?}");
}
#[test]
fn wasm_backend_runs_a_float_kernel_and_refuses_the_other_float() {
let b = WasmBackend::new().expect("init");
let handle = b.register_kernel("add", F64_ADD_WASM).expect("register");
b.dispatch_kernel(handle, 1, &[KernelArg::F64(1.5), KernelArg::F64(2.25)])
.expect("two f64 arguments reach an (f64, f64) kernel");
let err = b
.dispatch_kernel(handle, 1, &[KernelArg::F32(1.5), KernelArg::F64(2.25)])
.expect_err("an f32 where the kernel takes an f64 must error");
assert!(matches!(err, BackendError::Launch(_)), "{err:?}");
}
#[test]
fn wasm_backend_runs_a_kernel_returning_more_than_it_takes() {
let b = WasmBackend::new().expect("init");
let handle = b.register_kernel("seven", SEVEN_WASM).expect("register");
b.dispatch_kernel(handle, 1, &[])
.expect("a kernel taking nothing runs");
}
#[test]
fn wasm_backend_register_kernel_with_missing_export_errors() {
let b = WasmBackend::new().expect("init");
let err = b
.register_kernel("does_not_exist", ADD_WASM)
.expect_err("missing export must error");
assert!(matches!(err, BackendError::KernelCompile(_)));
}
#[test]
fn wasm_backend_register_invalid_module_errors() {
let b = WasmBackend::new().expect("init");
let err = b
.register_kernel("x", b"not a wasm module")
.expect_err("invalid bytes must error");
assert!(matches!(err, BackendError::KernelCompile(_)));
}
#[test]
fn wasm_backend_host_slice_arg_returns_not_supported() {
let b = WasmBackend::new().expect("init");
let handle = b.register_kernel("add", ADD_WASM).expect("register");
let buf: [u8; 4] = [1, 2, 3, 4];
let err = b
.dispatch_kernel(handle, 1, &[KernelArg::HostSlice(&buf)])
.expect_err("HostSlice must be unsupported in reference impl");
assert!(matches!(err, BackendError::NotSupported));
}
}