#![allow(clippy::missing_errors_doc)]
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use super::chase_lev_backend::{DispatchHandle, SharedMemoryChaseLevBackend};
use super::chase_lev_mmf::ARGS_INLINE_BYTES as CHASE_LEV_INLINE_BYTES;
use super::khpd::KHPD_ARGS_INLINE_BYTES;
use super::khpd_backend::SharedMemoryKhpdBackend;
use super::lcrq_lifo::LOH_ARGS_INLINE_BYTES;
use super::loh_backend::SharedMemoryLohBackend;
use super::urd_backend::SharedMemoryUrdBackend;
use crate::backend::BackendError;
use crate::cpu_info::has_waitpkg;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum DequeVariant {
ChaseLev,
Loh,
Khpd,
Urd,
}
impl DequeVariant {
pub const fn inline_args_bytes(self) -> usize {
match self {
Self::ChaseLev => CHASE_LEV_INLINE_BYTES,
Self::Loh => LOH_ARGS_INLINE_BYTES,
Self::Khpd => KHPD_ARGS_INLINE_BYTES,
Self::Urd => KHPD_ARGS_INLINE_BYTES,
}
}
pub const fn label(self) -> &'static str {
match self {
Self::ChaseLev => "chase_lev",
Self::Loh => "loh",
Self::Khpd => "khpd",
Self::Urd => "urd",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct WorkloadShape {
pub n_drain_threads: u32,
pub args_inline_bytes: u8,
pub expected_burst_size: u32,
pub k_unified: u8,
pub k_hardware_class: u8,
}
impl WorkloadShape {
pub const fn request_reply(args_inline_bytes: u8) -> Self {
Self {
n_drain_threads: 1,
args_inline_bytes,
expected_burst_size: 1,
k_unified: 0,
k_hardware_class: 0,
}
}
pub const fn producer_fast(args_inline_bytes: u8, burst: u32) -> Self {
Self {
n_drain_threads: 1,
args_inline_bytes,
expected_burst_size: burst,
k_unified: 0,
k_hardware_class: 0,
}
}
pub const fn multi_thief(args_inline_bytes: u8, n_drain_threads: u32, burst: u32) -> Self {
Self {
n_drain_threads,
args_inline_bytes,
expected_burst_size: burst,
k_unified: n_drain_threads.ilog2() as u8,
k_hardware_class: 0,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct DispatcherRoutingTable {
cells: HashMap<WorkloadShape, DequeVariant>,
}
impl DispatcherRoutingTable {
pub fn empty() -> Self {
Self::default()
}
pub fn default_heuristic() -> Self {
let mut cells: HashMap<WorkloadShape, DequeVariant> = HashMap::new();
cells.insert(WorkloadShape::request_reply(8), DequeVariant::ChaseLev);
cells.insert(
WorkloadShape::producer_fast(8, 64),
DequeVariant::Khpd,
);
cells.insert(
WorkloadShape::multi_thief(8, 4, 64),
DequeVariant::Urd,
);
Self { cells }
}
pub fn pick(&self, shape: &WorkloadShape) -> DequeVariant {
if let Some(&v) = self.cells.get(shape) {
return v;
}
Self::pick_heuristic(shape)
}
pub fn pick_heuristic(shape: &WorkloadShape) -> DequeVariant {
if shape.n_drain_threads >= 2 && has_waitpkg() {
if (shape.args_inline_bytes as usize) <= KHPD_ARGS_INLINE_BYTES {
return DequeVariant::Urd;
}
}
if shape.n_drain_threads >= 2
&& (shape.args_inline_bytes as usize) <= KHPD_ARGS_INLINE_BYTES
{
return DequeVariant::Urd;
}
if shape.expected_burst_size >= 8 {
if (shape.args_inline_bytes as usize) <= KHPD_ARGS_INLINE_BYTES {
return DequeVariant::Khpd;
}
if (shape.args_inline_bytes as usize) <= LOH_ARGS_INLINE_BYTES {
return DequeVariant::Loh;
}
}
DequeVariant::ChaseLev
}
pub fn update_cell(&mut self, shape: WorkloadShape, variant: DequeVariant) {
self.cells.insert(shape, variant);
}
pub fn len(&self) -> usize {
self.cells.len()
}
pub fn is_empty(&self) -> bool {
self.cells.is_empty()
}
pub fn cells(&self) -> &HashMap<WorkloadShape, DequeVariant> {
&self.cells
}
}
#[derive(Debug, Clone, Copy)]
pub struct DispatchedHandle {
pub variant: DequeVariant,
pub handle: DispatchHandle,
}
pub struct CrossProcessDispatcher {
table: DispatcherRoutingTable,
chase_lev: Option<Arc<SharedMemoryChaseLevBackend>>,
loh: Option<Arc<SharedMemoryLohBackend>>,
khpd: Option<Arc<SharedMemoryKhpdBackend>>,
urd: Option<Arc<SharedMemoryUrdBackend>>,
dispatched: AtomicU64,
}
impl CrossProcessDispatcher {
pub fn builder() -> CrossProcessDispatcherBuilder {
CrossProcessDispatcherBuilder::default()
}
pub fn table(&self) -> &DispatcherRoutingTable {
&self.table
}
pub fn table_mut(&mut self) -> &mut DispatcherRoutingTable {
&mut self.table
}
pub fn has(&self, variant: DequeVariant) -> bool {
match variant {
DequeVariant::ChaseLev => self.chase_lev.is_some(),
DequeVariant::Loh => self.loh.is_some(),
DequeVariant::Khpd => self.khpd.is_some(),
DequeVariant::Urd => self.urd.is_some(),
}
}
pub fn dispatched(&self) -> u64 {
self.dispatched.load(Ordering::Relaxed)
}
pub fn pick_with_fallback(&self, shape: &WorkloadShape) -> Result<DequeVariant, BackendError> {
let primary = self.table.pick(shape);
if self.has(primary) {
return Ok(primary);
}
if let Some(_be) = &self.chase_lev
&& (shape.args_inline_bytes as usize) <= CHASE_LEV_INLINE_BYTES
{
return Ok(DequeVariant::ChaseLev);
}
Err(BackendError::NotSupported)
}
pub fn dispatch_marshal(
&self,
shape: &WorkloadShape,
closure_id: u32,
args: &[u8],
) -> Result<DispatchedHandle, BackendError> {
let variant = self.pick_with_fallback(shape)?;
if args.len() > variant.inline_args_bytes() {
return Err(BackendError::Launch(format!(
"dispatcher: args length {} exceeds {} inline ceiling {}",
args.len(),
variant.label(),
variant.inline_args_bytes(),
)));
}
let handle = match variant {
DequeVariant::ChaseLev => self
.chase_lev
.as_ref()
.ok_or(BackendError::NotSupported)?
.dispatch_marshal(closure_id, args)?,
DequeVariant::Loh => self
.loh
.as_ref()
.ok_or(BackendError::NotSupported)?
.dispatch_marshal(closure_id, args)?,
DequeVariant::Khpd => self
.khpd
.as_ref()
.ok_or(BackendError::NotSupported)?
.dispatch_marshal(closure_id, args)?,
DequeVariant::Urd => self
.urd
.as_ref()
.ok_or(BackendError::NotSupported)?
.dispatch_marshal(closure_id, args)?,
};
self.dispatched.fetch_add(1, Ordering::Relaxed);
Ok(DispatchedHandle { variant, handle })
}
pub fn dispatch_marshal_batch(
&self,
shape: &WorkloadShape,
items: &[(u32, &[u8])],
) -> Result<Vec<DispatchedHandle>, BackendError> {
let variant = self.pick_with_fallback(shape)?;
for (_, args) in items {
if args.len() > variant.inline_args_bytes() {
return Err(BackendError::Launch(format!(
"dispatcher: batch item args length {} exceeds {} inline ceiling {}",
args.len(),
variant.label(),
variant.inline_args_bytes(),
)));
}
}
let handles = match variant {
DequeVariant::ChaseLev => {
let be = self.chase_lev.as_ref().ok_or(BackendError::NotSupported)?;
let mut out = Vec::with_capacity(items.len());
for (cid, args) in items {
out.push(be.dispatch_marshal(*cid, args)?);
}
out
}
DequeVariant::Loh => self
.loh
.as_ref()
.ok_or(BackendError::NotSupported)?
.dispatch_marshal_batch(items)?,
DequeVariant::Khpd => self
.khpd
.as_ref()
.ok_or(BackendError::NotSupported)?
.dispatch_marshal_batch(items)?,
DequeVariant::Urd => self
.urd
.as_ref()
.ok_or(BackendError::NotSupported)?
.dispatch_marshal_batch(items)?,
};
self.dispatched
.fetch_add(items.len() as u64, Ordering::Relaxed);
Ok(handles.into_iter().map(|h| DispatchedHandle { variant, handle: h }).collect())
}
pub fn wait_handle(
&self,
handle: DispatchedHandle,
iter_budget: u32,
) -> Result<Result<Vec<u8>, String>, BackendError> {
match handle.variant {
DequeVariant::ChaseLev => self
.chase_lev
.as_ref()
.ok_or(BackendError::NotSupported)?
.wait_handle(handle.handle, iter_budget),
DequeVariant::Loh => self
.loh
.as_ref()
.ok_or(BackendError::NotSupported)?
.wait_handle(handle.handle, iter_budget),
DequeVariant::Khpd => self
.khpd
.as_ref()
.ok_or(BackendError::NotSupported)?
.wait_handle(handle.handle, iter_budget),
DequeVariant::Urd => self
.urd
.as_ref()
.ok_or(BackendError::NotSupported)?
.wait_handle(handle.handle, iter_budget),
}
}
pub fn poll_handle(
&self,
handle: DispatchedHandle,
) -> Result<Option<Result<Vec<u8>, String>>, BackendError> {
match handle.variant {
DequeVariant::ChaseLev => self
.chase_lev
.as_ref()
.ok_or(BackendError::NotSupported)?
.poll_handle(handle.handle),
DequeVariant::Loh => self
.loh
.as_ref()
.ok_or(BackendError::NotSupported)?
.poll_handle(handle.handle),
DequeVariant::Khpd => self
.khpd
.as_ref()
.ok_or(BackendError::NotSupported)?
.poll_handle(handle.handle),
DequeVariant::Urd => self
.urd
.as_ref()
.ok_or(BackendError::NotSupported)?
.poll_handle(handle.handle),
}
}
pub fn chase_lev_backend(&self) -> Option<&SharedMemoryChaseLevBackend> {
self.chase_lev.as_deref()
}
pub fn loh_backend(&self) -> Option<&SharedMemoryLohBackend> {
self.loh.as_deref()
}
pub fn khpd_backend(&self) -> Option<&SharedMemoryKhpdBackend> {
self.khpd.as_deref()
}
pub fn urd_backend(&self) -> Option<&SharedMemoryUrdBackend> {
self.urd.as_deref()
}
}
#[derive(Default)]
pub struct CrossProcessDispatcherBuilder {
table: DispatcherRoutingTable,
chase_lev: Option<Arc<SharedMemoryChaseLevBackend>>,
loh: Option<Arc<SharedMemoryLohBackend>>,
khpd: Option<Arc<SharedMemoryKhpdBackend>>,
urd: Option<Arc<SharedMemoryUrdBackend>>,
}
impl CrossProcessDispatcherBuilder {
pub fn with_table(mut self, table: DispatcherRoutingTable) -> Self {
self.table = table;
self
}
pub fn with_chase_lev(mut self, be: Arc<SharedMemoryChaseLevBackend>) -> Self {
self.chase_lev = Some(be);
self
}
pub fn with_loh(mut self, be: Arc<SharedMemoryLohBackend>) -> Self {
self.loh = Some(be);
self
}
pub fn with_khpd(mut self, be: Arc<SharedMemoryKhpdBackend>) -> Self {
self.khpd = Some(be);
self
}
pub fn with_urd(mut self, be: Arc<SharedMemoryUrdBackend>) -> Self {
self.urd = Some(be);
self
}
pub fn build(self) -> CrossProcessDispatcher {
CrossProcessDispatcher {
table: self.table,
chase_lev: self.chase_lev,
loh: self.loh,
khpd: self.khpd,
urd: self.urd,
dispatched: AtomicU64::new(0),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::shared_mem::pass_registry::{hash_name, register, unregister};
use std::path::PathBuf;
fn temp_pair(label: &str) -> (PathBuf, PathBuf) {
let mut d = std::env::temp_dir();
let mut l = std::env::temp_dir();
let pid = std::process::id();
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
d.push(format!("flynnel_vd_{pid}_{nonce}_{label}_d.bin"));
l.push(format!("flynnel_vd_{pid}_{nonce}_{label}_l.bin"));
(d, l)
}
#[test]
fn heuristic_picks_chase_lev_for_request_reply() {
let table = DispatcherRoutingTable::default_heuristic();
let shape = WorkloadShape::request_reply(8);
assert_eq!(table.pick(&shape), DequeVariant::ChaseLev);
}
#[test]
fn heuristic_picks_khpd_for_producer_fast_small_args() {
let table = DispatcherRoutingTable::default_heuristic();
let shape = WorkloadShape::producer_fast(8, 64);
assert_eq!(table.pick(&shape), DequeVariant::Khpd);
}
#[test]
fn heuristic_picks_loh_for_producer_fast_args_over_khpd() {
let shape = WorkloadShape::producer_fast(16, 64);
let table = DispatcherRoutingTable::empty();
assert_eq!(table.pick(&shape), DequeVariant::Loh);
}
#[test]
fn heuristic_picks_chase_lev_when_args_exceed_loh() {
let shape = WorkloadShape::producer_fast(45, 64);
let table = DispatcherRoutingTable::empty();
assert_eq!(table.pick(&shape), DequeVariant::ChaseLev);
}
#[test]
fn heuristic_picks_urd_for_multi_thief_small_args() {
let table = DispatcherRoutingTable::empty();
let shape = WorkloadShape::multi_thief(8, 4, 64);
assert_eq!(table.pick(&shape), DequeVariant::Urd);
}
#[test]
fn update_cell_overrides_heuristic() {
let mut table = DispatcherRoutingTable::empty();
let shape = WorkloadShape::request_reply(8);
assert_eq!(table.pick(&shape), DequeVariant::ChaseLev);
table.update_cell(shape, DequeVariant::Urd);
assert_eq!(table.pick(&shape), DequeVariant::Urd);
assert_eq!(table.len(), 1);
}
#[test]
fn inline_args_bytes_matches_per_backend_ceilings() {
assert_eq!(DequeVariant::ChaseLev.inline_args_bytes(), CHASE_LEV_INLINE_BYTES);
assert_eq!(DequeVariant::Loh.inline_args_bytes(), LOH_ARGS_INLINE_BYTES);
assert_eq!(DequeVariant::Khpd.inline_args_bytes(), KHPD_ARGS_INLINE_BYTES);
assert_eq!(DequeVariant::Urd.inline_args_bytes(), KHPD_ARGS_INLINE_BYTES);
}
#[test]
fn dispatcher_with_no_backends_rejects_dispatch() {
let d = CrossProcessDispatcher::builder()
.with_table(DispatcherRoutingTable::default_heuristic())
.build();
let shape = WorkloadShape::request_reply(8);
let r = d.dispatch_marshal(&shape, 0, &[]);
assert!(matches!(r, Err(BackendError::NotSupported)));
}
#[test]
fn dispatcher_round_trips_through_chase_lev() {
let (d, l) = temp_pair("cl_route");
let cl =
Arc::new(SharedMemoryChaseLevBackend::create(0, &d, &l, 4, 8).expect("create"));
let dispatcher = CrossProcessDispatcher::builder()
.with_table(DispatcherRoutingTable::default_heuristic())
.with_chase_lev(Arc::clone(&cl))
.build();
let id = hash_name("flynnel.test.vd.cl_adder");
register(id, |args| {
let a = u32::from_le_bytes(args[0..4].try_into().unwrap());
let b = u32::from_le_bytes(args[4..8].try_into().unwrap());
Ok((a + b).to_le_bytes().to_vec())
});
let mut payload = [0u8; 8];
payload[..4].copy_from_slice(&13u32.to_le_bytes());
payload[4..].copy_from_slice(&29u32.to_le_bytes());
let shape = WorkloadShape::request_reply(8);
let handle = dispatcher
.dispatch_marshal(&shape, id, &payload)
.expect("dispatch");
assert_eq!(handle.variant, DequeVariant::ChaseLev);
cl.drain_owner().expect("drain").expect("had work");
let r = dispatcher
.wait_handle(handle, 1024)
.expect("wait")
.expect("ok branch");
let v = u32::from_le_bytes(r[..4].try_into().unwrap());
assert_eq!(v, 42);
unregister(id);
std::fs::remove_file(&d).ok();
std::fs::remove_file(&l).ok();
}
#[test]
fn dispatcher_rejects_oversize_args() {
let (d, l) = temp_pair("oversize");
let cl =
Arc::new(SharedMemoryChaseLevBackend::create(0, &d, &l, 4, 8).expect("create"));
let dispatcher = CrossProcessDispatcher::builder()
.with_chase_lev(Arc::clone(&cl))
.build();
let shape = WorkloadShape::request_reply(8);
let big = vec![0u8; 64];
let r = dispatcher.dispatch_marshal(&shape, 0, &big);
assert!(matches!(r, Err(BackendError::Launch(_))));
std::fs::remove_file(&d).ok();
std::fs::remove_file(&l).ok();
}
#[test]
fn fallback_to_chase_lev_when_primary_variant_absent() {
let (d, l) = temp_pair("fallback");
let cl =
Arc::new(SharedMemoryChaseLevBackend::create(0, &d, &l, 4, 8).expect("create"));
let dispatcher = CrossProcessDispatcher::builder()
.with_chase_lev(Arc::clone(&cl))
.build();
let shape = WorkloadShape::multi_thief(8, 4, 64);
let picked = dispatcher.pick_with_fallback(&shape).expect("pick");
assert_eq!(picked, DequeVariant::ChaseLev);
std::fs::remove_file(&d).ok();
std::fs::remove_file(&l).ok();
}
}