use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering};
use std::sync::Arc;
use std::time::Instant;
use crate::backend::{Backend, BackendError, KernelArg, KernelHandle, backend_by_id};
use crate::sched::hazard::HazardDomain;
use crate::sched::call_site::{Placement, caller_site};
use crate::sched::plan::JobPlan;
pub const LAUNCH_AMORTIZATION_FACTOR: u64 = 4;
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub struct AccelOpId(u32);
type CpuImpl = Arc<dyn Fn(u32, &[KernelArg<'_>]) + Send + Sync>;
struct AccelOp {
name: String,
bytes_per_item: u32,
cpu: CpuImpl,
kernels: AtomicPtr<Bindings>,
}
type Bindings = Vec<(Backend, KernelHandle)>;
const MAX_OPS: usize = 1024;
const READERS: usize = 256;
const RETIRED: usize = 64;
static OPS: [AtomicPtr<AccelOp>; MAX_OPS] =
[const { AtomicPtr::new(core::ptr::null_mut()) }; MAX_OPS];
static NEXT_OP: AtomicU32 = AtomicU32::new(0);
static BINDINGS: HazardDomain<Bindings, READERS, RETIRED> = HazardDomain::new();
fn binding_reader() -> usize {
thread_local! {
static READER: core::cell::Cell<usize> = const { core::cell::Cell::new(usize::MAX) };
}
READER.with(|cell| {
let held = cell.get();
if held != usize::MAX {
return held;
}
let fresh = BINDINGS.claim_reader();
cell.set(fresh);
fresh
})
}
fn op_by_id(op: AccelOpId) -> &'static AccelOp {
let slot = OPS
.get(op.0 as usize)
.expect("AccelOpId not issued by register_accel_op");
let published = slot.load(Ordering::Acquire);
assert!(
!published.is_null(),
"AccelOpId {} was not issued by register_accel_op",
op.0
);
unsafe { &*published }
}
fn update_bindings(op: &'static AccelOp, edit: impl Fn(&mut Bindings)) {
loop {
let current = op.kernels.load(Ordering::Acquire);
let mut next: Bindings = unsafe { &*current }.clone();
edit(&mut next);
let fresh = Box::into_raw(Box::new(next));
match op
.kernels
.compare_exchange(current, fresh, Ordering::SeqCst, Ordering::Acquire)
{
Ok(replaced) => {
unsafe { BINDINGS.retire(replaced) };
return;
}
Err(winner) => {
debug_assert!(!winner.is_null(), "a binding list is never null");
drop(unsafe { Box::from_raw(fresh) });
}
}
}
}
pub fn register_accel_op<F>(name: &str, bytes_per_item: u32, cpu: F) -> AccelOpId
where
F: Fn(u32, &[KernelArg<'_>]) + Send + Sync + 'static,
{
let index = NEXT_OP.fetch_add(1, Ordering::AcqRel) as usize;
assert!(
index < MAX_OPS,
"accel op table sized for {MAX_OPS} ops, {name:?} asked to be the {index}th"
);
let op: &'static AccelOp = Box::leak(Box::new(AccelOp {
name: name.to_string(),
bytes_per_item,
cpu: Arc::new(cpu),
kernels: AtomicPtr::new(Box::into_raw(Box::new(Bindings::new()))),
}));
OPS[index].store(op as *const AccelOp as *mut AccelOp, Ordering::Release);
AccelOpId(index as u32)
}
pub fn bind_accel_kernel(
op: AccelOpId,
backend: Backend,
entry: &str,
source: &[u8],
) -> Result<(), BackendError> {
let be = backend_by_id(&backend).ok_or(BackendError::DeviceUnavailable(backend))?;
let handle = be.register_kernel(entry, source)?;
bind_accel_kernel_handle(op, backend, handle);
Ok(())
}
pub fn bind_accel_kernel_handle(op: AccelOpId, backend: Backend, handle: KernelHandle) {
let op = op_by_id(op);
update_bindings(op, |bindings| {
match bindings.iter_mut().find(|(b, _)| *b == backend) {
Some(slot) => slot.1 = handle,
None => bindings.push((backend, handle)),
}
});
}
pub fn accel_target(plan: &JobPlan, op: AccelOpId) -> Option<(Backend, KernelHandle)> {
let op = op_by_id(op);
let guard = BINDINGS.protect(binding_reader(), &op.kernels);
let bound = guard.get()?;
let bound_and_registered = |b: Backend| -> Option<(Backend, KernelHandle)> {
let handle = bound.iter().find(|(k, _)| *k == b).map(|(_, h)| *h)?;
backend_by_id(&b)?;
Some((b, handle))
};
if let Some(hint) = plan.backend_hint
&& let Some(found) = bound_and_registered(hint)
{
return Some(found);
}
let active = crate::sched::adaptive_backend::active_backend_id();
if active != Backend::Cpu
&& let Some(found) = bound_and_registered(active)
{
return Some(found);
}
bound
.iter()
.find(|(b, _)| backend_by_id(b).is_some())
.map(|(b, h)| (*b, *h))
}
pub(crate) fn cost_gate_pass(
plan: &JobPlan,
caps: &crate::backend::BackendCapabilities,
count: u32,
bytes_per_item: u32,
) -> Option<bool> {
if !plan.estimated_per_item_ns_explicit {
return None;
}
let per_item = plan.estimated_per_item_ns? as u64;
let est_total_ns = per_item.saturating_mul(count as u64);
let launch_ns =
LAUNCH_AMORTIZATION_FACTOR.saturating_mul(caps.launch_latency_ns as u64);
let bytes = (count as u64).saturating_mul(bytes_per_item as u64);
let h2d_ns = bytes
.saturating_mul(1_000_000_000)
.checked_div(caps.h2d_bw_bytes_per_sec)
.unwrap_or(0);
Some(est_total_ns >= launch_ns.saturating_add(h2d_ns))
}
#[derive(Debug, Clone)]
pub struct AccelReport {
pub placement: Placement,
pub backend: Option<Backend>,
pub cpu_ns: Option<u64>,
pub backend_ns: Option<u64>,
pub gate_blocked: bool,
pub fell_back: bool,
pub fallback_error: Option<String>,
}
#[track_caller]
pub fn dispatch_accel(
plan: &JobPlan,
op: AccelOpId,
count: u32,
cpu_args: &[KernelArg<'_>],
kernel_args: &[KernelArg<'_>],
) -> AccelReport {
let op_arc = op_by_id(op);
let site_ref = plan.site.unwrap_or_else(caller_site);
let site = site_ref.get();
let run_cpu = |record: bool| -> u64 {
let t0 = Instant::now();
(op_arc.cpu)(count, cpu_args);
let ns = t0.elapsed().as_nanos() as u64;
if record {
site.record_placement(count, Some(ns), None);
}
ns
};
let Some((backend_id, handle)) = accel_target(plan, op) else {
let cpu_ns = run_cpu(true);
return AccelReport {
placement: Placement::Cpu,
backend: None,
cpu_ns: Some(cpu_ns),
backend_ns: None,
gate_blocked: false,
fell_back: false,
fallback_error: None,
};
};
let backend = backend_by_id(&backend_id).expect("accel_target checked registration");
if cost_gate_pass(plan, &backend.capabilities(), count, op_arc.bytes_per_item)
== Some(false)
{
let cpu_ns = run_cpu(true);
return AccelReport {
placement: Placement::Cpu,
backend: Some(backend_id),
cpu_ns: Some(cpu_ns),
backend_ns: None,
gate_blocked: true,
fell_back: false,
fallback_error: None,
};
}
let run_kernel = || -> Result<u64, BackendError> {
let t0 = Instant::now();
backend.dispatch_kernel_sync(handle, count, kernel_args)?;
Ok(t0.elapsed().as_nanos() as u64)
};
match site.choose_placement(count) {
Placement::Cpu => {
let cpu_ns = run_cpu(true);
AccelReport {
placement: Placement::Cpu,
backend: Some(backend_id),
cpu_ns: Some(cpu_ns),
backend_ns: None,
gate_blocked: false,
fell_back: false,
fallback_error: None,
}
}
Placement::Backend => match run_kernel() {
Ok(ns) => {
site.record_placement(count, None, Some(ns));
AccelReport {
placement: Placement::Backend,
backend: Some(backend_id),
cpu_ns: None,
backend_ns: Some(ns),
gate_blocked: false,
fell_back: false,
fallback_error: None,
}
}
Err(e) => {
let cpu_ns = run_cpu(true);
AccelReport {
placement: Placement::Cpu,
backend: Some(backend_id),
cpu_ns: Some(cpu_ns),
backend_ns: None,
gate_blocked: false,
fell_back: true,
fallback_error: Some(format!("{e:?}")),
}
}
},
Placement::Race => {
let kernel_outcome = run_kernel();
let cpu_ns = run_cpu(false);
match kernel_outcome {
Ok(dev_ns) => {
site.record_placement(count, Some(cpu_ns), Some(dev_ns));
AccelReport {
placement: Placement::Race,
backend: Some(backend_id),
cpu_ns: Some(cpu_ns),
backend_ns: Some(dev_ns),
gate_blocked: false,
fell_back: false,
fallback_error: None,
}
}
Err(e) => {
site.record_placement(count, Some(cpu_ns), None);
AccelReport {
placement: Placement::Cpu,
backend: Some(backend_id),
cpu_ns: Some(cpu_ns),
backend_ns: None,
gate_blocked: false,
fell_back: true,
fallback_error: Some(format!("{e:?}")),
}
}
}
}
}
}
pub fn accel_op_name(op: AccelOpId) -> String {
op_by_id(op).name.clone()
}
#[derive(Clone, Debug)]
pub struct RegisteredAccelOp {
pub op: AccelOpId,
pub name: String,
pub bytes_per_item: u32,
pub kernels: Vec<Backend>,
}
pub fn registered_accel_ops() -> Vec<RegisteredAccelOp> {
let reader = binding_reader();
let mut out = Vec::new();
for (index, slot) in OPS.iter().enumerate() {
let published = slot.load(Ordering::Acquire);
if published.is_null() {
continue;
}
let op = unsafe { &*published };
let guard = BINDINGS.protect(reader, &op.kernels);
let kernels = match guard.get() {
Some(bindings) => bindings.iter().map(|(backend, _)| *backend).collect(),
None => Vec::new(),
};
out.push(RegisteredAccelOp {
op: AccelOpId(index as u32),
name: op.name.clone(),
bytes_per_item: op.bytes_per_item,
kernels,
});
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::{BackendCapabilities, DispatchBackend, register_backend};
use std::sync::atomic::{AtomicU32, Ordering};
struct StubAccel {
id: Backend,
caps: BackendCapabilities,
kernel_calls: AtomicU32,
fail: bool,
}
impl DispatchBackend for StubAccel {
fn id(&self) -> Backend {
self.id
}
fn capabilities(&self) -> BackendCapabilities {
self.caps
}
fn dispatch_parallel_for(&self, _count: u32, _work: &(dyn Fn(u32) + Send + Sync)) {}
fn dispatch_one(&self, work: Box<dyn FnOnce() + Send>) {
work();
}
fn register_kernel(
&self,
_name: &str,
_source: &[u8],
) -> Result<KernelHandle, BackendError> {
Ok(KernelHandle(0xACCE1))
}
fn dispatch_kernel(
&self,
_handle: KernelHandle,
_count: u32,
_args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
if self.fail {
return Err(BackendError::Launch("stub failure".into()));
}
self.kernel_calls.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
fn fast_caps() -> BackendCapabilities {
BackendCapabilities {
simt_width: 32,
max_threads_in_flight: 4096,
launch_latency_ns: 1_000,
h2d_bw_bytes_per_sec: 10_000_000_000,
}
}
#[test]
fn unbound_op_runs_cpu() {
let cpu_calls = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&cpu_calls);
let op = register_accel_op("unbound", 4, move |_n, _a| {
c.fetch_add(1, Ordering::SeqCst);
});
let plan = JobPlan::bare(0, 64);
let report = dispatch_accel(&plan, op, 64, &[], &[]);
assert_eq!(report.placement, Placement::Cpu);
assert!(report.backend.is_none());
assert_eq!(cpu_calls.load(Ordering::SeqCst), 1);
}
#[test]
fn cold_bucket_races_then_warm_exploits_faster_backend() {
let stub = Arc::new(StubAccel {
id: Backend::Custom(0x7001),
caps: fast_caps(),
kernel_calls: AtomicU32::new(0),
fail: false,
});
register_backend(Arc::clone(&stub) as _);
let cpu_calls = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&cpu_calls);
let op = register_accel_op("race_then_exploit", 4, move |_n, _a| {
c.fetch_add(1, Ordering::SeqCst);
std::thread::sleep(std::time::Duration::from_millis(3));
});
bind_accel_kernel(op, Backend::Custom(0x7001), "k", b"")
.expect("stub registers any kernel");
let plan = JobPlan::bare(0, 4096).with_backend(Backend::Custom(0x7001));
let first = dispatch_accel(&plan, op, 4096, &[], &[]);
assert_eq!(first.placement, Placement::Race, "cold bucket races");
assert_eq!(cpu_calls.load(Ordering::SeqCst), 1);
assert_eq!(stub.kernel_calls.load(Ordering::SeqCst), 1);
for _ in 0..8 {
let r = dispatch_accel(&plan, op, 4096, &[], &[]);
assert_eq!(r.placement, Placement::Backend, "warm bucket exploits");
}
assert_eq!(cpu_calls.load(Ordering::SeqCst), 1, "CPU stays cold");
assert_eq!(stub.kernel_calls.load(Ordering::SeqCst), 9);
}
#[test]
fn gate_blocks_work_below_launch_amortization() {
let stub = Arc::new(StubAccel {
id: Backend::Custom(0x7002),
caps: BackendCapabilities {
launch_latency_ns: 100_000,
..fast_caps()
},
kernel_calls: AtomicU32::new(0),
fail: false,
});
register_backend(Arc::clone(&stub) as _);
let cpu_calls = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&cpu_calls);
let op = register_accel_op("gated", 4, move |_n, _a| {
c.fetch_add(1, Ordering::SeqCst);
});
bind_accel_kernel(op, Backend::Custom(0x7002), "k", b"").expect("stub binds");
let plan = JobPlan::bare(0, 100)
.with_backend(Backend::Custom(0x7002))
.with_estimated_per_item_ns(10);
let r = dispatch_accel(&plan, op, 100, &[], &[]);
assert!(r.gate_blocked, "gate must reject sub-breakeven work");
assert_eq!(r.placement, Placement::Cpu);
assert_eq!(stub.kernel_calls.load(Ordering::SeqCst), 0);
assert_eq!(cpu_calls.load(Ordering::SeqCst), 1);
}
#[test]
fn active_backend_tag_steers_hintless_dispatch() {
let stub = Arc::new(StubAccel {
id: Backend::Custom(0x7004),
caps: fast_caps(),
kernel_calls: AtomicU32::new(0),
fail: false,
});
register_backend(Arc::clone(&stub) as _);
let op = register_accel_op("tag_steered", 4, |_n, _a| {});
bind_accel_kernel(op, Backend::Custom(0x7004), "k", b"").expect("stub binds");
let hint_less = JobPlan::bare(0, 4096);
crate::sched::adaptive_backend::migrate_backend(Backend::Custom(0x7004));
let steered = accel_target(&hint_less, op);
crate::sched::adaptive_backend::migrate_backend(Backend::Cpu);
assert_eq!(
steered.map(|(b, _)| b),
Some(Backend::Custom(0x7004)),
"hint-less resolution must honor the active-backend tag",
);
let hinted = JobPlan::bare(0, 4096).with_backend(Backend::Custom(0x7004));
assert_eq!(
accel_target(&hinted, op).map(|(b, _)| b),
Some(Backend::Custom(0x7004)),
);
assert_eq!(
accel_target(&hint_less, op).map(|(b, _)| b),
Some(Backend::Custom(0x7004)),
"first bound-and-registered backend resolves under a Cpu tag",
);
}
#[test]
fn binding_to_unregistered_backend_stays_cpu() {
let cpu_calls = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&cpu_calls);
let op = register_accel_op("ghost_backend", 4, move |_n, _a| {
c.fetch_add(1, Ordering::SeqCst);
});
bind_accel_kernel_handle(op, Backend::Custom(0x7BAD), KernelHandle(1));
let plan = JobPlan::bare(0, 64);
let r = dispatch_accel(&plan, op, 64, &[], &[]);
assert_eq!(r.placement, Placement::Cpu);
assert!(r.backend.is_none(), "unregistered binding is no target");
assert_eq!(cpu_calls.load(Ordering::SeqCst), 1);
}
#[test]
fn kernel_failure_falls_back_to_cpu() {
let stub = Arc::new(StubAccel {
id: Backend::Custom(0x7003),
caps: fast_caps(),
kernel_calls: AtomicU32::new(0),
fail: true,
});
register_backend(Arc::clone(&stub) as _);
let cpu_calls = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&cpu_calls);
let op = register_accel_op("flaky", 4, move |_n, _a| {
c.fetch_add(1, Ordering::SeqCst);
});
bind_accel_kernel(op, Backend::Custom(0x7003), "k", b"").expect("stub binds");
let plan = JobPlan::bare(0, 512).with_backend(Backend::Custom(0x7003));
let r = dispatch_accel(&plan, op, 512, &[], &[]);
assert!(r.fell_back, "launch failure must be covered by CPU");
assert_eq!(r.placement, Placement::Cpu);
assert_eq!(cpu_calls.load(Ordering::SeqCst), 1);
}
#[test]
fn cost_gate_math() {
let plan = JobPlan::bare(0, 1000).with_estimated_per_item_ns(1000);
let caps = fast_caps();
assert_eq!(cost_gate_pass(&plan, &caps, 1000, 4), Some(true));
let hint_less = JobPlan::bare(0, 1000);
assert_eq!(cost_gate_pass(&hint_less, &caps, 1000, 4), None);
let heavy_bytes = cost_gate_pass(&plan, &caps, 1000, u32::MAX);
assert_eq!(heavy_bytes, Some(false));
}
#[test]
fn accel_op_name_round_trips() {
let op = register_accel_op("named_op", 0, |_n, _a| {});
assert_eq!(accel_op_name(op), "named_op");
}
#[test]
fn the_registry_lists_an_op_with_its_bytes_and_its_bindings() {
let op = register_accel_op("listed_op", 12, |_n, _a| {});
let listed = registered_accel_ops();
let mine = listed
.iter()
.find(|r| r.name == "listed_op")
.expect("an op that registered is in the registry");
assert_eq!(mine.op, op, "the id listed is the id register handed back");
assert_eq!(mine.bytes_per_item, 12);
assert!(
mine.kernels.is_empty(),
"an op with nothing bound reports no kernels, which is what says every \
dispatch of it runs the CPU implementation"
);
let stub = Arc::new(StubAccel {
id: Backend::Custom(0x7005),
caps: BackendCapabilities::cpu_defaults(),
kernel_calls: AtomicU32::new(0),
fail: false,
});
register_backend(Arc::clone(&stub) as _);
bind_accel_kernel(op, Backend::Custom(0x7005), "k", b"").expect("stub binds");
let after = registered_accel_ops();
let mine = after
.iter()
.find(|r| r.name == "listed_op")
.expect("still registered");
assert_eq!(mine.kernels, vec![Backend::Custom(0x7005)]);
}
#[test]
fn every_listed_id_resolves_to_the_name_it_was_listed_under() {
register_accel_op("paired_a", 1, |_n, _a| {});
register_accel_op("paired_b", 2, |_n, _a| {});
for row in registered_accel_ops() {
assert_eq!(
accel_op_name(row.op),
row.name,
"the id listed for {} resolves to a different op",
row.name
);
}
}
}