use crate::device::backend::HardwareHint;
use crate::device::{Device, DeviceSelection};
use crate::error::Result;
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MatmulPrecision {
F32,
F16,
Bf16,
Tf32,
F64,
}
impl MatmulPrecision {
#[must_use]
pub const fn is_tensor_core_eligible(self) -> bool {
matches!(self, Self::F16 | Self::Bf16 | Self::Tf32)
}
}
const CPU_FALLBACK_THRESHOLD: usize = 1_000;
const EIGENDECOMP_CPU_THRESHOLD: usize = 128;
const LINEAR_SOLVE_CPU_THRESHOLD: usize = 256;
const MATMUL_LARGE_DIM: usize = 256;
const MATMUL_MEDIUM_DIM: usize = 64;
const WORKGROUP_FFT: [u32; 3] = [256, 1, 1];
const WORKGROUP_PHYSICS: [u32; 3] = [64, 1, 1];
#[derive(Debug, Clone)]
pub enum ComputeWorkload {
DenseMatmul {
m: usize,
n: usize,
k: usize,
precision: Option<MatmulPrecision>,
},
SparseInference {
input_sparsity: f64,
model_name: String,
},
ReservoirState {
reservoir_size: usize,
input_dim: usize,
},
FFT {
size: usize,
batch_count: usize,
},
PhysicsForce {
particle_count: usize,
force_type: String,
},
Eigendecomp {
matrix_size: usize,
},
LinearSolve {
system_size: usize,
},
BinaryPrescreen {
input_count: usize,
threshold: f64,
},
GenericWgsl {
shader_name: String,
},
}
#[derive(Debug, Clone)]
pub enum KernelTarget {
Npu {
model_id: String,
input_shape: Vec<usize>,
output_shape: Vec<usize>,
},
Wgsl {
shader: String,
device: DeviceSelection,
workgroup_size: [u32; 3],
},
Hybrid {
npu_prefilter: Option<String>,
wgsl_compute: String,
device: DeviceSelection,
},
Sovereign {
shader: String,
device: DeviceSelection,
hardware_hint: HardwareHint,
workgroup_size: [u32; 3],
},
}
pub struct KernelRouter {
npu_models: HashMap<String, NpuModelInfo>,
has_gpu: bool,
has_npu: bool,
has_tpu: bool,
}
#[derive(Debug, Clone)]
pub struct NpuModelInfo {
pub path: String,
pub input_shape: Vec<usize>,
pub output_shape: Vec<usize>,
pub sparsity_threshold: f64,
}
const DEFAULT_NPU_INPUT_DIM: usize = 128;
const DEFAULT_NPU_OUTPUT_DIM: usize = 10;
const DEFAULT_SPARSITY_THRESHOLD: f64 = 0.7;
impl NpuModelInfo {
fn from_discovered_path(path: String) -> Self {
Self {
path,
input_shape: vec![DEFAULT_NPU_INPUT_DIM],
output_shape: vec![DEFAULT_NPU_OUTPUT_DIM],
sparsity_threshold: DEFAULT_SPARSITY_THRESHOLD,
}
}
}
impl KernelRouter {
pub fn new() -> Result<Self> {
let has_gpu = Device::GPU.is_available();
let has_npu = Device::NPU.is_available();
let has_tpu = Device::TPU.is_available();
let npu_models = Self::discover_npu_models()?;
Ok(Self {
npu_models,
has_gpu,
has_npu,
has_tpu,
})
}
#[must_use]
pub fn has_tpu(&self) -> bool {
self.has_tpu
}
pub fn route(&self, workload: &ComputeWorkload) -> Result<KernelTarget> {
match workload {
ComputeWorkload::DenseMatmul { m, n, k, precision } => {
let size = m * n * k;
if let Some(prec) = precision
&& prec.is_tensor_core_eligible()
&& self.has_gpu
{
return Ok(KernelTarget::Sovereign {
shader: "matmul_mma".to_string(),
device: DeviceSelection::Gpu,
hardware_hint: HardwareHint::TensorCore,
workgroup_size: self.optimal_workgroup_for_matmul(*m, *n, *k),
});
}
let device = self.select_wgsl_device(size);
Ok(KernelTarget::Wgsl {
shader: "matmul".to_string(),
device,
workgroup_size: self.optimal_workgroup_for_matmul(*m, *n, *k),
})
}
ComputeWorkload::FFT { size, batch_count } => {
let total = size * batch_count;
let device = self.select_wgsl_device(total);
Ok(KernelTarget::Wgsl {
shader: "fft".to_string(),
device,
workgroup_size: WORKGROUP_FFT,
})
}
ComputeWorkload::PhysicsForce {
particle_count,
force_type,
} => {
let device = self.select_wgsl_device(*particle_count);
Ok(KernelTarget::Wgsl {
shader: format!("forces/{force_type}"),
device,
workgroup_size: WORKGROUP_PHYSICS,
})
}
ComputeWorkload::Eigendecomp { matrix_size } => {
let device = if *matrix_size < EIGENDECOMP_CPU_THRESHOLD {
DeviceSelection::Cpu
} else {
self.select_wgsl_device(matrix_size * matrix_size)
};
Ok(KernelTarget::Wgsl {
shader: "linalg/eigh".to_string(),
device,
workgroup_size: [16, 16, 1],
})
}
ComputeWorkload::LinearSolve { system_size } => {
let device = if *system_size < LINEAR_SOLVE_CPU_THRESHOLD {
DeviceSelection::Cpu
} else {
self.select_wgsl_device(system_size * system_size)
};
Ok(KernelTarget::Wgsl {
shader: "linalg/solve".to_string(),
device,
workgroup_size: [16, 16, 1],
})
}
ComputeWorkload::GenericWgsl { shader_name } => {
let device = self.select_wgsl_device(CPU_FALLBACK_THRESHOLD);
Ok(KernelTarget::Wgsl {
shader: shader_name.clone(),
device,
workgroup_size: [64, 1, 1],
})
}
ComputeWorkload::SparseInference {
input_sparsity,
model_name,
} => {
if self.has_npu
&& let Some(model_info) = self.npu_models.get(model_name)
&& *input_sparsity >= model_info.sparsity_threshold
{
return Ok(KernelTarget::Npu {
model_id: model_info.path.clone(),
input_shape: model_info.input_shape.clone(),
output_shape: model_info.output_shape.clone(),
});
}
let device = self.select_wgsl_device(CPU_FALLBACK_THRESHOLD * 10);
Ok(KernelTarget::Wgsl {
shader: format!("snn/{model_name}"),
device,
workgroup_size: [64, 1, 1],
})
}
ComputeWorkload::ReservoirState {
reservoir_size,
input_dim,
} => {
if self.has_npu
&& let Some(model_info) = self.npu_models.get("reservoir")
{
return Ok(KernelTarget::Npu {
model_id: model_info.path.clone(),
input_shape: vec![*input_dim],
output_shape: vec![*reservoir_size],
});
}
let device = self.select_wgsl_device(reservoir_size * input_dim);
Ok(KernelTarget::Wgsl {
shader: "reservoir/esn".to_string(),
device,
workgroup_size: [64, 1, 1],
})
}
ComputeWorkload::BinaryPrescreen {
input_count,
threshold: _,
} => {
if self.has_npu
&& let Some(model_info) = self.npu_models.get("prescreen")
{
return Ok(KernelTarget::Npu {
model_id: model_info.path.clone(),
input_shape: vec![*input_count],
output_shape: vec![*input_count],
});
}
let device = self.select_wgsl_device(*input_count);
Ok(KernelTarget::Wgsl {
shader: "prescreen/binary_threshold".to_string(),
device,
workgroup_size: WORKGROUP_FFT,
})
}
}
}
#[must_use]
pub fn can_route_to_npu(&self, workload: &ComputeWorkload) -> bool {
if !self.has_npu {
return false;
}
match workload {
ComputeWorkload::SparseInference { model_name, .. } => {
self.npu_models.contains_key(model_name)
}
ComputeWorkload::ReservoirState { .. } => self.npu_models.contains_key("reservoir"),
ComputeWorkload::BinaryPrescreen { .. } => self.npu_models.contains_key("prescreen"),
_ => false,
}
}
fn select_wgsl_device(&self, workload_size: usize) -> DeviceSelection {
if workload_size < CPU_FALLBACK_THRESHOLD {
return DeviceSelection::Cpu;
}
if self.has_gpu {
DeviceSelection::Gpu
} else {
DeviceSelection::Cpu
}
}
fn optimal_workgroup_for_matmul(&self, m: usize, n: usize, _k: usize) -> [u32; 3] {
if m >= MATMUL_LARGE_DIM && n >= MATMUL_LARGE_DIM {
[16, 16, 1]
} else if m >= MATMUL_MEDIUM_DIM && n >= MATMUL_MEDIUM_DIM {
[8, 8, 1]
} else {
[4, 4, 1]
}
}
fn discover_npu_models() -> Result<HashMap<String, NpuModelInfo>> {
let mut models = HashMap::new();
for dir in super::akida::akida_model_dirs() {
let path = dir.as_path();
if path.exists()
&& let Ok(entries) = std::fs::read_dir(path)
{
for entry in entries.flatten() {
if let Some(ext) = entry.path().extension()
&& ext == "fbz"
{
if let Some(name) = entry.path().file_stem() {
let model_name = name.to_string_lossy().to_string();
models.insert(
model_name.clone(),
NpuModelInfo::from_discovered_path(
entry.path().to_string_lossy().to_string(),
),
);
}
}
}
}
}
Ok(models)
}
pub fn register_npu_model(&mut self, name: &str, info: NpuModelInfo) {
self.npu_models.insert(name.to_string(), info);
}
#[must_use]
pub fn available_npu_models(&self) -> Vec<&str> {
self.npu_models
.keys()
.map(std::string::String::as_str)
.collect()
}
}
impl Default for KernelRouter {
fn default() -> Self {
Self::new().unwrap_or_else(|e| {
tracing::warn!(
error = %e,
"KernelRouter: NPU discovery failed, routing without NPU models"
);
Self {
npu_models: HashMap::new(),
has_gpu: Device::GPU.is_available(),
has_npu: false,
has_tpu: false,
}
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_kernel_router_creation() {
let router = KernelRouter::new();
assert!(router.is_ok());
}
#[test]
fn test_dense_matmul_routes_to_wgsl() {
let router = KernelRouter::default();
let workload = ComputeWorkload::DenseMatmul {
m: 1024,
n: 1024,
k: 1024,
precision: None,
};
let target = router.route(&workload).unwrap();
match target {
KernelTarget::Wgsl { shader, device, .. } => {
assert_eq!(shader, "matmul");
assert!(device.supports_wgsl());
}
_ => panic!("Dense matmul (f32) should route to WGSL"),
}
}
#[test]
fn test_dense_matmul_f16_routes_to_sovereign_tensor_core() {
let router = KernelRouter::default();
let workload = ComputeWorkload::DenseMatmul {
m: 1024,
n: 1024,
k: 1024,
precision: Some(MatmulPrecision::F16),
};
let target = router.route(&workload).unwrap();
match target {
KernelTarget::Sovereign {
hardware_hint,
shader,
..
} => {
assert_eq!(hardware_hint, HardwareHint::TensorCore);
assert_eq!(shader, "matmul_mma");
}
KernelTarget::Wgsl { .. } => {
}
_ => panic!("F16 matmul should route to Sovereign/TensorCore or Wgsl"),
}
}
#[test]
fn test_dense_matmul_f64_routes_to_wgsl() {
let router = KernelRouter::default();
let workload = ComputeWorkload::DenseMatmul {
m: 512,
n: 512,
k: 512,
precision: Some(MatmulPrecision::F64),
};
let target = router.route(&workload).unwrap();
match target {
KernelTarget::Wgsl { shader, .. } => {
assert_eq!(shader, "matmul");
}
_ => panic!("F64 matmul should route to WGSL compute path"),
}
}
#[test]
fn test_physics_force_routes_to_wgsl() {
let router = KernelRouter::default();
let workload = ComputeWorkload::PhysicsForce {
particle_count: 10_000,
force_type: "lennard_jones".to_string(),
};
let target = router.route(&workload).unwrap();
match target {
KernelTarget::Wgsl { device, .. } => {
assert!(device.supports_wgsl());
}
_ => panic!("Physics force should route to WGSL"),
}
}
#[test]
fn test_small_eigendecomp_routes_to_cpu() {
let router = KernelRouter::default();
let workload = ComputeWorkload::Eigendecomp { matrix_size: 32 };
let target = router.route(&workload).unwrap();
match target {
KernelTarget::Wgsl { device, .. } => {
assert_eq!(device, DeviceSelection::Cpu);
}
_ => panic!("Small eigendecomp should route to CPU"),
}
}
#[test]
fn test_sparse_inference_fallback() {
let router = KernelRouter::default(); let workload = ComputeWorkload::SparseInference {
input_sparsity: 0.95,
model_name: "nonexistent".to_string(),
};
let target = router.route(&workload).unwrap();
match target {
KernelTarget::Wgsl { .. } => {}
_ => panic!("Missing NPU model should fall back to WGSL"),
}
}
#[test]
fn test_can_route_to_npu() {
let router = KernelRouter::default();
let workload = ComputeWorkload::SparseInference {
input_sparsity: 0.95,
model_name: "test".to_string(),
};
assert!(!router.can_route_to_npu(&workload));
let dense = ComputeWorkload::DenseMatmul {
m: 1024,
n: 1024,
k: 1024,
precision: None,
};
assert!(!router.can_route_to_npu(&dense));
}
#[test]
fn test_register_npu_model() {
let mut router = KernelRouter::default();
router.register_npu_model(
"custom_model",
NpuModelInfo {
path: "/path/to/model.fbz".to_string(),
input_shape: vec![128],
output_shape: vec![10],
sparsity_threshold: 0.8,
},
);
assert!(router.available_npu_models().contains(&"custom_model"));
}
}