use crate::planner::{self, Interval};
use crate::{Encoding, ExecutionContext, Registry, Target, TensorRef};
use core::alloc::Layout;
use core::marker::PhantomData;
use core::mem::MaybeUninit;
use nanoglia_core::backend::{BackendKind, Kernel, KernelPlan};
use nanoglia_core::le::LeSlice;
use nanoglia_core::{
ALIGN, Allocator, Block, DataType, Detail, Dims, Error, ErrorCode, MAX_OP_INPUTS,
MAX_OP_OUTPUTS, OpCode, OpDesc, Quantization, Result, TensorDesc, align_up,
};
use nanoglia_tflite::{Model, SubGraph};
#[derive(Clone, Copy, Debug)]
pub(crate) enum Location {
Unused,
Const { offset: u32, len: u32 },
Arena { offset: u32, len: u32 },
}
#[derive(Clone, Copy, Debug)]
pub(crate) struct OpEntry<'k> {
pub kernel: &'k dyn Kernel,
pub persist_offset: u32,
pub persist_len: u32,
pub io_start: u32,
pub inputs: u16,
pub outputs: u16,
}
#[derive(Clone, Copy, Debug)]
pub(crate) struct Section<T> {
offset: usize,
len: usize,
_type: PhantomData<T>,
}
impl<T> Section<T> {
fn new(layout: &mut Layout, len: usize) -> Result<Self> {
let array = Layout::array::<T>(len).map_err(|_| Error::out_of_memory(usize::MAX))?;
let (extended, offset) = layout
.extend(array)
.map_err(|_| Error::out_of_memory(usize::MAX))?;
*layout = extended;
Ok(Self {
offset,
len,
_type: PhantomData,
})
}
fn get<'b>(&self, block: &'b [u8]) -> &'b [T] {
debug_assert!(self.offset + self.len * size_of::<T>() <= block.len());
unsafe {
core::slice::from_raw_parts(block.as_ptr().add(self.offset).cast::<T>(), self.len)
}
}
fn uninit<'b>(&self, block: &'b mut [u8]) -> &'b mut [MaybeUninit<T>] {
debug_assert!(self.offset + self.len * size_of::<T>() <= block.len());
unsafe {
core::slice::from_raw_parts_mut(
block.as_mut_ptr().add(self.offset).cast::<MaybeUninit<T>>(),
self.len,
)
}
}
}
pub struct Graph<'m, A: Allocator> {
model: &'m [u8],
subgraph: SubGraph<'m>,
block: Block<A>,
ops: Section<OpEntry<'m>>,
locations: Section<Location>,
io: Section<i32>,
arena_size: usize,
scratch_offset: usize,
scratch_size: usize,
}
impl<A: Allocator> core::fmt::Debug for Graph<'_, A> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("Graph")
.field("operators", &self.ops.len)
.field("tensors", &self.locations.len)
.field("graph_bytes", &self.block.len())
.field("context_bytes", &self.arena_size)
.finish()
}
}
const NO_TENSOR: TensorDesc<'static> = TensorDesc {
index: u32::MAX,
dtype: DataType::I8,
dims: Dims::scalar(),
quant: Quantization::NONE,
data: None,
name: "",
};
struct OpDescs<'m> {
inputs: [Option<TensorDesc<'m>>; MAX_OP_INPUTS],
outputs: [TensorDesc<'m>; MAX_OP_OUTPUTS],
}
fn malformed() -> Error {
Error::new(
ErrorCode::InvalidEncoding,
Detail::MalformedModel { offset: 0 },
)
}
pub(crate) fn tensor_index(raw: i32, count: usize) -> Result<Option<usize>> {
match raw {
-1 => Ok(None),
i if i >= 0 && (i as usize) < count => Ok(Some(i as usize)),
_ => Err(malformed()),
}
}
impl<'m, A: Allocator> Graph<'m, A> {
pub fn load_in(
builders: &[&'m [u8]],
encoding: Encoding,
target: Target,
alloc: A,
) -> Result<Self> {
Self::load_with(&Registry::default(), builders, encoding, target, alloc)
}
pub fn load_with(
registry: &Registry<'m>,
builders: &[&'m [u8]],
encoding: Encoding,
target: Target,
alloc: A,
) -> Result<Self> {
let (bytes, subgraph) = open(builders, encoding)?;
let sizing = Sizing::compute(registry, target, &subgraph)?;
let tensor_count = subgraph.tensor_count();
let mut block = Block::new_zeroed(sizing.block_size, alloc)?;
let memory = block.as_mut_slice();
let (intervals, order) = sizing.sections.working_memory(memory);
let activations = plan_activations(&subgraph, intervals, order)?;
for t in 0..tensor_count {
let desc = subgraph.tensor(t)?;
let location = if let Some(data) = desc.data {
let offset = data.as_ptr() as usize - bytes.as_ptr() as usize;
let align = desc.dtype.size().unwrap_or(1);
if data.as_ptr().align_offset(align) != 0 {
return Err(Error::invalid_argument(Detail::Misaligned {
tensor: t as u32,
}));
}
Location::Const {
offset: offset as u32,
len: data.len() as u32,
}
} else {
let (intervals, _) = sizing.sections.working_memory(memory);
let iv = intervals[t];
if iv.first == u32::MAX {
Location::Unused
} else {
Location::Arena {
offset: iv.offset,
len: iv.size,
}
}
};
sizing.sections.locations.uninit(memory)[t].write(location);
}
let mut persist_offset = sizing.persist_start;
let mut io_next = 0usize;
for index in 0..subgraph.operator_count() {
let mut descs = OpDescs {
inputs: [None; MAX_OP_INPUTS],
outputs: [NO_TENSOR; MAX_OP_OUTPUTS],
};
let (desc, input_ids, output_ids) = describe(&subgraph, index, &mut descs)?;
let kernel = select(registry, target, &desc)?.kernel;
let plan = kernel.plan(&desc)?;
if plan.scratch > sizing.scratch_size {
return Err(Error::new(ErrorCode::Unknown, Detail::None));
}
let persist_len = plan.persistent;
let region = persist_offset
.checked_add(persist_len)
.and_then(|end| memory.get_mut(persist_offset..end))
.ok_or_else(|| Error::new(ErrorCode::Unknown, Detail::None))?;
kernel.prepare(&desc, region)?;
let io_slots = sizing.sections.io.uninit(memory);
let io_start = io_next;
for raw in input_ids.iter().chain(output_ids.iter()) {
io_slots[io_next].write(raw);
io_next += 1;
}
sizing.sections.ops.uninit(memory)[index].write(OpEntry {
kernel,
persist_offset: persist_offset as u32,
persist_len: persist_len as u32,
io_start: io_start as u32,
inputs: input_ids.len() as u16,
outputs: output_ids.len() as u16,
});
persist_offset = align_up(persist_len)
.and_then(|len| persist_offset.checked_add(len))
.ok_or_else(|| Error::out_of_memory(usize::MAX))?;
}
let scratch_offset =
align_up(activations).ok_or_else(|| Error::out_of_memory(usize::MAX))?;
let scratch_size =
align_up(sizing.scratch_size).ok_or_else(|| Error::out_of_memory(usize::MAX))?;
let arena_size = scratch_offset
.checked_add(scratch_size)
.ok_or_else(|| Error::out_of_memory(usize::MAX))?;
Ok(Self {
model: bytes,
subgraph,
block,
ops: sizing.sections.ops,
locations: sizing.sections.locations,
io: sizing.sections.io,
arena_size,
scratch_offset,
scratch_size,
})
}
pub fn memory_requirements(&self) -> MemoryRequirements {
MemoryRequirements {
graph_bytes: self.block.len(),
context_bytes: self.arena_size,
}
}
pub fn init_execution_context_in<B: Allocator>(
&self,
alloc: B,
) -> Result<ExecutionContext<'_, 'm, A, B>> {
ExecutionContext::new(self, alloc)
}
pub fn input_count(&self) -> usize {
self.subgraph.inputs().len()
}
pub fn output_count(&self) -> usize {
self.subgraph.outputs().len()
}
pub fn input(&self, index: usize) -> Result<TensorDesc<'m>> {
self.io_desc(self.subgraph.inputs(), index)
}
pub fn output(&self, index: usize) -> Result<TensorDesc<'m>> {
self.io_desc(self.subgraph.outputs(), index)
}
pub fn inputs(&self) -> impl Iterator<Item = TensorDesc<'m>> + '_ {
(0..self.input_count()).filter_map(|i| self.input(i).ok())
}
pub fn outputs(&self) -> impl Iterator<Item = TensorDesc<'m>> + '_ {
(0..self.output_count()).filter_map(|i| self.output(i).ok())
}
pub fn input_index(&self, name: &str) -> Result<usize> {
self.find(self.subgraph.inputs(), name)
}
pub fn output_index(&self, name: &str) -> Result<usize> {
self.find(self.subgraph.outputs(), name)
}
pub fn graph_bytes(&self) -> usize {
self.block.len()
}
pub fn context_bytes(&self) -> usize {
self.arena_size
}
pub fn operator_count(&self) -> usize {
self.ops.len
}
pub fn kernel_name(&self, index: usize) -> Option<&'static str> {
self.ops().get(index).map(|op| op.kernel.name())
}
fn io_desc(&self, list: LeSlice<'m, i32>, index: usize) -> Result<TensorDesc<'m>> {
let raw = list.get(index).ok_or_else(|| {
Error::invalid_argument(Detail::IndexOutOfRange {
index: index as u32,
})
})?;
let t = tensor_index(raw, self.locations.len)?.ok_or_else(malformed)?;
self.subgraph.tensor(t)
}
fn find(&self, list: LeSlice<'m, i32>, name: &str) -> Result<usize> {
for (i, raw) in list.iter().enumerate() {
if let Some(t) = tensor_index(raw, self.locations.len)?
&& self.subgraph.tensor(t)?.name == name
{
return Ok(i);
}
}
Err(Error::new(ErrorCode::NotFound, Detail::TensorNotFound))
}
pub(crate) fn ops(&self) -> &[OpEntry<'m>] {
self.ops.get(self.block.as_slice())
}
pub(crate) fn io_table(&self) -> &[i32] {
self.io.get(self.block.as_slice())
}
pub(crate) fn location(&self, raw: i32) -> Location {
match tensor_index(raw, self.locations.len) {
Ok(Some(t)) => self.locations.get(self.block.as_slice())[t],
_ => Location::Unused,
}
}
pub(crate) fn persist(&self, op: &OpEntry<'_>) -> &[u8] {
let start = op.persist_offset as usize;
&self.block.as_slice()[start..start + op.persist_len as usize]
}
pub(crate) fn model(&self) -> &'m [u8] {
self.model
}
pub(crate) fn tensors(&self) -> GraphTensors<'_> {
GraphTensors {
subgraph: &self.subgraph,
locations: self.locations.get(self.block.as_slice()),
model: self.model,
}
}
pub(crate) fn scratch(&self) -> (usize, usize) {
(self.scratch_offset, self.scratch_size)
}
pub(crate) fn input_tensor(&self, index: usize) -> Result<(TensorDesc<'m>, Location)> {
let raw = self.subgraph.inputs().get(index).ok_or_else(|| {
Error::invalid_argument(Detail::IndexOutOfRange {
index: index as u32,
})
})?;
Ok((
self.io_desc(self.subgraph.inputs(), index)?,
self.location(raw),
))
}
pub(crate) fn output_tensor(&self, index: usize) -> Result<(TensorDesc<'m>, Location)> {
let raw = self.subgraph.outputs().get(index).ok_or_else(|| {
Error::invalid_argument(Detail::IndexOutOfRange {
index: index as u32,
})
})?;
Ok((
self.io_desc(self.subgraph.outputs(), index)?,
self.location(raw),
))
}
}
#[cfg(feature = "alloc")]
impl<'m> Graph<'m, nanoglia_core::allocator_api2::alloc::Global> {
pub fn load(builders: &[&'m [u8]], encoding: Encoding, target: Target) -> Result<Self> {
Self::load_in(
builders,
encoding,
target,
nanoglia_core::allocator_api2::alloc::Global,
)
}
pub fn init_execution_context(
&self,
) -> Result<
ExecutionContext<
'_,
'm,
nanoglia_core::allocator_api2::alloc::Global,
nanoglia_core::allocator_api2::alloc::Global,
>,
> {
self.init_execution_context_in(nanoglia_core::allocator_api2::alloc::Global)
}
}
#[derive(Clone, Copy)]
pub(crate) struct GraphTensors<'a> {
subgraph: &'a SubGraph<'a>,
locations: &'a [Location],
model: &'a [u8],
}
impl<'a> GraphTensors<'a> {
pub(crate) fn opcode(&self, index: usize) -> Result<OpCode<'a>> {
Ok(self.subgraph.operator(index)?.code)
}
pub(crate) fn view(&self, raw: i32, arena: &'a [u8]) -> Result<TensorRef<'a>> {
let t = tensor_index(raw, self.locations.len())?
.ok_or_else(|| Error::new(ErrorCode::NotFound, Detail::TensorNotFound))?;
let desc = self.subgraph.tensor(t)?;
let range =
|offset: u32, len: u32| offset as usize..(offset as usize).saturating_add(len as usize);
let data = match self.locations.get(t).copied().unwrap_or(Location::Unused) {
Location::Unused => Some(&[][..]),
Location::Const { offset, len } => self.model.get(range(offset, len)),
Location::Arena { offset, len } => arena.get(range(offset, len)),
};
Ok(TensorRef::new(desc, data.ok_or_else(malformed)?))
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MemoryRequirements {
pub graph_bytes: usize,
pub context_bytes: usize,
}
impl MemoryRequirements {
pub const fn arena_bytes(&self, contexts: usize) -> usize {
let graph = round_up_saturating(self.graph_bytes);
let context = round_up_saturating(self.context_bytes);
(ALIGN - 1)
.saturating_add(graph)
.saturating_add(contexts.saturating_mul(context))
}
pub fn compute_in<A: Allocator>(
builders: &[&[u8]],
encoding: Encoding,
target: Target,
alloc: A,
) -> Result<Self> {
Self::compute_with(&Registry::default(), builders, encoding, target, alloc)
}
pub fn compute_with<A: Allocator>(
registry: &Registry<'_>,
builders: &[&[u8]],
encoding: Encoding,
target: Target,
alloc: A,
) -> Result<Self> {
let (_, subgraph) = open(builders, encoding)?;
let mut layout = Layout::from_size_align(0, ALIGN).map_err(|_| Error::out_of_memory(0))?;
let working = WorkingSections::new(&mut layout, subgraph.tensor_count())?;
let mut block = Block::new_zeroed(layout.size(), alloc)?;
let breakdown = measure(registry, target, &subgraph, &working, block.as_mut_slice())?;
Ok(breakdown.requirements())
}
#[cfg(feature = "alloc")]
pub fn compute(builders: &[&[u8]], encoding: Encoding, target: Target) -> Result<Self> {
Self::compute_in(
builders,
encoding,
target,
nanoglia_core::allocator_api2::alloc::Global,
)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub struct MemoryBreakdown {
pub graph_bytes: usize,
pub context_bytes: usize,
pub persistent_bytes: usize,
pub activation_bytes: usize,
pub scratch_bytes: usize,
}
impl MemoryBreakdown {
pub const fn requirements(&self) -> MemoryRequirements {
MemoryRequirements {
graph_bytes: self.graph_bytes,
context_bytes: self.context_bytes,
}
}
}
pub(crate) fn measure(
registry: &Registry<'_>,
target: Target,
subgraph: &SubGraph<'_>,
working: &WorkingSections,
memory: &mut [u8],
) -> Result<MemoryBreakdown> {
let sizing = Sizing::compute(registry, target, subgraph)?;
let (intervals, order) = working.working_memory(memory);
let activations = plan_activations(subgraph, intervals, order)?;
for t in 0..subgraph.tensor_count() {
let desc = subgraph.tensor(t)?;
if let Some(data) = desc.data
&& data.as_ptr().align_offset(desc.dtype.size().unwrap_or(1)) != 0
{
return Err(Error::invalid_argument(Detail::Misaligned {
tensor: t as u32,
}));
}
}
let overflow = || Error::out_of_memory(usize::MAX);
let activation_bytes = align_up(activations).ok_or_else(overflow)?;
let scratch_bytes = align_up(sizing.scratch_size).ok_or_else(overflow)?;
Ok(MemoryBreakdown {
graph_bytes: sizing.block_size,
context_bytes: activation_bytes
.checked_add(scratch_bytes)
.ok_or_else(overflow)?,
persistent_bytes: sizing.block_size - sizing.persist_start,
activation_bytes,
scratch_bytes,
})
}
const fn round_up_saturating(n: usize) -> usize {
match n.checked_next_multiple_of(ALIGN) {
Some(rounded) => rounded,
None => usize::MAX,
}
}
pub(crate) fn open<'m>(
builders: &[&'m [u8]],
encoding: Encoding,
) -> Result<(&'m [u8], SubGraph<'m>)> {
let [bytes] = builders else {
return Err(Error::invalid_argument(Detail::None));
};
let bytes: &'m [u8] = bytes;
match encoding {
Encoding::TensorFlowLite => {}
Encoding::Autodetect if Model::is_tflite(bytes) => {}
Encoding::Autodetect => {
return Err(Error::new(
ErrorCode::InvalidEncoding,
Detail::UnknownEncoding,
));
}
}
let model = Model::parse(bytes)?;
if model.subgraph_count() != 1 {
return Err(Error::unsupported(Detail::UnsupportedStructure));
}
let subgraph = model.subgraph(0)?;
if subgraph.operator_count() > u32::MAX as usize || subgraph.tensor_count() > u32::MAX as usize
{
return Err(Error::unsupported(Detail::UnsupportedStructure));
}
Ok((bytes, subgraph))
}
struct BlockSections<'k> {
ops: Section<OpEntry<'k>>,
locations: Section<Location>,
io: Section<i32>,
working: WorkingSections,
}
impl BlockSections<'_> {
fn working_memory<'b>(&self, memory: &'b mut [u8]) -> (&'b mut [Interval], &'b mut [u32]) {
self.working.working_memory(memory)
}
}
pub(crate) struct WorkingSections {
intervals: Section<Interval>,
order: Section<u32>,
}
impl WorkingSections {
pub(crate) fn new(layout: &mut Layout, tensor_count: usize) -> Result<Self> {
Ok(Self {
intervals: Section::new(layout, tensor_count)?,
order: Section::new(layout, tensor_count)?,
})
}
fn working_memory<'b>(&self, memory: &'b mut [u8]) -> (&'b mut [Interval], &'b mut [u32]) {
assert!(
self.intervals.offset + self.intervals.len * size_of::<Interval>() <= self.order.offset
);
assert!(self.order.offset + self.order.len * size_of::<u32>() <= memory.len());
unsafe {
let base = memory.as_mut_ptr();
(
core::slice::from_raw_parts_mut(
base.add(self.intervals.offset).cast::<Interval>(),
self.intervals.len,
),
core::slice::from_raw_parts_mut(
base.add(self.order.offset).cast::<u32>(),
self.order.len,
),
)
}
}
}
struct Sizing<'k> {
sections: BlockSections<'k>,
persist_start: usize,
block_size: usize,
scratch_size: usize,
}
impl Sizing<'_> {
fn compute(registry: &Registry<'_>, target: Target, subgraph: &SubGraph<'_>) -> Result<Self> {
let op_count = subgraph.operator_count();
let tensor_count = subgraph.tensor_count();
let mut persist_total = 0usize;
let mut scratch_size = 0usize;
let mut io_total = 0usize;
let mut npu_ops = 0usize;
for index in 0..op_count {
let choice = choose(registry, target, subgraph, index, None)?;
npu_ops += usize::from(choice.kind == BackendKind::Npu);
let plan = choice.plan;
persist_total = persist_total
.checked_add(
align_up(plan.persistent).ok_or_else(|| Error::out_of_memory(usize::MAX))?,
)
.ok_or_else(|| Error::out_of_memory(usize::MAX))?;
scratch_size = scratch_size.max(plan.scratch);
io_total += choice.io;
}
if target == Target::Npu && npu_ops == 0 {
return Err(Error::unsupported(Detail::None));
}
let mut layout = Layout::from_size_align(0, ALIGN).map_err(|_| Error::out_of_memory(0))?;
let ops = Section::new(&mut layout, op_count)?;
let locations = Section::new(&mut layout, tensor_count)?;
let io = Section::new(&mut layout, io_total)?;
let working = WorkingSections::new(&mut layout, tensor_count)?;
let persist_start =
align_up(layout.size()).ok_or_else(|| Error::out_of_memory(usize::MAX))?;
let block_size = persist_start
.checked_add(persist_total)
.ok_or_else(|| Error::out_of_memory(usize::MAX))?;
Ok(Self {
sections: BlockSections {
ops,
locations,
io,
working,
},
persist_start,
block_size,
scratch_size,
})
}
}
fn plan_activations(
subgraph: &SubGraph<'_>,
intervals: &mut [Interval],
order: &mut [u32],
) -> Result<usize> {
let tensor_count = subgraph.tensor_count();
let op_count = subgraph.operator_count();
intervals.fill(Interval {
size: 0,
first: u32::MAX,
last: 0,
offset: 0,
});
for raw in subgraph.inputs().iter() {
let t = tensor_index(raw, tensor_count)?.ok_or_else(malformed)?;
intervals[t].first = 0;
}
for index in 0..op_count {
let op_index = index as u32;
let op = subgraph.operator(index)?;
for raw in op.inputs.iter() {
if let Some(t) = tensor_index(raw, tensor_count)? {
if desc_is_constant(subgraph, t)? {
continue;
}
let iv = &mut intervals[t];
if iv.first == u32::MAX {
return Err(malformed());
}
iv.last = iv.last.max(op_index);
}
}
for raw in op.outputs.iter() {
let t = tensor_index(raw, tensor_count)?.ok_or_else(malformed)?;
if desc_is_constant(subgraph, t)? {
return Err(malformed());
}
let iv = &mut intervals[t];
if iv.first != u32::MAX {
return Err(malformed());
}
iv.first = op_index;
iv.last = iv.last.max(op_index);
}
}
for raw in subgraph.outputs().iter() {
let t = tensor_index(raw, tensor_count)?.ok_or_else(malformed)?;
if desc_is_constant(subgraph, t)? {
continue;
}
let iv = &mut intervals[t];
if iv.first == u32::MAX {
return Err(malformed());
}
iv.last = op_count as u32;
}
for (t, iv) in intervals.iter_mut().enumerate() {
if iv.first == u32::MAX {
continue;
}
let desc = subgraph.tensor(t)?;
if desc.data.is_some() {
continue;
}
let len = desc
.byte_len()
.ok_or_else(|| Error::unsupported(Detail::ShapeMismatch { tensor: t as u32 }))?;
iv.size = u32::try_from(len).map_err(|_| Error::out_of_memory(len))?;
}
planner::plan(intervals, order).ok_or_else(|| Error::out_of_memory(usize::MAX))
}
fn desc_is_constant(subgraph: &SubGraph<'_>, t: usize) -> Result<bool> {
Ok(subgraph.tensor(t)?.data.is_some())
}
fn describe<'m, 'd>(
subgraph: &SubGraph<'m>,
index: usize,
descs: &'d mut OpDescs<'m>,
) -> Result<(OpDesc<'m, 'd>, LeSlice<'m, i32>, LeSlice<'m, i32>)> {
let op = subgraph.operator(index)?;
let tensor_count = subgraph.tensor_count();
if op.inputs.len() > MAX_OP_INPUTS || op.outputs.len() > MAX_OP_OUTPUTS {
return Err(Error::unsupported(Detail::UnsupportedStructure));
}
for (slot, raw) in descs.inputs.iter_mut().zip(op.inputs.iter()) {
*slot = match tensor_index(raw, tensor_count)? {
Some(t) => {
if subgraph.tensor_is_variable(t)? {
return Err(Error::unsupported(Detail::UnsupportedStructure));
}
Some(subgraph.tensor(t)?)
}
None => None,
};
}
for (slot, raw) in descs.outputs.iter_mut().zip(op.outputs.iter()) {
let t = tensor_index(raw, tensor_count)?.ok_or_else(malformed)?;
*slot = subgraph.tensor(t)?;
}
let desc = OpDesc {
index: index as u32,
code: op.code,
options: op.options,
inputs: &descs.inputs[..op.inputs.len()],
outputs: &descs.outputs[..op.outputs.len()],
};
Ok((desc, op.inputs, op.outputs))
}
#[derive(Clone, Copy)]
struct Selected<'r> {
kernel: &'r dyn Kernel,
backend: &'static str,
kind: BackendKind,
}
fn select<'r>(
registry: &Registry<'r>,
target: Target,
desc: &OpDesc<'_, '_>,
) -> Result<Selected<'r>> {
for &backend in registry.backends() {
let kind = backend.kind();
if target == Target::Cpu && kind == BackendKind::Npu {
continue;
}
if let Some(kernel) = backend.select(desc) {
return Ok(Selected {
kernel,
backend: backend.name(),
kind,
});
}
}
Err(Error::unsupported(Detail::UnsupportedOp {
op_index: desc.index,
opcode: desc.opcode_number(),
}))
}
#[derive(Clone, Copy)]
pub(crate) struct Choice<'r> {
pub kernel: &'r dyn Kernel,
pub backend: &'static str,
pub kind: BackendKind,
pub plan: KernelPlan,
pub io: usize,
}
pub(crate) fn choose<'r>(
registry: &Registry<'r>,
target: Target,
subgraph: &SubGraph<'_>,
index: usize,
persist: Option<&mut [u8]>,
) -> Result<Choice<'r>> {
let mut descs = OpDescs {
inputs: [None; MAX_OP_INPUTS],
outputs: [NO_TENSOR; MAX_OP_OUTPUTS],
};
let (desc, _, _) = describe(subgraph, index, &mut descs)?;
let selected = select(registry, target, &desc)?;
let plan = selected.kernel.plan(&desc)?;
if let Some(persist) = persist {
let region = persist
.get_mut(..plan.persistent)
.ok_or_else(|| Error::new(ErrorCode::Unknown, Detail::None))?;
region.fill(0);
selected.kernel.prepare(&desc, region)?;
}
Ok(Choice {
kernel: selected.kernel,
backend: selected.backend,
kind: selected.kind,
plan,
io: desc.inputs.len() + desc.outputs.len(),
})
}