use crate::graph::{Graph, GraphTensors, Location};
use crate::{GraphHandle, TensorMut, TensorRef};
use core::future::Future;
use core::ops::ControlFlow;
use core::pin::Pin;
use core::task::{Context, Poll};
use nanoglia_core::backend::{EvalCtx, Kernel};
use nanoglia_core::{
Allocator, Block, Detail, Error, ErrorCode, MAX_OP_INPUTS, MAX_OP_OUTPUTS, OpCode, Result,
};
pub struct GraphExecutionContext<G: GraphHandle, B: Allocator> {
graph: G,
arena: Block<B>,
next: Option<usize>,
}
impl<G: GraphHandle, B: Allocator> core::fmt::Debug for GraphExecutionContext<G, B> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("GraphExecutionContext")
.field("arena_bytes", &self.arena.len())
.finish()
}
}
pub type ExecutionContext<'g, 'm, A, B> = GraphExecutionContext<&'g Graph<'m, A>, B>;
impl<'g, 'm, A: Allocator, B: Allocator> ExecutionContext<'g, 'm, A, B> {
pub(crate) fn new(graph: &'g Graph<'m, A>, alloc: B) -> Result<Self> {
Self::new_in(graph, alloc)
}
}
impl<G: GraphHandle, B: Allocator> GraphExecutionContext<G, B> {
pub fn new_in(graph: G, alloc: B) -> Result<Self> {
let bytes = graph.graph().context_bytes();
Ok(Self {
graph,
arena: Block::new_zeroed(bytes, alloc)?,
next: None,
})
}
pub fn graph(&self) -> &Graph<'_, G::Allocator> {
self.graph.graph()
}
pub fn graph_handle(&self) -> &G {
&self.graph
}
pub fn set_input(&mut self, index: usize, data: &[u8]) -> Result<()> {
let mut input = self.input_mut(index)?;
let bytes = input.bytes_mut();
if bytes.len() != data.len() {
return Err(Error::invalid_argument(Detail::SizeMismatch {
expected: bytes.len() as u32,
actual: data.len() as u32,
}));
}
bytes.copy_from_slice(data);
Ok(())
}
pub fn input_mut(&mut self, index: usize) -> Result<TensorMut<'_>> {
let (desc, location) = self.graph.graph().input_tensor(index)?;
let Location::Arena { offset, len } = location else {
return Err(Error::invalid_argument(Detail::IndexOutOfRange {
index: index as u32,
}));
};
let start = offset as usize;
let data = &mut self.arena.as_mut_slice()[start..start + len as usize];
Ok(TensorMut::new(desc, data))
}
pub fn output(&self, index: usize) -> Result<TensorRef<'_>> {
let (desc, location) = self.graph.graph().output_tensor(index)?;
let data: &[u8] = match location {
Location::Arena { offset, len } => {
&self.arena.as_slice()[offset as usize..offset as usize + len as usize]
}
Location::Const { offset, len } => {
&self.graph.graph().model()[offset as usize..offset as usize + len as usize]
}
Location::Unused => &[],
};
Ok(TensorRef::new(desc, data))
}
pub fn output_by_name(&self, name: &str) -> Result<TensorRef<'_>> {
self.output(self.graph.graph().output_index(name)?)
}
#[inline]
pub fn compute(&mut self) -> Result<()> {
self.compute_with(&mut ())
}
#[inline]
pub fn compute_with<O: Observer + ?Sized>(&mut self, observer: &mut O) -> Result<()> {
self.next = None;
for index in 0..self.graph.graph().operator_count() {
self.run_operator(index, observer)?;
}
Ok(())
}
#[inline]
pub fn step(&mut self) -> Result<Step> {
self.step_with(&mut ())
}
pub fn step_with<O: Observer + ?Sized>(&mut self, observer: &mut O) -> Result<Step> {
let count = self.graph.graph().operator_count();
let index = self.next.take().unwrap_or(0);
if index >= count {
return Ok(Step::Done);
}
self.run_operator(index, observer)?;
if index + 1 == count {
Ok(Step::Done)
} else {
self.next = Some(index + 1);
Ok(Step::Pending)
}
}
pub fn cancel(&mut self) {
self.next = None;
}
pub fn next_operator(&self) -> Option<usize> {
self.next
}
pub async fn compute_async(&mut self) -> Result<()> {
self.compute_async_with(&mut ()).await
}
pub async fn compute_async_with<O: Observer + ?Sized>(
&mut self,
observer: &mut O,
) -> Result<()> {
self.next = None;
loop {
match self.step_with(observer) {
Ok(Step::Done) => return Ok(()),
Ok(Step::Pending) => YieldNow(false).await,
Err(error) => return Err(error),
}
}
}
#[inline]
fn run_operator<O: Observer + ?Sized>(&mut self, index: usize, observer: &mut O) -> Result<()> {
let graph = self.graph.graph();
let model = graph.model();
let io = graph.io_table();
let tensors = graph.tensors();
let (scratch_offset, scratch_len) = graph.scratch();
let arena_len = self.arena.len();
let base = self.arena.as_mut_ptr();
let Some(op) = graph.ops().get(index) else {
return Err(Error::invalid_argument(Detail::IndexOutOfRange {
index: index as u32,
}));
};
{
let start = op.io_start as usize;
let inputs = &io[start..start + op.inputs as usize];
let outputs =
&io[start + op.inputs as usize..start + op.inputs as usize + op.outputs as usize];
let info = |arena| OpInfo {
index,
kernel: op.kernel,
inputs,
outputs,
tensors,
arena,
};
let flow = observer.before(&info(unsafe {
core::slice::from_raw_parts(base, arena_len)
}));
if flow.is_break() {
return Err(Error::new(
ErrorCode::Timeout,
Detail::Interrupted {
op_index: index as u32,
},
));
}
let mut input_bytes: [&[u8]; MAX_OP_INPUTS] = [&[]; MAX_OP_INPUTS];
for (slot, &raw) in input_bytes.iter_mut().zip(inputs) {
*slot = match graph.location(raw) {
Location::Unused => &[],
Location::Const { offset, len } => {
&model[offset as usize..offset as usize + len as usize]
}
Location::Arena { offset, len } => {
debug_assert!(offset as usize + len as usize <= arena_len);
unsafe {
core::slice::from_raw_parts(base.add(offset as usize), len as usize)
}
}
};
}
let mut output_bytes: [&mut [u8]; MAX_OP_OUTPUTS] =
core::array::from_fn(|_| <&mut [u8]>::default());
for (slot, &raw) in output_bytes.iter_mut().zip(outputs) {
if let Location::Arena { offset, len } = graph.location(raw) {
debug_assert!(offset as usize + len as usize <= arena_len);
*slot = unsafe {
core::slice::from_raw_parts_mut(base.add(offset as usize), len as usize)
};
}
}
debug_assert!(scratch_offset + scratch_len <= arena_len);
let scratch =
unsafe { core::slice::from_raw_parts_mut(base.add(scratch_offset), scratch_len) };
let mut ctx = EvalCtx {
inputs: &input_bytes[..op.inputs as usize],
outputs: &mut output_bytes[..op.outputs as usize],
persist: graph.persist(op),
scratch,
};
let result = op.kernel.eval(&mut ctx);
observer.after(
&info(unsafe { core::slice::from_raw_parts(base, arena_len) }),
&result,
);
result
}
}
}
pub trait Observer {
#[inline(always)]
fn before(&mut self, op: &OpInfo<'_>) -> ControlFlow<()> {
let _ = op;
ControlFlow::Continue(())
}
#[inline(always)]
fn after(&mut self, op: &OpInfo<'_>, result: &Result<()>) {
let _ = (op, result);
}
}
impl Observer for () {}
impl<O: Observer + ?Sized> Observer for &mut O {
#[inline(always)]
fn before(&mut self, op: &OpInfo<'_>) -> ControlFlow<()> {
(**self).before(op)
}
#[inline(always)]
fn after(&mut self, op: &OpInfo<'_>, result: &Result<()>) {
(**self).after(op, result);
}
}
pub struct OpInfo<'a> {
index: usize,
kernel: &'a dyn Kernel,
inputs: &'a [i32],
outputs: &'a [i32],
tensors: GraphTensors<'a>,
arena: &'a [u8],
}
impl core::fmt::Debug for OpInfo<'_> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("OpInfo")
.field("index", &self.index)
.field("kernel", &self.kernel.name())
.finish()
}
}
impl<'a> OpInfo<'a> {
pub fn index(&self) -> usize {
self.index
}
pub fn kernel_name(&self) -> &'static str {
self.kernel.name()
}
pub fn opcode(&self) -> Result<OpCode<'a>> {
self.tensors.opcode(self.index)
}
pub fn input_count(&self) -> usize {
self.inputs.len()
}
pub fn output_count(&self) -> usize {
self.outputs.len()
}
pub fn input(&self, i: usize) -> Result<TensorRef<'_>> {
self.tensor(self.inputs, i)
}
pub fn output(&self, i: usize) -> Result<TensorRef<'_>> {
self.tensor(self.outputs, i)
}
fn tensor(&self, list: &[i32], i: usize) -> Result<TensorRef<'_>> {
let raw = *list
.get(i)
.ok_or_else(|| Error::invalid_argument(Detail::IndexOutOfRange { index: i as u32 }))?;
self.tensors.view(raw, self.arena)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Step {
Pending,
Done,
}
struct YieldNow(bool);
impl Future for YieldNow {
type Output = ();
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
if self.0 {
Poll::Ready(())
} else {
self.0 = true;
cx.waker().wake_by_ref();
Poll::Pending
}
}
}