use std::collections::HashMap;
use std::path::{Path, PathBuf};
use anyhow::Result;
use ndarray::ArrayD;
use ort::ep::{CoreML, ExecutionProvider};
use ort::session::builder::GraphOptimizationLevel;
use ort::session::Session;
use ort::value::{DynValue, Value};
use super::{Model, Runtime, Tensor};
pub struct OrtRuntime {
pub optimization_level: GraphOptimizationLevel,
pub intra_threads: usize,
pub profiling_path: Option<PathBuf>,
}
impl Default for OrtRuntime {
fn default() -> Self {
Self {
optimization_level: GraphOptimizationLevel::Level3,
intra_threads: 0,
profiling_path: None,
}
}
}
impl OrtRuntime {
pub fn is_coreml_available(&self) -> bool {
CoreML::default().is_available().unwrap_or(false)
}
}
impl Runtime for OrtRuntime {
type Model = OrtModel;
#[allow(clippy::needless_match)]
fn load_model(&self, path: &Path) -> Result<OrtModel> {
let optimization_level = match self.optimization_level {
GraphOptimizationLevel::Disable => GraphOptimizationLevel::Disable,
GraphOptimizationLevel::Level1 => GraphOptimizationLevel::Level1,
GraphOptimizationLevel::Level2 => GraphOptimizationLevel::Level2,
GraphOptimizationLevel::Level3 => GraphOptimizationLevel::Level3,
GraphOptimizationLevel::All => GraphOptimizationLevel::All,
};
let mut builder = Session::builder()?
.with_optimization_level(optimization_level)?
.with_intra_threads(self.intra_threads)?
.with_execution_providers([CoreML::default().build()])?;
if let Some(ref profile_path) = self.profiling_path {
builder = builder.with_profiling(profile_path)?;
}
let session = builder.commit_from_file(path)?;
Ok(OrtModel { session })
}
}
pub struct OrtModel {
session: Session,
}
impl OrtModel {
pub fn end_profiling(&mut self) -> Result<String> {
Ok(self.session.end_profiling()?)
}
}
impl Model for OrtModel {
fn run(&mut self, inputs: &[(&str, &Tensor)]) -> Result<HashMap<String, Tensor>> {
let ort_inputs: Vec<(String, DynValue)> = inputs
.iter()
.map(|(name, tensor)| {
let shape: Vec<usize> = tensor.shape.clone();
let array = ArrayD::from_shape_vec(shape, tensor.data.clone())?;
let value: DynValue = Value::from_array(array)?.into_dyn();
Ok((name.to_string(), value))
})
.collect::<Result<Vec<_>>>()?;
let input_refs: Vec<(&str, &DynValue)> = ort_inputs
.iter()
.map(|(name, value)| (name.as_str(), value))
.collect();
let outputs = self.session.run(input_refs)?;
let mut result = HashMap::new();
for (name, value) in outputs.iter() {
let (shape, data) = value.try_extract_tensor::<f32>()?;
let tensor = Tensor {
shape: shape.iter().map(|&d| d as usize).collect(),
data: data.to_vec(),
};
result.insert(name.to_string(), tensor);
}
Ok(result)
}
}