use std::collections::HashMap;
use std::fmt;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use fandhe_ai_onnx_interop::onnx::export::{
ExportError, ExportOptions, build_model_proto, try_encode_model,
};
use fandhe_ai_onnx_interop::onnx::export_nn::graph_from_layers;
use fandhe_ai_onnx_interop::onnx::external_data::{
DEFAULT_MAX_EXTERNAL_DATA_TOTAL_BYTES, DEFAULT_MAX_EXTERNAL_FILES, ExternalDataError,
ExternalDataOptions, build_graph_with_external_data,
};
use fandhe_ai_onnx_interop::onnx::graph::{Graph, GraphError, build_graph};
use fandhe_ai_onnx_interop::onnx::interp::{
InterpError, Value as InterpValue, run as interp_run, run_with_ops as interp_run_with_ops,
};
use fandhe_ai_onnx_interop::onnx::proto::{DecodeModelError, decode_model};
use fandhe_ai_tensor_core::f16;
use crate::Device;
use crate::Tensor;
use crate::compat::Sequential;
pub(crate) static CUDA_ONNX_GPU_EXEC: AtomicBool = AtomicBool::new(false);
#[cfg(target_os = "macos")]
pub(crate) static METAL_ONNX_GPU_EXEC: AtomicBool = AtomicBool::new(false);
pub(crate) fn set_cuda_onnx_gpu_execution_enabled(enabled: bool) {
CUDA_ONNX_GPU_EXEC.store(enabled, Ordering::SeqCst);
}
pub(crate) fn cuda_onnx_gpu_execution_enabled() -> bool {
CUDA_ONNX_GPU_EXEC.load(Ordering::SeqCst)
}
#[cfg(target_os = "macos")]
pub(crate) fn set_metal_onnx_gpu_execution_enabled(enabled: bool) {
METAL_ONNX_GPU_EXEC.store(enabled, Ordering::SeqCst);
}
#[cfg(target_os = "macos")]
pub(crate) fn metal_onnx_gpu_execution_enabled() -> bool {
METAL_ONNX_GPU_EXEC.load(Ordering::SeqCst)
}
#[cfg(target_os = "macos")]
fn resolve_metal_ops_if_enabled()
-> Result<Option<Box<dyn fandhe_ai_tensor_core::BackendOps + Send>>, OnnxError> {
if !metal_onnx_gpu_execution_enabled() {
return Ok(None);
}
let ops = crate::resolve_ops(Device::Metal).map_err(|e| OnnxError::Execution {
message: e.to_string(),
})?;
Ok(Some(ops))
}
#[cfg(not(target_os = "macos"))]
fn resolve_metal_ops_if_enabled()
-> Result<Option<Box<dyn fandhe_ai_tensor_core::BackendOps + Send>>, OnnxError> {
Ok(None)
}
#[derive(Debug)]
pub struct OnnxModel {
graph: Graph,
}
impl OnnxModel {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, OnnxError> {
let model = decode_model(bytes).map_err(map_decode_error)?;
let graph = build_graph(&model).map_err(map_graph_error)?;
Ok(Self { graph })
}
pub fn from_path(path: impl AsRef<Path>) -> Result<Self, OnnxError> {
Self::from_path_with_limits(path, &OnnxExternalDataLimits::default())
}
pub fn from_path_with_limits(
path: impl AsRef<Path>,
limits: &OnnxExternalDataLimits,
) -> Result<Self, OnnxError> {
let path = path.as_ref();
let bytes = std::fs::read(path).map_err(OnnxError::Io)?;
let model = decode_model(&bytes).map_err(map_decode_error)?;
drop(bytes);
let base_dir = match path.parent() {
Some(parent) if !parent.as_os_str().is_empty() => parent,
_ => Path::new("."),
};
let graph = build_graph_with_external_data(&model, base_dir, &limits.to_internal())
.map_err(map_graph_error)?;
Ok(Self { graph })
}
pub fn from_sequential(model: &Sequential) -> Result<Self, OnnxError> {
let graph = graph_from_layers(model.layers()).map_err(map_export_error)?;
Ok(Self { graph })
}
pub fn run(
&self,
feeds: HashMap<String, OnnxValue>,
) -> Result<HashMap<String, OnnxValue>, OnnxError> {
let interp_feeds: HashMap<String, InterpValue> = feeds
.into_iter()
.map(|(k, v)| (k, onnx_value_to_interp(v)))
.collect();
let outputs = if cuda_onnx_gpu_execution_enabled() {
let ops = crate::resolve_ops(Device::Cuda(0)).map_err(|e| OnnxError::Execution {
message: e.to_string(),
})?;
interp_run_with_ops(&self.graph, interp_feeds, ops.as_ref())
.map_err(map_interp_error)?
} else if let Some(ops) = resolve_metal_ops_if_enabled()? {
interp_run_with_ops(&self.graph, interp_feeds, ops.as_ref())
.map_err(map_interp_error)?
} else {
interp_run(&self.graph, interp_feeds).map_err(map_interp_error)?
};
Ok(outputs
.into_iter()
.map(|(k, v)| (k, interp_value_to_onnx(v)))
.collect())
}
pub fn to_bytes(&self, options: &OnnxExportOptions) -> Result<Vec<u8>, OnnxError> {
let model =
build_model_proto(&self.graph, &options.to_internal()).map_err(map_export_error)?;
try_encode_model(&model).map_err(map_export_error)
}
pub fn to_path(
&self,
path: impl AsRef<Path>,
options: &OnnxExportOptions,
) -> Result<(), OnnxError> {
let bytes = self.to_bytes(options)?;
std::fs::write(path.as_ref(), bytes).map_err(OnnxError::Io)
}
}
#[non_exhaustive]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OnnxExportOptions {
pub ir_version: i64,
pub opset_version: i64,
}
impl Default for OnnxExportOptions {
fn default() -> Self {
OnnxExportOptions {
ir_version: 8,
opset_version: 17,
}
}
}
impl OnnxExportOptions {
fn to_internal(&self) -> ExportOptions {
ExportOptions {
ir_version: self.ir_version,
opset_version: self.opset_version,
..ExportOptions::default()
}
}
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct OnnxExternalDataLimits {
pub max_total_bytes: u64,
pub max_external_files: usize,
}
impl Default for OnnxExternalDataLimits {
fn default() -> Self {
OnnxExternalDataLimits {
max_total_bytes: DEFAULT_MAX_EXTERNAL_DATA_TOTAL_BYTES,
max_external_files: DEFAULT_MAX_EXTERNAL_FILES,
}
}
}
impl OnnxExternalDataLimits {
fn to_internal(self) -> ExternalDataOptions {
ExternalDataOptions {
max_total_bytes: self.max_total_bytes,
max_external_files: self.max_external_files,
}
}
}
#[derive(Clone, Debug)]
pub enum OnnxValue {
F32(Tensor<f32>),
I64(Tensor<i64>),
Bool(Tensor<bool>),
F16(Tensor<f16>),
}
fn onnx_value_to_interp(v: OnnxValue) -> InterpValue {
match v {
OnnxValue::F32(t) => InterpValue::F32(t),
OnnxValue::I64(t) => InterpValue::I64(t),
OnnxValue::Bool(t) => InterpValue::Bool(t),
OnnxValue::F16(t) => InterpValue::F16(t),
}
}
fn interp_value_to_onnx(v: InterpValue) -> OnnxValue {
match v {
InterpValue::F32(t) => OnnxValue::F32(t),
InterpValue::I64(t) => OnnxValue::I64(t),
InterpValue::Bool(t) => OnnxValue::Bool(t),
InterpValue::F16(t) => OnnxValue::F16(t),
}
}
#[non_exhaustive]
#[derive(Debug)]
pub enum OnnxError {
Io(std::io::Error),
Decode { message: String },
UnsupportedDataType { tensor_name: String, data_type: i32 },
SparseInitializerNotSupported { tensor_name: String, count: usize },
UnsupportedOp { op_type: String },
MissingFeed { input: String },
UnknownFeed { name: String },
UnsupportedLayer { index: usize, layer_kind: String },
InvalidModel { message: String },
Execution { message: String },
}
impl fmt::Display for OnnxError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OnnxError::Io(e) => write!(f, "ONNX ファイル I/O 失敗(読み込みまたは書き込み): {e}"),
OnnxError::Decode { message } => write!(f, "ONNX protobuf デコード失敗: {message}"),
OnnxError::UnsupportedDataType {
tensor_name,
data_type,
} => write!(
f,
"未対応の ONNX data_type(tensor={tensor_name}): {data_type}"
),
OnnxError::SparseInitializerNotSupported { tensor_name, count } => write!(
f,
"未対応の ONNX sparse_initializer(tensor={tensor_name}・count={count}): sparse テンソルは非対応"
),
OnnxError::UnsupportedOp { op_type } => write!(f, "未対応の ONNX op_type: {op_type}"),
OnnxError::MissingFeed { input } => {
write!(f, "グラフ入力 '{input}' に対応する feed がありません")
}
OnnxError::UnknownFeed { name } => {
write!(
f,
"feed '{name}' はグラフ入力にも initializer にも属しません"
)
}
OnnxError::UnsupportedLayer { index, layer_kind } => {
write!(f, "未対応の層(index={index}): {layer_kind}")
}
OnnxError::InvalidModel { message } => write!(f, "不正な ONNX モデル: {message}"),
OnnxError::Execution { message } => write!(f, "ONNX 実行時エラー: {message}"),
}
}
}
impl std::error::Error for OnnxError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
OnnxError::Io(e) => Some(e),
_ => None,
}
}
}
fn map_decode_error(e: DecodeModelError) -> OnnxError {
match e {
DecodeModelError::Wire(err) => OnnxError::Decode {
message: err.to_string(),
},
DecodeModelError::SparseInitializerNotSupported { tensor_name, count } => {
OnnxError::SparseInitializerNotSupported { tensor_name, count }
}
}
}
fn map_graph_error(e: GraphError) -> OnnxError {
match e {
GraphError::UnknownDataType {
tensor_name,
data_type,
} => OnnxError::UnsupportedDataType {
tensor_name,
data_type,
},
GraphError::SparseInitializerNotSupported { tensor_name, count } => {
OnnxError::SparseInitializerNotSupported { tensor_name, count }
}
GraphError::ExternalData(ExternalDataError::Io {
kind, raw_os_error, ..
}) => OnnxError::Io(match raw_os_error {
Some(code) => std::io::Error::from_raw_os_error(code),
None => std::io::Error::from(kind),
}),
GraphError::ExternalData(ExternalDataError::AllocationFailed { .. }) => {
OnnxError::Io(std::io::Error::from(std::io::ErrorKind::OutOfMemory))
}
other => OnnxError::InvalidModel {
message: other.to_string(),
},
}
}
fn map_interp_error(e: InterpError) -> OnnxError {
match e {
InterpError::UnsupportedOp(op_type) => OnnxError::UnsupportedOp { op_type },
InterpError::MissingFeed { input } => OnnxError::MissingFeed { input },
InterpError::UnknownFeed { name } => OnnxError::UnknownFeed { name },
InterpError::Graph(graph_err) => map_graph_error(graph_err),
InterpError::AllocationFailed { .. } => {
OnnxError::Io(std::io::Error::from(std::io::ErrorKind::OutOfMemory))
}
other => OnnxError::Execution {
message: other.to_string(),
},
}
}
fn map_export_error(e: ExportError) -> OnnxError {
match e {
ExportError::UnsupportedOp {
op_type, domain, ..
} => {
let op_type = if domain.is_empty() {
op_type
} else {
format!("{domain}::{op_type}")
};
OnnxError::UnsupportedOp { op_type }
}
ExportError::UnsupportedLayer { index, layer_kind } => OnnxError::UnsupportedLayer {
index,
layer_kind: layer_kind.to_string(),
},
ExportError::AllocationFailed { .. } => {
OnnxError::Io(std::io::Error::from(std::io::ErrorKind::OutOfMemory))
}
other => OnnxError::InvalidModel {
message: other.to_string(),
},
}
}
#[cfg(test)]
mod map_graph_error_tests {
use super::{
ExportError, ExternalDataError, GraphError, InterpError, OnnxError, map_export_error,
map_graph_error, map_interp_error,
};
fn assert_out_of_memory(e: OnnxError) {
match e {
OnnxError::Io(io) => assert_eq!(io.kind(), std::io::ErrorKind::OutOfMemory),
other => panic!("OnnxError::Io(OutOfMemory) を期待したが {other:?}"),
}
}
#[test]
fn interp_allocation_failed_maps_to_io_out_of_memory() {
assert_out_of_memory(map_interp_error(InterpError::AllocationFailed {
tensor_name: "w".to_string(),
bytes: 1 << 40,
}));
}
#[test]
fn export_allocation_failed_maps_to_io_out_of_memory() {
assert_out_of_memory(map_export_error(ExportError::AllocationFailed {
tensor_name: "w".to_string(),
bytes: 1 << 40,
}));
}
#[test]
fn too_many_external_files_maps_to_invalid_model() {
let e = map_graph_error(GraphError::ExternalData(
ExternalDataError::TooManyExternalFiles { limit: 1 },
));
match e {
OnnxError::InvalidModel { message } => assert!(message.contains("limit=1")),
other => panic!("OnnxError::InvalidModel を期待したが {other:?}"),
}
}
#[test]
fn allocation_failed_maps_to_io_out_of_memory() {
let e = map_graph_error(GraphError::ExternalData(
ExternalDataError::AllocationFailed {
tensor_name: "w".to_string(),
bytes: 1 << 40,
},
));
match e {
OnnxError::Io(io) => assert_eq!(io.kind(), std::io::ErrorKind::OutOfMemory),
other => panic!("OnnxError::Io(OutOfMemory) を期待したが {other:?}"),
}
}
#[test]
fn external_io_error_keeps_kind() {
let e = map_graph_error(GraphError::ExternalData(ExternalDataError::Io {
tensor_name: "w".to_string(),
kind: std::io::ErrorKind::NotFound,
raw_os_error: None,
}));
match e {
OnnxError::Io(io) => assert_eq!(io.kind(), std::io::ErrorKind::NotFound),
other => panic!("OnnxError::Io(NotFound) を期待したが {other:?}"),
}
}
#[test]
fn external_io_error_keeps_raw_os_error() {
let os_err = std::io::Error::from_raw_os_error(2);
let code = os_err.raw_os_error().expect("OS 由来のエラーのはず");
let e = map_graph_error(GraphError::ExternalData(ExternalDataError::Io {
tensor_name: "w".to_string(),
kind: os_err.kind(),
raw_os_error: Some(code),
}));
match e {
OnnxError::Io(io) => {
assert_eq!(io.raw_os_error(), Some(code));
assert_eq!(io.kind(), os_err.kind());
}
other => panic!("OnnxError::Io を期待したが {other:?}"),
}
}
#[test]
fn other_external_errors_map_to_invalid_model() {
let e = map_graph_error(GraphError::ExternalData(
ExternalDataError::TotalSizeLimitExceeded {
limit: 8,
requested: 16,
},
));
assert!(matches!(e, OnnxError::InvalidModel { .. }), "{e:?}");
}
}