use std::path::Path;
use std::sync::Arc;
use onnx_runtime_ir::{Graph, WeightRef};
use onnx_runtime_shape_inference::{InferenceRegistry, MergePolicy};
use onnx_runtime_tracer::{Args, SpanGuard};
use crate::graph_builder::BuiltGraph;
pub mod encoder;
pub mod epcontext;
pub mod function_inline;
pub(crate) mod graph_builder;
pub mod proto;
pub mod weights;
pub mod writer;
mod pathsafe;
pub use encoder::{
DEFAULT_IR_VERSION, DEFAULT_OPSET_VERSION, Model, ModelMetadata, encode_model,
encode_model_proto, write_model,
};
pub use epcontext::{
EmbedMode, EpContextBlob, EpContextNode, ep_context_node_ids, ep_context_nodes,
is_ep_context_op, resolve_ep_context,
};
pub use error::LoaderError;
pub use weights::{
ExpertQuantization, ExpertStorageOrder, ExpertTensorLayout, ExpertWeightRegion,
NonPageableReason, Pageability, WeightRegionCatalog, WeightStore, qmoe_expert_tensor_layout,
};
pub use writer::{EpContextDumpConfig, EpContextPartition, dump_ep_context};
fn trace_span(name: &'static str, cat: &'static str) -> Option<SpanGuard> {
onnx_runtime_tracer::global_context()
.filter(|trace| trace.is_enabled())
.map(|trace| trace.span(name, cat))
}
mod error;
pub fn load_model(path: impl AsRef<Path>) -> Result<Graph, LoaderError> {
Ok(load_model_with_weights(path)?.0)
}
pub fn load_model_bytes(bytes: &[u8]) -> Result<Graph, LoaderError> {
Ok(load_model_bytes_with_weights(bytes, Path::new("."))?.0)
}
pub fn load_model_with_weights(
path: impl AsRef<Path>,
) -> Result<(Graph, Arc<WeightStore>), LoaderError> {
let path = path.as_ref();
let bytes = read_model_binary(path)?;
let model_dir = path.parent().unwrap_or_else(|| Path::new("."));
build_from_bytes_with_weights(&bytes, model_dir, None)
}
pub fn read_model_binary(path: impl AsRef<Path>) -> Result<Vec<u8>, LoaderError> {
let path = path.as_ref();
let mut span = trace_span("load.read_model_binary", "load");
let raw = std::fs::read(path).map_err(|source| LoaderError::Io {
path: path.to_path_buf(),
source,
})?;
let raw_len = raw.len();
if is_textproto_path(path) {
let text = String::from_utf8(raw)
.map_err(|e| LoaderError::TextProtoParse(format!("model is not valid UTF-8: {e}")))?;
let binary = proto::textproto_to_binary(&text)?;
if let Some(span) = span.as_mut() {
span.set_args(
Args::new()
.bytes(binary.len() as u64)
.with("raw_bytes", raw_len as u64)
.with("textproto", true)
.with("path", path.display().to_string()),
);
}
Ok(binary)
} else {
if let Some(span) = span.as_mut() {
span.set_args(
Args::new()
.bytes(raw_len as u64)
.with("textproto", false)
.with("path", path.display().to_string()),
);
}
Ok(raw)
}
}
pub fn is_textproto_path(path: impl AsRef<Path>) -> bool {
path.as_ref()
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("textproto"))
}
pub fn load_model_bytes_with_weights(
bytes: &[u8],
base_dir: impl AsRef<Path>,
) -> Result<(Graph, Arc<WeightStore>), LoaderError> {
build_from_bytes_with_weights(bytes, base_dir.as_ref(), None)
}
pub fn load_model_bytes_with_weights_filtered(
bytes: &[u8],
base_dir: impl AsRef<Path>,
keep_as_op: &function_inline::KeepAsOp<'_>,
) -> Result<(Graph, Arc<WeightStore>), LoaderError> {
build_from_bytes_with_weights(bytes, base_dir.as_ref(), Some(keep_as_op))
}
fn build_from_bytes_with_weights(
bytes: &[u8],
model_dir: &Path,
keep_as_op: Option<&function_inline::KeepAsOp<'_>>,
) -> Result<(Graph, Arc<WeightStore>), LoaderError> {
let model = parse_model(bytes)?;
validate_proto(&model)?;
let BuiltGraph {
mut graph,
name_map,
} = build_graph(&model, keep_as_op)?;
validate_opset_imports(&graph)?;
let store = load_weights(&model, model_dir, &name_map)?;
attach_weights(&mut graph, &store);
validate_ir(&graph)?;
validate_loaded_model(&graph)?;
infer_shapes(&mut graph)?;
Ok((graph, Arc::new(store)))
}
fn parse_model(bytes: &[u8]) -> Result<proto::onnx::ModelProto, LoaderError> {
let mut span = trace_span("load.parse_model", "load");
let model = proto::decode_model(bytes)?;
if let Some(span) = span.as_mut() {
span.set_args(Args::new().bytes(bytes.len() as u64));
}
Ok(model)
}
fn validate_proto(model: &proto::onnx::ModelProto) -> Result<(), LoaderError> {
let mut span = trace_span("load.validate_model_proto", "load");
validate_model_proto(model)?;
if let Some(span) = span.as_mut() {
span.set_args(
Args::new()
.with("graph_count", if model.graph.is_some() { 1_u64 } else { 0 })
.with("metadata_props", model.metadata_props.len() as u64),
);
}
Ok(())
}
fn build_graph(
model: &proto::onnx::ModelProto,
keep_as_op: Option<&function_inline::KeepAsOp<'_>>,
) -> Result<BuiltGraph, LoaderError> {
let mut span = trace_span("load.build_graph", "load");
let built = graph_builder::build_graph(model, keep_as_op)?;
if let Some(span) = span.as_mut() {
span.set_args(
Args::new()
.with("nodes", built.graph.num_nodes() as u64)
.with("values", built.graph.values.len() as u64)
.with("inputs", built.graph.inputs.len() as u64)
.with("outputs", built.graph.outputs.len() as u64)
.with("initializers", built.graph.initializers.len() as u64),
);
}
Ok(built)
}
fn load_weights(
model: &proto::onnx::ModelProto,
model_dir: &Path,
name_map: &std::collections::HashMap<String, onnx_runtime_ir::ValueId>,
) -> Result<WeightStore, LoaderError> {
let mut span = trace_span("load.external_weights", "load");
let store = weights::load_weights(model, model_dir, name_map)?;
if let Some(span) = span.as_mut() {
let mut inline_count = 0_u64;
let mut inline_bytes = 0_u64;
let mut external_count = 0_u64;
let mut external_bytes = 0_u64;
for weight in store.weights.values() {
match weight {
WeightRef::Inline(tensor) => {
inline_count += 1;
inline_bytes += tensor.data.len() as u64;
}
WeightRef::External { length, .. } => {
external_count += 1;
external_bytes += *length as u64;
}
}
}
span.set_args(
Args::new()
.with("initializers", store.weights.len() as u64)
.with("inline_initializers", inline_count)
.with("inline_bytes", inline_bytes)
.with("external_initializers", external_count)
.with("external_bytes", external_bytes)
.with("model_dir", model_dir.display().to_string()),
);
}
Ok(store)
}
fn attach_weights(graph: &mut Graph, store: &WeightStore) {
for (&value_id, weight) in &store.weights {
graph.set_initializer(value_id, weight.clone());
}
}
fn validate_ir(graph: &Graph) -> Result<(), LoaderError> {
let mut span = trace_span("load.validate_graph", "load");
graph
.validate()
.map_err(|errors| LoaderError::GraphBuild(format!("{errors:?}")))?;
if let Some(span) = span.as_mut() {
span.set_args(
Args::new()
.with("nodes", graph.num_nodes() as u64)
.with("values", graph.values.len() as u64)
.with("initializers", graph.initializers.len() as u64),
);
}
Ok(())
}
fn validate_loaded_model(graph: &Graph) -> Result<(), LoaderError> {
let mut span = trace_span("load.validate_model", "load");
validate_model(graph)?;
if let Some(span) = span.as_mut() {
span.set_args(Args::new().with("nodes", graph.num_nodes() as u64));
}
Ok(())
}
fn infer_shapes(graph: &mut Graph) -> Result<(), LoaderError> {
let registry = InferenceRegistry::default_registry();
let opset_imports = graph.opset_imports.clone();
let mut span = trace_span("load.shape_inference", "load");
registry.infer_graph(graph, &opset_imports, MergePolicy::Permissive)?;
if let Some(span) = span.as_mut() {
span.set_args(
Args::new()
.with("nodes", graph.num_nodes() as u64)
.with("values", graph.values.len() as u64)
.with("opset_domains", graph.opset_imports.len() as u64),
);
}
Ok(())
}
pub fn validate_model(graph: &Graph) -> Result<(), LoaderError> {
validate_opset_imports(graph)?;
validate_einsum_nodes(graph)?;
validate_no_control_flow(graph)?;
validate_no_dangling_refs(graph)?;
validate_no_initializer_producer(graph)?;
Ok(())
}
pub fn validate_einsum_nodes(graph: &Graph) -> Result<(), LoaderError> {
use onnx_runtime_ir::{Attribute, EinsumInput, EinsumPlan, EinsumSchema, EinsumShapePlan};
fn check_graph(
graph: &Graph,
imports: &std::collections::HashMap<String, u64>,
) -> Result<(), LoaderError> {
for (_, node) in graph.nodes.iter() {
if !node.is_default_domain() || node.op_type != "Einsum" {
continue;
}
let imported_opset = node
.local_opset()
.or_else(|| imports.get("").copied())
.unwrap_or(1);
let equation = match node.attr("equation") {
Some(Attribute::String(bytes)) => {
std::str::from_utf8(bytes).map_err(|error| LoaderError::InvalidEinsum {
node: node_label(node),
detail: format!(
"attribute `equation` is not valid UTF-8 at byte offset {}",
error.valid_up_to()
),
})?
}
_ => {
return Err(LoaderError::InvalidEinsum {
node: node_label(node),
detail: "missing required STRING attribute `equation`".to_string(),
});
}
};
if node.outputs.len() != 1 {
return Err(LoaderError::InvalidEinsum {
node: node_label(node),
detail: format!(
"equation `{equation}` requires exactly 1 output, but the node declares {} outputs",
node.outputs.len()
),
});
}
let output_id = node.outputs[0];
let output = graph.values.get(output_id).ok_or_else(|| LoaderError::InvalidEinsum {
node: node_label(node),
detail: format!(
"equation `{equation}` declares 1 output, but output #0 references missing value {output_id:?}"
),
})?;
if output.name.as_deref().is_none_or(str::is_empty) {
return Err(LoaderError::InvalidEinsum {
node: node_label(node),
detail: format!(
"equation `{equation}` declares 1 output, but required output #0 has an empty or omitted name"
),
});
}
let mut metadata = Vec::with_capacity(node.inputs.len());
for (input, slot) in node.inputs.iter().enumerate() {
let value_id = slot.ok_or_else(|| LoaderError::InvalidEinsum {
node: node_label(node),
detail: format!("input #{input} is omitted from a variadic required operand"),
})?;
let value =
graph
.values
.get(value_id)
.ok_or_else(|| LoaderError::InvalidEinsum {
node: node_label(node),
detail: format!("input #{input} references missing value {value_id:?}"),
})?;
metadata.push(EinsumInput::from_optional(
graph.value_type_is_known(value_id).then_some(value.dtype),
graph
.value_shape_is_known(value_id)
.then_some(value.shape.as_slice()),
));
}
let plan = match EinsumPlan::build_for_opset(equation, &metadata, imported_opset) {
Ok(plan) => Some(plan),
Err(error) if error.is_incomplete_metadata() => {
if let Some(shapes) = metadata
.iter()
.map(|input| input.shape())
.collect::<Option<Vec<_>>>()
{
EinsumShapePlan::build_for_opset(equation, &shapes, imported_opset)
.map_err(|error| LoaderError::InvalidEinsum {
node: node_label(node),
detail: error.to_string(),
})?;
}
None
}
Err(error) => {
return Err(LoaderError::InvalidEinsum {
node: node_label(node),
detail: error.to_string(),
});
}
};
if graph.value_type_is_known(output_id) {
let schema = EinsumSchema::resolve(imported_opset).map_err(|error| {
LoaderError::InvalidEinsum {
node: node_label(node),
detail: error.to_string(),
}
})?;
if !schema.supports_dtype(output.dtype) {
return Err(LoaderError::InvalidEinsum {
node: node_label(node),
detail: format!(
"output dtype {:?} is not admitted by {schema}; cast the output and \
every operand to one homogeneous schema-supported dtype",
output.dtype
),
});
}
let input_dtype = plan
.as_ref()
.map(EinsumPlan::dtype)
.or_else(|| metadata.iter().find_map(|input| input.dtype()));
if let Some(input_dtype) = input_dtype
&& output.dtype != input_dtype
{
return Err(LoaderError::InvalidEinsum {
node: node_label(node),
detail: format!(
"output dtype {:?} does not match known homogeneous input dtype {:?}",
output.dtype, input_dtype
),
});
}
}
}
for subgraph in graph.subgraphs.values() {
check_graph(subgraph, imports)?;
}
Ok(())
}
check_graph(graph, &graph.opset_imports)
}
pub fn validate_model_proto(model: &proto::onnx::ModelProto) -> Result<(), LoaderError> {
use std::collections::HashSet;
use proto::onnx::GraphProto;
if model.ir_version < 1 {
return Err(LoaderError::InvalidIrVersion {
ir_version: model.ir_version,
});
}
if model.ir_version >= 3 && model.opset_import.is_empty() {
return Err(LoaderError::MissingModelOpsetImport {
ir_version: model.ir_version,
});
}
fn node_description(node: &proto::onnx::NodeProto, index: usize) -> String {
if node.name.is_empty() {
format!("<unnamed node #{index}>")
} else {
format!("{:?}", node.name)
}
}
fn check_graph(graph: &GraphProto) -> Result<(), LoaderError> {
let mut producers = std::collections::HashMap::new();
for input in &graph.input {
if !input.name.is_empty() {
producers.insert(input.name.clone(), "graph input".to_string());
}
}
for (index, node) in graph.node.iter().enumerate() {
let node_description = node_description(node, index);
for output in &node.output {
if output.is_empty() {
continue;
}
let producer = format!("output of {node_description}");
if let Some(first) = producers.insert(output.clone(), producer.clone()) {
return Err(LoaderError::DuplicateValueProducer {
tensor: output.clone(),
first,
second: producer,
});
}
}
for attribute in &node.attribute {
if !attribute.ref_attr_name.is_empty() {
return Err(LoaderError::RefAttributeOutsideFunction {
op_type: node.op_type.clone(),
node: node_description.clone(),
domain: display_domain(&node.domain),
attr: attribute.name.clone(),
ref_attr_name: attribute.ref_attr_name.clone(),
});
}
}
}
let sources: HashSet<&str> = graph
.input
.iter()
.map(|input| input.name.as_str())
.chain(
graph
.initializer
.iter()
.map(|initializer| initializer.name.as_str()),
)
.chain(
graph
.node
.iter()
.flat_map(|node| node.output.iter().map(String::as_str)),
)
.collect();
for output in &graph.output {
if !output.name.is_empty() && !sources.contains(output.name.as_str()) {
return Err(LoaderError::GraphOutputMissingProducer {
tensor: output.name.clone(),
});
}
}
let outer_initializers: HashSet<&str> = graph
.initializer
.iter()
.map(|initializer| initializer.name.as_str())
.collect();
for node in &graph.node {
for attribute in &node.attribute {
let subgraphs = attribute.g.iter().chain(attribute.graphs.iter());
for subgraph in subgraphs {
if let Some(input) = subgraph
.input
.iter()
.find(|input| outer_initializers.contains(input.name.as_str()))
{
return Err(LoaderError::SubgraphInputShadowsInitializer {
tensor: input.name.clone(),
});
}
check_graph(subgraph)?;
}
}
}
Ok(())
}
if let Some(graph) = &model.graph {
check_graph(graph)?;
}
Ok(())
}
fn node_label(node: &onnx_runtime_ir::Node) -> String {
if node.name.is_empty() {
format!("<unnamed node #{}>", node.id.0)
} else {
format!("{:?}", node.name)
}
}
fn display_domain(domain: &str) -> String {
if domain.is_empty() {
"ai.onnx".to_string()
} else {
domain.to_string()
}
}
pub fn validate_no_control_flow(graph: &Graph) -> Result<(), LoaderError> {
use onnx_runtime_ir::Attribute;
fn is_implemented_control_flow(node: &onnx_runtime_ir::Node) -> bool {
node.is_default_domain() && matches!(node.op_type.as_str(), "If" | "Loop" | "Scan")
}
fn check_graph(graph: &Graph) -> Result<(), LoaderError> {
for (_, node) in graph.nodes.iter() {
let mut subgraph_attrs: Vec<&String> = node
.attributes
.iter()
.filter(|(_, v)| matches!(v, Attribute::Graph(_) | Attribute::Graphs(_)))
.map(|(k, _)| k)
.collect();
subgraph_attrs.sort();
if let Some(attr) = subgraph_attrs.first() {
if !is_implemented_control_flow(node) {
return Err(LoaderError::UnsupportedControlFlow {
op_type: node.op_type.clone(),
node: node_label(node),
domain: display_domain(&node.domain),
attr: (*attr).clone(),
});
}
}
}
for subgraph in graph.subgraphs.values() {
check_graph(subgraph)?;
}
Ok(())
}
check_graph(graph)
}
pub fn validate_no_dangling_refs(graph: &Graph) -> Result<(), LoaderError> {
use std::collections::HashSet;
let graph_inputs: HashSet<_> = graph.inputs.iter().copied().collect();
for (_, node) in graph.nodes.iter() {
for vid in node.input_values() {
let Some(value) = graph.values.get(vid) else {
continue;
};
let is_sourced = value.producer.is_some()
|| graph_inputs.contains(&vid)
|| graph.initializers.contains_key(&vid);
if !is_sourced {
let tensor = value
.name
.clone()
.unwrap_or_else(|| format!("<anonymous value #{}>", vid.0));
return Err(LoaderError::DanglingTensorRef {
op_type: node.op_type.clone(),
node: node_label(node),
domain: display_domain(&node.domain),
tensor,
});
}
}
}
Ok(())
}
pub fn validate_no_initializer_producer(graph: &Graph) -> Result<(), LoaderError> {
for &vid in graph.initializers.keys() {
let Some(value) = graph.values.get(vid) else {
continue;
};
if let Some(producer) = value.producer {
let tensor = value
.name
.clone()
.unwrap_or_else(|| format!("<anonymous value #{}>", vid.0));
let node = if graph.nodes.contains(producer) {
node_label(graph.node(producer))
} else {
format!("<node #{}>", producer.0)
};
return Err(LoaderError::InitializerHasProducer { tensor, node });
}
}
Ok(())
}
pub fn validate_opset_imports(graph: &Graph) -> Result<(), LoaderError> {
fn has_import(imports: &std::collections::HashMap<String, u64>, domain: &str) -> bool {
imports.contains_key(domain)
}
fn validate_graph(
graph: &Graph,
imports: &std::collections::HashMap<String, u64>,
) -> Result<(), LoaderError> {
for (_, node) in graph.nodes.iter() {
if !has_import(imports, &node.domain) {
let domain = if node.domain.is_empty() {
"ai.onnx".to_string()
} else {
node.domain.clone()
};
let node_name = if node.name.is_empty() {
format!("<unnamed node #{}>", node.id.0)
} else {
format!("{:?}", node.name)
};
return Err(LoaderError::MissingOpsetImport {
op_type: node.op_type.clone(),
node: node_name,
domain,
});
}
}
for subgraph in graph.subgraphs.values() {
validate_graph(subgraph, imports)?;
}
Ok(())
}
validate_graph(graph, &graph.opset_imports)
}