1use std::path::Path;
41use std::sync::Arc;
42
43use onnx_runtime_ir::{Graph, WeightRef};
44use onnx_runtime_shape_inference::{InferenceRegistry, MergePolicy};
45use onnx_runtime_tracer::{Args, SpanGuard};
46
47use crate::graph_builder::BuiltGraph;
48
49pub mod encoder;
50pub mod epcontext;
51pub mod function_inline;
52pub(crate) mod graph_builder;
53pub mod proto;
54pub mod weights;
55pub mod writer;
56
57mod pathsafe;
58
59pub use encoder::{
60 DEFAULT_IR_VERSION, Model, ModelMetadata, encode_model, encode_model_proto, write_model,
61};
62pub use epcontext::{
63 EmbedMode, EpContextBlob, EpContextNode, ep_context_node_ids, ep_context_nodes,
64 is_ep_context_op, resolve_ep_context,
65};
66pub use error::LoaderError;
67pub use weights::{
68 ExpertQuantization, ExpertStorageOrder, ExpertTensorLayout, ExpertWeightRegion,
69 NonPageableReason, Pageability, WeightRegionCatalog, WeightStore,
70};
71pub use writer::{EpContextDumpConfig, EpContextPartition, dump_ep_context};
72
73fn trace_span(name: &'static str, cat: &'static str) -> Option<SpanGuard> {
74 onnx_runtime_tracer::global_context()
75 .filter(|trace| trace.is_enabled())
76 .map(|trace| trace.span(name, cat))
77}
78
79mod error {
80 use std::path::PathBuf;
81
82 #[derive(Debug, thiserror::Error)]
84 pub enum LoaderError {
85 #[error("failed to read model file {path}: {source}")]
86 Io {
87 path: PathBuf,
88 #[source]
89 source: std::io::Error,
90 },
91
92 #[error("failed to parse ONNX protobuf: {0}")]
93 ProtobufParse(String),
94
95 #[error("failed to parse ONNX protobuf TextFormat: {0}")]
96 TextProtoParse(String),
97
98 #[error("unsupported opset: domain={domain}, version={version}")]
99 UnsupportedOpset { domain: String, version: u64 },
100
101 #[error(
102 "illegal ONNX model: operator {domain}::{op_type} at node {node} uses domain \
103 '{domain}' but no corresponding opset_import is declared. RULES #1: the model must \
104 declare an opset_import for domain '{domain}'; if you built this graph \
105 programmatically, add it before loading; if this is a file, the model is \
106 malformed/invalid per the ONNX spec"
107 )]
108 MissingOpsetImport {
109 op_type: String,
110 node: String,
111 domain: String,
112 },
113
114 #[error(
115 "unsupported ONNX model: operator {domain}::{op_type} at node {node} carries a \
116 subgraph attribute '{attr}' (control-flow / nested-graph op) that this runtime cannot \
117 execute. RULES #1: ep-cpu recursively executes the standard control-flow ops \
118 If/Loop/Scan (ai.onnx), but not {op_type}, so the model cannot be run as-is. \
119 Expected: express control flow with If/Loop/Scan, lower/unroll {op_type} into \
120 supported ops, or register a kernel able to execute its subgraph body"
121 )]
122 UnsupportedControlFlow {
123 op_type: String,
124 node: String,
125 domain: String,
126 attr: String,
127 },
128
129 #[error(
130 "illegal ONNX model: operator {domain}::{op_type} at node {node} consumes tensor \
131 '{tensor}', but no producer exists — it is not a graph input, not an initializer, and \
132 not produced by any upstream node. RULES #1: every consumed tensor must be sourced; \
133 the graph is structurally malformed. Expected: add '{tensor}' as a graph input or \
134 initializer, or add a node that produces it; if this is a file, the model is invalid \
135 per the ONNX spec"
136 )]
137 DanglingTensorRef {
138 op_type: String,
139 node: String,
140 domain: String,
141 tensor: String,
142 },
143
144 #[error(
145 "illegal ONNX model: tensor '{tensor}' is declared as an initializer but is also \
146 produced as an output of node {node} — an initializer must be a constant source with \
147 no producer. RULES #1: initializer names must be unique and must not collide with any \
148 node output name; a producer-backed initializer would let a kernel write through \
149 read-only weight storage. Expected: rename the node output or the initializer so they \
150 no longer share a name; if this is a file, the model is malformed per the ONNX spec"
151 )]
152 InitializerHasProducer { tensor: String, node: String },
153
154 #[error(
155 "illegal ONNX model: value '{tensor}' has multiple producers ({first} and {second}). \
156 RULES #1: ONNX graphs are in SSA form, so a value name may be assigned only once. \
157 Expected: give each graph input and node output a unique name"
158 )]
159 DuplicateValueProducer {
160 tensor: String,
161 first: String,
162 second: String,
163 },
164
165 #[error(
166 "illegal ONNX model: operator {domain}::{op_type} at node {node} has attribute \
167 '{attr}' referring to function attribute '{ref_attr_name}' outside a FunctionProto. \
168 RULES #1: ref_attr_name is only bound while inlining a FunctionProto; it has no \
169 executable value in a main graph or control-flow subgraph. Expected: replace it with \
170 a concrete attribute value or move the node into a FunctionProto"
171 )]
172 RefAttributeOutsideFunction {
173 op_type: String,
174 node: String,
175 domain: String,
176 attr: String,
177 ref_attr_name: String,
178 },
179
180 #[error(
181 "illegal ONNX model: ir_version {ir_version} is invalid. RULES #1: ir_version is \
182 required and ONNX IR versions start at 1. Expected: emit a model with ir_version >= 1"
183 )]
184 InvalidIrVersion { ir_version: i64 },
185
186 #[error(
187 "illegal ONNX model: ir_version {ir_version} requires at least one opset_import \
188 (ONNX IR>=3). Expected: add an opset_import for every operator domain used by the \
189 model"
190 )]
191 MissingModelOpsetImport { ir_version: i64 },
192
193 #[error(
194 "illegal ONNX model: initializer '{tensor}' in an outer graph is shadowed by a \
195 subgraph input of the same name. RULES #1: this runtime does not permit ambiguous \
196 initializer/subgraph binding. Expected: rename the subgraph formal input or the \
197 outer initializer"
198 )]
199 SubgraphInputShadowsInitializer { tensor: String },
200
201 #[error(
202 "illegal ONNX model: graph output '{tensor}' has no producer in its graph. RULES #1: \
203 every output must be a graph input, initializer, or node output in the same scope. \
204 Expected: produce '{tensor}' locally or declare it as an input/initializer"
205 )]
206 GraphOutputMissingProducer { tensor: String },
207
208 #[error("external data file not found: {path}")]
209 ExternalDataNotFound { path: PathBuf },
210
211 #[error("external data path rejected ({reason}): {path}")]
212 ExternalDataPath { path: String, reason: &'static str },
213
214 #[error("weight mmap failed: {0}")]
215 Mmap(String),
216
217 #[error("EPContext node error: {0}")]
218 EpContext(String),
219
220 #[error("EPContext external path rejected ({reason}): {path}")]
221 EpContextPath { path: String, reason: &'static str },
222
223 #[error("graph construction failed: {0}")]
224 GraphBuild(String),
225
226 #[error(
227 "illegal ONNX model: model-local function {function} is recursive (call chain: \
228 {chain}). RULES #1: ONNX function bodies may reference other model-local functions \
229 but MUST NOT be recursive — inlining cannot terminate. Expected: break the cycle so \
230 no function transitively calls itself"
231 )]
232 RecursiveFunction { function: String, chain: String },
233
234 #[error(
235 "illegal ONNX model: call to model-local function {function} at node {node} passes \
236 {actual} {kind}(s) but the function declares only {formal}. RULES #1: a function \
237 call may omit trailing optional {kind}s but must not supply more than are declared. \
238 Expected: remove the extra {kind}(s) or fix the function signature"
239 )]
240 FunctionArityMismatch {
241 function: String,
242 node: String,
243 kind: &'static str,
244 formal: usize,
245 actual: usize,
246 },
247
248 #[error(
249 "illegal ONNX model: call to model-local function {function} at node {node} is missing \
250 required attribute '{attribute}', and the function declares no default for it. \
251 RULES #1: an attribute listed in FunctionProto.attribute has no default and must be \
252 supplied at every call site. Expected: set '{attribute}' on the call node, or give \
253 the function a default via attribute_proto"
254 )]
255 MissingRequiredFunctionAttribute {
256 function: String,
257 node: String,
258 attribute: String,
259 },
260
261 #[error("unsupported ONNX data_type {raw} at {context}")]
262 UnsupportedDataType { raw: i32, context: String },
263
264 #[error("shape inference failed: {0}")]
265 ShapeInference(#[from] onnx_runtime_shape_inference::ShapeInferError),
266
267 #[error(transparent)]
268 Ir(#[from] onnx_runtime_ir::IrError),
269 }
270}
271
272pub fn load_model(path: impl AsRef<Path>) -> Result<Graph, LoaderError> {
287 Ok(load_model_with_weights(path)?.0)
288}
289
290pub fn load_model_bytes(bytes: &[u8]) -> Result<Graph, LoaderError> {
301 Ok(load_model_bytes_with_weights(bytes, Path::new("."))?.0)
302}
303
304pub fn load_model_with_weights(
316 path: impl AsRef<Path>,
317) -> Result<(Graph, Arc<WeightStore>), LoaderError> {
318 let path = path.as_ref();
319 let bytes = read_model_binary(path)?;
320 let model_dir = path.parent().unwrap_or_else(|| Path::new("."));
321 build_from_bytes_with_weights(&bytes, model_dir)
322}
323
324pub fn read_model_binary(path: impl AsRef<Path>) -> Result<Vec<u8>, LoaderError> {
336 let path = path.as_ref();
337 let mut span = trace_span("load.read_model_binary", "load");
338 let raw = std::fs::read(path).map_err(|source| LoaderError::Io {
339 path: path.to_path_buf(),
340 source,
341 })?;
342 let raw_len = raw.len();
343 if is_textproto_path(path) {
344 let text = String::from_utf8(raw)
345 .map_err(|e| LoaderError::TextProtoParse(format!("model is not valid UTF-8: {e}")))?;
346 let binary = proto::textproto_to_binary(&text)?;
347 if let Some(span) = span.as_mut() {
348 span.set_args(
349 Args::new()
350 .bytes(binary.len() as u64)
351 .with("raw_bytes", raw_len as u64)
352 .with("textproto", true)
353 .with("path", path.display().to_string()),
354 );
355 }
356 Ok(binary)
357 } else {
358 if let Some(span) = span.as_mut() {
359 span.set_args(
360 Args::new()
361 .bytes(raw_len as u64)
362 .with("textproto", false)
363 .with("path", path.display().to_string()),
364 );
365 }
366 Ok(raw)
367 }
368}
369
370pub fn is_textproto_path(path: impl AsRef<Path>) -> bool {
372 path.as_ref()
373 .extension()
374 .is_some_and(|ext| ext.eq_ignore_ascii_case("textproto"))
375}
376
377pub fn load_model_bytes_with_weights(
384 bytes: &[u8],
385 base_dir: impl AsRef<Path>,
386) -> Result<(Graph, Arc<WeightStore>), LoaderError> {
387 build_from_bytes_with_weights(bytes, base_dir.as_ref())
388}
389
390fn build_from_bytes_with_weights(
391 bytes: &[u8],
392 model_dir: &Path,
393) -> Result<(Graph, Arc<WeightStore>), LoaderError> {
394 let model = {
395 let mut span = trace_span("load.parse_model", "load");
396 let model = proto::decode_model(bytes)?;
397 if let Some(span) = span.as_mut() {
398 span.set_args(Args::new().bytes(bytes.len() as u64));
399 }
400 model
401 };
402 {
403 let mut span = trace_span("load.validate_model_proto", "load");
404 validate_model_proto(&model)?;
405 if let Some(span) = span.as_mut() {
406 span.set_args(
407 Args::new()
408 .with("graph_count", if model.graph.is_some() { 1_u64 } else { 0 })
409 .with("metadata_props", model.metadata_props.len() as u64),
410 );
411 }
412 }
413 let BuiltGraph {
414 mut graph,
415 name_map,
416 } = {
417 let mut span = trace_span("load.build_graph", "load");
418 let built = graph_builder::build_graph(&model)?;
419 if let Some(span) = span.as_mut() {
420 span.set_args(
421 Args::new()
422 .with("nodes", built.graph.num_nodes() as u64)
423 .with("values", built.graph.values.len() as u64)
424 .with("inputs", built.graph.inputs.len() as u64)
425 .with("outputs", built.graph.outputs.len() as u64)
426 .with("initializers", built.graph.initializers.len() as u64),
427 );
428 }
429 built
430 };
431
432 validate_opset_imports(&graph)?;
435
436 let store = {
437 let mut span = trace_span("load.external_weights", "load");
438 let store = weights::load_weights(&model, model_dir, &name_map)?;
439 if let Some(span) = span.as_mut() {
440 let mut inline_count = 0_u64;
441 let mut inline_bytes = 0_u64;
442 let mut external_count = 0_u64;
443 let mut external_bytes = 0_u64;
444 for weight in store.weights.values() {
445 match weight {
446 WeightRef::Inline(tensor) => {
447 inline_count += 1;
448 inline_bytes += tensor.data.len() as u64;
449 }
450 WeightRef::External { length, .. } => {
451 external_count += 1;
452 external_bytes += *length as u64;
453 }
454 }
455 }
456 span.set_args(
457 Args::new()
458 .with("initializers", store.weights.len() as u64)
459 .with("inline_initializers", inline_count)
460 .with("inline_bytes", inline_bytes)
461 .with("external_initializers", external_count)
462 .with("external_bytes", external_bytes)
463 .with("model_dir", model_dir.display().to_string()),
464 );
465 }
466 store
467 };
468 for (&vid, weight) in &store.weights {
470 graph.set_initializer(vid, weight.clone());
471 }
472
473 {
481 let mut span = trace_span("load.validate_graph", "load");
482 graph
483 .validate()
484 .map_err(|errs| LoaderError::GraphBuild(format!("{errs:?}")))?;
485 if let Some(span) = span.as_mut() {
486 span.set_args(
487 Args::new()
488 .with("nodes", graph.num_nodes() as u64)
489 .with("values", graph.values.len() as u64)
490 .with("initializers", graph.initializers.len() as u64),
491 );
492 }
493 }
494
495 {
500 let mut span = trace_span("load.validate_model", "load");
501 validate_model(&graph)?;
502 if let Some(span) = span.as_mut() {
503 span.set_args(Args::new().with("nodes", graph.num_nodes() as u64));
504 }
505 }
506
507 let registry = InferenceRegistry::default_registry();
516 let opset_imports = graph.opset_imports.clone();
517 {
518 let mut span = trace_span("load.shape_inference", "load");
519 registry.infer_graph(&mut graph, &opset_imports, MergePolicy::Permissive)?;
520 if let Some(span) = span.as_mut() {
521 span.set_args(
522 Args::new()
523 .with("nodes", graph.num_nodes() as u64)
524 .with("values", graph.values.len() as u64)
525 .with("opset_domains", graph.opset_imports.len() as u64),
526 );
527 }
528 }
529
530 Ok((graph, Arc::new(store)))
531}
532
533pub fn validate_model(graph: &Graph) -> Result<(), LoaderError> {
566 validate_opset_imports(graph)?;
567 validate_no_control_flow(graph)?;
568 validate_no_dangling_refs(graph)?;
569 validate_no_initializer_producer(graph)?;
570 Ok(())
571}
572
573pub fn validate_model_proto(model: &proto::onnx::ModelProto) -> Result<(), LoaderError> {
580 use std::collections::HashSet;
581
582 use proto::onnx::GraphProto;
583
584 if model.ir_version < 1 {
587 return Err(LoaderError::InvalidIrVersion {
588 ir_version: model.ir_version,
589 });
590 }
591 if model.ir_version >= 3 && model.opset_import.is_empty() {
598 return Err(LoaderError::MissingModelOpsetImport {
599 ir_version: model.ir_version,
600 });
601 }
602
603 fn node_description(node: &proto::onnx::NodeProto, index: usize) -> String {
604 if node.name.is_empty() {
605 format!("<unnamed node #{index}>")
606 } else {
607 format!("{:?}", node.name)
608 }
609 }
610
611 fn check_graph(graph: &GraphProto) -> Result<(), LoaderError> {
612 let mut producers = std::collections::HashMap::new();
613 for input in &graph.input {
614 if !input.name.is_empty() {
615 producers.insert(input.name.clone(), "graph input".to_string());
616 }
617 }
618 for (index, node) in graph.node.iter().enumerate() {
619 let node_description = node_description(node, index);
620 for output in &node.output {
621 if output.is_empty() {
622 continue;
623 }
624 let producer = format!("output of {node_description}");
625 if let Some(first) = producers.insert(output.clone(), producer.clone()) {
626 return Err(LoaderError::DuplicateValueProducer {
627 tensor: output.clone(),
628 first,
629 second: producer,
630 });
631 }
632 }
633 for attribute in &node.attribute {
634 if !attribute.ref_attr_name.is_empty() {
635 return Err(LoaderError::RefAttributeOutsideFunction {
636 op_type: node.op_type.clone(),
637 node: node_description.clone(),
638 domain: display_domain(&node.domain),
639 attr: attribute.name.clone(),
640 ref_attr_name: attribute.ref_attr_name.clone(),
641 });
642 }
643 }
644 }
645
646 let sources: HashSet<&str> = graph
647 .input
648 .iter()
649 .map(|input| input.name.as_str())
650 .chain(
651 graph
652 .initializer
653 .iter()
654 .map(|initializer| initializer.name.as_str()),
655 )
656 .chain(
657 graph
658 .node
659 .iter()
660 .flat_map(|node| node.output.iter().map(String::as_str)),
661 )
662 .collect();
663 for output in &graph.output {
664 if !output.name.is_empty() && !sources.contains(output.name.as_str()) {
665 return Err(LoaderError::GraphOutputMissingProducer {
666 tensor: output.name.clone(),
667 });
668 }
669 }
670
671 let outer_initializers: HashSet<&str> = graph
672 .initializer
673 .iter()
674 .map(|initializer| initializer.name.as_str())
675 .collect();
676 for node in &graph.node {
677 for attribute in &node.attribute {
678 let subgraphs = attribute.g.iter().chain(attribute.graphs.iter());
679 for subgraph in subgraphs {
680 if let Some(input) = subgraph
681 .input
682 .iter()
683 .find(|input| outer_initializers.contains(input.name.as_str()))
684 {
685 return Err(LoaderError::SubgraphInputShadowsInitializer {
686 tensor: input.name.clone(),
687 });
688 }
689 check_graph(subgraph)?;
690 }
691 }
692 }
693 Ok(())
694 }
695
696 if let Some(graph) = &model.graph {
697 check_graph(graph)?;
698 }
699 Ok(())
700}
701
702fn node_label(node: &onnx_runtime_ir::Node) -> String {
705 if node.name.is_empty() {
706 format!("<unnamed node #{}>", node.id.0)
707 } else {
708 format!("{:?}", node.name)
709 }
710}
711
712fn display_domain(domain: &str) -> String {
714 if domain.is_empty() {
715 "ai.onnx".to_string()
716 } else {
717 domain.to_string()
718 }
719}
720
721pub fn validate_no_control_flow(graph: &Graph) -> Result<(), LoaderError> {
735 use onnx_runtime_ir::Attribute;
736
737 fn is_implemented_control_flow(node: &onnx_runtime_ir::Node) -> bool {
741 node.is_default_domain() && matches!(node.op_type.as_str(), "If" | "Loop" | "Scan")
742 }
743
744 fn check_graph(graph: &Graph) -> Result<(), LoaderError> {
745 for (_, node) in graph.nodes.iter() {
746 let mut subgraph_attrs: Vec<&String> = node
748 .attributes
749 .iter()
750 .filter(|(_, v)| matches!(v, Attribute::Graph(_) | Attribute::Graphs(_)))
751 .map(|(k, _)| k)
752 .collect();
753 subgraph_attrs.sort();
754 if let Some(attr) = subgraph_attrs.first() {
755 if !is_implemented_control_flow(node) {
758 return Err(LoaderError::UnsupportedControlFlow {
759 op_type: node.op_type.clone(),
760 node: node_label(node),
761 domain: display_domain(&node.domain),
762 attr: (*attr).clone(),
763 });
764 }
765 }
766 }
767 for subgraph in graph.subgraphs.values() {
770 check_graph(subgraph)?;
771 }
772 Ok(())
773 }
774
775 check_graph(graph)
776}
777
778pub fn validate_no_dangling_refs(graph: &Graph) -> Result<(), LoaderError> {
791 use std::collections::HashSet;
792
793 let graph_inputs: HashSet<_> = graph.inputs.iter().copied().collect();
794
795 for (_, node) in graph.nodes.iter() {
796 for vid in node.input_values() {
797 let Some(value) = graph.values.get(vid) else {
798 continue;
801 };
802 let is_sourced = value.producer.is_some()
803 || graph_inputs.contains(&vid)
804 || graph.initializers.contains_key(&vid);
805 if !is_sourced {
806 let tensor = value
807 .name
808 .clone()
809 .unwrap_or_else(|| format!("<anonymous value #{}>", vid.0));
810 return Err(LoaderError::DanglingTensorRef {
811 op_type: node.op_type.clone(),
812 node: node_label(node),
813 domain: display_domain(&node.domain),
814 tensor,
815 });
816 }
817 }
818 }
819 Ok(())
820}
821
822pub fn validate_no_initializer_producer(graph: &Graph) -> Result<(), LoaderError> {
840 for &vid in graph.initializers.keys() {
841 let Some(value) = graph.values.get(vid) else {
842 continue;
843 };
844 if let Some(producer) = value.producer {
845 let tensor = value
846 .name
847 .clone()
848 .unwrap_or_else(|| format!("<anonymous value #{}>", vid.0));
849 let node = if graph.nodes.contains(producer) {
850 node_label(graph.node(producer))
851 } else {
852 format!("<node #{}>", producer.0)
853 };
854 return Err(LoaderError::InitializerHasProducer { tensor, node });
855 }
856 }
857 Ok(())
858}
859pub fn validate_opset_imports(graph: &Graph) -> Result<(), LoaderError> {
863 fn has_import(imports: &std::collections::HashMap<String, u64>, domain: &str) -> bool {
867 imports.contains_key(domain)
868 }
869
870 fn validate_graph(
871 graph: &Graph,
872 imports: &std::collections::HashMap<String, u64>,
873 ) -> Result<(), LoaderError> {
874 for (_, node) in graph.nodes.iter() {
875 if !has_import(imports, &node.domain) {
876 let domain = if node.domain.is_empty() {
877 "ai.onnx".to_string()
878 } else {
879 node.domain.clone()
880 };
881 let node_name = if node.name.is_empty() {
882 format!("<unnamed node #{}>", node.id.0)
883 } else {
884 format!("{:?}", node.name)
885 };
886 return Err(LoaderError::MissingOpsetImport {
887 op_type: node.op_type.clone(),
888 node: node_name,
889 domain,
890 });
891 }
892 }
893 for subgraph in graph.subgraphs.values() {
894 validate_graph(subgraph, imports)?;
895 }
896 Ok(())
897 }
898
899 validate_graph(graph, &graph.opset_imports)
900}