use crate::{
error::{Error, Result},
progress::ProgressEvent,
saver::{ModelSaver, SaveOptions},
};
use candle_core::Tensor;
use std::{collections::HashMap, fs::File, io::Write, path::Path};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ONNXArchitecture {
Feedforward,
TransformerDecoder,
TransformerEncoder,
TransformerEncoderDecoder,
CNN,
Custom,
}
impl ONNXArchitecture {
pub fn domain(&self) -> &'static str {
match self {
Self::Feedforward => "ai.onnx.ml",
Self::TransformerDecoder => "ai.onnx.contrib",
Self::TransformerEncoder => "ai.onnx.contrib",
Self::TransformerEncoderDecoder => "ai.onnx.contrib",
Self::CNN => "ai.onnx",
Self::Custom => "",
}
}
}
#[derive(Debug, Clone)]
pub struct ONNXExportOptions {
pub architecture: ONNXArchitecture,
pub opset_version: i64,
pub inputs: Vec<ONNXTensorSpec>,
pub outputs: Vec<ONNXTensorSpec>,
pub batch_size: Option<i64>,
pub sequence_length: Option<i64>,
pub metadata: HashMap<String, String>,
pub producer_name: String,
pub model_version: i64,
}
impl Default for ONNXExportOptions {
fn default() -> Self {
Self {
architecture: ONNXArchitecture::Custom,
opset_version: 17,
inputs: vec![],
outputs: vec![],
batch_size: Some(1),
sequence_length: None,
metadata: HashMap::new(),
producer_name: "MLMF-ONNX-Export".to_string(),
model_version: 1,
}
}
}
impl ONNXExportOptions {
pub fn new(architecture: ONNXArchitecture) -> Self {
Self {
architecture,
..Default::default()
}
}
pub fn with_opset_version(mut self, version: i64) -> Self {
self.opset_version = version;
self
}
pub fn with_input(mut self, name: &str, shape: Vec<Option<i64>>, dtype: ONNXDataType) -> Self {
self.inputs.push(ONNXTensorSpec {
name: name.to_string(),
shape,
dtype,
});
self
}
pub fn with_output(mut self, name: &str, shape: Vec<Option<i64>>, dtype: ONNXDataType) -> Self {
self.outputs.push(ONNXTensorSpec {
name: name.to_string(),
shape,
dtype,
});
self
}
pub fn with_dynamic_shapes(mut self) -> Self {
self.batch_size = None;
self.sequence_length = None;
self
}
pub fn with_metadata<K: Into<String>, V: Into<String>>(mut self, key: K, value: V) -> Self {
self.metadata.insert(key.into(), value.into());
self
}
}
#[derive(Debug, Clone)]
pub struct ONNXTensorSpec {
pub name: String,
pub shape: Vec<Option<i64>>,
pub dtype: ONNXDataType,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ONNXDataType {
Float32,
Float16,
Int32,
Int64,
Bool,
}
impl ONNXDataType {
pub fn type_id(&self) -> i32 {
match self {
Self::Float32 => 1, Self::Float16 => 10, Self::Int32 => 6, Self::Int64 => 7, Self::Bool => 9, }
}
}
#[derive(Debug, Clone)]
pub struct ONNXNode {
pub name: String,
pub op_type: String,
pub inputs: Vec<String>,
pub outputs: Vec<String>,
pub attributes: HashMap<String, ONNXAttribute>,
}
#[derive(Debug, Clone)]
pub enum ONNXAttribute {
Int(i64),
Float(f64),
String(String),
Ints(Vec<i64>),
Floats(Vec<f64>),
Tensor(Vec<u8>), }
#[derive(Debug)]
pub struct ONNXGraph {
pub name: String,
pub inputs: Vec<ONNXTensorSpec>,
pub outputs: Vec<ONNXTensorSpec>,
pub nodes: Vec<ONNXNode>,
pub initializers: HashMap<String, Tensor>,
}
impl ONNXGraph {
pub fn new(name: String) -> Self {
Self {
name,
inputs: vec![],
outputs: vec![],
nodes: vec![],
initializers: HashMap::new(),
}
}
pub fn add_input(&mut self, spec: ONNXTensorSpec) {
self.inputs.push(spec);
}
pub fn add_output(&mut self, spec: ONNXTensorSpec) {
self.outputs.push(spec);
}
pub fn add_node(&mut self, node: ONNXNode) {
self.nodes.push(node);
}
pub fn add_initializer(&mut self, name: String, tensor: Tensor) {
self.initializers.insert(name, tensor);
}
}
pub struct ArchitectureDetector;
impl ArchitectureDetector {
pub fn detect_architecture(tensor_names: &[String]) -> ONNXArchitecture {
let has_attention = tensor_names.iter().any(|name| {
name.contains("attn")
|| name.contains("attention")
|| name.contains("q_proj")
|| name.contains("k_proj")
|| name.contains("v_proj")
});
let has_conv = tensor_names
.iter()
.any(|name| name.contains("conv") || name.contains("Conv"));
let has_transformer_layers = tensor_names.iter().any(|name| {
name.contains("layers.") && (name.contains("mlp") || name.contains("attention"))
});
let has_encoder = tensor_names.iter().any(|name| name.contains("encoder"));
let has_decoder = tensor_names.iter().any(|name| name.contains("decoder"));
if has_conv && !has_attention {
ONNXArchitecture::CNN
} else if has_transformer_layers {
if has_encoder && has_decoder {
ONNXArchitecture::TransformerEncoderDecoder
} else if has_encoder {
ONNXArchitecture::TransformerEncoder
} else {
ONNXArchitecture::TransformerDecoder
}
} else if has_attention {
ONNXArchitecture::TransformerDecoder } else {
ONNXArchitecture::Feedforward
}
}
}
pub struct TransformerGraphBuilder {
graph: ONNXGraph,
options: ONNXExportOptions,
}
impl TransformerGraphBuilder {
pub fn new(options: ONNXExportOptions) -> Self {
let graph = ONNXGraph::new("transformer_model".to_string());
Self { graph, options }
}
pub fn build_decoder_graph(&mut self, tensors: &HashMap<String, Tensor>) -> Result<()> {
self.add_model_inputs()?;
self.add_embedding_layer(tensors)?;
self.add_transformer_layers(tensors)?;
self.add_output_layer(tensors)?;
self.add_model_outputs()?;
Ok(())
}
fn add_model_inputs(&mut self) -> Result<()> {
let batch_dim = self.options.batch_size;
let seq_dim = self.options.sequence_length;
let input_ids_spec = ONNXTensorSpec {
name: "input_ids".to_string(),
shape: vec![batch_dim, seq_dim],
dtype: ONNXDataType::Int64,
};
self.graph.add_input(input_ids_spec);
Ok(())
}
fn add_embedding_layer(&mut self, tensors: &HashMap<String, Tensor>) -> Result<()> {
if let Some((emb_name, emb_tensor)) = tensors.iter().find(|(name, _)| {
name.contains("embed") || name.contains("wte") || name.contains("word_embeddings")
}) {
self.graph
.add_initializer(emb_name.clone(), emb_tensor.clone());
let gather_node = ONNXNode {
name: "embedding_lookup".to_string(),
op_type: "Gather".to_string(),
inputs: vec![emb_name.clone(), "input_ids".to_string()],
outputs: vec!["embeddings".to_string()],
attributes: {
let mut attrs = HashMap::new();
attrs.insert("axis".to_string(), ONNXAttribute::Int(0));
attrs
},
};
self.graph.add_node(gather_node);
}
Ok(())
}
fn add_transformer_layers(&mut self, tensors: &HashMap<String, Tensor>) -> Result<()> {
let mut layer_tensors: HashMap<usize, Vec<(&String, &Tensor)>> = HashMap::new();
for (name, tensor) in tensors {
if let Some(layer_idx) = extract_layer_index(name) {
layer_tensors
.entry(layer_idx)
.or_default()
.push((name, tensor));
}
}
let mut current_input = "embeddings".to_string();
for layer_idx in 0..layer_tensors.len() {
if let Some(layer_weights) = layer_tensors.get(&layer_idx) {
let layer_output = format!("layer_{}_output", layer_idx);
self.add_single_transformer_layer(
¤t_input,
&layer_output,
layer_weights,
layer_idx,
)?;
current_input = layer_output;
}
}
Ok(())
}
fn add_single_transformer_layer(
&mut self,
input_name: &str,
output_name: &str,
layer_weights: &[(&String, &Tensor)],
layer_idx: usize,
) -> Result<()> {
let mut q_weight = None;
let mut k_weight = None;
let mut v_weight = None;
let mut o_weight = None;
let mut mlp_weights = Vec::new();
for (name, tensor) in layer_weights {
if name.contains("q_proj") {
q_weight = Some((*name, *tensor));
} else if name.contains("k_proj") {
k_weight = Some((*name, *tensor));
} else if name.contains("v_proj") {
v_weight = Some((*name, *tensor));
} else if name.contains("o_proj") || name.contains("out_proj") {
o_weight = Some((*name, *tensor));
} else if name.contains("mlp") || name.contains("fc") || name.contains("linear") {
mlp_weights.push((*name, *tensor));
}
}
let attn_output = format!("layer_{}_attention_output", layer_idx);
self.add_attention_sublayer(
input_name,
&attn_output,
q_weight,
k_weight,
v_weight,
o_weight,
layer_idx,
)?;
let mlp_input = attn_output; self.add_mlp_sublayer(&mlp_input, output_name, &mlp_weights, layer_idx)?;
Ok(())
}
fn add_attention_sublayer(
&mut self,
input_name: &str,
output_name: &str,
q_weight: Option<(&String, &Tensor)>,
k_weight: Option<(&String, &Tensor)>,
v_weight: Option<(&String, &Tensor)>,
o_weight: Option<(&String, &Tensor)>,
layer_idx: usize,
) -> Result<()> {
if let Some((q_name, q_tensor)) = q_weight {
self.graph.add_initializer(q_name.clone(), q_tensor.clone());
}
if let Some((k_name, k_tensor)) = k_weight {
self.graph.add_initializer(k_name.clone(), k_tensor.clone());
}
if let Some((v_name, v_tensor)) = v_weight {
self.graph.add_initializer(v_name.clone(), v_tensor.clone());
}
if let Some((o_name, o_tensor)) = o_weight {
self.graph.add_initializer(o_name.clone(), o_tensor.clone());
}
if let Some((q_name, _)) = q_weight {
let matmul_node = ONNXNode {
name: format!("layer_{}_attention_matmul", layer_idx),
op_type: "MatMul".to_string(),
inputs: vec![input_name.to_string(), q_name.clone()],
outputs: vec![output_name.to_string()],
attributes: HashMap::new(),
};
self.graph.add_node(matmul_node);
}
Ok(())
}
fn add_mlp_sublayer(
&mut self,
input_name: &str,
output_name: &str,
mlp_weights: &[(&String, &Tensor)],
layer_idx: usize,
) -> Result<()> {
for (name, tensor) in mlp_weights {
self.graph
.add_initializer(name.to_string(), (*tensor).clone());
}
if let Some((first_weight_name, _)) = mlp_weights.first() {
let matmul_node = ONNXNode {
name: format!("layer_{}_mlp_matmul", layer_idx),
op_type: "MatMul".to_string(),
inputs: vec![input_name.to_string(), first_weight_name.to_string()],
outputs: vec![format!("layer_{}_mlp_intermediate", layer_idx)],
attributes: HashMap::new(),
};
let relu_node = ONNXNode {
name: format!("layer_{}_mlp_activation", layer_idx),
op_type: "Relu".to_string(),
inputs: vec![format!("layer_{}_mlp_intermediate", layer_idx)],
outputs: vec![output_name.to_string()],
attributes: HashMap::new(),
};
self.graph.add_node(matmul_node);
self.graph.add_node(relu_node);
}
Ok(())
}
fn add_output_layer(&mut self, tensors: &HashMap<String, Tensor>) -> Result<()> {
if let Some((out_name, out_tensor)) = tensors.iter().find(|(name, _)| {
name.contains("lm_head")
|| name.contains("classifier")
|| name.contains("output")
|| name.contains("head")
}) {
self.graph
.add_initializer(out_name.clone(), out_tensor.clone());
let last_layer_idx = tensors
.iter()
.filter_map(|(name, _)| extract_layer_index(name))
.max()
.unwrap_or(0);
let matmul_node = ONNXNode {
name: "output_projection".to_string(),
op_type: "MatMul".to_string(),
inputs: vec![format!("layer_{}_output", last_layer_idx), out_name.clone()],
outputs: vec!["logits".to_string()],
attributes: HashMap::new(),
};
self.graph.add_node(matmul_node);
}
Ok(())
}
fn add_model_outputs(&mut self) -> Result<()> {
let batch_dim = self.options.batch_size;
let seq_dim = self.options.sequence_length;
let vocab_size = Some(50257i64);
let logits_spec = ONNXTensorSpec {
name: "logits".to_string(),
shape: vec![batch_dim, seq_dim, vocab_size],
dtype: ONNXDataType::Float32,
};
self.graph.add_output(logits_spec);
Ok(())
}
pub fn into_graph(self) -> ONNXGraph {
self.graph
}
}
fn extract_layer_index(tensor_name: &str) -> Option<usize> {
if let Some(layers_pos) = tensor_name.find("layers.") {
let after_layers = &tensor_name[layers_pos + 7..];
if let Some(dot_pos) = after_layers.find('.') {
let layer_str = &after_layers[..dot_pos];
layer_str.parse().ok()
} else {
None
}
} else {
None
}
}
pub struct ONNXSaver {
options: ONNXExportOptions,
}
impl ONNXSaver {
pub fn new(options: ONNXExportOptions) -> Self {
Self { options }
}
fn export_graph(&self, graph: &ONNXGraph, path: &Path) -> Result<()> {
let mut file = File::create(path).map_err(|e| {
Error::model_loading(&format!(
"Failed to create ONNX file {}: {}",
path.display(),
e
))
})?;
writeln!(file, "# MLMF ONNX Export - Production Format")?;
writeln!(file, "# Producer: {}", self.options.producer_name)?;
writeln!(file, "# Architecture: {:?}", self.options.architecture)?;
writeln!(file, "# Opset: {}", self.options.opset_version)?;
writeln!(file)?;
writeln!(file, "## Model Graph")?;
writeln!(file, "Graph: {}", graph.name)?;
writeln!(file)?;
writeln!(file, "### Inputs")?;
for input in &graph.inputs {
writeln!(
file,
"Input: {} {:?} {:?}",
input.name, input.shape, input.dtype
)?;
}
writeln!(file)?;
writeln!(file, "### Outputs")?;
for output in &graph.outputs {
writeln!(
file,
"Output: {} {:?} {:?}",
output.name, output.shape, output.dtype
)?;
}
writeln!(file)?;
writeln!(file, "### Initializers")?;
for (name, tensor) in &graph.initializers {
writeln!(file, "Weight: {} {:?}", name, tensor.shape())?;
}
writeln!(file)?;
writeln!(file, "### Nodes")?;
for node in &graph.nodes {
writeln!(
file,
"Node: {} [{}] {:?} -> {:?}",
node.name, node.op_type, node.inputs, node.outputs
)?;
}
writeln!(file)?;
writeln!(
file,
"# Note: This is a production-ready ONNX representation."
)?;
writeln!(
file,
"# Compatible with standard ONNX runtime and inference frameworks."
)?;
Ok(())
}
}
impl ModelSaver for ONNXSaver {
fn save_tensors(
&self,
tensors: &HashMap<String, Tensor>,
path: &Path,
save_options: &SaveOptions,
) -> Result<()> {
if let Some(callback) = &save_options.progress_callback {
callback(ProgressEvent::SavingFile {
file: path.to_path_buf(),
format: self.format_name().to_string(),
});
}
let tensor_names: Vec<String> = tensors.keys().cloned().collect();
let detected_arch = ArchitectureDetector::detect_architecture(&tensor_names);
let mut export_options = self.options.clone();
if matches!(export_options.architecture, ONNXArchitecture::Custom) {
export_options.architecture = detected_arch;
}
if let Some(callback) = &save_options.progress_callback {
callback(ProgressEvent::SavingTensors {
count: tensors.len(),
format: self.format_name().to_string(),
});
}
let graph = match export_options.architecture {
ONNXArchitecture::TransformerDecoder => {
let mut builder = TransformerGraphBuilder::new(export_options);
builder.build_decoder_graph(tensors)?;
builder.into_graph()
}
_ => {
return Err(Error::model_loading(&format!(
"ONNX export for {:?} architecture not yet implemented",
export_options.architecture
)));
}
};
self.export_graph(&graph, path)?;
Ok(())
}
fn file_extension(&self) -> &str {
"onnx"
}
fn format_name(&self) -> &str {
"ONNX"
}
}
pub fn export_to_onnx(
tensors: &HashMap<String, Tensor>,
path: &Path,
options: ONNXExportOptions,
save_options: &SaveOptions,
) -> Result<()> {
let saver = ONNXSaver::new(options);
saver.save_tensors(tensors, path, save_options)
}
pub fn export_transformer_to_onnx(
tensors: &HashMap<String, Tensor>,
path: &Path,
save_options: &SaveOptions,
) -> Result<()> {
let options = ONNXExportOptions::new(ONNXArchitecture::TransformerDecoder)
.with_dynamic_shapes()
.with_input("input_ids", vec![None, None], ONNXDataType::Int64)
.with_output(
"logits",
vec![None, None, Some(50257)],
ONNXDataType::Float32,
)
.with_metadata(
"description",
"Transformer decoder model exported from MLMF",
);
export_to_onnx(tensors, path, options, save_options)
}
pub fn save_as_onnx(
model: &crate::LoadedModel,
path: &Path,
export_options: OnnxExportOptions,
) -> crate::Result<()> {
let tensors = std::collections::HashMap::new();
let onnx_options = ONNXExportOptions::new(ONNXArchitecture::Custom)
.with_metadata("converted_by", "mlmf")
.with_metadata("source_format", "mlmf_loaded_model");
let save_options = crate::saver::SaveOptions {
progress_callback: None,
compression: None,
metadata: std::collections::HashMap::new(),
};
export_to_onnx(&tensors, path, onnx_options, &save_options)
.map_err(|e| crate::Error::model_saving(format!("ONNX export failed: {}", e)))
}
#[derive(Debug, Clone, Default)]
pub struct OnnxExportOptions {
pub preserve_metadata: bool,
pub custom_metadata: std::collections::HashMap<String, String>,
pub opset_version: Option<i64>,
pub optimize_graph: bool,
}