use std::fmt;
use std::io;
use std::path::Path;
use fandhe_ai_autodiff::AutodiffError;
use fandhe_ai_autodiff::nn::MultiheadAttentionConfig;
mod compiled;
pub(super) mod functional_io;
use self::compiled::{
CompiledMeta, OPTIMIZER_PREFIX, check_lbfgs_history, check_slot_shapes, parse_compiled,
render_compiled, split_optimizer_tensors,
};
use super::sequential::{LayerSpec, ReparamSpec, Sequential, spectral_state_of};
use crate::fs_guard::{LeafError, MAX_MODEL_FILE_BYTES, OpenedLeaf, open_leaf_checked};
#[cfg(unix)]
use crate::fs_write::{self, MAX_TMP_NAME_ATTEMPTS, name_entropy, remove_own_tmp};
use crate::interop::safetensors::{load_safetensors_f32_from_bytes, save_safetensors_f32_to_bytes};
use crate::{EmbeddingBagMode, GlobalPoolMode, InterpolateMode, Tensor};
const MAX_SPECTRAL_POWER_ITERATIONS: usize = 4096;
const MANIFEST_FILE_NAME: &str = "manifest.json";
const FORMAT_NAME: &str = "fandhe-ai.compat.sequential";
const FORMAT_VERSION: u64 = 2;
const LEGACY_FORMAT_VERSION: u64 = 1;
const MAX_MANIFEST_BYTES: u64 = 1024 * 1024;
const MAX_LAYERS: usize = 4096;
const MAX_LBFGS_HISTORY: usize = 65536;
const MAX_JSON_DEPTH: usize = 4;
const MAX_ARRAY_LEN: usize = 2 * MAX_LAYERS;
const MAX_NUMBER_LEXEME_LEN: usize = 64;
const MAX_OBJECT_KEYS: usize = 16;
#[derive(Debug)]
#[non_exhaustive]
pub enum ModelIoError {
Io(std::io::Error),
Manifest {
message: String,
},
Safetensors(String),
UnsupportedModel {
reason: String,
},
Mismatch {
message: String,
},
Autodiff(AutodiffError),
TooLarge {
what: &'static str,
limit: u64,
},
}
impl fmt::Display for ModelIoError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
ModelIoError::Io(e) => write!(f, "I/O エラー: {e}"),
ModelIoError::Manifest { message } => write!(f, "manifest が不正です: {message}"),
ModelIoError::Safetensors(m) => write!(f, "safetensors の処理に失敗しました: {m}"),
ModelIoError::UnsupportedModel { reason } => {
write!(f, "未対応のモデルです: {reason}")
}
ModelIoError::Mismatch { message } => {
write!(f, "manifest と safetensors が一致しません: {message}")
}
ModelIoError::Autodiff(e) => write!(f, "モデルの構築に失敗しました: {e}"),
ModelIoError::TooLarge { what, limit } => {
write!(f, "{what} が固定上限 {limit} を超えています")
}
}
}
}
impl std::error::Error for ModelIoError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
ModelIoError::Io(e) => Some(e),
ModelIoError::Autodiff(e) => Some(e),
_ => None,
}
}
}
pub fn save_model(model: &Sequential, dir: impl AsRef<Path>) -> Result<(), ModelIoError> {
save_platform_check()?;
let prepared = prepare_save(model)?;
write_prepared(dir.as_ref(), &prepared)
}
pub fn load_model(dir: impl AsRef<Path>) -> Result<Sequential, ModelIoError> {
load_from_dir_with_limits(dir.as_ref(), MAX_MANIFEST_BYTES, MAX_MODEL_FILE_BYTES)
}
fn save_platform_supported(is_unix: bool) -> io::Result<()> {
if is_unix {
Ok(())
} else {
Err(io::Error::from(io::ErrorKind::Unsupported))
}
}
fn save_platform_check() -> Result<(), ModelIoError> {
save_platform_supported(cfg!(unix)).map_err(ModelIoError::Io)
}
fn spec_kind(spec: &LayerSpec) -> Option<&'static str> {
match spec {
LayerSpec::Linear { .. } => Some("linear"),
LayerSpec::Relu => Some("relu"),
LayerSpec::Sigmoid => Some("sigmoid"),
LayerSpec::Tanh => Some("tanh"),
LayerSpec::Silu => Some("silu"),
LayerSpec::Hardswish => Some("hardswish"),
LayerSpec::Gelu => Some("gelu"),
LayerSpec::GeluTanh => Some("gelu_tanh"),
LayerSpec::LeakyRelu { .. } => Some("leaky_relu"),
LayerSpec::Elu { .. } => Some("elu"),
LayerSpec::Softmax { .. } => Some("softmax"),
LayerSpec::LogSoftmax { .. } => Some("log_softmax"),
LayerSpec::Softplus { .. } => Some("softplus"),
LayerSpec::Mish => Some("mish"),
LayerSpec::Selu => Some("selu"),
LayerSpec::Softsign => Some("softsign"),
LayerSpec::Hardsigmoid => Some("hardsigmoid"),
LayerSpec::LogSigmoid => Some("log_sigmoid"),
LayerSpec::Tanhshrink => Some("tanhshrink"),
LayerSpec::Celu { .. } => Some("celu"),
LayerSpec::Softmin { .. } => Some("softmin"),
LayerSpec::Threshold { .. } => Some("threshold"),
LayerSpec::RRelu { .. } => Some("rrelu"),
LayerSpec::Hardtanh { .. } => Some("hardtanh"),
LayerSpec::Relu6 => Some("relu6"),
LayerSpec::Glu { .. } => Some("glu"),
LayerSpec::PRelu { .. } => Some("prelu"),
LayerSpec::Flatten { .. } => Some("flatten"),
LayerSpec::Dropout { .. } => Some("dropout"),
LayerSpec::Dropout2d { .. } => Some("dropout2d"),
LayerSpec::AlphaDropout { .. } => Some("alpha_dropout"),
LayerSpec::Conv2d { .. } => Some("conv2d"),
LayerSpec::Conv3d { .. } => Some("conv3d"),
LayerSpec::ConvTranspose2d { .. } => Some("conv_transpose2d"),
LayerSpec::ConvTranspose3d { .. } => Some("conv_transpose3d"),
LayerSpec::Conv1d { .. } => Some("conv1d"),
LayerSpec::ConvTranspose1d { .. } => Some("conv_transpose1d"),
LayerSpec::Unflatten { .. } => Some("unflatten"),
LayerSpec::LayerNorm { .. } => Some("layer_norm"),
LayerSpec::RmsNorm { .. } => Some("rms_norm"),
LayerSpec::BatchNorm1d { .. } => Some("batch_norm1d"),
LayerSpec::BatchNorm2d { .. } => Some("batch_norm2d"),
LayerSpec::Embedding { .. } => Some("embedding"),
LayerSpec::EmbeddingBag { .. } => Some("embedding_bag"),
LayerSpec::MultiheadAttention { .. } => Some("multihead_attention"),
LayerSpec::MultiheadAttentionConfig { .. } => Some("multihead_attention_config"),
LayerSpec::TransformerEncoder { .. } => Some("transformer_encoder"),
LayerSpec::TransformerDecoderLayer { .. } => Some("transformer_decoder_layer"),
LayerSpec::Transformer { .. } => Some("transformer"),
LayerSpec::MaxPool2d { .. } => Some("max_pool2d"),
LayerSpec::MaxPool1d { .. } => Some("max_pool1d"),
LayerSpec::MaxPool3d { .. } => Some("max_pool3d"),
LayerSpec::Unfold { .. } => Some("unfold"),
LayerSpec::Fold { .. } => Some("fold"),
LayerSpec::LocalResponseNorm { .. } => Some("local_response_norm"),
LayerSpec::AvgPool3d { .. } => Some("avg_pool3d"),
LayerSpec::AvgPool2d { .. } => Some("avg_pool2d"),
LayerSpec::AvgPool1d { .. } => Some("avg_pool1d"),
LayerSpec::AdaptiveAvgPool2d { .. } => Some("adaptive_avg_pool2d"),
LayerSpec::AdaptiveAvgPool1d { .. } => Some("adaptive_avg_pool1d"),
LayerSpec::AdaptiveMaxPool2d { .. } => Some("adaptive_max_pool2d"),
LayerSpec::AdaptiveMaxPool1d { .. } => Some("adaptive_max_pool1d"),
LayerSpec::GlobalPool { .. } => Some("global_pool"),
LayerSpec::Upsample { .. } => Some("upsample"),
LayerSpec::ZeroPad2d { .. } => Some("zero_pad2d"),
LayerSpec::Identity => Some("identity"),
LayerSpec::GroupNorm { .. } => Some("group_norm"),
LayerSpec::InstanceNorm { .. } => Some("instance_norm"),
LayerSpec::PixelShuffle { .. } => Some("pixel_shuffle"),
LayerSpec::PixelUnshuffle { .. } => Some("pixel_unshuffle"),
LayerSpec::Reparam { base, reparam } => reparam_kind(base, reparam),
LayerSpec::Unsupported { .. } => None,
}
}
const REPARAM_KINDS: [(&str, &str, bool); 14] = [
("linear_weight_norm", "linear", false),
("linear_spectral_norm", "linear", true),
("conv1d_weight_norm", "conv1d", false),
("conv1d_spectral_norm", "conv1d", true),
("conv2d_weight_norm", "conv2d", false),
("conv2d_spectral_norm", "conv2d", true),
("conv3d_weight_norm", "conv3d", false),
("conv3d_spectral_norm", "conv3d", true),
("conv_transpose1d_weight_norm", "conv_transpose1d", false),
("conv_transpose1d_spectral_norm", "conv_transpose1d", true),
("conv_transpose2d_weight_norm", "conv_transpose2d", false),
("conv_transpose2d_spectral_norm", "conv_transpose2d", true),
("conv_transpose3d_weight_norm", "conv_transpose3d", false),
("conv_transpose3d_spectral_norm", "conv_transpose3d", true),
];
fn reparam_kind(base: &LayerSpec, reparam: &ReparamSpec) -> Option<&'static str> {
let base_kind = spec_kind(base)?;
let spectral = matches!(reparam, ReparamSpec::SpectralNorm { .. });
REPARAM_KINDS
.iter()
.find(|(_, b, sp)| *b == base_kind && *sp == spectral)
.map(|(k, _, _)| *k)
}
fn reparam_kind_parts(kind: &str) -> Option<(&'static str, bool)> {
REPARAM_KINDS
.iter()
.find(|(k, _, _)| *k == kind)
.map(|(_, b, sp)| (*b, *sp))
}
fn spectral_row_axis(base: &LayerSpec) -> Option<usize> {
match base {
LayerSpec::Linear { .. }
| LayerSpec::ConvTranspose1d { .. }
| LayerSpec::ConvTranspose2d { .. }
| LayerSpec::ConvTranspose3d { .. } => Some(1),
LayerSpec::Conv1d { .. } | LayerSpec::Conv2d { .. } | LayerSpec::Conv3d { .. } => Some(0),
_ => None,
}
}
fn spectral_uv_lens(base: &LayerSpec) -> Option<(usize, usize)> {
let axis = spectral_row_axis(base)?;
let weight = layer_parameters(base).into_iter().next()?.1;
let rows = *weight.get(axis)?;
let numel = weight
.iter()
.try_fold(1usize, |acc, d| acc.checked_mul(*d))?;
if rows == 0 || numel == 0 {
return None;
}
Some((rows, numel / rows))
}
fn spec_f32_fields_are_finite(spec: &LayerSpec) -> bool {
match spec {
LayerSpec::LeakyRelu { negative_slope } => negative_slope.is_finite(),
LayerSpec::Reparam { base, reparam } => {
spec_f32_fields_are_finite(base)
&& match reparam {
ReparamSpec::SpectralNorm { eps, .. } => eps.is_finite(),
ReparamSpec::WeightNorm { .. } => true,
}
}
LayerSpec::LocalResponseNorm { alpha, beta, k, .. } => {
alpha.is_finite() && beta.is_finite() && k.is_finite()
}
LayerSpec::Elu { alpha } => alpha.is_finite(),
LayerSpec::Celu { alpha } => alpha.is_finite(),
LayerSpec::Threshold { threshold, value } => threshold.is_finite() && value.is_finite(),
LayerSpec::RRelu { lower, upper } => lower.is_finite() && upper.is_finite(),
LayerSpec::Softplus { beta, threshold } => beta.is_finite() && threshold.is_finite(),
LayerSpec::Hardtanh { min_val, max_val } => min_val.is_finite() && max_val.is_finite(),
LayerSpec::Dropout { p } | LayerSpec::Dropout2d { p } | LayerSpec::AlphaDropout { p } => {
p.is_finite()
}
LayerSpec::LayerNorm { eps, .. }
| LayerSpec::RmsNorm { eps, .. }
| LayerSpec::GroupNorm { eps, .. }
| LayerSpec::Transformer { eps, .. }
| LayerSpec::InstanceNorm { eps } => eps.is_finite(),
LayerSpec::BatchNorm1d { eps, momentum, .. }
| LayerSpec::BatchNorm2d { eps, momentum, .. } => eps.is_finite() && momentum.is_finite(),
_ => true,
}
}
fn mha_parameters(embed: usize, with_prefix: bool, bias: bool) -> Vec<(&'static str, Vec<usize>)> {
let names: [&'static str; 8] = if with_prefix {
[
"self_attn.q_proj.weight",
"self_attn.q_proj.bias",
"self_attn.k_proj.weight",
"self_attn.k_proj.bias",
"self_attn.v_proj.weight",
"self_attn.v_proj.bias",
"self_attn.out_proj.weight",
"self_attn.out_proj.bias",
]
} else {
[
"q_proj.weight",
"q_proj.bias",
"k_proj.weight",
"k_proj.bias",
"v_proj.weight",
"v_proj.bias",
"out_proj.weight",
"out_proj.bias",
]
};
names
.into_iter()
.enumerate()
.filter(|(i, _)| bias || i % 2 == 0)
.map(|(i, name)| {
let shape = if i % 2 == 0 {
vec![embed, embed]
} else {
vec![embed]
};
(name, shape)
})
.collect()
}
fn layer_parameters(spec: &LayerSpec) -> Vec<(&'static str, Vec<usize>)> {
match spec {
LayerSpec::Reparam { base, reparam } => {
let mut rest = layer_parameters(base).into_iter();
let Some((_, weight)) = rest.next() else {
return Vec::new();
};
let mut out: Vec<(&'static str, Vec<usize>)> = match reparam {
ReparamSpec::WeightNorm { dim } => {
let g: Vec<usize> = match dim {
None => Vec::new(),
Some(d) => weight
.iter()
.enumerate()
.map(|(axis, n)| if axis == *d { *n } else { 1 })
.collect(),
};
vec![
("parametrizations.weight.original0", g),
("parametrizations.weight.original1", weight),
]
}
ReparamSpec::SpectralNorm { .. } => {
vec![("parametrizations.weight.original", weight)]
}
};
out.extend(rest);
out
}
LayerSpec::Linear {
in_features,
out_features,
} => vec![
("weight", vec![*in_features, *out_features]),
("bias", vec![*out_features]),
],
LayerSpec::Conv2d {
in_channels,
out_channels,
kernel_size,
groups,
..
} => vec![
(
"weight",
vec![
*out_channels,
in_channels.checked_div(*groups).unwrap_or(0),
kernel_size[0],
kernel_size[1],
],
),
("bias", vec![*out_channels]),
],
LayerSpec::Conv3d {
in_channels,
out_channels,
kernel_size,
groups,
..
} => vec![
(
"weight",
vec![
*out_channels,
in_channels.checked_div(*groups).unwrap_or(0),
kernel_size[0],
kernel_size[1],
kernel_size[2],
],
),
("bias", vec![*out_channels]),
],
LayerSpec::ConvTranspose2d {
in_channels,
out_channels,
kernel_size,
groups,
..
} => vec![
(
"weight",
vec![
*in_channels,
out_channels.checked_div(*groups).unwrap_or(0),
kernel_size[0],
kernel_size[1],
],
),
("bias", vec![*out_channels]),
],
LayerSpec::ConvTranspose3d {
in_channels,
out_channels,
kernel_size,
groups,
..
} => vec![
(
"weight",
vec![
*in_channels,
out_channels.checked_div(*groups).unwrap_or(0),
kernel_size[0],
kernel_size[1],
kernel_size[2],
],
),
("bias", vec![*out_channels]),
],
LayerSpec::ConvTranspose1d {
in_channels,
out_channels,
kernel_size,
groups,
..
} => vec![
(
"weight",
vec![
*in_channels,
out_channels.checked_div(*groups).unwrap_or(0),
*kernel_size,
],
),
("bias", vec![*out_channels]),
],
LayerSpec::Conv1d {
in_channels,
out_channels,
kernel_size,
groups,
..
} => vec![
(
"weight",
vec![
*out_channels,
in_channels.checked_div(*groups).unwrap_or(0),
*kernel_size,
],
),
("bias", vec![*out_channels]),
],
LayerSpec::LayerNorm {
normalized_size, ..
} => vec![
("weight", vec![*normalized_size]),
("bias", vec![*normalized_size]),
],
LayerSpec::RmsNorm {
normalized_size, ..
} => vec![("weight", vec![*normalized_size])],
LayerSpec::BatchNorm1d { num_features, .. }
| LayerSpec::BatchNorm2d { num_features, .. } => vec![
("weight", vec![*num_features]),
("bias", vec![*num_features]),
],
LayerSpec::Embedding {
num_embeddings,
embedding_dim,
..
}
| LayerSpec::EmbeddingBag {
num_embeddings,
embedding_dim,
..
} => vec![("weight", vec![*num_embeddings, *embedding_dim])],
LayerSpec::PRelu { num_parameters } => vec![("weight", vec![*num_parameters])],
LayerSpec::MultiheadAttention { embed_dim, .. } => mha_parameters(*embed_dim, false, true),
LayerSpec::MultiheadAttentionConfig {
embed_dim, bias, ..
} => mha_parameters(*embed_dim, false, *bias),
LayerSpec::TransformerEncoder {
d_model,
dim_feedforward,
..
} => transformer_encoder_layer_parameters(*d_model, *dim_feedforward),
LayerSpec::TransformerDecoderLayer {
d_model,
dim_feedforward,
..
} => transformer_decoder_layer_parameters(*d_model, *dim_feedforward),
LayerSpec::Transformer { .. } => Vec::new(),
LayerSpec::Relu
| LayerSpec::Sigmoid
| LayerSpec::Tanh
| LayerSpec::Silu
| LayerSpec::Hardswish
| LayerSpec::Gelu
| LayerSpec::GeluTanh
| LayerSpec::LeakyRelu { .. }
| LayerSpec::Elu { .. }
| LayerSpec::Softmax { .. }
| LayerSpec::LogSoftmax { .. }
| LayerSpec::Softplus { .. }
| LayerSpec::Mish
| LayerSpec::Selu
| LayerSpec::Softsign
| LayerSpec::Hardsigmoid
| LayerSpec::LogSigmoid
| LayerSpec::Tanhshrink
| LayerSpec::Celu { .. }
| LayerSpec::Softmin { .. }
| LayerSpec::Threshold { .. }
| LayerSpec::RRelu { .. }
| LayerSpec::Hardtanh { .. }
| LayerSpec::Relu6
| LayerSpec::Glu { .. }
| LayerSpec::Flatten { .. }
| LayerSpec::Dropout { .. }
| LayerSpec::Dropout2d { .. }
| LayerSpec::AlphaDropout { .. }
| LayerSpec::MaxPool2d { .. }
| LayerSpec::MaxPool1d { .. }
| LayerSpec::MaxPool3d { .. }
| LayerSpec::Unfold { .. }
| LayerSpec::Fold { .. }
| LayerSpec::LocalResponseNorm { .. }
| LayerSpec::AvgPool3d { .. }
| LayerSpec::AvgPool2d { .. }
| LayerSpec::AvgPool1d { .. }
| LayerSpec::AdaptiveAvgPool2d { .. }
| LayerSpec::AdaptiveAvgPool1d { .. }
| LayerSpec::AdaptiveMaxPool2d { .. }
| LayerSpec::AdaptiveMaxPool1d { .. }
| LayerSpec::GlobalPool { .. }
| LayerSpec::Upsample { .. }
| LayerSpec::Unflatten { .. }
| LayerSpec::ZeroPad2d { .. }
| LayerSpec::Identity
| LayerSpec::GroupNorm { .. }
| LayerSpec::InstanceNorm { .. }
| LayerSpec::PixelShuffle { .. }
| LayerSpec::PixelUnshuffle { .. }
| LayerSpec::Unsupported { .. } => Vec::new(),
}
}
fn transformer_encoder_layer_parameters(dm: usize, ff: usize) -> Vec<(&'static str, Vec<usize>)> {
let mut out = mha_parameters(dm, true, true);
out.extend([
("linear1.weight", vec![dm, ff]),
("linear1.bias", vec![ff]),
("linear2.weight", vec![ff, dm]),
("linear2.bias", vec![dm]),
("norm1.weight", vec![dm]),
("norm1.bias", vec![dm]),
("norm2.weight", vec![dm]),
("norm2.bias", vec![dm]),
]);
out
}
fn transformer_decoder_layer_parameters(dm: usize, ff: usize) -> Vec<(&'static str, Vec<usize>)> {
let mut out = mha_parameters(dm, true, true);
out.extend([
("multihead_attn.q_proj.weight", vec![dm, dm]),
("multihead_attn.q_proj.bias", vec![dm]),
("multihead_attn.k_proj.weight", vec![dm, dm]),
("multihead_attn.k_proj.bias", vec![dm]),
("multihead_attn.v_proj.weight", vec![dm, dm]),
("multihead_attn.v_proj.bias", vec![dm]),
("multihead_attn.out_proj.weight", vec![dm, dm]),
("multihead_attn.out_proj.bias", vec![dm]),
("linear1.weight", vec![dm, ff]),
("linear1.bias", vec![ff]),
("linear2.weight", vec![ff, dm]),
("linear2.bias", vec![dm]),
("norm1.weight", vec![dm]),
("norm1.bias", vec![dm]),
("norm2.weight", vec![dm]),
("norm2.bias", vec![dm]),
("norm3.weight", vec![dm]),
("norm3.bias", vec![dm]),
]);
out
}
fn transformer_parameters(
d_model: usize,
dim_feedforward: usize,
num_encoder_layers: usize,
num_decoder_layers: usize,
) -> Vec<(String, Vec<usize>)> {
let mut out = Vec::new();
for i in 0..num_encoder_layers {
for (name, shape) in transformer_encoder_layer_parameters(d_model, dim_feedforward) {
out.push((format!("encoder.layers.{i}.{name}"), shape));
}
}
out.push(("encoder.norm.weight".to_string(), vec![d_model]));
out.push(("encoder.norm.bias".to_string(), vec![d_model]));
for i in 0..num_decoder_layers {
for (name, shape) in transformer_decoder_layer_parameters(d_model, dim_feedforward) {
out.push((format!("decoder.layers.{i}.{name}"), shape));
}
}
out.push(("decoder.norm.weight".to_string(), vec![d_model]));
out.push(("decoder.norm.bias".to_string(), vec![d_model]));
out
}
fn layer_parameter_count(spec: &LayerSpec) -> Option<usize> {
match spec {
LayerSpec::Transformer {
num_encoder_layers,
num_decoder_layers,
..
} => num_encoder_layers
.checked_mul(16)?
.checked_add(num_decoder_layers.checked_mul(26)?)?
.checked_add(4),
other => Some(layer_parameters(other).len()),
}
}
fn expected_parameter_keys(specs: &[LayerSpec]) -> Vec<(String, Vec<usize>)> {
let mut keys = Vec::new();
for (i, spec) in specs.iter().enumerate() {
if let LayerSpec::Transformer {
d_model,
dim_feedforward,
num_encoder_layers,
num_decoder_layers,
..
} = spec
{
for (name, shape) in transformer_parameters(
*d_model,
*dim_feedforward,
*num_encoder_layers,
*num_decoder_layers,
) {
keys.push((format!("{i}.{name}"), shape));
}
continue;
}
for (name, shape) in layer_parameters(spec) {
keys.push((format!("{i}.{name}"), shape));
}
}
keys
}
const SPECTRAL_U_NAME: &str = "parametrizations.weight.0._u";
const SPECTRAL_V_NAME: &str = "parametrizations.weight.0._v";
fn expected_buffer_keys(specs: &[LayerSpec]) -> Vec<(String, Vec<usize>)> {
let mut keys = Vec::new();
for (i, spec) in specs.iter().enumerate() {
if let LayerSpec::BatchNorm1d { num_features, .. }
| LayerSpec::BatchNorm2d { num_features, .. } = spec
{
keys.push((format!("{i}.running_mean"), vec![*num_features]));
keys.push((format!("{i}.running_var"), vec![*num_features]));
}
if let LayerSpec::Reparam {
base,
reparam: ReparamSpec::SpectralNorm { .. },
} = spec
&& let Some((rows, cols)) = spectral_uv_lens(base)
{
keys.push((format!("{i}.{SPECTRAL_U_NAME}"), vec![rows]));
keys.push((format!("{i}.{SPECTRAL_V_NAME}"), vec![cols]));
}
}
keys
}
type ParamFields = Vec<(String, String)>;
fn put_num(f: &mut ParamFields, key: &str, v: usize) {
f.push((key.to_string(), v.to_string()));
}
fn put_real(f: &mut ParamFields, key: &str, v: f32) {
f.push((key.to_string(), format!("{v:?}")));
}
fn put_bool(f: &mut ParamFields, key: &str, v: bool) {
f.push((key.to_string(), v.to_string()));
}
fn put_opt(f: &mut ParamFields, key: &str, v: Option<usize>) {
f.push((
key.to_string(),
v.map_or_else(|| "null".to_string(), |x| x.to_string()),
));
}
fn put_pair(f: &mut ParamFields, base: &str, v: [usize; 2]) {
put_num(f, &format!("{base}_h"), v[0]);
put_num(f, &format!("{base}_w"), v[1]);
}
fn put_triple(f: &mut ParamFields, base: &str, v: [usize; 3]) {
put_num(f, &format!("{base}_d"), v[0]);
put_num(f, &format!("{base}_h"), v[1]);
put_num(f, &format!("{base}_w"), v[2]);
}
fn put_triple_str(f: &mut ParamFields, key: &str, v: [usize; 3]) {
f.push((key.to_string(), format!("\"{},{},{}\"", v[0], v[1], v[2])));
}
fn put_opt_pair(f: &mut ParamFields, base: &str, v: Option<[usize; 2]>) {
put_opt(f, &format!("{base}_h"), v.map(|x| x[0]));
put_opt(f, &format!("{base}_w"), v.map(|x| x[1]));
}
fn put_opt_triple(f: &mut ParamFields, base: &str, v: Option<[usize; 3]>) {
put_opt(f, &format!("{base}_d"), v.map(|x| x[0]));
put_opt(f, &format!("{base}_h"), v.map(|x| x[1]));
put_opt(f, &format!("{base}_w"), v.map(|x| x[2]));
}
const UPSAMPLE_MAX_SIZE_LEN: usize = 3;
const UNFLATTEN_MAX_SIZE_LEN: usize = 8;
fn upsample_mode_to_wire(mode: InterpolateMode) -> Option<(&'static str, bool)> {
match mode {
InterpolateMode::Nearest => Some(("nearest", false)),
InterpolateMode::NearestExact => Some(("nearest_exact", false)),
InterpolateMode::Area => Some(("area", false)),
InterpolateMode::Linear { align_corners } => Some(("linear", align_corners)),
InterpolateMode::Bilinear { align_corners } => Some(("bilinear", align_corners)),
InterpolateMode::Bicubic { align_corners } => Some(("bicubic", align_corners)),
InterpolateMode::Trilinear { align_corners } => Some(("trilinear", align_corners)),
_ => None,
}
}
fn upsample_mode_from_wire(name: &str, align_corners: bool) -> Option<InterpolateMode> {
match name {
"nearest" => Some(InterpolateMode::Nearest),
"nearest_exact" => Some(InterpolateMode::NearestExact),
"area" => Some(InterpolateMode::Area),
"linear" => Some(InterpolateMode::Linear { align_corners }),
"bilinear" => Some(InterpolateMode::Bilinear { align_corners }),
"bicubic" => Some(InterpolateMode::Bicubic { align_corners }),
"trilinear" => Some(InterpolateMode::Trilinear { align_corners }),
_ => None,
}
}
fn global_pool_mode_to_wire(mode: GlobalPoolMode) -> Option<&'static str> {
match mode {
GlobalPoolMode::Avg => Some("avg"),
GlobalPoolMode::Max => Some("max"),
_ => None,
}
}
fn global_pool_mode_from_wire(name: &str) -> Option<GlobalPoolMode> {
match name {
"avg" => Some(GlobalPoolMode::Avg),
"max" => Some(GlobalPoolMode::Max),
_ => None,
}
}
fn embedding_bag_mode_to_wire(mode: EmbeddingBagMode) -> Option<&'static str> {
match mode {
EmbeddingBagMode::Sum => Some("sum"),
EmbeddingBagMode::Mean => Some("mean"),
EmbeddingBagMode::Max => Some("max"),
_ => None,
}
}
fn embedding_bag_mode_from_wire(name: &str) -> Option<EmbeddingBagMode> {
match name {
"sum" => Some(EmbeddingBagMode::Sum),
"mean" => Some(EmbeddingBagMode::Mean),
"max" => Some(EmbeddingBagMode::Max),
_ => None,
}
}
fn upsample_is_savable(size: &[usize], mode: InterpolateMode) -> bool {
(1..=UPSAMPLE_MAX_SIZE_LEN).contains(&size.len()) && upsample_mode_to_wire(mode).is_some()
}
fn render_params(spec: &LayerSpec) -> String {
render_fields(¶m_fields(spec))
}
fn param_fields(spec: &LayerSpec) -> ParamFields {
if let LayerSpec::Reparam { base, reparam } = spec {
let mut f = param_fields(base);
match reparam {
ReparamSpec::WeightNorm { dim } => put_opt(&mut f, "dim", *dim),
ReparamSpec::SpectralNorm {
n_power_iterations,
eps,
} => f.push((
"spectral".to_string(),
format!("\"{n_power_iterations}:{eps:?}\""),
)),
}
return f;
}
let mut f: ParamFields = Vec::new();
match spec {
LayerSpec::Linear {
in_features,
out_features,
} => {
put_num(&mut f, "in_features", *in_features);
put_num(&mut f, "out_features", *out_features);
}
LayerSpec::LeakyRelu { negative_slope } => {
put_real(&mut f, "negative_slope", *negative_slope)
}
LayerSpec::Elu { alpha } => put_real(&mut f, "alpha", *alpha),
LayerSpec::Celu { alpha } => put_real(&mut f, "alpha", *alpha),
LayerSpec::Softmin { dim } => put_num(&mut f, "dim", *dim),
LayerSpec::Threshold { threshold, value } => {
put_real(&mut f, "threshold", *threshold);
put_real(&mut f, "value", *value);
}
LayerSpec::RRelu { lower, upper } => {
put_real(&mut f, "lower", *lower);
put_real(&mut f, "upper", *upper);
}
LayerSpec::Softmax { dim } | LayerSpec::LogSoftmax { dim } => put_num(&mut f, "dim", *dim),
LayerSpec::Softplus { beta, threshold } => {
put_real(&mut f, "beta", *beta);
put_real(&mut f, "threshold", *threshold);
}
LayerSpec::Hardtanh { min_val, max_val } => {
put_real(&mut f, "min_val", *min_val);
put_real(&mut f, "max_val", *max_val);
}
LayerSpec::Glu { dim } => put_num(&mut f, "dim", *dim),
LayerSpec::PRelu { num_parameters } => put_num(&mut f, "num_parameters", *num_parameters),
LayerSpec::Flatten { start_dim, end_dim } => {
put_num(&mut f, "start_dim", *start_dim);
put_num(&mut f, "end_dim", *end_dim);
}
LayerSpec::Dropout { p } | LayerSpec::Dropout2d { p } | LayerSpec::AlphaDropout { p } => {
put_real(&mut f, "p", *p)
}
LayerSpec::Conv2d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
dilation,
groups,
} => {
put_num(&mut f, "in_channels", *in_channels);
put_num(&mut f, "out_channels", *out_channels);
put_pair(&mut f, "kernel_size", *kernel_size);
put_pair(&mut f, "stride", *stride);
put_pair(&mut f, "padding", *padding);
put_pair(&mut f, "dilation", *dilation);
put_num(&mut f, "groups", *groups);
}
LayerSpec::Conv3d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
dilation,
groups,
} => {
put_num(&mut f, "in_channels", *in_channels);
put_num(&mut f, "out_channels", *out_channels);
put_triple(&mut f, "kernel_size", *kernel_size);
put_triple(&mut f, "stride", *stride);
put_triple(&mut f, "padding", *padding);
put_triple(&mut f, "dilation", *dilation);
put_num(&mut f, "groups", *groups);
}
LayerSpec::ConvTranspose2d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
output_padding,
dilation,
groups,
} => {
put_num(&mut f, "in_channels", *in_channels);
put_num(&mut f, "out_channels", *out_channels);
put_pair(&mut f, "kernel_size", *kernel_size);
put_pair(&mut f, "stride", *stride);
put_pair(&mut f, "padding", *padding);
put_pair(&mut f, "output_padding", *output_padding);
put_pair(&mut f, "dilation", *dilation);
put_num(&mut f, "groups", *groups);
}
LayerSpec::ConvTranspose3d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
output_padding,
dilation,
groups,
} => {
put_num(&mut f, "in_channels", *in_channels);
put_num(&mut f, "out_channels", *out_channels);
put_triple_str(&mut f, "kernel_size", *kernel_size);
put_triple_str(&mut f, "stride", *stride);
put_triple_str(&mut f, "padding", *padding);
put_triple_str(&mut f, "output_padding", *output_padding);
put_triple_str(&mut f, "dilation", *dilation);
put_num(&mut f, "groups", *groups);
}
LayerSpec::Conv1d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
dilation,
groups,
} => {
put_num(&mut f, "in_channels", *in_channels);
put_num(&mut f, "out_channels", *out_channels);
put_num(&mut f, "kernel_size", *kernel_size);
put_num(&mut f, "stride", *stride);
put_num(&mut f, "padding", *padding);
put_num(&mut f, "dilation", *dilation);
put_num(&mut f, "groups", *groups);
}
LayerSpec::ConvTranspose1d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
output_padding,
dilation,
groups,
} => {
put_num(&mut f, "in_channels", *in_channels);
put_num(&mut f, "out_channels", *out_channels);
put_num(&mut f, "kernel_size", *kernel_size);
put_num(&mut f, "stride", *stride);
put_num(&mut f, "padding", *padding);
put_num(&mut f, "output_padding", *output_padding);
put_num(&mut f, "dilation", *dilation);
put_num(&mut f, "groups", *groups);
}
LayerSpec::Unflatten { dim, sizes } => {
put_num(&mut f, "dim", *dim);
put_num(&mut f, "size_len", sizes.len());
for i in 0..UNFLATTEN_MAX_SIZE_LEN {
put_opt(&mut f, &format!("size_{i}"), sizes.get(i).copied());
}
}
LayerSpec::LayerNorm {
normalized_size,
eps,
}
| LayerSpec::RmsNorm {
normalized_size,
eps,
} => {
put_num(&mut f, "normalized_size", *normalized_size);
put_real(&mut f, "eps", *eps);
}
LayerSpec::BatchNorm1d {
num_features,
eps,
momentum,
}
| LayerSpec::BatchNorm2d {
num_features,
eps,
momentum,
} => {
put_num(&mut f, "num_features", *num_features);
put_real(&mut f, "eps", *eps);
put_real(&mut f, "momentum", *momentum);
}
LayerSpec::Embedding {
num_embeddings,
embedding_dim,
padding_idx,
} => {
put_num(&mut f, "num_embeddings", *num_embeddings);
put_num(&mut f, "embedding_dim", *embedding_dim);
put_opt(&mut f, "padding_idx", *padding_idx);
}
LayerSpec::EmbeddingBag {
num_embeddings,
embedding_dim,
mode,
padding_idx,
} => {
let name = embedding_bag_mode_to_wire(*mode).unwrap_or("");
put_num(&mut f, "num_embeddings", *num_embeddings);
put_num(&mut f, "embedding_dim", *embedding_dim);
f.push(("mode".to_string(), format!("\"{name}\"")));
put_opt(&mut f, "padding_idx", *padding_idx);
}
LayerSpec::MultiheadAttention {
embed_dim,
num_heads,
} => {
put_num(&mut f, "embed_dim", *embed_dim);
put_num(&mut f, "num_heads", *num_heads);
}
LayerSpec::MultiheadAttentionConfig {
embed_dim,
num_heads,
bias,
batch_first,
} => {
put_num(&mut f, "embed_dim", *embed_dim);
put_num(&mut f, "num_heads", *num_heads);
put_bool(&mut f, "bias", *bias);
put_bool(&mut f, "batch_first", *batch_first);
}
LayerSpec::TransformerEncoder {
d_model,
num_heads,
dim_feedforward,
} => {
put_num(&mut f, "d_model", *d_model);
put_num(&mut f, "num_heads", *num_heads);
put_num(&mut f, "dim_feedforward", *dim_feedforward);
}
LayerSpec::TransformerDecoderLayer {
d_model,
num_heads,
dim_feedforward,
} => {
put_num(&mut f, "d_model", *d_model);
put_num(&mut f, "num_heads", *num_heads);
put_num(&mut f, "dim_feedforward", *dim_feedforward);
}
LayerSpec::Transformer {
d_model,
num_heads,
num_encoder_layers,
num_decoder_layers,
dim_feedforward,
eps,
} => {
put_num(&mut f, "d_model", *d_model);
put_num(&mut f, "num_heads", *num_heads);
put_num(&mut f, "num_encoder_layers", *num_encoder_layers);
put_num(&mut f, "num_decoder_layers", *num_decoder_layers);
put_num(&mut f, "dim_feedforward", *dim_feedforward);
put_real(&mut f, "eps", *eps);
}
LayerSpec::MaxPool2d {
kernel_size,
stride,
padding,
dilation,
} => {
put_pair(&mut f, "kernel_size", *kernel_size);
put_opt_pair(&mut f, "stride", *stride);
put_pair(&mut f, "padding", *padding);
put_pair(&mut f, "dilation", *dilation);
}
LayerSpec::Unfold {
kernel_size,
stride,
padding,
dilation,
} => {
put_pair(&mut f, "kernel_size", *kernel_size);
put_pair(&mut f, "stride", *stride);
put_pair(&mut f, "padding", *padding);
put_pair(&mut f, "dilation", *dilation);
}
LayerSpec::Fold {
output_size,
kernel_size,
stride,
padding,
dilation,
} => {
put_pair(&mut f, "output_size", *output_size);
put_pair(&mut f, "kernel_size", *kernel_size);
put_pair(&mut f, "stride", *stride);
put_pair(&mut f, "padding", *padding);
put_pair(&mut f, "dilation", *dilation);
}
LayerSpec::LocalResponseNorm {
size,
alpha,
beta,
k,
} => {
put_num(&mut f, "size", *size);
put_real(&mut f, "alpha", *alpha);
put_real(&mut f, "beta", *beta);
put_real(&mut f, "k", *k);
}
LayerSpec::MaxPool3d {
kernel_size,
stride,
padding,
dilation,
} => {
put_triple(&mut f, "kernel_size", *kernel_size);
put_opt_triple(&mut f, "stride", *stride);
put_triple(&mut f, "padding", *padding);
put_triple(&mut f, "dilation", *dilation);
}
LayerSpec::AvgPool3d {
kernel_size,
stride,
padding,
count_include_pad,
} => {
put_triple(&mut f, "kernel_size", *kernel_size);
put_opt_triple(&mut f, "stride", *stride);
put_triple(&mut f, "padding", *padding);
put_bool(&mut f, "count_include_pad", *count_include_pad);
}
LayerSpec::MaxPool1d {
kernel_size,
stride,
padding,
dilation,
} => {
put_num(&mut f, "kernel_size", *kernel_size);
put_opt(&mut f, "stride", *stride);
put_num(&mut f, "padding", *padding);
put_num(&mut f, "dilation", *dilation);
}
LayerSpec::AvgPool2d {
kernel_size,
stride,
padding,
count_include_pad,
} => {
put_pair(&mut f, "kernel_size", *kernel_size);
put_opt_pair(&mut f, "stride", *stride);
put_pair(&mut f, "padding", *padding);
put_bool(&mut f, "count_include_pad", *count_include_pad);
}
LayerSpec::AvgPool1d {
kernel_size,
stride,
padding,
count_include_pad,
} => {
put_num(&mut f, "kernel_size", *kernel_size);
put_opt(&mut f, "stride", *stride);
put_num(&mut f, "padding", *padding);
put_bool(&mut f, "count_include_pad", *count_include_pad);
}
LayerSpec::AdaptiveAvgPool2d { output_size } => {
put_pair(&mut f, "output_size", *output_size)
}
LayerSpec::AdaptiveAvgPool1d { output_size } => {
put_num(&mut f, "output_size", *output_size)
}
LayerSpec::AdaptiveMaxPool2d { output_size } => {
put_pair(&mut f, "output_size", *output_size)
}
LayerSpec::AdaptiveMaxPool1d { output_size } => {
put_num(&mut f, "output_size", *output_size)
}
LayerSpec::GlobalPool { mode, keepdims } => {
let name = global_pool_mode_to_wire(*mode).unwrap_or("");
f.push(("mode".to_string(), format!("\"{name}\"")));
put_bool(&mut f, "keepdims", *keepdims);
}
LayerSpec::Upsample { size, mode } => {
let (name, align_corners) = upsample_mode_to_wire(*mode).unwrap_or(("", false));
f.push(("mode".to_string(), format!("\"{name}\"")));
put_bool(&mut f, "align_corners", align_corners);
put_num(&mut f, "size_len", size.len());
for i in 0..UPSAMPLE_MAX_SIZE_LEN {
put_opt(&mut f, &format!("size_{i}"), size.get(i).copied());
}
}
LayerSpec::GroupNorm { groups, eps } => {
put_num(&mut f, "groups", *groups);
put_real(&mut f, "eps", *eps);
}
LayerSpec::InstanceNorm { eps } => put_real(&mut f, "eps", *eps),
LayerSpec::PixelShuffle { upscale_factor } => {
put_num(&mut f, "upscale_factor", *upscale_factor)
}
LayerSpec::PixelUnshuffle { downscale_factor } => {
put_num(&mut f, "downscale_factor", *downscale_factor)
}
LayerSpec::ZeroPad2d { padding } => {
put_num(&mut f, "left", padding[0]);
put_num(&mut f, "right", padding[1]);
put_num(&mut f, "top", padding[2]);
put_num(&mut f, "bottom", padding[3]);
}
LayerSpec::Relu
| LayerSpec::Sigmoid
| LayerSpec::Tanh
| LayerSpec::Silu
| LayerSpec::Hardswish
| LayerSpec::Gelu
| LayerSpec::GeluTanh
| LayerSpec::Mish
| LayerSpec::Selu
| LayerSpec::Softsign
| LayerSpec::Hardsigmoid
| LayerSpec::LogSigmoid
| LayerSpec::Tanhshrink
| LayerSpec::Relu6
| LayerSpec::Identity
| LayerSpec::Reparam { .. }
| LayerSpec::Unsupported { .. } => {}
}
f
}
fn render_fields(f: &ParamFields) -> String {
let body: Vec<String> = f.iter().map(|(k, v)| format!("\"{k}\":{v}")).collect();
format!("{{{}}}", body.join(","))
}
fn is_valid_safetensors_file_name(name: &str) -> bool {
let Some(rest) = name.strip_prefix("model.") else {
return false;
};
let Some(generation) = rest.strip_suffix(".safetensors") else {
return false;
};
generation.len() == 32
&& generation
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
struct PreparedSave {
training: bool,
specs: Vec<LayerSpec>,
parameter_keys: Vec<(String, Vec<usize>)>,
compiled: Option<CompiledMeta>,
buffer_keys: Vec<(String, Vec<usize>)>,
safetensors: Vec<u8>,
}
fn prepare_save(model: &Sequential) -> Result<PreparedSave, ModelIoError> {
let snapshot = model.snapshot_compiled().map_err(ModelIoError::Autodiff)?;
let compiled = snapshot
.as_ref()
.map(CompiledMeta::from_snapshot)
.transpose()?;
let specs = model.specs();
let CheckedState {
mut state,
expected,
expected_buffers,
} = collect_checked_state(model)?;
if let Some(snap) = snapshot {
check_slot_shapes(&snap.optimizer_state, &model.trainable_parameters())?;
model
.check_lbfgs_restorable(&snap)
.map_err(|e| ModelIoError::Mismatch {
message: format!(
"Lbfgs の状態を復元できない構成です({})",
clip(&e.to_string())
),
})?;
for (key, tensor) in snap.optimizer_state {
if state
.insert(format!("{OPTIMIZER_PREFIX}{key}"), tensor)
.is_some()
{
return Err(ModelIoError::UnsupportedModel {
reason: "optimizer 状態のキーがパラメータのキーと衝突しました".into(),
});
}
}
}
let safetensors = save_safetensors_f32_to_bytes(&state, None)
.map_err(|e| ModelIoError::Safetensors(e.to_string()))?;
if safetensors.len() as u64 > MAX_MODEL_FILE_BYTES {
return Err(ModelIoError::TooLarge {
what: "model safetensors",
limit: MAX_MODEL_FILE_BYTES,
});
}
let prepared = PreparedSave {
training: model.training(),
specs: specs.to_vec(),
parameter_keys: expected,
compiled,
buffer_keys: expected_buffers,
safetensors,
};
verify_round_trip(&prepared)?;
Ok(prepared)
}
struct CheckedState {
state: std::collections::HashMap<String, Tensor<f32>>,
expected: Vec<(String, Vec<usize>)>,
expected_buffers: Vec<(String, Vec<usize>)>,
}
fn collect_checked_state(model: &Sequential) -> Result<CheckedState, ModelIoError> {
let specs = model.specs();
if specs.len() != model.layers().len() {
return Err(ModelIoError::UnsupportedModel {
reason: "層構成の記録と実際の層数が一致しません".into(),
});
}
if specs.len() > MAX_LAYERS {
return Err(ModelIoError::TooLarge {
what: "層数",
limit: MAX_LAYERS as u64,
});
}
for (i, spec) in specs.iter().enumerate() {
if let LayerSpec::Unsupported { kind } = spec {
return Err(ModelIoError::UnsupportedModel {
reason: format!("層 {i}({kind})は保存に未対応です"),
});
}
if !spec_f32_fields_are_finite(spec) {
return Err(ModelIoError::UnsupportedModel {
reason: format!("層 {i} の f32 引数が有限ではないため保存できません"),
});
}
if let LayerSpec::Reparam {
base,
reparam:
ReparamSpec::SpectralNorm {
n_power_iterations, ..
},
} = spec
{
if !(1..=MAX_SPECTRAL_POWER_ITERATIONS).contains(n_power_iterations) {
return Err(ModelIoError::UnsupportedModel {
reason: format!(
"層 {i}(spectral_norm)の n_power_iterations({n_power_iterations})が 1..={MAX_SPECTRAL_POWER_ITERATIONS} の範囲外のため保存できません"
),
});
}
if spectral_uv_lens(base).is_none() {
return Err(ModelIoError::UnsupportedModel {
reason: format!(
"層 {i}(spectral_norm)の weight 形状から u/v の長さを導けません"
),
});
}
}
if let LayerSpec::Upsample { size, mode } = spec
&& !upsample_is_savable(size, *mode)
{
return Err(ModelIoError::UnsupportedModel {
reason: format!(
"層 {i}(upsample)の size 軸数({})または mode が保存に未対応です",
size.len()
),
});
}
if let LayerSpec::Unflatten { sizes, .. } = spec
&& !(1..=UNFLATTEN_MAX_SIZE_LEN).contains(&sizes.len())
{
return Err(ModelIoError::UnsupportedModel {
reason: format!(
"層 {i}(unflatten)の sizes 軸数({})が 1..={UNFLATTEN_MAX_SIZE_LEN} の範囲外のため保存できません",
sizes.len()
),
});
}
if let LayerSpec::GlobalPool { mode, .. } = spec
&& global_pool_mode_to_wire(*mode).is_none()
{
return Err(ModelIoError::UnsupportedModel {
reason: format!("層 {i}(global_pool)の mode が保存に未対応です"),
});
}
if let LayerSpec::EmbeddingBag { mode, .. } = spec
&& embedding_bag_mode_to_wire(*mode).is_none()
{
return Err(ModelIoError::UnsupportedModel {
reason: format!("層 {i}(embedding_bag)の mode が保存に未対応です"),
});
}
check_layer_state(model, i, spec)?;
}
let expected = expected_parameter_keys(specs);
let expected_buffers = expected_buffer_keys(specs);
let mut state = model.state_dict();
let consistent = state.len() == expected.len()
&& expected
.iter()
.all(|(k, shape)| state.get(k).is_some_and(|t| t.shape() == shape.as_slice()));
if !consistent {
return Err(ModelIoError::UnsupportedModel {
reason: "state_dict のキー・shape が層構成から導いた期待と一致しません".into(),
});
}
for (i, spec) in specs.iter().enumerate() {
let (mean, var) = match spec {
LayerSpec::BatchNorm1d { .. } => {
let bn = model.layers()[i]
.as_batch_norm1d()
.ok_or_else(|| bn_downcast_error(i))?;
(bn.running_mean(), bn.running_var())
}
LayerSpec::BatchNorm2d { .. } => {
let bn = model.layers()[i]
.as_batch_norm2d()
.ok_or_else(|| bn_downcast_error(i))?;
(bn.running_mean(), bn.running_var())
}
_ => continue,
};
for (name, t) in [("running_mean", mean), ("running_var", var)] {
let key = format!("{i}.{name}");
if state.insert(key.clone(), t).is_some() {
return Err(ModelIoError::UnsupportedModel {
reason: format!("buffer キー {key} が parameter キーと衝突します"),
});
}
}
}
for (i, spec) in specs.iter().enumerate() {
let LayerSpec::Reparam {
reparam: ReparamSpec::SpectralNorm { .. },
..
} = spec
else {
continue;
};
let st = spectral_state_of(model.layers()[i].as_ref()).ok_or_else(|| {
ModelIoError::UnsupportedModel {
reason: format!(
"層 {i} を spectral_norm 適用済みの層として取り出せないため u/v を保存できません"
),
}
})?;
for (name, vec) in [(SPECTRAL_U_NAME, st.u()), (SPECTRAL_V_NAME, st.v())] {
if vec.iter().any(|x| !x.is_finite()) {
return Err(ModelIoError::UnsupportedModel {
reason: format!(
"層 {i} の spectral_norm の {name} に非有限値が含まれるため保存できません"
),
});
}
let t = Tensor::new(vec.to_vec(), &[vec.len()]).map_err(|e| {
ModelIoError::UnsupportedModel {
reason: format!("層 {i} の spectral_norm の {name} を構築できません({e})"),
}
})?;
let key = format!("{i}.{name}");
if state.insert(key.clone(), t).is_some() {
return Err(ModelIoError::UnsupportedModel {
reason: format!("buffer キー {key} が既存のキーと衝突します"),
});
}
}
}
let buffers_consistent = expected_buffers
.iter()
.all(|(k, shape)| state.get(k).is_some_and(|t| t.shape() == shape.as_slice()));
if !buffers_consistent || state.len() != expected.len() + expected_buffers.len() {
return Err(ModelIoError::UnsupportedModel {
reason: "buffer(BatchNorm の running stats・spectral_norm の u/v)の shape が層構成と一致しません".into(),
});
}
Ok(CheckedState {
state,
expected,
expected_buffers,
})
}
fn check_layer_state(model: &Sequential, i: usize, spec: &LayerSpec) -> Result<(), ModelIoError> {
let layer = &model.layers()[i];
let mode_dependent = matches!(
spec,
LayerSpec::Dropout { .. }
| LayerSpec::Dropout2d { .. }
| LayerSpec::AlphaDropout { .. }
| LayerSpec::RRelu { .. }
| LayerSpec::BatchNorm1d { .. }
| LayerSpec::BatchNorm2d { .. }
| LayerSpec::Reparam {
reparam: ReparamSpec::SpectralNorm { .. },
..
}
);
if mode_dependent && layer.training() != model.training() {
return Err(ModelIoError::UnsupportedModel {
reason: format!(
"層 {i} のモード(training)がモデル全体と異なります(load は全層をモデルのモードへ揃えるため復元後に出力がずれます)。`train()`/`eval()` で揃えてから保存してください"
),
});
}
Ok(())
}
fn bn_downcast_error(i: usize) -> ModelIoError {
ModelIoError::UnsupportedModel {
reason: format!(
"層 {i} を BatchNorm として取り出せないため running stats を保存できません"
),
}
}
fn verify_round_trip(prepared: &PreparedSave) -> Result<(), ModelIoError> {
let probe_name = format!("model.{}.safetensors", "0".repeat(32));
let text = render_manifest(prepared, &probe_name, prepared.safetensors.len() as u64);
if text.len() as u64 > MAX_MANIFEST_BYTES {
return Err(ModelIoError::TooLarge {
what: "manifest.json",
limit: MAX_MANIFEST_BYTES,
});
}
let parsed = match parse_manifest(text.as_bytes()) {
Ok(p) => p,
Err(e @ ModelIoError::TooLarge { .. }) => return Err(e),
Err(e) => {
return Err(ModelIoError::UnsupportedModel {
reason: format!("保存する構成を load が読み戻せません({e})"),
});
}
};
let same_specs = parsed.specs.len() == prepared.specs.len()
&& parsed
.specs
.iter()
.zip(&prepared.specs)
.all(|(a, b)| spec_kind(a) == spec_kind(b) && render_params(a) == render_params(b));
let same_compiled = parsed.compiled.as_ref().map(render_compiled)
== prepared.compiled.as_ref().map(render_compiled);
if parsed.training != prepared.training
|| parsed.parameter_keys != prepared.parameter_keys
|| parsed.buffer_keys != prepared.buffer_keys
|| !same_specs
|| !same_compiled
{
return Err(ModelIoError::UnsupportedModel {
reason: "保存する構成と読み戻した構成が一致しません".into(),
});
}
Ok(())
}
fn render_manifest(p: &PreparedSave, safetensors_file: &str, safetensors_bytes: u64) -> String {
let mut s = String::new();
s.push_str(&format!(
"{{\"format\":\"{FORMAT_NAME}\",\"format_version\":{FORMAT_VERSION},\"training\":{},\"num_layers\":{},\"layers\":[",
p.training,
p.specs.len()
));
for (i, spec) in p.specs.iter().enumerate() {
if i > 0 {
s.push(',');
}
let kind = spec_kind(spec).unwrap_or("unsupported");
let params = render_params(spec);
s.push_str(&format!(
"{{\"index\":{i},\"kind\":\"{kind}\",\"params\":{params}}}"
));
}
s.push_str("],\"parameter_keys\":");
s.push_str(&render_key_shapes(&p.parameter_keys));
s.push_str(",\"buffer_keys\":");
s.push_str(&render_key_shapes(&p.buffer_keys));
let compiled = p
.compiled
.as_ref()
.map_or_else(|| "null".to_string(), render_compiled);
s.push_str(&format!(
",\"safetensors_file\":\"{safetensors_file}\",\"safetensors_bytes\":{safetensors_bytes},\"compiled\":{compiled}}}"
));
s
}
fn render_key_shapes(keys: &[(String, Vec<usize>)]) -> String {
let items: Vec<String> = keys
.iter()
.map(|(key, shape)| {
let dims: Vec<String> = shape.iter().map(usize::to_string).collect();
format!("{{\"key\":\"{key}\",\"shape\":[{}]}}", dims.join(","))
})
.collect();
format!("[{}]", items.join(","))
}
#[cfg(not(unix))]
fn write_prepared(dir: &Path, prepared: &PreparedSave) -> Result<(), ModelIoError> {
let _ = (dir, prepared);
Err(ModelIoError::Io(io::Error::from(
io::ErrorKind::Unsupported,
)))
}
#[cfg(unix)]
fn write_prepared(dir: &Path, p: &PreparedSave) -> Result<(), ModelIoError> {
write_prepared_with(dir, p, generation_id, tmp_manifest_name)
}
#[cfg(unix)]
fn write_prepared_with(
dir: &Path,
p: &PreparedSave,
next_gen: impl FnMut() -> String,
next_tmp: impl FnMut() -> String,
) -> Result<(), ModelIoError> {
write_generation_with(
dir,
&p.safetensors,
|file, bytes| render_manifest(p, file, bytes),
next_gen,
next_tmp,
)
}
#[cfg(unix)]
fn write_generation_with(
dir: &Path,
safetensors: &[u8],
render: impl FnOnce(&str, u64) -> String,
mut next_gen: impl FnMut() -> String,
mut next_tmp: impl FnMut() -> String,
) -> Result<(), ModelIoError> {
use std::io::Write;
std::fs::create_dir_all(dir).map_err(ModelIoError::Io)?;
let (mut st_file, st_name) = create_new_with_retry(
dir,
|| format!("model.{}.safetensors", next_gen()),
MAX_TMP_NAME_ATTEMPTS,
)?;
st_file.write_all(safetensors).map_err(ModelIoError::Io)?;
let manifest = render(&st_name, safetensors.len() as u64);
let (mut tmp_file, tmp_name) =
create_new_with_retry(dir, &mut next_tmp, MAX_TMP_NAME_ATTEMPTS)?;
let tmp_path = dir.join(&tmp_name);
let committed = tmp_file
.write_all(manifest.as_bytes())
.and_then(|()| std::fs::rename(&tmp_path, dir.join(MANIFEST_FILE_NAME)));
if let Err(e) = committed {
remove_own_tmp(&tmp_file, &tmp_path);
return Err(ModelIoError::Io(e));
}
Ok(())
}
#[cfg(unix)]
fn create_new_with_retry(
dir: &Path,
next_name: impl FnMut() -> String,
max_attempts: usize,
) -> Result<(std::fs::File, String), ModelIoError> {
fs_write::create_new_with_retry(dir, next_name, max_attempts).map_err(ModelIoError::Io)
}
#[cfg(unix)]
fn generation_id() -> String {
use std::hash::{BuildHasher, Hasher};
let (pid, nanos, counter) = name_entropy();
let mut id = String::with_capacity(32);
for lane in 0..2u64 {
let mut hasher = std::collections::hash_map::RandomState::new().build_hasher();
hasher.write_u32(pid);
hasher.write_u128(nanos);
hasher.write_u64(counter);
hasher.write_u64(lane);
id.push_str(&format!("{:016x}", hasher.finish()));
}
id
}
#[cfg(unix)]
fn tmp_manifest_name() -> String {
let (pid, nanos, counter) = name_entropy();
format!(".manifest.json.tmp-{pid}-{counter}-{nanos}")
}
fn map_leaf_error(what: &'static str) -> impl Fn(LeafError) -> ModelIoError {
move |e| match e {
LeafError::Io(e) => ModelIoError::Io(e),
LeafError::TooLarge { limit } => ModelIoError::TooLarge { what, limit },
}
}
fn load_from_dir_with_limits(
dir: &Path,
max_manifest: u64,
max_model: u64,
) -> Result<Sequential, ModelIoError> {
let manifest_bytes = open_leaf_checked(&dir.join(MANIFEST_FILE_NAME), max_manifest)
.and_then(OpenedLeaf::read_exact_len)
.map_err(map_leaf_error("manifest.json"))?;
let manifest = parse_manifest(&manifest_bytes)?;
let LoadedTensors {
mut tensors,
optimizer_state,
} = read_checked_tensors(
dir,
max_model,
&manifest.safetensors_file,
manifest.safetensors_bytes,
&manifest.parameter_keys,
&manifest.buffer_keys,
manifest.compiled.as_ref(),
)?;
let mut buffers = std::collections::HashMap::new();
for (key, _) in &manifest.buffer_keys {
if let Some(t) = tensors.remove(key) {
buffers.insert(key.clone(), t);
}
}
if manifest.buffer_keys.is_empty() {
for (key, shape) in expected_buffer_keys(&manifest.specs) {
let n: usize = shape.iter().product();
let v = if key.ends_with(".running_var") {
1.0
} else {
0.0
};
let t = Tensor::new(vec![v; n], &shape).map_err(|e| ModelIoError::Mismatch {
message: format!("旧形式 BatchNorm の初期 running stats を構築できません({e})"),
})?;
buffers.insert(key, t);
}
}
let mut model = build_model(&manifest.specs, &mut buffers, &tensors)?;
model
.load_state_dict(tensors)
.map_err(ModelIoError::Autodiff)?;
model.set_training(manifest.training);
if let Some(meta) = manifest.compiled {
check_slot_shapes(&optimizer_state, &model.trainable_parameters())?;
model
.restore_compiled(meta.into_snapshot(optimizer_state))
.map_err(ModelIoError::Autodiff)?;
}
Ok(model)
}
struct LoadedTensors {
tensors: std::collections::HashMap<String, Tensor<f32>>,
optimizer_state: std::collections::HashMap<String, Tensor<f32>>,
}
fn read_checked_tensors(
dir: &Path,
max_model: u64,
safetensors_file: &str,
safetensors_bytes: u64,
parameter_keys: &[(String, Vec<usize>)],
buffer_keys: &[(String, Vec<usize>)],
compiled: Option<&CompiledMeta>,
) -> Result<LoadedTensors, ModelIoError> {
let opened = open_leaf_checked(&dir.join(safetensors_file), max_model)
.map_err(map_leaf_error("model safetensors"))?;
if opened.len() != safetensors_bytes {
return Err(ModelIoError::Mismatch {
message: format!(
"safetensors の実バイト数 {} が manifest の記載 {} と一致しません",
opened.len(),
safetensors_bytes
),
});
}
let bytes = opened
.read_exact_len()
.map_err(map_leaf_error("model safetensors"))?;
let mut tensors = load_safetensors_f32_from_bytes(&bytes)
.map_err(|e| ModelIoError::Safetensors(e.to_string()))?;
drop(bytes);
let (optimizer_full_keys, optimizer_state) = split_optimizer_tensors(&mut tensors);
let expected_optimizer_keys: &[String] = compiled.map_or(&[], |c| c.state_keys.as_slice());
if optimizer_full_keys != expected_optimizer_keys {
return Err(ModelIoError::Mismatch {
message: "safetensors の optimizer 状態のキー集合が manifest の optimizer_state_keys と一致しません"
.into(),
});
}
if let Some(meta) = compiled {
check_lbfgs_history(meta, &optimizer_state)?;
}
let consistent = tensors.len() == parameter_keys.len() + buffer_keys.len()
&& parameter_keys
.iter()
.chain(buffer_keys)
.all(|(key, shape)| {
tensors
.get(key)
.is_some_and(|t| t.shape() == shape.as_slice())
});
if !consistent {
return Err(ModelIoError::Mismatch {
message: "safetensors のキー集合または shape が manifest と一致しません".into(),
});
}
Ok(LoadedTensors {
tensors,
optimizer_state,
})
}
fn build_model(
specs: &[LayerSpec],
buffers: &mut std::collections::HashMap<String, Tensor<f32>>,
tensors: &std::collections::HashMap<String, Tensor<f32>>,
) -> Result<Sequential, ModelIoError> {
let mut model = Sequential::new();
for (i, spec) in specs.iter().enumerate() {
let (base_spec, reparam) = match spec {
LayerSpec::Reparam { base, reparam } => (base.as_ref(), Some(*reparam)),
other => (other, None),
};
model = match base_spec {
LayerSpec::Linear {
in_features,
out_features,
} => model
.add_linear(*in_features, *out_features, 0)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Relu => model.add_relu(),
LayerSpec::Sigmoid => model.add_sigmoid(),
LayerSpec::Tanh => model.add_tanh(),
LayerSpec::Silu => model.add_silu(),
LayerSpec::Hardswish => model.add_hardswish(),
LayerSpec::Gelu => model.add_gelu(),
LayerSpec::GeluTanh => model.add_gelu_tanh(),
LayerSpec::LeakyRelu { negative_slope } => model.add_leaky_relu(*negative_slope),
LayerSpec::Elu { alpha } => model.add_elu(*alpha),
LayerSpec::Softmax { dim } => model.add_softmax(*dim),
LayerSpec::LogSoftmax { dim } => model.add_log_softmax(*dim),
LayerSpec::Softplus { beta, threshold } => model
.add_softplus(*beta, *threshold)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Mish => model.add_mish(),
LayerSpec::Selu => model.add_selu(),
LayerSpec::Softsign => model.add_softsign(),
LayerSpec::Hardsigmoid => model.add_hardsigmoid(),
LayerSpec::LogSigmoid => model.add_log_sigmoid(),
LayerSpec::Tanhshrink => model.add_tanhshrink(),
LayerSpec::Celu { alpha } => model.add_celu(*alpha).map_err(ModelIoError::Autodiff)?,
LayerSpec::Softmin { dim } => model.add_softmin(*dim),
LayerSpec::Threshold { threshold, value } => model.add_threshold(*threshold, *value),
LayerSpec::RRelu { lower, upper } => model
.add_rrelu(*lower, *upper)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Hardtanh { min_val, max_val } => model
.add_hardtanh(*min_val, *max_val)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Relu6 => model.add_relu6(),
LayerSpec::Glu { dim } => model.add_glu(*dim),
LayerSpec::PRelu { num_parameters } => model
.add_prelu(*num_parameters, 0.25)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Flatten { start_dim, end_dim } => model.add_flatten(*start_dim, *end_dim),
LayerSpec::Dropout { p } => model.add_dropout(*p).map_err(ModelIoError::Autodiff)?,
LayerSpec::Dropout2d { p } => {
model.add_dropout2d(*p).map_err(ModelIoError::Autodiff)?
}
LayerSpec::AlphaDropout { p } => model
.add_alpha_dropout(*p)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Conv2d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
dilation,
groups,
} => model
.add_conv2d(
*in_channels,
*out_channels,
*kernel_size,
*stride,
*padding,
*dilation,
*groups,
0,
)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Conv3d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
dilation,
groups,
} => model
.add_conv3d(
*in_channels,
*out_channels,
*kernel_size,
*stride,
*padding,
*dilation,
*groups,
0,
)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::ConvTranspose2d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
output_padding,
dilation,
groups,
} => model
.add_conv_transpose2d(
*in_channels,
*out_channels,
*kernel_size,
*stride,
*padding,
*output_padding,
*dilation,
*groups,
0,
)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::ConvTranspose3d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
output_padding,
dilation,
groups,
} => model
.add_conv_transpose3d(
*in_channels,
*out_channels,
*kernel_size,
*stride,
*padding,
*output_padding,
*dilation,
*groups,
0,
)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Conv1d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
dilation,
groups,
} => model
.add_conv1d(
*in_channels,
*out_channels,
*kernel_size,
*stride,
*padding,
*dilation,
*groups,
0,
)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::ConvTranspose1d {
in_channels,
out_channels,
kernel_size,
stride,
padding,
output_padding,
dilation,
groups,
} => model
.add_conv_transpose1d(
*in_channels,
*out_channels,
*kernel_size,
*stride,
*padding,
*output_padding,
*dilation,
*groups,
0,
)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Unflatten { dim, sizes } => model
.add_unflatten(*dim, sizes.clone())
.map_err(ModelIoError::Autodiff)?,
LayerSpec::LayerNorm {
normalized_size,
eps,
} => model
.add_layer_norm(*normalized_size, *eps)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::RmsNorm {
normalized_size,
eps,
} => model
.add_rms_norm(*normalized_size, *eps)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::BatchNorm1d {
num_features,
eps,
momentum,
} => {
let (w, b, m, v) = take_bn_tensors(i, buffers, tensors)?;
let next = model
.add_batch_norm1d_restored(w, b, m, v, *eps, *momentum)
.map_err(ModelIoError::Autodiff)?;
check_restored_features(&next, i, *num_features)?;
next
}
LayerSpec::BatchNorm2d {
num_features,
eps,
momentum,
} => {
let (w, b, m, v) = take_bn_tensors(i, buffers, tensors)?;
let next = model
.add_batch_norm2d_restored(w, b, m, v, *eps, *momentum)
.map_err(ModelIoError::Autodiff)?;
check_restored_features(&next, i, *num_features)?;
next
}
LayerSpec::Embedding {
num_embeddings,
embedding_dim,
padding_idx,
} => model
.add_embedding(*num_embeddings, *embedding_dim, *padding_idx, 0)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::EmbeddingBag {
num_embeddings,
embedding_dim,
mode,
padding_idx,
} => model
.add_embedding_bag(*num_embeddings, *embedding_dim, *mode, *padding_idx, 0)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::MultiheadAttention {
embed_dim,
num_heads,
} => model
.add_multihead_attention(*embed_dim, *num_heads, 0)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::MultiheadAttentionConfig {
embed_dim,
num_heads,
bias,
batch_first,
} => model
.add_multihead_attention_with_config(
MultiheadAttentionConfig::new(*embed_dim, *num_heads)
.with_bias(*bias)
.with_batch_first(*batch_first),
0,
)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::TransformerEncoder {
d_model,
num_heads,
dim_feedforward,
} => model
.add_transformer_encoder(*d_model, *num_heads, *dim_feedforward, 0)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::TransformerDecoderLayer {
d_model,
num_heads,
dim_feedforward,
} => model
.add_transformer_decoder_layer(*d_model, *num_heads, *dim_feedforward, 0)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Transformer {
d_model,
num_heads,
num_encoder_layers,
num_decoder_layers,
dim_feedforward,
eps,
} => model
.add_transformer(
fandhe_ai_autodiff::nn::TransformerConfig::new(*d_model, *num_heads)
.with_num_encoder_layers(*num_encoder_layers)
.with_num_decoder_layers(*num_decoder_layers)
.with_dim_feedforward(*dim_feedforward)
.with_eps(*eps),
0,
)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::MaxPool2d {
kernel_size,
stride,
padding,
dilation,
} => model
.add_max_pool2d(*kernel_size, *stride, *padding, *dilation)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Unfold {
kernel_size,
stride,
padding,
dilation,
} => model
.add_unfold(*kernel_size, *stride, *padding, *dilation)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Fold {
output_size,
kernel_size,
stride,
padding,
dilation,
} => model
.add_fold(*output_size, *kernel_size, *stride, *padding, *dilation)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::LocalResponseNorm {
size,
alpha,
beta,
k,
} => model
.add_local_response_norm(*size, *alpha, *beta, *k)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::MaxPool3d {
kernel_size,
stride,
padding,
dilation,
} => model
.add_max_pool3d(*kernel_size, *stride, *padding, *dilation)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::AvgPool3d {
kernel_size,
stride,
padding,
count_include_pad,
} => model
.add_avg_pool3d(*kernel_size, *stride, *padding, *count_include_pad)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::MaxPool1d {
kernel_size,
stride,
padding,
dilation,
} => model
.add_max_pool1d(*kernel_size, *stride, *padding, *dilation)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::AvgPool2d {
kernel_size,
stride,
padding,
count_include_pad,
} => model
.add_avg_pool2d(*kernel_size, *stride, *padding, *count_include_pad)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::AvgPool1d {
kernel_size,
stride,
padding,
count_include_pad,
} => model
.add_avg_pool1d(*kernel_size, *stride, *padding, *count_include_pad)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::AdaptiveAvgPool2d { output_size } => model
.add_adaptive_avg_pool2d(*output_size)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::AdaptiveAvgPool1d { output_size } => model
.add_adaptive_avg_pool1d(*output_size)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::AdaptiveMaxPool2d { output_size } => model
.add_adaptive_max_pool2d(*output_size)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::AdaptiveMaxPool1d { output_size } => model
.add_adaptive_max_pool1d(*output_size)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::GlobalPool { mode, keepdims } => model.add_global_pool(*mode, *keepdims),
LayerSpec::Upsample { size, mode } => model
.add_upsample(size.clone(), *mode)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::ZeroPad2d { padding } => model.add_zero_pad2d(*padding),
LayerSpec::Identity => model.add_identity(),
LayerSpec::GroupNorm { groups, eps } => model
.add_group_norm(*groups, *eps)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::InstanceNorm { eps } => model
.add_instance_norm(*eps)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::PixelShuffle { upscale_factor } => model
.add_pixel_shuffle(*upscale_factor)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::PixelUnshuffle { downscale_factor } => model
.add_pixel_unshuffle(*downscale_factor)
.map_err(ModelIoError::Autodiff)?,
LayerSpec::Reparam { .. } => {
return Err(ModelIoError::UnsupportedModel {
reason: format!("層 {i} の再パラメータ化がネストしており復元できません"),
});
}
LayerSpec::Unsupported { kind } => {
return Err(ModelIoError::UnsupportedModel {
reason: format!("層 {kind} は復元に未対応です"),
});
}
};
match reparam {
None => {}
Some(ReparamSpec::WeightNorm { dim }) => {
model = model
.parametrize_weight_norm(i, dim)
.map_err(ModelIoError::Autodiff)?;
}
Some(ReparamSpec::SpectralNorm {
n_power_iterations,
eps,
}) => {
model = model
.parametrize_spectral_norm(i, n_power_iterations, eps, 0)
.map_err(ModelIoError::Autodiff)?;
let (u, v) = take_spectral_vectors(i, buffers)?;
model
.restore_spectral_norm_vectors(i, &u, &v)
.map_err(ModelIoError::Autodiff)?;
}
}
}
Ok(model)
}
fn take_spectral_vectors(
i: usize,
buffers: &mut std::collections::HashMap<String, Tensor<f32>>,
) -> Result<(Vec<f32>, Vec<f32>), ModelIoError> {
let mut take = |name: &str| -> Result<Vec<f32>, ModelIoError> {
let key = format!("{i}.{name}");
let t = buffers.remove(&key).ok_or_else(|| ModelIoError::Mismatch {
message: format!("spectral_norm の復元に必要なキー {key} が見つかりません"),
})?;
let t = t.contiguous();
Ok(t.host_slice().to_vec())
};
let u = take(SPECTRAL_U_NAME)?;
let v = take(SPECTRAL_V_NAME)?;
Ok((u, v))
}
type BnTensors = (Tensor<f32>, Tensor<f32>, Tensor<f32>, Tensor<f32>);
fn take_bn_tensors(
i: usize,
buffers: &mut std::collections::HashMap<String, Tensor<f32>>,
tensors: &std::collections::HashMap<String, Tensor<f32>>,
) -> Result<BnTensors, ModelIoError> {
let missing = |key: String| ModelIoError::Mismatch {
message: format!("BatchNorm の復元に必要なキー {key} が見つかりません"),
};
let mean = buffers
.remove(&format!("{i}.running_mean"))
.ok_or_else(|| missing(format!("{i}.running_mean")))?;
let var = buffers
.remove(&format!("{i}.running_var"))
.ok_or_else(|| missing(format!("{i}.running_var")))?;
let weight = tensors
.get(&format!("{i}.weight"))
.cloned()
.ok_or_else(|| missing(format!("{i}.weight")))?;
let bias = tensors
.get(&format!("{i}.bias"))
.cloned()
.ok_or_else(|| missing(format!("{i}.bias")))?;
Ok((weight, bias, mean, var))
}
fn check_restored_features(
model: &Sequential,
i: usize,
expected: usize,
) -> Result<(), ModelIoError> {
match model.specs().get(i) {
Some(LayerSpec::BatchNorm1d { num_features, .. })
| Some(LayerSpec::BatchNorm2d { num_features, .. })
if *num_features == expected =>
{
Ok(())
}
_ => Err(ModelIoError::Mismatch {
message: format!(
"層 {i} の BatchNorm の復元後の num_features が manifest と一致しません"
),
}),
}
}
struct ParsedManifest {
training: bool,
specs: Vec<LayerSpec>,
parameter_keys: Vec<(String, Vec<usize>)>,
buffer_keys: Vec<(String, Vec<usize>)>,
safetensors_file: String,
safetensors_bytes: u64,
compiled: Option<CompiledMeta>,
}
#[derive(Debug, Clone)]
enum Json {
Null,
Bool(bool),
Num(u64),
Real(String),
Str(String),
Arr(Vec<Json>),
Obj(Vec<(String, Json)>),
}
fn manifest_error(message: impl Into<String>) -> ModelIoError {
ModelIoError::Manifest {
message: message.into(),
}
}
fn clip(s: &str) -> String {
s.chars().take(64).collect()
}
struct Parser<'a> {
src: &'a [u8],
pos: usize,
}
impl<'a> Parser<'a> {
fn fail(&self, what: &str) -> ModelIoError {
manifest_error(format!(
"JSON 構文エラー: {what}(バイト位置 {})",
self.pos
))
}
fn peek(&self) -> Option<u8> {
self.src.get(self.pos).copied()
}
fn skip_ws(&mut self) {
while matches!(self.peek(), Some(b' ' | b'\n' | b'\r' | b'\t')) {
self.pos += 1;
}
}
fn expect(&mut self, byte: u8, what: &str) -> Result<(), ModelIoError> {
if self.peek() == Some(byte) {
self.pos += 1;
Ok(())
} else {
Err(self.fail(what))
}
}
fn document(mut self) -> Result<Json, ModelIoError> {
let value = self.value(0)?;
self.skip_ws();
if self.pos != self.src.len() {
return Err(self.fail("値の後ろに余分な内容があります"));
}
Ok(value)
}
fn value(&mut self, depth: usize) -> Result<Json, ModelIoError> {
self.skip_ws();
match self.peek() {
Some(b'{') => self.object(depth),
Some(b'[') => self.array(depth),
Some(b'"') => self.string().map(Json::Str),
Some(b'0'..=b'9' | b'-') => self.number(),
Some(b't') => self.literal(b"true", Json::Bool(true)),
Some(b'f') => self.literal(b"false", Json::Bool(false)),
Some(b'n') => self.literal(b"null", Json::Null),
_ => Err(self.fail("値として解釈できません")),
}
}
fn literal(&mut self, word: &[u8], value: Json) -> Result<Json, ModelIoError> {
if self.src[self.pos..].starts_with(word) {
self.pos += word.len();
Ok(value)
} else {
Err(self.fail("不正なリテラル"))
}
}
fn number(&mut self) -> Result<Json, ModelIoError> {
let start = self.pos;
let negative = self.peek() == Some(b'-');
if negative {
self.pos += 1;
}
match self.peek() {
Some(b'0') => {
self.pos += 1;
if matches!(self.peek(), Some(b'0'..=b'9')) {
return Err(self.fail("数値に先頭ゼロは使えません"));
}
}
Some(b'1'..=b'9') => self.skip_digits(),
_ => return Err(self.fail("数値の整数部が不正です")),
}
let mut is_real = negative;
if self.peek() == Some(b'.') {
self.pos += 1;
self.require_digits("小数点の後ろに数字がありません")?;
is_real = true;
}
if matches!(self.peek(), Some(b'e' | b'E')) {
self.pos += 1;
if matches!(self.peek(), Some(b'+' | b'-')) {
self.pos += 1;
}
self.require_digits("指数部に数字がありません")?;
is_real = true;
}
if self.pos - start > MAX_NUMBER_LEXEME_LEN {
return Err(self.fail("数値の字句が長すぎます"));
}
let lexeme = std::str::from_utf8(&self.src[start..self.pos])
.map_err(|_| self.fail("数値が UTF-8 ではありません"))?;
if is_real {
return Ok(Json::Real(lexeme.to_string()));
}
lexeme
.parse::<u64>()
.map(Json::Num)
.map_err(|_| self.fail("整数が u64 に収まりません"))
}
fn skip_digits(&mut self) {
while matches!(self.peek(), Some(b'0'..=b'9')) {
self.pos += 1;
}
}
fn require_digits(&mut self, what: &str) -> Result<(), ModelIoError> {
if !matches!(self.peek(), Some(b'0'..=b'9')) {
return Err(self.fail(what));
}
self.skip_digits();
Ok(())
}
fn string(&mut self) -> Result<String, ModelIoError> {
self.expect(b'"', "文字列の開始")?;
let start = self.pos;
loop {
match self.peek() {
Some(b'"') => break,
Some(b) if b < 0x20 || b == 0x7f || b == b'\\' => {
return Err(self.fail("文字列に制御文字またはエスケープは使えません"));
}
Some(_) => self.pos += 1,
None => return Err(self.fail("文字列が閉じていません")),
}
}
let text = std::str::from_utf8(&self.src[start..self.pos])
.map_err(|_| self.fail("文字列が UTF-8 ではありません"))?
.to_string();
self.pos += 1;
Ok(text)
}
fn array(&mut self, depth: usize) -> Result<Json, ModelIoError> {
if depth >= MAX_JSON_DEPTH {
return Err(self.fail("ネストが深すぎます"));
}
self.pos += 1;
let mut items = Vec::new();
self.skip_ws();
if self.peek() == Some(b']') {
self.pos += 1;
return Ok(Json::Arr(items));
}
loop {
if items.len() >= MAX_ARRAY_LEN {
return Err(ModelIoError::TooLarge {
what: "JSON 配列の要素数",
limit: MAX_ARRAY_LEN as u64,
});
}
items.push(self.value(depth + 1)?);
self.skip_ws();
match self.peek() {
Some(b',') => self.pos += 1,
Some(b']') => {
self.pos += 1;
return Ok(Json::Arr(items));
}
_ => return Err(self.fail("配列の区切りが不正です")),
}
}
}
fn object(&mut self, depth: usize) -> Result<Json, ModelIoError> {
if depth >= MAX_JSON_DEPTH {
return Err(self.fail("ネストが深すぎます"));
}
self.pos += 1;
let mut entries: Vec<(String, Json)> = Vec::new();
self.skip_ws();
if self.peek() == Some(b'}') {
self.pos += 1;
return Ok(Json::Obj(entries));
}
loop {
self.skip_ws();
let key = self.string()?;
if entries.iter().any(|(k, _)| *k == key) {
return Err(manifest_error(format!("重複キー {}", clip(&key))));
}
if entries.len() >= MAX_OBJECT_KEYS {
return Err(ModelIoError::TooLarge {
what: "JSON object のキー数",
limit: MAX_OBJECT_KEYS as u64,
});
}
self.skip_ws();
self.expect(b':', "キーの後ろの ':'")?;
let value = self.value(depth + 1)?;
entries.push((key, value));
self.skip_ws();
match self.peek() {
Some(b',') => self.pos += 1,
Some(b'}') => {
self.pos += 1;
return Ok(Json::Obj(entries));
}
_ => return Err(self.fail("object の区切りが不正です")),
}
}
}
}
fn exact_fields<'a>(
value: &'a Json,
ctx: &str,
keys: &[&str],
) -> Result<Vec<&'a Json>, ModelIoError> {
let Json::Obj(entries) = value else {
return Err(manifest_error(format!(
"{ctx} は object である必要があります"
)));
};
if let Some((unknown, _)) = entries.iter().find(|(k, _)| !keys.contains(&k.as_str())) {
return Err(manifest_error(format!(
"{ctx} に未知のキー {} があります",
clip(unknown)
)));
}
keys.iter()
.map(|k| {
entries
.iter()
.find(|(ek, _)| ek == k)
.map(|(_, v)| v)
.ok_or_else(|| manifest_error(format!("{ctx} にキー {k} がありません")))
})
.collect()
}
fn as_u64(value: &Json, ctx: &str) -> Result<u64, ModelIoError> {
match value {
Json::Num(n) => Ok(*n),
_ => Err(manifest_error(format!(
"{ctx} は非負整数である必要があります"
))),
}
}
fn as_usize(value: &Json, ctx: &str) -> Result<usize, ModelIoError> {
usize::try_from(as_u64(value, ctx)?)
.map_err(|_| manifest_error(format!("{ctx} が usize に収まりません")))
}
fn as_str<'a>(value: &'a Json, ctx: &str) -> Result<&'a str, ModelIoError> {
match value {
Json::Str(s) => Ok(s),
_ => Err(manifest_error(format!(
"{ctx} は文字列である必要があります"
))),
}
}
fn as_arr<'a>(value: &'a Json, ctx: &str) -> Result<&'a [Json], ModelIoError> {
match value {
Json::Arr(a) => Ok(a),
_ => Err(manifest_error(format!("{ctx} は配列である必要があります"))),
}
}
fn as_f32(value: &Json, ctx: &str) -> Result<f32, ModelIoError> {
let Json::Real(raw) = value else {
return Err(manifest_error(format!(
"{ctx} は小数表記の数値(f32)である必要があります"
)));
};
let v: f32 = raw
.parse()
.map_err(|_| manifest_error(format!("{ctx} を f32 として読めません")))?;
if !v.is_finite() {
return Err(manifest_error(format!("{ctx} が有限の f32 ではありません")));
}
if format!("{v:?}") != *raw {
return Err(manifest_error(format!(
"{ctx} が正準な f32 表記ではありません"
)));
}
Ok(v)
}
fn as_bool(value: &Json, ctx: &str) -> Result<bool, ModelIoError> {
match value {
Json::Bool(b) => Ok(*b),
_ => Err(manifest_error(format!(
"{ctx} は bool である必要があります"
))),
}
}
fn as_opt_usize(value: &Json, ctx: &str) -> Result<Option<usize>, ModelIoError> {
match value {
Json::Null => Ok(None),
other => as_usize(other, ctx).map(Some),
}
}
struct Params<'a> {
entries: &'a [(String, Json)],
}
impl<'a> Params<'a> {
fn new(params: &'a Json, keys: &[&str]) -> Result<Self, ModelIoError> {
Self::named("layers[].params", params, keys)
}
fn named(ctx: &str, params: &'a Json, keys: &[&str]) -> Result<Self, ModelIoError> {
exact_fields(params, ctx, keys)?;
match params {
Json::Obj(entries) => Ok(Params { entries }),
_ => Err(manifest_error(format!(
"{ctx} は object である必要があります"
))),
}
}
fn get(&self, key: &str) -> Result<&'a Json, ModelIoError> {
self.entries
.iter()
.find(|(k, _)| k == key)
.map(|(_, v)| v)
.ok_or_else(|| manifest_error(format!("params にキー {key} がありません")))
}
fn usize(&self, key: &str) -> Result<usize, ModelIoError> {
as_usize(self.get(key)?, key)
}
fn f32(&self, key: &str) -> Result<f32, ModelIoError> {
as_f32(self.get(key)?, key)
}
fn bool(&self, key: &str) -> Result<bool, ModelIoError> {
as_bool(self.get(key)?, key)
}
fn opt_usize(&self, key: &str) -> Result<Option<usize>, ModelIoError> {
as_opt_usize(self.get(key)?, key)
}
fn pair(&self, base: &str) -> Result<[usize; 2], ModelIoError> {
Ok([
self.usize(&format!("{base}_h"))?,
self.usize(&format!("{base}_w"))?,
])
}
fn triple(&self, base: &str) -> Result<[usize; 3], ModelIoError> {
Ok([
self.usize(&format!("{base}_d"))?,
self.usize(&format!("{base}_h"))?,
self.usize(&format!("{base}_w"))?,
])
}
fn opt_pair(&self, base: &str) -> Result<Option<[usize; 2]>, ModelIoError> {
let h = self.opt_usize(&format!("{base}_h"))?;
let w = self.opt_usize(&format!("{base}_w"))?;
match (h, w) {
(Some(h), Some(w)) => Ok(Some([h, w])),
(None, None) => Ok(None),
_ => Err(manifest_error(format!(
"{base}_h と {base}_w は両方 null か両方整数である必要があります"
))),
}
}
}
impl Params<'_> {
fn triple_str(&self, key: &str) -> Result<[usize; 3], ModelIoError> {
let raw = as_str(self.get(key)?, key)?;
let parts: Vec<&str> = raw.split(',').collect();
if parts.len() != 3 {
return Err(manifest_error(format!(
"{key} は \"d,h,w\" 形式である必要があります"
)));
}
let mut out = [0usize; 3];
for (slot, part) in out.iter_mut().zip(&parts) {
if part.is_empty() || !part.bytes().all(|b| b.is_ascii_digit()) {
return Err(manifest_error(format!(
"{key} の要素は非負 10 進整数である必要があります"
)));
}
*slot = part
.parse::<usize>()
.map_err(|_| manifest_error(format!("{key} が usize に収まりません")))?;
}
if format!("{},{},{}", out[0], out[1], out[2]) != raw {
return Err(manifest_error(format!("{key} が正準な表記ではありません")));
}
Ok(out)
}
fn opt_triple(&self, base: &str) -> Result<Option<[usize; 3]>, ModelIoError> {
let d = self.opt_usize(&format!("{base}_d"))?;
let h = self.opt_usize(&format!("{base}_h"))?;
let w = self.opt_usize(&format!("{base}_w"))?;
match (d, h, w) {
(Some(d), Some(h), Some(w)) => Ok(Some([d, h, w])),
(None, None, None) => Ok(None),
_ => Err(manifest_error(format!(
"{base}_d・{base}_h・{base}_w は全て null か全て整数である必要があります"
))),
}
}
}
fn check_conv_groups(
in_channels: usize,
out_channels: usize,
groups: usize,
) -> Result<(), ModelIoError> {
if groups == 0 || !in_channels.is_multiple_of(groups) || !out_channels.is_multiple_of(groups) {
return Err(manifest_error(
"conv の groups は 1 以上で in_channels・out_channels を割り切る必要があります",
));
}
Ok(())
}
fn reparam_spec_from_params(
base_kind: &str,
spectral: bool,
params: &Json,
) -> Result<LayerSpec, ModelIoError> {
let extra_key = if spectral { "spectral" } else { "dim" };
let Json::Obj(entries) = params else {
return Err(manifest_error(
"layers[].params は object である必要があります",
));
};
let extra = entries
.iter()
.find(|(k, _)| k == extra_key)
.map(|(_, v)| v)
.ok_or_else(|| manifest_error(format!("params にキー {extra_key} がありません")))?;
let base_json = Json::Obj(
entries
.iter()
.filter(|(k, _)| k != extra_key)
.cloned()
.collect(),
);
let base = spec_from_kind(base_kind, &base_json)?;
let reparam = if spectral {
let raw = as_str(extra, extra_key)?;
let canon_err = || manifest_error("spectral が正準な \"<n>:<eps>\" 表記ではありません");
let (n_raw, eps_raw) = raw.split_once(':').ok_or_else(canon_err)?;
if n_raw.is_empty() || !n_raw.bytes().all(|b| b.is_ascii_digit()) {
return Err(canon_err());
}
let n: usize = n_raw.parse().map_err(|_| canon_err())?;
let eps: f32 = eps_raw.parse().map_err(|_| canon_err())?;
if n.to_string() != n_raw || format!("{eps:?}") != eps_raw {
return Err(canon_err());
}
if !(1..=MAX_SPECTRAL_POWER_ITERATIONS).contains(&n) {
return Err(manifest_error(format!(
"spectral の n_power_iterations は 1..={MAX_SPECTRAL_POWER_ITERATIONS} の範囲である必要があります"
)));
}
if !eps.is_finite() || eps < 0.0 {
return Err(manifest_error(
"spectral の eps は有限かつ非負である必要があります",
));
}
if spectral_uv_lens(&base).is_none() {
return Err(manifest_error(
"spectral_norm の weight 形状から u/v の長さを導けません",
));
}
ReparamSpec::SpectralNorm {
n_power_iterations: n,
eps,
}
} else {
let dim = as_opt_usize(extra, extra_key)?;
let rank = layer_parameters(&base)
.into_iter()
.next()
.map_or(0, |(_, shape)| shape.len());
if let Some(d) = dim
&& d >= rank
{
return Err(manifest_error(format!(
"weight_norm の dim({d})が weight の rank({rank})以上です"
)));
}
ReparamSpec::WeightNorm { dim }
};
Ok(LayerSpec::Reparam {
base: Box::new(base),
reparam,
})
}
fn spec_from_kind(kind: &str, params: &Json) -> Result<LayerSpec, ModelIoError> {
let none = |spec: LayerSpec| -> Result<LayerSpec, ModelIoError> {
Params::new(params, &[])?;
Ok(spec)
};
if let Some((base_kind, spectral)) = reparam_kind_parts(kind) {
return reparam_spec_from_params(base_kind, spectral, params);
}
match kind {
"linear" => {
let p = Params::new(params, &["in_features", "out_features"])?;
Ok(LayerSpec::Linear {
in_features: p.usize("in_features")?,
out_features: p.usize("out_features")?,
})
}
"relu" => none(LayerSpec::Relu),
"sigmoid" => none(LayerSpec::Sigmoid),
"tanh" => none(LayerSpec::Tanh),
"silu" => none(LayerSpec::Silu),
"hardswish" => none(LayerSpec::Hardswish),
"gelu" => none(LayerSpec::Gelu),
"gelu_tanh" => none(LayerSpec::GeluTanh),
"leaky_relu" => {
let p = Params::new(params, &["negative_slope"])?;
Ok(LayerSpec::LeakyRelu {
negative_slope: p.f32("negative_slope")?,
})
}
"elu" => {
let p = Params::new(params, &["alpha"])?;
Ok(LayerSpec::Elu {
alpha: p.f32("alpha")?,
})
}
"celu" => {
let p = Params::new(params, &["alpha"])?;
Ok(LayerSpec::Celu {
alpha: p.f32("alpha")?,
})
}
"softmin" => {
let p = Params::new(params, &["dim"])?;
Ok(LayerSpec::Softmin {
dim: p.usize("dim")?,
})
}
"threshold" => {
let p = Params::new(params, &["threshold", "value"])?;
Ok(LayerSpec::Threshold {
threshold: p.f32("threshold")?,
value: p.f32("value")?,
})
}
"rrelu" => {
let p = Params::new(params, &["lower", "upper"])?;
Ok(LayerSpec::RRelu {
lower: p.f32("lower")?,
upper: p.f32("upper")?,
})
}
"softmax" => {
let p = Params::new(params, &["dim"])?;
Ok(LayerSpec::Softmax {
dim: p.usize("dim")?,
})
}
"log_softmax" => {
let p = Params::new(params, &["dim"])?;
Ok(LayerSpec::LogSoftmax {
dim: p.usize("dim")?,
})
}
"softplus" => {
let p = Params::new(params, &["beta", "threshold"])?;
Ok(LayerSpec::Softplus {
beta: p.f32("beta")?,
threshold: p.f32("threshold")?,
})
}
"mish" => none(LayerSpec::Mish),
"selu" => none(LayerSpec::Selu),
"softsign" => none(LayerSpec::Softsign),
"hardsigmoid" => none(LayerSpec::Hardsigmoid),
"log_sigmoid" => none(LayerSpec::LogSigmoid),
"tanhshrink" => none(LayerSpec::Tanhshrink),
"relu6" => none(LayerSpec::Relu6),
"hardtanh" => {
let p = Params::new(params, &["min_val", "max_val"])?;
Ok(LayerSpec::Hardtanh {
min_val: p.f32("min_val")?,
max_val: p.f32("max_val")?,
})
}
"glu" => {
let p = Params::new(params, &["dim"])?;
Ok(LayerSpec::Glu {
dim: p.usize("dim")?,
})
}
"prelu" => {
let p = Params::new(params, &["num_parameters"])?;
Ok(LayerSpec::PRelu {
num_parameters: p.usize("num_parameters")?,
})
}
"flatten" => {
let p = Params::new(params, &["start_dim", "end_dim"])?;
Ok(LayerSpec::Flatten {
start_dim: p.usize("start_dim")?,
end_dim: p.usize("end_dim")?,
})
}
"dropout" => {
let p = Params::new(params, &["p"])?;
Ok(LayerSpec::Dropout { p: p.f32("p")? })
}
"dropout2d" => {
let p = Params::new(params, &["p"])?;
Ok(LayerSpec::Dropout2d { p: p.f32("p")? })
}
"alpha_dropout" => {
let p = Params::new(params, &["p"])?;
Ok(LayerSpec::AlphaDropout { p: p.f32("p")? })
}
"conv2d" => {
let p = Params::new(
params,
&[
"in_channels",
"out_channels",
"kernel_size_h",
"kernel_size_w",
"stride_h",
"stride_w",
"padding_h",
"padding_w",
"dilation_h",
"dilation_w",
"groups",
],
)?;
let spec = LayerSpec::Conv2d {
in_channels: p.usize("in_channels")?,
out_channels: p.usize("out_channels")?,
kernel_size: p.pair("kernel_size")?,
stride: p.pair("stride")?,
padding: p.pair("padding")?,
dilation: p.pair("dilation")?,
groups: p.usize("groups")?,
};
if let LayerSpec::Conv2d {
in_channels,
out_channels,
groups,
..
} = &spec
{
check_conv_groups(*in_channels, *out_channels, *groups)?;
}
Ok(spec)
}
"conv3d" => {
let keys: Vec<String> = ["kernel_size", "stride", "padding", "dilation"]
.iter()
.flat_map(|b| ["d", "h", "w"].map(|a| format!("{b}_{a}")))
.chain(["in_channels", "out_channels", "groups"].map(String::from))
.collect();
let key_refs: Vec<&str> = keys.iter().map(String::as_str).collect();
let p = Params::new(params, &key_refs)?;
let in_channels = p.usize("in_channels")?;
let out_channels = p.usize("out_channels")?;
let groups = p.usize("groups")?;
check_conv_groups(in_channels, out_channels, groups)?;
Ok(LayerSpec::Conv3d {
in_channels,
out_channels,
kernel_size: p.triple("kernel_size")?,
stride: p.triple("stride")?,
padding: p.triple("padding")?,
dilation: p.triple("dilation")?,
groups,
})
}
"conv_transpose2d" => {
let p = Params::new(
params,
&[
"in_channels",
"out_channels",
"kernel_size_h",
"kernel_size_w",
"stride_h",
"stride_w",
"padding_h",
"padding_w",
"output_padding_h",
"output_padding_w",
"dilation_h",
"dilation_w",
"groups",
],
)?;
let in_channels = p.usize("in_channels")?;
let out_channels = p.usize("out_channels")?;
let groups = p.usize("groups")?;
check_conv_groups(in_channels, out_channels, groups)?;
Ok(LayerSpec::ConvTranspose2d {
in_channels,
out_channels,
kernel_size: p.pair("kernel_size")?,
stride: p.pair("stride")?,
padding: p.pair("padding")?,
output_padding: p.pair("output_padding")?,
dilation: p.pair("dilation")?,
groups,
})
}
"conv_transpose3d" => {
let p = Params::new(
params,
&[
"in_channels",
"out_channels",
"kernel_size",
"stride",
"padding",
"output_padding",
"dilation",
"groups",
],
)?;
let in_channels = p.usize("in_channels")?;
let out_channels = p.usize("out_channels")?;
let groups = p.usize("groups")?;
check_conv_groups(in_channels, out_channels, groups)?;
Ok(LayerSpec::ConvTranspose3d {
in_channels,
out_channels,
kernel_size: p.triple_str("kernel_size")?,
stride: p.triple_str("stride")?,
padding: p.triple_str("padding")?,
output_padding: p.triple_str("output_padding")?,
dilation: p.triple_str("dilation")?,
groups,
})
}
"conv1d" => {
let p = Params::new(
params,
&[
"in_channels",
"out_channels",
"kernel_size",
"stride",
"padding",
"dilation",
"groups",
],
)?;
let in_channels = p.usize("in_channels")?;
let out_channels = p.usize("out_channels")?;
let groups = p.usize("groups")?;
check_conv_groups(in_channels, out_channels, groups)?;
Ok(LayerSpec::Conv1d {
in_channels,
out_channels,
kernel_size: p.usize("kernel_size")?,
stride: p.usize("stride")?,
padding: p.usize("padding")?,
dilation: p.usize("dilation")?,
groups,
})
}
"conv_transpose1d" => {
let p = Params::new(
params,
&[
"in_channels",
"out_channels",
"kernel_size",
"stride",
"padding",
"output_padding",
"dilation",
"groups",
],
)?;
let in_channels = p.usize("in_channels")?;
let out_channels = p.usize("out_channels")?;
let groups = p.usize("groups")?;
check_conv_groups(in_channels, out_channels, groups)?;
Ok(LayerSpec::ConvTranspose1d {
in_channels,
out_channels,
kernel_size: p.usize("kernel_size")?,
stride: p.usize("stride")?,
padding: p.usize("padding")?,
output_padding: p.usize("output_padding")?,
dilation: p.usize("dilation")?,
groups,
})
}
"unflatten" => {
let p = Params::new(
params,
&[
"dim", "size_len", "size_0", "size_1", "size_2", "size_3", "size_4", "size_5",
"size_6", "size_7",
],
)?;
let dim = p.usize("dim")?;
let size_len = p.usize("size_len")?;
if !(1..=UNFLATTEN_MAX_SIZE_LEN).contains(&size_len) {
return Err(manifest_error(
"unflatten の size_len は 1..=8 である必要があります",
));
}
let mut sizes = Vec::with_capacity(size_len);
for i in 0..UNFLATTEN_MAX_SIZE_LEN {
let v = p.opt_usize(&format!("size_{i}"))?;
match (i < size_len, v) {
(true, Some(x)) => sizes.push(x),
(false, None) => {}
_ => {
return Err(manifest_error(
"unflatten の size_i は size_len 未満が整数、以上が null である必要があります",
));
}
}
}
Ok(LayerSpec::Unflatten { dim, sizes })
}
"layer_norm" => {
let p = Params::new(params, &["normalized_size", "eps"])?;
Ok(LayerSpec::LayerNorm {
normalized_size: p.usize("normalized_size")?,
eps: p.f32("eps")?,
})
}
"rms_norm" => {
let p = Params::new(params, &["normalized_size", "eps"])?;
Ok(LayerSpec::RmsNorm {
normalized_size: p.usize("normalized_size")?,
eps: p.f32("eps")?,
})
}
"group_norm" => {
let p = Params::new(params, &["groups", "eps"])?;
Ok(LayerSpec::GroupNorm {
groups: p.usize("groups")?,
eps: p.f32("eps")?,
})
}
"instance_norm" => {
let p = Params::new(params, &["eps"])?;
Ok(LayerSpec::InstanceNorm { eps: p.f32("eps")? })
}
"pixel_shuffle" => {
let p = Params::new(params, &["upscale_factor"])?;
Ok(LayerSpec::PixelShuffle {
upscale_factor: p.usize("upscale_factor")?,
})
}
"pixel_unshuffle" => {
let p = Params::new(params, &["downscale_factor"])?;
Ok(LayerSpec::PixelUnshuffle {
downscale_factor: p.usize("downscale_factor")?,
})
}
"batch_norm1d" => {
let p = Params::new(params, &["num_features", "eps", "momentum"])?;
Ok(LayerSpec::BatchNorm1d {
num_features: p.usize("num_features")?,
eps: p.f32("eps")?,
momentum: p.f32("momentum")?,
})
}
"batch_norm2d" => {
let p = Params::new(params, &["num_features", "eps", "momentum"])?;
Ok(LayerSpec::BatchNorm2d {
num_features: p.usize("num_features")?,
eps: p.f32("eps")?,
momentum: p.f32("momentum")?,
})
}
"embedding" => {
let p = Params::new(params, &["num_embeddings", "embedding_dim", "padding_idx"])?;
Ok(LayerSpec::Embedding {
num_embeddings: p.usize("num_embeddings")?,
embedding_dim: p.usize("embedding_dim")?,
padding_idx: p.opt_usize("padding_idx")?,
})
}
"embedding_bag" => {
let p = Params::new(
params,
&["num_embeddings", "embedding_dim", "mode", "padding_idx"],
)?;
let mode_name = as_str(p.get("mode")?, "mode")?;
let mode = embedding_bag_mode_from_wire(mode_name)
.ok_or_else(|| manifest_error("embedding_bag の mode が allowlist 外です"))?;
Ok(LayerSpec::EmbeddingBag {
num_embeddings: p.usize("num_embeddings")?,
embedding_dim: p.usize("embedding_dim")?,
mode,
padding_idx: p.opt_usize("padding_idx")?,
})
}
"multihead_attention" => {
let p = Params::new(params, &["embed_dim", "num_heads"])?;
Ok(LayerSpec::MultiheadAttention {
embed_dim: p.usize("embed_dim")?,
num_heads: p.usize("num_heads")?,
})
}
"multihead_attention_config" => {
let p = Params::new(params, &["embed_dim", "num_heads", "bias", "batch_first"])?;
Ok(LayerSpec::MultiheadAttentionConfig {
embed_dim: p.usize("embed_dim")?,
num_heads: p.usize("num_heads")?,
bias: p.bool("bias")?,
batch_first: p.bool("batch_first")?,
})
}
"transformer_encoder" => {
let p = Params::new(params, &["d_model", "num_heads", "dim_feedforward"])?;
Ok(LayerSpec::TransformerEncoder {
d_model: p.usize("d_model")?,
num_heads: p.usize("num_heads")?,
dim_feedforward: p.usize("dim_feedforward")?,
})
}
"transformer_decoder_layer" => {
let p = Params::new(params, &["d_model", "num_heads", "dim_feedforward"])?;
Ok(LayerSpec::TransformerDecoderLayer {
d_model: p.usize("d_model")?,
num_heads: p.usize("num_heads")?,
dim_feedforward: p.usize("dim_feedforward")?,
})
}
"transformer" => {
let p = Params::new(
params,
&[
"d_model",
"num_heads",
"num_encoder_layers",
"num_decoder_layers",
"dim_feedforward",
"eps",
],
)?;
let num_encoder_layers = p.usize("num_encoder_layers")?;
let num_decoder_layers = p.usize("num_decoder_layers")?;
if num_encoder_layers == 0 || num_decoder_layers == 0 {
return Err(manifest_error(
"transformer の層数は 1 以上である必要があります",
));
}
Ok(LayerSpec::Transformer {
d_model: p.usize("d_model")?,
num_heads: p.usize("num_heads")?,
num_encoder_layers,
num_decoder_layers,
dim_feedforward: p.usize("dim_feedforward")?,
eps: p.f32("eps")?,
})
}
"max_pool2d" => {
let p = Params::new(
params,
&[
"kernel_size_h",
"kernel_size_w",
"stride_h",
"stride_w",
"padding_h",
"padding_w",
"dilation_h",
"dilation_w",
],
)?;
Ok(LayerSpec::MaxPool2d {
kernel_size: p.pair("kernel_size")?,
stride: p.opt_pair("stride")?,
padding: p.pair("padding")?,
dilation: p.pair("dilation")?,
})
}
"unfold" => {
let p = Params::new(
params,
&[
"kernel_size_h",
"kernel_size_w",
"stride_h",
"stride_w",
"padding_h",
"padding_w",
"dilation_h",
"dilation_w",
],
)?;
Ok(LayerSpec::Unfold {
kernel_size: p.pair("kernel_size")?,
stride: p.pair("stride")?,
padding: p.pair("padding")?,
dilation: p.pair("dilation")?,
})
}
"fold" => {
let p = Params::new(
params,
&[
"output_size_h",
"output_size_w",
"kernel_size_h",
"kernel_size_w",
"stride_h",
"stride_w",
"padding_h",
"padding_w",
"dilation_h",
"dilation_w",
],
)?;
Ok(LayerSpec::Fold {
output_size: p.pair("output_size")?,
kernel_size: p.pair("kernel_size")?,
stride: p.pair("stride")?,
padding: p.pair("padding")?,
dilation: p.pair("dilation")?,
})
}
"local_response_norm" => {
let p = Params::new(params, &["size", "alpha", "beta", "k"])?;
Ok(LayerSpec::LocalResponseNorm {
size: p.usize("size")?,
alpha: p.f32("alpha")?,
beta: p.f32("beta")?,
k: p.f32("k")?,
})
}
"max_pool3d" => {
let p = Params::new(
params,
&[
"kernel_size_d",
"kernel_size_h",
"kernel_size_w",
"stride_d",
"stride_h",
"stride_w",
"padding_d",
"padding_h",
"padding_w",
"dilation_d",
"dilation_h",
"dilation_w",
],
)?;
Ok(LayerSpec::MaxPool3d {
kernel_size: p.triple("kernel_size")?,
stride: p.opt_triple("stride")?,
padding: p.triple("padding")?,
dilation: p.triple("dilation")?,
})
}
"avg_pool3d" => {
let p = Params::new(
params,
&[
"kernel_size_d",
"kernel_size_h",
"kernel_size_w",
"stride_d",
"stride_h",
"stride_w",
"padding_d",
"padding_h",
"padding_w",
"count_include_pad",
],
)?;
Ok(LayerSpec::AvgPool3d {
kernel_size: p.triple("kernel_size")?,
stride: p.opt_triple("stride")?,
padding: p.triple("padding")?,
count_include_pad: p.bool("count_include_pad")?,
})
}
"max_pool1d" => {
let p = Params::new(params, &["kernel_size", "stride", "padding", "dilation"])?;
Ok(LayerSpec::MaxPool1d {
kernel_size: p.usize("kernel_size")?,
stride: p.opt_usize("stride")?,
padding: p.usize("padding")?,
dilation: p.usize("dilation")?,
})
}
"avg_pool2d" => {
let p = Params::new(
params,
&[
"kernel_size_h",
"kernel_size_w",
"stride_h",
"stride_w",
"padding_h",
"padding_w",
"count_include_pad",
],
)?;
Ok(LayerSpec::AvgPool2d {
kernel_size: p.pair("kernel_size")?,
stride: p.opt_pair("stride")?,
padding: p.pair("padding")?,
count_include_pad: p.bool("count_include_pad")?,
})
}
"avg_pool1d" => {
let p = Params::new(
params,
&["kernel_size", "stride", "padding", "count_include_pad"],
)?;
Ok(LayerSpec::AvgPool1d {
kernel_size: p.usize("kernel_size")?,
stride: p.opt_usize("stride")?,
padding: p.usize("padding")?,
count_include_pad: p.bool("count_include_pad")?,
})
}
"adaptive_avg_pool2d" => {
let p = Params::new(params, &["output_size_h", "output_size_w"])?;
Ok(LayerSpec::AdaptiveAvgPool2d {
output_size: p.pair("output_size")?,
})
}
"adaptive_avg_pool1d" => {
let p = Params::new(params, &["output_size"])?;
Ok(LayerSpec::AdaptiveAvgPool1d {
output_size: p.usize("output_size")?,
})
}
"adaptive_max_pool2d" => {
let p = Params::new(params, &["output_size_h", "output_size_w"])?;
Ok(LayerSpec::AdaptiveMaxPool2d {
output_size: p.pair("output_size")?,
})
}
"adaptive_max_pool1d" => {
let p = Params::new(params, &["output_size"])?;
Ok(LayerSpec::AdaptiveMaxPool1d {
output_size: p.usize("output_size")?,
})
}
"global_pool" => {
let p = Params::new(params, &["mode", "keepdims"])?;
let mode_name = as_str(p.get("mode")?, "mode")?;
let mode = global_pool_mode_from_wire(mode_name)
.ok_or_else(|| manifest_error("global_pool の mode が allowlist 外です"))?;
Ok(LayerSpec::GlobalPool {
mode,
keepdims: p.bool("keepdims")?,
})
}
"identity" => {
Params::new(params, &[])?;
Ok(LayerSpec::Identity)
}
"zero_pad2d" => {
let p = Params::new(params, &["left", "right", "top", "bottom"])?;
Ok(LayerSpec::ZeroPad2d {
padding: [
p.usize("left")?,
p.usize("right")?,
p.usize("top")?,
p.usize("bottom")?,
],
})
}
"upsample" => {
let p = Params::new(
params,
&[
"mode",
"align_corners",
"size_len",
"size_0",
"size_1",
"size_2",
],
)?;
let align_corners = p.bool("align_corners")?;
let mode_name = as_str(p.get("mode")?, "mode")?;
let mode = upsample_mode_from_wire(mode_name, align_corners)
.ok_or_else(|| manifest_error("upsample の mode が allowlist 外です"))?;
if align_corners
&& matches!(
mode,
InterpolateMode::Nearest
| InterpolateMode::NearestExact
| InterpolateMode::Area
)
{
return Err(manifest_error(
"upsample の mode は align_corners=true を持てません",
));
}
let size_len = p.usize("size_len")?;
if !(1..=UPSAMPLE_MAX_SIZE_LEN).contains(&size_len) {
return Err(manifest_error(
"upsample の size_len は 1..=3 である必要があります",
));
}
let mut size = Vec::with_capacity(size_len);
for i in 0..UPSAMPLE_MAX_SIZE_LEN {
let v = p.opt_usize(&format!("size_{i}"))?;
match (i < size_len, v) {
(true, Some(x)) => size.push(x),
(false, None) => {}
_ => {
return Err(manifest_error(
"upsample の size_i は size_len 未満が整数、以上が null である必要があります",
));
}
}
}
Ok(LayerSpec::Upsample { size, mode })
}
other => Err(ModelIoError::UnsupportedModel {
reason: format!("未対応の層 kind {}", clip(other)),
}),
}
}
fn parse_key_shapes(arr: &[Json], ctx: &str) -> Result<Vec<(String, Vec<usize>)>, ModelIoError> {
let mut keys = Vec::with_capacity(arr.len());
for entry in arr {
let kf = exact_fields(entry, &format!("{ctx}[]"), &["key", "shape"])?;
let key = as_str(kf[0], &format!("{ctx}[].key"))?.to_string();
let shape = as_arr(kf[1], &format!("{ctx}[].shape"))?
.iter()
.map(|d| as_usize(d, &format!("{ctx}[].shape[]")))
.collect::<Result<Vec<_>, _>>()?;
keys.push((key, shape));
}
Ok(keys)
}
struct ParsedLayers {
specs: Vec<LayerSpec>,
parameter_keys: Vec<(String, Vec<usize>)>,
buffer_keys: Vec<(String, Vec<usize>)>,
}
fn parse_layers_and_keys(
layers: &[Json],
parameter_keys: &[Json],
buffer_keys: &[Json],
allow_legacy_empty_buffers: bool,
) -> Result<ParsedLayers, ModelIoError> {
let mut specs = Vec::with_capacity(layers.len());
for (i, layer) in layers.iter().enumerate() {
let lf = exact_fields(layer, "layers[]", &["index", "kind", "params"])?;
if as_usize(lf[0], "layers[].index")? != i {
return Err(manifest_error("layers[].index が連番ではありません"));
}
specs.push(spec_from_kind(as_str(lf[1], "layers[].kind")?, lf[2])?);
}
let keys = parse_key_shapes(parameter_keys, "parameter_keys")?;
let buffer_keys = parse_key_shapes(buffer_keys, "buffer_keys")?;
let mut expected_count: Option<usize> = Some(0);
for spec in &specs {
expected_count = expected_count
.zip(layer_parameter_count(spec))
.and_then(|(acc, n)| acc.checked_add(n));
}
if expected_count != Some(keys.len()) {
return Err(ModelIoError::Mismatch {
message: "parameter_keys が層構成から導いた期待キー・shape と一致しません".into(),
});
}
if keys != expected_parameter_keys(&specs) {
return Err(ModelIoError::Mismatch {
message: "parameter_keys が層構成から導いた期待キー・shape と一致しません".into(),
});
}
let expected_buffers = expected_buffer_keys(&specs);
let legacy_empty = allow_legacy_empty_buffers && buffer_keys.is_empty();
if allow_legacy_empty_buffers && specs.iter().any(|s| matches!(s, LayerSpec::Reparam { .. })) {
return Err(ModelIoError::Mismatch {
message: "旧形式(format_version 1)の manifest に再パラメータ化層は含められません"
.into(),
});
}
if !legacy_empty && buffer_keys != expected_buffers {
return Err(ModelIoError::Mismatch {
message: "buffer_keys が層構成から導いた期待キー・shape と一致しません".into(),
});
}
Ok(ParsedLayers {
specs,
parameter_keys: keys,
buffer_keys,
})
}
fn parse_manifest(bytes: &[u8]) -> Result<ParsedManifest, ModelIoError> {
let text = std::str::from_utf8(bytes)
.map_err(|_| manifest_error("manifest が UTF-8 ではありません"))?;
let root = Parser {
src: text.as_bytes(),
pos: 0,
}
.document()?;
let f = exact_fields(
&root,
"manifest",
&[
"format",
"format_version",
"training",
"num_layers",
"layers",
"parameter_keys",
"buffer_keys",
"safetensors_file",
"safetensors_bytes",
"compiled",
],
)?;
if as_str(f[0], "format")? != FORMAT_NAME {
return Err(manifest_error("format が想定と異なります"));
}
let format_version = as_u64(f[1], "format_version")?;
if format_version != FORMAT_VERSION && format_version != LEGACY_FORMAT_VERSION {
return Err(manifest_error("format_version が未対応です"));
}
let Json::Bool(training) = f[2] else {
return Err(manifest_error("training は bool である必要があります"));
};
let num_layers = as_usize(f[3], "num_layers")?;
let layers = as_arr(f[4], "layers")?;
let parameter_keys = as_arr(f[5], "parameter_keys")?;
let buffer_keys = as_arr(f[6], "buffer_keys")?;
let safetensors_file = as_str(f[7], "safetensors_file")?;
let safetensors_bytes = as_u64(f[8], "safetensors_bytes")?;
if layers.len() > MAX_LAYERS {
return Err(ModelIoError::TooLarge {
what: "層数",
limit: MAX_LAYERS as u64,
});
}
if num_layers != layers.len() {
return Err(manifest_error(
"num_layers が layers の要素数と一致しません",
));
}
if !is_valid_safetensors_file_name(safetensors_file) {
return Err(manifest_error(
"safetensors_file が model.<32 桁 16 進>.safetensors の形式ではありません",
));
}
let ParsedLayers {
specs,
parameter_keys: keys,
buffer_keys,
} = parse_layers_and_keys(
layers,
parameter_keys,
buffer_keys,
format_version == LEGACY_FORMAT_VERSION,
)?;
let compiled = parse_compiled(f[9])?;
Ok(ParsedManifest {
training: *training,
specs,
parameter_keys: keys,
buffer_keys,
safetensors_file: safetensors_file.to_string(),
safetensors_bytes,
compiled,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(unix)]
fn temp_dir(label: &str) -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!(
"fandhe-ai-model-io-{}-{}-{label}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0)
));
std::fs::create_dir(&dir).expect("一時ディレクトリを作成できるはず");
dir
}
fn parse_json(s: &str) -> Result<Json, ModelIoError> {
Parser {
src: s.as_bytes(),
pos: 0,
}
.document()
}
fn is_manifest_err<T>(r: Result<T, ModelIoError>) -> bool {
matches!(r, Err(ModelIoError::Manifest { .. }))
}
#[test]
fn parser_accepts_v1_shapes() {
assert!(parse_json(r#"{"a":[1,2,{"b":null}],"c":true,"d":"x"}"#).is_ok());
assert!(parse_json(" [ ] ").is_ok());
}
#[test]
fn parser_rejects_duplicate_keys() {
assert!(is_manifest_err(parse_json(r#"{"a":1,"a":2}"#)));
}
#[test]
fn parser_rejects_leading_zero_fraction_exponent_and_sign() {
for s in [
"01",
"-01",
"18446744073709551616",
"-",
"1.",
".5",
"1e",
"1e+",
"--1",
"+1",
] {
assert!(is_manifest_err(parse_json(s)), "{s} は拒否されるはず");
}
assert!(matches!(parse_json("0"), Ok(Json::Num(0))));
assert!(matches!(
parse_json("18446744073709551615"),
Ok(Json::Num(u64::MAX))
));
}
#[test]
fn parser_keeps_fraction_exponent_and_sign_as_real_lexemes() {
for s in ["1.5", "1e3", "-1", "0.5", "-0.0", "1e-5", "2.5E+3"] {
match parse_json(s) {
Ok(Json::Real(raw)) => assert_eq!(raw, s),
other => panic!("{s} は Real のはず: {other:?}"),
}
assert!(is_manifest_err(as_u64(
&parse_json(s).expect("読めるはず"),
"x"
)));
}
let long = format!("0.{}", "1".repeat(MAX_NUMBER_LEXEME_LEN));
assert!(is_manifest_err(parse_json(&long)));
}
#[test]
fn as_f32_accepts_only_finite_canonical_real() {
let real = |s: &str| parse_json(s).expect("字句として読めるはず");
for v in [
0.5f32,
1.0,
-0.0,
1e-5,
1e20,
f32::MIN_POSITIVE,
0.1,
f32::MAX,
] {
let text = format!("{v:?}");
let got = as_f32(&real(&text), "x").expect("正準形は受理されるはず");
assert_eq!(got.to_bits(), v.to_bits(), "{text}");
}
for s in [
"0.50", "1E0", "1", "0", "-1", "1e39", "true", "null", "\"0.5\"",
] {
assert!(
is_manifest_err(as_f32(&real(s), "x")),
"{s} は拒否されるはず"
);
}
}
#[test]
fn parser_rejects_trailing_garbage_and_trailing_comma() {
assert!(is_manifest_err(parse_json("{} x")));
assert!(is_manifest_err(parse_json("[1,]")));
assert!(is_manifest_err(parse_json(r#"{"a":1,}"#)));
}
#[test]
fn parser_rejects_escapes_control_chars_and_bom() {
assert!(is_manifest_err(parse_json(r#""a\nb""#)));
assert!(is_manifest_err(parse_json("\"a\u{1}b\"")));
assert!(is_manifest_err(parse_json("\u{feff}{}")));
}
#[test]
fn parser_rejects_excess_depth_before_recursing() {
let ok = format!(
"{}{}",
"[".repeat(MAX_JSON_DEPTH),
"]".repeat(MAX_JSON_DEPTH)
);
assert!(parse_json(&ok).is_ok());
let deep = format!(
"{}{}",
"[".repeat(MAX_JSON_DEPTH + 1),
"]".repeat(MAX_JSON_DEPTH + 1)
);
assert!(is_manifest_err(parse_json(&deep)));
assert!(is_manifest_err(parse_json(&"[".repeat(100_000))));
}
#[test]
fn parser_caps_array_length_and_object_keys() {
let many = format!("[{}]", vec!["0"; MAX_ARRAY_LEN + 1].join(","));
assert!(matches!(
parse_json(&many),
Err(ModelIoError::TooLarge { .. })
));
let keys: Vec<String> = (0..=MAX_OBJECT_KEYS)
.map(|i| format!("\"k{i}\":0"))
.collect();
assert!(matches!(
parse_json(&format!("{{{}}}", keys.join(","))),
Err(ModelIoError::TooLarge { .. })
));
}
#[test]
fn safetensors_file_name_pattern_is_exact() {
let good = format!("model.{}.safetensors", "0123456789abcdef".repeat(2));
assert!(is_valid_safetensors_file_name(&good));
for bad in [
"../model.0123456789abcdef0123456789abcdef.safetensors",
"model.0123456789ABCDEF0123456789ABCDEF.safetensors",
"model.0123456789abcdef.safetensors",
"model.0123456789abcdef0123456789abcde/.safetensors",
"model.0123456789abcdef0123456789abcdef.safetensors/",
"model.safetensors",
"",
] {
assert!(
!is_valid_safetensors_file_name(bad),
"{bad} は拒否されるはず"
);
}
}
#[test]
fn sequential_manifest_text_is_byte_stable() {
let model = Sequential::new()
.add_linear(2, 3, 0)
.expect("linear")
.add_relu();
let prepared = prepare_save(&model).expect("prepare");
let text = render_manifest(
&prepared,
"model.00000000000000000000000000000000.safetensors",
prepared.safetensors.len() as u64,
);
let expected = format!(
"{{\"format\":\"fandhe-ai.compat.sequential\",\"format_version\":2,\"training\":true,\"num_layers\":2,\"layers\":[{{\"index\":0,\"kind\":\"linear\",\"params\":{{\"in_features\":2,\"out_features\":3}}}},{{\"index\":1,\"kind\":\"relu\",\"params\":{{}}}}],\"parameter_keys\":[{{\"key\":\"0.weight\",\"shape\":[2,3]}},{{\"key\":\"0.bias\",\"shape\":[3]}}],\"buffer_keys\":[],\"safetensors_file\":\"model.00000000000000000000000000000000.safetensors\",\"safetensors_bytes\":{},\"compiled\":null}}",
prepared.safetensors.len()
);
assert_eq!(text, expected);
}
#[test]
fn non_unix_platform_is_rejected_as_unsupported() {
let err = save_platform_supported(false).expect_err("非 unix は拒否されるはず");
assert_eq!(err.kind(), io::ErrorKind::Unsupported);
assert!(save_platform_supported(true).is_ok());
}
fn build_model_fresh(specs: &[LayerSpec]) -> Result<Sequential, ModelIoError> {
let mut buffers = std::collections::HashMap::new();
let mut tensors = std::collections::HashMap::new();
for (i, spec) in specs.iter().enumerate() {
if let LayerSpec::BatchNorm1d { num_features, .. }
| LayerSpec::BatchNorm2d { num_features, .. } = spec
{
let n = *num_features;
let t = |v: f32| Tensor::new(vec![v; n], &[n]).expect("構築できるはず");
tensors.insert(format!("{i}.weight"), t(1.0));
tensors.insert(format!("{i}.bias"), t(0.0));
buffers.insert(format!("{i}.running_mean"), t(0.0));
buffers.insert(format!("{i}.running_var"), t(1.0));
}
}
build_model(specs, &mut buffers, &tensors)
}
fn sample_model() -> Sequential {
Sequential::new()
.add_linear(3, 4, 1)
.and_then(|m| m.add_relu().add_linear(4, 2, 2))
.expect("構築できるはず")
}
#[cfg(unix)]
pub(super) fn sample_model_for_threats() -> Sequential {
sample_model()
}
#[test]
fn prepare_save_rejects_unsupported_and_compiled_models() {
struct Custom;
impl crate::nn::Module for Custom {
fn forward<'t>(
&self,
_tape: crate::TapeRef<'t>,
input: &crate::Var<'t>,
) -> Result<crate::Var<'t>, AutodiffError> {
Ok(*input)
}
}
let custom = Sequential::new().add_relu().add_module(Custom);
assert!(matches!(
prepare_save(&custom),
Err(ModelIoError::UnsupportedModel { .. })
));
let nan = Sequential::new().add_relu().add_leaky_relu(f32::NAN);
assert!(matches!(
prepare_save(&nan),
Err(ModelIoError::UnsupportedModel { .. })
));
let inf = Sequential::new().add_elu(f32::INFINITY);
assert!(matches!(
prepare_save(&inf),
Err(ModelIoError::UnsupportedModel { .. })
));
}
fn all_kind_specs() -> Vec<LayerSpec> {
vec![
LayerSpec::Linear {
in_features: 3,
out_features: 4,
},
LayerSpec::Relu,
LayerSpec::Sigmoid,
LayerSpec::Tanh,
LayerSpec::Silu,
LayerSpec::Hardswish,
LayerSpec::Gelu,
LayerSpec::GeluTanh,
LayerSpec::LeakyRelu {
negative_slope: 0.125,
},
LayerSpec::Elu { alpha: 1.5 },
LayerSpec::Softmax { dim: 1 },
LayerSpec::LogSoftmax { dim: 0 },
LayerSpec::Softplus {
beta: 2.0,
threshold: 20.0,
},
LayerSpec::Mish,
LayerSpec::Selu,
LayerSpec::Softsign,
LayerSpec::Hardsigmoid,
LayerSpec::LogSigmoid,
LayerSpec::Tanhshrink,
LayerSpec::Celu { alpha: -0.75 },
LayerSpec::Softmin { dim: 1 },
LayerSpec::Threshold {
threshold: -0.5,
value: 2.0,
},
LayerSpec::RRelu {
lower: -0.125,
upper: 0.3,
},
LayerSpec::Hardtanh {
min_val: -2.0,
max_val: 3.0,
},
LayerSpec::Relu6,
LayerSpec::Glu { dim: 1 },
LayerSpec::PRelu { num_parameters: 3 },
LayerSpec::Flatten {
start_dim: 1,
end_dim: 3,
},
LayerSpec::Dropout { p: 0.25 },
LayerSpec::Dropout2d { p: 0.25 },
LayerSpec::AlphaDropout { p: 0.25 },
LayerSpec::Conv2d {
in_channels: 4,
out_channels: 6,
kernel_size: [3, 2],
stride: [1, 2],
padding: [1, 0],
dilation: [1, 1],
groups: 2,
},
LayerSpec::Conv3d {
in_channels: 4,
out_channels: 6,
kernel_size: [2, 3, 2],
stride: [1, 2, 1],
padding: [0, 1, 1],
dilation: [1, 1, 2],
groups: 2,
},
LayerSpec::ConvTranspose2d {
in_channels: 4,
out_channels: 6,
kernel_size: [3, 2],
stride: [2, 2],
padding: [1, 0],
output_padding: [1, 0],
dilation: [1, 1],
groups: 2,
},
LayerSpec::ConvTranspose3d {
in_channels: 4,
out_channels: 6,
kernel_size: [2, 3, 1],
stride: [2, 3, 1],
padding: [0, 1, 0],
output_padding: [1, 2, 0],
dilation: [1, 2, 1],
groups: 2,
},
LayerSpec::ConvTranspose3d {
in_channels: 2,
out_channels: 3,
kernel_size: [10, 100, 1],
stride: [11, 101, 2],
padding: [0, 0, 0],
output_padding: [10, 100, 1],
dilation: [1, 1, 1],
groups: 1,
},
LayerSpec::Conv1d {
in_channels: 4,
out_channels: 2,
kernel_size: 3,
stride: 1,
padding: 1,
dilation: 1,
groups: 2,
},
LayerSpec::LayerNorm {
normalized_size: 5,
eps: 1e-5,
},
LayerSpec::RmsNorm {
normalized_size: 5,
eps: 1e-6,
},
LayerSpec::BatchNorm1d {
num_features: 3,
eps: 1e-5,
momentum: 0.1,
},
LayerSpec::BatchNorm2d {
num_features: 3,
eps: 1e-5,
momentum: 0.1,
},
LayerSpec::Embedding {
num_embeddings: 7,
embedding_dim: 4,
padding_idx: Some(0),
},
LayerSpec::Embedding {
num_embeddings: 7,
embedding_dim: 4,
padding_idx: None,
},
LayerSpec::EmbeddingBag {
num_embeddings: 7,
embedding_dim: 4,
mode: EmbeddingBagMode::Sum,
padding_idx: Some(0),
},
LayerSpec::EmbeddingBag {
num_embeddings: 7,
embedding_dim: 4,
mode: EmbeddingBagMode::Mean,
padding_idx: None,
},
LayerSpec::EmbeddingBag {
num_embeddings: 7,
embedding_dim: 4,
mode: EmbeddingBagMode::Max,
padding_idx: None,
},
LayerSpec::MultiheadAttention {
embed_dim: 8,
num_heads: 2,
},
LayerSpec::MultiheadAttentionConfig {
embed_dim: 8,
num_heads: 2,
bias: false,
batch_first: false,
},
LayerSpec::TransformerEncoder {
d_model: 8,
num_heads: 2,
dim_feedforward: 16,
},
LayerSpec::TransformerDecoderLayer {
d_model: 8,
num_heads: 2,
dim_feedforward: 16,
},
LayerSpec::Transformer {
d_model: 8,
num_heads: 2,
num_encoder_layers: 1,
num_decoder_layers: 2,
dim_feedforward: 16,
eps: 1e-5,
},
LayerSpec::MaxPool3d {
kernel_size: [2, 3, 2],
stride: Some([1, 2, 1]),
padding: [1, 1, 0],
dilation: [1, 1, 2],
},
LayerSpec::MaxPool3d {
kernel_size: [2, 2, 2],
stride: None,
padding: [0, 0, 0],
dilation: [1, 1, 1],
},
LayerSpec::AvgPool3d {
kernel_size: [2, 3, 2],
stride: Some([1, 2, 1]),
padding: [1, 1, 0],
count_include_pad: false,
},
LayerSpec::AvgPool3d {
kernel_size: [2, 2, 2],
stride: None,
padding: [0, 0, 0],
count_include_pad: true,
},
LayerSpec::MaxPool2d {
kernel_size: [2, 2],
stride: Some([2, 1]),
padding: [0, 0],
dilation: [1, 1],
},
LayerSpec::MaxPool2d {
kernel_size: [2, 2],
stride: None,
padding: [1, 1],
dilation: [1, 1],
},
LayerSpec::MaxPool1d {
kernel_size: 2,
stride: Some(2),
padding: 0,
dilation: 1,
},
LayerSpec::MaxPool1d {
kernel_size: 2,
stride: None,
padding: 0,
dilation: 1,
},
LayerSpec::AvgPool2d {
kernel_size: [2, 2],
stride: Some([1, 1]),
padding: [1, 1],
count_include_pad: true,
},
LayerSpec::AvgPool2d {
kernel_size: [2, 2],
stride: None,
padding: [0, 0],
count_include_pad: false,
},
LayerSpec::AvgPool1d {
kernel_size: 2,
stride: Some(1),
padding: 1,
count_include_pad: true,
},
LayerSpec::AvgPool1d {
kernel_size: 2,
stride: None,
padding: 0,
count_include_pad: false,
},
LayerSpec::AdaptiveAvgPool2d {
output_size: [2, 3],
},
LayerSpec::AdaptiveAvgPool1d { output_size: 4 },
LayerSpec::AdaptiveMaxPool2d {
output_size: [2, 3],
},
LayerSpec::AdaptiveMaxPool1d { output_size: 4 },
LayerSpec::GlobalPool {
mode: GlobalPoolMode::Avg,
keepdims: true,
},
LayerSpec::GlobalPool {
mode: GlobalPoolMode::Max,
keepdims: false,
},
LayerSpec::ConvTranspose1d {
in_channels: 4,
out_channels: 6,
kernel_size: 3,
stride: 2,
padding: 1,
output_padding: 1,
dilation: 2,
groups: 2,
},
LayerSpec::Unflatten {
dim: 1,
sizes: vec![2, 3, 4],
},
LayerSpec::Upsample {
size: vec![6, 6],
mode: InterpolateMode::Nearest,
},
LayerSpec::Upsample {
size: vec![8, 8],
mode: InterpolateMode::Bilinear {
align_corners: true,
},
},
LayerSpec::ZeroPad2d {
padding: [1, 2, 3, 4],
},
LayerSpec::Identity,
LayerSpec::GroupNorm {
groups: 2,
eps: 1e-5,
},
LayerSpec::InstanceNorm { eps: 1e-5 },
LayerSpec::PixelShuffle { upscale_factor: 2 },
LayerSpec::PixelUnshuffle {
downscale_factor: 2,
},
LayerSpec::Unfold {
kernel_size: [2, 3],
stride: [1, 2],
padding: [1, 0],
dilation: [1, 2],
},
LayerSpec::Unfold {
kernel_size: [1, 1],
stride: [1, 1],
padding: [0, 0],
dilation: [1, 1],
},
LayerSpec::Fold {
output_size: [5, 7],
kernel_size: [2, 3],
stride: [1, 2],
padding: [1, 0],
dilation: [1, 2],
},
LayerSpec::Fold {
output_size: [123, 456],
kernel_size: [1, 1],
stride: [1, 1],
padding: [0, 0],
dilation: [1, 1],
},
LayerSpec::LocalResponseNorm {
size: 5,
alpha: 1e-4,
beta: 0.75,
k: 2.0,
},
LayerSpec::LocalResponseNorm {
size: 1,
alpha: 0.0,
beta: 0.0,
k: 1.0,
},
]
}
fn fold_params(key: &str, val: Option<&str>) -> String {
let mut fields: Vec<(String, String)> = [
("output_size_h", "5"),
("output_size_w", "7"),
("kernel_size_h", "2"),
("kernel_size_w", "3"),
("stride_h", "1"),
("stride_w", "2"),
("padding_h", "1"),
("padding_w", "0"),
("dilation_h", "1"),
("dilation_w", "2"),
]
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
match (fields.iter().position(|(k, _)| k == key), val) {
(Some(i), Some(v)) => fields[i].1 = v.to_string(),
(Some(i), None) => {
fields.remove(i);
}
(None, Some(v)) => fields.push((key.to_string(), v.to_string())),
(None, None) => {}
}
let body: Vec<String> = fields.iter().map(|(k, v)| format!("\"{k}\":{v}")).collect();
format!("{{{}}}", body.join(","))
}
#[test]
fn fold_and_unfold_reject_untrusted_params() {
let parse = |s: &str| parse_json(s).expect("JSON として読めるはず");
assert!(parse_json(&fold_params("stride_h", Some("18446744073709551616"))).is_err());
let bad: [(&str, Option<&str>); 11] = [
("output_size_h", None),
("dilation_w", None),
("x", Some("1")),
("stride_h", Some("-1")),
("stride_h", Some("1.5")),
("stride_h", Some(r#""1""#)),
("stride_h", Some("true")),
("stride_h", Some("null")),
("stride_h", Some("[1]")),
("kernel_size_w", None),
("padding_h", Some("-0.5")),
];
for (key, val) in bad {
let json = parse(&fold_params(key, val));
assert!(
spec_from_kind("fold", &json).is_err(),
"fold は {key}={val:?} を拒否するはず"
);
}
assert!(spec_from_kind("unfold", &parse(&fold_params("x", None))).is_err());
let unfold_ok = parse(
r#"{"kernel_size_h":2,"kernel_size_w":3,"stride_h":1,"stride_w":2,"padding_h":1,"padding_w":0,"dilation_h":1,"dilation_w":2}"#,
);
assert!(spec_from_kind("unfold", &unfold_ok).is_ok());
let unfold_missing = parse(
r#"{"kernel_size_h":2,"kernel_size_w":3,"stride_h":1,"stride_w":2,"padding_h":1,"padding_w":0,"dilation_h":1}"#,
);
assert!(spec_from_kind("unfold", &unfold_missing).is_err());
assert!(
spec_from_kind(
"fold",
&parse(&fold_params("output_size_h", Some("100000")))
)
.is_ok()
);
assert!(spec_from_kind("fold", &parse(&fold_params("x", None))).is_ok());
}
fn ct3d(key: &str, val: Option<&str>) -> String {
let mut fields: Vec<(String, String)> = vec![
("in_channels".into(), "4".into()),
("out_channels".into(), "2".into()),
("kernel_size".into(), r#""3,3,3""#.into()),
("stride".into(), r#""2,2,2""#.into()),
("padding".into(), r#""0,0,0""#.into()),
("output_padding".into(), r#""0,0,0""#.into()),
("dilation".into(), r#""1,1,1""#.into()),
("groups".into(), "1".into()),
];
match (fields.iter().position(|(k, _)| k == key), val) {
(Some(i), Some(v)) => fields[i].1 = v.to_string(),
(Some(i), None) => {
fields.remove(i);
}
(None, Some(v)) => fields.push((key.to_string(), v.to_string())),
(None, None) => {}
}
let body: Vec<String> = fields.iter().map(|(k, v)| format!("\"{k}\":{v}")).collect();
format!("{{{}}}", body.join(","))
}
#[test]
fn conv_transpose3d_rejects_untrusted_params() {
let bad: [(&str, Option<&str>); 18] = [
("kernel_size", Some(r#""3,3""#)),
("kernel_size", Some(r#""3,3,3,3""#)),
("kernel_size", Some(r#""3,,3""#)),
("kernel_size", Some(r#""03,3,3""#)),
("kernel_size", Some(r#""3, 3,3""#)),
("kernel_size", Some(r#""+3,3,3""#)),
("kernel_size", Some(r#""-3,3,3""#)),
("kernel_size", Some(r#""1.5,3,3""#)),
("kernel_size", Some("3")),
("kernel_size", Some("true")),
("kernel_size", Some("[3,3,3]")),
("kernel_size", Some(r#""18446744073709551616,1,1""#)),
("groups", Some("0")),
("groups", Some("3")),
("dilation", None),
("x", Some("1")),
("output_padding", Some("null")),
("kernel_size", Some(r#""3,3,3""#)),
];
for (key, val) in bad {
let json = parse_json(&ct3d(key, val)).expect("字句としては正しい JSON");
assert!(
matches!(
spec_from_kind("conv_transpose3d", &json),
Err(ModelIoError::Manifest { .. })
),
"{key}={val:?}"
);
}
let ok = parse_json(&ct3d("kernel_size", Some(r#""0,10,333""#))).expect("json");
assert!(spec_from_kind("conv_transpose3d", &ok).is_ok());
}
#[test]
fn all_thirty_kinds_are_covered_and_round_trip_through_params_schema() {
let specs = all_kind_specs();
let kinds: std::collections::BTreeSet<&str> = specs.iter().filter_map(spec_kind).collect();
assert_eq!(kinds.len(), 70, "kind allowlist は 70 種: {kinds:?}");
for spec in &specs {
let kind = spec_kind(spec).expect("保存可能な層");
let text = render_params(spec);
let json = parse_json(&text).expect("自分が書いた params は読めるはず");
if let Json::Obj(entries) = &json {
assert!(entries.len() <= MAX_OBJECT_KEYS, "{kind}");
}
let back = spec_from_kind(kind, &json).expect("往復できるはず");
assert_eq!(&back, spec, "{kind}");
assert_eq!(render_params(&back), text, "{kind}");
}
}
#[test]
fn expected_keys_match_real_state_dict_for_all_kinds() {
let specs = all_kind_specs();
let model = build_model_fresh(&specs).expect("70 種を構築できるはず");
let state = model.state_dict();
let expected = expected_parameter_keys(&specs);
assert_eq!(state.len(), expected.len());
for (k, shape) in &expected {
let t = state.get(k).unwrap_or_else(|| panic!("キー {k} がない"));
assert_eq!(t.shape(), shape.as_slice(), "{k}");
}
let prepared = prepare_save(&model).expect("70 種を含むモデルを検証できるはず");
assert_eq!(prepared.parameter_keys, expected);
}
#[test]
fn reparam_specs_round_trip_and_match_real_layers() {
let mut bases: Vec<LayerSpec> = Vec::new();
for spec in all_kind_specs() {
let kind = spec_kind(&spec).expect("保存可能な層");
let is_base = REPARAM_KINDS.iter().any(|(_, b, _)| *b == kind);
if is_base && !bases.iter().any(|b| spec_kind(b) == Some(kind)) {
bases.push(spec);
}
}
assert_eq!(bases.len(), 7, "基底は 7 種");
let mut seen = std::collections::BTreeSet::new();
for base in &bases {
for reparam in [
ReparamSpec::WeightNorm { dim: None },
ReparamSpec::WeightNorm { dim: Some(0) },
ReparamSpec::WeightNorm { dim: Some(1) },
ReparamSpec::SpectralNorm {
n_power_iterations: 3,
eps: 1e-12,
},
] {
let spec = LayerSpec::Reparam {
base: Box::new(base.clone()),
reparam,
};
let kind = spec_kind(&spec).expect("14 kind の表にある");
seen.insert(kind);
let text = render_params(&spec);
let json = parse_json(&text).expect("自分が書いた params は読めるはず");
if let Json::Obj(entries) = &json {
assert!(entries.len() <= MAX_OBJECT_KEYS, "{kind}: キー数上限");
}
let back = spec_from_kind(kind, &json).expect("往復できるはず");
assert_eq!(back, spec, "{kind}");
assert_eq!(render_params(&back), text, "{kind}");
let specs = [spec.clone()];
let mut buffers = std::collections::HashMap::new();
for (key, shape) in expected_buffer_keys(&specs) {
let n: usize = shape.iter().product();
buffers.insert(key, Tensor::new(vec![0.5; n], &shape).expect("buffer"));
}
let model = build_model(&specs, &mut buffers, &Default::default())
.unwrap_or_else(|e| panic!("{kind}: 構築できるはず: {e}"));
assert_eq!(model.specs(), specs.as_slice(), "{kind}: 記録した構成");
let state = model.state_dict();
let expected = expected_parameter_keys(&specs);
assert_eq!(state.len(), expected.len(), "{kind}");
for (k, shape) in &expected {
let t = state
.get(k)
.unwrap_or_else(|| panic!("{kind}: キー {k} がない"));
assert_eq!(t.shape(), shape.as_slice(), "{kind}: {k}");
}
let checked = collect_checked_state(&model)
.unwrap_or_else(|e| panic!("{kind}: 保存検査を通るはず: {e}"));
assert_eq!(
checked.expected_buffers,
expected_buffer_keys(&specs),
"{kind}"
);
}
}
assert_eq!(seen.len(), 14, "{seen:?}");
}
#[test]
fn spec_from_kind_rejects_schema_violations() {
let p = |s: &str| parse_json(s).expect("JSON として読めるはず");
assert!(matches!(
spec_from_kind("no_such_kind", &p("{}")),
Err(ModelIoError::UnsupportedModel { .. })
));
for (kind, params) in [
("dropout", r#"{"p":0.5,"q":1}"#),
("dropout", "{}"),
("dropout", r#"{"p":1}"#),
("dropout", r#"{"p":"0.5"}"#),
("dropout", r#"{"p":0.50}"#),
("dropout", r#"{"p":1e39}"#),
("celu", "{}"),
("threshold", r#"{"threshold":0.5}"#),
("rrelu", r#"{"lower":0.1}"#),
("celu", r#"{"alpha":1.0,"x":1}"#),
("softmin", r#"{"dim":1,"x":1}"#),
("celu", r#"{"alpha":1}"#),
("rrelu", r#"{"lower":0,"upper":1.0}"#),
("threshold", r#"{"threshold":0.50,"value":1.0}"#),
("celu", r#"{"alpha":1E0}"#),
("celu", r#"{"alpha":"NaN"}"#),
("threshold", r#"{"threshold":null,"value":1.0}"#),
("softmin", r#"{"dim":1.5}"#),
("softmin", r#"{"dim":-1}"#),
("softmin", "null"),
("rrelu", "[]"),
("softmax", r#"{"dim":1.0}"#),
("softmax", r#"{"dim":-1}"#),
("relu", r#"{"x":1}"#),
(
"embedding",
r#"{"num_embeddings":3,"embedding_dim":2,"padding_idx":1.0}"#,
),
(
"avg_pool1d",
r#"{"kernel_size":2,"stride":null,"padding":0,"count_include_pad":1}"#,
),
(
"max_pool2d",
r#"{"kernel_size_h":2,"kernel_size_w":2,"stride_h":null,"stride_w":1,"padding_h":0,"padding_w":0,"dilation_h":1,"dilation_w":1}"#,
),
(
"conv1d",
r#"{"in_channels":4,"out_channels":2,"kernel_size":3,"stride":1,"padding":0,"dilation":1,"groups":0}"#,
),
(
"conv1d",
r#"{"in_channels":4,"out_channels":2,"kernel_size":3,"stride":1,"padding":0,"dilation":1,"groups":3}"#,
),
(
"conv2d",
r#"{"in_channels":4,"out_channels":3,"kernel_size_h":3,"kernel_size_w":3,"stride_h":1,"stride_w":1,"padding_h":0,"padding_w":0,"dilation_h":1,"dilation_w":1,"groups":2}"#,
),
(
"conv_transpose2d",
r#"{"in_channels":4,"out_channels":2,"kernel_size_h":3,"kernel_size_w":3,"stride_h":2,"stride_w":2,"padding_h":0,"padding_w":0,"output_padding_h":0,"dilation_h":1,"dilation_w":1,"groups":1}"#,
),
(
"conv_transpose2d",
r#"{"in_channels":4,"out_channels":3,"kernel_size_h":3,"kernel_size_w":3,"stride_h":2,"stride_w":2,"padding_h":0,"padding_w":0,"output_padding_h":0,"output_padding_w":0,"dilation_h":1,"dilation_w":1,"groups":2}"#,
),
(
"conv_transpose2d",
r#"{"in_channels":4,"out_channels":2,"kernel_size_h":3,"kernel_size_w":3,"stride_h":2,"stride_w":2,"padding_h":0,"padding_w":0,"output_padding_h":-1,"output_padding_w":0,"dilation_h":1,"dilation_w":1,"groups":1}"#,
),
(
"conv_transpose2d",
r#"{"in_channels":4,"out_channels":2,"kernel_size_h":3,"kernel_size_w":3,"stride_h":2,"stride_w":2,"padding_h":0,"padding_w":0,"output_padding_h":0.5,"output_padding_w":0,"dilation_h":1,"dilation_w":1,"groups":1}"#,
),
(
"conv_transpose2d",
r#"{"in_channels":4,"out_channels":2,"kernel_size_h":3,"kernel_size_w":3,"stride_h":2,"stride_w":2,"padding_h":0,"padding_w":0,"output_padding_h":0,"output_padding_w":0,"dilation_h":1,"dilation_w":1,"groups":1,"x":1}"#,
),
(
"conv3d",
r#"{"in_channels":4,"out_channels":2,"kernel_size_h":3,"kernel_size_w":3,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"dilation_d":1,"dilation_h":1,"dilation_w":1,"groups":1}"#,
),
(
"conv3d",
r#"{"in_channels":4,"out_channels":3,"kernel_size_d":3,"kernel_size_h":3,"kernel_size_w":3,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"dilation_d":1,"dilation_h":1,"dilation_w":1,"groups":2}"#,
),
(
"conv3d",
r#"{"in_channels":4,"out_channels":2,"kernel_size_d":3,"kernel_size_h":3,"kernel_size_w":3,"stride_d":-1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"dilation_d":1,"dilation_h":1,"dilation_w":1,"groups":1}"#,
),
(
"conv3d",
r#"{"in_channels":4,"out_channels":2,"kernel_size_d":3,"kernel_size_h":3,"kernel_size_w":3,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0.5,"dilation_d":1,"dilation_h":1,"dilation_w":1,"groups":1}"#,
),
(
"conv3d",
r#"{"in_channels":4,"out_channels":2,"kernel_size_d":3,"kernel_size_h":3,"kernel_size_w":3,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"dilation_d":1,"dilation_h":1,"dilation_w":1,"groups":1,"x":1}"#,
),
(
"max_pool3d",
r#"{"kernel_size_h":2,"kernel_size_w":2,"stride_d":null,"stride_h":null,"stride_w":null,"padding_d":0,"padding_h":0,"padding_w":0,"dilation_d":1,"dilation_h":1,"dilation_w":1}"#,
),
(
"max_pool3d",
r#"{"kernel_size_d":2,"kernel_size_h":2,"kernel_size_w":2,"stride_d":null,"stride_h":1,"stride_w":null,"padding_d":0,"padding_h":0,"padding_w":0,"dilation_d":1,"dilation_h":1,"dilation_w":1}"#,
),
(
"max_pool3d",
r#"{"kernel_size_d":2,"kernel_size_h":2,"kernel_size_w":2,"stride_d":-1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"dilation_d":1,"dilation_h":1,"dilation_w":1}"#,
),
(
"max_pool3d",
r#"{"kernel_size_d":2,"kernel_size_h":2,"kernel_size_w":2,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0.5,"dilation_d":1,"dilation_h":1,"dilation_w":1}"#,
),
(
"max_pool3d",
r#"{"kernel_size_d":2,"kernel_size_h":2,"kernel_size_w":2,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"dilation_d":1,"dilation_h":1,"dilation_w":1,"x":1}"#,
),
(
"avg_pool3d",
r#"{"kernel_size_d":2,"kernel_size_h":2,"kernel_size_w":2,"stride_d":null,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"count_include_pad":true}"#,
),
(
"avg_pool3d",
r#"{"kernel_size_d":2,"kernel_size_h":2,"kernel_size_w":2,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"count_include_pad":1}"#,
),
(
"avg_pool3d",
r#"{"kernel_size_d":2,"kernel_size_h":2,"kernel_size_w":2,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0}"#,
),
(
"avg_pool3d",
r#"{"kernel_size_d":2,"kernel_size_h":2,"kernel_size_w":2,"stride_d":1,"stride_h":1,"stride_w":1,"padding_d":0,"padding_h":0,"padding_w":0,"count_include_pad":true,"dilation_d":1}"#,
),
("group_norm", r#"{"groups":2}"#),
("group_norm", r#"{"eps":1e-5}"#),
("group_norm", r#"{"groups":2,"eps":1e-5,"x":1}"#),
("group_norm", r#"{"groups":-1,"eps":1e-5}"#),
("group_norm", r#"{"groups":1.5,"eps":1e-5}"#),
("group_norm", r#"{"groups":2,"eps":"a"}"#),
("instance_norm", r#"{}"#),
("instance_norm", r#"{"eps":1e-5,"x":1}"#),
("instance_norm", r#"{"eps":"a"}"#),
("adaptive_max_pool2d", r#"{"output_size_h":2}"#),
(
"adaptive_max_pool2d",
r#"{"output_size_h":2,"output_size_w":2,"x":1}"#,
),
(
"adaptive_max_pool2d",
r#"{"output_size_h":-1,"output_size_w":2}"#,
),
(
"adaptive_max_pool2d",
r#"{"output_size_h":1.5,"output_size_w":2}"#,
),
("adaptive_max_pool1d", r#"{}"#),
("adaptive_max_pool1d", r#"{"output_size":"a"}"#),
("adaptive_max_pool1d", r#"{"output_size":-1}"#),
("adaptive_max_pool1d", r#"{"output_size":2,"x":1}"#),
("global_pool", r#"{"mode":"max"}"#),
("global_pool", r#"{"keepdims":true}"#),
("global_pool", r#"{"mode":"min","keepdims":true}"#),
("global_pool", r#"{"mode":1,"keepdims":true}"#),
("global_pool", r#"{"mode":"max","keepdims":1}"#),
("global_pool", r#"{"mode":"max","keepdims":"true"}"#),
("global_pool", r#"{"mode":"max","keepdims":true,"x":1}"#),
(
"transformer",
r#"{"d_model":8,"num_heads":2,"num_encoder_layers":1,"num_decoder_layers":1,"dim_feedforward":16}"#,
),
(
"transformer",
r#"{"d_model":8,"num_heads":2,"num_encoder_layers":1,"num_decoder_layers":1,"dim_feedforward":16,"eps":1e-5,"x":1}"#,
),
(
"transformer",
r#"{"d_model":8,"num_heads":2,"num_encoder_layers":"1","num_decoder_layers":1,"dim_feedforward":16,"eps":1e-5}"#,
),
(
"transformer",
r#"{"d_model":8,"num_heads":2,"num_encoder_layers":0,"num_decoder_layers":1,"dim_feedforward":16,"eps":1e-5}"#,
),
(
"transformer",
r#"{"d_model":8,"num_heads":2,"num_encoder_layers":1,"num_decoder_layers":0,"dim_feedforward":16,"eps":1e-5}"#,
),
("dropout2d", r#"{}"#),
("dropout2d", r#"{"p":"a"}"#),
("dropout2d", r#"{"p":0.5,"x":1}"#),
("alpha_dropout", r#"{}"#),
("alpha_dropout", r#"{"p":"a"}"#),
("alpha_dropout", r#"{"p":0.5,"x":1}"#),
(
"embedding_bag",
r#"{"num_embeddings":3,"embedding_dim":2,"padding_idx":null}"#,
),
(
"embedding_bag",
r#"{"num_embeddings":3,"embedding_dim":2,"mode":"min","padding_idx":null}"#,
),
(
"embedding_bag",
r#"{"num_embeddings":3,"embedding_dim":2,"mode":1,"padding_idx":null}"#,
),
(
"embedding_bag",
r#"{"num_embeddings":3,"embedding_dim":2,"mode":"sum"}"#,
),
(
"embedding_bag",
r#"{"num_embeddings":3,"embedding_dim":2,"mode":"sum","padding_idx":null,"x":1}"#,
),
(
"embedding_bag",
r#"{"num_embeddings":-1,"embedding_dim":2,"mode":"sum","padding_idx":null}"#,
),
("mish", r#"{"x":1}"#),
("selu", r#"{"x":1}"#),
("selu", "null"),
("softsign", r#"{"x":1}"#),
("softsign", "[]"),
("hardsigmoid", r#"{"x":1}"#),
("hardsigmoid", "null"),
("log_sigmoid", r#"{"x":1}"#),
("log_sigmoid", "[]"),
("tanhshrink", r#"{"x":1}"#),
("tanhshrink", "null"),
("relu6", r#"{"x":1}"#),
("hardtanh", r#"{}"#),
("hardtanh", r#"{"min_val":-1.0}"#),
("hardtanh", r#"{"min_val":-1.0,"max_val":"a"}"#),
("hardtanh", r#"{"min_val":-1.0,"max_val":1.0,"x":1}"#),
("glu", r#"{}"#),
("glu", r#"{"dim":"a"}"#),
("glu", r#"{"dim":-1}"#),
("glu", r#"{"dim":1.5}"#),
("glu", r#"{"dim":1,"x":1}"#),
("prelu", r#"{}"#),
("prelu", r#"{"num_parameters":"a"}"#),
("prelu", r#"{"num_parameters":-1}"#),
("prelu", r#"{"num_parameters":1.5}"#),
("prelu", r#"{"num_parameters":1,"x":1}"#),
("pixel_shuffle", r#"{}"#),
("pixel_shuffle", r#"{"upscale_factor":2,"x":1}"#),
("pixel_shuffle", r#"{"upscale_factor":-1}"#),
("pixel_shuffle", r#"{"upscale_factor":1.5}"#),
("pixel_shuffle", r#"{"upscale_factor":"a"}"#),
("pixel_unshuffle", r#"{}"#),
("pixel_unshuffle", r#"{"downscale_factor":2,"x":1}"#),
("pixel_unshuffle", r#"{"downscale_factor":-1}"#),
("pixel_unshuffle", r#"{"downscale_factor":1.5}"#),
("pixel_unshuffle", r#"{"downscale_factor":"a"}"#),
("identity", r#"{"x":1}"#),
("zero_pad2d", r#"{"left":1,"right":1,"top":1}"#),
("zero_pad2d", r#"{"left":-1,"right":1,"top":1,"bottom":1}"#),
("zero_pad2d", r#"{"left":1.5,"right":1,"top":1,"bottom":1}"#),
(
"upsample",
r#"{"mode":"nearest","align_corners":false,"size_len":2,"size_0":4,"size_1":4}"#,
),
(
"upsample",
r#"{"mode":"cubic","align_corners":false,"size_len":2,"size_0":4,"size_1":4,"size_2":null}"#,
),
(
"upsample",
r#"{"mode":"nearest","align_corners":true,"size_len":2,"size_0":4,"size_1":4,"size_2":null}"#,
),
(
"upsample",
r#"{"mode":"nearest","align_corners":false,"size_len":0,"size_0":null,"size_1":null,"size_2":null}"#,
),
(
"upsample",
r#"{"mode":"nearest","align_corners":false,"size_len":4,"size_0":1,"size_1":1,"size_2":1}"#,
),
(
"upsample",
r#"{"mode":"nearest","align_corners":false,"size_len":2,"size_0":4,"size_1":4,"size_2":4}"#,
),
(
"upsample",
r#"{"mode":"nearest","align_corners":false,"size_len":2,"size_0":4,"size_1":null,"size_2":null}"#,
),
(
"upsample",
r#"{"mode":"nearest","align_corners":false,"size_len":2,"size_0":-4,"size_1":4,"size_2":null}"#,
),
(
"upsample",
r#"{"mode":"nearest","align_corners":false,"size_len":2,"size_0":4.5,"size_1":4,"size_2":null}"#,
),
(
"conv_transpose1d",
r#"{"in_channels":4,"out_channels":2,"kernel_size":3,"stride":1,"padding":0,"output_padding":0,"dilation":1,"groups":0}"#,
),
(
"conv_transpose1d",
r#"{"in_channels":4,"out_channels":3,"kernel_size":3,"stride":1,"padding":0,"output_padding":0,"dilation":1,"groups":2}"#,
),
(
"conv_transpose1d",
r#"{"in_channels":4,"out_channels":2,"kernel_size":-3,"stride":1,"padding":0,"output_padding":0,"dilation":1,"groups":1}"#,
),
(
"conv_transpose1d",
r#"{"in_channels":4,"out_channels":2,"kernel_size":3.5,"stride":1,"padding":0,"output_padding":0,"dilation":1,"groups":1}"#,
),
(
"conv_transpose1d",
r#"{"in_channels":4,"out_channels":2,"kernel_size":3,"stride":1,"padding":0,"dilation":1,"groups":1}"#,
),
(
"conv_transpose1d",
r#"{"in_channels":4,"out_channels":2,"kernel_size":3,"stride":1,"padding":0,"output_padding":0,"dilation":1,"groups":1,"x":1}"#,
),
(
"unflatten",
r#"{"dim":1,"size_len":0,"size_0":null,"size_1":null,"size_2":null,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null}"#,
),
(
"unflatten",
r#"{"dim":1,"size_len":9,"size_0":1,"size_1":1,"size_2":1,"size_3":1,"size_4":1,"size_5":1,"size_6":1,"size_7":1}"#,
),
(
"unflatten",
r#"{"dim":1,"size_len":2,"size_0":2,"size_1":2,"size_2":2,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null}"#,
),
(
"unflatten",
r#"{"dim":1,"size_len":2,"size_0":2,"size_1":null,"size_2":null,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null}"#,
),
(
"unflatten",
r#"{"dim":1,"size_len":2,"size_0":-2,"size_1":2,"size_2":null,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null}"#,
),
(
"unflatten",
r#"{"dim":1,"size_len":2,"size_0":2.5,"size_1":2,"size_2":null,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null}"#,
),
(
"unflatten",
r#"{"dim":-1,"size_len":1,"size_0":2,"size_1":null,"size_2":null,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null}"#,
),
(
"unflatten",
r#"{"dim":1.5,"size_len":1,"size_0":2,"size_1":null,"size_2":null,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null}"#,
),
(
"unflatten",
r#"{"size_len":1,"size_0":2,"size_1":null,"size_2":null,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null}"#,
),
(
"unflatten",
r#"{"dim":1,"size_len":1,"size_0":2,"size_1":null,"size_2":null,"size_3":null,"size_4":null,"size_5":null,"size_6":null,"size_7":null,"size_8":null}"#,
),
] {
assert!(
is_manifest_err(spec_from_kind(kind, &p(params))),
"{kind} {params} は Manifest で拒否されるはず"
);
}
}
#[test]
fn upsample_zero_pad2d_identity_round_trip_and_reject_unsavable() {
let p = |s: &str| parse_json(s).expect("JSON として読めるはず");
for spec in [
LayerSpec::Identity,
LayerSpec::ZeroPad2d {
padding: [0, 1, 2, 3],
},
LayerSpec::Upsample {
size: vec![5],
mode: InterpolateMode::Linear {
align_corners: false,
},
},
LayerSpec::Upsample {
size: vec![2, 3, 4],
mode: InterpolateMode::Trilinear {
align_corners: true,
},
},
LayerSpec::Upsample {
size: vec![2, 3],
mode: InterpolateMode::Area,
},
] {
let kind = spec_kind(&spec).expect("保存可能");
let back = spec_from_kind(kind, &p(&render_params(&spec))).expect("往復できるはず");
assert_eq!(back, spec);
}
let model = Sequential::new()
.add_upsample(vec![1, 2, 3, 4], InterpolateMode::Nearest)
.expect("Nearest は軸数を構築時に見ない");
assert!(matches!(
prepare_save(&model),
Err(ModelIoError::UnsupportedModel { .. })
));
}
#[cfg(unix)]
#[test]
fn unsavable_upsample_leaves_no_trace_in_dir() {
let model = Sequential::new()
.add_upsample(vec![1, 2, 3, 4], InterpolateMode::Nearest)
.expect("Nearest は軸数を構築時に見ない");
let dir = temp_dir("upsample-unsavable");
let target = dir.join("m");
assert!(matches!(
save_model(&model, &target),
Err(ModelIoError::UnsupportedModel { .. })
));
let leftovers = std::fs::read_dir(&dir).expect("読めるはず").count();
assert_eq!(leftovers, 0, "拒否時は dir に何も作らない");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn out_of_range_values_are_rejected_by_constructors_on_build() {
for spec in [
LayerSpec::Dropout { p: 2.0 },
LayerSpec::Dropout2d { p: 2.0 },
LayerSpec::AlphaDropout { p: 2.0 },
LayerSpec::EmbeddingBag {
num_embeddings: 3,
embedding_dim: 2,
mode: EmbeddingBagMode::Sum,
padding_idx: Some(3),
},
LayerSpec::EmbeddingBag {
num_embeddings: 0,
embedding_dim: 2,
mode: EmbeddingBagMode::Sum,
padding_idx: None,
},
LayerSpec::Softplus {
beta: -1.0,
threshold: 20.0,
},
LayerSpec::Hardtanh {
min_val: 1.0,
max_val: 1.0,
},
LayerSpec::Hardtanh {
min_val: f32::NAN,
max_val: 1.0,
},
LayerSpec::Celu { alpha: 0.0 },
LayerSpec::Celu { alpha: -0.0 },
LayerSpec::RRelu {
lower: 0.5,
upper: 0.1,
},
LayerSpec::PRelu { num_parameters: 0 },
LayerSpec::MaxPool1d {
kernel_size: 0,
stride: None,
padding: 0,
dilation: 1,
},
LayerSpec::Embedding {
num_embeddings: 3,
embedding_dim: 2,
padding_idx: Some(3),
},
LayerSpec::BatchNorm1d {
num_features: 2,
eps: 1e-5,
momentum: 2.0,
},
LayerSpec::GroupNorm {
groups: 0,
eps: 1e-5,
},
LayerSpec::GroupNorm {
groups: 2,
eps: -1.0,
},
LayerSpec::InstanceNorm { eps: -1.0 },
LayerSpec::PixelShuffle { upscale_factor: 0 },
LayerSpec::PixelUnshuffle {
downscale_factor: 0,
},
LayerSpec::AdaptiveMaxPool2d {
output_size: [0, 2],
},
LayerSpec::AdaptiveMaxPool2d {
output_size: [2, 0],
},
LayerSpec::AdaptiveMaxPool1d { output_size: 0 },
] {
assert!(
matches!(
build_model_fresh(std::slice::from_ref(&spec)),
Err(ModelIoError::Autodiff(_))
),
"{spec:?} は構築時に拒否されるはず"
);
}
}
#[test]
fn prepare_save_rejects_models_whose_manifest_cannot_be_read_back() {
let mut model = Sequential::new();
for _ in 0..(MAX_ARRAY_LEN / 16 + 1) {
model = model
.add_transformer_encoder(2, 1, 1, 0)
.expect("構築できるはず");
}
assert!(matches!(
prepare_save(&model),
Err(ModelIoError::TooLarge {
what: "JSON 配列の要素数",
..
})
));
}
#[test]
fn prepare_save_rejects_mode_mismatch() {
let mut model = Sequential::new()
.add_linear(2, 2, 1)
.expect("構築できるはず");
model.eval();
let model = model.add_dropout(0.5).expect("構築できるはず");
assert!(matches!(
prepare_save(&model),
Err(ModelIoError::UnsupportedModel { .. })
));
let mut ok = Sequential::new()
.add_linear(2, 2, 1)
.expect("構築できるはず");
ok.eval();
let ok = ok.add_relu();
assert!(prepare_save(&ok).is_ok());
let mut rr = Sequential::new()
.add_linear(2, 2, 1)
.expect("構築できるはず");
rr.eval();
let rr = rr.add_rrelu(0.1, 0.3).expect("構築できるはず");
assert!(matches!(
prepare_save(&rr),
Err(ModelIoError::UnsupportedModel { .. })
));
let mut nm = Sequential::new()
.add_linear(2, 2, 1)
.expect("構築できるはず");
nm.eval();
let nm = nm
.add_celu(1.0)
.expect("構築できるはず")
.add_threshold(0.5, 1.0)
.add_softmin(1);
assert!(prepare_save(&nm).is_ok());
}
#[test]
fn prepare_save_rejects_non_finite_threshold() {
for (t, v) in [(f32::NEG_INFINITY, 0.0), (0.0, f32::NAN)] {
let model = Sequential::new().add_threshold(t, v);
assert!(matches!(
prepare_save(&model),
Err(ModelIoError::UnsupportedModel { .. })
));
}
}
fn bn_model() -> Sequential {
let mut model = Sequential::new()
.add_linear(3, 4, 1)
.and_then(|m| m.add_batch_norm1d(4, 1e-5, 0.1))
.and_then(|m| m.add_relu().add_batch_norm2d(4, 1e-5, 0.1))
.expect("構築できるはず");
model.train();
model
}
#[test]
fn expected_buffer_keys_match_real_bn_buffers() {
let specs = all_kind_specs();
let model = build_model_fresh(&specs).expect("70 種を構築できるはず");
let expected = expected_buffer_keys(&specs);
assert_eq!(expected.len(), 4);
for (i, spec) in specs.iter().enumerate() {
let (mean, var) = match spec {
LayerSpec::BatchNorm1d { .. } => {
let bn = model.layers()[i].as_batch_norm1d().expect("BN1d");
(bn.running_mean(), bn.running_var())
}
LayerSpec::BatchNorm2d { .. } => {
let bn = model.layers()[i].as_batch_norm2d().expect("BN2d");
(bn.running_mean(), bn.running_var())
}
_ => continue,
};
for (name, t) in [("running_mean", mean), ("running_var", var)] {
let key = format!("{i}.{name}");
let (_, shape) = expected
.iter()
.find(|(k, _)| *k == key)
.unwrap_or_else(|| panic!("期待キー {key} がない"));
assert_eq!(t.shape(), shape.as_slice(), "{key}");
}
}
}
#[test]
fn manifest_with_buffers_round_trips_through_strict_parser() {
let model = bn_model();
let prepared = prepare_save(&model).expect("検証を通るはず");
assert_eq!(prepared.buffer_keys.len(), 4);
let name = format!("model.{}.safetensors", "c".repeat(32));
let text = render_manifest(&prepared, &name, prepared.safetensors.len() as u64);
let parsed = parse_manifest(text.as_bytes()).expect("読み戻せるはず");
assert_eq!(parsed.buffer_keys, prepared.buffer_keys);
assert_eq!(parsed.parameter_keys, prepared.parameter_keys);
}
#[test]
fn parse_manifest_rejects_buffer_key_tampering() {
let model = bn_model();
let prepared = prepare_save(&model).expect("検証を通るはず");
let name = format!("model.{}.safetensors", "d".repeat(32));
let good = render_manifest(&prepared, &name, 1);
let m1 = "{\"key\":\"1.running_mean\",\"shape\":[4]}";
let v1 = "{\"key\":\"1.running_var\",\"shape\":[4]}";
assert!(good.contains(&format!("{m1},{v1}")));
let mismatch = |text: String| {
assert!(
matches!(
parse_manifest(text.as_bytes()),
Err(ModelIoError::Mismatch { .. })
),
"Mismatch のはず: {text}"
);
};
mismatch(good.replace(&format!("{m1},{v1}"), v1));
mismatch(good.replace(&format!("{m1},{v1}"), &format!("{m1},{v1},{m1}")));
mismatch(good.replace(&format!("{m1},{v1}"), &format!("{v1},{m1}")));
mismatch(good.replace("1.running_mean", "1.running_meen"));
mismatch(good.replace(m1, "{\"key\":\"1.running_mean\",\"shape\":[5]}"));
let bad_type = good.replace(m1, "{\"key\":\"1.running_mean\",\"shape\":\"4\"}");
assert!(is_manifest_err(parse_manifest(bad_type.as_bytes())));
}
#[test]
fn parse_manifest_accepts_legacy_empty_buffer_keys_for_batch_norm() {
let model = bn_model();
let prepared = prepare_save(&model).expect("検証を通るはず");
let name = format!("model.{}.safetensors", "e".repeat(32));
let good = render_manifest(&prepared, &name, 1);
let start = good
.find("\"buffer_keys\":[")
.expect("buffer_keys があるはず");
let end = good
.find(",\"safetensors_file\"")
.expect("safetensors_file があるはず");
let stripped = format!("{}\"buffer_keys\":[]{}", &good[..start], &good[end..]);
assert!(stripped.contains("\"buffer_keys\":[]"), "{stripped}");
let legacy = stripped.replacen("\"format_version\":2", "\"format_version\":1", 1);
assert!(legacy.contains("\"format_version\":1"), "{legacy}");
let parsed = parse_manifest(legacy.as_bytes()).expect("旧形式は受理されるはず");
assert!(parsed.buffer_keys.is_empty());
assert!(stripped.contains("\"format_version\":2"), "{stripped}");
assert!(matches!(
parse_manifest(stripped.as_bytes()),
Err(ModelIoError::Mismatch { .. })
));
}
#[test]
fn manifest_round_trips_through_strict_parser() {
let model = sample_model();
let prepared = prepare_save(&model).expect("検証を通るはず");
let name = format!("model.{}.safetensors", "a".repeat(32));
let text = render_manifest(&prepared, &name, 123);
let parsed = parse_manifest(text.as_bytes()).expect("自分が書いた manifest は読めるはず");
assert_eq!(parsed.specs, prepared.specs);
assert_eq!(parsed.parameter_keys, prepared.parameter_keys);
assert_eq!(parsed.safetensors_file, name);
assert_eq!(parsed.safetensors_bytes, 123);
assert!(parsed.training);
}
#[test]
fn parse_manifest_rejects_unknown_keys_and_unsupported_features() {
let model = sample_model();
let prepared = prepare_save(&model).expect("検証を通るはず");
let name = format!("model.{}.safetensors", "b".repeat(32));
let good = render_manifest(&prepared, &name, 1);
let with_unknown = good.replacen("\"training\"", "\"extra\":1,\"training\"", 1);
assert!(is_manifest_err(parse_manifest(with_unknown.as_bytes())));
let compiled = good.replace("\"compiled\":null", "\"compiled\":{}");
assert!(is_manifest_err(parse_manifest(compiled.as_bytes())));
let buffers = good.replace("\"buffer_keys\":[]", "\"buffer_keys\":[{}]");
assert!(is_manifest_err(parse_manifest(buffers.as_bytes())));
let extra = good.replace(
"\"buffer_keys\":[]",
"\"buffer_keys\":[{\"key\":\"0.running_mean\",\"shape\":[4]}]",
);
assert!(matches!(
parse_manifest(extra.as_bytes()),
Err(ModelIoError::Mismatch { .. })
));
let kind = good.replace("\"relu\"", "\"no_such_kind\"");
assert!(matches!(
parse_manifest(kind.as_bytes()),
Err(ModelIoError::UnsupportedModel { .. })
));
let wrong_params = good.replace("\"relu\"", "\"conv2d\"");
assert!(is_manifest_err(parse_manifest(wrong_params.as_bytes())));
let bad_file = good.replace(&name, "../evil");
assert!(is_manifest_err(parse_manifest(bad_file.as_bytes())));
}
#[test]
fn parse_manifest_rejects_layer_count_over_bound() {
let layers: Vec<String> = (0..=MAX_LAYERS)
.map(|i| format!("{{\"index\":{i},\"kind\":\"relu\",\"params\":{{}}}}"))
.collect();
let text = format!(
"{{\"format\":\"{FORMAT_NAME}\",\"format_version\":1,\"training\":true,\"num_layers\":{},\"layers\":[{}],\"parameter_keys\":[],\"buffer_keys\":[],\"safetensors_file\":\"model.{}.safetensors\",\"safetensors_bytes\":0,\"compiled\":null}}",
MAX_LAYERS + 1,
layers.join(","),
"0".repeat(32)
);
assert!(matches!(
parse_manifest(text.as_bytes()),
Err(ModelIoError::TooLarge { what: "層数", .. })
));
}
#[cfg(unix)]
#[test]
fn create_new_with_retry_gives_up_after_bound_without_touching_existing() {
let dir = temp_dir("retry");
let existing = dir.join("fixed");
std::fs::write(&existing, b"keep").expect("書き込めるはず");
let mut calls = 0usize;
let result = create_new_with_retry(
&dir,
|| {
calls += 1;
"fixed".to_string()
},
MAX_TMP_NAME_ATTEMPTS,
);
assert!(matches!(
result,
Err(ModelIoError::Io(e)) if e.kind() == io::ErrorKind::AlreadyExists
));
assert_eq!(calls, MAX_TMP_NAME_ATTEMPTS);
assert_eq!(std::fs::read(&existing).expect("読めるはず"), b"keep");
let _ = std::fs::remove_dir_all(&dir);
}
#[cfg(unix)]
#[test]
fn create_new_with_retry_retries_past_a_collision() {
let dir = temp_dir("retry-ok");
std::fs::write(dir.join("first"), b"keep").expect("書き込めるはず");
let mut names = ["first", "second"].into_iter();
let (_file, name) = create_new_with_retry(
&dir,
|| names.next().unwrap_or("third").to_string(),
MAX_TMP_NAME_ATTEMPTS,
)
.expect("2 回目で成功するはず");
assert_eq!(name, "second");
assert_eq!(
std::fs::read(dir.join("first")).expect("読めるはず"),
b"keep"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[cfg(unix)]
#[test]
fn generation_id_is_32_lowercase_hex() {
let id = generation_id();
assert!(is_valid_safetensors_file_name(&format!(
"model.{id}.safetensors"
)));
assert_ne!(id, generation_id());
}
#[cfg(unix)]
#[test]
fn remove_own_tmp_skips_replaced_path() {
let dir = temp_dir("own-tmp");
let path = dir.join("tmp");
let file = std::fs::File::create(&path).expect("作れるはず");
std::fs::remove_file(&path).expect("消せるはず");
std::fs::write(&path, b"other").expect("書き込めるはず");
remove_own_tmp(&file, &path);
assert!(path.exists(), "別ファイルへ差し替えられた場合は削除しない");
let own = dir.join("own");
let own_file = std::fs::File::create(&own).expect("作れるはず");
remove_own_tmp(&own_file, &own);
assert!(!own.exists(), "自分が作ったままのファイルは削除される");
let _ = std::fs::remove_dir_all(&dir);
}
#[cfg(unix)]
#[test]
fn load_with_small_limits_reports_too_large() {
let dir = temp_dir("limits");
let model = sample_model();
save_model(&model, &dir).expect("保存できるはず");
assert!(matches!(
load_from_dir_with_limits(&dir, 8, MAX_MODEL_FILE_BYTES),
Err(ModelIoError::TooLarge {
what: "manifest.json",
limit: 8
})
));
assert!(matches!(
load_from_dir_with_limits(&dir, MAX_MANIFEST_BYTES, 8),
Err(ModelIoError::TooLarge {
what: "model safetensors",
limit: 8
})
));
let _ = std::fs::remove_dir_all(&dir);
}
#[cfg(unix)]
#[test]
fn manifest_of_max_layers_fits_the_manifest_bound() {
let specs: Vec<LayerSpec> = (0..MAX_LAYERS)
.map(|_| LayerSpec::Linear {
in_features: 65_536,
out_features: 65_536,
})
.collect();
let prepared = PreparedSave {
training: true,
parameter_keys: expected_parameter_keys(&specs),
buffer_keys: expected_buffer_keys(&specs),
specs,
compiled: None,
safetensors: Vec::new(),
};
let text = render_manifest(
&prepared,
&format!("model.{}.safetensors", "0".repeat(32)),
u64::MAX,
);
assert!(
(text.len() as u64) <= MAX_MANIFEST_BYTES,
"len={}",
text.len()
);
}
}
#[cfg(all(test, unix))]
mod fs_threat_tests;