use crate::attention::{AttentionCache, CrossAttentionCache, MultiHeadAttention};
use crate::batch::Layout;
use crate::ffn::{SwiGlu, SwiGluCache};
use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::norm::RmsNorm;
use crate::param::{Linear, Param};
use crate::rope::Rope;
use rand::Rng;
use rand::rngs::StdRng;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
pub struct ShapeVaeConfig {
pub d_model: usize,
pub latents: usize,
pub latent_dim: usize,
pub num_heads: usize,
pub head_dim: usize,
pub d_ff: usize,
pub encoder_blocks: usize,
#[serde(default)]
pub decoder_blocks: usize,
pub frequencies: usize,
pub eps: f32,
#[serde(default)]
pub free_bits: f32,
}
impl Default for ShapeVaeConfig {
fn default() -> Self {
Self {
d_model: 512,
latents: 512,
latent_dim: 64,
num_heads: 8,
head_dim: 64,
d_ff: 2048,
encoder_blocks: 4,
decoder_blocks: 0,
frequencies: 8,
eps: 1e-5,
free_bits: 0.0,
}
}
}
impl ShapeVaeConfig {
pub fn feature_dim(&self) -> usize {
3 + 6 * self.frequencies
}
fn check(&self) -> Result<(), NetworkError> {
if self.latents == 0 || self.latent_dim == 0 {
return Err(NetworkError::InvalidConfig(
"a shape autoencoder needs at least one latent vector of at least one dimension"
.into(),
));
}
if !self.free_bits.is_finite() || self.free_bits < 0.0 {
return Err(NetworkError::InvalidConfig(format!(
"a free-bits floor is a non-negative number of nats, got {}",
self.free_bits
)));
}
if !self.head_dim.is_multiple_of(2) {
return Err(NetworkError::InvalidConfig(format!(
"head_dim must be even, got {}",
self.head_dim
)));
}
Ok(())
}
}
pub fn fourier_features(points: &Matrix, bands: usize) -> Result<Matrix, NetworkError> {
if points.cols != 3 {
return Err(NetworkError::InvalidConfig(format!(
"points are 3-D, so a [n, 3] matrix was wanted and this one is [{}, {}]",
points.rows, points.cols
)));
}
let width = 3 + 6 * bands;
let mut features = Matrix::new(points.rows, width);
use rayon::prelude::*;
let rows = features
.data
.par_chunks_mut(width)
.zip(points.data.par_chunks(3));
rows.with_min_len(1024).for_each(|(out, point)| {
out[..3].copy_from_slice(point);
for band in 0..bands {
let frequency = (1u32 << band) as f32 * std::f32::consts::PI;
for axis in 0..3 {
let (sin, cos) = (frequency * point[axis]).sin_cos();
out[3 + band * 6 + axis * 2] = sin;
out[3 + band * 6 + axis * 2 + 1] = cos;
}
}
});
Ok(features)
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub(crate) struct Block {
pub(crate) attention_norm: RmsNorm,
pub(crate) attention: MultiHeadAttention,
pub(crate) mlp_norm: RmsNorm,
pub(crate) mlp: SwiGlu,
}
struct BlockCache {
input: Matrix,
attention: AttentionCache,
residual: Matrix,
mlp: SwiGluCache,
}
impl Block {
fn new(config: &ShapeVaeConfig, rng: &mut StdRng) -> Result<Self, NetworkError> {
let rope = Rope::new(config.head_dim, 1, 10000.0)?;
let mut attention = MultiHeadAttention::new(
config.d_model,
config.num_heads,
config.num_heads,
config.head_dim,
rope,
rng,
)?;
attention.set_causal(false);
attention.set_rope_enabled(false);
Ok(Self {
attention_norm: RmsNorm::new(config.d_model, config.eps),
attention,
mlp_norm: RmsNorm::new(config.d_model, config.eps),
mlp: SwiGlu::new(config.d_model, config.d_ff, rng),
})
}
fn forward_train(&self, input: &Matrix) -> Result<(Matrix, BlockCache), NetworkError> {
let normed = self.attention_norm.forward(input);
let (attended, attention) = self.attention.forward_train(&normed, Layout::default())?;
let residual = add(input, &attended);
let normed = self.mlp_norm.forward(&residual);
let (wide, mlp) = self.mlp.forward_train(&normed);
let output = add(&residual, &wide);
Ok((
output,
BlockCache {
input: input.clone(),
attention,
residual,
mlp,
},
))
}
fn backward(
&mut self,
cache: &BlockCache,
grad_output: &Matrix,
) -> Result<Matrix, NetworkError> {
let grad_wide = self.mlp.backward(&cache.mlp, grad_output);
let grad_residual = add(
grad_output,
&self.mlp_norm.backward(&cache.residual, &grad_wide),
);
let grad_attended = self.attention.backward(&cache.attention, &grad_residual)?;
Ok(add(
&grad_residual,
&self.attention_norm.backward(&cache.input, &grad_attended),
))
}
fn params_mut(&mut self) -> Vec<&mut Param> {
let mut params = self.attention_norm.params_mut();
params.extend(self.attention.params_mut());
params.extend(self.mlp_norm.params_mut());
params.extend(self.mlp.params_mut());
params
}
}
pub struct EncoderCache {
inner: EncoderCached,
}
enum EncoderCached {
Host(Box<HostEncoderCache>),
#[cfg(feature = "cuda")]
Device(Box<crate::gpu_shape::GpuEncoderCache>),
}
struct HostEncoderCache {
features: Matrix,
cross: CrossAttentionCache,
blocks: Vec<BlockCache>,
hidden: Matrix,
normed: Matrix,
}
pub struct DecoderCache {
inner: DecoderCached,
}
enum DecoderCached {
Host(Box<HostDecoderCache>),
#[cfg(feature = "cuda")]
Device(Box<crate::gpu_shape::GpuDecoderCache>),
}
struct HostDecoderCache {
latent: Matrix,
blocks: Vec<BlockCache>,
features: Matrix,
cross: CrossAttentionCache,
residual: Matrix,
mlp: SwiGluCache,
hidden: Matrix,
normed: Matrix,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ShapeVae {
pub(crate) config: ShapeVaeConfig,
pub(crate) surface_in: Linear,
pub(crate) latent_queries: Param,
pub(crate) read_surface: MultiHeadAttention,
pub(crate) blocks: Vec<Block>,
pub(crate) encoder_norm: RmsNorm,
pub(crate) to_moments: Linear,
pub(crate) from_latent: Linear,
#[serde(default)]
pub(crate) decoder_blocks: Vec<Block>,
pub(crate) query_in: Linear,
pub(crate) read_latent: MultiHeadAttention,
pub(crate) decoder_norm: RmsNorm,
pub(crate) decoder_mlp: SwiGlu,
pub(crate) head_norm: RmsNorm,
pub(crate) head: Linear,
pub(crate) color_head: Linear,
pub(crate) head_bias: Param,
#[cfg(feature = "cuda")]
#[serde(skip)]
device: Option<std::sync::Arc<crate::gpu_transformer::GpuContext>>,
}
impl PartialEq for ShapeVae {
fn eq(&self, other: &Self) -> bool {
self.config == other.config
&& self.surface_in == other.surface_in
&& self.latent_queries == other.latent_queries
&& self.read_surface == other.read_surface
&& self.blocks == other.blocks
&& self.encoder_norm == other.encoder_norm
&& self.to_moments == other.to_moments
&& self.from_latent == other.from_latent
&& self.decoder_blocks == other.decoder_blocks
&& self.query_in == other.query_in
&& self.read_latent == other.read_latent
&& self.decoder_norm == other.decoder_norm
&& self.decoder_mlp == other.decoder_mlp
&& self.head_norm == other.head_norm
&& self.head == other.head
&& self.color_head == other.color_head
&& self.head_bias == other.head_bias
}
}
impl ShapeVae {
pub fn new(config: ShapeVaeConfig, rng: &mut StdRng) -> Result<Self, NetworkError> {
config.check()?;
let features = config.feature_dim();
let rope = Rope::new(config.head_dim, 1, 10000.0)?;
let mut blocks = Vec::with_capacity(config.encoder_blocks);
for _ in 0..config.encoder_blocks {
blocks.push(Block::new(&config, rng)?);
}
let mut decoder_blocks = Vec::with_capacity(config.decoder_blocks);
Ok(Self {
surface_in: Linear::new(features + 3, config.d_model, rng),
latent_queries: Param::he_uniform(config.latents, config.d_model, config.d_model, rng),
read_surface: MultiHeadAttention::cross(
config.d_model,
config.d_model,
config.num_heads,
config.num_heads,
config.head_dim,
rope.clone(),
rng,
)?,
blocks,
encoder_norm: RmsNorm::new(config.d_model, config.eps),
to_moments: Linear::new(config.d_model, 2 * config.latent_dim, rng),
from_latent: Linear::new(config.latent_dim, config.d_model, rng),
query_in: Linear::new(features, config.d_model, rng),
read_latent: MultiHeadAttention::cross(
config.d_model,
config.d_model,
config.num_heads,
config.num_heads,
config.head_dim,
rope,
rng,
)?,
decoder_norm: RmsNorm::new(config.d_model, config.eps),
decoder_mlp: SwiGlu::new(config.d_model, config.d_ff, rng),
head_norm: RmsNorm::new(config.d_model, config.eps),
head: Linear::new(config.d_model, 1, rng),
color_head: Linear::new(config.d_model, 3, rng),
head_bias: Param::zeros(1, 1),
decoder_blocks: {
for _ in 0..config.decoder_blocks {
decoder_blocks.push(Block::new(&config, rng)?);
}
decoder_blocks
},
#[cfg(feature = "cuda")]
device: None,
config,
})
}
pub fn config(&self) -> &ShapeVaeConfig {
&self.config
}
pub fn encode(&self, surface: &Matrix) -> Result<(Matrix, Matrix), NetworkError> {
let (mean, log_variance, _) = self.encode_train(surface)?;
Ok((mean, log_variance))
}
pub fn encode_sample(
&self,
surface: &Matrix,
rng: &mut StdRng,
) -> Result<Matrix, NetworkError> {
let (mean, log_variance) = self.encode(surface)?;
Ok(Self::sample(&mean, &log_variance, rng).0)
}
pub fn encode_train(
&self,
surface: &Matrix,
) -> Result<(Matrix, Matrix, EncoderCache), NetworkError> {
if surface.cols != 6 || surface.rows == 0 {
return Err(NetworkError::InvalidConfig(format!(
"a surface point is a position and a normal, so [n, 6] was wanted and this is [{}, {}]",
surface.rows, surface.cols
)));
}
if surface.rows == 0 {
return Err(NetworkError::InvalidConfig(
"a shape cannot be encoded from no surface points".into(),
));
}
#[cfg(feature = "cuda")]
if let Some(context) = &self.device {
let (mean, log_variance, cache) =
crate::gpu_shape::encode_train(self, context, surface)?;
return Ok((
mean,
log_variance,
EncoderCache {
inner: EncoderCached::Device(Box::new(cache)),
},
));
}
let features = self.surface_features(surface)?;
let tokens = self.surface_in.forward(&features);
let queries = self.latent_queries.value.clone();
let (crossed, cross) =
self.read_surface
.forward_train_cross(&queries, &tokens, queries.rows, tokens.rows)?;
let residual = add(&queries, &crossed);
let mut hidden = residual.clone();
let mut blocks = Vec::with_capacity(self.blocks.len());
for block in &self.blocks {
let (output, cache) = block.forward_train(&hidden)?;
hidden = output;
blocks.push(cache);
}
let normed = self.encoder_norm.forward(&hidden);
let moments = self.to_moments.forward(&normed);
let (mean, log_variance) = split(&moments);
Ok((
mean,
log_variance,
EncoderCache {
inner: EncoderCached::Host(Box::new(HostEncoderCache {
features,
cross,
blocks,
hidden,
normed,
})),
},
))
}
#[cfg(feature = "cuda")]
pub fn encode_train_batch(
&self,
surface: &Matrix,
shapes: usize,
) -> Result<(Matrix, Matrix, EncoderCache), NetworkError> {
if surface.cols != 6 {
return Err(NetworkError::InvalidConfig(format!(
"a surface point is a position and a normal, so [n, 6] was wanted and this is [{}, {}]",
surface.rows, surface.cols
)));
}
let context = self.device_context()?;
let (mean, log_variance, cache) =
crate::gpu_shape::encode_train_batch(self, &context, surface, shapes)?;
Ok((
mean,
log_variance,
EncoderCache {
inner: EncoderCached::Device(Box::new(cache)),
},
))
}
pub fn encode_backward(
&mut self,
cache: &EncoderCache,
grad_mean: &Matrix,
grad_log_variance: &Matrix,
) -> Result<(), NetworkError> {
#[allow(clippy::infallible_destructuring_match)]
let cache = match &cache.inner {
EncoderCached::Host(cache) => cache,
#[cfg(feature = "cuda")]
EncoderCached::Device(cache) => {
let context = self.device_context()?;
return crate::gpu_shape::encode_backward(
self,
&context,
cache,
grad_mean,
grad_log_variance,
);
}
};
let grad_moments = join(grad_mean, grad_log_variance);
let grad_normed = self.to_moments.backward(&cache.normed, &grad_moments);
let mut grad = self.encoder_norm.backward(&cache.hidden, &grad_normed);
for (block, cache) in self.blocks.iter_mut().zip(&cache.blocks).rev() {
grad = block.backward(cache, &grad)?;
}
let (grad_queries, grad_tokens) = self.read_surface.backward_cross(&cache.cross, &grad)?;
let grad_queries = add(&grad, &grad_queries);
if !self.latent_queries.is_frozen() {
for (slot, value) in self
.latent_queries
.grad
.data
.iter_mut()
.zip(&grad_queries.data)
{
*slot += value;
}
}
self.surface_in.backward(&cache.features, &grad_tokens);
Ok(())
}
pub fn sample<R: Rng>(mean: &Matrix, log_variance: &Matrix, rng: &mut R) -> (Matrix, Matrix) {
let mut noise = Matrix::new(mean.rows, mean.cols);
let mut latent = Matrix::new(mean.rows, mean.cols);
for index in 0..mean.data.len() {
let first: f32 = rng.gen_range(f32::EPSILON..1.0);
let second: f32 = rng.gen_range(0.0..1.0);
let sample = (-2.0 * first.ln()).sqrt() * (std::f32::consts::TAU * second).cos();
noise.data[index] = sample;
latent.data[index] = mean.data[index] + (0.5 * log_variance.data[index]).exp() * sample;
}
(latent, noise)
}
pub fn sample_backward(
grad_latent: &Matrix,
log_variance: &Matrix,
noise: &Matrix,
) -> (Matrix, Matrix) {
let mut grad_log_variance = Matrix::new(grad_latent.rows, grad_latent.cols);
for index in 0..grad_latent.data.len() {
grad_log_variance.data[index] = grad_latent.data[index]
* 0.5
* (0.5 * log_variance.data[index]).exp()
* noise.data[index];
}
(grad_latent.clone(), grad_log_variance)
}
pub fn decode(
&self,
latent: &Matrix,
queries: &Matrix,
chunk: usize,
) -> Result<Vec<f32>, NetworkError> {
if chunk == 0 {
return Err(NetworkError::InvalidConfig(
"a decode chunk holds at least one query point".into(),
));
}
let kv = self.host_keys(latent)?;
let mut distances = Vec::with_capacity(queries.rows);
for start in (0..queries.rows).step_by(chunk) {
let rows = chunk.min(queries.rows - start);
let slice = Matrix::from_vec(
rows,
3,
queries.data[start * 3..(start + rows) * 3].to_vec(),
);
#[cfg(feature = "cuda")]
if self.device.is_some() {
distances.extend(self.decode_train(latent, &slice)?.0);
continue;
}
let (chunk_distances, _) = self.decode_chunk(latent, &kv, Vec::new(), &slice)?;
distances.extend(chunk_distances);
}
Ok(distances)
}
pub fn decode_train(
&self,
latent: &Matrix,
queries: &Matrix,
) -> Result<(Vec<f32>, DecoderCache), NetworkError> {
#[cfg(feature = "cuda")]
if let Some(context) = &self.device {
let (distances, cache) =
crate::gpu_shape::decode_train(self, context, latent, queries)?;
return Ok((
distances,
DecoderCache {
inner: DecoderCached::Device(Box::new(cache)),
},
));
}
let (kv, blocks) = self.latent_keys(latent)?;
self.decode_chunk(latent, &kv, blocks, queries)
}
#[cfg(feature = "cuda")]
pub fn decode_train_batch(
&self,
latent: &Matrix,
queries: &Matrix,
shapes: usize,
) -> Result<(Vec<f32>, DecoderCache), NetworkError> {
if queries.cols != 3 || queries.rows == 0 {
return Err(NetworkError::InvalidConfig(format!(
"query points are 3-D, so [n, 3] was wanted and this is [{}, {}]",
queries.rows, queries.cols
)));
}
let context = self.device_context()?;
let (distances, cache) =
crate::gpu_shape::decode_train_batch(self, &context, latent, queries, shapes)?;
Ok((
distances,
DecoderCache {
inner: DecoderCached::Device(Box::new(cache)),
},
))
}
pub fn colors(&self, cache: &DecoderCache) -> Result<Matrix, NetworkError> {
match &cache.inner {
DecoderCached::Host(cache) => Ok(self.color_head.forward(&cache.normed)),
#[cfg(feature = "cuda")]
DecoderCached::Device(cache) => crate::gpu_shape::colors(self, cache),
}
}
pub fn decode_colors(
&self,
latent: &Matrix,
points: &Matrix,
chunk: usize,
) -> Result<Vec<[f32; 3]>, NetworkError> {
if chunk == 0 {
return Err(NetworkError::InvalidConfig(
"a decode chunk holds at least one query point".into(),
));
}
let kv = self.host_keys(latent)?;
let mut colors = Vec::with_capacity(points.rows);
for start in (0..points.rows).step_by(chunk) {
let rows = chunk.min(points.rows - start);
let slice =
Matrix::from_vec(rows, 3, points.data[start * 3..(start + rows) * 3].to_vec());
#[cfg(feature = "cuda")]
let (_, cache) = if self.device.is_some() {
self.decode_train(latent, &slice)?
} else {
self.decode_chunk(latent, &kv, Vec::new(), &slice)?
};
#[cfg(not(feature = "cuda"))]
let (_, cache) = self.decode_chunk(latent, &kv, Vec::new(), &slice)?;
let decoded = self.colors(&cache)?;
colors.extend(
decoded
.data
.as_chunks::<3>()
.0
.iter()
.map(|c| [c[0], c[1], c[2]]),
);
}
Ok(colors)
}
pub fn decode_backward(
&mut self,
cache: &DecoderCache,
grad_distances: &[f32],
) -> Result<Matrix, NetworkError> {
self.decode_backward_colored(cache, grad_distances, None)
}
pub fn decode_backward_colored(
&mut self,
cache: &DecoderCache,
grad_distances: &[f32],
grad_colors: Option<&Matrix>,
) -> Result<Matrix, NetworkError> {
#[allow(clippy::infallible_destructuring_match)]
let cache = match &cache.inner {
DecoderCached::Host(cache) => cache,
#[cfg(feature = "cuda")]
DecoderCached::Device(cache) => {
let context = self.device_context()?;
return crate::gpu_shape::decode_backward(
self,
&context,
cache,
grad_distances,
grad_colors,
);
}
};
if grad_distances.len() != cache.normed.rows {
return Err(NetworkError::InvalidConfig(format!(
"{} query points were decoded and {} gradients came back",
cache.normed.rows,
grad_distances.len()
)));
}
let grad_output = Matrix::from_vec(grad_distances.len(), 1, grad_distances.to_vec());
if !self.head_bias.is_frozen() {
self.head_bias.grad.data[0] += grad_distances.iter().sum::<f32>();
}
let mut grad_normed = self.head.backward(&cache.normed, &grad_output);
if let Some(grad_colors) = grad_colors {
if grad_colors.rows != cache.normed.rows || grad_colors.cols != 3 {
return Err(NetworkError::InvalidConfig(format!(
"the colour gradient is [{}, {}] and should be [{}, 3]",
grad_colors.rows, grad_colors.cols, cache.normed.rows
)));
}
grad_normed = add(
&grad_normed,
&self.color_head.backward(&cache.normed, grad_colors),
);
}
let grad_hidden = self.head_norm.backward(&cache.hidden, &grad_normed);
let grad_wide = self.decoder_mlp.backward(&cache.mlp, &grad_hidden);
let grad_residual = add(
&grad_hidden,
&self.decoder_norm.backward(&cache.residual, &grad_wide),
);
let (grad_embedded, grad_kv) = self
.read_latent
.backward_cross(&cache.cross, &grad_residual)?;
let grad_embedded = add(&grad_residual, &grad_embedded);
self.query_in.backward(&cache.features, &grad_embedded);
if cache.blocks.len() != self.decoder_blocks.len() {
return Err(NetworkError::InvalidConfig(
"this cache was decoded for its values and cannot be trained through".into(),
));
}
let mut grad_kv = grad_kv;
for (block, block_cache) in self.decoder_blocks.iter_mut().zip(&cache.blocks).rev() {
grad_kv = block.backward(block_cache, &grad_kv)?;
}
Ok(self.from_latent.backward(&cache.latent, &grad_kv))
}
pub fn field<'a>(
&'a self,
latent: &'a Matrix,
chunk: usize,
) -> impl Fn(&Matrix) -> Result<Vec<f32>, NetworkError> + Sync + 'a {
move |queries| self.decode(latent, queries, chunk)
}
#[cfg(feature = "cuda")]
pub fn to_cuda(&mut self, device: usize, memory_budget_mib: usize) -> Result<(), NetworkError> {
self.to_cuda_with_precision(device, memory_budget_mib, true)
}
#[cfg(feature = "cuda")]
pub fn to_cuda_with_precision(
&mut self,
device: usize,
memory_budget_mib: usize,
mixed_precision: bool,
) -> Result<(), NetworkError> {
let context = crate::gpu_transformer::GpuContext::with_precision(device, mixed_precision)?;
crate::gpu_shape::to_cuda(self, &context, memory_budget_mib)?;
self.device = Some(context);
Ok(())
}
#[cfg(feature = "cuda")]
pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
crate::gpu_shape::to_cpu(self)?;
self.device = None;
Ok(())
}
pub fn on_device(&self) -> bool {
#[cfg(feature = "cuda")]
return self.device.is_some();
#[cfg(not(feature = "cuda"))]
false
}
#[cfg(feature = "cuda")]
pub(crate) fn device_context(
&self,
) -> Result<std::sync::Arc<crate::gpu_transformer::GpuContext>, NetworkError> {
self.device.clone().ok_or_else(|| {
NetworkError::InvalidConfig(
"this cache came from a device the model has since left".into(),
)
})
}
pub fn params_mut(&mut self) -> Vec<&mut Param> {
let mut params = self.surface_in.params_mut();
params.push(&mut self.latent_queries);
params.extend(self.read_surface.params_mut());
for block in &mut self.blocks {
params.extend(block.params_mut());
}
params.extend(self.encoder_norm.params_mut());
params.extend(self.to_moments.params_mut());
params.extend(self.from_latent.params_mut());
params.extend(self.query_in.params_mut());
params.extend(self.read_latent.params_mut());
params.extend(self.decoder_norm.params_mut());
params.extend(self.decoder_mlp.params_mut());
params.extend(self.head_norm.params_mut());
params.extend(self.head.params_mut());
params.extend(self.color_head.params_mut());
params.push(&mut self.head_bias);
for block in &mut self.decoder_blocks {
params.extend(block.params_mut());
}
params
}
pub fn num_parameters(&mut self) -> usize {
self.params_mut()
.iter()
.map(|param| param.value.data.len())
.sum()
}
pub fn save<P: AsRef<std::path::Path>>(
&mut self,
path: P,
metadata: &BTreeMap<String, String>,
) -> Result<(), NetworkError> {
let mut metadata = metadata.clone();
metadata.insert("config".to_string(), serde_json::to_string(&self.config)?);
crate::checkpoint::save(
path,
&mut crate::checkpoint::positional(self.params_mut(), "vae"),
&metadata,
)
}
pub fn load<P: AsRef<std::path::Path>>(
path: P,
rng: &mut StdRng,
) -> Result<(Self, BTreeMap<String, String>), NetworkError> {
let mut metadata = crate::safetensors::SafeTensors::open(path.as_ref())?
.metadata()
.clone();
let config = metadata.remove("config").ok_or_else(|| {
NetworkError::InvalidSnapshot(
"the checkpoint carries no shape autoencoder configuration".into(),
)
})?;
let mut model = Self::new(serde_json::from_str(&config)?, rng)?;
crate::checkpoint::load(
path,
&mut crate::checkpoint::positional(model.params_mut(), "vae"),
)?;
Ok((model, metadata))
}
pub(crate) fn surface_features(&self, surface: &Matrix) -> Result<Matrix, NetworkError> {
let positions = Matrix::from_vec(
surface.rows,
3,
surface
.data
.as_chunks::<6>()
.0
.iter()
.flat_map(|point| point[..3].iter().copied())
.collect(),
);
let encoded = fourier_features(&positions, self.config.frequencies)?;
let width = encoded.cols + 3;
let mut features = Matrix::new(surface.rows, width);
for row in 0..surface.rows {
let out = features.row_mut(row);
out[..encoded.cols].copy_from_slice(encoded.row(row));
out[encoded.cols..].copy_from_slice(&surface.row(row)[3..]);
}
Ok(features)
}
fn host_keys(&self, latent: &Matrix) -> Result<Matrix, NetworkError> {
#[cfg(feature = "cuda")]
if self.device.is_some() {
return Ok(Matrix::from_vec(0, 0, Vec::new()));
}
Ok(self.latent_keys(latent)?.0)
}
fn latent_keys(&self, latent: &Matrix) -> Result<(Matrix, Vec<BlockCache>), NetworkError> {
if latent.rows != self.config.latents || latent.cols != self.config.latent_dim {
return Err(NetworkError::InvalidConfig(format!(
"the latent is [{}, {}] and this model's is [{}, {}]",
latent.rows, latent.cols, self.config.latents, self.config.latent_dim
)));
}
let mut kv = self.from_latent.forward(latent);
let mut caches = Vec::with_capacity(self.decoder_blocks.len());
for block in &self.decoder_blocks {
let (next, cache) = block.forward_train(&kv)?;
kv = next;
caches.push(cache);
}
Ok((kv, caches))
}
fn decode_chunk(
&self,
latent: &Matrix,
kv: &Matrix,
blocks: Vec<BlockCache>,
queries: &Matrix,
) -> Result<(Vec<f32>, DecoderCache), NetworkError> {
let features = fourier_features(queries, self.config.frequencies)?;
let embedded = self.query_in.forward(&features);
let (crossed, cross) =
self.read_latent
.forward_train_cross(&embedded, kv, embedded.rows, kv.rows)?;
let residual = add(&embedded, &crossed);
let normed = self.decoder_norm.forward(&residual);
let (wide, mlp) = self.decoder_mlp.forward_train(&normed);
let hidden = add(&residual, &wide);
let normed = self.head_norm.forward(&hidden);
let distances = self.head.forward(&normed);
let bias = self.head_bias.value.data[0];
Ok((
distances.data.iter().map(|value| value + bias).collect(),
DecoderCache {
inner: DecoderCached::Host(Box::new(HostDecoderCache {
latent: latent.clone(),
blocks,
features,
cross,
residual,
mlp,
hidden,
normed,
})),
},
))
}
}
fn add(left: &Matrix, right: &Matrix) -> Matrix {
let mut out = left.clone();
for (slot, value) in out.data.iter_mut().zip(&right.data) {
*slot += value;
}
out
}
pub(crate) fn split(moments: &Matrix) -> (Matrix, Matrix) {
let half = moments.cols / 2;
let mut left = Matrix::new(moments.rows, half);
let mut right = Matrix::new(moments.rows, half);
for row in 0..moments.rows {
let source = moments.row(row);
left.row_mut(row).copy_from_slice(&source[..half]);
right.row_mut(row).copy_from_slice(&source[half..]);
}
(left, right)
}
pub(crate) fn join(left: &Matrix, right: &Matrix) -> Matrix {
let mut out = Matrix::new(left.rows, left.cols + right.cols);
for row in 0..left.rows {
let slot = out.row_mut(row);
slot[..left.cols].copy_from_slice(left.row(row));
slot[left.cols..].copy_from_slice(right.row(row));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use rand::SeedableRng;
fn tiny() -> ShapeVaeConfig {
ShapeVaeConfig {
d_model: 16,
latents: 4,
latent_dim: 3,
num_heads: 2,
head_dim: 8,
d_ff: 16,
encoder_blocks: 1,
decoder_blocks: 1,
frequencies: 2,
eps: 1e-5,
free_bits: 0.0,
}
}
fn rows(count: usize, width: usize, salt: usize) -> Matrix {
Matrix::from_vec(
count,
width,
(0..count * width)
.map(|i| ((i * 37 + salt * 11) % 23) as f32 / 23.0 - 0.5)
.collect(),
)
}
fn objective(values: &[f32], weights: &Matrix) -> f32 {
values
.iter()
.zip(weights.data.iter().cycle())
.map(|(value, weight)| value * weight)
.sum()
}
#[test]
fn the_latent_is_the_shape_the_config_asks_for() {
let mut rng = StdRng::seed_from_u64(0);
let model = ShapeVae::new(tiny(), &mut rng).unwrap();
for points in [16, 400] {
let (mean, log_variance) = model.encode(&rows(points, 6, 1)).unwrap();
assert_eq!((mean.rows, mean.cols), (4, 3));
assert_eq!((log_variance.rows, log_variance.cols), (4, 3));
}
}
#[test]
fn a_surface_that_is_not_points_and_normals_is_refused() {
let mut rng = StdRng::seed_from_u64(1);
let model = ShapeVae::new(tiny(), &mut rng).unwrap();
assert!(model.encode(&rows(8, 3, 1)).is_err());
assert!(model.encode(&rows(0, 6, 1)).is_err());
}
#[test]
fn a_latent_of_the_wrong_shape_is_refused() {
let mut rng = StdRng::seed_from_u64(2);
let model = ShapeVae::new(tiny(), &mut rng).unwrap();
let error = model
.decode(&rows(4, 5, 1), &rows(6, 3, 2), 8)
.unwrap_err()
.to_string();
assert!(error.contains("[4, 5]"), "{error}");
}
#[test]
fn encoding_a_sample_draws_around_the_mean_and_repeats_for_a_seed() {
let mut rng = StdRng::seed_from_u64(12);
let model = ShapeVae::new(tiny(), &mut rng).unwrap();
let surface = rows(16, 6, 2);
let (mean, log_variance) = model.encode(&surface).unwrap();
let first = model
.encode_sample(&surface, &mut StdRng::seed_from_u64(1))
.unwrap();
let again = model
.encode_sample(&surface, &mut StdRng::seed_from_u64(1))
.unwrap();
let other = model
.encode_sample(&surface, &mut StdRng::seed_from_u64(2))
.unwrap();
assert_eq!(first.rows, mean.rows);
assert_eq!(first.cols, mean.cols);
assert_eq!(first.data, again.data, "a seed repeats its draw");
assert_ne!(first.data, other.data, "another seed draws elsewhere");
assert_ne!(first.data, mean.data, "a draw is not the mean");
for index in 0..mean.data.len() {
let sigma = (0.5 * log_variance.data[index]).exp();
let offset = (first.data[index] - mean.data[index]).abs();
assert!(offset < 6.0 * sigma, "{offset} against a sigma of {sigma}");
}
}
#[test]
fn a_checkpoint_written_before_the_free_bits_floor_existed_reads_as_zero() {
let older = r#"{"d_model":32,"latents":4,"latent_dim":8,"num_heads":2,
"head_dim":16,"d_ff":64,"encoder_blocks":1,"frequencies":2,"eps":1e-5}"#;
let config: ShapeVaeConfig = serde_json::from_str(older).unwrap();
assert_eq!(config.free_bits, 0.0, "an older run keeps its behaviour");
let mut broken = tiny();
broken.free_bits = -1.0;
assert!(matches!(
ShapeVae::new(broken, &mut StdRng::seed_from_u64(0)),
Err(NetworkError::InvalidConfig(_))
));
}
#[test]
fn chunked_decoding_matches_decoding_in_one_go() {
let mut rng = StdRng::seed_from_u64(3);
let model = ShapeVae::new(tiny(), &mut rng).unwrap();
let latent = rows(4, 3, 4);
let queries = rows(37, 3, 5);
let whole = model.decode(&latent, &queries, 64).unwrap();
let pieces = model.decode(&latent, &queries, 8).unwrap();
assert_eq!(whole.len(), 37);
for (whole, piece) in whole.iter().zip(&pieces) {
assert!((whole - piece).abs() < 1e-5, "{whole} vs {piece}");
}
}
#[test]
fn a_chunk_size_of_zero_is_refused() {
let mut rng = StdRng::seed_from_u64(4);
let model = ShapeVae::new(tiny(), &mut rng).unwrap();
assert!(model.decode(&rows(4, 3, 1), &rows(4, 3, 2), 0).is_err());
}
#[test]
fn the_decoder_gradients_match_finite_differences() {
let mut rng = StdRng::seed_from_u64(5);
let mut model = ShapeVae::new(tiny(), &mut rng).unwrap();
let latent = rows(4, 3, 6);
let queries = rows(9, 3, 7);
let upstream = rows(1, 9, 8);
let (distances, cache) = model.decode_train(&latent, &queries).unwrap();
let grad: Vec<f32> = upstream.data[..distances.len()].to_vec();
model.decode_backward(&cache, &grad).unwrap();
let epsilon = 1e-3;
let probes: [(&str, usize); 5] = [
("head", 3),
("query_in", 11),
("from_latent", 5),
("decoder_block", 7),
("head_bias", 0),
];
for (name, index) in probes {
let analytic = match name {
"head" => model.head.weight.grad.data[index],
"query_in" => model.query_in.weight.grad.data[index],
"from_latent" => model.from_latent.weight.grad.data[index],
"decoder_block" => model.decoder_blocks[0].mlp.params_mut()[0].grad.data[index],
_ => model.head_bias.grad.data[index],
};
let mut shifted = |delta: f32| {
let slot = match name {
"head" => &mut model.head.weight.value.data[index],
"query_in" => &mut model.query_in.weight.value.data[index],
"from_latent" => &mut model.from_latent.weight.value.data[index],
"decoder_block" => {
&mut model.decoder_blocks[0]
.mlp
.params_mut()
.remove(0)
.value
.data[index]
}
_ => &mut model.head_bias.value.data[index],
};
let original = *slot;
*slot = original + delta;
let moved = model.decode(&latent, &queries, 64).unwrap();
let slot = match name {
"head" => &mut model.head.weight.value.data[index],
"query_in" => &mut model.query_in.weight.value.data[index],
"from_latent" => &mut model.from_latent.weight.value.data[index],
"decoder_block" => {
&mut model.decoder_blocks[0]
.mlp
.params_mut()
.remove(0)
.value
.data[index]
}
_ => &mut model.head_bias.value.data[index],
};
*slot = original;
objective(&moved, &upstream)
};
let numeric = (shifted(epsilon) - shifted(-epsilon)) / (2.0 * epsilon);
assert!(
(analytic - numeric).abs() < 1e-2,
"{name}[{index}]: {analytic} vs {numeric}"
);
}
}
#[test]
fn the_colour_gradients_match_finite_differences() {
let mut rng = StdRng::seed_from_u64(15);
let mut model = ShapeVae::new(tiny(), &mut rng).unwrap();
let latent = rows(4, 3, 16);
let queries = rows(9, 3, 17);
let upstream = rows(9, 3, 18);
let (distances, cache) = model.decode_train(&latent, &queries).unwrap();
model
.decode_backward_colored(&cache, &vec![0.0; distances.len()], Some(&upstream))
.unwrap();
let loss = |model: &ShapeVae| {
let (_, cache) = model.decode_train(&latent, &queries).unwrap();
let colors = model.colors(&cache).unwrap();
colors
.data
.iter()
.zip(&upstream.data)
.map(|(color, weight)| color * weight)
.sum::<f32>()
};
let epsilon = 1e-3;
for (name, index) in [("color_head", 5), ("query_in", 11)] {
let analytic = match name {
"color_head" => model.color_head.weight.grad.data[index],
_ => model.query_in.weight.grad.data[index],
};
let mut shifted = |delta: f32| {
let slot = match name {
"color_head" => &mut model.color_head.weight.value.data[index],
_ => &mut model.query_in.weight.value.data[index],
};
let original = *slot;
*slot = original + delta;
let moved = loss(&model);
let slot = match name {
"color_head" => &mut model.color_head.weight.value.data[index],
_ => &mut model.query_in.weight.value.data[index],
};
*slot = original;
moved
};
let numeric = (shifted(epsilon) - shifted(-epsilon)) / (2.0 * epsilon);
assert!(
(analytic - numeric).abs() < 1e-2,
"{name}[{index}]: {analytic} vs {numeric}"
);
}
}
#[test]
fn the_latent_gradient_matches_finite_differences() {
let mut rng = StdRng::seed_from_u64(6);
let mut model = ShapeVae::new(tiny(), &mut rng).unwrap();
let mut latent = rows(4, 3, 9);
let queries = rows(7, 3, 10);
let upstream = rows(1, 7, 11);
let (distances, cache) = model.decode_train(&latent, &queries).unwrap();
let grad: Vec<f32> = upstream.data[..distances.len()].to_vec();
let grad_latent = model.decode_backward(&cache, &grad).unwrap();
let epsilon = 1e-3;
for index in [0, 4, 7, 11] {
let original = latent.data[index];
latent.data[index] = original + epsilon;
let high = objective(&model.decode(&latent, &queries, 64).unwrap(), &upstream);
latent.data[index] = original - epsilon;
let low = objective(&model.decode(&latent, &queries, 64).unwrap(), &upstream);
latent.data[index] = original;
let numeric = (high - low) / (2.0 * epsilon);
assert!(
(grad_latent.data[index] - numeric).abs() < 1e-2,
"index {index}: {} vs {numeric}",
grad_latent.data[index]
);
}
}
#[test]
fn the_encoder_gradients_match_finite_differences() {
let mut rng = StdRng::seed_from_u64(7);
let mut model = ShapeVae::new(tiny(), &mut rng).unwrap();
let surface = rows(12, 6, 12);
let queries = rows(6, 3, 13);
let upstream = rows(1, 6, 14);
let (mean, _, encoder) = model.encode_train(&surface).unwrap();
let (distances, decoder) = model.decode_train(&mean, &queries).unwrap();
let grad: Vec<f32> = upstream.data[..distances.len()].to_vec();
let grad_mean = model.decode_backward(&decoder, &grad).unwrap();
let grad_log_variance = Matrix::new(grad_mean.rows, grad_mean.cols);
model
.encode_backward(&encoder, &grad_mean, &grad_log_variance)
.unwrap();
let epsilon = 1e-3;
for index in [0, 9, 23] {
let analytic = model.latent_queries.grad.data[index];
let original = model.latent_queries.value.data[index];
model.latent_queries.value.data[index] = original + epsilon;
let high = reconstruct(&model, &surface, &queries, &upstream);
model.latent_queries.value.data[index] = original - epsilon;
let low = reconstruct(&model, &surface, &queries, &upstream);
model.latent_queries.value.data[index] = original;
let numeric = (high - low) / (2.0 * epsilon);
assert!(
(analytic - numeric).abs() < 1e-2,
"index {index}: {analytic} vs {numeric}"
);
}
}
fn reconstruct(model: &ShapeVae, surface: &Matrix, queries: &Matrix, upstream: &Matrix) -> f32 {
let (mean, _) = model.encode(surface).unwrap();
objective(&model.decode(&mean, queries, 64).unwrap(), upstream)
}
#[test]
fn the_reparameterization_gradient_matches_finite_differences() {
let mean = rows(3, 4, 15);
let mut log_variance = rows(3, 4, 16);
let noise = rows(3, 4, 17);
let upstream = rows(3, 4, 18);
let mut latent = Matrix::new(3, 4);
let draw = |log_variance: &Matrix, latent: &mut Matrix| {
for index in 0..latent.data.len() {
latent.data[index] =
mean.data[index] + (0.5 * log_variance.data[index]).exp() * noise.data[index];
}
objective(&latent.data, &upstream)
};
draw(&log_variance, &mut latent);
let (grad_mean, grad_log_variance) =
ShapeVae::sample_backward(&upstream, &log_variance, &noise);
assert_eq!(grad_mean.data, upstream.data);
let epsilon = 1e-3;
for index in [0, 5, 11] {
let original = log_variance.data[index];
log_variance.data[index] = original + epsilon;
let high = draw(&log_variance, &mut latent);
log_variance.data[index] = original - epsilon;
let low = draw(&log_variance, &mut latent);
log_variance.data[index] = original;
let numeric = (high - low) / (2.0 * epsilon);
assert!(
(grad_log_variance.data[index] - numeric).abs() < 1e-4,
"index {index}: {} vs {numeric}",
grad_log_variance.data[index]
);
}
}
#[test]
fn sampling_scatters_around_the_mean_and_collapses_onto_it() {
let mut rng = StdRng::seed_from_u64(8);
let mean = rows(8, 8, 19);
let (spread, noise) = ShapeVae::sample(&mean, &Matrix::new(8, 8), &mut rng);
let moved = spread
.data
.iter()
.zip(&mean.data)
.map(|(sample, mean)| (sample - mean).abs())
.fold(0.0f32, f32::max);
assert!(moved > 0.5, "the samples barely moved: {moved}");
assert_eq!(noise.rows, 8);
let mut silent = Matrix::new(8, 8);
silent.data.fill(-40.0);
let (collapsed, _) = ShapeVae::sample(&mean, &silent, &mut rng);
for (sample, mean) in collapsed.data.iter().zip(&mean.data) {
assert!((sample - mean).abs() < 1e-6);
}
}
#[test]
fn fourier_features_separate_points_a_linear_map_would_confuse() {
let points = Matrix::from_vec(2, 3, vec![0.1, 0.2, 0.3, 0.1, 0.2, 0.35]);
let features = fourier_features(&points, 4).unwrap();
assert_eq!(features.cols, 3 + 24);
let raw = (features.row(0)[2] - features.row(1)[2]).abs();
let high = (features.row(0)[3 + 18 + 4] - features.row(1)[3 + 18 + 4]).abs();
assert!(high > 4.0 * raw, "{high} against {raw}");
assert!(fourier_features(&Matrix::new(2, 4), 2).is_err());
}
#[test]
fn the_field_closure_drives_marching_tetrahedra() {
let mut rng = StdRng::seed_from_u64(9);
let mut model = ShapeVae::new(tiny(), &mut rng).unwrap();
model.head_bias.value.data[0] = -model
.decode(&rows(4, 3, 20), &Matrix::new(1, 3), 8)
.unwrap()[0];
let latent = rows(4, 3, 20);
let mesh = crate::mesh::marching_tetrahedra(
model.field(&latent, 4096),
12,
([-1.0; 3], [1.0; 3]),
0.0,
)
.unwrap();
assert!(!mesh.indices.is_empty(), "the field never crossed zero");
}
#[test]
fn a_training_step_moves_every_parameter() {
use crate::losses::{clamped_l1, kl_divergence};
use crate::optimizers::Optimizer;
let mut rng = StdRng::seed_from_u64(10);
let mut model = ShapeVae::new(tiny(), &mut rng).unwrap();
let surface = rows(12, 6, 21);
let queries = rows(10, 3, 22);
let targets: Vec<f32> = rows(1, 10, 23).data;
let before: Vec<Vec<f32>> = model
.params_mut()
.iter()
.map(|param| param.value.data.clone())
.collect();
let (mean, log_variance, encoder) = model.encode_train(&surface).unwrap();
let (latent, noise) = ShapeVae::sample(&mean, &log_variance, &mut rng);
let (distances, decoder) = model.decode_train(&latent, &queries).unwrap();
let (_, grad_distances) = clamped_l1(&distances, &targets, 0.1);
let grad_colors = rows(queries.rows, 3, 13);
let grad_latent = model
.decode_backward_colored(&decoder, &grad_distances, Some(&grad_colors))
.unwrap();
let (mut grad_mean, mut grad_log_variance) =
ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
let (_, kl_mean, kl_log_variance) = kl_divergence(&mean.data, &log_variance.data);
for (slot, value) in grad_mean.data.iter_mut().zip(&kl_mean) {
*slot += value * 1e-3;
}
for (slot, value) in grad_log_variance.data.iter_mut().zip(&kl_log_variance) {
*slot += value * 1e-3;
}
model
.encode_backward(&encoder, &grad_mean, &grad_log_variance)
.unwrap();
let optimizer = Optimizer::adam(1e-2);
for param in model.params_mut() {
param.step(&optimizer, 1, 1.0);
}
for (index, (param, before)) in model.params_mut().iter().zip(&before).enumerate() {
let moved = param
.value
.data
.iter()
.zip(before)
.any(|(after, before)| after != before);
assert!(
moved,
"parameter {index} ({}x{}) never received a gradient",
param.value.rows, param.value.cols
);
}
}
#[test]
fn a_single_sphere_can_be_overfitted() {
use crate::losses::clamped_l1;
use crate::mesh::Bvh;
use crate::optimizers::Optimizer;
let mut rng = StdRng::seed_from_u64(11);
let sphere = unit_sphere();
let bvh = Bvh::build(&sphere);
let (points, normals) = bvh.sample_surface(256, &mut rng);
let mut surface = Matrix::new(points.len(), 6);
for (row, (point, normal)) in points.iter().zip(&normals).enumerate() {
let slot = surface.row_mut(row);
slot[..3].copy_from_slice(point);
slot[3..].copy_from_slice(normal);
}
let mut query_points = Vec::new();
for _ in 0..256 {
let radius = 0.6 + rng.gen_range(0.0..0.8);
let (point, _) = bvh.sample_surface(1, &mut rng);
query_points.push([
point[0][0] * radius,
point[0][1] * radius,
point[0][2] * radius,
]);
}
let targets = bvh.signed_distance(&query_points);
let queries = Matrix::from_vec(
query_points.len(),
3,
query_points.iter().flatten().copied().collect(),
);
let config = ShapeVaeConfig {
d_model: 48,
latents: 8,
latent_dim: 8,
num_heads: 3,
head_dim: 16,
d_ff: 96,
encoder_blocks: 1,
decoder_blocks: 1,
frequencies: 6,
eps: 1e-5,
free_bits: 0.0,
};
let mut model = ShapeVae::new(config, &mut rng).unwrap();
let optimizer = Optimizer::adam(3e-3);
let mut first = 0.0;
let mut last = 0.0;
for step in 1..=120 {
let (mean, _, encoder) = model.encode_train(&surface).unwrap();
let (distances, decoder) = model.decode_train(&mean, &queries).unwrap();
let (loss, grad) = clamped_l1(&distances, &targets, 0.2);
if step == 1 {
first = loss;
}
last = loss;
let grad_mean = model.decode_backward(&decoder, &grad).unwrap();
let zero = Matrix::new(grad_mean.rows, grad_mean.cols);
model.encode_backward(&encoder, &grad_mean, &zero).unwrap();
for param in model.params_mut() {
param.step(&optimizer, step, 1.0);
param.grad.zeros();
}
}
assert!(
last < first * 0.25,
"the loss went from {first} to {last}, which is not learning"
);
assert!(last < 0.1, "final loss {last}");
}
fn unit_sphere() -> crate::mesh::Mesh {
let (rings, segments) = (12, 24);
let mut mesh = crate::mesh::Mesh::default();
for ring in 0..=rings {
let phi = std::f32::consts::PI * ring as f32 / rings as f32;
for segment in 0..=segments {
let theta = std::f32::consts::TAU * segment as f32 / segments as f32;
mesh.positions
.push([phi.sin() * theta.cos(), phi.cos(), phi.sin() * theta.sin()]);
}
}
let stride = segments + 1;
for ring in 0..rings {
for segment in 0..segments {
let a = (ring * stride + segment) as u32;
let b = a + 1;
let c = a + stride as u32;
let d = c + 1;
mesh.indices.push([a, b, c]);
mesh.indices.push([b, d, c]);
}
}
mesh.recompute_normals();
mesh
}
#[test]
fn a_checkpoint_round_trip_restores_the_model_and_its_metadata() {
let path = std::env::temp_dir().join(format!(
"rusting-brain-shape-vae-{}.safetensors",
std::process::id()
));
let mut rng = StdRng::seed_from_u64(4);
let mut model = ShapeVae::new(tiny(), &mut rng).unwrap();
for (index, param) in model.params_mut().into_iter().enumerate() {
param.value.data[0] += 0.25 + index as f32 / 100.0;
}
let metadata = BTreeMap::from([("step".to_string(), "42".to_string())]);
model.save(&path, &metadata).unwrap();
let (restored, read) = ShapeVae::load(&path, &mut StdRng::seed_from_u64(99)).unwrap();
std::fs::remove_file(&path).ok();
assert_eq!(restored, model, "the restored model is not the saved one");
assert_eq!(read, metadata, "the caller's metadata did not survive");
}
}