use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::param::Param;
use crate::safetensors::{self, SafeTensors};
use std::collections::BTreeMap;
use std::path::Path;
pub fn save<P: AsRef<Path>>(
path: P,
params: &mut [(String, &mut Param)],
metadata: &BTreeMap<String, String>,
) -> Result<(), NetworkError> {
for (_, param) in params.iter_mut() {
#[cfg(feature = "cuda")]
param.sync_from_device()?;
param.moments()?;
}
let mut entries: Vec<(String, [usize; 2], &[f32])> = Vec::new();
for (name, param) in params.iter() {
let value = ¶m.value;
entries.push((name.clone(), [value.rows, value.cols], &value.data));
if param.is_frozen() {
continue;
}
let (first, second) = param.host_moments();
entries.push((
format!("{name}.moment1"),
[first.rows, first.cols],
&first.data,
));
entries.push((
format!("{name}.moment2"),
[second.rows, second.cols],
&second.data,
));
}
let borrowed: Vec<(&str, &[usize], &[f32])> = entries
.iter()
.map(|(name, shape, data)| (name.as_str(), &shape[..], *data))
.collect();
safetensors::write(path, &borrowed, metadata)
}
pub fn load<P: AsRef<Path>>(
path: P,
params: &mut [(String, &mut Param)],
) -> Result<BTreeMap<String, String>, NetworkError> {
let mut file = SafeTensors::open(path)?;
for (name, param) in params.iter_mut() {
if param.is_on_device() {
return Err(NetworkError::InvalidSnapshot(format!(
"{name} is resident on a device; load the checkpoint before moving the model there"
)));
}
let value = file.matrix(name)?;
let (rows, cols) = (param.value.rows, param.value.cols);
if value.rows != rows || value.cols != cols {
return Err(NetworkError::InvalidSnapshot(format!(
"{name} is {}x{} in the checkpoint and {rows}x{cols} in the model",
value.rows, value.cols
)));
}
param.value = value;
let moments: Vec<Matrix> = ["moment1", "moment2"]
.iter()
.filter_map(|suffix| file.matrix(&format!("{name}.{suffix}")).ok())
.collect();
if let [first, second] = &moments[..]
&& first.rows == rows
&& first.cols == cols
&& second.rows == rows
&& second.cols == cols
{
param.set_moments(first.clone(), second.clone())?;
}
}
Ok(file.metadata().clone())
}
pub fn positional<'a>(params: Vec<&'a mut Param>, prefix: &str) -> Vec<(String, &'a mut Param)> {
params
.into_iter()
.enumerate()
.map(|(index, param)| (format!("{prefix}.{index:04}"), param))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::optimizers::Optimizer;
fn stepped(rows: usize, cols: usize, seed: f32) -> Param {
let mut param = Param::new(Matrix::from_vec(
rows,
cols,
(0..rows * cols).map(|i| seed + i as f32).collect(),
));
param.grad = Matrix::from_vec(rows, cols, vec![0.5; rows * cols]);
param.step(&Optimizer::adam(1e-2), 1, 1.0);
param
}
fn scratch(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!(
"rusting-brain-{name}-{}.safetensors",
std::process::id()
))
}
#[test]
fn a_round_trip_restores_weights_and_moments() {
let path = scratch("round-trip");
let mut first = stepped(3, 2, 1.0);
let mut second = stepped(2, 4, 10.0);
let mut saved = [
("vae.encoder.weight".to_string(), &mut first),
("flow.block0.weight".to_string(), &mut second),
];
save(
&path,
&mut saved,
&BTreeMap::from([("step".to_string(), "1200".to_string())]),
)
.unwrap();
let expected: Vec<(Matrix, Matrix, Matrix)> = saved
.iter_mut()
.map(|(_, param)| {
let value = param.value.clone();
let (first, second) = param.moments().unwrap();
(value, first.clone(), second.clone())
})
.collect();
let mut blank_first = Param::new(Matrix::new(3, 2));
let mut blank_second = Param::new(Matrix::new(2, 4));
let mut restored = [
("vae.encoder.weight".to_string(), &mut blank_first),
("flow.block0.weight".to_string(), &mut blank_second),
];
let metadata = load(&path, &mut restored).unwrap();
assert_eq!(metadata.get("step").map(String::as_str), Some("1200"));
for ((_, param), (value, first, second)) in restored.iter_mut().zip(&expected) {
assert_eq!(param.value.data, value.data, "weights");
let (one, two) = param.moments().unwrap();
assert_eq!(&one.data, &first.data, "first moment");
assert_eq!(&two.data, &second.data, "second moment");
assert!(one.data.iter().any(|v| *v != 0.0), "a moment to compare");
}
std::fs::remove_file(&path).ok();
}
#[test]
fn a_frozen_parameter_saves_weights_alone() {
let path = scratch("frozen");
let mut param = stepped(2, 2, 1.0);
param.freeze();
save(
&path,
&mut [("tower.weight".to_string(), &mut param)],
&BTreeMap::new(),
)
.unwrap();
let file = SafeTensors::open(&path).unwrap();
let names: Vec<&str> = file.names().collect();
assert_eq!(names, ["tower.weight"], "no moments for a frozen parameter");
std::fs::remove_file(&path).ok();
}
#[test]
fn a_shape_mismatch_names_the_parameter() {
let path = scratch("mismatch");
let mut param = stepped(3, 2, 1.0);
save(
&path,
&mut [("w".to_string(), &mut param)],
&BTreeMap::new(),
)
.unwrap();
let mut wrong = Param::new(Matrix::new(2, 3));
let error = load(&path, &mut [("w".to_string(), &mut wrong)]).unwrap_err();
assert!(
error.to_string().contains("3x2 in the checkpoint"),
"{error}"
);
std::fs::remove_file(&path).ok();
}
#[test]
fn a_missing_parameter_is_an_error() {
let path = scratch("missing");
let mut param = stepped(2, 2, 1.0);
save(
&path,
&mut [("w".to_string(), &mut param)],
&BTreeMap::new(),
)
.unwrap();
let mut other = Param::new(Matrix::new(2, 2));
assert!(load(&path, &mut [("elsewhere".to_string(), &mut other)]).is_err());
std::fs::remove_file(&path).ok();
}
}