use crate::{
Device, RustTorchError, Tensor,
nn::VarStore,
optim::{Optimizer, OptimizerState},
};
use safetensors::{SafeTensors, tensor::TensorView};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use std::{
collections::{BTreeMap, BTreeSet, HashMap},
fs::{self, File, OpenOptions},
io::{self, Read, Write},
path::{Path, PathBuf},
sync::atomic::{AtomicU64, Ordering},
};
use zip::{ZipArchive, ZipWriter, write::FileOptions};
const VERSION: u32 = 1;
const WEIGHTS: &str = "model.safetensors";
const OPTIMIZER: &str = "optimizer.json";
const METADATA: &str = "metadata.json";
static TEMPORARY_ID: AtomicU64 = AtomicU64::new(0);
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum CheckpointError {
#[error("invalid training checkpoint: {0}")]
Invalid(String),
#[error(transparent)]
Io(#[from] io::Error),
#[error(transparent)]
Json(#[from] serde_json::Error),
#[error(transparent)]
Archive(#[from] zip::result::ZipError),
#[error(transparent)]
Weights(#[from] safetensors::SafeTensorError),
#[error(transparent)]
Model(#[from] RustTorchError),
#[error(transparent)]
Backend(#[from] tch::TchError),
}
pub type CheckpointResult<T> = std::result::Result<T, CheckpointError>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CheckpointLimits {
pub archive_bytes: usize,
pub weight_bytes: usize,
pub json_bytes: usize,
pub tensors: usize,
}
impl Default for CheckpointLimits {
fn default() -> Self {
Self {
archive_bytes: 1024 * 1024 * 1024,
weight_bytes: 512 * 1024 * 1024,
json_bytes: 256 * 1024 * 1024,
tensors: 100_000,
}
}
}
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Metadata<S> {
version: u32,
state: S,
}
#[derive(Debug)]
pub struct TrainingCheckpoint<S> {
weights: BTreeMap<String, Tensor>,
optimizer: OptimizerState,
state: S,
}
impl<S: DeserializeOwned> TrainingCheckpoint<S> {
pub fn read(path: impl AsRef<Path>, limits: CheckpointLimits) -> CheckpointResult<Self> {
let path = path.as_ref();
if !fs::symlink_metadata(path)?.file_type().is_file() {
return Err(invalid("checkpoint must be a regular file"));
}
let file = File::open(path)?;
check_size("archive", file.metadata()?.len(), limits.archive_bytes)?;
let bytes = read_bounded(file, limits.archive_bytes)?;
crate::deployment::validate_zip_envelope(&bytes, 3, limits.archive_bytes)?;
let mut archive = ZipArchive::new(io::Cursor::new(bytes))?;
if archive.len() != 3 {
return Err(invalid("archive must contain exactly three members"));
}
let mut members = BTreeMap::new();
let mut total = 0_usize;
for index in 0..archive.len() {
let mut entry = archive.by_index(index)?;
let name = entry.name().to_owned();
let limit = match name.as_str() {
WEIGHTS => limits.weight_bytes,
OPTIMIZER | METADATA => limits.json_bytes,
_ => return Err(invalid("unexpected archive member")),
};
if entry.is_dir()
|| entry
.unix_mode()
.is_some_and(|mode| mode & 0o170000 != 0 && mode & 0o170000 != 0o100000)
{
return Err(invalid("archive members must be ordinary files"));
}
if members.contains_key(&name) {
return Err(invalid("duplicate archive member"));
}
check_size(&name, entry.size(), limit)?;
total = total
.checked_add(
usize::try_from(entry.size())
.map_err(|_| invalid("entry size exceeds usize"))?,
)
.filter(|&total| total <= limits.archive_bytes)
.ok_or_else(|| invalid("combined member payload exceeds archive limit"))?;
let content = read_bounded(&mut entry, limit)?;
if content.len() as u64 != entry.size() {
return Err(invalid(
"archive member's decoded size differs from its directory",
));
}
members.insert(name, content);
}
let metadata: Metadata<S> = serde_json::from_slice(&members[METADATA])?;
if metadata.version != VERSION {
return Err(invalid("unsupported checkpoint schema version"));
}
let optimizer = serde_json::from_slice(&members[OPTIMIZER])?;
let tensors = SafeTensors::deserialize(&members[WEIGHTS])?;
if tensors.len() > limits.tensors {
return Err(invalid("too many named tensors"));
}
let mut weights = BTreeMap::new();
for (name, view) in tensors.tensors() {
if name.is_empty() || name.len() > 4096 || view.shape().len() > 64 {
return Err(invalid("invalid tensor name or rank"));
}
let shape = view
.shape()
.iter()
.map(|&dimension| {
i64::try_from(dimension).map_err(|_| invalid("tensor dimension exceeds i64"))
})
.collect::<CheckpointResult<Vec<_>>>()?;
let kind = tch::Kind::try_from(view.dtype())?;
let count = view
.shape()
.iter()
.try_fold(1usize, |n, &d| n.checked_mul(d))
.ok_or_else(|| invalid("tensor shape product overflow"))?;
if count.checked_mul(kind.elt_size_in_bytes()) != Some(view.data().len()) {
return Err(invalid("tensor byte count mismatch"));
}
validate_tensor_bytes(kind, view.data())?;
let mut bytes = view.data().to_vec();
if cfg!(target_endian = "big") {
for element in bytes.chunks_exact_mut(kind.elt_size_in_bytes()) {
element.reverse();
}
}
weights.insert(name, Tensor::f_from_data_size(&bytes, &shape, kind)?);
}
Ok(Self {
weights,
optimizer,
state: metadata.state,
})
}
}
impl<S> TrainingCheckpoint<S> {
pub const fn state(&self) -> &S {
&self.state
}
pub fn restore(self, store: &VarStore, optimizer: &mut Optimizer) -> CheckpointResult<S> {
let destination = store.variables().into_iter().collect::<BTreeMap<_, _>>();
if destination.keys().collect::<BTreeSet<_>>() != self.weights.keys().collect() {
return Err(invalid("model parameter/buffer names differ"));
}
ensure_optimizer_store(store, optimizer)?;
let mut prepared = Vec::new();
let mut ranges: HashMap<Device, Vec<(usize, usize, &str)>> = HashMap::new();
for (name, target) in &destination {
let saved = &self.weights[name];
if target.size() != saved.size()
|| target.kind() != saved.kind()
|| !target.is_contiguous()
{
return Err(invalid(&format!(
"model tensor metadata differs for {name}"
)));
}
let bytes = target
.numel()
.checked_mul(target.kind().elt_size_in_bytes())
.ok_or_else(|| invalid("model byte count overflow"))?;
if bytes != 0 {
let start = target.data_ptr() as usize;
let end = start
.checked_add(bytes)
.ok_or_else(|| invalid("model storage range overflow"))?;
ranges
.entry(target.device())
.or_default()
.push((start, end, name));
}
prepared.push((target.shallow_clone(), saved.f_to_device(target.device())?));
}
for device_ranges in ranges.values_mut() {
device_ranges.sort_unstable();
for pair in device_ranges.windows(2) {
let (start, end, name) = pair[1];
let (previous_start, previous_end, previous_name) = pair[0];
let saved = &self.weights[name];
let previous = &self.weights[previous_name];
if start < previous_end
&& (start != previous_start
|| end != previous_end
|| saved.kind() != previous.kind()
|| saved.size() != previous.size()
|| !saved.f_equal(previous)?)
{
return Err(invalid(
"overlapping model tensors have incompatible checkpoint values",
));
}
}
}
let prepared_optimizer = optimizer.prepare_state_dict(&self.optimizer)?;
crate::no_grad(|| -> CheckpointResult<()> {
for (mut target, value) in prepared {
target.f_copy_(&value)?;
}
Ok(())
})?;
optimizer.apply_prepared_state(prepared_optimizer);
Ok(self.state)
}
}
pub fn save<S: Serialize>(
path: impl AsRef<Path>,
store: &VarStore,
optimizer: &mut Optimizer,
state: &S,
limits: CheckpointLimits,
) -> CheckpointResult<()> {
let path = path.as_ref();
if path.file_name().is_none() {
return Err(invalid("checkpoint destination needs a file name"));
}
if fs::symlink_metadata(path).is_ok() {
return Err(invalid("checkpoint destination already exists"));
}
let variables = store.variables().into_iter().collect::<BTreeMap<_, _>>();
if variables.len() > limits.tensors {
return Err(invalid("too many named tensors"));
}
let mut size = 0_usize;
let mut payloads = Vec::new();
for (name, tensor) in variables {
if name.is_empty() || name.len() > 4096 || tensor.size().len() > 64 {
return Err(invalid("invalid tensor name or rank"));
}
let count = tensor.numel();
let bytes = count
.checked_mul(tensor.kind().elt_size_in_bytes())
.ok_or_else(|| invalid("tensor byte size overflow"))?;
size = size
.checked_add(bytes)
.filter(|&n| n <= limits.weight_bytes)
.ok_or_else(|| invalid("model exceeds checkpoint weight limit"))?;
let tensor = tensor
.f_detach()?
.f_to_device(Device::Cpu)?
.f_contiguous()?;
let mut data = Vec::new();
data.try_reserve_exact(bytes)
.map_err(|_| invalid("cannot reserve model payload"))?;
data.resize(bytes, 0);
tensor.f_copy_data_u8(&mut data, count)?;
validate_tensor_bytes(tensor.kind(), &data)?;
if cfg!(target_endian = "big") {
for element in data.chunks_exact_mut(tensor.kind().elt_size_in_bytes()) {
element.reverse();
}
}
payloads.push((
name,
tensor.kind().try_into()?,
tensor
.size()
.into_iter()
.map(|d| d as usize)
.collect::<Vec<_>>(),
data,
));
}
let views = payloads
.iter()
.map(|(name, dtype, shape, bytes)| {
Ok((
name.as_str(),
TensorView::new(*dtype, shape.clone(), bytes)?,
))
})
.collect::<std::result::Result<Vec<_>, safetensors::SafeTensorError>>()?;
let weights = safetensors::tensor::serialize(views, &None)?;
check_size(WEIGHTS, weights.len() as u64, limits.weight_bytes)?;
let optimizer_state = optimizer.state_dict()?;
ensure_optimizer_store(store, optimizer)?;
let optimizer_json = json_bounded(&optimizer_state, limits.json_bytes)?;
let metadata = json_bounded(
&Metadata {
version: VERSION,
state,
},
limits.json_bytes,
)?;
let parent = path
.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new("."));
let (temporary, file) = create_temporary(parent)?;
let cleanup = Temporary(temporary.clone());
let mut zip = ZipWriter::new(file);
let options = FileOptions::default()
.compression_method(zip::CompressionMethod::Stored)
.unix_permissions(0o600);
for (name, bytes) in [
(WEIGHTS, weights.as_slice()),
(OPTIMIZER, optimizer_json.as_slice()),
(METADATA, metadata.as_slice()),
] {
zip.start_file(name, options)?;
zip.write_all(bytes)?;
}
let file = zip.finish()?;
check_size("archive", file.metadata()?.len(), limits.archive_bytes)?;
file.sync_all()?;
drop(file);
fs::hard_link(&temporary, path)?;
drop(cleanup);
Ok(())
}
fn ensure_optimizer_store(store: &VarStore, optimizer: &Optimizer) -> CheckpointResult<()> {
if optimizer.belongs_to(store) {
Ok(())
} else {
Err(invalid("optimizer belongs to a different model store"))
}
}
fn validate_tensor_bytes(kind: tch::Kind, bytes: &[u8]) -> CheckpointResult<()> {
if kind == tch::Kind::Bool && bytes.iter().any(|&value| value > 1) {
return Err(invalid("boolean tensor payload must contain only 0 or 1"));
}
Ok(())
}
fn create_temporary(parent: &Path) -> io::Result<(PathBuf, File)> {
for _ in 0..100 {
let id = TEMPORARY_ID.fetch_add(1, Ordering::Relaxed);
let path = parent.join(format!(
".rusttorch-checkpoint-{}-{id}.tmp",
std::process::id()
));
match OpenOptions::new().write(true).create_new(true).open(&path) {
Ok(file) => return Ok((path, file)),
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(error),
}
}
Err(io::Error::new(
io::ErrorKind::AlreadyExists,
"checkpoint temporary-name collision limit reached",
))
}
struct Temporary(PathBuf);
impl Drop for Temporary {
fn drop(&mut self) {
let _ = fs::remove_file(&self.0);
}
}
struct BoundedJson {
bytes: Vec<u8>,
limit: usize,
}
impl Write for BoundedJson {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
if self
.bytes
.len()
.checked_add(bytes.len())
.is_none_or(|len| len > self.limit)
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"checkpoint JSON exceeds resource limit",
));
}
self.bytes
.try_reserve(bytes.len())
.map_err(io::Error::other)?;
self.bytes.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
fn json_bounded(value: &impl Serialize, limit: usize) -> CheckpointResult<Vec<u8>> {
let mut writer = BoundedJson {
bytes: Vec::new(),
limit,
};
serde_json::to_writer(&mut writer, value)?;
Ok(writer.bytes)
}
fn read_bounded(reader: impl Read, limit: usize) -> CheckpointResult<Vec<u8>> {
let mut bytes = Vec::new();
reader
.take((limit as u64).saturating_add(1))
.read_to_end(&mut bytes)?;
if bytes.len() > limit {
return Err(invalid("checkpoint payload exceeds resource limit"));
}
Ok(bytes)
}
fn check_size(name: &str, size: u64, limit: usize) -> CheckpointResult<()> {
if size > limit as u64 {
Err(invalid(&format!("{name} exceeds resource limit")))
} else {
Ok(())
}
}
fn invalid(reason: &str) -> CheckpointError {
CheckpointError::Invalid(reason.into())
}