use crate::amp::LossScalerState;
use crate::layers::WeightQuantization;
use crate::model::{CausalLM, CausalLmConfig};
use crate::optimizer::{AdamW, AdamWConfig, AdamWState, ParameterMoments};
use crate::profile;
use crate::tokenizer::BpeTokenizer;
use crate::trainer::{LearningRateSchedule, TrainerConfig, TrainerError};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, File};
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::{Path, PathBuf};
pub const CHECKPOINT_MAGIC: &[u8; 8] = b"OXLMCKP2";
pub const CHECKPOINT_VERSION: u32 = 2;
pub const LEGACY_CHECKPOINT_VERSION: u32 = 1;
pub const QUANTIZATION_SCHEMA_VERSION: u32 = 3;
pub const QUANTIZATION_SCHEMA_VERSION_MIN: u32 = 1;
const DTYPE_F32: &str = "F32";
const DTYPE_U32: &str = "U32";
const KNOWN_QUANTIZE_METHODS: &[&str] = &[
"rtn", "gptq", "awq", "smoothquant", "hqq", "spqr", "quip", "aqlm", "qat", "exported",
];
pub fn validate_quantization_metadata(
quantization: &CheckpointQuantization,
) -> Result<(), TrainerError> {
if quantization.schema_version < QUANTIZATION_SCHEMA_VERSION_MIN
|| quantization.schema_version > QUANTIZATION_SCHEMA_VERSION
{
return Err(TrainerError::InvalidData(format!(
"unsupported quantization schema version {} (supported {}..={})",
quantization.schema_version,
QUANTIZATION_SCHEMA_VERSION_MIN,
QUANTIZATION_SCHEMA_VERSION
)));
}
let method = quantization.method.trim().to_ascii_lowercase();
if !KNOWN_QUANTIZE_METHODS.iter().any(|known| *known == method) {
return Err(TrainerError::InvalidData(format!(
"unknown quantization method '{}'; expected one of {}",
quantization.method,
KNOWN_QUANTIZE_METHODS.join("|")
)));
}
if let Some(format) = quantization.format.as_deref() {
WeightQuantization::parse(format).map_err(|message| {
TrainerError::InvalidData(format!(
"unknown quantization format in artifact metadata: {message}"
))
})?;
}
for layer in &quantization.layers {
if let Some(granularity) = layer.granularity.as_deref() {
crate::weight_quant::QuantGranularity::parse(granularity).map_err(|message| {
TrainerError::InvalidData(format!(
"layer {}: unknown granularity: {message}",
layer.name
))
})?;
}
if let Some(scale_method) = layer.scale_method.as_deref() {
crate::weight_quant::ScaleMethod::parse(scale_method).map_err(|message| {
TrainerError::InvalidData(format!(
"layer {}: unknown scale method: {message}",
layer.name
))
})?;
}
}
if let Some(activation) = &quantization.activation {
crate::act_quant::ActivationQuantization::parse(&activation.format).map_err(|message| {
TrainerError::InvalidData(format!(
"unknown activation quantization format: {message}"
))
})?;
crate::act_quant::ActGranularity::parse(&activation.granularity).map_err(|message| {
TrainerError::InvalidData(format!(
"unknown activation granularity: {message}"
))
})?;
crate::act_quant::ActQuantMode::parse(&activation.mode).map_err(|message| {
TrainerError::InvalidData(format!("unknown activation mode: {message}"))
})?;
}
if let Some(kv) = &quantization.kv_cache {
crate::kv_quant::KvQuantization::parse(&kv.format).map_err(|message| {
TrainerError::InvalidData(format!("unknown KV quantization format: {message}"))
})?;
crate::kv_quant::KvGranularity::parse(&kv.granularity).map_err(|message| {
TrainerError::InvalidData(format!("unknown KV granularity: {message}"))
})?;
}
Ok(())
}
fn map_quantization_serde_error(error: serde_json::Error) -> TrainerError {
let message = error.to_string();
let lowered = message.to_ascii_lowercase();
if lowered.contains("unknown variant")
|| lowered.contains("quantization")
|| lowered.contains("weightquantization")
{
TrainerError::InvalidData(format!(
"unsupported quantization artifact ({message}); schema_version must be {}..={} with a known scheme",
QUANTIZATION_SCHEMA_VERSION_MIN, QUANTIZATION_SCHEMA_VERSION
))
} else {
TrainerError::Json(error)
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct RngState {
pub model_seed: u64,
pub dropout_stream: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct DataLoaderState {
pub seed: u64,
pub cursor: usize,
pub sequence_length: usize,
pub batch_size: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
struct TensorDescriptor {
name: String,
shape: Vec<usize>,
dtype: String,
offset: u64,
nbytes: u64,
crc32: u32,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct CheckpointQuantization {
pub schema_version: u32,
pub method: String,
pub bits: u32,
#[serde(default)]
pub group_size: Option<usize>,
#[serde(default = "default_true")]
pub symmetric: bool,
#[serde(default)]
pub zero_points: bool,
#[serde(default)]
pub scale_method: Option<String>,
#[serde(default)]
pub granularity: Option<String>,
#[serde(default)]
pub format: Option<String>,
#[serde(default)]
pub inference_only: bool,
#[serde(default)]
pub excluded: Vec<String>,
#[serde(default)]
pub included: Option<Vec<String>>,
pub layers: Vec<QuantizedLayerEntry>,
#[serde(default)]
pub activation: Option<CheckpointActivationQuant>,
#[serde(default)]
pub kv_cache: Option<CheckpointKvQuant>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct CheckpointActivationQuant {
pub format: String,
pub granularity: String,
pub mode: String,
#[serde(default)]
pub layers: Vec<ActivationLayerEntry>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct ActivationLayerEntry {
pub name: String,
#[serde(default)]
pub scale: Option<f32>,
pub scales_name: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct CheckpointKvQuant {
pub format: String,
pub granularity: String,
}
fn default_true() -> bool {
true
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct QuantizedLayerEntry {
pub name: String,
pub format: WeightQuantization,
pub in_features: usize,
pub out_features: usize,
pub packed_name: String,
pub scales_name: String,
#[serde(default)]
pub zero_points_name: Option<String>,
#[serde(default)]
pub group_size: Option<usize>,
#[serde(default = "default_true")]
pub symmetric: bool,
#[serde(default)]
pub scale_method: Option<String>,
#[serde(default)]
pub granularity: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
struct CheckpointMetadata {
version: u32,
model_config: CausalLmConfig,
optimizer_config: AdamWConfig,
optimizer_step: usize,
tokenizer_json: Vec<u8>,
schedule: LearningRateSchedule,
trainer_config: TrainerConfig,
training_step: usize,
tokens_processed: usize,
#[serde(default)]
best_validation_loss: Option<f32>,
#[serde(default)]
evaluations_without_improvement: usize,
#[serde(default)]
loss_scaler: Option<LossScalerState>,
rng: RngState,
data_loader: DataLoaderState,
tensors: Vec<TensorDescriptor>,
#[serde(default)]
quantization: Option<CheckpointQuantization>,
}
#[derive(Serialize, Deserialize)]
struct LegacyCheckpointFile {
version: u32,
model_config: CausalLmConfig,
weights: Vec<Vec<f32>>,
optimizer_config: AdamWConfig,
optimizer_state: AdamWState,
tokenizer_json: Vec<u8>,
schedule: LearningRateSchedule,
trainer_config: TrainerConfig,
training_step: usize,
cursor: usize,
tokens_processed: usize,
#[serde(default)]
best_validation_loss: Option<f32>,
#[serde(default)]
evaluations_without_improvement: usize,
#[serde(default)]
loss_scaler: Option<LossScalerState>,
}
pub struct LoadedCheckpoint {
pub version: u32,
pub model: CausalLM,
pub optimizer: AdamW,
pub tokenizer: BpeTokenizer,
pub schedule: LearningRateSchedule,
pub trainer_config: TrainerConfig,
pub training_step: usize,
pub cursor: usize,
pub tokens_processed: usize,
pub best_validation_loss: Option<f32>,
pub evaluations_without_improvement: usize,
pub loss_scaler: Option<LossScalerState>,
pub rng: RngState,
pub data_loader: DataLoaderState,
}
enum TensorData {
F32(Vec<f32>),
U32(Vec<u32>),
}
struct NamedTensor {
name: String,
shape: Vec<usize>,
data: TensorData,
}
impl NamedTensor {
fn f32(name: String, shape: Vec<usize>, data: Vec<f32>) -> Self {
Self {
name,
shape,
data: TensorData::F32(data),
}
}
fn u32(name: String, shape: Vec<usize>, data: Vec<u32>) -> Self {
Self {
name,
shape,
data: TensorData::U32(data),
}
}
fn as_f32(&self) -> Result<&[f32], TrainerError> {
match &self.data {
TensorData::F32(values) => Ok(values),
TensorData::U32(_) => Err(TrainerError::InvalidData(format!(
"tensor {} is U32, expected F32",
self.name
))),
}
}
fn as_u32(&self) -> Result<&[u32], TrainerError> {
match &self.data {
TensorData::U32(values) => Ok(values),
TensorData::F32(_) => Err(TrainerError::InvalidData(format!(
"tensor {} is F32, expected U32",
self.name
))),
}
}
fn dtype(&self) -> &'static str {
match self.data {
TensorData::F32(_) => DTYPE_F32,
TensorData::U32(_) => DTYPE_U32,
}
}
fn to_le_bytes(&self) -> Vec<u8> {
match &self.data {
TensorData::F32(values) => f32_slice_to_le_bytes(values),
TensorData::U32(values) => u32_slice_to_le_bytes(values),
}
}
}
#[allow(clippy::too_many_arguments)]
pub fn save_checkpoint(
path: impl AsRef<Path>,
model: &CausalLM,
optimizer: &AdamW,
tokenizer: &BpeTokenizer,
schedule: &LearningRateSchedule,
trainer_config: &TrainerConfig,
training_step: usize,
cursor: usize,
tokens_processed: usize,
) -> Result<(), TrainerError> {
let micro_batches_processed =
training_step.saturating_mul(trainer_config.gradient_accumulation_steps);
save_checkpoint_with_best(
path,
model,
optimizer,
tokenizer,
schedule,
trainer_config,
training_step,
cursor,
tokens_processed,
micro_batches_processed,
None,
0,
None,
)
}
#[allow(clippy::too_many_arguments)]
pub fn save_checkpoint_with_best(
path: impl AsRef<Path>,
model: &CausalLM,
optimizer: &AdamW,
tokenizer: &BpeTokenizer,
schedule: &LearningRateSchedule,
trainer_config: &TrainerConfig,
training_step: usize,
cursor: usize,
tokens_processed: usize,
micro_batches_processed: usize,
best_validation_loss: Option<f32>,
evaluations_without_improvement: usize,
loss_scaler: Option<LossScalerState>,
) -> Result<(), TrainerError> {
let _span = profile::span("checkpoint.save");
let path = path.as_ref();
let rng = RngState {
model_seed: model.config.seed,
dropout_stream: micro_batches_processed,
};
let data_loader = DataLoaderState {
seed: model.config.seed,
cursor,
sequence_length: trainer_config.sequence_length,
batch_size: trainer_config.batch_size,
};
let tensors = collect_named_tensors(model, optimizer)?;
let (metadata, payload) = build_metadata_and_payload(
model,
optimizer,
tokenizer,
schedule,
trainer_config,
training_step,
tokens_processed,
best_validation_loss,
evaluations_without_improvement,
loss_scaler,
rng,
data_loader,
&tensors,
)?;
let temp_path = temporary_path(path);
write_binary_checkpoint(&temp_path, &metadata, &payload)?;
validate_binary_file(&temp_path)?;
fs::rename(&temp_path, path).map_err(|error| {
let _ = fs::remove_file(&temp_path);
TrainerError::Io(error)
})?;
Ok(())
}
pub fn load_checkpoint(path: impl AsRef<Path>) -> Result<LoadedCheckpoint, TrainerError> {
let _span = profile::span("checkpoint.load");
let path = path.as_ref();
let mut file = File::open(path)?;
let mut magic = [0u8; 8];
match file.read_exact(&mut magic) {
Ok(()) if &magic == CHECKPOINT_MAGIC => load_binary_checkpoint(path),
Ok(()) => load_legacy_json_checkpoint(path),
Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => {
load_legacy_json_checkpoint(path)
}
Err(error) => Err(TrainerError::Io(error)),
}
}
pub fn save_inference_checkpoint(
path: impl AsRef<Path>,
model: &CausalLM,
tokenizer: &BpeTokenizer,
quantization: &CheckpointQuantization,
) -> Result<(), TrainerError> {
let _span = profile::span("checkpoint.save_inference");
validate_quantization_metadata(quantization)?;
let path = path.as_ref();
let quantized_modules: BTreeSet<String> = quantization
.layers
.iter()
.map(|layer| layer.name.clone())
.collect();
let mut tensors = Vec::new();
for (name, parameter) in model.named_parameters() {
let skip_fp32_master = name.ends_with(".weight")
&& quantized_modules
.iter()
.any(|module| name == format!("{module}.weight"));
if skip_fp32_master {
continue;
}
tensors.push(NamedTensor::f32(
name,
parameter.shape(),
parameter.to_cpu()?.data(),
));
}
for entry in &quantization.layers {
let linear = model
.named_linears()
.into_iter()
.find(|(name, _)| name == &entry.name)
.map(|(_, linear)| linear)
.ok_or_else(|| {
TrainerError::InvalidData(format!(
"quantization metadata references unknown linear {}",
entry.name
))
})?;
let quant = linear.quantized_weight().ok_or_else(|| {
TrainerError::InvalidData(format!(
"linear {} is listed as quantized but has no packed weight",
entry.name
))
})?;
tensors.push(NamedTensor::u32(
entry.packed_name.clone(),
vec![quant.packed_values().len()],
quant.packed_values().to_vec(),
));
tensors.push(NamedTensor::f32(
entry.scales_name.clone(),
vec![quant.scales().len()],
quant.scales().to_vec(),
));
if let Some(zero_points) = quant.zero_points() {
let name = entry.zero_points_name.clone().ok_or_else(|| {
TrainerError::InvalidData(format!(
"asymmetric quantized layer {} missing zero_points_name",
entry.name
))
})?;
tensors.push(NamedTensor::f32(
name,
vec![zero_points.len()],
zero_points.to_vec(),
));
}
}
if let Some(activation) = &quantization.activation {
for entry in &activation.layers {
if let (Some(scale), Some(name)) = (entry.scale, entry.scales_name.as_ref()) {
tensors.push(NamedTensor::f32(name.clone(), vec![1], vec![scale]));
}
}
}
let optimizer_config = AdamWConfig::default();
let schedule = LearningRateSchedule {
base_learning_rate: 0.0,
minimum_learning_rate: 0.0,
warmup_steps: 0,
total_steps: 0,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: 1.min(model.config.max_sequence_length).max(1),
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
let rng = RngState {
model_seed: model.config.seed,
dropout_stream: 0,
};
let data_loader = DataLoaderState {
seed: model.config.seed,
cursor: 0,
sequence_length: trainer_config.sequence_length,
batch_size: trainer_config.batch_size,
};
let (metadata, payload) = build_metadata_and_payload_with_quantization(
model.config.clone(),
&optimizer_config,
0,
tokenizer,
&schedule,
&trainer_config,
0,
0,
None,
0,
None,
rng,
data_loader,
&tensors,
Some(quantization.clone()),
)?;
let temp_path = temporary_path(path);
write_binary_checkpoint(&temp_path, &metadata, &payload)?;
validate_binary_file(&temp_path)?;
fs::rename(&temp_path, path).map_err(|error| {
let _ = fs::remove_file(&temp_path);
TrainerError::Io(error)
})?;
Ok(())
}
fn collect_named_tensors(
model: &CausalLM,
optimizer: &AdamW,
) -> Result<Vec<NamedTensor>, TrainerError> {
let state = optimizer.checkpoint_state()?;
let named = model.named_parameters();
let trainable: Vec<_> = named
.iter()
.filter(|(_, parameter)| parameter.requires_grad())
.collect();
if trainable.len() != state.moments.len() {
return Err(TrainerError::InvalidData(
"optimizer moment count does not match trainable parameters".into(),
));
}
let mut tensors = Vec::with_capacity(named.len() + trainable.len() * 2);
for (name, parameter) in &named {
tensors.push(NamedTensor::f32(
name.clone(),
parameter.shape(),
parameter.to_cpu()?.data(),
));
}
for ((name, _), moments) in trainable.iter().zip(state.moments.iter()) {
let first = moments.first.to_f32_vec();
let second = moments.second.to_f32_vec();
tensors.push(NamedTensor::f32(
format!("optimizer.m.{name}"),
vec![first.len()],
first,
));
tensors.push(NamedTensor::f32(
format!("optimizer.v.{name}"),
vec![second.len()],
second,
));
}
Ok(tensors)
}
#[allow(clippy::too_many_arguments)]
fn build_metadata_and_payload(
model: &CausalLM,
optimizer: &AdamW,
tokenizer: &BpeTokenizer,
schedule: &LearningRateSchedule,
trainer_config: &TrainerConfig,
training_step: usize,
tokens_processed: usize,
best_validation_loss: Option<f32>,
evaluations_without_improvement: usize,
loss_scaler: Option<LossScalerState>,
rng: RngState,
data_loader: DataLoaderState,
tensors: &[NamedTensor],
) -> Result<(CheckpointMetadata, Vec<u8>), TrainerError> {
build_metadata_and_payload_with_quantization(
model.config.clone(),
optimizer.config(),
optimizer.state().step,
tokenizer,
schedule,
trainer_config,
training_step,
tokens_processed,
best_validation_loss,
evaluations_without_improvement,
loss_scaler,
rng,
data_loader,
tensors,
None,
)
}
#[allow(clippy::too_many_arguments)]
fn build_metadata_and_payload_with_quantization(
model_config: CausalLmConfig,
optimizer_config: &AdamWConfig,
optimizer_step: usize,
tokenizer: &BpeTokenizer,
schedule: &LearningRateSchedule,
trainer_config: &TrainerConfig,
training_step: usize,
tokens_processed: usize,
best_validation_loss: Option<f32>,
evaluations_without_improvement: usize,
loss_scaler: Option<LossScalerState>,
rng: RngState,
data_loader: DataLoaderState,
tensors: &[NamedTensor],
quantization: Option<CheckpointQuantization>,
) -> Result<(CheckpointMetadata, Vec<u8>), TrainerError> {
if let Some(quantization) = &quantization {
validate_quantization_metadata(quantization)?;
}
let mut payload = Vec::new();
let mut descriptors = Vec::with_capacity(tensors.len());
for tensor in tensors {
let offset = payload.len() as u64;
let bytes = tensor.to_le_bytes();
let crc32 = crc32(&bytes);
let nbytes = bytes.len() as u64;
payload.extend_from_slice(&bytes);
descriptors.push(TensorDescriptor {
name: tensor.name.clone(),
shape: tensor.shape.clone(),
dtype: tensor.dtype().into(),
offset,
nbytes,
crc32,
});
}
let metadata = CheckpointMetadata {
version: CHECKPOINT_VERSION,
model_config,
optimizer_config: optimizer_config.clone(),
optimizer_step,
tokenizer_json: tokenizer.to_json_bytes()?,
schedule: schedule.clone(),
trainer_config: trainer_config.clone(),
training_step,
tokens_processed,
best_validation_loss,
evaluations_without_improvement,
loss_scaler,
rng,
data_loader,
tensors: descriptors,
quantization,
};
Ok((metadata, payload))
}
fn write_binary_checkpoint(
path: &Path,
metadata: &CheckpointMetadata,
payload: &[u8],
) -> Result<(), TrainerError> {
let metadata_bytes = serde_json::to_vec(metadata)?;
let mut writer = BufWriter::new(File::create(path)?);
writer.write_all(CHECKPOINT_MAGIC)?;
writer.write_all(&CHECKPOINT_VERSION.to_le_bytes())?;
writer.write_all(&(metadata_bytes.len() as u64).to_le_bytes())?;
writer.write_all(&metadata_bytes)?;
writer.write_all(payload)?;
writer.flush()?;
writer.get_ref().sync_all()?;
Ok(())
}
fn validate_binary_file(path: &Path) -> Result<(), TrainerError> {
let (metadata, payload) = read_binary_parts(path)?;
verify_tensors(&metadata, &payload)?;
Ok(())
}
fn load_binary_checkpoint(path: &Path) -> Result<LoadedCheckpoint, TrainerError> {
let (metadata, payload) = read_binary_parts(path)?;
if metadata.version != CHECKPOINT_VERSION {
return Err(TrainerError::InvalidData(format!(
"unsupported checkpoint version {}",
metadata.version
)));
}
if let Some(quantization) = &metadata.quantization {
validate_quantization_metadata(quantization)?;
}
let tensors = verify_tensors(&metadata, &payload)?;
let quantized_weight_names: BTreeSet<String> = metadata
.quantization
.as_ref()
.map(|quantization| {
quantization
.layers
.iter()
.map(|layer| format!("{}.weight", layer.name))
.collect()
})
.unwrap_or_default();
let mut model = CausalLM::new(metadata.model_config.clone())?;
for (name, parameter) in model.named_parameters() {
if name.contains(".lora.") || name.contains(".qat_scale") {
continue;
}
if quantized_weight_names.contains(&name) {
continue;
}
let Some(storage) = tensors.get(&name) else {
return Err(TrainerError::InvalidData(format!(
"checkpoint is missing tensor {name}"
)));
};
if storage.shape != parameter.shape() {
return Err(TrainerError::InvalidData(format!(
"checkpoint tensor {name} has shape {:?} but model expects {:?}",
storage.shape, parameter.shape()
)));
}
let values = storage.as_f32()?;
if values.len() != parameter.numel() {
return Err(TrainerError::InvalidData(format!(
"checkpoint tensor {name} has the wrong length"
)));
}
parameter.set_data(values.to_vec());
}
if let Some(quantization) = &metadata.quantization {
restore_quantized_linears(&mut model, quantization, &tensors)?;
}
if let Some(lora) = &metadata.trainer_config.low_memory.lora {
model.install_lora(lora)?;
for (name, parameter) in model.named_parameters() {
if !name.contains(".lora.") {
continue;
}
let Some(storage) = tensors.get(&name) else {
return Err(TrainerError::InvalidData(format!(
"checkpoint is missing LoRA tensor {name}"
)));
};
let values = storage.as_f32()?;
if values.len() != parameter.numel() {
return Err(TrainerError::InvalidData(format!(
"checkpoint LoRA tensor {name} has the wrong length"
)));
}
parameter.set_data(values.to_vec());
}
}
if let Some(qat) = &metadata.trainer_config.low_memory.qat {
model.enable_qat_with(qat.clone())?;
for (name, parameter) in model.named_parameters() {
if !name.contains(".qat_scale") {
continue;
}
let Some(storage) = tensors.get(&name) else {
continue;
};
let values = storage.as_f32()?;
if values.len() != parameter.numel() {
return Err(TrainerError::InvalidData(format!(
"checkpoint QAT scale tensor {name} has the wrong length"
)));
}
parameter.set_data(values.to_vec());
}
}
if !metadata.trainer_config.low_memory.freeze.is_empty() {
crate::peft::freeze_parameters(
&model.named_parameters(),
&metadata.trainer_config.low_memory.freeze,
);
}
let inference_only = metadata
.quantization
.as_ref()
.is_some_and(|quantization| quantization.inference_only);
let optimizer = if inference_only {
for (_, parameter) in model.named_parameters() {
parameter.set_requires_grad(false);
}
AdamW::new(&[], metadata.optimizer_config.clone())?
} else {
let moments = moments_from_named(&model, &tensors)?;
let trainable: Vec<_> = model
.named_parameters()
.into_iter()
.filter(|(_, parameter)| parameter.requires_grad())
.collect();
AdamW::from_state_grouped(
&trainable,
metadata.optimizer_config.clone(),
AdamWState {
step: metadata.optimizer_step,
moments,
},
)?
};
let tokenizer = BpeTokenizer::from_json_bytes(&metadata.tokenizer_json)?;
Ok(LoadedCheckpoint {
version: metadata.version,
model,
optimizer,
tokenizer,
schedule: metadata.schedule,
trainer_config: metadata.trainer_config,
training_step: metadata.training_step,
cursor: metadata.data_loader.cursor,
tokens_processed: metadata.tokens_processed,
best_validation_loss: metadata.best_validation_loss,
evaluations_without_improvement: metadata.evaluations_without_improvement,
loss_scaler: metadata.loss_scaler,
rng: metadata.rng,
data_loader: metadata.data_loader,
})
}
fn restore_quantized_linears(
model: &mut CausalLM,
quantization: &CheckpointQuantization,
tensors: &BTreeMap<String, NamedTensor>,
) -> Result<(), TrainerError> {
for entry in &quantization.layers {
let packed = tensors.get(&entry.packed_name).ok_or_else(|| {
TrainerError::InvalidData(format!(
"checkpoint is missing packed tensor {}",
entry.packed_name
))
})?;
let scales = tensors.get(&entry.scales_name).ok_or_else(|| {
TrainerError::InvalidData(format!(
"checkpoint is missing scale tensor {}",
entry.scales_name
))
})?;
let zero_points = if let Some(name) = &entry.zero_points_name {
Some(
tensors
.get(name)
.ok_or_else(|| {
TrainerError::InvalidData(format!(
"checkpoint is missing zero-point tensor {name}"
))
})?
.as_f32()?
.to_vec(),
)
} else {
None
};
let granularity = entry
.granularity
.as_deref()
.or(quantization.granularity.as_deref())
.map(crate::weight_quant::QuantGranularity::parse)
.transpose()
.map_err(TrainerError::InvalidData)?
.unwrap_or_else(|| {
entry
.group_size
.or(quantization.group_size)
.map(|size| crate::weight_quant::QuantGranularity::PerGroup { size })
.unwrap_or(crate::weight_quant::QuantGranularity::PerChannel)
});
let scale_method = entry
.scale_method
.as_deref()
.or(quantization.scale_method.as_deref())
.map(crate::weight_quant::ScaleMethod::parse)
.transpose()
.map_err(TrainerError::InvalidData)?
.unwrap_or(crate::weight_quant::ScaleMethod::AbsMax);
let scheme = crate::weight_quant::QuantScheme {
format: entry.format,
granularity,
symmetric: entry.symmetric,
scale_method,
percentile: 99.99,
};
let weight = crate::layers::QuantizedLinearWeight::from_parts_with_scheme(
scheme,
packed.as_u32()?.to_vec(),
scales.as_f32()?.to_vec(),
zero_points,
entry.in_features,
entry.out_features,
)
.map_err(TrainerError::InvalidData)?;
model
.set_quantized_linear(&entry.name, weight)
.map_err(TrainerError::InvalidData)?;
}
if let Some(activation) = &quantization.activation {
let format = crate::act_quant::ActivationQuantization::parse(&activation.format)
.map_err(TrainerError::InvalidData)?;
let granularity = crate::act_quant::ActGranularity::parse(&activation.granularity)
.map_err(TrainerError::InvalidData)?;
let mode = crate::act_quant::ActQuantMode::parse(&activation.mode)
.map_err(TrainerError::InvalidData)?;
let scheme = crate::act_quant::ActQuantScheme {
format,
granularity,
mode,
};
for entry in &activation.layers {
let config = match mode {
crate::act_quant::ActQuantMode::Static => {
let scale = if let Some(name) = &entry.scales_name {
tensors
.get(name)
.and_then(|tensor| tensor.as_f32().ok())
.and_then(|values| values.first().copied())
.or(entry.scale)
.unwrap_or(1.0)
} else {
entry.scale.unwrap_or(1.0)
};
crate::act_quant::ActQuantConfig::static_per_tensor(scale)
}
crate::act_quant::ActQuantMode::Dynamic => {
crate::act_quant::ActQuantConfig::dynamic(scheme)
.map_err(TrainerError::InvalidData)?
}
};
if let Some((_, linear)) = model
.named_linears_mut()
.into_iter()
.find(|(name, _)| name == &entry.name)
{
linear
.enable_activation_quant(config)
.map_err(TrainerError::InvalidData)?;
}
}
if mode == crate::act_quant::ActQuantMode::Dynamic && activation.layers.is_empty() {
let quantized: BTreeSet<_> = quantization.layers.iter().map(|l| l.name.clone()).collect();
for (name, linear) in model.named_linears_mut() {
if quantized.contains(&name) {
let config = crate::act_quant::ActQuantConfig::dynamic(scheme)
.map_err(TrainerError::InvalidData)?;
linear
.enable_activation_quant(config)
.map_err(TrainerError::InvalidData)?;
}
}
}
}
if let Some(kv) = &quantization.kv_cache {
let format = crate::kv_quant::KvQuantization::parse(&kv.format)
.map_err(TrainerError::InvalidData)?;
let granularity = crate::kv_quant::KvGranularity::parse(&kv.granularity)
.map_err(TrainerError::InvalidData)?;
model.enable_kv_cache_quant(crate::kv_quant::KvQuantScheme {
format,
granularity,
});
}
Ok(())
}
fn load_legacy_json_checkpoint(path: &Path) -> Result<LoadedCheckpoint, TrainerError> {
let checkpoint: LegacyCheckpointFile =
serde_json::from_reader(BufReader::new(File::open(path)?))?;
if checkpoint.version != LEGACY_CHECKPOINT_VERSION {
return Err(TrainerError::InvalidData(format!(
"unsupported checkpoint version {}",
checkpoint.version
)));
}
let rng = RngState {
model_seed: checkpoint.model_config.seed,
dropout_stream: checkpoint
.training_step
.saturating_mul(checkpoint.trainer_config.gradient_accumulation_steps),
};
let data_loader = DataLoaderState {
seed: checkpoint.model_config.seed,
cursor: checkpoint.cursor,
sequence_length: checkpoint.trainer_config.sequence_length,
batch_size: checkpoint.trainer_config.batch_size,
};
let model = CausalLM::new(checkpoint.model_config)?;
let parameters = model.parameters();
if parameters.len() != checkpoint.weights.len() {
return Err(TrainerError::InvalidData(
"checkpoint parameter count does not match model".into(),
));
}
for (index, (parameter, values)) in parameters
.iter()
.zip(checkpoint.weights.into_iter())
.enumerate()
{
if parameter.numel() != values.len() {
return Err(TrainerError::InvalidData(format!(
"checkpoint parameter {index} has the wrong length"
)));
}
parameter.set_data(values);
}
let optimizer = AdamW::from_state_grouped(
&model.named_parameters(),
checkpoint.optimizer_config,
checkpoint.optimizer_state,
)?;
let tokenizer = BpeTokenizer::from_json_bytes(&checkpoint.tokenizer_json)?;
Ok(LoadedCheckpoint {
version: checkpoint.version,
model,
optimizer,
tokenizer,
schedule: checkpoint.schedule,
trainer_config: checkpoint.trainer_config,
training_step: checkpoint.training_step,
cursor: data_loader.cursor,
tokens_processed: checkpoint.tokens_processed,
best_validation_loss: checkpoint.best_validation_loss,
evaluations_without_improvement: checkpoint.evaluations_without_improvement,
loss_scaler: checkpoint.loss_scaler,
rng,
data_loader,
})
}
fn read_binary_parts(path: &Path) -> Result<(CheckpointMetadata, Vec<u8>), TrainerError> {
let mut file = File::open(path)?;
let mut magic = [0u8; 8];
file.read_exact(&mut magic)?;
if &magic != CHECKPOINT_MAGIC {
return Err(TrainerError::InvalidData(
"checkpoint has invalid binary magic".into(),
));
}
let mut version_bytes = [0u8; 4];
file.read_exact(&mut version_bytes)?;
let version = u32::from_le_bytes(version_bytes);
if version != CHECKPOINT_VERSION {
return Err(TrainerError::InvalidData(format!(
"unsupported checkpoint version {version}"
)));
}
let mut len_bytes = [0u8; 8];
file.read_exact(&mut len_bytes)?;
let metadata_len = usize::try_from(u64::from_le_bytes(len_bytes)).map_err(|_| {
TrainerError::InvalidData("checkpoint metadata length is too large".into())
})?;
let mut metadata_bytes = vec![0u8; metadata_len];
file.read_exact(&mut metadata_bytes)?;
let metadata: CheckpointMetadata =
serde_json::from_slice(&metadata_bytes).map_err(map_quantization_serde_error)?;
let mut payload = Vec::new();
file.read_to_end(&mut payload)?;
Ok((metadata, payload))
}
fn verify_tensors(
metadata: &CheckpointMetadata,
payload: &[u8],
) -> Result<BTreeMap<String, NamedTensor>, TrainerError> {
let mut tensors = BTreeMap::new();
for entry in &metadata.tensors {
let start = usize::try_from(entry.offset).map_err(|_| {
TrainerError::InvalidData(format!("tensor {} offset is too large", entry.name))
})?;
let nbytes = usize::try_from(entry.nbytes).map_err(|_| {
TrainerError::InvalidData(format!("tensor {} byte length is too large", entry.name))
})?;
let end = start.checked_add(nbytes).ok_or_else(|| {
TrainerError::InvalidData(format!("tensor {} byte range overflowed", entry.name))
})?;
if end > payload.len() {
return Err(TrainerError::InvalidData(format!(
"tensor {} payload is truncated",
entry.name
)));
}
let bytes = &payload[start..end];
if crc32(bytes) != entry.crc32 {
return Err(TrainerError::InvalidData(format!(
"tensor {} failed checksum validation",
entry.name
)));
}
let expected_numel: usize = entry.shape.iter().copied().product();
let named = match entry.dtype.as_str() {
DTYPE_F32 => {
let data = le_bytes_to_f32_vec(bytes)?;
if data.len() != expected_numel {
return Err(TrainerError::InvalidData(format!(
"tensor {} shape {:?} does not match {} values",
entry.name,
entry.shape,
data.len()
)));
}
NamedTensor::f32(entry.name.clone(), entry.shape.clone(), data)
}
DTYPE_U32 => {
let data = le_bytes_to_u32_vec(bytes)?;
if data.len() != expected_numel {
return Err(TrainerError::InvalidData(format!(
"tensor {} shape {:?} does not match {} values",
entry.name,
entry.shape,
data.len()
)));
}
NamedTensor::u32(entry.name.clone(), entry.shape.clone(), data)
}
other => {
return Err(TrainerError::InvalidData(format!(
"unsupported tensor dtype {other} for {}",
entry.name
)));
}
};
if tensors.insert(entry.name.clone(), named).is_some() {
return Err(TrainerError::InvalidData(format!(
"duplicate tensor name {}",
entry.name
)));
}
}
Ok(tensors)
}
fn moments_from_named(
model: &CausalLM,
tensors: &BTreeMap<String, NamedTensor>,
) -> Result<Vec<ParameterMoments>, TrainerError> {
let mut moments = Vec::new();
for (name, parameter) in model.named_parameters() {
if !parameter.requires_grad() {
continue;
}
let first_name = format!("optimizer.m.{name}");
let second_name = format!("optimizer.v.{name}");
let first = tensors.get(&first_name).ok_or_else(|| {
TrainerError::InvalidData(format!("checkpoint is missing tensor {first_name}"))
})?;
let second = tensors.get(&second_name).ok_or_else(|| {
TrainerError::InvalidData(format!("checkpoint is missing tensor {second_name}"))
})?;
moments.push(ParameterMoments {
first: crate::optimizer::MomentStorage::Fp32(first.as_f32()?.to_vec()),
second: crate::optimizer::MomentStorage::Fp32(second.as_f32()?.to_vec()),
});
}
Ok(moments)
}
fn temporary_path(path: &Path) -> PathBuf {
let mut temp = path.as_os_str().to_os_string();
temp.push(".tmp");
PathBuf::from(temp)
}
fn f32_slice_to_le_bytes(values: &[f32]) -> Vec<u8> {
let mut bytes = Vec::with_capacity(values.len() * 4);
for value in values {
bytes.extend_from_slice(&value.to_le_bytes());
}
bytes
}
fn u32_slice_to_le_bytes(values: &[u32]) -> Vec<u8> {
let mut bytes = Vec::with_capacity(values.len() * 4);
for value in values {
bytes.extend_from_slice(&value.to_le_bytes());
}
bytes
}
fn le_bytes_to_f32_vec(bytes: &[u8]) -> Result<Vec<f32>, TrainerError> {
if !bytes.len().is_multiple_of(4) {
return Err(TrainerError::InvalidData(
"tensor byte length is not a multiple of 4".into(),
));
}
Ok(bytes
.chunks_exact(4)
.map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap()))
.collect())
}
fn le_bytes_to_u32_vec(bytes: &[u8]) -> Result<Vec<u32>, TrainerError> {
if !bytes.len().is_multiple_of(4) {
return Err(TrainerError::InvalidData(
"tensor byte length is not a multiple of 4".into(),
));
}
Ok(bytes
.chunks_exact(4)
.map(|chunk| u32::from_le_bytes(chunk.try_into().unwrap()))
.collect())
}
fn crc32(bytes: &[u8]) -> u32 {
let mut crc = 0xffff_ffffu32;
for byte in bytes {
let index = ((crc ^ u32::from(*byte)) & 0xff) as usize;
crc = (crc >> 8) ^ CRC32_TABLE[index];
}
!crc
}
const CRC32_TABLE: [u32; 256] = {
let mut table = [0u32; 256];
let mut index = 0;
while index < 256 {
let mut value = index as u32;
let mut bit = 0;
while bit < 8 {
if value & 1 != 0 {
value = 0xedb8_8320 ^ (value >> 1);
} else {
value >>= 1;
}
bit += 1;
}
table[index] = value;
index += 1;
}
table
};
#[cfg(test)]
mod tests {
use super::*;
use crate::optimizer::AdamWConfig;
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
use crate::trainer::TrainerConfig;
use std::time::{SystemTime, UNIX_EPOCH};
fn tiny_model_and_tokenizer() -> (CausalLM, BpeTokenizer, AdamW) {
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 7,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let optimizer =
AdamW::new_grouped(&model.named_parameters(), AdamWConfig::default()).unwrap();
(model, tokenizer, optimizer)
}
fn temp_path(label: &str) -> PathBuf {
std::env::temp_dir().join(format!(
"oxidelm-ckpt-{label}-{}.bin",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
))
}
#[test]
fn binary_checkpoint_round_trip_preserves_named_tensors_and_state() {
let (model, tokenizer, optimizer) = tiny_model_and_tokenizer();
let schedule = LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
};
let trainer_config = TrainerConfig {
batch_size: 2,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 3,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
let path = temp_path("roundtrip");
save_checkpoint_with_best(
&path,
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
5,
11,
100,
15,
Some(1.25),
3,
None,
)
.unwrap();
let mut header = [0u8; 8];
File::open(&path).unwrap().read_exact(&mut header).unwrap();
assert_eq!(&header, CHECKPOINT_MAGIC);
assert!(!PathBuf::from(format!("{}.tmp", path.display())).exists());
let loaded = load_checkpoint(&path).unwrap();
let _ = fs::remove_file(&path);
assert_eq!(loaded.version, CHECKPOINT_VERSION);
assert_eq!(loaded.training_step, 5);
assert_eq!(loaded.cursor, 11);
assert_eq!(loaded.tokens_processed, 100);
assert_eq!(loaded.best_validation_loss, Some(1.25));
assert_eq!(loaded.evaluations_without_improvement, 3);
assert_eq!(loaded.rng.model_seed, 7);
assert_eq!(loaded.rng.dropout_stream, 15);
assert_eq!(loaded.data_loader.cursor, 11);
assert_eq!(loaded.data_loader.batch_size, 2);
assert_eq!(loaded.data_loader.sequence_length, 4);
for ((name, parameter), (_, restored)) in model
.named_parameters()
.into_iter()
.zip(loaded.model.named_parameters())
{
assert_eq!(parameter.data(), restored.data(), "{name}");
}
}
#[test]
fn corrupt_checksum_is_rejected() {
let (model, tokenizer, optimizer) = tiny_model_and_tokenizer();
let schedule = LearningRateSchedule {
base_learning_rate: 1e-3,
minimum_learning_rate: 1e-4,
warmup_steps: 0,
total_steps: 2,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
let path = temp_path("corrupt");
save_checkpoint(
&path,
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
1,
0,
0,
)
.unwrap();
let mut bytes = fs::read(&path).unwrap();
let last = bytes.len() - 1;
bytes[last] ^= 0xff;
fs::write(&path, bytes).unwrap();
let error = match load_checkpoint(&path) {
Ok(_) => panic!("expected checksum failure"),
Err(error) => error,
};
let _ = fs::remove_file(&path);
assert!(error.to_string().contains("checksum"));
}
#[test]
fn rejects_future_quantization_schema_version() {
let quantization = CheckpointQuantization {
schema_version: QUANTIZATION_SCHEMA_VERSION + 1,
method: "rtn".into(),
bits: 8,
group_size: None,
symmetric: true,
zero_points: false,
scale_method: Some("absmax".into()),
granularity: Some("channel".into()),
format: Some("int8".into()),
inference_only: true,
excluded: vec![],
included: None,
layers: vec![],
activation: None,
kv_cache: None,
};
let error = validate_quantization_metadata(&quantization).unwrap_err();
let message = error.to_string();
assert!(
message.contains("unsupported quantization schema version"),
"{message}"
);
assert!(
message.contains(&format!("supported {QUANTIZATION_SCHEMA_VERSION_MIN}..={QUANTIZATION_SCHEMA_VERSION}")),
"{message}"
);
}
#[test]
fn rejects_unknown_quantization_method() {
let quantization = CheckpointQuantization {
schema_version: QUANTIZATION_SCHEMA_VERSION,
method: "not-a-real-method".into(),
bits: 4,
group_size: Some(32),
symmetric: true,
zero_points: false,
scale_method: None,
granularity: None,
format: Some("int4".into()),
inference_only: true,
excluded: vec![],
included: None,
layers: vec![],
activation: None,
kv_cache: None,
};
let error = validate_quantization_metadata(&quantization).unwrap_err();
assert!(
error.to_string().contains("unknown quantization method"),
"{error}"
);
}
#[test]
fn rejects_zero_quantization_schema_version() {
let quantization = CheckpointQuantization {
schema_version: 0,
method: "rtn".into(),
bits: 8,
group_size: None,
symmetric: true,
zero_points: false,
scale_method: None,
granularity: None,
format: None,
inference_only: true,
excluded: vec![],
included: None,
layers: vec![],
activation: None,
kv_cache: None,
};
assert!(validate_quantization_metadata(&quantization).is_err());
}
}