use crate::gpu::GpuError;
use crate::inspect::named_parameters;
use crate::model::{CausalLM, CausalLmConfig, ModelConfigError};
use crate::optimizer::{AdamW, OptimizerError};
use crate::profile;
use crate::rope::RopeScaling;
use crate::tokenizer::TokenizerError;
use crate::trainer::{load_checkpoint, save_checkpoint_with_best, TrainerError};
use crate::Tensor;
use memmap2::MmapOptions;
use safetensors::tensor::{serialize_to_file, TensorView, View};
use safetensors::{Dtype, SafeTensorError, SafeTensors};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::{BTreeMap, HashMap};
use std::error::Error;
use std::fmt;
use std::fs::File;
use std::io::{BufReader, BufWriter};
use std::path::Path;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum WeightFormat {
OxideLm,
HuggingFace,
}
#[derive(Clone, Debug)]
pub struct TensorStorage {
pub shape: Vec<usize>,
pub data: Vec<f32>,
}
impl TensorStorage {
pub fn numel(&self) -> usize {
self.shape.iter().copied().product()
}
fn validate(&self) -> Result<(), StateDictError> {
if self.numel() != self.data.len() {
return Err(StateDictError::Invalid(format!(
"tensor storage has shape {:?} ({} elements) but {} values",
self.shape,
self.numel(),
self.data.len()
)));
}
Ok(())
}
}
pub type StateDict = BTreeMap<String, TensorStorage>;
#[derive(Debug)]
pub enum StateDictError {
Invalid(String),
Io(std::io::Error),
Json(serde_json::Error),
SafeTensors(SafeTensorError),
Model(ModelConfigError),
Optimizer(OptimizerError),
Tokenizer(TokenizerError),
Trainer(TrainerError),
Gpu(GpuError),
}
impl fmt::Display for StateDictError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Invalid(message) => write!(formatter, "state dict error: {message}"),
Self::Io(error) => error.fmt(formatter),
Self::Json(error) => error.fmt(formatter),
Self::SafeTensors(error) => write!(formatter, "safetensors error: {error}"),
Self::Model(error) => error.fmt(formatter),
Self::Optimizer(error) => error.fmt(formatter),
Self::Tokenizer(error) => error.fmt(formatter),
Self::Trainer(error) => error.fmt(formatter),
Self::Gpu(error) => error.fmt(formatter),
}
}
}
impl Error for StateDictError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Io(error) => Some(error),
Self::Json(error) => Some(error),
Self::SafeTensors(error) => Some(error),
Self::Model(error) => Some(error),
Self::Optimizer(error) => Some(error),
Self::Tokenizer(error) => Some(error),
Self::Trainer(error) => Some(error),
Self::Gpu(error) => Some(error),
Self::Invalid(_) => None,
}
}
}
macro_rules! state_dict_from {
($source:ty, $variant:ident) => {
impl From<$source> for StateDictError {
fn from(error: $source) -> Self {
Self::$variant(error)
}
}
};
}
state_dict_from!(std::io::Error, Io);
state_dict_from!(serde_json::Error, Json);
state_dict_from!(SafeTensorError, SafeTensors);
state_dict_from!(ModelConfigError, Model);
state_dict_from!(OptimizerError, Optimizer);
state_dict_from!(TokenizerError, Tokenizer);
state_dict_from!(TrainerError, Trainer);
state_dict_from!(GpuError, Gpu);
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct HfLlamaConfig {
pub architectures: Vec<String>,
pub model_type: String,
pub vocab_size: usize,
pub hidden_size: usize,
pub intermediate_size: usize,
pub num_hidden_layers: usize,
pub num_attention_heads: usize,
#[serde(default)]
pub num_key_value_heads: usize,
pub max_position_embeddings: usize,
pub rms_norm_eps: f32,
pub rope_theta: f32,
#[serde(default)]
pub rope_scaling: Option<RopeScaling>,
pub hidden_act: String,
pub tie_word_embeddings: bool,
pub eos_token_id: usize,
pub torch_dtype: String,
#[serde(default = "default_initializer_range")]
pub initializer_range: f32,
#[serde(default)]
pub attention_dropout: f32,
#[serde(default)]
pub residual_dropout: f32,
#[serde(default)]
pub embedding_dropout: f32,
#[serde(default)]
pub attention_bias: bool,
#[serde(default)]
pub mlp_bias: bool,
}
fn default_initializer_range() -> f32 {
crate::model::DEFAULT_INITIALIZER_STD
}
impl HfLlamaConfig {
pub fn from_causal_lm(config: &CausalLmConfig) -> Self {
Self {
architectures: vec!["LlamaForCausalLM".into()],
model_type: "llama".into(),
vocab_size: config.vocab_size,
hidden_size: config.hidden_size,
intermediate_size: config.intermediate_size,
num_hidden_layers: config.num_layers,
num_attention_heads: config.num_heads,
num_key_value_heads: config.num_key_value_heads,
max_position_embeddings: config.max_sequence_length,
rms_norm_eps: config.rms_norm_epsilon,
rope_theta: config.rope_theta,
rope_scaling: config.rope_scaling.clone(),
hidden_act: "silu".into(),
tie_word_embeddings: config.tie_word_embeddings,
eos_token_id: config.eos_token_id,
torch_dtype: "float32".into(),
initializer_range: config.initializer_std,
attention_dropout: config.attention_dropout,
residual_dropout: config.residual_dropout,
embedding_dropout: config.embedding_dropout,
attention_bias: config.use_bias,
mlp_bias: config.use_bias,
}
}
pub fn to_causal_lm(&self, seed: u64) -> Result<CausalLmConfig, StateDictError> {
if self.hidden_act != "silu" && self.hidden_act != "swiglu" {
return Err(StateDictError::Invalid(format!(
"unsupported hidden_act '{}'; OxideLM expects silu/SwiGLU",
self.hidden_act
)));
}
if self.attention_bias != self.mlp_bias {
return Err(StateDictError::Invalid(
"OxideLM requires attention_bias and mlp_bias to match".into(),
));
}
let num_key_value_heads = if self.num_key_value_heads == 0 {
self.num_attention_heads
} else {
self.num_key_value_heads
};
let mut rope_scaling = self.rope_scaling.clone();
if let Some(scaling) = &mut rope_scaling {
scaling.infer_original_context(self.max_position_embeddings);
}
let config = CausalLmConfig {
vocab_size: self.vocab_size,
hidden_size: self.hidden_size,
num_layers: self.num_hidden_layers,
num_heads: self.num_attention_heads,
num_key_value_heads,
intermediate_size: self.intermediate_size,
max_sequence_length: self.max_position_embeddings,
rms_norm_epsilon: self.rms_norm_eps,
rope_theta: self.rope_theta,
rope_scaling,
eos_token_id: self.eos_token_id,
seed,
initializer_std: self.initializer_range,
tie_word_embeddings: self.tie_word_embeddings,
attention_dropout: self.attention_dropout,
residual_dropout: self.residual_dropout,
embedding_dropout: self.embedding_dropout,
use_bias: self.attention_bias,
};
config.validate()?;
Ok(config)
}
}
impl CausalLM {
pub fn state_dict(&self) -> Result<StateDict, StateDictError> {
let mut state = StateDict::new();
for (name, tensor) in named_parameters(self) {
let cpu = tensor.to_cpu()?;
let storage = TensorStorage {
shape: cpu.shape(),
data: cpu.data(),
};
storage.validate()?;
state.insert(name, storage);
}
Ok(state)
}
pub fn load_state_dict(&self, state: &StateDict) -> Result<(), StateDictError> {
let expected = named_parameters(self);
let expected_names: BTreeMap<&str, &Tensor> = expected
.iter()
.map(|(name, tensor)| (name.as_str(), tensor))
.collect();
for name in state.keys() {
if !expected_names.contains_key(name.as_str()) {
return Err(StateDictError::Invalid(format!(
"unexpected parameter '{name}' in state dict"
)));
}
}
for (name, tensor) in &expected {
let storage = state.get(name).ok_or_else(|| {
StateDictError::Invalid(format!("missing parameter '{name}' in state dict"))
})?;
storage.validate()?;
if storage.shape != tensor.shape() {
return Err(StateDictError::Invalid(format!(
"parameter '{name}' has shape {:?} but model expects {:?}",
storage.shape,
tensor.shape()
)));
}
tensor.set_data(storage.data.clone());
}
Ok(())
}
}
pub fn to_hf_state_dict(
config: &CausalLmConfig,
state: &StateDict,
) -> Result<StateDict, StateDictError> {
let mut hf = StateDict::new();
for (name, storage) in state {
let hf_name = oxidelm_to_hf_name(name).ok_or_else(|| {
StateDictError::Invalid(format!("no Hugging Face mapping for '{name}'"))
})?;
let converted = if is_linear_weight(name) {
transpose_storage(storage)?
} else {
storage.clone()
};
hf.insert(hf_name, converted);
}
if config.tie_word_embeddings {
let embed = state.get("tok_embeddings.weight").ok_or_else(|| {
StateDictError::Invalid("tied model is missing tok_embeddings.weight".into())
})?;
hf.insert("lm_head.weight".into(), embed.clone());
}
Ok(hf)
}
pub fn from_hf_state_dict(
config: &CausalLmConfig,
hf_state: &StateDict,
) -> Result<StateDict, StateDictError> {
let mut state = StateDict::new();
let expected = CausalLM::new(config.clone())?.state_dict()?;
for name in expected.keys() {
let hf_name = oxidelm_to_hf_name(name).ok_or_else(|| {
StateDictError::Invalid(format!("no Hugging Face mapping for '{name}'"))
})?;
let storage = if let Some(value) = hf_state.get(&hf_name) {
value
} else if name == "tok_embeddings.weight" {
hf_state.get("lm_head.weight").ok_or_else(|| {
StateDictError::Invalid(
"Hugging Face state dict is missing model.embed_tokens.weight".into(),
)
})?
} else {
return Err(StateDictError::Invalid(format!(
"Hugging Face state dict is missing '{hf_name}'"
)));
};
let converted = if is_linear_weight(name) {
transpose_storage(storage)?
} else {
storage.clone()
};
if converted.shape != expected[name].shape {
return Err(StateDictError::Invalid(format!(
"after conversion '{name}' has shape {:?} but model expects {:?}",
converted.shape, expected[name].shape
)));
}
state.insert(name.clone(), converted);
}
if config.tie_word_embeddings {
if let (Some(embed), Some(lm_head)) = (
hf_state.get("model.embed_tokens.weight"),
hf_state.get("lm_head.weight"),
) {
if embed.data != lm_head.data || embed.shape != lm_head.shape {
return Err(StateDictError::Invalid(
"tie_word_embeddings is set but embed_tokens and lm_head differ".into(),
));
}
}
}
Ok(state)
}
pub fn save_safetensors(
path: impl AsRef<Path>,
state: &StateDict,
metadata: Option<HashMap<String, String>>,
) -> Result<(), StateDictError> {
let _span = profile::span("checkpoint.save");
let mut tensors = Vec::with_capacity(state.len());
let mut views = Vec::with_capacity(state.len());
for (name, storage) in state {
storage.validate()?;
let bytes = f32_slice_to_le_bytes(&storage.data);
views.push(OwnedTensor {
shape: storage.shape.clone(),
data: bytes,
dtype: Dtype::F32,
});
tensors.push(name.clone());
}
let payload: Vec<(String, &OwnedTensor)> = tensors
.into_iter()
.zip(views.iter())
.map(|(name, view)| (name, view))
.collect();
serialize_to_file(payload, metadata, path.as_ref())?;
Ok(())
}
pub fn load_safetensors(path: impl AsRef<Path>) -> Result<StateDict, StateDictError> {
let _span = profile::span("checkpoint.load");
let file = File::open(path.as_ref())?;
let mmap = unsafe { MmapOptions::new().map(&file)? };
let tensors = SafeTensors::deserialize(&mmap)?;
let mut state = StateDict::new();
for name in tensors.names() {
let tensor = tensors.tensor(name)?;
let storage = tensor_view_to_storage(&tensor)?;
state.insert(name.to_string(), storage);
}
Ok(state)
}
pub fn save_hf_config(
path: impl AsRef<Path>,
config: &CausalLmConfig,
) -> Result<(), StateDictError> {
let hf = HfLlamaConfig::from_causal_lm(config);
let file = File::create(path)?;
serde_json::to_writer_pretty(BufWriter::new(file), &hf)?;
Ok(())
}
pub fn load_hf_config(path: impl AsRef<Path>) -> Result<HfLlamaConfig, StateDictError> {
let file = File::open(path)?;
Ok(serde_json::from_reader(BufReader::new(file))?)
}
pub fn export_checkpoint_weights(
checkpoint_path: impl AsRef<Path>,
weights_path: impl AsRef<Path>,
format: WeightFormat,
config_out: Option<&Path>,
) -> Result<(), StateDictError> {
let checkpoint = load_checkpoint(checkpoint_path)?;
let native = checkpoint.model.state_dict()?;
let (export, metadata) = match format {
WeightFormat::OxideLm => {
let mut metadata = HashMap::new();
metadata.insert("format".into(), "oxidelm".into());
metadata.insert(
"tie_word_embeddings".into(),
checkpoint.model.config.tie_word_embeddings.to_string(),
);
(native, Some(metadata))
}
WeightFormat::HuggingFace => {
let mut metadata = HashMap::new();
metadata.insert("format".into(), "huggingface-llama".into());
(
to_hf_state_dict(&checkpoint.model.config, &native)?,
Some(metadata),
)
}
};
save_safetensors(weights_path.as_ref(), &export, metadata)?;
if let Some(config_path) = config_out {
match format {
WeightFormat::HuggingFace => {
save_hf_config(config_path, &checkpoint.model.config)?;
}
WeightFormat::OxideLm => {
let file = File::create(config_path)?;
serde_json::to_writer_pretty(BufWriter::new(file), &checkpoint.model.config)?;
}
}
}
Ok(())
}
pub fn import_checkpoint_weights(
weights_path: impl AsRef<Path>,
checkpoint_path: impl AsRef<Path>,
format: WeightFormat,
) -> Result<(), StateDictError> {
let checkpoint = load_checkpoint(checkpoint_path.as_ref())?;
let loaded = load_safetensors(weights_path)?;
let native = match format {
WeightFormat::OxideLm => loaded,
WeightFormat::HuggingFace => from_hf_state_dict(&checkpoint.model.config, &loaded)?,
};
checkpoint.model.load_state_dict(&native)?;
let optimizer = AdamW::new_grouped(
&checkpoint.model.named_parameters(),
checkpoint.optimizer.config().clone(),
)?;
save_checkpoint_with_best(
checkpoint_path.as_ref(),
&checkpoint.model,
&optimizer,
&checkpoint.tokenizer,
&checkpoint.schedule,
&checkpoint.trainer_config,
checkpoint.training_step,
checkpoint.cursor,
checkpoint.tokens_processed,
checkpoint.rng.dropout_stream,
checkpoint.best_validation_loss,
checkpoint.evaluations_without_improvement,
checkpoint.loss_scaler,
)?;
Ok(())
}
pub fn export_quantized_safetensors(
checkpoint_path: impl AsRef<Path>,
weights_path: impl AsRef<Path>,
format: WeightFormat,
) -> Result<(), StateDictError> {
let checkpoint = load_checkpoint(checkpoint_path)?;
let (mut tensors, quant_meta) = collect_quantized_tensors(&checkpoint.model, format)?;
let mut metadata = HashMap::new();
metadata.insert("format".into(), "oxidelm-quantized".into());
metadata.insert(
"weight_layout".into(),
match format {
WeightFormat::OxideLm => "oxidelm".into(),
WeightFormat::HuggingFace => "huggingface-llama".into(),
},
);
if let Some(quant) = quant_meta {
metadata.insert("quantization".into(), serde_json::to_string(&quant)?);
}
save_mixed_safetensors(weights_path.as_ref(), &mut tensors, Some(metadata))?;
Ok(())
}
pub fn import_quantized_safetensors(
weights_path: impl AsRef<Path>,
checkpoint_path: impl AsRef<Path>,
format: WeightFormat,
) -> Result<(), StateDictError> {
let file = File::open(weights_path.as_ref())?;
let mmap = unsafe { MmapOptions::new().map(&file)? };
let (_, file_metadata) = SafeTensors::read_metadata(&mmap).map_err(StateDictError::SafeTensors)?;
let metadata = file_metadata.metadata().clone().unwrap_or_default();
let tensors = SafeTensors::deserialize(&mmap)?;
let quant_json = metadata.get("quantization").ok_or_else(|| {
StateDictError::Invalid(
"quantized SafeTensors missing metadata key 'quantization'".into(),
)
})?;
let quantization: crate::checkpoint::CheckpointQuantization =
serde_json::from_str(quant_json)?;
let mut checkpoint = load_checkpoint(checkpoint_path.as_ref())?;
let mut fp32 = StateDict::new();
for name in tensors.names() {
let tensor = tensors.tensor(name)?;
if tensor.dtype() != Dtype::F32 {
continue;
}
if name.ends_with(".qscales") || name.ends_with(".qzeros") {
continue;
}
let storage = tensor_view_to_storage(&tensor)?;
fp32.insert(name.to_string(), storage);
}
let native = match format {
WeightFormat::OxideLm => fp32,
WeightFormat::HuggingFace => from_hf_state_dict(&checkpoint.model.config, &fp32)?,
};
let mut filtered = StateDict::new();
for (name, storage) in native {
let is_quant_master = quantization.layers.iter().any(|layer| {
name == format!("{}.weight", layer.name)
|| hf_module_weight_name(&layer.name)
.map(|hf| name == hf)
.unwrap_or(false)
});
if !is_quant_master {
filtered.insert(name, storage);
}
}
for (name, parameter) in checkpoint.model.named_parameters() {
if let Some(storage) = filtered.get(&name) {
if parameter.shape() == storage.shape {
parameter.set_data(storage.data.clone());
}
}
}
for layer in &quantization.layers {
let (packed_name, scales_name, zeros_name) = match format {
WeightFormat::OxideLm => (
layer.packed_name.clone(),
layer.scales_name.clone(),
layer.zero_points_name.clone(),
),
WeightFormat::HuggingFace => {
let stem = hf_module_stem(&layer.name).ok_or_else(|| {
StateDictError::Invalid(format!(
"cannot map module {} to HF name",
layer.name
))
})?;
(
format!("{stem}.weight.qpacked"),
format!("{stem}.weight.qscales"),
layer
.zero_points_name
.as_ref()
.map(|_| format!("{stem}.weight.qzeros")),
)
}
};
let packed = read_u32_tensor(&tensors, &packed_name)?;
let scales = read_f32_tensor(&tensors, &scales_name)?;
let zeros = match &zeros_name {
Some(name) => Some(read_f32_tensor(&tensors, name)?),
None => None,
};
let scheme = crate::layers::QuantScheme {
format: layer.format,
granularity: layer
.granularity
.as_deref()
.and_then(|value| crate::layers::QuantGranularity::parse(value).ok())
.unwrap_or_else(|| match layer.group_size {
Some(size) if size > 1 => crate::layers::QuantGranularity::PerGroup { size },
Some(1) => crate::layers::QuantGranularity::PerTensor,
_ => crate::layers::QuantGranularity::PerChannel,
}),
symmetric: layer.symmetric,
scale_method: layer
.scale_method
.as_deref()
.and_then(|value| crate::layers::ScaleMethod::parse(value).ok())
.unwrap_or(crate::layers::ScaleMethod::AbsMax),
percentile: 99.99,
};
let weight = crate::layers::QuantizedLinearWeight::from_parts_with_scheme(
scheme,
packed,
scales,
zeros,
layer.in_features,
layer.out_features,
)
.map_err(StateDictError::Invalid)?;
checkpoint
.model
.set_quantized_linear(&layer.name, weight)
.map_err(StateDictError::Invalid)?;
if let Some((_, linear)) = checkpoint
.model
.named_linears_mut()
.into_iter()
.find(|(name, _)| name == &layer.name)
{
linear.materialize_dequantized_master();
}
}
crate::checkpoint::save_inference_checkpoint(
checkpoint_path.as_ref(),
&checkpoint.model,
&checkpoint.tokenizer,
&quantization,
)?;
Ok(())
}
fn collect_quantized_tensors(
model: &CausalLM,
format: WeightFormat,
) -> Result<(Vec<(String, OwnedTensor)>, Option<crate::checkpoint::CheckpointQuantization>), StateDictError>
{
let mut tensors = Vec::new();
let mut layers = Vec::new();
let quantized_modules: std::collections::BTreeSet<String> = model
.named_linears()
.into_iter()
.filter(|(_, linear)| linear.quantized_weight().is_some())
.map(|(name, _)| name)
.collect();
for (name, parameter) in model.named_parameters() {
let skip = name.ends_with(".weight")
&& quantized_modules
.iter()
.any(|module| name == format!("{module}.weight"));
if skip {
continue;
}
let export_name = match format {
WeightFormat::OxideLm => name.clone(),
WeightFormat::HuggingFace => oxidelm_to_hf_name(&name).unwrap_or(name.clone()),
};
let mut data = parameter.data();
let mut shape = parameter.shape();
if format == WeightFormat::HuggingFace && is_linear_weight(&name) {
let transposed = transpose_storage(&TensorStorage {
shape: shape.clone(),
data: data.clone(),
})?;
data = transposed.data;
shape = transposed.shape;
}
tensors.push((
export_name,
OwnedTensor {
shape,
data: f32_slice_to_le_bytes(&data),
dtype: Dtype::F32,
},
));
}
for (name, linear) in model.named_linears() {
let Some(quant) = linear.quantized_weight() else {
continue;
};
let stem = match format {
WeightFormat::OxideLm => name.clone(),
WeightFormat::HuggingFace => hf_module_stem(&name).ok_or_else(|| {
StateDictError::Invalid(format!("cannot map module {name} to HF name"))
})?,
};
let packed_name = format!("{stem}.weight.qpacked");
let scales_name = format!("{stem}.weight.qscales");
let zeros_name = if quant.scheme().symmetric {
None
} else {
Some(format!("{stem}.weight.qzeros"))
};
tensors.push((
packed_name.clone(),
OwnedTensor {
shape: vec![quant.packed_values().len()],
data: u32_slice_to_le_bytes(quant.packed_values()),
dtype: Dtype::U32,
},
));
tensors.push((
scales_name.clone(),
OwnedTensor {
shape: vec![quant.scales().len()],
data: f32_slice_to_le_bytes(quant.scales()),
dtype: Dtype::F32,
},
));
if let (Some(zeros_name), Some(zeros)) = (&zeros_name, quant.zero_points()) {
tensors.push((
zeros_name.clone(),
OwnedTensor {
shape: vec![zeros.len()],
data: f32_slice_to_le_bytes(zeros),
dtype: Dtype::F32,
},
));
}
let scheme = quant.scheme();
layers.push(crate::checkpoint::QuantizedLayerEntry {
name: name.clone(),
format: quant.format(),
in_features: quant.in_features(),
out_features: quant.out_features(),
packed_name: format!("{name}.weight.qpacked"),
scales_name: format!("{name}.weight.qscales"),
zero_points_name: if scheme.symmetric {
None
} else {
Some(format!("{name}.weight.qzeros"))
},
group_size: scheme.granularity.group_size(),
symmetric: scheme.symmetric,
scale_method: Some(scheme.scale_method.as_str().into()),
granularity: Some(scheme.granularity.as_str()),
});
}
let quant_meta = if layers.is_empty() {
None
} else {
let bits = layers.first().map(|layer| layer.format.bits()).unwrap_or(8);
Some(crate::checkpoint::CheckpointQuantization {
schema_version: crate::checkpoint::QUANTIZATION_SCHEMA_VERSION,
method: "exported".into(),
bits,
group_size: layers.first().and_then(|layer| layer.group_size),
symmetric: layers.first().map(|layer| layer.symmetric).unwrap_or(true),
zero_points: layers.iter().any(|layer| layer.zero_points_name.is_some()),
scale_method: layers.first().and_then(|layer| layer.scale_method.clone()),
granularity: layers.first().and_then(|layer| layer.granularity.clone()),
format: layers
.first()
.map(|layer| layer.format.as_str().into()),
inference_only: true,
excluded: Vec::new(),
included: None,
layers,
activation: None,
kv_cache: None,
})
};
Ok((tensors, quant_meta))
}
fn hf_module_stem(oxidelm_name: &str) -> Option<String> {
let weight_name = format!("{oxidelm_name}.weight");
let hf = oxidelm_to_hf_name(&weight_name)?;
Some(hf.trim_end_matches(".weight").to_string())
}
fn hf_module_weight_name(oxidelm_module: &str) -> Option<String> {
oxidelm_to_hf_name(&format!("{oxidelm_module}.weight"))
}
fn save_mixed_safetensors(
path: &Path,
tensors: &mut [(String, OwnedTensor)],
metadata: Option<HashMap<String, String>>,
) -> Result<(), StateDictError> {
let payload: Vec<(String, &OwnedTensor)> = tensors
.iter()
.map(|(name, tensor)| (name.clone(), tensor))
.collect();
serialize_to_file(payload, metadata, path)?;
Ok(())
}
fn read_u32_tensor(tensors: &SafeTensors<'_>, name: &str) -> Result<Vec<u32>, StateDictError> {
let tensor = tensors.tensor(name).map_err(|error| {
StateDictError::Invalid(format!("missing tensor {name}: {error}"))
})?;
if tensor.dtype() != Dtype::U32 {
return Err(StateDictError::Invalid(format!(
"tensor {name} expected U32, got {:?}",
tensor.dtype()
)));
}
le_bytes_to_u32_vec(tensor.data())
}
fn read_f32_tensor(tensors: &SafeTensors<'_>, name: &str) -> Result<Vec<f32>, StateDictError> {
let tensor = tensors.tensor(name).map_err(|error| {
StateDictError::Invalid(format!("missing tensor {name}: {error}"))
})?;
if tensor.dtype() != Dtype::F32 {
return Err(StateDictError::Invalid(format!(
"tensor {name} expected F32, got {:?}",
tensor.dtype()
)));
}
le_bytes_to_f32_vec(tensor.data())
}
fn u32_slice_to_le_bytes(values: &[u32]) -> Vec<u8> {
let mut bytes = Vec::with_capacity(values.len() * 4);
for value in values {
bytes.extend_from_slice(&value.to_le_bytes());
}
bytes
}
fn le_bytes_to_u32_vec(bytes: &[u8]) -> Result<Vec<u32>, StateDictError> {
if !bytes.len().is_multiple_of(4) {
return Err(StateDictError::Invalid(
"u32 tensor byte length is not a multiple of 4".into(),
));
}
Ok(bytes
.chunks_exact(4)
.map(|chunk| u32::from_le_bytes(chunk.try_into().unwrap()))
.collect())
}
fn oxidelm_to_hf_name(name: &str) -> Option<String> {
if name == "tok_embeddings.weight" {
return Some("model.embed_tokens.weight".into());
}
if name == "norm.weight" {
return Some("model.norm.weight".into());
}
if name == "lm_head.weight" {
return Some("lm_head.weight".into());
}
if name == "lm_head.bias" {
return Some("lm_head.bias".into());
}
let rest = name.strip_prefix("layers.")?;
let (index, suffix) = rest.split_once('.')?;
let hf_suffix = match suffix {
"attention_norm.weight" => "input_layernorm.weight",
"attention.q_proj.weight" => "self_attn.q_proj.weight",
"attention.q_proj.bias" => "self_attn.q_proj.bias",
"attention.k_proj.weight" => "self_attn.k_proj.weight",
"attention.k_proj.bias" => "self_attn.k_proj.bias",
"attention.v_proj.weight" => "self_attn.v_proj.weight",
"attention.v_proj.bias" => "self_attn.v_proj.bias",
"attention.o_proj.weight" => "self_attn.o_proj.weight",
"attention.o_proj.bias" => "self_attn.o_proj.bias",
"ffn_norm.weight" => "post_attention_layernorm.weight",
"feed_forward.gate_proj.weight" => "mlp.gate_proj.weight",
"feed_forward.gate_proj.bias" => "mlp.gate_proj.bias",
"feed_forward.up_proj.weight" => "mlp.up_proj.weight",
"feed_forward.up_proj.bias" => "mlp.up_proj.bias",
"feed_forward.down_proj.weight" => "mlp.down_proj.weight",
"feed_forward.down_proj.bias" => "mlp.down_proj.bias",
_ => return None,
};
Some(format!("model.layers.{index}.{hf_suffix}"))
}
fn is_linear_weight(name: &str) -> bool {
(name.ends_with("q_proj.weight")
|| name.ends_with("k_proj.weight")
|| name.ends_with("v_proj.weight")
|| name.ends_with("o_proj.weight")
|| name.ends_with("gate_proj.weight")
|| name.ends_with("up_proj.weight")
|| name.ends_with("down_proj.weight")
|| name == "lm_head.weight")
&& !name.ends_with(".bias")
}
fn transpose_storage(storage: &TensorStorage) -> Result<TensorStorage, StateDictError> {
storage.validate()?;
if storage.shape.len() != 2 {
return Err(StateDictError::Invalid(format!(
"cannot transpose tensor with shape {:?}",
storage.shape
)));
}
let rows = storage.shape[0];
let cols = storage.shape[1];
let mut data = vec![0.0; storage.data.len()];
for row in 0..rows {
for col in 0..cols {
data[col * rows + row] = storage.data[row * cols + col];
}
}
Ok(TensorStorage {
shape: vec![cols, rows],
data,
})
}
fn f32_slice_to_le_bytes(values: &[f32]) -> Vec<u8> {
let mut bytes = Vec::with_capacity(values.len() * 4);
for value in values {
bytes.extend_from_slice(&value.to_le_bytes());
}
bytes
}
fn le_bytes_to_f32_vec(bytes: &[u8]) -> Result<Vec<f32>, StateDictError> {
if !bytes.len().is_multiple_of(4) {
return Err(StateDictError::Invalid(
"tensor byte length is not a multiple of 4".into(),
));
}
Ok(bytes
.chunks_exact(4)
.map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap()))
.collect())
}
fn tensor_view_to_storage(tensor: &TensorView<'_>) -> Result<TensorStorage, StateDictError> {
if tensor.dtype() != Dtype::F32 {
return Err(StateDictError::Invalid(format!(
"unsupported dtype {:?}; only F32 is supported",
tensor.dtype()
)));
}
let shape = tensor.shape().to_vec();
let data = le_bytes_to_f32_vec(tensor.data())?;
let storage = TensorStorage { shape, data };
storage.validate()?;
Ok(storage)
}
struct OwnedTensor {
shape: Vec<usize>,
data: Vec<u8>,
dtype: Dtype,
}
impl View for &OwnedTensor {
fn dtype(&self) -> Dtype {
self.dtype
}
fn shape(&self) -> &[usize] {
&self.shape
}
fn data(&self) -> Cow<'_, [u8]> {
Cow::Borrowed(&self.data)
}
fn data_len(&self) -> usize {
self.data.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::optimizer::AdamWConfig;
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
use crate::trainer::{save_checkpoint, LearningRateSchedule, TrainerConfig};
use crate::IndexTensor;
fn tiny_config(tie_word_embeddings: bool) -> CausalLmConfig {
CausalLmConfig {
vocab_size: 16,
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 15,
seed: 11,
initializer_std: 0.02,
tie_word_embeddings,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
}
}
fn temp_dir(label: &str) -> std::path::PathBuf {
let directory = std::env::temp_dir().join(format!(
"oxidelm-state-dict-{label}-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&directory).unwrap();
directory
}
fn write_tiny_checkpoint(path: &Path, tie: bool) -> CausalLM {
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world",
FIRST_MERGE_ID + 4,
1,
)
.unwrap();
let mut config = tiny_config(tie);
config.vocab_size = tokenizer.vocab_size();
config.eos_token_id = tokenizer.special_tokens.eos;
let model = CausalLM::new(config).unwrap();
let parameters = model.parameters();
let optimizer = AdamW::new(¶meters, AdamWConfig::default()).unwrap();
save_checkpoint(
path,
&model,
&optimizer,
&tokenizer,
&LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
},
&TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
0,
0,
0,
)
.unwrap();
model
}
#[test]
fn state_dict_round_trip_preserves_logits() {
let model = CausalLM::new(tiny_config(true)).unwrap();
let input = IndexTensor::new(vec![1, 2, 3, 4], vec![1, 4]);
let before = model.forward(&input).unwrap().data();
let state = model.state_dict().unwrap();
let reloaded = CausalLM::new(tiny_config(true)).unwrap();
reloaded.load_state_dict(&state).unwrap();
let after = reloaded.forward(&input).unwrap().data();
assert_eq!(before, after);
}
#[test]
fn oxidelm_safetensors_round_trip() {
let directory = temp_dir("oxide");
let path = directory.join("weights.safetensors");
let model = CausalLM::new(tiny_config(false)).unwrap();
let state = model.state_dict().unwrap();
save_safetensors(&path, &state, None).unwrap();
let loaded = load_safetensors(&path).unwrap();
assert_eq!(state.len(), loaded.len());
for (name, storage) in &state {
let other = loaded.get(name).unwrap();
assert_eq!(storage.shape, other.shape);
assert_eq!(storage.data, other.data);
}
let _ = std::fs::remove_dir_all(directory);
}
#[test]
fn huggingface_conversion_round_trip_tied_and_untied() {
for tie in [true, false] {
let model = CausalLM::new(tiny_config(tie)).unwrap();
let native = model.state_dict().unwrap();
let hf = to_hf_state_dict(&model.config, &native).unwrap();
assert!(hf.contains_key("model.embed_tokens.weight"));
assert!(hf.contains_key("model.layers.0.self_attn.q_proj.weight"));
assert!(hf.contains_key("lm_head.weight"));
let q = &hf["model.layers.0.self_attn.q_proj.weight"];
assert_eq!(q.shape, vec![8, 8]);
let k = &hf["model.layers.0.self_attn.k_proj.weight"];
assert_eq!(k.shape, vec![4, 8]);
let restored = from_hf_state_dict(&model.config, &hf).unwrap();
for (name, storage) in &native {
let other = restored.get(name).unwrap();
assert_eq!(storage.shape, other.shape);
assert_eq!(storage.data, other.data);
}
}
}
#[test]
fn export_import_checkpoint_weights_hf() {
let directory = temp_dir("checkpoint");
let checkpoint = directory.join("model.bin");
let weights = directory.join("model.safetensors");
let config_path = directory.join("config.json");
let original = write_tiny_checkpoint(&checkpoint, true);
let input = IndexTensor::new(vec![1, 2, 3, 4], vec![1, 4]);
let before = original.forward(&input).unwrap().data();
export_checkpoint_weights(
&checkpoint,
&weights,
WeightFormat::HuggingFace,
Some(&config_path),
)
.unwrap();
let hf_config = load_hf_config(&config_path).unwrap();
assert_eq!(hf_config.model_type, "llama");
assert_eq!(hf_config.num_hidden_layers, 1);
assert_eq!(hf_config.num_key_value_heads, 1);
assert_eq!(hf_config.initializer_range, 0.02);
let loaded = load_checkpoint(&checkpoint).unwrap();
for parameter in loaded.model.parameters() {
let zeros = vec![0.0; parameter.numel()];
parameter.set_data(zeros);
}
save_checkpoint(
&checkpoint,
&loaded.model,
&AdamW::new(&loaded.model.parameters(), AdamWConfig::default()).unwrap(),
&loaded.tokenizer,
&loaded.schedule,
&loaded.trainer_config,
loaded.training_step,
loaded.cursor,
loaded.tokens_processed,
)
.unwrap();
import_checkpoint_weights(&weights, &checkpoint, WeightFormat::HuggingFace).unwrap();
let restored = load_checkpoint(&checkpoint).unwrap();
let after = restored.model.forward(&input).unwrap().data();
assert_eq!(before, after);
let _ = std::fs::remove_dir_all(directory);
}
#[test]
fn quantized_safetensors_round_trip() {
use crate::quantize::{quantize_checkpoint, QuantizeMethod, QuantizeOptions};
let directory = temp_dir("quant-st");
let fp32 = directory.join("fp32.bin");
let quantized = directory.join("int8.bin");
let weights = directory.join("quant.safetensors");
let restored = directory.join("restored.bin");
write_tiny_checkpoint(&fp32, true);
quantize_checkpoint(
&fp32,
&quantized,
&QuantizeOptions {
method: QuantizeMethod::Rtn,
bits: 8,
group_size: Some(32),
..QuantizeOptions::default()
},
)
.unwrap();
export_quantized_safetensors(&quantized, &weights, WeightFormat::OxideLm).unwrap();
std::fs::copy(&quantized, &restored).unwrap();
import_quantized_safetensors(&weights, &restored, WeightFormat::OxideLm).unwrap();
let loaded = load_checkpoint(&restored).unwrap();
assert!(loaded
.model
.named_linears()
.iter()
.any(|(_, linear)| linear.quantized_weight().is_some()));
let _ = std::fs::remove_dir_all(directory);
}
#[test]
fn linear_transpose_is_involution() {
let storage = TensorStorage {
shape: vec![2, 3],
data: vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
};
let once = transpose_storage(&storage).unwrap();
assert_eq!(once.shape, vec![3, 2]);
assert_eq!(once.data, vec![1.0, 4.0, 2.0, 5.0, 3.0, 6.0]);
let twice = transpose_storage(&once).unwrap();
assert_eq!(twice.shape, storage.shape);
assert_eq!(twice.data, storage.data);
}
}