use crate::checkpoint::{
save_checkpoint, save_inference_checkpoint, QUANTIZATION_SCHEMA_VERSION,
};
use crate::layers::{QuantGranularity, QuantScheme, ScaleMethod, WeightQuantization};
use crate::model::{CausalLM, CausalLmConfig};
use crate::optimizer::{AdamW, AdamWConfig};
use crate::quantize::{build_quantization_metadata_for_model, should_quantize_module, QuantizeOptions};
use crate::tokenizer::BpeTokenizer;
use crate::trainer::{LearningRateSchedule, TrainerConfig, TrainerError};
use half::f16;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::error::Error;
use std::fmt;
use std::fs::File;
use std::io::{BufWriter, Read, Write};
use std::path::Path;
pub const GGUF_MAGIC: &[u8; 4] = b"GGUF";
pub const GGUF_VERSION: u32 = 3;
const GGUF_DEFAULT_ALIGNMENT: u64 = 32;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "UPPERCASE")]
#[allow(non_camel_case_types)]
#[repr(u32)]
pub enum GgmlType {
F32 = 0,
F16 = 1,
Q4_0 = 2,
Q8_0 = 8,
Q4_K = 12,
Q5_K = 13,
Q6_K = 14,
}
impl GgmlType {
pub fn as_str(self) -> &'static str {
match self {
Self::F32 => "F32",
Self::F16 => "F16",
Self::Q4_0 => "Q4_0",
Self::Q8_0 => "Q8_0",
Self::Q4_K => "Q4_K",
Self::Q5_K => "Q5_K",
Self::Q6_K => "Q6_K",
}
}
pub fn parse(value: &str) -> Result<Self, GgufError> {
match value.trim().to_ascii_uppercase().as_str() {
"F32" | "FP32" => Ok(Self::F32),
"F16" | "FP16" => Ok(Self::F16),
"Q4_0" | "Q4.0" | "Q40" => Ok(Self::Q4_0),
"Q8_0" | "Q8.0" | "Q80" => Ok(Self::Q8_0),
"Q4_K" | "Q4K" | "Q4_K_M" | "Q4_K_S" => Ok(Self::Q4_K),
"Q5_K" | "Q5K" | "Q5_K_M" | "Q5_K_S" => Ok(Self::Q5_K),
"Q6_K" | "Q6K" => Ok(Self::Q6_K),
other => Err(GgufError::Invalid(format!(
"unsupported GGUF type '{other}'; expected Q4_0|Q4_K|Q5_K|Q6_K|Q8_0"
))),
}
}
pub fn from_u32(value: u32) -> Result<Self, GgufError> {
match value {
0 => Ok(Self::F32),
1 => Ok(Self::F16),
2 => Ok(Self::Q4_0),
8 => Ok(Self::Q8_0),
12 => Ok(Self::Q4_K),
13 => Ok(Self::Q5_K),
14 => Ok(Self::Q6_K),
other => Err(GgufError::Invalid(format!(
"unsupported GGML type id {other}"
))),
}
}
pub fn block_size(self) -> usize {
match self {
Self::F32 | Self::F16 => 1,
Self::Q4_0 | Self::Q8_0 => 32,
Self::Q4_K | Self::Q5_K | Self::Q6_K => 256,
}
}
pub fn type_size(self) -> usize {
match self {
Self::F32 => 4,
Self::F16 => 2,
Self::Q4_0 => 18,
Self::Q8_0 => 34,
Self::Q4_K => 144,
Self::Q5_K => 176,
Self::Q6_K => 210,
}
}
pub fn native_scheme(self) -> Option<QuantScheme> {
match self {
Self::Q8_0 => Some(QuantScheme {
format: WeightQuantization::Int8,
granularity: QuantGranularity::PerGroup { size: 32 },
symmetric: true,
scale_method: ScaleMethod::AbsMax,
percentile: 99.99,
}),
Self::Q4_0 | Self::Q4_K => Some(QuantScheme {
format: WeightQuantization::Int4,
granularity: QuantGranularity::PerGroup { size: 32 },
symmetric: false,
scale_method: ScaleMethod::MinMax,
percentile: 99.99,
}),
Self::Q5_K | Self::Q6_K => None,
Self::F32 | Self::F16 => None,
}
}
}
#[derive(Debug)]
pub enum GgufError {
Invalid(String),
Io(std::io::Error),
Json(serde_json::Error),
Tokenizer(crate::tokenizer::TokenizerError),
Trainer(TrainerError),
Model(crate::model::ModelConfigError),
}
impl fmt::Display for GgufError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Invalid(message) => write!(f, "gguf error: {message}"),
Self::Io(error) => error.fmt(f),
Self::Json(error) => error.fmt(f),
Self::Tokenizer(error) => error.fmt(f),
Self::Trainer(error) => error.fmt(f),
Self::Model(error) => error.fmt(f),
}
}
}
impl Error for GgufError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Io(error) => Some(error),
Self::Json(error) => Some(error),
Self::Tokenizer(error) => Some(error),
Self::Trainer(error) => Some(error),
Self::Model(error) => Some(error),
Self::Invalid(_) => None,
}
}
}
impl From<std::io::Error> for GgufError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<serde_json::Error> for GgufError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
impl From<crate::tokenizer::TokenizerError> for GgufError {
fn from(error: crate::tokenizer::TokenizerError) -> Self {
Self::Tokenizer(error)
}
}
impl From<TrainerError> for GgufError {
fn from(error: TrainerError) -> Self {
Self::Trainer(error)
}
}
impl From<crate::model::ModelConfigError> for GgufError {
fn from(error: crate::model::ModelConfigError) -> Self {
Self::Model(error)
}
}
#[derive(Clone, Debug)]
pub struct GgufExportOptions {
pub quant_type: GgmlType,
pub keep_embed_f32: bool,
}
impl Default for GgufExportOptions {
fn default() -> Self {
Self {
quant_type: GgmlType::Q4_0,
keep_embed_f32: true,
}
}
}
#[derive(Clone, Debug, Default)]
pub struct GgufImportOptions {
pub tokenizer_path: Option<std::path::PathBuf>,
pub pack_native: bool,
}
pub fn scheme_mapping_docs() -> &'static str {
"Q8_0→int8/group:32/sym; Q4_0→int4/group:32/asym; Q4_K→int4/group:32/asym; Q5_K/Q6_K→FP32 import (no native 5/6-bit)"
}
pub fn export_checkpoint_gguf(
checkpoint_path: impl AsRef<Path>,
gguf_path: impl AsRef<Path>,
options: &GgufExportOptions,
) -> Result<(), GgufError> {
let loaded = crate::trainer::load_checkpoint(checkpoint_path.as_ref())?;
write_model_gguf(&loaded.model, &loaded.tokenizer, gguf_path, options)
}
pub fn import_gguf_checkpoint(
gguf_path: impl AsRef<Path>,
checkpoint_path: impl AsRef<Path>,
options: &GgufImportOptions,
) -> Result<(), GgufError> {
let (config, tensors, tokenizer_json) = read_gguf_model(gguf_path.as_ref())?;
let tokenizer = if let Some(bytes) = tokenizer_json {
BpeTokenizer::from_json_bytes(&bytes)?
} else if let Some(path) = &options.tokenizer_path {
let bytes = std::fs::read(path)?;
BpeTokenizer::from_json_bytes(&bytes)?
} else {
return Err(GgufError::Invalid(
"GGUF missing oxidelm.tokenizer_json; pass --tokenizer <tokenizer.json>".into(),
));
};
let mut model = CausalLM::new(config)?;
apply_gguf_tensors(&mut model, &tensors)?;
if options.pack_native {
let mut quant_options = QuantizeOptions::default();
quant_options.bits = 4;
quant_options.group_size = Some(32);
quant_options.symmetric = false;
quant_options.scale_method = ScaleMethod::MinMax;
for (name, linear) in model.named_linears_mut() {
if should_quantize_module(&name, &quant_options) {
if let Some(scheme) = GgmlType::Q4_0.native_scheme() {
linear.quantize_weight_with(scheme);
linear.materialize_dequantized_master();
}
}
}
let names: Vec<String> = model
.named_linears()
.into_iter()
.filter(|(_, linear)| linear.quantized_weight().is_some())
.map(|(name, _)| name)
.collect();
let metadata = build_quantization_metadata_for_model(&model, &quant_options, &names)
.map_err(|error| GgufError::Invalid(error.to_string()))?;
save_inference_checkpoint(checkpoint_path.as_ref(), &model, &tokenizer, &metadata)?;
return Ok(());
}
let optimizer = AdamW::new(&model.parameters(), AdamWConfig::default())
.map_err(|error| GgufError::Invalid(error.to_string()))?;
let schedule = LearningRateSchedule {
base_learning_rate: 0.0,
minimum_learning_rate: 0.0,
warmup_steps: 0,
total_steps: 1,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: model.config.max_sequence_length.min(32).max(2),
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
save_checkpoint(
checkpoint_path.as_ref(),
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
0,
0,
0,
)?;
Ok(())
}
fn write_model_gguf(
model: &CausalLM,
tokenizer: &BpeTokenizer,
path: impl AsRef<Path>,
options: &GgufExportOptions,
) -> Result<(), GgufError> {
if !matches!(
options.quant_type,
GgmlType::Q4_0 | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K | GgmlType::Q8_0
) {
return Err(GgufError::Invalid(format!(
"export type must be Q4_0|Q4_K|Q5_K|Q6_K|Q8_0, got {}",
options.quant_type.as_str()
)));
}
let mut kv: Vec<(String, GgufValue)> = Vec::new();
kv.push(("general.architecture".into(), GgufValue::String("llama".into())));
kv.push(("general.name".into(), GgufValue::String("oxidelm".into())));
kv.push((
"general.quantization_version".into(),
GgufValue::U32(QUANTIZATION_SCHEMA_VERSION),
));
kv.push((
"oxidelm.scheme_mapping".into(),
GgufValue::String(scheme_mapping_docs().into()),
));
kv.push((
"oxidelm.tokenizer_json".into(),
GgufValue::String(String::from_utf8_lossy(&tokenizer.to_json_bytes()?).into()),
));
kv.push((
"oxidelm.model_config".into(),
GgufValue::String(serde_json::to_string(&model.config)?),
));
kv.push(("llama.context_length".into(), GgufValue::U32(model.config.max_sequence_length as u32)));
kv.push(("llama.embedding_length".into(), GgufValue::U32(model.config.hidden_size as u32)));
kv.push(("llama.block_count".into(), GgufValue::U32(model.config.num_layers as u32)));
kv.push(("llama.feed_forward_length".into(), GgufValue::U32(model.config.intermediate_size as u32)));
kv.push(("llama.attention.head_count".into(), GgufValue::U32(model.config.num_heads as u32)));
kv.push((
"llama.attention.head_count_kv".into(),
GgufValue::U32(model.config.num_key_value_heads as u32),
));
kv.push(("llama.rope.freq_base".into(), GgufValue::F32(model.config.rope_theta)));
kv.push(("llama.attention.layer_norm_rms_epsilon".into(), GgufValue::F32(model.config.rms_norm_epsilon)));
kv.push(("llama.vocab_size".into(), GgufValue::U32(model.config.vocab_size as u32)));
let mut tensors = Vec::new();
push_f32_tensor(
&mut tensors,
"token_embd.weight",
&[model.config.vocab_size, model.config.hidden_size],
&model.tok_embeddings.weight.data(),
)?;
push_f32_tensor(
&mut tensors,
"output_norm.weight",
&[model.config.hidden_size],
&model.norm.weight.data(),
)?;
if !model.config.tie_word_embeddings {
let values = linear_row_major_out_in(&model.lm_head);
let qtype = if options.keep_embed_f32 {
GgmlType::F32
} else {
options.quant_type
};
push_quant_or_f32(
&mut tensors,
"output.weight",
&[model.lm_head.out_features, model.lm_head.in_features],
&values,
qtype,
)?;
}
for (layer_index, layer) in model.layers.iter().enumerate() {
let prefix = format!("blk.{layer_index}");
push_f32_tensor(
&mut tensors,
&format!("{prefix}.attn_norm.weight"),
&[model.config.hidden_size],
&layer.attention_norm.weight.data(),
)?;
push_f32_tensor(
&mut tensors,
&format!("{prefix}.ffn_norm.weight"),
&[model.config.hidden_size],
&layer.ffn_norm.weight.data(),
)?;
for (suffix, linear) in [
("attn_q", &layer.attention.q_proj),
("attn_k", &layer.attention.k_proj),
("attn_v", &layer.attention.v_proj),
("attn_output", &layer.attention.o_proj),
("ffn_gate", &layer.feed_forward.gate_proj),
("ffn_up", &layer.feed_forward.up_proj),
("ffn_down", &layer.feed_forward.down_proj),
] {
let values = linear_row_major_out_in(linear);
push_quant_or_f32(
&mut tensors,
&format!("{prefix}.{suffix}.weight"),
&[linear.out_features, linear.in_features],
&values,
options.quant_type,
)?;
}
}
write_gguf_file(path.as_ref(), &kv, &tensors)
}
fn linear_row_major_out_in(linear: &crate::layers::Linear) -> Vec<f32> {
let storage = if let Some(quant) = linear.quantized_weight() {
quant.dequantized_values(linear.transpose_weight)
} else {
linear.weight.data()
};
let mut out = vec![0.0; linear.in_features * linear.out_features];
for input in 0..linear.in_features {
for output in 0..linear.out_features {
let value = if linear.transpose_weight {
storage[output * linear.in_features + input]
} else {
storage[input * linear.out_features + output]
};
out[output * linear.in_features + input] = value;
}
}
out
}
fn push_f32_tensor(
tensors: &mut Vec<PreparedTensor>,
name: &str,
shape: &[usize],
data: &[f32],
) -> Result<(), GgufError> {
let expected: usize = shape.iter().copied().product();
if data.len() != expected {
return Err(GgufError::Invalid(format!(
"tensor {name} shape {:?} expects {expected} floats, got {}",
shape,
data.len()
)));
}
let mut bytes = Vec::with_capacity(data.len() * 4);
for value in data {
bytes.extend_from_slice(&value.to_le_bytes());
}
tensors.push(PreparedTensor {
name: name.into(),
dims: shape.to_vec(),
ggml_type: GgmlType::F32,
data: bytes,
});
Ok(())
}
fn push_quant_or_f32(
tensors: &mut Vec<PreparedTensor>,
name: &str,
shape: &[usize],
data: &[f32],
quant_type: GgmlType,
) -> Result<(), GgufError> {
if quant_type == GgmlType::F32 {
return push_f32_tensor(tensors, name, shape, data);
}
let bytes = quantize_weights(data, quant_type)?;
tensors.push(PreparedTensor {
name: name.into(),
dims: shape.to_vec(),
ggml_type: quant_type,
data: bytes,
});
Ok(())
}
struct PreparedTensor {
name: String,
dims: Vec<usize>,
ggml_type: GgmlType,
data: Vec<u8>,
}
#[derive(Clone, Debug)]
enum GgufValue {
U32(u32),
F32(f32),
String(String),
}
fn write_gguf_file(
path: &Path,
kv: &[(String, GgufValue)],
tensors: &[PreparedTensor],
) -> Result<(), GgufError> {
let mut header = Vec::new();
header.extend_from_slice(GGUF_MAGIC);
header.extend_from_slice(&GGUF_VERSION.to_le_bytes());
header.extend_from_slice(&(tensors.len() as u64).to_le_bytes());
header.extend_from_slice(&(kv.len() as u64).to_le_bytes());
for (key, value) in kv {
write_string(&mut header, key);
match value {
GgufValue::U32(v) => {
header.extend_from_slice(&4u32.to_le_bytes()); header.extend_from_slice(&v.to_le_bytes());
}
GgufValue::F32(v) => {
header.extend_from_slice(&6u32.to_le_bytes()); header.extend_from_slice(&v.to_le_bytes());
}
GgufValue::String(v) => {
header.extend_from_slice(&8u32.to_le_bytes()); write_string(&mut header, v);
}
}
}
let mut tensor_infos = Vec::new();
for tensor in tensors {
let mut info = Vec::new();
write_string(&mut info, &tensor.name);
info.extend_from_slice(&(tensor.dims.len() as u32).to_le_bytes());
for dim in tensor.dims.iter().rev() {
info.extend_from_slice(&(*dim as u64).to_le_bytes());
}
info.extend_from_slice(&(tensor.ggml_type as u32).to_le_bytes());
info.extend_from_slice(&0u64.to_le_bytes());
tensor_infos.push(info);
}
let mut info_bytes: Vec<u8> = tensor_infos.iter().flatten().copied().collect();
let data_start_unaligned = header.len() + info_bytes.len();
let alignment = GGUF_DEFAULT_ALIGNMENT as usize;
let padding = (alignment - (data_start_unaligned % alignment)) % alignment;
let mut offset = 0u64;
let mut cursor = 0usize;
for (index, tensor) in tensors.iter().enumerate() {
let info = &mut tensor_infos[index];
let off_pos = info.len() - 8;
info[off_pos..].copy_from_slice(&offset.to_le_bytes());
offset += tensor.data.len() as u64;
let pad = (alignment - (tensor.data.len() % alignment)) % alignment;
offset += pad as u64;
cursor += 1;
let _ = cursor;
}
info_bytes = tensor_infos.iter().flatten().copied().collect();
let mut writer = BufWriter::new(File::create(path)?);
writer.write_all(&header)?;
writer.write_all(&info_bytes)?;
writer.write_all(&vec![0u8; padding])?;
for tensor in tensors {
writer.write_all(&tensor.data)?;
let pad = (alignment - (tensor.data.len() % alignment)) % alignment;
if pad > 0 {
writer.write_all(&vec![0u8; pad])?;
}
}
writer.flush()?;
Ok(())
}
fn write_string(buf: &mut Vec<u8>, value: &str) {
let bytes = value.as_bytes();
buf.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
buf.extend_from_slice(bytes);
}
struct GgufTensor {
name: String,
dims: Vec<usize>,
ggml_type: GgmlType,
data: Vec<u8>,
}
fn read_gguf_model(
path: &Path,
) -> Result<(CausalLmConfig, BTreeMap<String, Vec<f32>>, Option<Vec<u8>>), GgufError> {
let mut file = File::open(path)?;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes)?;
if bytes.len() < 24 || &bytes[0..4] != GGUF_MAGIC {
return Err(GgufError::Invalid("not a GGUF file".into()));
}
let version = u32::from_le_bytes(bytes[4..8].try_into().unwrap());
if version != 2 && version != 3 {
return Err(GgufError::Invalid(format!(
"unsupported GGUF version {version}"
)));
}
let tensor_count = u64::from_le_bytes(bytes[8..16].try_into().unwrap()) as usize;
let kv_count = u64::from_le_bytes(bytes[16..24].try_into().unwrap()) as usize;
let mut cursor = 24usize;
let mut kv = BTreeMap::new();
for _ in 0..kv_count {
let key = read_string(&bytes, &mut cursor)?;
let value = read_value(&bytes, &mut cursor)?;
kv.insert(key, value);
}
let mut infos = Vec::with_capacity(tensor_count);
for _ in 0..tensor_count {
let name = read_string(&bytes, &mut cursor)?;
let n_dims = u32::from_le_bytes(bytes[cursor..cursor + 4].try_into().unwrap()) as usize;
cursor += 4;
let mut dims = Vec::with_capacity(n_dims);
for _ in 0..n_dims {
let dim = u64::from_le_bytes(bytes[cursor..cursor + 8].try_into().unwrap()) as usize;
cursor += 8;
dims.push(dim);
}
dims.reverse(); let ggml_type = GgmlType::from_u32(u32::from_le_bytes(
bytes[cursor..cursor + 4].try_into().unwrap(),
))?;
cursor += 4;
let offset = u64::from_le_bytes(bytes[cursor..cursor + 8].try_into().unwrap());
cursor += 8;
infos.push((name, dims, ggml_type, offset));
}
let alignment = match kv.get("general.alignment") {
Some(GgufValue::U32(value)) => *value as usize,
_ => GGUF_DEFAULT_ALIGNMENT as usize,
};
let data_base = align_up(cursor, alignment);
let mut tensors = Vec::new();
for (name, dims, ggml_type, offset) in infos {
let n_elements: usize = dims.iter().copied().product();
let nbytes = nbytes_for(ggml_type, n_elements)?;
let start = data_base + offset as usize;
let end = start + nbytes;
if end > bytes.len() {
return Err(GgufError::Invalid(format!(
"tensor {name} exceeds file size"
)));
}
tensors.push(GgufTensor {
name,
dims,
ggml_type,
data: bytes[start..end].to_vec(),
});
}
let config = config_from_kv(&kv, &tensors)?;
let tokenizer_json = match kv.get("oxidelm.tokenizer_json") {
Some(GgufValue::String(value)) => Some(value.as_bytes().to_vec()),
_ => None,
};
let mut fp32 = BTreeMap::new();
for tensor in &tensors {
let values = dequantize_tensor(tensor)?;
fp32.insert(tensor.name.clone(), values);
}
Ok((config, fp32, tokenizer_json))
}
fn nbytes_for(ggml_type: GgmlType, n_elements: usize) -> Result<usize, GgufError> {
let block = ggml_type.block_size();
if n_elements % block != 0 {
return Err(GgufError::Invalid(format!(
"element count {n_elements} not divisible by block size {block} for {}",
ggml_type.as_str()
)));
}
Ok((n_elements / block) * ggml_type.type_size())
}
fn config_from_kv(
kv: &BTreeMap<String, GgufValue>,
tensors: &[GgufTensor],
) -> Result<CausalLmConfig, GgufError> {
if let Some(GgufValue::String(json)) = kv.get("oxidelm.model_config") {
return Ok(serde_json::from_str(json)?);
}
let vocab = u32_kv(kv, "llama.vocab_size").or_else(|| {
tensors
.iter()
.find(|t| t.name == "token_embd.weight")
.and_then(|t| t.dims.first().copied().map(|v| v as u32))
});
let hidden = u32_kv(kv, "llama.embedding_length");
let layers = u32_kv(kv, "llama.block_count");
let intermediate = u32_kv(kv, "llama.feed_forward_length");
let heads = u32_kv(kv, "llama.attention.head_count");
let kv_heads = u32_kv(kv, "llama.attention.head_count_kv").or(heads);
let ctx = u32_kv(kv, "llama.context_length").unwrap_or(2048);
let rope = f32_kv(kv, "llama.rope.freq_base").unwrap_or(10_000.0);
let eps = f32_kv(kv, "llama.attention.layer_norm_rms_epsilon").unwrap_or(1e-5);
let (Some(vocab), Some(hidden), Some(layers), Some(intermediate), Some(heads), Some(kv_heads)) =
(vocab, hidden, layers, intermediate, heads, kv_heads)
else {
return Err(GgufError::Invalid(
"GGUF missing llama.* architecture metadata and oxidelm.model_config".into(),
));
};
let tie = !tensors.iter().any(|t| t.name == "output.weight");
Ok(CausalLmConfig {
vocab_size: vocab as usize,
hidden_size: hidden as usize,
num_layers: layers as usize,
num_heads: heads as usize,
num_key_value_heads: kv_heads as usize,
intermediate_size: intermediate as usize,
max_sequence_length: ctx as usize,
rms_norm_epsilon: eps,
rope_theta: rope,
rope_scaling: None,
eos_token_id: 0,
seed: 0,
initializer_std: crate::model::DEFAULT_INITIALIZER_STD,
tie_word_embeddings: tie,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
}
fn u32_kv(kv: &BTreeMap<String, GgufValue>, key: &str) -> Option<u32> {
match kv.get(key) {
Some(GgufValue::U32(v)) => Some(*v),
_ => None,
}
}
fn f32_kv(kv: &BTreeMap<String, GgufValue>, key: &str) -> Option<f32> {
match kv.get(key) {
Some(GgufValue::F32(v)) => Some(*v),
_ => None,
}
}
fn apply_gguf_tensors(
model: &mut CausalLM,
tensors: &BTreeMap<String, Vec<f32>>,
) -> Result<(), GgufError> {
let emb = tensors
.get("token_embd.weight")
.ok_or_else(|| GgufError::Invalid("missing token_embd.weight".into()))?;
model.tok_embeddings.weight.set_data(emb.clone());
let norm = tensors
.get("output_norm.weight")
.ok_or_else(|| GgufError::Invalid("missing output_norm.weight".into()))?;
model.norm.weight.set_data(norm.clone());
if !model.config.tie_word_embeddings {
if let Some(values) = tensors.get("output.weight") {
set_linear_from_out_in(&mut model.lm_head, values)?;
}
}
for (layer_index, layer) in model.layers.iter_mut().enumerate() {
let prefix = format!("blk.{layer_index}");
if let Some(values) = tensors.get(&format!("{prefix}.attn_norm.weight")) {
layer.attention_norm.weight.set_data(values.clone());
}
if let Some(values) = tensors.get(&format!("{prefix}.ffn_norm.weight")) {
layer.ffn_norm.weight.set_data(values.clone());
}
for (suffix, linear) in [
("attn_q", &mut layer.attention.q_proj),
("attn_k", &mut layer.attention.k_proj),
("attn_v", &mut layer.attention.v_proj),
("attn_output", &mut layer.attention.o_proj),
("ffn_gate", &mut layer.feed_forward.gate_proj),
("ffn_up", &mut layer.feed_forward.up_proj),
("ffn_down", &mut layer.feed_forward.down_proj),
] {
let key = format!("{prefix}.{suffix}.weight");
let values = tensors
.get(&key)
.ok_or_else(|| GgufError::Invalid(format!("missing {key}")))?;
set_linear_from_out_in(linear, values)?;
}
}
Ok(())
}
fn set_linear_from_out_in(
linear: &mut crate::layers::Linear,
out_in: &[f32],
) -> Result<(), GgufError> {
let expected = linear.in_features * linear.out_features;
if out_in.len() != expected {
return Err(GgufError::Invalid(format!(
"linear expects {expected} weights, got {}",
out_in.len()
)));
}
let mut storage = vec![0.0; expected];
for input in 0..linear.in_features {
for output in 0..linear.out_features {
let value = out_in[output * linear.in_features + input];
if linear.transpose_weight {
storage[output * linear.in_features + input] = value;
} else {
storage[input * linear.out_features + output] = value;
}
}
}
linear.weight.set_data(storage);
Ok(())
}
fn read_string(bytes: &[u8], cursor: &mut usize) -> Result<String, GgufError> {
if *cursor + 8 > bytes.len() {
return Err(GgufError::Invalid("truncated string length".into()));
}
let len = u64::from_le_bytes(bytes[*cursor..*cursor + 8].try_into().unwrap()) as usize;
*cursor += 8;
if *cursor + len > bytes.len() {
return Err(GgufError::Invalid("truncated string".into()));
}
let value = String::from_utf8_lossy(&bytes[*cursor..*cursor + len]).into_owned();
*cursor += len;
Ok(value)
}
fn read_value(bytes: &[u8], cursor: &mut usize) -> Result<GgufValue, GgufError> {
if *cursor + 4 > bytes.len() {
return Err(GgufError::Invalid("truncated value type".into()));
}
let ty = u32::from_le_bytes(bytes[*cursor..*cursor + 4].try_into().unwrap());
*cursor += 4;
match ty {
4 => {
let v = u32::from_le_bytes(bytes[*cursor..*cursor + 4].try_into().unwrap());
*cursor += 4;
Ok(GgufValue::U32(v))
}
5 => {
let v = i32::from_le_bytes(bytes[*cursor..*cursor + 4].try_into().unwrap());
*cursor += 4;
Ok(GgufValue::U32(v as u32))
}
6 => {
let v = f32::from_le_bytes(bytes[*cursor..*cursor + 4].try_into().unwrap());
*cursor += 4;
Ok(GgufValue::F32(v))
}
8 => Ok(GgufValue::String(read_string(bytes, cursor)?)),
9 => {
let elem_ty = u32::from_le_bytes(bytes[*cursor..*cursor + 4].try_into().unwrap());
*cursor += 4;
let count = u64::from_le_bytes(bytes[*cursor..*cursor + 8].try_into().unwrap()) as usize;
*cursor += 8;
skip_array_elements(bytes, cursor, elem_ty, count)?;
Ok(GgufValue::String(String::new()))
}
10 => {
let v = u64::from_le_bytes(bytes[*cursor..*cursor + 8].try_into().unwrap());
*cursor += 8;
Ok(GgufValue::U32(v as u32))
}
other => Err(GgufError::Invalid(format!(
"unsupported GGUF value type {other}"
))),
}
}
fn skip_array_elements(
bytes: &[u8],
cursor: &mut usize,
elem_ty: u32,
count: usize,
) -> Result<(), GgufError> {
for _ in 0..count {
match elem_ty {
0 | 1 => *cursor += 1,
2 | 3 => *cursor += 2,
4 | 5 | 6 => *cursor += 4,
7 => *cursor += 1,
8 => {
let _ = read_string(bytes, cursor)?;
}
10 | 11 | 12 => *cursor += 8,
other => {
return Err(GgufError::Invalid(format!(
"unsupported GGUF array element type {other}"
)));
}
}
if *cursor > bytes.len() {
return Err(GgufError::Invalid("truncated array".into()));
}
}
Ok(())
}
fn align_up(value: usize, alignment: usize) -> usize {
(value + alignment - 1) / alignment * alignment
}
fn dequantize_tensor(tensor: &GgufTensor) -> Result<Vec<f32>, GgufError> {
let n: usize = tensor.dims.iter().copied().product();
match tensor.ggml_type {
GgmlType::F32 => {
let mut out = Vec::with_capacity(n);
for chunk in tensor.data.chunks_exact(4) {
out.push(f32::from_le_bytes(chunk.try_into().unwrap()));
}
if out.len() != n {
return Err(GgufError::Invalid("F32 length mismatch".into()));
}
Ok(out)
}
GgmlType::F16 => {
let mut out = Vec::with_capacity(n);
for chunk in tensor.data.chunks_exact(2) {
let bits = u16::from_le_bytes(chunk.try_into().unwrap());
out.push(f16::from_bits(bits).to_f32());
}
Ok(out)
}
other => dequantize_blocks(&tensor.data, n, other),
}
}
fn quantize_weights(values: &[f32], ggml_type: GgmlType) -> Result<Vec<u8>, GgufError> {
let block = ggml_type.block_size();
let mut padded = values.to_vec();
let rem = padded.len() % block;
if rem != 0 {
padded.extend(std::iter::repeat_n(0.0, block - rem));
}
match ggml_type {
GgmlType::Q4_0 => Ok(encode_q4_0(&padded)),
GgmlType::Q8_0 => Ok(encode_q8_0(&padded)),
GgmlType::Q4_K => Ok(encode_q4_k(&padded)),
GgmlType::Q5_K => Ok(encode_q5_k(&padded)),
GgmlType::Q6_K => Ok(encode_q6_k(&padded)),
other => Err(GgufError::Invalid(format!(
"cannot quantize to {}",
other.as_str()
))),
}
}
fn dequantize_blocks(data: &[u8], n: usize, ggml_type: GgmlType) -> Result<Vec<f32>, GgufError> {
let mut out = match ggml_type {
GgmlType::Q4_0 => decode_q4_0(data)?,
GgmlType::Q8_0 => decode_q8_0(data)?,
GgmlType::Q4_K => decode_q4_k(data)?,
GgmlType::Q5_K => decode_q5_k(data)?,
GgmlType::Q6_K => decode_q6_k(data)?,
other => {
return Err(GgufError::Invalid(format!(
"cannot dequantize {}",
other.as_str()
)));
}
};
if out.len() < n {
return Err(GgufError::Invalid("dequant produced too few values".into()));
}
out.truncate(n);
Ok(out)
}
fn encode_q4_0(values: &[f32]) -> Vec<u8> {
let mut out = Vec::with_capacity((values.len() / 32) * 18);
for chunk in values.chunks_exact(32) {
let absmax = chunk.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
let d = if absmax > 0.0 { absmax / 8.0 } else { 0.0 };
out.extend_from_slice(&f16::from_f32(d).to_bits().to_le_bytes());
let inv = if d > 0.0 { 1.0 / d } else { 0.0 };
for j in 0..16 {
let q0 = ((chunk[j] * inv).round() as i32 + 8).clamp(0, 15) as u8;
let q1 = ((chunk[j + 16] * inv).round() as i32 + 8).clamp(0, 15) as u8;
out.push(q0 | (q1 << 4));
}
}
out
}
fn decode_q4_0(data: &[u8]) -> Result<Vec<f32>, GgufError> {
if data.len() % 18 != 0 {
return Err(GgufError::Invalid("invalid Q4_0 payload".into()));
}
let mut out = vec![0.0; (data.len() / 18) * 32];
let mut out_off = 0usize;
for block in data.chunks_exact(18) {
let d = f16::from_bits(u16::from_le_bytes(block[0..2].try_into().unwrap())).to_f32();
for j in 0..16 {
let byte = block[2 + j];
out[out_off + j] = ((byte & 0x0F) as i8 as f32 - 8.0) * d;
out[out_off + j + 16] = (((byte >> 4) & 0x0F) as i8 as f32 - 8.0) * d;
}
out_off += 32;
}
Ok(out)
}
fn encode_q8_0(values: &[f32]) -> Vec<u8> {
let mut out = Vec::with_capacity((values.len() / 32) * 34);
for chunk in values.chunks_exact(32) {
let absmax = chunk.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
let d = if absmax > 0.0 { absmax / 127.0 } else { 0.0 };
out.extend_from_slice(&f16::from_f32(d).to_bits().to_le_bytes());
let inv = if d > 0.0 { 1.0 / d } else { 0.0 };
for value in chunk {
let q = (value * inv).round().clamp(-127.0, 127.0) as i8;
out.push(q as u8);
}
}
out
}
fn decode_q8_0(data: &[u8]) -> Result<Vec<f32>, GgufError> {
if data.len() % 34 != 0 {
return Err(GgufError::Invalid("invalid Q8_0 payload".into()));
}
let mut out = Vec::with_capacity((data.len() / 34) * 32);
for block in data.chunks_exact(34) {
let d = f16::from_bits(u16::from_le_bytes(block[0..2].try_into().unwrap())).to_f32();
for byte in &block[2..] {
out.push((*byte as i8 as f32) * d);
}
}
Ok(out)
}
fn get_scale_min_k4(j: usize, scales: &[u8]) -> (u8, u8) {
if j < 4 {
(scales[j] & 63, scales[j + 4] & 63)
} else {
let d = (scales[j + 4] & 0xF) | ((scales[j - 4] >> 6) << 4);
let m = (scales[j + 4] >> 4) | ((scales[j] >> 6) << 4);
(d, m)
}
}
fn encode_q4_k(values: &[f32]) -> Vec<u8> {
let mut out = Vec::with_capacity((values.len() / 256) * 144);
for chunk in values.chunks_exact(256) {
let mut scales = [0u8; 12];
let mut mins_f = [0.0f32; 8];
let mut scales_f = [0.0f32; 8];
let mut max_scale = 0.0f32;
let mut max_min = 0.0f32;
for j in 0..8 {
let group = &chunk[j * 32..(j + 1) * 32];
let min_v = group.iter().copied().fold(f32::INFINITY, f32::min);
let max_v = group.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let scale = ((max_v - min_v) / 15.0).max(0.0);
scales_f[j] = scale;
mins_f[j] = -min_v;
max_scale = max_scale.max(scale);
max_min = max_min.max(-min_v);
}
let inv_scale = if max_scale > 0.0 { 63.0 / max_scale } else { 0.0 };
let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 };
for j in 0..8 {
let ls = ((inv_scale * scales_f[j]).round() as u8).min(63);
let lm = ((inv_min * mins_f[j]).round() as u8).min(63);
if j < 4 {
scales[j] = ls;
scales[j + 4] = lm;
} else {
scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4);
scales[j - 4] |= (ls >> 4) << 6;
scales[j] |= (lm >> 4) << 6;
}
}
let d = max_scale / 63.0;
let dmin = max_min / 63.0;
out.extend_from_slice(&f16::from_f32(d).to_bits().to_le_bytes());
out.extend_from_slice(&f16::from_f32(dmin).to_bits().to_le_bytes());
out.extend_from_slice(&scales);
let mut qs = [0u8; 128];
let mut codes = [0u8; 256];
for j in 0..8 {
let (sc, m) = get_scale_min_k4(j, &scales);
let dj = d * sc as f32;
let dm = dmin * m as f32;
if dj == 0.0 {
continue;
}
for ii in 0..32 {
let l = ((chunk[j * 32 + ii] + dm) / dj).round() as i32;
codes[j * 32 + ii] = l.clamp(0, 15) as u8;
}
}
for j in (0..256).step_by(64) {
for l in 0..32 {
qs[j / 2 + l] = codes[j + l] | (codes[j + l + 32] << 4);
}
}
out.extend_from_slice(&qs);
}
out
}
fn decode_q4_k(data: &[u8]) -> Result<Vec<f32>, GgufError> {
if data.len() % 144 != 0 {
return Err(GgufError::Invalid("invalid Q4_K payload".into()));
}
let mut out = Vec::with_capacity((data.len() / 144) * 256);
for block in data.chunks_exact(144) {
let d = f16::from_bits(u16::from_le_bytes(block[0..2].try_into().unwrap())).to_f32();
let min = f16::from_bits(u16::from_le_bytes(block[2..4].try_into().unwrap())).to_f32();
let scales = &block[4..16];
let mut q = &block[16..];
let mut is = 0usize;
for _ in 0..4 {
let (sc, m) = get_scale_min_k4(is, scales);
let d1 = d * sc as f32;
let m1 = min * m as f32;
let (sc2, m2) = get_scale_min_k4(is + 1, scales);
let d2 = d * sc2 as f32;
let m2v = min * m2 as f32;
for l in 0..32 {
out.push(d1 * (q[l] & 0xF) as f32 - m1);
}
for l in 0..32 {
out.push(d2 * (q[l] >> 4) as f32 - m2v);
}
q = &q[32..];
is += 2;
}
}
Ok(out)
}
fn encode_q5_k(values: &[f32]) -> Vec<u8> {
let mut out = Vec::with_capacity((values.len() / 256) * 176);
for chunk in values.chunks_exact(256) {
let mut scales = [0u8; 12];
let mut mins_f = [0.0f32; 8];
let mut scales_f = [0.0f32; 8];
let mut max_scale = 0.0f32;
let mut max_min = 0.0f32;
for j in 0..8 {
let group = &chunk[j * 32..(j + 1) * 32];
let min_v = group.iter().copied().fold(f32::INFINITY, f32::min);
let max_v = group.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let scale = ((max_v - min_v) / 31.0).max(0.0);
scales_f[j] = scale;
mins_f[j] = -min_v;
max_scale = max_scale.max(scale);
max_min = max_min.max(-min_v);
}
let inv_scale = if max_scale > 0.0 { 63.0 / max_scale } else { 0.0 };
let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 };
for j in 0..8 {
let ls = ((inv_scale * scales_f[j]).round() as u8).min(63);
let lm = ((inv_min * mins_f[j]).round() as u8).min(63);
if j < 4 {
scales[j] = ls;
scales[j + 4] = lm;
} else {
scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4);
scales[j - 4] |= (ls >> 4) << 6;
scales[j] |= (lm >> 4) << 6;
}
}
let d = max_scale / 63.0;
let dmin = max_min / 63.0;
out.extend_from_slice(&f16::from_f32(d).to_bits().to_le_bytes());
out.extend_from_slice(&f16::from_f32(dmin).to_bits().to_le_bytes());
out.extend_from_slice(&scales);
let mut codes = [0u8; 256];
for j in 0..8 {
let (sc, m) = get_scale_min_k4(j, &scales);
let dj = d * sc as f32;
let dm = dmin * m as f32;
if dj == 0.0 {
continue;
}
for ii in 0..32 {
let l = ((chunk[j * 32 + ii] + dm) / dj).round() as i32;
codes[j * 32 + ii] = l.clamp(0, 31) as u8;
}
}
let mut qh = [0u8; 32];
let mut qs = [0u8; 128];
let mut m1 = 1u8;
let mut m2 = 2u8;
let mut ql_off = 0usize;
for n in (0..256).step_by(64) {
for j in 0..32 {
let mut l1 = codes[n + j];
if l1 > 15 {
l1 -= 16;
qh[j] |= m1;
}
let mut l2 = codes[n + j + 32];
if l2 > 15 {
l2 -= 16;
qh[j] |= m2;
}
qs[ql_off + j] = l1 | (l2 << 4);
}
m1 <<= 2;
m2 <<= 2;
ql_off += 32;
}
out.extend_from_slice(&qh);
out.extend_from_slice(&qs);
}
out
}
fn decode_q5_k(data: &[u8]) -> Result<Vec<f32>, GgufError> {
if data.len() % 176 != 0 {
return Err(GgufError::Invalid("invalid Q5_K payload".into()));
}
let mut out = Vec::with_capacity((data.len() / 176) * 256);
for block in data.chunks_exact(176) {
let d = f16::from_bits(u16::from_le_bytes(block[0..2].try_into().unwrap())).to_f32();
let min = f16::from_bits(u16::from_le_bytes(block[2..4].try_into().unwrap())).to_f32();
let scales = &block[4..16];
let qh = &block[16..48];
let mut ql = &block[48..];
let mut is = 0usize;
let mut u1 = 1u8;
let mut u2 = 2u8;
for _ in 0..4 {
let (sc, m) = get_scale_min_k4(is, scales);
let d1 = d * sc as f32;
let m1 = min * m as f32;
let (sc2, m2) = get_scale_min_k4(is + 1, scales);
let d2 = d * sc2 as f32;
let m2v = min * m2 as f32;
for l in 0..32 {
let q = (ql[l] & 0xF) + if qh[l] & u1 != 0 { 16 } else { 0 };
out.push(d1 * q as f32 - m1);
}
for l in 0..32 {
let q = (ql[l] >> 4) + if qh[l] & u2 != 0 { 16 } else { 0 };
out.push(d2 * q as f32 - m2v);
}
ql = &ql[32..];
is += 2;
u1 <<= 2;
u2 <<= 2;
}
}
Ok(out)
}
fn encode_q6_k(values: &[f32]) -> Vec<u8> {
let mut out = Vec::with_capacity((values.len() / 256) * 210);
for chunk in values.chunks_exact(256) {
let mut scales = [0i8; 16];
let mut scales_f = [0.0f32; 16];
let mut max_abs_scale = 0.0f32;
let mut max_scale = 0.0f32;
for ib in 0..16 {
let group = &chunk[ib * 16..(ib + 1) * 16];
let absmax = group.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
let scale = if absmax > 0.0 { absmax / 32.0 } else { 0.0 };
scales_f[ib] = scale;
if scale.abs() > max_abs_scale {
max_abs_scale = scale.abs();
max_scale = scale;
}
}
if max_abs_scale < 1e-12 {
out.extend(std::iter::repeat_n(0u8, 210));
continue;
}
let iscale = -128.0 / max_scale;
let d = 1.0 / iscale;
for ib in 0..16 {
scales[ib] = (iscale * scales_f[ib]).round().clamp(-128.0, 127.0) as i8;
}
let mut codes = [0u8; 256];
for j in 0..16 {
let dj = d * scales[j] as f32;
if dj == 0.0 {
continue;
}
for ii in 0..16 {
let l = (chunk[j * 16 + ii] / dj).round() as i32;
codes[j * 16 + ii] = (l.clamp(-32, 31) + 32) as u8;
}
}
let mut ql = [0u8; 128];
let mut qh = [0u8; 64];
for (block_i, j) in (0..256).step_by(128).enumerate() {
let ql_off = block_i * 64;
let qh_off = block_i * 32;
for l in 0..32 {
let q1 = codes[j + l] & 0xF;
let q2 = codes[j + l + 32] & 0xF;
let q3 = codes[j + l + 64] & 0xF;
let q4 = codes[j + l + 96] & 0xF;
ql[ql_off + l] = q1 | (q3 << 4);
ql[ql_off + l + 32] = q2 | (q4 << 4);
qh[qh_off + l] = (codes[j + l] >> 4)
| ((codes[j + l + 32] >> 4) << 2)
| ((codes[j + l + 64] >> 4) << 4)
| ((codes[j + l + 96] >> 4) << 6);
}
}
out.extend_from_slice(&ql);
out.extend_from_slice(&qh);
for s in scales {
out.push(s as u8);
}
out.extend_from_slice(&f16::from_f32(d).to_bits().to_le_bytes());
}
out
}
fn decode_q6_k(data: &[u8]) -> Result<Vec<f32>, GgufError> {
if data.len() % 210 != 0 {
return Err(GgufError::Invalid("invalid Q6_K payload".into()));
}
let mut out = vec![0.0; (data.len() / 210) * 256];
let mut out_off = 0usize;
for block in data.chunks_exact(210) {
let ql = &block[0..128];
let qh = &block[128..192];
let scales = &block[192..208];
let d = f16::from_bits(u16::from_le_bytes(block[208..210].try_into().unwrap())).to_f32();
let mut ql_off = 0usize;
let mut qh_off = 0usize;
let mut sc_off = 0usize;
for _ in 0..2 {
for l in 0..32 {
let is = l / 16;
let q1 = ((ql[ql_off + l] & 0xF) | (((qh[qh_off + l] >> 0) & 3) << 4)) as i8 - 32;
let q2 = ((ql[ql_off + l + 32] & 0xF) | (((qh[qh_off + l] >> 2) & 3) << 4)) as i8 - 32;
let q3 = ((ql[ql_off + l] >> 4) | (((qh[qh_off + l] >> 4) & 3) << 4)) as i8 - 32;
let q4 = ((ql[ql_off + l + 32] >> 4) | (((qh[qh_off + l] >> 6) & 3) << 4)) as i8 - 32;
let sc = |idx: usize| scales[sc_off + idx] as i8 as f32;
out[out_off + l] = d * sc(is) * q1 as f32;
out[out_off + l + 32] = d * sc(is + 2) * q2 as f32;
out[out_off + l + 64] = d * sc(is + 4) * q3 as f32;
out[out_off + l + 96] = d * sc(is + 6) * q4 as f32;
}
out_off += 128;
ql_off += 64;
qh_off += 32;
sc_off += 8;
}
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn q4_0_and_q8_0_round_trip_bounded() {
let values: Vec<f32> = (0..64).map(|i| (i as f32 - 32.0) * 0.05).collect();
for ty in [GgmlType::Q4_0, GgmlType::Q8_0] {
let encoded = quantize_weights(&values, ty).unwrap();
let decoded = dequantize_blocks(&encoded, values.len(), ty).unwrap();
let mse: f32 = values
.iter()
.zip(decoded.iter())
.map(|(a, b)| (a - b).powi(2))
.sum::<f32>()
/ values.len() as f32;
assert!(mse < 0.05, "{} mse={mse}", ty.as_str());
}
}
#[test]
fn qk_round_trip_bounded() {
let values: Vec<f32> = (0..256).map(|i| ((i % 17) as f32 - 8.0) * 0.1).collect();
for ty in [GgmlType::Q4_K, GgmlType::Q5_K, GgmlType::Q6_K] {
let encoded = quantize_weights(&values, ty).unwrap();
let decoded = dequantize_blocks(&encoded, values.len(), ty).unwrap();
let mse: f32 = values
.iter()
.zip(decoded.iter())
.map(|(a, b)| (a - b).powi(2))
.sum::<f32>()
/ values.len() as f32;
assert!(mse < 0.2, "{} mse={mse}", ty.as_str());
}
}
#[test]
fn native_scheme_mapping() {
assert!(GgmlType::Q8_0.native_scheme().is_some());
assert!(GgmlType::Q4_0.native_scheme().is_some());
assert!(GgmlType::Q5_K.native_scheme().is_none());
}
#[test]
fn gguf_checkpoint_round_trip() {
use crate::optimizer::{AdamW, AdamWConfig};
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
use crate::trainer::{save_checkpoint, LearningRateSchedule, TrainerConfig};
use std::time::{SystemTime, UNIX_EPOCH};
let stamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let directory = std::env::temp_dir().join(format!("oxidelm-gguf-{stamp}"));
std::fs::create_dir_all(&directory).unwrap();
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let config = CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 32,
num_layers: 1,
num_heads: 4,
num_key_value_heads: 2,
intermediate_size: 64,
max_sequence_length: 16,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 0,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
};
let model = CausalLM::new(config).unwrap();
let optimizer = AdamW::new(&model.parameters(), AdamWConfig::default()).unwrap();
let checkpoint = directory.join("model.bin");
save_checkpoint(
&checkpoint,
&model,
&optimizer,
&tokenizer,
&LearningRateSchedule {
base_learning_rate: 0.0,
minimum_learning_rate: 0.0,
warmup_steps: 0,
total_steps: 1,
},
&TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
0,
0,
0,
)
.unwrap();
let gguf = directory.join("model.gguf");
export_checkpoint_gguf(
&checkpoint,
&gguf,
&GgufExportOptions {
quant_type: GgmlType::Q8_0,
keep_embed_f32: true,
},
)
.unwrap();
let restored = directory.join("restored.bin");
import_gguf_checkpoint(&gguf, &restored, &GgufImportOptions::default()).unwrap();
let loaded = crate::trainer::load_checkpoint(&restored).unwrap();
assert_eq!(loaded.model.config.hidden_size, 32);
assert_eq!(loaded.model.layers.len(), 1);
let _ = std::fs::remove_dir_all(directory);
}
}