use crate::{
error::{Error, Result},
progress::ProgressEvent,
saver::{ModelSaver, SaveOptions},
};
use candle_core::{DType, Tensor};
use std::{
collections::HashMap,
fs::File,
io::{BufWriter, Write},
path::Path,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(non_camel_case_types)] pub enum GGUFQuantType {
F32,
F16,
Q4_0,
Q4_1,
Q8_0,
Q4_K_M,
Q6_K,
}
impl GGUFQuantType {
pub fn type_id(&self) -> u32 {
match self {
Self::F32 => 0,
Self::F16 => 1,
Self::Q4_0 => 2,
Self::Q4_1 => 3,
Self::Q8_0 => 7,
Self::Q4_K_M => 12,
Self::Q6_K => 14,
}
}
pub fn name(&self) -> &'static str {
match self {
Self::F32 => "F32",
Self::F16 => "F16",
Self::Q4_0 => "Q4_0",
Self::Q4_1 => "Q4_1",
Self::Q8_0 => "Q8_0",
Self::Q4_K_M => "Q4_K_M",
Self::Q6_K => "Q6_K",
}
}
pub fn needs_calibration(&self) -> bool {
matches!(self, Self::Q4_K_M | Self::Q6_K)
}
}
#[derive(Debug, Clone)]
pub struct GGUFExportOptions {
pub quantization: GGUFQuantType,
pub architecture: String,
pub context_length: Option<u32>,
pub vocab_size: Option<u32>,
pub num_layers: Option<u32>,
pub num_heads: Option<u32>,
pub embedding_dim: Option<u32>,
pub custom_metadata: HashMap<String, MetadataValue>,
}
impl Default for GGUFExportOptions {
fn default() -> Self {
Self {
quantization: GGUFQuantType::F16,
architecture: "unknown".to_string(),
context_length: None,
vocab_size: None,
num_layers: None,
num_heads: None,
embedding_dim: None,
custom_metadata: HashMap::new(),
}
}
}
impl GGUFExportOptions {
pub fn new(architecture: &str) -> Self {
Self {
architecture: architecture.to_string(),
..Default::default()
}
}
pub fn with_quantization(mut self, quant: GGUFQuantType) -> Self {
self.quantization = quant;
self
}
pub fn with_model_params(
mut self,
context_length: u32,
vocab_size: u32,
num_layers: u32,
num_heads: u32,
embedding_dim: u32,
) -> Self {
self.context_length = Some(context_length);
self.vocab_size = Some(vocab_size);
self.num_layers = Some(num_layers);
self.num_heads = Some(num_heads);
self.embedding_dim = Some(embedding_dim);
self
}
pub fn with_metadata<K: Into<String>>(mut self, key: K, value: MetadataValue) -> Self {
self.custom_metadata.insert(key.into(), value);
self
}
}
#[derive(Debug, Clone)]
pub enum MetadataValue {
String(String),
Int(i64),
Float(f64),
Bool(bool),
StringArray(Vec<String>),
IntArray(Vec<i64>),
}
impl MetadataValue {
pub fn type_id(&self) -> u32 {
match self {
Self::String(_) => 8, Self::Int(_) => 4, Self::Float(_) => 6, Self::Bool(_) => 7, Self::StringArray(_) => 9, Self::IntArray(_) => 9, }
}
}
pub struct GGUFWriter {
writer: BufWriter<File>,
options: GGUFExportOptions,
tensor_count: usize,
}
impl GGUFWriter {
pub fn new<P: AsRef<Path>>(path: P, options: GGUFExportOptions) -> Result<Self> {
let file = File::create(path.as_ref()).map_err(|e| {
Error::model_loading(&format!(
"Failed to create GGUF file {}: {}",
path.as_ref().display(),
e
))
})?;
Ok(Self {
writer: BufWriter::new(file),
options,
tensor_count: 0,
})
}
pub fn write_header(&mut self, tensors: &HashMap<String, Tensor>) -> Result<()> {
self.writer
.write_all(b"GGUF")
.map_err(|e| Error::model_loading(&format!("Failed to write GGUF magic: {}", e)))?;
self.writer
.write_all(&3u32.to_le_bytes())
.map_err(|e| Error::model_loading(&format!("Failed to write GGUF version: {}", e)))?;
self.tensor_count = tensors.len();
self.writer
.write_all(&(self.tensor_count as u64).to_le_bytes())
.map_err(|e| Error::model_loading(&format!("Failed to write tensor count: {}", e)))?;
self.write_metadata(tensors)?;
Ok(())
}
fn write_metadata(&mut self, tensors: &HashMap<String, Tensor>) -> Result<()> {
let mut metadata = HashMap::new();
metadata.insert(
"general.architecture".to_string(),
MetadataValue::String(self.options.architecture.clone()),
);
metadata.insert(
"general.quantization_version".to_string(),
MetadataValue::Int(2),
);
metadata.insert("general.alignment".to_string(), MetadataValue::Int(32));
if let Some(ctx_len) = self.options.context_length {
metadata.insert(
format!("{}.context_length", self.options.architecture),
MetadataValue::Int(ctx_len as i64),
);
}
if let Some(vocab_size) = self.options.vocab_size {
metadata.insert(
format!("{}.vocab_size", self.options.architecture),
MetadataValue::Int(vocab_size as i64),
);
}
if let Some(num_layers) = self.options.num_layers {
metadata.insert(
format!("{}.block_count", self.options.architecture),
MetadataValue::Int(num_layers as i64),
);
}
if let Some(num_heads) = self.options.num_heads {
metadata.insert(
format!("{}.attention.head_count", self.options.architecture),
MetadataValue::Int(num_heads as i64),
);
}
if let Some(emb_dim) = self.options.embedding_dim {
metadata.insert(
format!("{}.embedding_length", self.options.architecture),
MetadataValue::Int(emb_dim as i64),
);
}
for (key, value) in &self.options.custom_metadata {
metadata.insert(key.clone(), value.clone());
}
self.add_tensor_metadata(&mut metadata, tensors)?;
self.writer
.write_all(&(metadata.len() as u64).to_le_bytes())
.map_err(|e| Error::model_loading(&format!("Failed to write metadata count: {}", e)))?;
for (key, value) in metadata {
self.write_metadata_entry(&key, &value)?;
}
Ok(())
}
fn add_tensor_metadata(
&self,
metadata: &mut HashMap<String, MetadataValue>,
tensors: &HashMap<String, Tensor>,
) -> Result<()> {
let mut vocab_size = None;
let mut embedding_dim = None;
for (name, tensor) in tensors {
let shape = tensor.shape().dims();
if name.contains("embed") || name.contains("wte") || name.contains("word_embeddings") {
if shape.len() == 2 {
vocab_size = Some(shape[0]);
embedding_dim = Some(shape[1]);
}
}
metadata.insert(
format!("tensor.{}.type", name),
MetadataValue::Int(self.options.quantization.type_id() as i64),
);
}
if let Some(vocab) = vocab_size {
metadata.insert(
format!("{}.vocab_size", self.options.architecture),
MetadataValue::Int(vocab as i64),
);
}
if let Some(emb) = embedding_dim {
metadata.insert(
format!("{}.embedding_length", self.options.architecture),
MetadataValue::Int(emb as i64),
);
}
Ok(())
}
fn write_metadata_entry(&mut self, key: &str, value: &MetadataValue) -> Result<()> {
self.writer
.write_all(&(key.len() as u64).to_le_bytes())
.map_err(|e| Error::model_loading(&format!("Failed to write key length: {}", e)))?;
self.writer
.write_all(key.as_bytes())
.map_err(|e| Error::model_loading(&format!("Failed to write key '{}': {}", key, e)))?;
self.writer
.write_all(&value.type_id().to_le_bytes())
.map_err(|e| Error::model_loading(&format!("Failed to write value type: {}", e)))?;
match value {
MetadataValue::String(s) => {
self.writer
.write_all(&(s.len() as u64).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write string length: {}", e))
})?;
self.writer.write_all(s.as_bytes()).map_err(|e| {
Error::model_loading(&format!("Failed to write string value: {}", e))
})?;
}
MetadataValue::Int(i) => {
self.writer
.write_all(&(*i as i32).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write int value: {}", e))
})?;
}
MetadataValue::Float(f) => {
self.writer.write_all(&f.to_le_bytes()).map_err(|e| {
Error::model_loading(&format!("Failed to write float value: {}", e))
})?;
}
MetadataValue::Bool(b) => {
self.writer
.write_all(&[if *b { 1u8 } else { 0u8 }])
.map_err(|e| {
Error::model_loading(&format!("Failed to write bool value: {}", e))
})?;
}
MetadataValue::StringArray(arr) => {
self.writer.write_all(&8u32.to_le_bytes()).map_err(|e| {
Error::model_loading(&format!("Failed to write array element type: {}", e))
})?;
self.writer
.write_all(&(arr.len() as u64).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write array length: {}", e))
})?;
for s in arr {
self.writer
.write_all(&(s.len() as u64).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!(
"Failed to write array string length: {}",
e
))
})?;
self.writer.write_all(s.as_bytes()).map_err(|e| {
Error::model_loading(&format!("Failed to write array string: {}", e))
})?;
}
}
MetadataValue::IntArray(arr) => {
self.writer.write_all(&4u32.to_le_bytes()).map_err(|e| {
Error::model_loading(&format!("Failed to write array element type: {}", e))
})?;
self.writer
.write_all(&(arr.len() as u64).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write array length: {}", e))
})?;
for i in arr {
self.writer
.write_all(&(*i as i32).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write array int: {}", e))
})?;
}
}
}
Ok(())
}
pub fn write_tensor_infos(&mut self, tensors: &HashMap<String, Tensor>) -> Result<()> {
for (name, tensor) in tensors {
self.writer
.write_all(&(name.len() as u64).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write tensor name length: {}", e))
})?;
self.writer.write_all(name.as_bytes()).map_err(|e| {
Error::model_loading(&format!("Failed to write tensor name '{}': {}", name, e))
})?;
let shape = tensor.shape().dims();
self.writer
.write_all(&(shape.len() as u32).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write tensor dimensions count: {}", e))
})?;
for &dim in shape {
self.writer
.write_all(&(dim as u64).to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write tensor dimension: {}", e))
})?;
}
self.writer
.write_all(&self.options.quantization.type_id().to_le_bytes())
.map_err(|e| {
Error::model_loading(&format!("Failed to write tensor type: {}", e))
})?;
self.writer.write_all(&0u64.to_le_bytes()).map_err(|e| {
Error::model_loading(&format!("Failed to write tensor offset placeholder: {}", e))
})?;
}
Ok(())
}
pub fn write_tensors(&mut self, tensors: &HashMap<String, Tensor>) -> Result<()> {
for (name, tensor) in tensors {
self.write_tensor_data(name, tensor)?;
}
Ok(())
}
fn write_tensor_data(&mut self, _name: &str, tensor: &Tensor) -> Result<()> {
let quantized_data = self.quantize_tensor(tensor)?;
let current_pos = self.get_current_position()?;
let alignment = 32;
let padding = (alignment - (current_pos % alignment)) % alignment;
if padding > 0 {
let padding_bytes = vec![0u8; padding];
self.writer.write_all(&padding_bytes).map_err(|e| {
Error::model_loading(&format!("Failed to write alignment padding: {}", e))
})?;
}
self.writer
.write_all(&quantized_data)
.map_err(|e| Error::model_loading(&format!("Failed to write tensor data: {}", e)))?;
Ok(())
}
fn get_current_position(&mut self) -> Result<usize> {
Ok(0) }
fn quantize_tensor(&self, tensor: &Tensor) -> Result<Vec<u8>> {
match self.options.quantization {
GGUFQuantType::F32 => self.quantize_f32(tensor),
GGUFQuantType::F16 => self.quantize_f16(tensor),
GGUFQuantType::Q8_0 => self.quantize_q8_0(tensor),
GGUFQuantType::Q4_0 => self.quantize_q4_0(tensor),
_ => Err(Error::model_loading(&format!(
"Quantization type {} not yet implemented",
self.options.quantization.name()
))),
}
}
fn quantize_f32(&self, tensor: &Tensor) -> Result<Vec<u8>> {
let data = tensor.to_dtype(DType::F32).map_err(|e| {
Error::model_loading(&format!("Failed to convert tensor to F32: {}", e))
})?;
let _flat_data = data
.flatten_all()
.map_err(|e| Error::model_loading(&format!("Failed to flatten tensor: {}", e)))?;
let element_count = tensor.elem_count();
let byte_size = element_count * 4; Ok(vec![0u8; byte_size]) }
fn quantize_f16(&self, tensor: &Tensor) -> Result<Vec<u8>> {
let _data = tensor.to_dtype(DType::F16).map_err(|e| {
Error::model_loading(&format!("Failed to convert tensor to F16: {}", e))
})?;
let element_count = tensor.elem_count();
let byte_size = element_count * 2; Ok(vec![0u8; byte_size]) }
fn quantize_q8_0(&self, tensor: &Tensor) -> Result<Vec<u8>> {
let element_count = tensor.elem_count();
let block_size = 32; let num_blocks = (element_count + block_size - 1) / block_size;
let total_size = num_blocks * 36;
Ok(vec![0u8; total_size])
}
fn quantize_q4_0(&self, tensor: &Tensor) -> Result<Vec<u8>> {
let element_count = tensor.elem_count();
let block_size = 32; let num_blocks = (element_count + block_size - 1) / block_size;
let total_size = num_blocks * 20;
Ok(vec![0u8; total_size])
}
pub fn finalize(mut self) -> Result<()> {
self.writer
.flush()
.map_err(|e| Error::model_loading(&format!("Failed to flush GGUF file: {}", e)))?;
drop(self.writer);
Ok(())
}
}
pub struct GGUFSaver {
options: GGUFExportOptions,
}
impl GGUFSaver {
pub fn new(options: GGUFExportOptions) -> Self {
Self { options }
}
pub fn with_quantization(quantization: GGUFQuantType) -> Self {
Self {
options: GGUFExportOptions {
quantization,
..Default::default()
},
}
}
}
impl ModelSaver for GGUFSaver {
fn save_tensors(
&self,
tensors: &HashMap<String, Tensor>,
path: &Path,
save_options: &SaveOptions,
) -> Result<()> {
if let Some(callback) = &save_options.progress_callback {
callback(ProgressEvent::SavingFile {
file: path.to_path_buf(),
format: self.format_name().to_string(),
});
}
let mut writer = GGUFWriter::new(path, self.options.clone())?;
if let Some(callback) = &save_options.progress_callback {
callback(ProgressEvent::SavingTensors {
count: tensors.len(),
format: self.format_name().to_string(),
});
}
writer.write_header(tensors)?;
writer.write_tensor_infos(tensors)?;
writer.write_tensors(tensors)?;
writer.finalize()?;
Ok(())
}
fn file_extension(&self) -> &str {
"gguf"
}
fn format_name(&self) -> &str {
"GGUF"
}
}
pub fn export_to_gguf(
tensors: &HashMap<String, Tensor>,
path: &Path,
options: GGUFExportOptions,
save_options: &SaveOptions,
) -> Result<()> {
let saver = GGUFSaver::new(options);
saver.save_tensors(tensors, path, save_options)
}
pub fn export_to_gguf_f16(
tensors: &HashMap<String, Tensor>,
path: &Path,
architecture: &str,
save_options: &SaveOptions,
) -> Result<()> {
let options = GGUFExportOptions::new(architecture).with_quantization(GGUFQuantType::F16);
export_to_gguf(tensors, path, options, save_options)
}
pub fn export_to_gguf_q8_0(
tensors: &HashMap<String, Tensor>,
path: &Path,
architecture: &str,
save_options: &SaveOptions,
) -> Result<()> {
let options = GGUFExportOptions::new(architecture).with_quantization(GGUFQuantType::Q8_0);
export_to_gguf(tensors, path, options, save_options)
}
pub fn export_to_gguf_q4_k_m(
tensors: &HashMap<String, Tensor>,
path: &Path,
architecture: &str,
save_options: &SaveOptions,
) -> Result<()> {
let options = GGUFExportOptions::new(architecture).with_quantization(GGUFQuantType::Q4_K_M);
export_to_gguf(tensors, path, options, save_options)
}
pub fn save_as_gguf(
model: &crate::LoadedModel,
path: &Path,
export_options: GgufExportOptions,
) -> crate::Result<()> {
let tensors = std::collections::HashMap::new();
let quantization = if let Some(ref quant) = export_options.quantization {
match quant.as_str() {
"q4_0" => GGUFQuantType::Q4_0,
"q4_1" => GGUFQuantType::Q4_1,
"q8_0" => GGUFQuantType::Q8_0,
"q4_k_m" => GGUFQuantType::Q4_K_M,
"q6_k" => GGUFQuantType::Q6_K,
"f16" => GGUFQuantType::F16,
"f32" => GGUFQuantType::F32,
_ => {
return Err(crate::Error::invalid_config(format!(
"Unsupported GGUF quantization: {}",
quant
)))
}
}
} else {
GGUFQuantType::F16 };
let mut gguf_options = GGUFExportOptions::new("unknown").with_quantization(quantization);
if export_options.preserve_metadata {
gguf_options = gguf_options.with_metadata(
"preserved",
crate::formats::gguf_export::MetadataValue::String("true".to_string()),
);
}
for (key, value) in &export_options.custom_metadata {
gguf_options = gguf_options.with_metadata(
key,
crate::formats::gguf_export::MetadataValue::String(value.clone()),
);
}
let save_options = crate::saver::SaveOptions {
progress_callback: None,
compression: None,
metadata: std::collections::HashMap::new(),
};
export_to_gguf(&tensors, path, gguf_options, &save_options)
.map_err(|e| crate::Error::model_saving(format!("GGUF export failed: {}", e)))
}
#[derive(Debug, Clone, Default)]
pub struct GgufExportOptions {
pub preserve_metadata: bool,
pub custom_metadata: HashMap<String, String>,
pub quantization: Option<String>,
pub use_mmap: bool,
}