use crate::dist::placement::Local;
use crate::err::{Error, Result};
use crate::exec::catalog::{QuantizationAttributes, op};
use crate::exec::request::TensorHandle;
use crate::exec::{self, ExecutionContext};
use crate::nn::{StateSnapshot, VisitState};
use crate::tensor::backend::{Backend, Execute, HostInterop, SupportsDType};
use crate::tensor::dtype::{DTypeId, Q8_0};
use alloc::collections::BTreeMap;
use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec::Vec;
use core::any::type_name;
use std::fs::File;
use std::io::{BufWriter, Seek, Write};
use std::path::Path;
#[allow(non_camel_case_types)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum QuantScheme {
F32,
F16,
Q8_0,
W4A16_Q4_0,
W4A16_Q4_K_M,
}
impl QuantScheme {
pub fn ggml_type_id(&self) -> u32 {
match self {
Self::F32 => 0,
Self::F16 => 1,
Self::W4A16_Q4_0 => 2,
Self::Q8_0 => 8,
Self::W4A16_Q4_K_M => 12,
}
}
pub fn file_type_id(&self) -> u32 {
match self {
Self::F32 => 0,
Self::F16 => 1,
Self::W4A16_Q4_0 => 2,
Self::Q8_0 => 7,
Self::W4A16_Q4_K_M => 15,
}
}
}
#[derive(Debug, Clone)]
pub enum GgufValue {
Uint8(u8),
Int8(i8),
Uint16(u16),
Int16(i16),
Uint32(u32),
Int32(i32),
Float32(f32),
Bool(bool),
Str(String),
Uint64(u64),
Int64(i64),
Float64(f64),
}
impl GgufValue {
fn type_id(&self) -> u32 {
match self {
Self::Uint8(_) => 0,
Self::Int8(_) => 1,
Self::Uint16(_) => 2,
Self::Int16(_) => 3,
Self::Uint32(_) => 4,
Self::Int32(_) => 5,
Self::Float32(_) => 6,
Self::Bool(_) => 7,
Self::Str(_) => 8,
Self::Uint64(_) => 10,
Self::Int64(_) => 11,
Self::Float64(_) => 12,
}
}
fn write_binary<W: Write>(&self, w: &mut W) -> Result<()> {
w.write_all(&self.type_id().to_le_bytes())?;
match self {
Self::Uint8(v) => w.write_all(&[*v])?,
Self::Int8(v) => w.write_all(&[*v as u8])?,
Self::Uint16(v) => w.write_all(&v.to_le_bytes())?,
Self::Int16(v) => w.write_all(&v.to_le_bytes())?,
Self::Uint32(v) => w.write_all(&v.to_le_bytes())?,
Self::Int32(v) => w.write_all(&v.to_le_bytes())?,
Self::Float32(v) => w.write_all(&v.to_le_bytes())?,
Self::Bool(v) => w.write_all(&[*v as u8])?,
Self::Str(s) => {
let bytes = s.as_bytes();
let len = u64::try_from(bytes.len())
.map_err(|_| Error::Msg("GGUF string is too large for the format".into()))?;
w.write_all(&len.to_le_bytes())?;
w.write_all(bytes)?;
}
Self::Uint64(v) => w.write_all(&v.to_le_bytes())?,
Self::Int64(v) => w.write_all(&v.to_le_bytes())?,
Self::Float64(v) => w.write_all(&v.to_le_bytes())?,
}
Ok(())
}
}
#[derive(Debug, Clone, Default)]
pub struct GgufMetadata {
pub entries: BTreeMap<String, GgufValue>,
}
impl GgufMetadata {
pub fn new(arch: &str) -> Self {
let mut meta = Self::default();
meta.set("general.architecture", GgufValue::Str(arch.to_string()));
meta.set(
"general.producer.name",
GgufValue::Str("incin-v0.2.0".to_string()),
);
meta.set("general.alignment", GgufValue::Uint32(32));
meta
}
pub fn set(&mut self, key: impl Into<String>, val: GgufValue) {
self.entries.insert(key.into(), val);
}
}
pub struct GgufExporter<
'a,
B: Backend + crate::tensor::backend::VariableBackend + HostInterop,
M: VisitState<B>,
> {
module: &'a M,
metadata: GgufMetadata,
quant: QuantScheme,
_phantom: core::marker::PhantomData<B>,
}
impl<'a, B, M> GgufExporter<'a, B, M>
where
B: Backend + crate::tensor::backend::VariableBackend + Execute<op::Quantize> + HostInterop,
<B as Execute<op::Quantize>>::Output: Into<B::Storage<Q8_0>>,
M: VisitState<B>,
{
pub fn from_module(module: &'a M) -> Self {
let full_name = type_name::<M>();
let short_name = full_name
.split("::")
.last()
.unwrap_or("model")
.to_lowercase();
let arch = if short_name.contains("llama") {
"llama"
} else if short_name.contains("resnet") {
"resnet"
} else {
"custom"
};
let mut metadata = GgufMetadata::new(arch);
metadata.set("general.name", GgufValue::Str(short_name));
Self {
module,
metadata,
quant: QuantScheme::F32,
_phantom: core::marker::PhantomData,
}
}
pub fn with_metadata_entry(mut self, key: impl Into<String>, val: GgufValue) -> Self {
self.metadata.set(key, val);
self
}
pub fn with_quantization(mut self, quant: QuantScheme) -> Self {
self.quant = quant;
self
}
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()>
where
B: SupportsDType<f32>,
{
if !matches!(self.quant, QuantScheme::F32 | QuantScheme::Q8_0) {
return Err(Error::Msg(format!(
"GGUF export: quantization scheme {:?} is not yet implemented (only F32 and Q8_0 are supported)",
self.quant
)));
}
let mut file = BufWriter::new(File::create(path)?);
file.write_all(b"GGUF")?;
file.write_all(&3u32.to_le_bytes())?;
let snapshot: StateSnapshot = crate::nn::collect_state::<B, _>(self.module)?;
let tensor_count = u64::try_from(snapshot.len())
.map_err(|_| Error::Msg("tensor count is too large for the GGUF format".into()))?;
let mut final_metadata = self.metadata.clone();
final_metadata.set(
"general.file_type",
GgufValue::Uint32(self.quant.file_type_id()),
);
let metadata_count = u64::try_from(final_metadata.entries.len())
.map_err(|_| Error::Msg("metadata count is too large for the GGUF format".into()))?;
file.write_all(&tensor_count.to_le_bytes())?;
file.write_all(&metadata_count.to_le_bytes())?;
for (key, val) in &final_metadata.entries {
let key_bytes = key.as_bytes();
let key_len = u64::try_from(key_bytes.len())
.map_err(|_| Error::Msg("GGUF metadata key is too large".into()))?;
file.write_all(&key_len.to_le_bytes())?;
file.write_all(key_bytes)?;
val.write_binary(&mut file)?;
}
let mut payload_bytes: Vec<u8> = Vec::new();
let mut tensor_headers = Vec::new();
let alignment = 32usize;
for (name, value) in snapshot.iter() {
let shape = value.shape().dims();
if value.dtype().builtin_id() != Some(DTypeId::F32) {
return Err(Error::Msg(format!(
"GGUF export currently requires F32 state, got {} for {}",
value.dtype().name(),
name
)));
}
let numel = crate::shapes::ShapeBuf::from_slice(shape)
.checked_numel(crate::shapes::error::OperationKind::Storage)?;
let can_quantize =
self.quant == QuantScheme::Q8_0 && numel > 0 && numel.is_multiple_of(32);
let (bytes, ggml_type) = if can_quantize {
let storage = B::from_bytes::<f32>(
value.bytes(),
shape,
DTypeId::F32.descriptor(),
&crate::tensor::device::DeviceId::cpu(),
)?;
let input = TensorHandle::from_storage::<B, f32, Local>(&storage);
let context = ExecutionContext::from_scope(B::default());
let quantized = exec::dispatch::execute::<op::Quantize, B>(
&context,
QuantizationAttributes {
dtype: DTypeId::Q8_0.descriptor(),
},
&[input],
)?
.into();
(
B::to_bytes::<Q8_0>(&quantized)?,
QuantScheme::Q8_0.ggml_type_id(),
)
} else {
(value.bytes().to_vec(), QuantScheme::F32.ggml_type_id())
};
let n_dims = u32::try_from(shape.len())
.map_err(|_| Error::Msg("tensor rank is too large for GGUF".into()))?;
let mut gguf_shape: Vec<u64> = shape
.iter()
.rev()
.map(|&dimension| {
u64::try_from(dimension)
.map_err(|_| Error::Msg("tensor dimension is too large for GGUF".into()))
})
.collect::<Result<_>>()?;
if gguf_shape.is_empty() {
gguf_shape.push(1);
}
let padding = (alignment - (payload_bytes.len() % alignment)) % alignment;
payload_bytes.extend(core::iter::repeat_n(0u8, padding));
let data_offset = u64::try_from(payload_bytes.len())
.map_err(|_| Error::Msg("GGUF payload offset exceeds the format".into()))?;
payload_bytes.extend(bytes);
tensor_headers.push((name, n_dims, gguf_shape, ggml_type, data_offset));
}
for (name, n_dims, shape, ggml_type, offset) in tensor_headers {
let name_bytes = name.as_str().as_bytes();
let name_len = u64::try_from(name_bytes.len())
.map_err(|_| Error::Msg("GGUF tensor name is too large".into()))?;
file.write_all(&name_len.to_le_bytes())?;
file.write_all(name_bytes)?;
file.write_all(&n_dims.to_le_bytes())?;
for dim in shape {
file.write_all(&dim.to_le_bytes())?;
}
file.write_all(&ggml_type.to_le_bytes())?;
file.write_all(&offset.to_le_bytes())?;
}
let current_pos = file.stream_position()?;
let current_pos = usize::try_from(current_pos)
.map_err(|_| Error::Msg("GGUF header position does not fit this platform".into()))?;
let header_padding = (alignment - (current_pos % alignment)) % alignment;
file.write_all(&vec![0u8; header_padding])?;
file.write_all(&payload_bytes)?;
file.flush()?;
Ok(())
}
}