incin-core 0.1.0

Core tensor operations and static shape checking for incin
Documentation
use crate::dist::placement::Local;
use crate::err::{Error, Result};
use crate::exec::catalog::{QuantizationAttributes, op};
use crate::exec::request::TensorHandle;
use crate::exec::{self, ExecutionContext};
use crate::nn::{StateSnapshot, VisitState};
use crate::tensor::backend::{Backend, Execute, HostInterop, SupportsDType};
use crate::tensor::dtype::{DTypeId, Q8_0};
use alloc::collections::BTreeMap;
use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec::Vec;
use core::any::type_name;
use std::fs::File;
use std::io::{BufWriter, Seek, Write};
use std::path::Path;

/// Supported GGUF quantization schemes.
#[allow(non_camel_case_types)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum QuantScheme {
    /// Full precision 32-bit floating point (GGML_TYPE_F32).
    F32,
    /// Half precision 16-bit floating point (GGML_TYPE_F16).
    F16,
    /// 8-bit symmetric quantization with 32-element blocks (GGML_TYPE_Q8_0).
    Q8_0,
    /// 4-bit symmetric quantization with 32-element blocks (GGML_TYPE_Q4_0, W4A16).
    W4A16_Q4_0,
    /// 4-bit K-quant medium quantization (GGML_TYPE_Q4_K, W4A16).
    W4A16_Q4_K_M,
}

impl QuantScheme {
    /// Returns the corresponding GGML type ID for GGUF serialization.
    pub fn ggml_type_id(&self) -> u32 {
        match self {
            Self::F32 => 0,
            Self::F16 => 1,
            Self::W4A16_Q4_0 => 2,
            Self::Q8_0 => 8,
            Self::W4A16_Q4_K_M => 12,
        }
    }

    /// Returns the GGUF file_type metadata integer.
    pub fn file_type_id(&self) -> u32 {
        match self {
            Self::F32 => 0,
            Self::F16 => 1,
            Self::W4A16_Q4_0 => 2,
            Self::Q8_0 => 7,
            Self::W4A16_Q4_K_M => 15,
        }
    }
}

/// Key-Value metadata entry for GGUF headers.
#[derive(Debug, Clone)]
pub enum GgufValue {
    /// GGUF `Uint8` metadata value.
    Uint8(u8),
    /// GGUF `Int8` metadata value.
    Int8(i8),
    /// GGUF `Uint16` metadata value.
    Uint16(u16),
    /// GGUF `Int16` metadata value.
    Int16(i16),
    /// GGUF `Uint32` metadata value.
    Uint32(u32),
    /// GGUF `Int32` metadata value.
    Int32(i32),
    /// GGUF `Float32` metadata value.
    Float32(f32),
    /// GGUF `Bool` metadata value.
    Bool(bool),
    /// GGUF string metadata value.
    Str(String),
    /// GGUF `Uint64` metadata value.
    Uint64(u64),
    /// GGUF `Int64` metadata value.
    Int64(i64),
    /// GGUF `Float64` metadata value.
    Float64(f64),
}

impl GgufValue {
    fn type_id(&self) -> u32 {
        match self {
            Self::Uint8(_) => 0,
            Self::Int8(_) => 1,
            Self::Uint16(_) => 2,
            Self::Int16(_) => 3,
            Self::Uint32(_) => 4,
            Self::Int32(_) => 5,
            Self::Float32(_) => 6,
            Self::Bool(_) => 7,
            Self::Str(_) => 8,
            Self::Uint64(_) => 10,
            Self::Int64(_) => 11,
            Self::Float64(_) => 12,
        }
    }

    fn write_binary<W: Write>(&self, w: &mut W) -> Result<()> {
        w.write_all(&self.type_id().to_le_bytes())?;
        match self {
            Self::Uint8(v) => w.write_all(&[*v])?,
            Self::Int8(v) => w.write_all(&[*v as u8])?,
            Self::Uint16(v) => w.write_all(&v.to_le_bytes())?,
            Self::Int16(v) => w.write_all(&v.to_le_bytes())?,
            Self::Uint32(v) => w.write_all(&v.to_le_bytes())?,
            Self::Int32(v) => w.write_all(&v.to_le_bytes())?,
            Self::Float32(v) => w.write_all(&v.to_le_bytes())?,
            Self::Bool(v) => w.write_all(&[*v as u8])?,
            Self::Str(s) => {
                let bytes = s.as_bytes();
                let len = u64::try_from(bytes.len())
                    .map_err(|_| Error::Msg("GGUF string is too large for the format".into()))?;
                w.write_all(&len.to_le_bytes())?;
                w.write_all(bytes)?;
            }
            Self::Uint64(v) => w.write_all(&v.to_le_bytes())?,
            Self::Int64(v) => w.write_all(&v.to_le_bytes())?,
            Self::Float64(v) => w.write_all(&v.to_le_bytes())?,
        }
        Ok(())
    }
}

/// GGUF metadata container for model architecture properties.
#[derive(Debug, Clone, Default)]
pub struct GgufMetadata {
    /// Metadata entries keyed by name.
    pub entries: BTreeMap<String, GgufValue>,
}

impl GgufMetadata {
    /// Creates metadata for one architecture.
    pub fn new(arch: &str) -> Self {
        let mut meta = Self::default();
        meta.set("general.architecture", GgufValue::Str(arch.to_string()));
        meta.set(
            "general.producer.name",
            GgufValue::Str("incin-v0.2.0".to_string()),
        );
        meta.set("general.alignment", GgufValue::Uint32(32));
        meta
    }

    /// Sets one metadata entry.
    pub fn set(&mut self, key: impl Into<String>, val: GgufValue) {
        self.entries.insert(key.into(), val);
    }
}

/// Exporter for saving `incin` modules to GGUF v3 format.
pub struct GgufExporter<
    'a,
    B: Backend + crate::tensor::backend::VariableBackend + HostInterop,
    M: VisitState<B>,
> {
    module: &'a M,
    metadata: GgufMetadata,
    quant: QuantScheme,
    _phantom: core::marker::PhantomData<B>,
}

impl<'a, B, M> GgufExporter<'a, B, M>
where
    B: Backend + crate::tensor::backend::VariableBackend + Execute<op::Quantize> + HostInterop,
    <B as Execute<op::Quantize>>::Output: Into<B::Storage<Q8_0>>,
    M: VisitState<B>,
{
    /// Creates a new exporter for the given module, auto-deriving architecture metadata.
    pub fn from_module(module: &'a M) -> Self {
        let full_name = type_name::<M>();
        let short_name = full_name
            .split("::")
            .last()
            .unwrap_or("model")
            .to_lowercase();
        let arch = if short_name.contains("llama") {
            "llama"
        } else if short_name.contains("resnet") {
            "resnet"
        } else {
            "custom"
        };

        let mut metadata = GgufMetadata::new(arch);
        metadata.set("general.name", GgufValue::Str(short_name));

        Self {
            module,
            metadata,
            quant: QuantScheme::F32,
            _phantom: core::marker::PhantomData,
        }
    }

    /// Sets a custom metadata property.
    pub fn with_metadata_entry(mut self, key: impl Into<String>, val: GgufValue) -> Self {
        self.metadata.set(key, val);
        self
    }

    /// Configures the quantization scheme for exported weights.
    pub fn with_quantization(mut self, quant: QuantScheme) -> Self {
        self.quant = quant;
        self
    }

    /// Exports the module and its weights to a `.gguf` file.
    pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()>
    where
        B: SupportsDType<f32>,
    {
        // Only F32 (passthrough) and Q8_0 (real block quantization) are
        // actually backed by a working conversion right now. Refuse the
        // rest rather than silently writing float bytes under a
        // quantized `ggml_type` header, which would produce a `.gguf`
        // file that lies about its own binary layout.
        if !matches!(self.quant, QuantScheme::F32 | QuantScheme::Q8_0) {
            return Err(Error::Msg(format!(
                "GGUF export: quantization scheme {:?} is not yet implemented (only F32 and Q8_0 are supported)",
                self.quant
            )));
        }

        let mut file = BufWriter::new(File::create(path)?);

        // 1. Magic bytes: "GGUF" = 0x46554747
        file.write_all(b"GGUF")?;
        // Version: 3
        file.write_all(&3u32.to_le_bytes())?;

        let snapshot: StateSnapshot = crate::nn::collect_state::<B, _>(self.module)?;
        let tensor_count = u64::try_from(snapshot.len())
            .map_err(|_| Error::Msg("tensor count is too large for the GGUF format".into()))?;

        // Auto-set file_type metadata
        let mut final_metadata = self.metadata.clone();
        final_metadata.set(
            "general.file_type",
            GgufValue::Uint32(self.quant.file_type_id()),
        );
        let metadata_count = u64::try_from(final_metadata.entries.len())
            .map_err(|_| Error::Msg("metadata count is too large for the GGUF format".into()))?;

        file.write_all(&tensor_count.to_le_bytes())?;
        file.write_all(&metadata_count.to_le_bytes())?;

        // 2. Write KV metadata entries
        for (key, val) in &final_metadata.entries {
            let key_bytes = key.as_bytes();
            let key_len = u64::try_from(key_bytes.len())
                .map_err(|_| Error::Msg("GGUF metadata key is too large".into()))?;
            file.write_all(&key_len.to_le_bytes())?;
            file.write_all(key_bytes)?;
            val.write_binary(&mut file)?;
        }

        // 3. Collect tensor information table & payloads
        let mut payload_bytes: Vec<u8> = Vec::new();
        let mut tensor_headers = Vec::new();
        let alignment = 32usize;

        for (name, value) in snapshot.iter() {
            let shape = value.shape().dims();
            if value.dtype().builtin_id() != Some(DTypeId::F32) {
                return Err(Error::Msg(format!(
                    "GGUF export currently requires F32 state, got {} for {}",
                    value.dtype().name(),
                    name
                )));
            }
            let numel = crate::shapes::ShapeBuf::from_slice(shape)
                .checked_numel(crate::shapes::error::OperationKind::Storage)?;

            // Q8_0 quantizes in blocks of 32 elements; tensors that don't
            // divide evenly (e.g. 1D biases/norm weights) are kept at F32,
            // matching how llama.cpp itself only quantizes eligible weight
            // tensors and leaves the rest at full precision.
            let can_quantize =
                self.quant == QuantScheme::Q8_0 && numel > 0 && numel.is_multiple_of(32);

            let (bytes, ggml_type) = if can_quantize {
                let storage = B::from_bytes::<f32>(
                    value.bytes(),
                    shape,
                    DTypeId::F32.descriptor(),
                    &crate::tensor::device::DeviceId::cpu(),
                )?;
                let input = TensorHandle::from_storage::<B, f32, Local>(&storage);
                let context = ExecutionContext::from_scope(B::default());
                let quantized = exec::dispatch::execute::<op::Quantize, B>(
                    &context,
                    QuantizationAttributes {
                        dtype: DTypeId::Q8_0.descriptor(),
                    },
                    &[input],
                )?
                .into();
                (
                    B::to_bytes::<Q8_0>(&quantized)?,
                    QuantScheme::Q8_0.ggml_type_id(),
                )
            } else {
                (value.bytes().to_vec(), QuantScheme::F32.ggml_type_id())
            };
            let n_dims = u32::try_from(shape.len())
                .map_err(|_| Error::Msg("tensor rank is too large for GGUF".into()))?;

            // GGUF stores dimensions in reverse (row-major contiguous first)
            let mut gguf_shape: Vec<u64> = shape
                .iter()
                .rev()
                .map(|&dimension| {
                    u64::try_from(dimension)
                        .map_err(|_| Error::Msg("tensor dimension is too large for GGUF".into()))
                })
                .collect::<Result<_>>()?;
            if gguf_shape.is_empty() {
                gguf_shape.push(1);
            }

            // Pad current payload to 32-byte alignment
            let padding = (alignment - (payload_bytes.len() % alignment)) % alignment;
            payload_bytes.extend(core::iter::repeat_n(0u8, padding));
            let data_offset = u64::try_from(payload_bytes.len())
                .map_err(|_| Error::Msg("GGUF payload offset exceeds the format".into()))?;

            payload_bytes.extend(bytes);

            tensor_headers.push((name, n_dims, gguf_shape, ggml_type, data_offset));
        }

        // Write Tensor Information Table
        for (name, n_dims, shape, ggml_type, offset) in tensor_headers {
            let name_bytes = name.as_str().as_bytes();
            let name_len = u64::try_from(name_bytes.len())
                .map_err(|_| Error::Msg("GGUF tensor name is too large".into()))?;
            file.write_all(&name_len.to_le_bytes())?;
            file.write_all(name_bytes)?;
            file.write_all(&n_dims.to_le_bytes())?;
            for dim in shape {
                file.write_all(&dim.to_le_bytes())?;
            }
            file.write_all(&ggml_type.to_le_bytes())?;
            file.write_all(&offset.to_le_bytes())?;
        }

        // Write 32-byte alignment padding before binary payload
        let current_pos = file.stream_position()?;
        let current_pos = usize::try_from(current_pos)
            .map_err(|_| Error::Msg("GGUF header position does not fit this platform".into()))?;
        let header_padding = (alignment - (current_pos % alignment)) % alignment;
        file.write_all(&vec![0u8; header_padding])?;

        // 4. Write Tensor Binary Payload
        file.write_all(&payload_bytes)?;
        file.flush()?;

        Ok(())
    }
}