pub mod mxfp;
pub mod nf4;
pub mod per_block_affine;
pub mod per_channel_affine;
pub mod per_tensor_affine;
pub use mxfp::{Mxfp, MXFP_CANONICAL_BLOCK_SIZE, MXFP_MAX_BLOCK_SIZE, MXFP_MIN_BLOCK_SIZE};
pub use nf4::{Nf4, NF4_LUT, NF4_MIN_BLOCK_SIZE};
pub use per_block_affine::{PerBlockAffine, PER_BLOCK_AFFINE_MIN_BLOCK_SIZE};
pub use per_channel_affine::PerChannelAffine;
pub use per_tensor_affine::PerTensorAffine;
use crate::{BufferHandle, ElementType, Error, Result};
const TIER2_MIN: u8 = 0x40;
const TIER2_MAX: u8 = 0x5F;
const RESERVED_MIN: u8 = 0x60;
const RESERVED_MAX: u8 = 0xEF;
const PRIVATE_MIN: u8 = 0xF0;
const PRIVATE_MAX: u8 = 0xFE;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[repr(u8)]
pub enum QuantizationSchemeTag {
PerTensorAffine = 0x01,
PerChannelAffine = 0x02,
PerBlockAffine = 0x03,
Nf4 = 0x04,
Mxfp = 0x05,
}
impl QuantizationSchemeTag {
#[inline]
pub fn tag(self) -> u8 {
self as u8
}
#[inline]
pub fn tier(self) -> u8 {
match self {
Self::PerTensorAffine | Self::PerChannelAffine | Self::PerBlockAffine => 1,
Self::Nf4 | Self::Mxfp => 2,
}
}
pub fn from_byte(b: u8) -> Result<Self> {
match b {
0x00 | 0xFF => Err(Error::InvalidQuantizationSchemeTag(b)),
0x01 => Ok(Self::PerTensorAffine),
0x02 => Ok(Self::PerChannelAffine),
0x03 => Ok(Self::PerBlockAffine),
0x04 => Ok(Self::Nf4),
0x05 => Ok(Self::Mxfp),
b if (0x06..=0x3F).contains(&b) => Err(Error::UnknownQuantizationSchemeTag(b)),
b if (TIER2_MIN..=TIER2_MAX).contains(&b) => {
Err(Error::UnknownQuantizationSchemeTag(b))
}
b if (RESERVED_MIN..=RESERVED_MAX).contains(&b) => {
Err(Error::ReservedQuantizationSchemeTag(b))
}
b if (PRIVATE_MIN..=PRIVATE_MAX).contains(&b) => {
Err(Error::PrivateQuantizationSchemeTag(b))
}
_ => Err(Error::UnknownQuantizationSchemeTag(b)),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct QuantizationHeader {
pub scheme_tag: QuantizationSchemeTag,
pub scheme_version: u8,
pub flags: u16,
}
impl QuantizationHeader {
pub(crate) const LEN: usize = 4;
pub(crate) fn parse(bytes: &[u8], supported_version: u8) -> Result<Self> {
if bytes.len() < Self::LEN {
return Err(Error::QuantizationDescriptorTooShort {
found: bytes.len(),
needed: Self::LEN,
});
}
let scheme_tag_byte = bytes[0];
let scheme_version = bytes[1];
let flags = u16::from_le_bytes([bytes[2], bytes[3]]);
let scheme_tag = QuantizationSchemeTag::from_byte(scheme_tag_byte)?;
if scheme_version > supported_version {
return Err(Error::UnsupportedSchemeVersion {
tag: scheme_tag_byte,
version: scheme_version,
supported: supported_version,
});
}
Ok(Self {
scheme_tag,
scheme_version,
flags,
})
}
pub(crate) fn write(&self, out: &mut [u8]) {
out[0] = self.scheme_tag.tag();
out[1] = self.scheme_version;
let flags_le = self.flags.to_le_bytes();
out[2] = flags_le[0];
out[3] = flags_le[1];
}
}
#[derive(Debug, Clone, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum QuantizationDescriptor {
PerTensorAffine(PerTensorAffine),
PerChannelAffine(PerChannelAffine),
PerBlockAffine(PerBlockAffine),
Nf4(Nf4),
Mxfp(Mxfp),
}
impl QuantizationDescriptor {
pub fn scheme_tag(&self) -> QuantizationSchemeTag {
match self {
Self::PerTensorAffine(_) => QuantizationSchemeTag::PerTensorAffine,
Self::PerChannelAffine(_) => QuantizationSchemeTag::PerChannelAffine,
Self::PerBlockAffine(_) => QuantizationSchemeTag::PerBlockAffine,
Self::Nf4(_) => QuantizationSchemeTag::Nf4,
Self::Mxfp(_) => QuantizationSchemeTag::Mxfp,
}
}
pub fn encoded_len(&self) -> usize {
match self {
Self::PerTensorAffine(_) => per_tensor_affine::ENCODED_LEN,
Self::PerChannelAffine(_) => per_channel_affine::ENCODED_LEN,
Self::PerBlockAffine(_) => per_block_affine::ENCODED_LEN,
Self::Nf4(_) => nf4::ENCODED_LEN,
Self::Mxfp(_) => mxfp::ENCODED_LEN,
}
}
pub fn encode_into(&self, out: &mut [u8]) -> Result<usize> {
let len = self.encoded_len();
if out.len() < len {
return Err(Error::InvalidQuantization(format!(
"output buffer too short: need {len} bytes, have {}",
out.len()
)));
}
let (scheme_version, flags) = self.header_version_and_flags();
let header = QuantizationHeader {
scheme_tag: self.scheme_tag(),
scheme_version,
flags,
};
header.write(out);
match self {
Self::PerTensorAffine(inner) => inner.encode_payload(out),
Self::PerChannelAffine(inner) => inner.encode_payload(out),
Self::PerBlockAffine(inner) => inner.encode_payload(out),
Self::Nf4(inner) => inner.encode_payload(out),
Self::Mxfp(inner) => inner.encode_payload(out),
}
Ok(len)
}
pub fn encode_to_vec(&self) -> Vec<u8> {
let len = self.encoded_len();
let mut buf = vec![0u8; len];
self.encode_into(&mut buf)
.expect("encode_into into exact-sized buffer is infallible");
buf
}
pub fn decode(bytes: &[u8]) -> Result<(Self, usize)> {
if bytes.is_empty() {
return Err(Error::QuantizationDescriptorTooShort {
found: 0,
needed: QuantizationHeader::LEN,
});
}
let tag_byte = bytes[0];
let tag = QuantizationSchemeTag::from_byte(tag_byte)?;
let supported_version = supported_version_for(tag);
let header = QuantizationHeader::parse(bytes, supported_version)?;
let descriptor = match tag {
QuantizationSchemeTag::PerTensorAffine => Self::PerTensorAffine(
per_tensor_affine::PerTensorAffine::decode_payload(header.flags, bytes)?,
),
QuantizationSchemeTag::PerChannelAffine => Self::PerChannelAffine(
per_channel_affine::PerChannelAffine::decode_payload(header.flags, bytes)?,
),
QuantizationSchemeTag::PerBlockAffine => Self::PerBlockAffine(
per_block_affine::PerBlockAffine::decode_payload(header.flags, bytes)?,
),
QuantizationSchemeTag::Nf4 => Self::Nf4(nf4::Nf4::decode_payload(header.flags, bytes)?),
QuantizationSchemeTag::Mxfp => {
Self::Mxfp(mxfp::Mxfp::decode_payload(header.flags, bytes)?)
}
};
let consumed = descriptor.encoded_len();
Ok((descriptor, consumed))
}
pub fn valid_storage_types(&self) -> &'static [ElementType] {
match self {
Self::PerTensorAffine(_) => PerTensorAffine::valid_storage_types(),
Self::PerChannelAffine(_) => PerChannelAffine::valid_storage_types(),
Self::PerBlockAffine(_) => PerBlockAffine::valid_storage_types(),
Self::Nf4(_) => Nf4::valid_storage_types(),
Self::Mxfp(_) => Mxfp::valid_storage_types(),
}
}
pub fn is_valid_storage_type(&self, ty: ElementType) -> bool {
self.valid_storage_types().contains(&ty)
}
fn header_version_and_flags(&self) -> (u8, u16) {
match self {
Self::PerTensorAffine(_) => (per_tensor_affine::SUPPORTED_VERSION, 0),
Self::PerChannelAffine(inner) => (per_channel_affine::SUPPORTED_VERSION, inner.flags()),
Self::PerBlockAffine(inner) => (per_block_affine::SUPPORTED_VERSION, inner.flags()),
Self::Nf4(_) => (nf4::SUPPORTED_VERSION, 0),
Self::Mxfp(_) => (mxfp::SUPPORTED_VERSION, 0),
}
}
}
fn supported_version_for(tag: QuantizationSchemeTag) -> u8 {
match tag {
QuantizationSchemeTag::PerTensorAffine => per_tensor_affine::SUPPORTED_VERSION,
QuantizationSchemeTag::PerChannelAffine => per_channel_affine::SUPPORTED_VERSION,
QuantizationSchemeTag::PerBlockAffine => per_block_affine::SUPPORTED_VERSION,
QuantizationSchemeTag::Nf4 => nf4::SUPPORTED_VERSION,
QuantizationSchemeTag::Mxfp => mxfp::SUPPORTED_VERSION,
}
}
pub fn validate_axis(axis: u32, rank: u32) -> Result<()> {
if axis >= rank {
return Err(Error::QuantizationAxisOutOfBounds { axis, rank });
}
Ok(())
}
pub fn validate_buffer_placement(
desc: &QuantizationDescriptor,
buffers: &[BufferHandle],
data_buffer_index: u32,
) -> Result<()> {
let data_device = buffers
.get(data_buffer_index as usize)
.ok_or(Error::QuantizationBufferIndexOutOfRange {
index: data_buffer_index,
buffer_count: buffers.len() as u32,
})?
.device_tag();
for param_index in param_buffer_indices(desc) {
if param_index as usize >= buffers.len() {
return Err(Error::QuantizationBufferIndexOutOfRange {
index: param_index,
buffer_count: buffers.len() as u32,
});
}
if param_index == data_buffer_index {
return Err(Error::QuantizationBufferAliasesData { index: param_index });
}
let param_device = buffers[param_index as usize].device_tag();
if param_device != data_device {
return Err(Error::DeviceTagMismatch {
expected: data_device.to_byte(),
found: param_device.to_byte(),
});
}
}
Ok(())
}
fn param_buffer_indices(desc: &QuantizationDescriptor) -> impl Iterator<Item = u32> {
let mut indices = [u32::MAX; 2];
let mut count = 0usize;
match desc {
QuantizationDescriptor::PerTensorAffine(_) => {
}
QuantizationDescriptor::PerChannelAffine(inner) => {
indices[count] = inner.scale_buffer_index();
count += 1;
if let Some(zp) = inner.zero_point_buffer_index() {
indices[count] = zp;
count += 1;
}
}
QuantizationDescriptor::PerBlockAffine(inner) => {
indices[count] = inner.scale_buffer_index();
count += 1;
if let Some(zp) = inner.zero_point_buffer_index() {
indices[count] = zp;
count += 1;
}
}
QuantizationDescriptor::Nf4(inner) => {
indices[count] = inner.scale_buffer_index();
count += 1;
}
QuantizationDescriptor::Mxfp(inner) => {
indices[count] = inner.scale_buffer_index();
count += 1;
}
}
indices.into_iter().take(count)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{BufferHandle, DeviceTag, ElementType, Error, SyncMode, MIN_BUFFER_ALIGNMENT};
#[test]
fn scheme_tag_from_byte_tier1_variants() {
assert_eq!(
QuantizationSchemeTag::from_byte(0x01).unwrap(),
QuantizationSchemeTag::PerTensorAffine
);
assert_eq!(
QuantizationSchemeTag::from_byte(0x02).unwrap(),
QuantizationSchemeTag::PerChannelAffine
);
assert_eq!(
QuantizationSchemeTag::from_byte(0x03).unwrap(),
QuantizationSchemeTag::PerBlockAffine
);
assert_eq!(
QuantizationSchemeTag::from_byte(0x04).unwrap(),
QuantizationSchemeTag::Nf4
);
assert_eq!(
QuantizationSchemeTag::from_byte(0x05).unwrap(),
QuantizationSchemeTag::Mxfp
);
}
#[test]
fn scheme_tag_from_byte_permanently_invalid_sentinels() {
assert!(matches!(
QuantizationSchemeTag::from_byte(0x00),
Err(Error::InvalidQuantizationSchemeTag(0x00))
));
assert!(matches!(
QuantizationSchemeTag::from_byte(0xFF),
Err(Error::InvalidQuantizationSchemeTag(0xFF))
));
}
#[test]
fn scheme_tag_from_byte_reserved_range() {
assert!(matches!(
QuantizationSchemeTag::from_byte(0x60),
Err(Error::ReservedQuantizationSchemeTag(0x60))
));
assert!(matches!(
QuantizationSchemeTag::from_byte(0xEF),
Err(Error::ReservedQuantizationSchemeTag(0xEF))
));
}
#[test]
fn scheme_tag_from_byte_private_range() {
assert!(matches!(
QuantizationSchemeTag::from_byte(0xF0),
Err(Error::PrivateQuantizationSchemeTag(0xF0))
));
assert!(matches!(
QuantizationSchemeTag::from_byte(0xFE),
Err(Error::PrivateQuantizationSchemeTag(0xFE))
));
}
#[test]
fn scheme_tag_from_byte_unknown_unallocated_tier1() {
assert!(matches!(
QuantizationSchemeTag::from_byte(0x06),
Err(Error::UnknownQuantizationSchemeTag(0x06))
));
}
#[test]
fn scheme_tag_tag_returns_raw_byte() {
assert_eq!(QuantizationSchemeTag::PerTensorAffine.tag(), 0x01);
assert_eq!(QuantizationSchemeTag::PerChannelAffine.tag(), 0x02);
assert_eq!(QuantizationSchemeTag::PerBlockAffine.tag(), 0x03);
assert_eq!(QuantizationSchemeTag::Nf4.tag(), 0x04);
assert_eq!(QuantizationSchemeTag::Mxfp.tag(), 0x05);
}
#[test]
fn scheme_tag_tier_correct_per_spec_table() {
assert_eq!(QuantizationSchemeTag::PerTensorAffine.tier(), 1);
assert_eq!(QuantizationSchemeTag::PerChannelAffine.tier(), 1);
assert_eq!(QuantizationSchemeTag::PerBlockAffine.tier(), 1);
assert_eq!(QuantizationSchemeTag::Nf4.tier(), 2);
assert_eq!(QuantizationSchemeTag::Mxfp.tier(), 2);
}
#[test]
fn encoded_len_per_tensor_affine_is_16() {
let desc = QuantizationDescriptor::PerTensorAffine(PerTensorAffine::new(1.0, 0).unwrap());
assert_eq!(desc.encoded_len(), 16);
}
#[test]
fn encoded_len_per_channel_affine_is_20() {
let desc = QuantizationDescriptor::PerChannelAffine(
PerChannelAffine::new_symmetric(0, 1).unwrap(),
);
assert_eq!(desc.encoded_len(), 20);
}
#[test]
fn encoded_len_per_block_affine_is_24() {
let desc = QuantizationDescriptor::PerBlockAffine(
PerBlockAffine::new_symmetric(0, 32, 1, ElementType::Float32).unwrap(),
);
assert_eq!(desc.encoded_len(), 24);
}
#[test]
fn encoded_len_nf4_is_16() {
let desc = QuantizationDescriptor::Nf4(Nf4::new(0, 64, 1).unwrap());
assert_eq!(desc.encoded_len(), 16);
}
#[test]
fn encoded_len_mxfp_is_16() {
let desc = QuantizationDescriptor::Mxfp(Mxfp::new(0, 32, 1).unwrap());
assert_eq!(desc.encoded_len(), 16);
}
fn round_trip(desc: &QuantizationDescriptor) {
let len = desc.encoded_len();
let mut buf = vec![0u8; len];
let written = desc.encode_into(&mut buf).unwrap();
assert_eq!(written, len, "encode_into must return encoded_len");
let (decoded, consumed) = QuantizationDescriptor::decode(&buf).unwrap();
assert_eq!(
consumed, len,
"decode must consume exactly encoded_len bytes"
);
assert_eq!(&decoded, desc, "decoded descriptor must equal original");
}
#[test]
fn round_trip_per_tensor_affine() {
round_trip(&QuantizationDescriptor::PerTensorAffine(
PerTensorAffine::new(0.5, 128).unwrap(),
));
}
#[test]
fn round_trip_per_channel_affine_symmetric() {
round_trip(&QuantizationDescriptor::PerChannelAffine(
PerChannelAffine::new_symmetric(0, 1).unwrap(),
));
}
#[test]
fn round_trip_per_channel_affine_asymmetric() {
round_trip(&QuantizationDescriptor::PerChannelAffine(
PerChannelAffine::new_asymmetric(2, 1, 3).unwrap(),
));
}
#[test]
fn round_trip_per_block_affine_symmetric() {
round_trip(&QuantizationDescriptor::PerBlockAffine(
PerBlockAffine::new_symmetric(0, 32, 1, ElementType::Float32).unwrap(),
));
}
#[test]
fn round_trip_per_block_affine_asymmetric() {
round_trip(&QuantizationDescriptor::PerBlockAffine(
PerBlockAffine::new_asymmetric(1, 64, 2, 3, ElementType::Float16).unwrap(),
));
}
#[test]
fn round_trip_nf4() {
round_trip(&QuantizationDescriptor::Nf4(Nf4::new(0, 64, 1).unwrap()));
}
#[test]
fn round_trip_mxfp() {
round_trip(&QuantizationDescriptor::Mxfp(Mxfp::new(0, 32, 1).unwrap()));
}
#[test]
fn encode_to_vec_matches_encode_into() {
let desc = QuantizationDescriptor::PerTensorAffine(PerTensorAffine::new(0.25, -7).unwrap());
let vec_bytes = desc.encode_to_vec();
let mut into_bytes = vec![0u8; desc.encoded_len()];
desc.encode_into(&mut into_bytes).unwrap();
assert_eq!(vec_bytes, into_bytes);
}
#[test]
fn validate_axis_within_rank_is_ok() {
assert!(validate_axis(0, 3).is_ok());
assert!(validate_axis(2, 3).is_ok());
}
#[test]
fn validate_axis_equal_to_rank_is_out_of_bounds() {
assert!(matches!(
validate_axis(3, 3),
Err(Error::QuantizationAxisOutOfBounds { axis: 3, rank: 3 })
));
}
#[test]
fn validate_axis_greater_than_rank_is_out_of_bounds() {
assert!(matches!(
validate_axis(10, 3),
Err(Error::QuantizationAxisOutOfBounds { axis: 10, rank: 3 })
));
}
#[test]
fn validate_axis_rank_zero_always_errs() {
assert!(matches!(
validate_axis(0, 0),
Err(Error::QuantizationAxisOutOfBounds { axis: 0, rank: 0 })
));
}
fn make_cpu_buffer() -> BufferHandle {
BufferHandle::new(
1024,
MIN_BUFFER_ALIGNMENT,
DeviceTag::Cpu,
SyncMode::ProducerSynced,
)
.unwrap()
}
#[test]
fn validate_buffer_placement_nf4_scale_only_valid() {
let buffers = [make_cpu_buffer(), make_cpu_buffer()];
let desc = QuantizationDescriptor::Nf4(Nf4::new(0, 64, 1).unwrap());
assert!(validate_buffer_placement(&desc, &buffers, 0).is_ok());
}
#[test]
fn validate_buffer_placement_mxfp_scale_only_valid() {
let buffers = [make_cpu_buffer(), make_cpu_buffer()];
let desc = QuantizationDescriptor::Mxfp(Mxfp::new(0, 32, 1).unwrap());
assert!(validate_buffer_placement(&desc, &buffers, 0).is_ok());
}
#[test]
fn validate_buffer_placement_per_block_asymmetric_scale_and_zp_valid() {
let buffers = [make_cpu_buffer(), make_cpu_buffer(), make_cpu_buffer()];
let desc = QuantizationDescriptor::PerBlockAffine(
PerBlockAffine::new_asymmetric(0, 32, 1, 2, ElementType::Float32).unwrap(),
);
assert!(validate_buffer_placement(&desc, &buffers, 0).is_ok());
}
#[test]
fn validate_buffer_placement_aliases_data_is_err() {
let buffers = [make_cpu_buffer(), make_cpu_buffer()];
let desc = QuantizationDescriptor::Nf4(Nf4::new(0, 64, 0).unwrap());
assert!(matches!(
validate_buffer_placement(&desc, &buffers, 0),
Err(Error::QuantizationBufferAliasesData { index: 0 })
));
}
#[test]
fn validate_buffer_placement_index_out_of_range_is_err() {
let buffers = [make_cpu_buffer()];
let desc = QuantizationDescriptor::Nf4(Nf4::new(0, 64, 5).unwrap());
assert!(matches!(
validate_buffer_placement(&desc, &buffers, 0),
Err(Error::QuantizationBufferIndexOutOfRange { .. })
));
}
#[test]
fn validate_buffer_placement_device_tag_mismatch_is_err() {
let cpu_buf = BufferHandle::new(
1024,
MIN_BUFFER_ALIGNMENT,
DeviceTag::Cpu,
SyncMode::ProducerSynced,
)
.unwrap();
let cuda_buf = BufferHandle::new(
1024,
MIN_BUFFER_ALIGNMENT,
DeviceTag::Cuda,
SyncMode::ProducerSynced,
)
.unwrap();
let buffers = [cpu_buf, cuda_buf];
let desc = QuantizationDescriptor::Nf4(Nf4::new(0, 64, 1).unwrap());
assert!(matches!(
validate_buffer_placement(&desc, &buffers, 0),
Err(Error::DeviceTagMismatch { .. })
));
}
}