pub mod avx2;
mod blocks;
mod dequant;
mod dot;
mod f16;
mod quantize;
#[cfg(test)]
mod test_support;
#[cfg(test)]
mod tests;
pub use blocks::{BlockQ8_0, BlockQ8K};
pub use dequant::{
dequantize_row_q4_k, dequantize_row_q5_k, dequantize_row_q6_k, dequantize_row_q8_0,
};
pub use dot::{vec_dot_q4_k_q8_k, vec_dot_q5_k_q8_k, vec_dot_q6_k_q8_k, vec_dot_q8_0_q8_0};
pub use f16::{f16_to_f32, f32_to_f16};
pub use quantize::{quantize_row_q8_0, quantize_row_q8_k};
pub use super::KernelError;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[allow(non_camel_case_types)]
pub enum QuantFormat {
Q4_K,
Q5_K,
Q6_K,
Q8_0,
Q8_K,
}
impl QuantFormat {
pub const fn block_weights(self) -> usize {
match self {
Self::Q8_0 => 32,
Self::Q4_K | Self::Q5_K | Self::Q6_K | Self::Q8_K => 256,
}
}
pub const fn block_bytes(self) -> usize {
match self {
Self::Q4_K => 144,
Self::Q5_K => 176,
Self::Q6_K => 210,
Self::Q8_0 => 34,
Self::Q8_K => 292,
}
}
pub fn row_bytes(self, in_dim: usize) -> Result<usize, KernelError> {
if !in_dim.is_multiple_of(self.block_weights()) {
return Err(KernelError::IndivisibleRow {
format: self,
in_dim,
block_weights: self.block_weights(),
});
}
Ok(in_dim / self.block_weights() * self.block_bytes())
}
}
const _: () = {
assert!(QuantFormat::Q4_K.block_bytes() == blocks::q4k::BYTES);
assert!(QuantFormat::Q5_K.block_bytes() == blocks::q5k::BYTES);
assert!(QuantFormat::Q6_K.block_bytes() == blocks::q6k::BYTES);
assert!(QuantFormat::Q8_0.block_bytes() == blocks::q8_0::BYTES);
};