use crate::weight_quant::{
absmax_scale, decode_scalar, encode_scalar, read_packed_public, write_packed_public,
WeightQuantization,
};
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum KvQuantization {
Int8,
Nf4,
#[serde(rename = "fp8_e4m3")]
Fp8E4M3,
#[serde(rename = "fp8_e5m2")]
Fp8E5M2,
}
impl KvQuantization {
pub fn as_str(self) -> &'static str {
match self {
Self::Int8 => "int8",
Self::Nf4 => "nf4",
Self::Fp8E4M3 => "fp8_e4m3",
Self::Fp8E5M2 => "fp8_e5m2",
}
}
pub fn bits(self) -> u32 {
match self {
Self::Int8 | Self::Fp8E4M3 | Self::Fp8E5M2 => 8,
Self::Nf4 => 4,
}
}
pub fn parse(value: &str) -> Result<Self, String> {
match value.trim().to_ascii_lowercase().as_str() {
"int8" | "i8" | "8" => Ok(Self::Int8),
"nf4" => Ok(Self::Nf4),
"fp8" | "fp8_e4m3" | "e4m3" => Ok(Self::Fp8E4M3),
"fp8_e5m2" | "e5m2" => Ok(Self::Fp8E5M2),
other => Err(format!(
"unknown KV quantization '{other}'; expected int8|nf4|fp8_e4m3|fp8_e5m2"
)),
}
}
pub fn weight_format(self) -> WeightQuantization {
match self {
Self::Int8 => WeightQuantization::Int8,
Self::Nf4 => WeightQuantization::Nf4,
Self::Fp8E4M3 => WeightQuantization::Fp8E4M3,
Self::Fp8E5M2 => WeightQuantization::Fp8E5M2,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum KvGranularity {
PerLayer,
PerHead,
}
impl KvGranularity {
pub fn as_str(self) -> &'static str {
match self {
Self::PerLayer => "layer",
Self::PerHead => "head",
}
}
pub fn parse(value: &str) -> Result<Self, String> {
match value.trim().to_ascii_lowercase().as_str() {
"layer" | "per-layer" | "per_layer" | "tensor" => Ok(Self::PerLayer),
"head" | "per-head" | "per_head" => Ok(Self::PerHead),
other => Err(format!(
"unknown KV granularity '{other}'; expected layer|head"
)),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct KvQuantScheme {
pub format: KvQuantization,
pub granularity: KvGranularity,
}
impl KvQuantScheme {
pub fn int8_per_head() -> Self {
Self {
format: KvQuantization::Int8,
granularity: KvGranularity::PerHead,
}
}
pub fn scale_count(self, heads: usize) -> usize {
match self.granularity {
KvGranularity::PerLayer => 1,
KvGranularity::PerHead => heads.max(1),
}
}
pub fn bytes_per_element(self) -> f32 {
self.format.bits() as f32 / 8.0
}
}
#[derive(Clone, Debug)]
pub struct PackedKvTensor {
pub scheme: KvQuantScheme,
pub packed: Vec<u32>,
pub scales: Vec<f32>,
pub batch: usize,
pub heads: usize,
pub seq: usize,
pub head_dim: usize,
}
impl PackedKvTensor {
pub fn numel(&self) -> usize {
self.batch
.saturating_mul(self.heads)
.saturating_mul(self.seq)
.saturating_mul(self.head_dim)
}
pub fn storage_bytes(&self) -> usize {
self.packed.len() * std::mem::size_of::<u32>()
+ self.scales.len() * std::mem::size_of::<f32>()
}
pub fn dequantize(&self) -> Vec<f32> {
let format = self.scheme.format.weight_format();
let mut values = vec![0.0; self.numel()];
let head_stride = self.seq * self.head_dim;
for batch in 0..self.batch {
for head in 0..self.heads {
let scale = match self.scheme.granularity {
KvGranularity::PerLayer => self.scales[0],
KvGranularity::PerHead => self.scales[head],
};
let base = (batch * self.heads + head) * head_stride;
for index in 0..head_stride {
let flat = base + index;
let raw = read_packed_public(&self.packed, flat, format);
values[flat] = decode_scalar(raw, scale, 0.0, format, true);
}
}
}
values
}
}
pub fn pack_kv(
values: &[f32],
batch: usize,
heads: usize,
seq: usize,
head_dim: usize,
scheme: KvQuantScheme,
) -> Result<PackedKvTensor, String> {
let expected = batch * heads * seq * head_dim;
if values.len() != expected {
return Err(format!(
"KV length {} != B·H·S·D ({expected})",
values.len()
));
}
let format = scheme.format.weight_format();
let scale_count = scheme.scale_count(heads);
let mut scales = vec![1.0; scale_count];
let head_stride = seq * head_dim;
match scheme.granularity {
KvGranularity::PerLayer => {
scales[0] = absmax_scale(values, format).max(1e-12);
}
KvGranularity::PerHead => {
for head in 0..heads {
let mut absmax = 0.0f32;
for batch_index in 0..batch {
let start = (batch_index * heads + head) * head_stride;
let end = start + head_stride;
for &value in &values[start..end] {
absmax = absmax.max(value.abs());
}
}
scales[head] = absmax_scale(&[absmax], format).max(1e-12);
}
}
}
let words = expected.div_ceil(format.values_per_word());
let mut packed = vec![0u32; words];
for batch_index in 0..batch {
for head in 0..heads {
let scale = match scheme.granularity {
KvGranularity::PerLayer => scales[0],
KvGranularity::PerHead => scales[head],
};
let base = (batch_index * heads + head) * head_stride;
for index in 0..head_stride {
let flat = base + index;
let encoded = encode_scalar(values[flat], scale, 0.0, format, true);
write_packed_public(&mut packed, flat, encoded, format);
}
}
}
Ok(PackedKvTensor {
scheme,
packed,
scales,
batch,
heads,
seq,
head_dim,
})
}
pub fn estimate_kv_cache_bytes_quantized(
num_layers: usize,
batch: usize,
seq: usize,
kv_heads: usize,
head_dim: usize,
scheme: KvQuantScheme,
) -> usize {
let elements_per_layer = batch
.saturating_mul(kv_heads)
.saturating_mul(seq)
.saturating_mul(head_dim)
.saturating_mul(2); let packed_bytes =
((elements_per_layer as f32) * scheme.bytes_per_element()).ceil() as usize;
let scale_bytes = scheme.scale_count(kv_heads) * 2 * std::mem::size_of::<f32>();
num_layers.saturating_mul(packed_bytes.saturating_add(scale_bytes))
}
#[derive(Clone, Debug, Default)]
pub struct KvRangeCollector {
pub key_absmax: f32,
pub value_absmax: f32,
pub per_head_key: Vec<f32>,
pub per_head_value: Vec<f32>,
}
impl KvRangeCollector {
pub fn new(heads: usize) -> Self {
Self {
key_absmax: 0.0,
value_absmax: 0.0,
per_head_key: vec![0.0; heads],
per_head_value: vec![0.0; heads],
}
}
pub fn observe(
&mut self,
key: &[f32],
value: &[f32],
batch: usize,
heads: usize,
seq: usize,
head_dim: usize,
) {
if self.per_head_key.len() != heads {
self.per_head_key = vec![0.0; heads];
self.per_head_value = vec![0.0; heads];
}
let head_stride = seq * head_dim;
for batch_index in 0..batch {
for head in 0..heads {
let start = (batch_index * heads + head) * head_stride;
let end = start + head_stride;
for &v in &key[start..end] {
let abs = v.abs();
self.key_absmax = self.key_absmax.max(abs);
self.per_head_key[head] = self.per_head_key[head].max(abs);
}
for &v in &value[start..end] {
let abs = v.abs();
self.value_absmax = self.value_absmax.max(abs);
self.per_head_value[head] = self.per_head_value[head].max(abs);
}
}
}
}
pub fn scales(&self, scheme: KvQuantScheme) -> (Vec<f32>, Vec<f32>) {
let to_scale = |absmax: f32| -> f32 {
if absmax <= 0.0 {
return 1.0;
}
match scheme.format {
KvQuantization::Int8 => absmax / 127.0,
_ => absmax,
}
};
match scheme.granularity {
KvGranularity::PerLayer => (vec![to_scale(self.key_absmax)], vec![to_scale(self.value_absmax)]),
KvGranularity::PerHead => (
self.per_head_key.iter().copied().map(to_scale).collect(),
self.per_head_value.iter().copied().map(to_scale).collect(),
),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn int8_per_head_roundtrip() {
let batch = 1;
let heads = 2;
let seq = 3;
let head_dim = 4;
let mut values = Vec::new();
for i in 0..(batch * heads * seq * head_dim) {
values.push((i as f32) * 0.01 - 0.2);
}
let packed = pack_kv(
&values,
batch,
heads,
seq,
head_dim,
KvQuantScheme::int8_per_head(),
)
.unwrap();
assert_eq!(packed.scales.len(), heads);
let recon = packed.dequantize();
for (original, restored) in values.iter().zip(recon.iter()) {
assert!(
(original - restored).abs() < 0.02,
"{original} vs {restored}"
);
}
}
#[test]
fn nf4_per_layer_packs() {
let values: Vec<f32> = (0..16).map(|i| (i as f32) * 0.05 - 0.4).collect();
let packed = pack_kv(
&values,
1,
1,
4,
4,
KvQuantScheme {
format: KvQuantization::Nf4,
granularity: KvGranularity::PerLayer,
},
)
.unwrap();
assert_eq!(packed.scales.len(), 1);
assert!(!packed.packed.is_empty());
}
}