use crate::matrix::Matrix;
use rayon::prelude::*;
#[derive(Clone, Debug, PartialEq)]
pub struct Quantized {
rows: usize,
cols: usize,
data: Vec<i8>,
scales: Vec<f32>,
}
impl Quantized {
pub(crate) fn from_matrix(value: &Matrix) -> Self {
let cols = value.cols.max(1);
let mut data = Vec::with_capacity(value.data.len());
let mut scales = Vec::with_capacity(value.rows);
for row in value.data.chunks(cols) {
let absmax = row.iter().fold(0.0f32, |acc, v| acc.max(v.abs()));
let scale = absmax / 127.0;
scales.push(scale);
let inverse = if scale > 0.0 { 1.0 / scale } else { 0.0 };
data.extend(
row.iter()
.map(|v| (v * inverse).round().clamp(-127.0, 127.0) as i8),
);
}
Self {
rows: value.rows,
cols: value.cols,
data,
scales,
}
}
pub(crate) fn matmul_rhs_transposed(&self, input: &Matrix) -> Matrix {
debug_assert_eq!(input.cols, self.cols);
let mut output = Matrix::new(input.rows, self.rows);
let cols = self.cols.max(1);
fn row_of<'a>(activations: &'a [f32]) -> impl Fn((&mut f32, (&[i8], &f32))) + Sync + 'a {
move |(slot, (weights, scale))| *slot = dot(activations, weights) * scale
}
const PARALLEL_VALUES: usize = 1 << 18;
if input.rows > 1 {
output
.data
.par_chunks_mut(self.rows)
.enumerate()
.for_each(|(token, target)| {
let activations = input.row(token);
target
.iter_mut()
.zip(self.data.chunks_exact(cols).zip(self.scales.iter()))
.for_each(row_of(activations));
});
} else if self.data.len() >= PARALLEL_VALUES {
for (token, target) in output.data.chunks_mut(self.rows).enumerate() {
let activations = input.row(token);
target
.par_iter_mut()
.zip(self.data.par_chunks_exact(cols).zip(self.scales.par_iter()))
.for_each(row_of(activations));
}
} else {
for (token, target) in output.data.chunks_mut(self.rows).enumerate() {
let activations = input.row(token);
target
.iter_mut()
.zip(self.data.chunks_exact(cols).zip(self.scales.iter()))
.for_each(row_of(activations));
}
}
output
}
pub(crate) fn dequantize_row(&self, index: usize, target: &mut [f32]) {
let cols = self.cols.max(1);
let scale = self.scales[index];
let row = &self.data[index * cols..index * cols + cols];
for (slot, &value) in target.iter_mut().zip(row) {
*slot = value as f32 * scale;
}
}
#[cfg(feature = "cuda")]
pub(crate) fn dequantize(&self) -> Matrix {
let mut output = Matrix::new(self.rows, self.cols);
for index in 0..self.rows {
self.dequantize_row(index, output.row_mut(index));
}
output
}
pub(crate) fn round_row(row: &mut [f32]) {
let absmax = row.iter().fold(0.0f32, |acc, v| acc.max(v.abs()));
let scale = absmax / 127.0;
if scale <= 0.0 {
return;
}
for value in row {
*value = (*value / scale).round().clamp(-127.0, 127.0) * scale;
}
}
pub(crate) fn bytes(&self) -> usize {
self.data.len() + self.scales.len() * 4
}
}
#[inline(always)]
fn dot_lanes<const LANES: usize>(activations: &[f32], weights: &[i8]) -> f32 {
debug_assert_eq!(activations.len(), weights.len());
let mut lanes = [0.0f32; LANES];
let tail = activations.len() - activations.len() % LANES;
for (x, y) in activations[..tail]
.chunks_exact(LANES)
.zip(weights[..tail].chunks_exact(LANES))
{
let mut widened = [0.0f32; LANES];
for lane in 0..LANES {
widened[lane] = f32::from(y[lane]);
}
for lane in 0..LANES {
lanes[lane] += x[lane] * widened[lane];
}
}
let mut rest = 0.0f32;
for (x, &y) in activations[tail..].iter().zip(&weights[tail..]) {
rest += x * f32::from(y);
}
lanes.iter().sum::<f32>() + rest
}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2,fma")]
unsafe fn dot_avx2(activations: &[f32], weights: &[i8]) -> f32 {
dot_lanes::<32>(activations, weights)
}
fn dot(activations: &[f32], weights: &[i8]) -> f32 {
#[cfg(target_arch = "x86_64")]
if std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma") {
return unsafe { dot_avx2(activations, weights) };
}
dot_lanes::<8>(activations, weights)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn quantizing_and_multiplying_agrees_with_the_float_matrix() {
let weights = Matrix::from_vec(3, 4, (0..12).map(|v| v as f32 * 0.37 - 2.0).collect());
let input = Matrix::from_vec(2, 4, vec![0.5, -1.5, 2.0, 0.25, 1.0, 1.0, -1.0, 0.0]);
let mut expected = Matrix::new(2, 3);
input.dot_rhs_transposed(&weights, &mut expected);
let actual = Quantized::from_matrix(&weights).matmul_rhs_transposed(&input);
assert_eq!(actual.rows, 2);
assert_eq!(actual.cols, 3);
for (got, want) in actual.data.iter().zip(&expected.data) {
assert!((got - want).abs() < 0.05, "{got} vs {want}");
}
}
#[test]
fn a_row_of_zeros_stays_zero() {
let quantized = Quantized::from_matrix(&Matrix::new(2, 3));
let mut row = [1.0f32; 3];
quantized.dequantize_row(0, &mut row);
assert_eq!(row, [0.0; 3]);
}
#[test]
fn dequantizing_a_row_recovers_it_within_the_step_size() {
let weights = Matrix::from_vec(2, 3, vec![0.5, -1.0, 0.25, 4.0, -2.0, 0.0]);
let quantized = Quantized::from_matrix(&weights);
let mut row = [0.0f32; 3];
quantized.dequantize_row(1, &mut row);
for (got, want) in row.iter().zip(&weights.data[3..]) {
assert!((got - want).abs() < 4.0 / 127.0, "{got} vs {want}");
}
}
}