use hurray_core::{
BufferHandle, DeviceTag, ElementType, LayoutDescriptor, PerBlockAffine, QuantizationDescriptor,
Shape, SyncMode, TensorDescriptor, DESCRIPTOR_VERSION_MAJOR, DESCRIPTOR_VERSION_MINOR,
MIN_BUFFER_ALIGNMENT,
};
const ROWS: usize = 2;
const COLS: usize = 16;
const BLOCK: usize = 8;
const BLOCKS_PER_ROW: usize = COLS / BLOCK;
const NUM_BLOCKS: usize = BLOCKS_PER_ROW * ROWS;
#[rustfmt::skip]
const WEIGHTS: [i8; ROWS * COLS] = [
9, 8, 7, 10, 8, 6, 12, 8, 3, 9, 8, 11, 7, 8, 5, 10,
8, 13, 8, 4, 9, 8, 7, 8, 10, 8, 6, 8, 9, 2, 8, 14,
];
const SCALES: [f32; NUM_BLOCKS] = [0.02, 0.015, 0.025, 0.01];
const FOREIGN_ZERO_POINTS_PACKED: [u8; NUM_BLOCKS / 2] = [0x67, 0x78];
fn normalize_zero_points(packed: &[u8], count: usize) -> Vec<i32> {
(0..count)
.map(|i| {
let byte = packed[i / 2];
let nibble = if i % 2 == 0 { byte & 0x0F } else { byte >> 4 };
i32::from(nibble) + 1
})
.collect()
}
fn block_of(row: usize, col: usize) -> usize {
row * BLOCKS_PER_ROW + col / BLOCK
}
fn dequantize(weights: &[i8], scales: &[f32], zero_points: &[i32]) -> Vec<f32> {
let mut out = Vec::with_capacity(weights.len());
for row in 0..ROWS {
for col in 0..COLS {
let b = block_of(row, col);
let q = i32::from(weights[row * COLS + col]);
out.push(scales[b] * (q - zero_points[b]) as f32);
}
}
out
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("Converting an externally quantized tensor\n");
let zero_points = normalize_zero_points(&FOREIGN_ZERO_POINTS_PACKED, NUM_BLOCKS);
assert_eq!(zero_points, vec![8, 7, 9, 8], "normalized zero points");
println!(" foreign plane : {FOREIGN_ZERO_POINTS_PACKED:02X?} (4-bit, biased)");
println!(" hurray plane : {zero_points:?} (int32, as subtracted)\n");
let handle = |len: u64| {
BufferHandle::new(
len,
MIN_BUFFER_ALIGNMENT,
DeviceTag::Cpu,
SyncMode::ProducerSynced,
)
};
let buffers = vec![
handle((ROWS * COLS) as u64)?,
handle((NUM_BLOCKS * 4) as u64)?,
handle((NUM_BLOCKS * 4) as u64)?,
];
let quant = QuantizationDescriptor::PerBlockAffine(PerBlockAffine::new_asymmetric(
1, BLOCK as u32, 1, 2, ElementType::Float32, )?);
let desc = TensorDescriptor::new(
DESCRIPTOR_VERSION_MAJOR,
DESCRIPTOR_VERSION_MINOR,
ElementType::Int8,
Shape::new(vec![ROWS as u64, COLS as u64])?,
0,
LayoutDescriptor::RowMajor,
buffers,
Some(quant.encode_to_vec()),
None, None, None, )?;
println!(" descriptor : {} bytes\n", desc.encode()?.len());
let correct = dequantize(&WEIGHTS, &SCALES, &zero_points);
let unnormalized: Vec<i32> = zero_points.iter().map(|z| z - 1).collect();
let wrong = dequantize(&WEIGHTS, &SCALES, &unnormalized);
for row in 0..ROWS {
for col in 0..COLS {
let i = row * COLS + col;
let drift = wrong[i] - correct[i];
let step = SCALES[block_of(row, col)];
assert!(
(drift - step).abs() < 1e-6,
"element [{row}, {col}]: drift {drift} is not one scale step {step}"
);
}
}
println!(
" element [0, 0]: correct {:.4}, un-normalized {:.4}",
correct[0], wrong[0]
);
println!(" every element off by exactly one scale step of its block\n");
println!("Nothing rejects the second descriptor. Only the arithmetic tells you.");
Ok(())
}