pub const BINARY_QUANTIZATION_DIM: usize = 768;
pub const QUANTIZED_VECTOR_SIZE: usize = 96;
#[repr(C, align(64))]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct QuantizedVector {
data: [u8; QUANTIZED_VECTOR_SIZE],
}
impl QuantizedVector {
#[must_use]
pub const fn from_bytes(data: [u8; QUANTIZED_VECTOR_SIZE]) -> Self {
Self { data }
}
#[must_use]
pub const fn data(&self) -> &[u8; QUANTIZED_VECTOR_SIZE] {
&self.data
}
#[must_use]
pub fn hamming_distance(&self, other: &Self) -> u32 {
crate::quantization::simd::hamming_distance(&self.data, &other.data)
}
#[must_use]
#[allow(clippy::cast_precision_loss)] pub fn similarity(&self, other: &Self) -> f32 {
let distance = self.hamming_distance(other);
1.0 - (distance as f32 / BINARY_QUANTIZATION_DIM as f32)
}
}
impl Default for QuantizedVector {
fn default() -> Self {
Self {
data: [0u8; QUANTIZED_VECTOR_SIZE],
}
}
}
#[derive(Clone, Debug, Default)]
pub struct BinaryQuantizer {
_marker: std::marker::PhantomData<()>,
}
impl BinaryQuantizer {
#[must_use]
pub const fn new() -> Self {
Self {
_marker: std::marker::PhantomData,
}
}
#[must_use]
pub fn quantize(&self, vector: &[f32]) -> QuantizedVector {
assert_eq!(
vector.len(),
BINARY_QUANTIZATION_DIM,
"Input must be {BINARY_QUANTIZATION_DIM}-dimensional, got {}",
vector.len()
);
let mut data = [0u8; QUANTIZED_VECTOR_SIZE];
for (i, &value) in vector.iter().enumerate() {
if value > 0.0 {
let byte_idx = i / 8;
let bit_idx = i % 8;
data[byte_idx] |= 1 << bit_idx;
}
}
QuantizedVector { data }
}
#[must_use]
pub fn quantize_flexible(&self, vector: &[f32]) -> QuantizedVector {
let mut data = [0u8; QUANTIZED_VECTOR_SIZE];
let len = vector.len().min(BINARY_QUANTIZATION_DIM);
for (i, &value) in vector.iter().take(len).enumerate() {
if value > 0.0 {
let byte_idx = i / 8;
let bit_idx = i % 8;
data[byte_idx] |= 1 << bit_idx;
}
}
QuantizedVector { data }
}
#[must_use]
pub fn quantize_to_bytes(vector: &[f32]) -> Vec<u8> {
let num_bytes = (vector.len() + 7) / 8;
let mut data = vec![0u8; num_bytes];
for (i, &value) in vector.iter().enumerate() {
if value > 0.0 {
let byte_idx = i / 8;
let bit_idx = i % 8;
data[byte_idx] |= 1 << bit_idx;
}
}
data
}
}
#[cfg(test)]
#[allow(clippy::similar_names)]
mod tests {
use super::*;
#[test]
fn test_quantize_zero_vector() {
let quantizer = BinaryQuantizer::new();
let zero = vec![0.0f32; BINARY_QUANTIZATION_DIM];
let quantized = quantizer.quantize(&zero);
assert_eq!(quantized.data, [0u8; QUANTIZED_VECTOR_SIZE]);
}
#[test]
fn test_quantize_positive_vector() {
let quantizer = BinaryQuantizer::new();
let positive = vec![1.0f32; BINARY_QUANTIZATION_DIM];
let quantized = quantizer.quantize(&positive);
assert_eq!(quantized.data, [0xFFu8; QUANTIZED_VECTOR_SIZE]);
}
#[test]
fn test_quantize_negative_vector() {
let quantizer = BinaryQuantizer::new();
let negative = vec![-1.0f32; BINARY_QUANTIZATION_DIM];
let quantized = quantizer.quantize(&negative);
assert_eq!(quantized.data, [0u8; QUANTIZED_VECTOR_SIZE]);
}
#[test]
fn test_quantize_mixed_vector() {
let quantizer = BinaryQuantizer::new();
let mut mixed = vec![-1.0f32; BINARY_QUANTIZATION_DIM];
mixed[0] = 1.0; mixed[8] = 1.0;
let quantized = quantizer.quantize(&mixed);
assert_eq!(quantized.data[0], 0b0000_0001); assert_eq!(quantized.data[1], 0b0000_0001); for i in 2..QUANTIZED_VECTOR_SIZE {
assert_eq!(quantized.data[i], 0);
}
}
#[test]
fn test_quantize_alternating() {
let quantizer = BinaryQuantizer::new();
let alternating: Vec<f32> = (0..BINARY_QUANTIZATION_DIM)
.map(|i| if i % 2 == 0 { 1.0 } else { -1.0 })
.collect();
let quantized = quantizer.quantize(&alternating);
for byte in &quantized.data {
assert_eq!(*byte, 0x55);
}
}
#[test]
fn test_hamming_distance_identical() {
let quantizer = BinaryQuantizer::new();
let vec = vec![0.5f32; BINARY_QUANTIZATION_DIM];
let q1 = quantizer.quantize(&vec);
let q2 = quantizer.quantize(&vec);
assert_eq!(q1.hamming_distance(&q2), 0);
}
#[test]
#[allow(clippy::cast_possible_truncation)]
fn test_hamming_distance_opposite() {
let q1 = QuantizedVector::from_bytes([0x00u8; QUANTIZED_VECTOR_SIZE]);
let q2 = QuantizedVector::from_bytes([0xFFu8; QUANTIZED_VECTOR_SIZE]);
assert_eq!(q1.hamming_distance(&q2), BINARY_QUANTIZATION_DIM as u32);
}
#[test]
#[allow(clippy::cast_possible_truncation)]
fn test_hamming_distance_symmetric() {
let q1 = QuantizedVector::from_bytes([0xAAu8; QUANTIZED_VECTOR_SIZE]); let q2 = QuantizedVector::from_bytes([0x55u8; QUANTIZED_VECTOR_SIZE]);
assert_eq!(q1.hamming_distance(&q2), q2.hamming_distance(&q1));
assert_eq!(q1.hamming_distance(&q2), BINARY_QUANTIZATION_DIM as u32);
}
#[test]
#[allow(clippy::cast_possible_truncation)]
fn test_hamming_distance_partial() {
let q1 = QuantizedVector::from_bytes([0xF0u8; QUANTIZED_VECTOR_SIZE]); let q2 = QuantizedVector::from_bytes([0x0Fu8; QUANTIZED_VECTOR_SIZE]);
assert_eq!(q1.hamming_distance(&q2), BINARY_QUANTIZATION_DIM as u32);
}
#[test]
fn test_quantize_deterministic() {
let quantizer = BinaryQuantizer::new();
let vec = vec![0.123f32; BINARY_QUANTIZATION_DIM];
let q1 = quantizer.quantize(&vec);
let q2 = quantizer.quantize(&vec);
assert_eq!(q1, q2); }
#[test]
fn test_alignment() {
let q = QuantizedVector::default();
let ptr = std::ptr::addr_of!(q) as usize;
assert_eq!(
ptr % 64,
0,
"QuantizedVector must be 64-byte aligned, got alignment {}",
ptr % 64
);
}
#[test]
fn test_struct_size() {
assert_eq!(
std::mem::size_of::<QuantizedVector>(),
128, "QuantizedVector size should be 128 bytes (96 data + padding)"
);
}
#[test]
fn test_struct_alignment() {
assert_eq!(
std::mem::align_of::<QuantizedVector>(),
64,
"QuantizedVector must have 64-byte alignment"
);
}
#[test]
fn test_similarity_identical() {
let q1 = QuantizedVector::from_bytes([0xAAu8; QUANTIZED_VECTOR_SIZE]);
let q2 = QuantizedVector::from_bytes([0xAAu8; QUANTIZED_VECTOR_SIZE]);
assert!((q1.similarity(&q2) - 1.0).abs() < f32::EPSILON);
}
#[test]
fn test_similarity_opposite() {
let q1 = QuantizedVector::from_bytes([0x00u8; QUANTIZED_VECTOR_SIZE]);
let q2 = QuantizedVector::from_bytes([0xFFu8; QUANTIZED_VECTOR_SIZE]);
assert!((q1.similarity(&q2) - 0.0).abs() < f32::EPSILON);
}
#[test]
fn test_quantize_flexible_short() {
let quantizer = BinaryQuantizer::new();
let short = vec![1.0f32; 16];
let quantized = quantizer.quantize_flexible(&short);
assert_eq!(quantized.data[0], 0xFF);
assert_eq!(quantized.data[1], 0xFF);
for i in 2..QUANTIZED_VECTOR_SIZE {
assert_eq!(quantized.data[i], 0);
}
}
#[test]
#[should_panic(expected = "Input must be 768-dimensional")]
fn test_quantize_wrong_dimension() {
let quantizer = BinaryQuantizer::new();
let wrong_dim = vec![1.0f32; 100];
let _ = quantizer.quantize(&wrong_dim);
}
#[test]
fn test_edge_case_nan() {
let quantizer = BinaryQuantizer::new();
let mut vec = vec![1.0f32; BINARY_QUANTIZATION_DIM];
vec[0] = f32::NAN;
let quantized = quantizer.quantize(&vec);
assert_eq!(quantized.data[0] & 1, 0);
}
#[test]
fn test_edge_case_infinity() {
let quantizer = BinaryQuantizer::new();
let mut vec = vec![1.0f32; BINARY_QUANTIZATION_DIM];
vec[0] = f32::INFINITY; vec[1] = f32::NEG_INFINITY;
let quantized = quantizer.quantize(&vec);
assert_eq!(quantized.data[0] & 0b01, 0b01);
assert_eq!(quantized.data[0] & 0b10, 0b00);
}
#[test]
fn test_edge_case_negative_zero() {
let quantizer = BinaryQuantizer::new();
let mut vec = vec![1.0f32; BINARY_QUANTIZATION_DIM];
vec[0] = -0.0f32;
let quantized = quantizer.quantize(&vec);
assert_eq!(quantized.data[0] & 1, 0);
}
#[test]
#[allow(clippy::cast_possible_truncation)]
fn test_hamming_bounds() {
let q1 = QuantizedVector::default();
let q2 = QuantizedVector::from_bytes([0xFFu8; QUANTIZED_VECTOR_SIZE]);
let distance = q1.hamming_distance(&q2);
assert!(distance <= BINARY_QUANTIZATION_DIM as u32);
}
}
#[cfg(test)]
#[allow(clippy::cast_possible_truncation)]
mod proptests {
use super::*;
use proptest::prelude::*;
fn valid_vector_strategy() -> impl Strategy<Value = Vec<f32>> {
proptest::collection::vec(-1.0f32..=1.0f32, BINARY_QUANTIZATION_DIM)
}
proptest! {
#[test]
fn prop_quantize_deterministic(v in valid_vector_strategy()) {
let quantizer = BinaryQuantizer::new();
let q1 = quantizer.quantize(&v);
let q2 = quantizer.quantize(&v);
prop_assert_eq!(q1, q2);
}
#[test]
fn prop_self_distance_zero(v in valid_vector_strategy()) {
let quantizer = BinaryQuantizer::new();
let q = quantizer.quantize(&v);
prop_assert_eq!(q.hamming_distance(&q), 0);
}
#[test]
fn prop_hamming_symmetric(
v1 in valid_vector_strategy(),
v2 in valid_vector_strategy()
) {
let quantizer = BinaryQuantizer::new();
let q1 = quantizer.quantize(&v1);
let q2 = quantizer.quantize(&v2);
prop_assert_eq!(q1.hamming_distance(&q2), q2.hamming_distance(&q1));
}
#[test]
fn prop_hamming_bounded(
v1 in valid_vector_strategy(),
v2 in valid_vector_strategy()
) {
let quantizer = BinaryQuantizer::new();
let q1 = quantizer.quantize(&v1);
let q2 = quantizer.quantize(&v2);
let dist = q1.hamming_distance(&q2);
prop_assert!(dist <= BINARY_QUANTIZATION_DIM as u32);
}
#[test]
fn prop_similarity_bounded(
v1 in valid_vector_strategy(),
v2 in valid_vector_strategy()
) {
let quantizer = BinaryQuantizer::new();
let q1 = quantizer.quantize(&v1);
let q2 = quantizer.quantize(&v2);
let sim = q1.similarity(&q2);
prop_assert!(sim >= 0.0);
prop_assert!(sim <= 1.0);
}
#[test]
fn prop_triangle_inequality(
v1 in valid_vector_strategy(),
v2 in valid_vector_strategy(),
v3 in valid_vector_strategy()
) {
let quantizer = BinaryQuantizer::new();
let q1 = quantizer.quantize(&v1);
let q2 = quantizer.quantize(&v2);
let q3 = quantizer.quantize(&v3);
let d12 = q1.hamming_distance(&q2);
let d23 = q2.hamming_distance(&q3);
let d13 = q1.hamming_distance(&q3);
prop_assert!(d13 <= d12 + d23,
"Triangle inequality violated: {} > {} + {}", d13, d12, d23);
}
#[test]
fn prop_output_size_constant(v in valid_vector_strategy()) {
let quantizer = BinaryQuantizer::new();
let q = quantizer.quantize(&v);
prop_assert_eq!(q.data().len(), QUANTIZED_VECTOR_SIZE);
}
#[test]
fn prop_all_positive_all_ones(scale in 0.001f32..10.0f32) {
let v: Vec<f32> = (0..BINARY_QUANTIZATION_DIM).map(|_| scale).collect();
let quantizer = BinaryQuantizer::new();
let q = quantizer.quantize(&v);
for byte in q.data() {
prop_assert_eq!(*byte, 0xFF);
}
}
#[test]
fn prop_all_negative_all_zeros(scale in 0.001f32..10.0f32) {
let v: Vec<f32> = (0..BINARY_QUANTIZATION_DIM).map(|_| -scale).collect();
let quantizer = BinaryQuantizer::new();
let q = quantizer.quantize(&v);
for byte in q.data() {
prop_assert_eq!(*byte, 0x00);
}
}
}
}