Skip to main content

lattice_simd/
int4.rs

1//! INT4 packed-vector quantization and approximate dot products.
2//!
3//! Nibble layout and offset correction are shared by scalar and NEON paths.
4//!
5//! See docs/simd.md for the packed format and corrected dot-product derivation.
6
7#[cfg(target_arch = "aarch64")]
8use std::arch::aarch64::*;
9
10#[cfg(target_arch = "x86_64")]
11use std::arch::x86_64::*;
12
13#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
14use super::simd_config;
15
16/// **Unstable**: INT4 quantization internals; scale/bias scheme may change.
17///
18/// Quantization parameters for INT4 conversion.
19///
20/// Uses symmetric unsigned quantization: the float range [-max_abs, max_abs]
21/// is mapped to the integer range [0, 15].
22#[derive(Debug, Clone, Copy)]
23pub struct Int4Params {
24    /// **Unstable**: scale factor; formula may change with quantization scheme update.
25    pub scale: f32,
26    /// **Unstable**: maximum absolute value; field may be removed.
27    pub max_abs: f32,
28}
29
30impl Int4Params {
31    /// **Unstable**: quantization parameter computation; may be folded into `Int4Vector::from_f32`.
32    pub fn from_vector(vector: &[f32]) -> Self {
33        let max_abs = max_abs_finite(vector);
34
35        // Epsilon guard: avoid division by near-zero
36        let scale = if max_abs > 1e-10 {
37            15.0 / (2.0 * max_abs)
38        } else {
39            1.0
40        };
41
42        Self { scale, max_abs }
43    }
44}
45
46fn max_abs_finite(vector: &[f32]) -> f32 {
47    #[cfg(target_arch = "x86_64")]
48    {
49        if simd_config().avx2_enabled {
50            // SAFETY: AVX2 was detected at runtime; the kernel bounds every load.
51            return unsafe { max_abs_finite_avx2(vector) };
52        }
53    }
54    #[cfg(target_arch = "aarch64")]
55    {
56        if simd_config().neon_enabled {
57            // SAFETY: NEON was detected at runtime; the kernel bounds every load.
58            return unsafe { max_abs_finite_neon(vector) };
59        }
60    }
61    max_abs_finite_scalar(vector)
62}
63
64fn max_abs_finite_scalar(vector: &[f32]) -> f32 {
65    vector
66        .iter()
67        .filter(|value| value.is_finite())
68        .map(|value| value.abs())
69        .fold(0.0, f32::max)
70}
71
72#[cfg(test)]
73thread_local! {
74    static INT4_MAX_ABS_SIMD_HITS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
75}
76
77#[cfg(target_arch = "x86_64")]
78#[target_feature(enable = "avx2")]
79unsafe fn max_abs_finite_avx2(vector: &[f32]) -> f32 {
80    #[cfg(test)]
81    INT4_MAX_ABS_SIMD_HITS.with(|hits| hits.set(hits.get() + 1));
82
83    let chunks = vector.len() / 8;
84    let sign = _mm256_set1_ps(-0.0);
85    let inf = _mm256_set1_ps(f32::INFINITY);
86    let mut maximum = _mm256_setzero_ps();
87
88    for i in 0..chunks {
89        let input = _mm256_loadu_ps(vector.as_ptr().add(i * 8));
90        let abs = _mm256_andnot_ps(sign, input);
91        let finite = _mm256_cmp_ps(abs, inf, _CMP_LT_OQ);
92        maximum = _mm256_max_ps(maximum, _mm256_and_ps(abs, finite));
93    }
94
95    let mut lanes = [0.0f32; 8];
96    _mm256_storeu_ps(lanes.as_mut_ptr(), maximum);
97    let mut max_abs = lanes.into_iter().fold(0.0f32, f32::max);
98    for &value in &vector[chunks * 8..] {
99        if value.is_finite() {
100            max_abs = max_abs.max(value.abs());
101        }
102    }
103    max_abs
104}
105
106#[cfg(target_arch = "aarch64")]
107#[target_feature(enable = "neon")]
108unsafe fn max_abs_finite_neon(vector: &[f32]) -> f32 {
109    #[cfg(test)]
110    INT4_MAX_ABS_SIMD_HITS.with(|hits| hits.set(hits.get() + 1));
111
112    let chunks = vector.len() / 4;
113    let inf = vdupq_n_f32(f32::INFINITY);
114    let zero = vdupq_n_f32(0.0);
115    let mut maximum = zero;
116
117    for i in 0..chunks {
118        let input = vld1q_f32(vector.as_ptr().add(i * 4));
119        let abs = vabsq_f32(input);
120        let finite = vcltq_f32(abs, inf);
121        maximum = vmaxq_f32(maximum, vbslq_f32(finite, abs, zero));
122    }
123
124    let mut max_abs = vmaxvq_f32(maximum);
125    for &value in &vector[chunks * 4..] {
126        if value.is_finite() {
127            max_abs = max_abs.max(value.abs());
128        }
129    }
130    max_abs
131}
132
133/// **Unstable**: INT4 quantization format is under active design; struct layout may change.
134///
135/// Quantized INT4 vector with packed nibble storage.
136#[derive(Debug, Clone)]
137pub struct Int4Vector {
138    /// **Unstable**: packed nibble data; bit packing scheme may change.
139    pub data: Vec<u8>,
140    /// **Unstable**: number of original dimensions.
141    pub dims: usize,
142    /// **Unstable**: quantization parameters; may be separated from the vector.
143    pub params: Int4Params,
144    /// **Unstable**: L2 norm; may be removed or moved.
145    pub norm: f32,
146}
147
148impl Int4Vector {
149    /// **Unstable**: quantization format; nibble packing may change.
150    ///
151    /// Each pair of consecutive dimensions is packed into one byte:
152    /// - High nibble (bits 7..4) = even-indexed value
153    /// - Low nibble (bits 3..0) = odd-indexed value
154    pub fn from_f32(vector: &[f32]) -> Self {
155        let params = Int4Params::from_vector(vector);
156        let dims = vector.len();
157
158        // Compute L2 norm
159        let mut norm_sq = 0.0f32;
160        for &v in vector {
161            if v.is_finite() {
162                norm_sq += v * v;
163            }
164        }
165        let norm = norm_sq.sqrt();
166
167        let data = quantize_int4(vector, params);
168
169        Self {
170            data,
171            dims,
172            params,
173            norm,
174        }
175    }
176
177    /// **Unstable**: dequantizes packed INT4 data, or returns empty for a malformed buffer.
178    ///
179    /// See [`docs/simd.md`](../docs/simd.md#int4-vectors) for format and precision bounds.
180    pub fn to_f32(&self) -> Vec<f32> {
181        let required_bytes = self.dims.div_ceil(2);
182        if self.data.len() < required_bytes {
183            return Vec::new();
184        }
185
186        let scale = if self.params.scale.is_finite() && self.params.scale != 0.0 {
187            self.params.scale
188        } else {
189            1.0
190        };
191
192        let mut result = Vec::with_capacity(self.dims);
193        for i in 0..self.dims {
194            let byte_idx = i / 2;
195            let q = if i % 2 == 0 {
196                (self.data[byte_idx] >> 4) & 0x0F
197            } else {
198                self.data[byte_idx] & 0x0F
199            };
200            result.push(q as f32 / scale - self.params.max_abs);
201        }
202        result
203    }
204
205    /// **Unstable**: INT4 dot product approximation; formula may change.
206    ///
207    /// Returns the dequantized dot product suitable for cosine distance computation.
208    #[inline]
209    pub fn dot_product(&self, other: &Int4Vector) -> f32 {
210        dot_product_int4(self, other)
211    }
212
213    /// **Unstable**: INT4 cosine similarity approximation; delegates to `dot_product`.
214    #[inline]
215    pub fn cosine_similarity(&self, other: &Int4Vector) -> f32 {
216        let denom = self.norm * other.norm;
217        if denom == 0.0 || !denom.is_finite() {
218            return 0.0;
219        }
220        self.dot_product(other) / denom
221    }
222
223    /// **Unstable**: complement of `cosine_similarity`; definition may evolve.
224    #[inline]
225    pub fn cosine_distance(&self, other: &Int4Vector) -> f32 {
226        1.0 - self.cosine_similarity(other)
227    }
228}
229
230fn quantize_int4(vector: &[f32], params: Int4Params) -> Vec<u8> {
231    #[cfg(target_arch = "x86_64")]
232    {
233        if simd_config().avx2_enabled {
234            // SAFETY: AVX2 was detected at runtime; the kernel bounds every load and store.
235            return unsafe { quantize_int4_avx2(vector, params) };
236        }
237    }
238    #[cfg(target_arch = "aarch64")]
239    {
240        if simd_config().neon_enabled {
241            // SAFETY: NEON was detected at runtime; the kernel bounds every load and store.
242            return unsafe { quantize_int4_neon(vector, params) };
243        }
244    }
245    quantize_int4_scalar(vector, params)
246}
247
248fn quantize_int4_scalar(vector: &[f32], params: Int4Params) -> Vec<u8> {
249    let mut data = vec![0u8; vector.len().div_ceil(2)];
250    quantize_int4_scalar_tail(vector, params, &mut data, 0);
251    data
252}
253
254fn quantize_int4_scalar_tail(vector: &[f32], params: Int4Params, data: &mut [u8], start: usize) {
255    for (i, &value) in vector.iter().enumerate().skip(start) {
256        let quantized = quantize_int4_value(value, params);
257        if i % 2 == 0 {
258            data[i / 2] |= quantized << 4;
259        } else {
260            data[i / 2] |= quantized;
261        }
262    }
263}
264
265#[inline]
266fn quantize_int4_value(value: f32, params: Int4Params) -> u8 {
267    let finite_value = if value.is_finite() { value } else { 0.0 };
268    ((finite_value + params.max_abs) * params.scale)
269        .round()
270        .clamp(0.0, 15.0) as u8
271}
272
273#[cfg(test)]
274thread_local! {
275    static INT4_QUANTIZE_SIMD_HITS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
276}
277
278#[cfg(target_arch = "x86_64")]
279#[target_feature(enable = "avx2")]
280unsafe fn quantize_int4_avx2(vector: &[f32], params: Int4Params) -> Vec<u8> {
281    #[cfg(test)]
282    INT4_QUANTIZE_SIMD_HITS.with(|hits| hits.set(hits.get() + 1));
283
284    let mut data = vec![0u8; vector.len().div_ceil(2)];
285    let chunks = vector.len() / 8;
286    let sign = _mm256_set1_ps(-0.0);
287    let inf = _mm256_set1_ps(f32::INFINITY);
288    let max_abs = _mm256_set1_ps(params.max_abs);
289    let scale = _mm256_set1_ps(params.scale);
290    let zero = _mm256_setzero_ps();
291    let high = _mm256_set1_ps(15.0);
292    let half = _mm256_set1_ps(0.5);
293    let one = _mm256_set1_epi32(1);
294
295    for i in 0..chunks {
296        let base = i * 8;
297        let input = _mm256_loadu_ps(vector.as_ptr().add(base));
298        let abs = _mm256_andnot_ps(sign, input);
299        let finite = _mm256_cmp_ps(abs, inf, _CMP_LT_OQ);
300        let values = _mm256_and_ps(input, finite);
301        let scaled = _mm256_mul_ps(_mm256_add_ps(values, max_abs), scale);
302        let clamped = _mm256_min_ps(_mm256_max_ps(scaled, zero), high);
303        let truncated = _mm256_cvttps_epi32(clamped);
304        let fraction = _mm256_sub_ps(clamped, _mm256_cvtepi32_ps(truncated));
305        let round_up = _mm256_castps_si256(_mm256_cmp_ps(fraction, half, _CMP_GE_OQ));
306        let rounded = _mm256_add_epi32(truncated, _mm256_and_si256(round_up, one));
307        let mut lanes = [0i32; 8];
308        _mm256_storeu_si256(lanes.as_mut_ptr().cast::<__m256i>(), rounded);
309        for pair in 0..4 {
310            data[base / 2 + pair] = ((lanes[pair * 2] as u8) << 4) | lanes[pair * 2 + 1] as u8;
311        }
312    }
313
314    quantize_int4_scalar_tail(vector, params, &mut data, chunks * 8);
315    data
316}
317
318#[cfg(target_arch = "aarch64")]
319#[target_feature(enable = "neon")]
320unsafe fn quantize_int4_neon(vector: &[f32], params: Int4Params) -> Vec<u8> {
321    #[cfg(test)]
322    INT4_QUANTIZE_SIMD_HITS.with(|hits| hits.set(hits.get() + 1));
323
324    let mut data = vec![0u8; vector.len().div_ceil(2)];
325    let chunks = vector.len() / 4;
326    let inf = vdupq_n_f32(f32::INFINITY);
327    let zero = vdupq_n_f32(0.0);
328    let max_abs = vdupq_n_f32(params.max_abs);
329    let scale = vdupq_n_f32(params.scale);
330    let high = vdupq_n_f32(15.0);
331
332    for i in 0..chunks {
333        let base = i * 4;
334        let input = vld1q_f32(vector.as_ptr().add(base));
335        let finite = vcaltq_f32(input, inf);
336        let values = vbslq_f32(finite, input, zero);
337        let scaled = vmulq_f32(vaddq_f32(values, max_abs), scale);
338        let clamped = vminq_f32(vmaxq_f32(scaled, zero), high);
339        let rounded = vcvtaq_s32_f32(clamped);
340        let mut lanes = [0i32; 4];
341        vst1q_s32(lanes.as_mut_ptr(), rounded);
342        data[base / 2] = ((lanes[0] as u8) << 4) | lanes[1] as u8;
343        data[base / 2 + 1] = ((lanes[2] as u8) << 4) | lanes[3] as u8;
344    }
345
346    quantize_int4_scalar_tail(vector, params, &mut data, chunks * 4);
347    data
348}
349
350/// **Unstable**: dequantized INT4 dot product; dispatch may change.
351#[inline]
352pub fn dot_product_int4(a: &Int4Vector, b: &Int4Vector) -> f32 {
353    if a.dims != b.dims {
354        return 0.0;
355    }
356
357    let scale_a = a.params.scale;
358    let scale_b = b.params.scale;
359    if scale_a == 0.0 || scale_b == 0.0 || !scale_a.is_finite() || !scale_b.is_finite() {
360        return 0.0;
361    }
362
363    let packed_len = a.dims.div_ceil(2);
364    if a.data.len() < packed_len || b.data.len() < packed_len {
365        return 0.0;
366    }
367
368    #[cfg(target_arch = "aarch64")]
369    {
370        let config = simd_config();
371        if config.neon_enabled {
372            // SAFETY: aarch64 NEON is available by config, the packed data length guard
373            // above prevents out-of-bounds loads, and the callee handles odd dimensions
374            // without reading the padding nibble as a real dimension.
375            let (raw_dot, sum_a, sum_b) =
376                unsafe { dot_product_int4_neon_unrolled(&a.data, &b.data, a.dims) };
377            return finish_int4_dot(raw_dot, sum_a, sum_b, a, b);
378        }
379    }
380
381    let (raw_dot, sum_a, sum_b) = dot_product_int4_packed_scalar(&a.data, &b.data, a.dims);
382    finish_int4_dot(raw_dot, sum_a, sum_b, a, b)
383}
384
385#[inline]
386fn finish_int4_dot(raw_dot: i32, sum_a: i32, sum_b: i32, a: &Int4Vector, b: &Int4Vector) -> f32 {
387    let raw_dot = raw_dot as f32;
388    let sum_a = sum_a as f32;
389    let sum_b = sum_b as f32;
390    let scale_a = a.params.scale;
391    let scale_b = b.params.scale;
392
393    raw_dot / (scale_a * scale_b)
394        - (b.params.max_abs * sum_a / scale_a)
395        - (a.params.max_abs * sum_b / scale_b)
396        + (a.dims as f32 * a.params.max_abs * b.params.max_abs)
397}
398
399#[inline]
400fn dot_product_int4_packed_scalar(a: &[u8], b: &[u8], dims: usize) -> (i32, i32, i32) {
401    let full_bytes = dims / 2;
402    let mut raw_dot = 0i32;
403    let mut sum_a = 0i32;
404    let mut sum_b = 0i32;
405
406    for i in 0..full_bytes {
407        let av = a[i];
408        let bv = b[i];
409        let ah = ((av >> 4) & 0x0f) as i32;
410        let al = (av & 0x0f) as i32;
411        let bh = ((bv >> 4) & 0x0f) as i32;
412        let bl = (bv & 0x0f) as i32;
413        raw_dot += ah * bh + al * bl;
414        sum_a += ah + al;
415        sum_b += bh + bl;
416    }
417
418    if dims % 2 == 1 {
419        let av = a[full_bytes];
420        let bv = b[full_bytes];
421        let ah = ((av >> 4) & 0x0f) as i32;
422        let bh = ((bv >> 4) & 0x0f) as i32;
423        raw_dot += ah * bh;
424        sum_a += ah;
425        sum_b += bh;
426    }
427
428    (raw_dot, sum_a, sum_b)
429}
430
431#[cfg(target_arch = "aarch64")]
432#[target_feature(enable = "neon")]
433#[inline]
434unsafe fn dot_product_int4_neon_unrolled(a: &[u8], b: &[u8], dims: usize) -> (i32, i32, i32) {
435    debug_assert!(a.len() >= dims.div_ceil(2));
436    debug_assert!(b.len() >= dims.div_ceil(2));
437
438    const BLOCK_BYTES: usize = 16;
439    const UNROLL: usize = 4;
440    const CHUNK_BYTES: usize = BLOCK_BYTES * UNROLL;
441
442    // Only bytes containing two valid dimensions are processed in SIMD.
443    // If dims is odd, the final high nibble is handled separately and the low
444    // padding nibble is ignored to preserve current to_f32 semantics.
445    let full_bytes = dims / 2;
446    let chunks = full_bytes / CHUNK_BYTES;
447
448    let mut raw0 = vdupq_n_u32(0);
449    let mut raw1 = vdupq_n_u32(0);
450    let mut raw2 = vdupq_n_u32(0);
451    let mut raw3 = vdupq_n_u32(0);
452    let mut sum_a = vdupq_n_u32(0);
453    let mut sum_b = vdupq_n_u32(0);
454    let mask = vdupq_n_u8(0x0f);
455
456    macro_rules! accumulate_block {
457        ($base:expr, $raw:ident) => {{
458            let a_bytes = vld1q_u8(a.as_ptr().add($base));
459            let b_bytes = vld1q_u8(b.as_ptr().add($base));
460
461            let a_hi = vshrq_n_u8::<4>(a_bytes);
462            let b_hi = vshrq_n_u8::<4>(b_bytes);
463            let a_lo = vandq_u8(a_bytes, mask);
464            let b_lo = vandq_u8(b_bytes, mask);
465
466            $raw = vpadalq_u16($raw, vmull_u8(vget_low_u8(a_hi), vget_low_u8(b_hi)));
467            $raw = vpadalq_u16($raw, vmull_u8(vget_high_u8(a_hi), vget_high_u8(b_hi)));
468            $raw = vpadalq_u16($raw, vmull_u8(vget_low_u8(a_lo), vget_low_u8(b_lo)));
469            $raw = vpadalq_u16($raw, vmull_u8(vget_high_u8(a_lo), vget_high_u8(b_lo)));
470
471            sum_a = vpadalq_u16(sum_a, vpaddlq_u8(a_hi));
472            sum_a = vpadalq_u16(sum_a, vpaddlq_u8(a_lo));
473            sum_b = vpadalq_u16(sum_b, vpaddlq_u8(b_hi));
474            sum_b = vpadalq_u16(sum_b, vpaddlq_u8(b_lo));
475        }};
476    }
477
478    for i in 0..chunks {
479        let base = i * CHUNK_BYTES;
480        accumulate_block!(base, raw0);
481        accumulate_block!(base + BLOCK_BYTES, raw1);
482        accumulate_block!(base + BLOCK_BYTES * 2, raw2);
483        accumulate_block!(base + BLOCK_BYTES * 3, raw3);
484    }
485
486    let raw_vec = vaddq_u32(vaddq_u32(raw0, raw1), vaddq_u32(raw2, raw3));
487    let mut raw_total = (vgetq_lane_u32::<0>(raw_vec)
488        + vgetq_lane_u32::<1>(raw_vec)
489        + vgetq_lane_u32::<2>(raw_vec)
490        + vgetq_lane_u32::<3>(raw_vec)) as i32;
491    let mut sum_a_total = (vgetq_lane_u32::<0>(sum_a)
492        + vgetq_lane_u32::<1>(sum_a)
493        + vgetq_lane_u32::<2>(sum_a)
494        + vgetq_lane_u32::<3>(sum_a)) as i32;
495    let mut sum_b_total = (vgetq_lane_u32::<0>(sum_b)
496        + vgetq_lane_u32::<1>(sum_b)
497        + vgetq_lane_u32::<2>(sum_b)
498        + vgetq_lane_u32::<3>(sum_b)) as i32;
499
500    let remainder_start = chunks * CHUNK_BYTES;
501    for byte_idx in remainder_start..full_bytes {
502        let av = *a.get_unchecked(byte_idx);
503        let bv = *b.get_unchecked(byte_idx);
504        let ah = ((av >> 4) & 0x0f) as i32;
505        let al = (av & 0x0f) as i32;
506        let bh = ((bv >> 4) & 0x0f) as i32;
507        let bl = (bv & 0x0f) as i32;
508
509        raw_total += ah * bh + al * bl;
510        sum_a_total += ah + al;
511        sum_b_total += bh + bl;
512    }
513
514    if dims % 2 == 1 {
515        let av = *a.get_unchecked(full_bytes);
516        let bv = *b.get_unchecked(full_bytes);
517        let ah = ((av >> 4) & 0x0f) as i32;
518        let bh = ((bv >> 4) & 0x0f) as i32;
519
520        raw_total += ah * bh;
521        sum_a_total += ah;
522        sum_b_total += bh;
523    }
524
525    (raw_total, sum_a_total, sum_b_total)
526}
527
528#[cfg(test)]
529mod tests;