1#[cfg(target_arch = "aarch64")]
8use std::arch::aarch64::*;
9
10#[cfg(target_arch = "x86_64")]
11use std::arch::x86_64::*;
12
13#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
14use super::simd_config;
15
16#[derive(Debug, Clone, Copy)]
23pub struct Int4Params {
24 pub scale: f32,
26 pub max_abs: f32,
28}
29
30impl Int4Params {
31 pub fn from_vector(vector: &[f32]) -> Self {
33 let max_abs = max_abs_finite(vector);
34
35 let scale = if max_abs > 1e-10 {
37 15.0 / (2.0 * max_abs)
38 } else {
39 1.0
40 };
41
42 Self { scale, max_abs }
43 }
44}
45
46fn max_abs_finite(vector: &[f32]) -> f32 {
47 #[cfg(target_arch = "x86_64")]
48 {
49 if simd_config().avx2_enabled {
50 return unsafe { max_abs_finite_avx2(vector) };
52 }
53 }
54 #[cfg(target_arch = "aarch64")]
55 {
56 if simd_config().neon_enabled {
57 return unsafe { max_abs_finite_neon(vector) };
59 }
60 }
61 max_abs_finite_scalar(vector)
62}
63
64fn max_abs_finite_scalar(vector: &[f32]) -> f32 {
65 vector
66 .iter()
67 .filter(|value| value.is_finite())
68 .map(|value| value.abs())
69 .fold(0.0, f32::max)
70}
71
72#[cfg(test)]
73thread_local! {
74 static INT4_MAX_ABS_SIMD_HITS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
75}
76
77#[cfg(target_arch = "x86_64")]
78#[target_feature(enable = "avx2")]
79unsafe fn max_abs_finite_avx2(vector: &[f32]) -> f32 {
80 #[cfg(test)]
81 INT4_MAX_ABS_SIMD_HITS.with(|hits| hits.set(hits.get() + 1));
82
83 let chunks = vector.len() / 8;
84 let sign = _mm256_set1_ps(-0.0);
85 let inf = _mm256_set1_ps(f32::INFINITY);
86 let mut maximum = _mm256_setzero_ps();
87
88 for i in 0..chunks {
89 let input = _mm256_loadu_ps(vector.as_ptr().add(i * 8));
90 let abs = _mm256_andnot_ps(sign, input);
91 let finite = _mm256_cmp_ps(abs, inf, _CMP_LT_OQ);
92 maximum = _mm256_max_ps(maximum, _mm256_and_ps(abs, finite));
93 }
94
95 let mut lanes = [0.0f32; 8];
96 _mm256_storeu_ps(lanes.as_mut_ptr(), maximum);
97 let mut max_abs = lanes.into_iter().fold(0.0f32, f32::max);
98 for &value in &vector[chunks * 8..] {
99 if value.is_finite() {
100 max_abs = max_abs.max(value.abs());
101 }
102 }
103 max_abs
104}
105
106#[cfg(target_arch = "aarch64")]
107#[target_feature(enable = "neon")]
108unsafe fn max_abs_finite_neon(vector: &[f32]) -> f32 {
109 #[cfg(test)]
110 INT4_MAX_ABS_SIMD_HITS.with(|hits| hits.set(hits.get() + 1));
111
112 let chunks = vector.len() / 4;
113 let inf = vdupq_n_f32(f32::INFINITY);
114 let zero = vdupq_n_f32(0.0);
115 let mut maximum = zero;
116
117 for i in 0..chunks {
118 let input = vld1q_f32(vector.as_ptr().add(i * 4));
119 let abs = vabsq_f32(input);
120 let finite = vcltq_f32(abs, inf);
121 maximum = vmaxq_f32(maximum, vbslq_f32(finite, abs, zero));
122 }
123
124 let mut max_abs = vmaxvq_f32(maximum);
125 for &value in &vector[chunks * 4..] {
126 if value.is_finite() {
127 max_abs = max_abs.max(value.abs());
128 }
129 }
130 max_abs
131}
132
133#[derive(Debug, Clone)]
137pub struct Int4Vector {
138 pub data: Vec<u8>,
140 pub dims: usize,
142 pub params: Int4Params,
144 pub norm: f32,
146}
147
148impl Int4Vector {
149 pub fn from_f32(vector: &[f32]) -> Self {
155 let params = Int4Params::from_vector(vector);
156 let dims = vector.len();
157
158 let mut norm_sq = 0.0f32;
160 for &v in vector {
161 if v.is_finite() {
162 norm_sq += v * v;
163 }
164 }
165 let norm = norm_sq.sqrt();
166
167 let data = quantize_int4(vector, params);
168
169 Self {
170 data,
171 dims,
172 params,
173 norm,
174 }
175 }
176
177 pub fn to_f32(&self) -> Vec<f32> {
181 let required_bytes = self.dims.div_ceil(2);
182 if self.data.len() < required_bytes {
183 return Vec::new();
184 }
185
186 let scale = if self.params.scale.is_finite() && self.params.scale != 0.0 {
187 self.params.scale
188 } else {
189 1.0
190 };
191
192 let mut result = Vec::with_capacity(self.dims);
193 for i in 0..self.dims {
194 let byte_idx = i / 2;
195 let q = if i % 2 == 0 {
196 (self.data[byte_idx] >> 4) & 0x0F
197 } else {
198 self.data[byte_idx] & 0x0F
199 };
200 result.push(q as f32 / scale - self.params.max_abs);
201 }
202 result
203 }
204
205 #[inline]
209 pub fn dot_product(&self, other: &Int4Vector) -> f32 {
210 dot_product_int4(self, other)
211 }
212
213 #[inline]
215 pub fn cosine_similarity(&self, other: &Int4Vector) -> f32 {
216 let denom = self.norm * other.norm;
217 if denom == 0.0 || !denom.is_finite() {
218 return 0.0;
219 }
220 self.dot_product(other) / denom
221 }
222
223 #[inline]
225 pub fn cosine_distance(&self, other: &Int4Vector) -> f32 {
226 1.0 - self.cosine_similarity(other)
227 }
228}
229
230fn quantize_int4(vector: &[f32], params: Int4Params) -> Vec<u8> {
231 #[cfg(target_arch = "x86_64")]
232 {
233 if simd_config().avx2_enabled {
234 return unsafe { quantize_int4_avx2(vector, params) };
236 }
237 }
238 #[cfg(target_arch = "aarch64")]
239 {
240 if simd_config().neon_enabled {
241 return unsafe { quantize_int4_neon(vector, params) };
243 }
244 }
245 quantize_int4_scalar(vector, params)
246}
247
248fn quantize_int4_scalar(vector: &[f32], params: Int4Params) -> Vec<u8> {
249 let mut data = vec![0u8; vector.len().div_ceil(2)];
250 quantize_int4_scalar_tail(vector, params, &mut data, 0);
251 data
252}
253
254fn quantize_int4_scalar_tail(vector: &[f32], params: Int4Params, data: &mut [u8], start: usize) {
255 for (i, &value) in vector.iter().enumerate().skip(start) {
256 let quantized = quantize_int4_value(value, params);
257 if i % 2 == 0 {
258 data[i / 2] |= quantized << 4;
259 } else {
260 data[i / 2] |= quantized;
261 }
262 }
263}
264
265#[inline]
266fn quantize_int4_value(value: f32, params: Int4Params) -> u8 {
267 let finite_value = if value.is_finite() { value } else { 0.0 };
268 ((finite_value + params.max_abs) * params.scale)
269 .round()
270 .clamp(0.0, 15.0) as u8
271}
272
273#[cfg(test)]
274thread_local! {
275 static INT4_QUANTIZE_SIMD_HITS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
276}
277
278#[cfg(target_arch = "x86_64")]
279#[target_feature(enable = "avx2")]
280unsafe fn quantize_int4_avx2(vector: &[f32], params: Int4Params) -> Vec<u8> {
281 #[cfg(test)]
282 INT4_QUANTIZE_SIMD_HITS.with(|hits| hits.set(hits.get() + 1));
283
284 let mut data = vec![0u8; vector.len().div_ceil(2)];
285 let chunks = vector.len() / 8;
286 let sign = _mm256_set1_ps(-0.0);
287 let inf = _mm256_set1_ps(f32::INFINITY);
288 let max_abs = _mm256_set1_ps(params.max_abs);
289 let scale = _mm256_set1_ps(params.scale);
290 let zero = _mm256_setzero_ps();
291 let high = _mm256_set1_ps(15.0);
292 let half = _mm256_set1_ps(0.5);
293 let one = _mm256_set1_epi32(1);
294
295 for i in 0..chunks {
296 let base = i * 8;
297 let input = _mm256_loadu_ps(vector.as_ptr().add(base));
298 let abs = _mm256_andnot_ps(sign, input);
299 let finite = _mm256_cmp_ps(abs, inf, _CMP_LT_OQ);
300 let values = _mm256_and_ps(input, finite);
301 let scaled = _mm256_mul_ps(_mm256_add_ps(values, max_abs), scale);
302 let clamped = _mm256_min_ps(_mm256_max_ps(scaled, zero), high);
303 let truncated = _mm256_cvttps_epi32(clamped);
304 let fraction = _mm256_sub_ps(clamped, _mm256_cvtepi32_ps(truncated));
305 let round_up = _mm256_castps_si256(_mm256_cmp_ps(fraction, half, _CMP_GE_OQ));
306 let rounded = _mm256_add_epi32(truncated, _mm256_and_si256(round_up, one));
307 let mut lanes = [0i32; 8];
308 _mm256_storeu_si256(lanes.as_mut_ptr().cast::<__m256i>(), rounded);
309 for pair in 0..4 {
310 data[base / 2 + pair] = ((lanes[pair * 2] as u8) << 4) | lanes[pair * 2 + 1] as u8;
311 }
312 }
313
314 quantize_int4_scalar_tail(vector, params, &mut data, chunks * 8);
315 data
316}
317
318#[cfg(target_arch = "aarch64")]
319#[target_feature(enable = "neon")]
320unsafe fn quantize_int4_neon(vector: &[f32], params: Int4Params) -> Vec<u8> {
321 #[cfg(test)]
322 INT4_QUANTIZE_SIMD_HITS.with(|hits| hits.set(hits.get() + 1));
323
324 let mut data = vec![0u8; vector.len().div_ceil(2)];
325 let chunks = vector.len() / 4;
326 let inf = vdupq_n_f32(f32::INFINITY);
327 let zero = vdupq_n_f32(0.0);
328 let max_abs = vdupq_n_f32(params.max_abs);
329 let scale = vdupq_n_f32(params.scale);
330 let high = vdupq_n_f32(15.0);
331
332 for i in 0..chunks {
333 let base = i * 4;
334 let input = vld1q_f32(vector.as_ptr().add(base));
335 let finite = vcaltq_f32(input, inf);
336 let values = vbslq_f32(finite, input, zero);
337 let scaled = vmulq_f32(vaddq_f32(values, max_abs), scale);
338 let clamped = vminq_f32(vmaxq_f32(scaled, zero), high);
339 let rounded = vcvtaq_s32_f32(clamped);
340 let mut lanes = [0i32; 4];
341 vst1q_s32(lanes.as_mut_ptr(), rounded);
342 data[base / 2] = ((lanes[0] as u8) << 4) | lanes[1] as u8;
343 data[base / 2 + 1] = ((lanes[2] as u8) << 4) | lanes[3] as u8;
344 }
345
346 quantize_int4_scalar_tail(vector, params, &mut data, chunks * 4);
347 data
348}
349
350#[inline]
352pub fn dot_product_int4(a: &Int4Vector, b: &Int4Vector) -> f32 {
353 if a.dims != b.dims {
354 return 0.0;
355 }
356
357 let scale_a = a.params.scale;
358 let scale_b = b.params.scale;
359 if scale_a == 0.0 || scale_b == 0.0 || !scale_a.is_finite() || !scale_b.is_finite() {
360 return 0.0;
361 }
362
363 let packed_len = a.dims.div_ceil(2);
364 if a.data.len() < packed_len || b.data.len() < packed_len {
365 return 0.0;
366 }
367
368 #[cfg(target_arch = "aarch64")]
369 {
370 let config = simd_config();
371 if config.neon_enabled {
372 let (raw_dot, sum_a, sum_b) =
376 unsafe { dot_product_int4_neon_unrolled(&a.data, &b.data, a.dims) };
377 return finish_int4_dot(raw_dot, sum_a, sum_b, a, b);
378 }
379 }
380
381 let (raw_dot, sum_a, sum_b) = dot_product_int4_packed_scalar(&a.data, &b.data, a.dims);
382 finish_int4_dot(raw_dot, sum_a, sum_b, a, b)
383}
384
385#[inline]
386fn finish_int4_dot(raw_dot: i32, sum_a: i32, sum_b: i32, a: &Int4Vector, b: &Int4Vector) -> f32 {
387 let raw_dot = raw_dot as f32;
388 let sum_a = sum_a as f32;
389 let sum_b = sum_b as f32;
390 let scale_a = a.params.scale;
391 let scale_b = b.params.scale;
392
393 raw_dot / (scale_a * scale_b)
394 - (b.params.max_abs * sum_a / scale_a)
395 - (a.params.max_abs * sum_b / scale_b)
396 + (a.dims as f32 * a.params.max_abs * b.params.max_abs)
397}
398
399#[inline]
400fn dot_product_int4_packed_scalar(a: &[u8], b: &[u8], dims: usize) -> (i32, i32, i32) {
401 let full_bytes = dims / 2;
402 let mut raw_dot = 0i32;
403 let mut sum_a = 0i32;
404 let mut sum_b = 0i32;
405
406 for i in 0..full_bytes {
407 let av = a[i];
408 let bv = b[i];
409 let ah = ((av >> 4) & 0x0f) as i32;
410 let al = (av & 0x0f) as i32;
411 let bh = ((bv >> 4) & 0x0f) as i32;
412 let bl = (bv & 0x0f) as i32;
413 raw_dot += ah * bh + al * bl;
414 sum_a += ah + al;
415 sum_b += bh + bl;
416 }
417
418 if dims % 2 == 1 {
419 let av = a[full_bytes];
420 let bv = b[full_bytes];
421 let ah = ((av >> 4) & 0x0f) as i32;
422 let bh = ((bv >> 4) & 0x0f) as i32;
423 raw_dot += ah * bh;
424 sum_a += ah;
425 sum_b += bh;
426 }
427
428 (raw_dot, sum_a, sum_b)
429}
430
431#[cfg(target_arch = "aarch64")]
432#[target_feature(enable = "neon")]
433#[inline]
434unsafe fn dot_product_int4_neon_unrolled(a: &[u8], b: &[u8], dims: usize) -> (i32, i32, i32) {
435 debug_assert!(a.len() >= dims.div_ceil(2));
436 debug_assert!(b.len() >= dims.div_ceil(2));
437
438 const BLOCK_BYTES: usize = 16;
439 const UNROLL: usize = 4;
440 const CHUNK_BYTES: usize = BLOCK_BYTES * UNROLL;
441
442 let full_bytes = dims / 2;
446 let chunks = full_bytes / CHUNK_BYTES;
447
448 let mut raw0 = vdupq_n_u32(0);
449 let mut raw1 = vdupq_n_u32(0);
450 let mut raw2 = vdupq_n_u32(0);
451 let mut raw3 = vdupq_n_u32(0);
452 let mut sum_a = vdupq_n_u32(0);
453 let mut sum_b = vdupq_n_u32(0);
454 let mask = vdupq_n_u8(0x0f);
455
456 macro_rules! accumulate_block {
457 ($base:expr, $raw:ident) => {{
458 let a_bytes = vld1q_u8(a.as_ptr().add($base));
459 let b_bytes = vld1q_u8(b.as_ptr().add($base));
460
461 let a_hi = vshrq_n_u8::<4>(a_bytes);
462 let b_hi = vshrq_n_u8::<4>(b_bytes);
463 let a_lo = vandq_u8(a_bytes, mask);
464 let b_lo = vandq_u8(b_bytes, mask);
465
466 $raw = vpadalq_u16($raw, vmull_u8(vget_low_u8(a_hi), vget_low_u8(b_hi)));
467 $raw = vpadalq_u16($raw, vmull_u8(vget_high_u8(a_hi), vget_high_u8(b_hi)));
468 $raw = vpadalq_u16($raw, vmull_u8(vget_low_u8(a_lo), vget_low_u8(b_lo)));
469 $raw = vpadalq_u16($raw, vmull_u8(vget_high_u8(a_lo), vget_high_u8(b_lo)));
470
471 sum_a = vpadalq_u16(sum_a, vpaddlq_u8(a_hi));
472 sum_a = vpadalq_u16(sum_a, vpaddlq_u8(a_lo));
473 sum_b = vpadalq_u16(sum_b, vpaddlq_u8(b_hi));
474 sum_b = vpadalq_u16(sum_b, vpaddlq_u8(b_lo));
475 }};
476 }
477
478 for i in 0..chunks {
479 let base = i * CHUNK_BYTES;
480 accumulate_block!(base, raw0);
481 accumulate_block!(base + BLOCK_BYTES, raw1);
482 accumulate_block!(base + BLOCK_BYTES * 2, raw2);
483 accumulate_block!(base + BLOCK_BYTES * 3, raw3);
484 }
485
486 let raw_vec = vaddq_u32(vaddq_u32(raw0, raw1), vaddq_u32(raw2, raw3));
487 let mut raw_total = (vgetq_lane_u32::<0>(raw_vec)
488 + vgetq_lane_u32::<1>(raw_vec)
489 + vgetq_lane_u32::<2>(raw_vec)
490 + vgetq_lane_u32::<3>(raw_vec)) as i32;
491 let mut sum_a_total = (vgetq_lane_u32::<0>(sum_a)
492 + vgetq_lane_u32::<1>(sum_a)
493 + vgetq_lane_u32::<2>(sum_a)
494 + vgetq_lane_u32::<3>(sum_a)) as i32;
495 let mut sum_b_total = (vgetq_lane_u32::<0>(sum_b)
496 + vgetq_lane_u32::<1>(sum_b)
497 + vgetq_lane_u32::<2>(sum_b)
498 + vgetq_lane_u32::<3>(sum_b)) as i32;
499
500 let remainder_start = chunks * CHUNK_BYTES;
501 for byte_idx in remainder_start..full_bytes {
502 let av = *a.get_unchecked(byte_idx);
503 let bv = *b.get_unchecked(byte_idx);
504 let ah = ((av >> 4) & 0x0f) as i32;
505 let al = (av & 0x0f) as i32;
506 let bh = ((bv >> 4) & 0x0f) as i32;
507 let bl = (bv & 0x0f) as i32;
508
509 raw_total += ah * bh + al * bl;
510 sum_a_total += ah + al;
511 sum_b_total += bh + bl;
512 }
513
514 if dims % 2 == 1 {
515 let av = *a.get_unchecked(full_bytes);
516 let bv = *b.get_unchecked(full_bytes);
517 let ah = ((av >> 4) & 0x0f) as i32;
518 let bh = ((bv >> 4) & 0x0f) as i32;
519
520 raw_total += ah * bh;
521 sum_a_total += ah;
522 sum_b_total += bh;
523 }
524
525 (raw_total, sum_a_total, sum_b_total)
526}
527
528#[cfg(test)]
529mod tests;