Skip to main content

simd_vector_engine/kernels/
avx512.rs

1// AVX-512, 512-битные ZMM, 16 лейнов по 32 бита.
2// Вызывать только после is_x86_feature_detected!("avx512f").
3// Сравнение возвращает __mmask16, а не вектор из 0/-1, поэтому movemask не нужен.
4
5#[cfg(target_arch = "x86")]
6use core::arch::x86::*;
7#[cfg(target_arch = "x86_64")]
8use core::arch::x86_64::*;
9
10use crate::aligned::is_aligned;
11
12#[inline]
13#[target_feature(enable = "avx512f")]
14unsafe fn load_i32x16(p: *const i32, aligned: bool) -> __m512i {
15    if aligned {
16        (p as *const __m512i).read() // vmovdqa64
17    } else {
18        (p as *const __m512i).read_unaligned() // vmovdqu64
19    }
20}
21
22#[inline]
23#[target_feature(enable = "avx512f")]
24unsafe fn load_f32x16(p: *const f32, aligned: bool) -> __m512 {
25    if aligned {
26        (p as *const __m512).read() // vmovaps
27    } else {
28        (p as *const __m512).read_unaligned() // vmovups
29    }
30}
31
32// 64 элемента за итерацию = 4 кэш-линии
33/// # Safety
34///
35/// The `avx512f` target feature must be available on the current CPU. Use
36/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
37/// calling this directly.
38#[target_feature(enable = "avx512f")]
39pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
40    let n = data.len();
41    let p = data.as_ptr();
42    let aligned = is_aligned(p, 64);
43
44    let thr = _mm512_set1_epi32(threshold); // vpbroadcastd zmm
45
46    let mut c0 = 0u64;
47    let mut c1 = 0u64;
48    let mut c2 = 0u64;
49    let mut c3 = 0u64;
50
51    let mut i = 0usize;
52    let block = n & !63usize;
53    while i < block {
54        let v0 = load_i32x16(p.add(i), aligned);
55        let v1 = load_i32x16(p.add(i + 16), aligned);
56        let v2 = load_i32x16(p.add(i + 32), aligned);
57        let v3 = load_i32x16(p.add(i + 48), aligned);
58
59        // vpcmpd k, zmm, zmm, 6 (imm 6 == _MM_CMPINT_NLE)
60        let m0: __mmask16 = _mm512_cmpgt_epi32_mask(v0, thr);
61        let m1: __mmask16 = _mm512_cmpgt_epi32_mask(v1, thr);
62        let m2: __mmask16 = _mm512_cmpgt_epi32_mask(v2, thr);
63        let m3: __mmask16 = _mm512_cmpgt_epi32_mask(v3, thr);
64
65        // kmovw + popcnt, счёт идёт в GPR
66        c0 += (m0 as u32).count_ones() as u64;
67        c1 += (m1 as u32).count_ones() as u64;
68        c2 += (m2 as u32).count_ones() as u64;
69        c3 += (m3 as u32).count_ones() as u64;
70
71        i += 64;
72    }
73
74    let mut total = (c0 + c1 + c2 + c3) as usize;
75    while i < n {
76        total += (*p.add(i) > threshold) as usize;
77        i += 1;
78    }
79    total
80}
81
82/// # Safety
83///
84/// The `avx512f` target feature must be available on the current CPU. Use
85/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
86/// calling this directly.
87#[target_feature(enable = "avx512f")]
88pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
89    let n = data.len();
90    let p = data.as_ptr();
91    let aligned = is_aligned(p, 64);
92
93    let thr = _mm512_set1_ps(threshold); // vbroadcastss zmm
94    let mut c0 = 0u64;
95    let mut c1 = 0u64;
96    let mut c2 = 0u64;
97    let mut c3 = 0u64;
98
99    let mut i = 0usize;
100    let block = n & !63usize;
101    while i < block {
102        let v0 = load_f32x16(p.add(i), aligned);
103        let v1 = load_f32x16(p.add(i + 16), aligned);
104        let v2 = load_f32x16(p.add(i + 32), aligned);
105        let v3 = load_f32x16(p.add(i + 48), aligned);
106
107        // vcmpps k, zmm, zmm, _CMP_GT_OQ — NaN даёт false, как `>` в Rust
108        let m0 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v0, thr);
109        let m1 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v1, thr);
110        let m2 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v2, thr);
111        let m3 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v3, thr);
112
113        c0 += (m0 as u32).count_ones() as u64;
114        c1 += (m1 as u32).count_ones() as u64;
115        c2 += (m2 as u32).count_ones() as u64;
116        c3 += (m3 as u32).count_ones() as u64;
117
118        i += 64;
119    }
120
121    let mut total = (c0 + c1 + c2 + c3) as usize;
122    while i < n {
123        total += (*p.add(i) > threshold) as usize;
124        i += 1;
125    }
126    total
127}
128
129/// # Safety
130///
131/// The `avx512f` target feature must be available on the current CPU. Use
132/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
133/// calling this directly.
134#[target_feature(enable = "avx512f")]
135pub unsafe fn sum_i32(data: &[i32]) -> i64 {
136    let n = data.len();
137    let p = data.as_ptr();
138    let aligned = is_aligned(p, 64);
139
140    let mut a0 = _mm512_setzero_si512(); // 8 × i64
141    let mut a1 = _mm512_setzero_si512();
142    let mut a2 = _mm512_setzero_si512();
143    let mut a3 = _mm512_setzero_si512();
144
145    let mut i = 0usize;
146    let block = n & !31usize;
147    while i < block {
148        let v0 = load_i32x16(p.add(i), aligned);
149        let v1 = load_i32x16(p.add(i + 16), aligned);
150
151        // vpmovsxdq: 8 × i32 -> 8 × i64; верхняя половина через vextracti64x4
152        let lo0 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v0));
153        let hi0 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(v0));
154        let lo1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v1));
155        let hi1 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(v1));
156
157        // vpaddq zmm
158        a0 = _mm512_add_epi64(a0, lo0);
159        a1 = _mm512_add_epi64(a1, hi0);
160        a2 = _mm512_add_epi64(a2, lo1);
161        a3 = _mm512_add_epi64(a3, hi1);
162
163        i += 32;
164    }
165
166    let acc = _mm512_add_epi64(_mm512_add_epi64(a0, a1), _mm512_add_epi64(a2, a3));
167    // Не одна инструкция, а секвенция shuffle+add, но один раз на весь массив.
168    let mut total = _mm512_reduce_add_epi64(acc);
169
170    while i < n {
171        total += *p.add(i) as i64;
172        i += 1;
173    }
174    total
175}
176
177/// Индекс первого вхождения. 4 маски клеятся в u64 и проверяются одним test.
178/// # Safety
179///
180/// The `avx512f` target feature must be available on the current CPU. Use
181/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
182/// calling this directly.
183#[target_feature(enable = "avx512f")]
184pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
185    let n = data.len();
186    let p = data.as_ptr();
187    let aligned = is_aligned(p, 64);
188    let key = _mm512_set1_epi32(needle);
189
190    let mut i = 0usize;
191    let block = n & !63usize;
192    while i < block {
193        let v0 = load_i32x16(p.add(i), aligned);
194        let v1 = load_i32x16(p.add(i + 16), aligned);
195        let v2 = load_i32x16(p.add(i + 32), aligned);
196        let v3 = load_i32x16(p.add(i + 48), aligned);
197
198        // vpcmpeqd k, zmm, zmm -> __mmask16
199        let m0 = _mm512_cmpeq_epi32_mask(v0, key);
200        let m1 = _mm512_cmpeq_epi32_mask(v1, key);
201        let m2 = _mm512_cmpeq_epi32_mask(v2, key);
202        let m3 = _mm512_cmpeq_epi32_mask(v3, key);
203
204        let bits = (m0 as u64) | ((m1 as u64) << 16) | ((m2 as u64) << 32) | ((m3 as u64) << 48);
205        if bits != 0 {
206            // tzcnt даёт смещение внутри 64-элементного блока
207            return Some(i + bits.trailing_zeros() as usize);
208        }
209        i += 64;
210    }
211
212    while i < n {
213        if *p.add(i) == needle {
214            return Some(i);
215        }
216        i += 1;
217    }
218    None
219}
220
221/// Compaction через `vpcompressd`: одна инструкция вместо 8 КБ LUT и vpermd из AVX2.
222///
223/// Пишет ровно popcnt элементов, без перелива за курсор.
224/// Размер `out` обеспечивает вызывающий (см. `crate::parallel`).
225/// # Safety
226///
227/// The `avx512f` target feature must be available on the current CPU. Use
228/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
229/// calling this directly.
230#[target_feature(enable = "avx512f")]
231pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
232    debug_assert!(data.is_empty() || !out.is_empty());
233    let n = data.len();
234    let p = data.as_ptr();
235    let o = out.as_mut_ptr();
236    let aligned = is_aligned(p, 64);
237    let thr = _mm512_set1_epi32(threshold);
238
239    let mut w = 0usize;
240    let mut i = 0usize;
241    let block = n & !15usize;
242    while i < block {
243        let v = load_i32x16(p.add(i), aligned);
244        let m = _mm512_cmpgt_epi32_mask(v, thr);
245        // vpcompressd [mem]{k}, zmm: пишет только отмеченные лейны, плотно
246        _mm512_mask_compressstoreu_epi32(o.add(w), m, v);
247        w += (m as u32).count_ones() as usize;
248        i += 16;
249    }
250
251    while i < n {
252        let x = *p.add(i);
253        if x > threshold {
254            *o.add(w) = x;
255            w += 1;
256        }
257        i += 1;
258    }
259    w
260}