Skip to main content

simd_vector_engine/kernels/
avx2.rs

1// AVX2, 256-битные YMM, 8 лейнов по 32 бита.
2// Вызывать только после is_x86_feature_detected!("avx2").
3// Развёртка на 4 вектора: у vpaddd латентность 1, throughput 2-3/такт,
4// с одним аккумулятором цикл упирается в зависимость, а не в память.
5
6#[cfg(target_arch = "x86")]
7use core::arch::x86::*;
8#[cfg(target_arch = "x86_64")]
9use core::arch::x86_64::*;
10
11use crate::aligned::is_aligned;
12
13#[inline]
14#[target_feature(enable = "avx2")]
15unsafe fn hsum_epi32(v: __m256i) -> i32 {
16    let hi = _mm256_extracti128_si256::<1>(v); // vextracti128
17    let lo = _mm256_castsi256_si128(v);
18    let s = _mm_add_epi32(lo, hi); // vpaddd, 8 -> 4
19    let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b01_00_11_10>(s)); // 4 -> 2
20    let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b10_11_00_01>(s)); // 2 -> 1
21    _mm_cvtsi128_si32(s) // vmovd
22}
23
24#[inline]
25#[target_feature(enable = "avx2")]
26unsafe fn hsum_epi64(v: __m256i) -> i64 {
27    let mut tmp = [0i64; 4];
28    _mm256_storeu_si256(tmp.as_mut_ptr() as *mut __m256i, v);
29    tmp[0] + tmp[1] + tmp[2] + tmp[3]
30}
31
32#[inline]
33#[target_feature(enable = "avx2")]
34unsafe fn load_i32x8(p: *const i32, aligned: bool) -> __m256i {
35    if aligned {
36        _mm256_load_si256(p as *const __m256i) // vmovdqa
37    } else {
38        _mm256_loadu_si256(p as *const __m256i) // vmovdqu
39    }
40}
41
42#[inline]
43#[target_feature(enable = "avx2")]
44unsafe fn load_f32x8(p: *const f32, aligned: bool) -> __m256 {
45    if aligned {
46        _mm256_load_ps(p) // vmovaps
47    } else {
48        _mm256_loadu_ps(p) // vmovups
49    }
50}
51
52/// # Safety
53///
54/// The `avx2` target feature must be available on the current CPU. Use
55/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
56/// calling this directly.
57#[target_feature(enable = "avx2")]
58pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
59    let n = data.len();
60    let p = data.as_ptr();
61    let aligned = is_aligned(p, 32);
62
63    let thr = _mm256_set1_epi32(threshold); // vpbroadcastd
64
65    let mut a0 = _mm256_setzero_si256();
66    let mut a1 = _mm256_setzero_si256();
67    let mut a2 = _mm256_setzero_si256();
68    let mut a3 = _mm256_setzero_si256();
69
70    let mut i = 0usize;
71    let block = n & !31usize;
72    while i < block {
73        let v0 = load_i32x8(p.add(i), aligned);
74        let v1 = load_i32x8(p.add(i + 8), aligned);
75        let v2 = load_i32x8(p.add(i + 16), aligned);
76        let v3 = load_i32x8(p.add(i + 24), aligned);
77
78        // vpcmpgtd, маска 0 / -1
79        let m0 = _mm256_cmpgt_epi32(v0, thr);
80        let m1 = _mm256_cmpgt_epi32(v1, thr);
81        let m2 = _mm256_cmpgt_epi32(v2, thr);
82        let m3 = _mm256_cmpgt_epi32(v3, thr);
83
84        // vpsubd: acc - (-1) == acc + 1
85        a0 = _mm256_sub_epi32(a0, m0);
86        a1 = _mm256_sub_epi32(a1, m1);
87        a2 = _mm256_sub_epi32(a2, m2);
88        a3 = _mm256_sub_epi32(a3, m3);
89
90        i += 32;
91    }
92
93    let acc = _mm256_add_epi32(_mm256_add_epi32(a0, a1), _mm256_add_epi32(a2, a3));
94    let mut total = hsum_epi32(acc) as usize;
95
96    while i < n {
97        total += (*p.add(i) > threshold) as usize;
98        i += 1;
99    }
100    total
101}
102
103/// # Safety
104///
105/// The `avx2` target feature must be available on the current CPU. Use
106/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
107/// calling this directly.
108#[target_feature(enable = "avx2")]
109pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
110    let n = data.len();
111    let p = data.as_ptr();
112    let aligned = is_aligned(p, 32);
113
114    let thr = _mm256_set1_ps(threshold); // vbroadcastss
115    let mut c0 = 0u32;
116    let mut c1 = 0u32;
117    let mut c2 = 0u32;
118    let mut c3 = 0u32;
119
120    let mut i = 0usize;
121    let block = n & !31usize;
122    while i < block {
123        let v0 = load_f32x8(p.add(i), aligned);
124        let v1 = load_f32x8(p.add(i + 8), aligned);
125        let v2 = load_f32x8(p.add(i + 16), aligned);
126        let v3 = load_f32x8(p.add(i + 24), aligned);
127
128        // vcmpps, _CMP_GT_OQ: NaN даёт false, как `>` в Rust
129        let m0 = _mm256_cmp_ps::<_CMP_GT_OQ>(v0, thr);
130        let m1 = _mm256_cmp_ps::<_CMP_GT_OQ>(v1, thr);
131        let m2 = _mm256_cmp_ps::<_CMP_GT_OQ>(v2, thr);
132        let m3 = _mm256_cmp_ps::<_CMP_GT_OQ>(v3, thr);
133
134        // vmovmskps + popcnt
135        c0 += (_mm256_movemask_ps(m0) as u32).count_ones();
136        c1 += (_mm256_movemask_ps(m1) as u32).count_ones();
137        c2 += (_mm256_movemask_ps(m2) as u32).count_ones();
138        c3 += (_mm256_movemask_ps(m3) as u32).count_ones();
139
140        i += 32;
141    }
142
143    let mut total = (c0 + c1 + c2 + c3) as usize;
144    while i < n {
145        total += (*p.add(i) > threshold) as usize;
146        i += 1;
147    }
148    total
149}
150
151/// # Safety
152///
153/// The `avx2` target feature must be available on the current CPU. Use
154/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
155/// calling this directly.
156#[target_feature(enable = "avx2")]
157pub unsafe fn sum_i32(data: &[i32]) -> i64 {
158    let n = data.len();
159    let p = data.as_ptr();
160    let aligned = is_aligned(p, 32);
161
162    // аккумуляторы в i64: 100M × i32 не влезут в 32 бита
163    let mut a0 = _mm256_setzero_si256();
164    let mut a1 = _mm256_setzero_si256();
165    let mut a2 = _mm256_setzero_si256();
166    let mut a3 = _mm256_setzero_si256();
167
168    let mut i = 0usize;
169    let block = n & !15usize;
170    while i < block {
171        let v0 = load_i32x8(p.add(i), aligned);
172        let v1 = load_i32x8(p.add(i + 8), aligned);
173
174        // vpmovsxdq: 4 × i32 -> 4 × i64
175        let lo0 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v0));
176        let hi0 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(v0));
177        let lo1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v1));
178        let hi1 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(v1));
179
180        // vpaddq
181        a0 = _mm256_add_epi64(a0, lo0);
182        a1 = _mm256_add_epi64(a1, hi0);
183        a2 = _mm256_add_epi64(a2, lo1);
184        a3 = _mm256_add_epi64(a3, hi1);
185
186        i += 16;
187    }
188
189    let acc = _mm256_add_epi64(_mm256_add_epi64(a0, a1), _mm256_add_epi64(a2, a3));
190    let mut total = hsum_epi64(acc);
191    while i < n {
192        total += *p.add(i) as i64;
193        i += 1;
194    }
195    total
196}
197
198/// # Safety
199///
200/// The `avx2` target feature must be available on the current CPU. Use
201/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
202/// calling this directly.
203#[target_feature(enable = "avx2")]
204pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
205    let n = data.len();
206    let p = data.as_ptr();
207    let aligned = is_aligned(p, 32);
208    let key = _mm256_set1_epi32(needle);
209
210    let mut i = 0usize;
211    let block = n & !31usize;
212    while i < block {
213        let v0 = load_i32x8(p.add(i), aligned);
214        let v1 = load_i32x8(p.add(i + 8), aligned);
215        let v2 = load_i32x8(p.add(i + 16), aligned);
216        let v3 = load_i32x8(p.add(i + 24), aligned);
217
218        // vpcmpeqd
219        let m0 = _mm256_cmpeq_epi32(v0, key);
220        let m1 = _mm256_cmpeq_epi32(v1, key);
221        let m2 = _mm256_cmpeq_epi32(v2, key);
222        let m3 = _mm256_cmpeq_epi32(v3, key);
223
224        let any = _mm256_or_si256(_mm256_or_si256(m0, m1), _mm256_or_si256(m2, m3));
225
226        // vptest: ZF=1 — совпадений нет
227        if _mm256_testz_si256(any, any) == 0 {
228            for (k, m) in [m0, m1, m2, m3].into_iter().enumerate() {
229                let bits = _mm256_movemask_ps(_mm256_castsi256_ps(m)) as u32;
230                if bits != 0 {
231                    return Some(i + k * 8 + bits.trailing_zeros() as usize);
232                }
233            }
234        }
235        i += 32;
236    }
237
238    while i < n {
239        if *p.add(i) == needle {
240            return Some(i);
241        }
242        i += 1;
243    }
244    None
245}
246
247/// `out` должен вмещать результат плюс 7 элементов запаса (см. parallel::fill_exact).
248/// # Safety
249///
250/// The `avx2` target feature must be available on the current CPU. Use
251/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
252/// calling this directly.
253#[target_feature(enable = "avx2")]
254pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
255    debug_assert!(data.is_empty() || !out.is_empty());
256    let n = data.len();
257    let p = data.as_ptr();
258    let o = out.as_mut_ptr();
259    let aligned = is_aligned(p, 32);
260    let thr = _mm256_set1_epi32(threshold);
261
262    let mut w = 0usize;
263    let mut i = 0usize;
264    let block = n & !7usize;
265    while i < block {
266        let v = load_i32x8(p.add(i), aligned);
267        let m = _mm256_cmpgt_epi32(v, thr);
268        let bits = _mm256_movemask_ps(_mm256_castsi256_ps(m)) as usize;
269
270        if bits != 0 {
271            // vpermd собирает отобранные лейны в начало регистра
272            let idx = _mm256_loadu_si256(COMPACT_LUT[bits].as_ptr() as *const __m256i);
273            let packed = _mm256_permutevar8x32_epi32(v, idx);
274            // пишет все 8 лейнов, курсор двигается на popcnt — мусор перезатрётся
275            _mm256_storeu_si256(o.add(w) as *mut __m256i, packed);
276            w += (bits as u32).count_ones() as usize;
277        }
278        i += 8;
279    }
280
281    while i < n {
282        let x = *p.add(i);
283        if x > threshold {
284            *o.add(w) = x;
285            w += 1;
286        }
287        i += 1;
288    }
289    w
290}
291
292// Порядок лейнов для vpermd по каждой из 256 масок. 8 КБ.
293static COMPACT_LUT: [[u32; 8]; 256] = build_compact_lut();
294
295const fn build_compact_lut() -> [[u32; 8]; 256] {
296    let mut lut = [[0u32; 8]; 256];
297    let mut mask = 0usize;
298    while mask < 256 {
299        let mut w = 0usize;
300        let mut lane = 0usize;
301        while lane < 8 {
302            if mask & (1 << lane) != 0 {
303                lut[mask][w] = lane as u32;
304                w += 1;
305            }
306            lane += 1;
307        }
308        while w < 8 {
309            lut[mask][w] = 0;
310            w += 1;
311        }
312        mask += 1;
313    }
314    lut
315}