simd_vector_engine/kernels/
avx2.rs1#[cfg(target_arch = "x86")]
7use core::arch::x86::*;
8#[cfg(target_arch = "x86_64")]
9use core::arch::x86_64::*;
10
11use crate::aligned::is_aligned;
12
13#[inline]
14#[target_feature(enable = "avx2")]
15unsafe fn hsum_epi32(v: __m256i) -> i32 {
16 let hi = _mm256_extracti128_si256::<1>(v); let lo = _mm256_castsi256_si128(v);
18 let s = _mm_add_epi32(lo, hi); let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b01_00_11_10>(s)); let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b10_11_00_01>(s)); _mm_cvtsi128_si32(s) }
23
24#[inline]
25#[target_feature(enable = "avx2")]
26unsafe fn hsum_epi64(v: __m256i) -> i64 {
27 let mut tmp = [0i64; 4];
28 _mm256_storeu_si256(tmp.as_mut_ptr() as *mut __m256i, v);
29 tmp[0] + tmp[1] + tmp[2] + tmp[3]
30}
31
32#[inline]
33#[target_feature(enable = "avx2")]
34unsafe fn load_i32x8(p: *const i32, aligned: bool) -> __m256i {
35 if aligned {
36 _mm256_load_si256(p as *const __m256i) } else {
38 _mm256_loadu_si256(p as *const __m256i) }
40}
41
42#[inline]
43#[target_feature(enable = "avx2")]
44unsafe fn load_f32x8(p: *const f32, aligned: bool) -> __m256 {
45 if aligned {
46 _mm256_load_ps(p) } else {
48 _mm256_loadu_ps(p) }
50}
51
52#[target_feature(enable = "avx2")]
58pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
59 let n = data.len();
60 let p = data.as_ptr();
61 let aligned = is_aligned(p, 32);
62
63 let thr = _mm256_set1_epi32(threshold); let mut a0 = _mm256_setzero_si256();
66 let mut a1 = _mm256_setzero_si256();
67 let mut a2 = _mm256_setzero_si256();
68 let mut a3 = _mm256_setzero_si256();
69
70 let mut i = 0usize;
71 let block = n & !31usize;
72 while i < block {
73 let v0 = load_i32x8(p.add(i), aligned);
74 let v1 = load_i32x8(p.add(i + 8), aligned);
75 let v2 = load_i32x8(p.add(i + 16), aligned);
76 let v3 = load_i32x8(p.add(i + 24), aligned);
77
78 let m0 = _mm256_cmpgt_epi32(v0, thr);
80 let m1 = _mm256_cmpgt_epi32(v1, thr);
81 let m2 = _mm256_cmpgt_epi32(v2, thr);
82 let m3 = _mm256_cmpgt_epi32(v3, thr);
83
84 a0 = _mm256_sub_epi32(a0, m0);
86 a1 = _mm256_sub_epi32(a1, m1);
87 a2 = _mm256_sub_epi32(a2, m2);
88 a3 = _mm256_sub_epi32(a3, m3);
89
90 i += 32;
91 }
92
93 let acc = _mm256_add_epi32(_mm256_add_epi32(a0, a1), _mm256_add_epi32(a2, a3));
94 let mut total = hsum_epi32(acc) as usize;
95
96 while i < n {
97 total += (*p.add(i) > threshold) as usize;
98 i += 1;
99 }
100 total
101}
102
103#[target_feature(enable = "avx2")]
109pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
110 let n = data.len();
111 let p = data.as_ptr();
112 let aligned = is_aligned(p, 32);
113
114 let thr = _mm256_set1_ps(threshold); let mut c0 = 0u32;
116 let mut c1 = 0u32;
117 let mut c2 = 0u32;
118 let mut c3 = 0u32;
119
120 let mut i = 0usize;
121 let block = n & !31usize;
122 while i < block {
123 let v0 = load_f32x8(p.add(i), aligned);
124 let v1 = load_f32x8(p.add(i + 8), aligned);
125 let v2 = load_f32x8(p.add(i + 16), aligned);
126 let v3 = load_f32x8(p.add(i + 24), aligned);
127
128 let m0 = _mm256_cmp_ps::<_CMP_GT_OQ>(v0, thr);
130 let m1 = _mm256_cmp_ps::<_CMP_GT_OQ>(v1, thr);
131 let m2 = _mm256_cmp_ps::<_CMP_GT_OQ>(v2, thr);
132 let m3 = _mm256_cmp_ps::<_CMP_GT_OQ>(v3, thr);
133
134 c0 += (_mm256_movemask_ps(m0) as u32).count_ones();
136 c1 += (_mm256_movemask_ps(m1) as u32).count_ones();
137 c2 += (_mm256_movemask_ps(m2) as u32).count_ones();
138 c3 += (_mm256_movemask_ps(m3) as u32).count_ones();
139
140 i += 32;
141 }
142
143 let mut total = (c0 + c1 + c2 + c3) as usize;
144 while i < n {
145 total += (*p.add(i) > threshold) as usize;
146 i += 1;
147 }
148 total
149}
150
151#[target_feature(enable = "avx2")]
157pub unsafe fn sum_i32(data: &[i32]) -> i64 {
158 let n = data.len();
159 let p = data.as_ptr();
160 let aligned = is_aligned(p, 32);
161
162 let mut a0 = _mm256_setzero_si256();
164 let mut a1 = _mm256_setzero_si256();
165 let mut a2 = _mm256_setzero_si256();
166 let mut a3 = _mm256_setzero_si256();
167
168 let mut i = 0usize;
169 let block = n & !15usize;
170 while i < block {
171 let v0 = load_i32x8(p.add(i), aligned);
172 let v1 = load_i32x8(p.add(i + 8), aligned);
173
174 let lo0 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v0));
176 let hi0 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(v0));
177 let lo1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v1));
178 let hi1 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(v1));
179
180 a0 = _mm256_add_epi64(a0, lo0);
182 a1 = _mm256_add_epi64(a1, hi0);
183 a2 = _mm256_add_epi64(a2, lo1);
184 a3 = _mm256_add_epi64(a3, hi1);
185
186 i += 16;
187 }
188
189 let acc = _mm256_add_epi64(_mm256_add_epi64(a0, a1), _mm256_add_epi64(a2, a3));
190 let mut total = hsum_epi64(acc);
191 while i < n {
192 total += *p.add(i) as i64;
193 i += 1;
194 }
195 total
196}
197
198#[target_feature(enable = "avx2")]
204pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
205 let n = data.len();
206 let p = data.as_ptr();
207 let aligned = is_aligned(p, 32);
208 let key = _mm256_set1_epi32(needle);
209
210 let mut i = 0usize;
211 let block = n & !31usize;
212 while i < block {
213 let v0 = load_i32x8(p.add(i), aligned);
214 let v1 = load_i32x8(p.add(i + 8), aligned);
215 let v2 = load_i32x8(p.add(i + 16), aligned);
216 let v3 = load_i32x8(p.add(i + 24), aligned);
217
218 let m0 = _mm256_cmpeq_epi32(v0, key);
220 let m1 = _mm256_cmpeq_epi32(v1, key);
221 let m2 = _mm256_cmpeq_epi32(v2, key);
222 let m3 = _mm256_cmpeq_epi32(v3, key);
223
224 let any = _mm256_or_si256(_mm256_or_si256(m0, m1), _mm256_or_si256(m2, m3));
225
226 if _mm256_testz_si256(any, any) == 0 {
228 for (k, m) in [m0, m1, m2, m3].into_iter().enumerate() {
229 let bits = _mm256_movemask_ps(_mm256_castsi256_ps(m)) as u32;
230 if bits != 0 {
231 return Some(i + k * 8 + bits.trailing_zeros() as usize);
232 }
233 }
234 }
235 i += 32;
236 }
237
238 while i < n {
239 if *p.add(i) == needle {
240 return Some(i);
241 }
242 i += 1;
243 }
244 None
245}
246
247#[target_feature(enable = "avx2")]
254pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
255 debug_assert!(data.is_empty() || !out.is_empty());
256 let n = data.len();
257 let p = data.as_ptr();
258 let o = out.as_mut_ptr();
259 let aligned = is_aligned(p, 32);
260 let thr = _mm256_set1_epi32(threshold);
261
262 let mut w = 0usize;
263 let mut i = 0usize;
264 let block = n & !7usize;
265 while i < block {
266 let v = load_i32x8(p.add(i), aligned);
267 let m = _mm256_cmpgt_epi32(v, thr);
268 let bits = _mm256_movemask_ps(_mm256_castsi256_ps(m)) as usize;
269
270 if bits != 0 {
271 let idx = _mm256_loadu_si256(COMPACT_LUT[bits].as_ptr() as *const __m256i);
273 let packed = _mm256_permutevar8x32_epi32(v, idx);
274 _mm256_storeu_si256(o.add(w) as *mut __m256i, packed);
276 w += (bits as u32).count_ones() as usize;
277 }
278 i += 8;
279 }
280
281 while i < n {
282 let x = *p.add(i);
283 if x > threshold {
284 *o.add(w) = x;
285 w += 1;
286 }
287 i += 1;
288 }
289 w
290}
291
292static COMPACT_LUT: [[u32; 8]; 256] = build_compact_lut();
294
295const fn build_compact_lut() -> [[u32; 8]; 256] {
296 let mut lut = [[0u32; 8]; 256];
297 let mut mask = 0usize;
298 while mask < 256 {
299 let mut w = 0usize;
300 let mut lane = 0usize;
301 while lane < 8 {
302 if mask & (1 << lane) != 0 {
303 lut[mask][w] = lane as u32;
304 w += 1;
305 }
306 lane += 1;
307 }
308 while w < 8 {
309 lut[mask][w] = 0;
310 w += 1;
311 }
312 mask += 1;
313 }
314 lut
315}