simd_vector_engine/kernels/
avx512.rs1#[cfg(target_arch = "x86")]
6use core::arch::x86::*;
7#[cfg(target_arch = "x86_64")]
8use core::arch::x86_64::*;
9
10use crate::aligned::is_aligned;
11
12#[inline]
13#[target_feature(enable = "avx512f")]
14unsafe fn load_i32x16(p: *const i32, aligned: bool) -> __m512i {
15 if aligned {
16 (p as *const __m512i).read() } else {
18 (p as *const __m512i).read_unaligned() }
20}
21
22#[inline]
23#[target_feature(enable = "avx512f")]
24unsafe fn load_f32x16(p: *const f32, aligned: bool) -> __m512 {
25 if aligned {
26 (p as *const __m512).read() } else {
28 (p as *const __m512).read_unaligned() }
30}
31
32#[target_feature(enable = "avx512f")]
39pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
40 let n = data.len();
41 let p = data.as_ptr();
42 let aligned = is_aligned(p, 64);
43
44 let thr = _mm512_set1_epi32(threshold); let mut c0 = 0u64;
47 let mut c1 = 0u64;
48 let mut c2 = 0u64;
49 let mut c3 = 0u64;
50
51 let mut i = 0usize;
52 let block = n & !63usize;
53 while i < block {
54 let v0 = load_i32x16(p.add(i), aligned);
55 let v1 = load_i32x16(p.add(i + 16), aligned);
56 let v2 = load_i32x16(p.add(i + 32), aligned);
57 let v3 = load_i32x16(p.add(i + 48), aligned);
58
59 let m0: __mmask16 = _mm512_cmpgt_epi32_mask(v0, thr);
61 let m1: __mmask16 = _mm512_cmpgt_epi32_mask(v1, thr);
62 let m2: __mmask16 = _mm512_cmpgt_epi32_mask(v2, thr);
63 let m3: __mmask16 = _mm512_cmpgt_epi32_mask(v3, thr);
64
65 c0 += (m0 as u32).count_ones() as u64;
67 c1 += (m1 as u32).count_ones() as u64;
68 c2 += (m2 as u32).count_ones() as u64;
69 c3 += (m3 as u32).count_ones() as u64;
70
71 i += 64;
72 }
73
74 let mut total = (c0 + c1 + c2 + c3) as usize;
75 while i < n {
76 total += (*p.add(i) > threshold) as usize;
77 i += 1;
78 }
79 total
80}
81
82#[target_feature(enable = "avx512f")]
88pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
89 let n = data.len();
90 let p = data.as_ptr();
91 let aligned = is_aligned(p, 64);
92
93 let thr = _mm512_set1_ps(threshold); let mut c0 = 0u64;
95 let mut c1 = 0u64;
96 let mut c2 = 0u64;
97 let mut c3 = 0u64;
98
99 let mut i = 0usize;
100 let block = n & !63usize;
101 while i < block {
102 let v0 = load_f32x16(p.add(i), aligned);
103 let v1 = load_f32x16(p.add(i + 16), aligned);
104 let v2 = load_f32x16(p.add(i + 32), aligned);
105 let v3 = load_f32x16(p.add(i + 48), aligned);
106
107 let m0 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v0, thr);
109 let m1 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v1, thr);
110 let m2 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v2, thr);
111 let m3 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v3, thr);
112
113 c0 += (m0 as u32).count_ones() as u64;
114 c1 += (m1 as u32).count_ones() as u64;
115 c2 += (m2 as u32).count_ones() as u64;
116 c3 += (m3 as u32).count_ones() as u64;
117
118 i += 64;
119 }
120
121 let mut total = (c0 + c1 + c2 + c3) as usize;
122 while i < n {
123 total += (*p.add(i) > threshold) as usize;
124 i += 1;
125 }
126 total
127}
128
129#[target_feature(enable = "avx512f")]
135pub unsafe fn sum_i32(data: &[i32]) -> i64 {
136 let n = data.len();
137 let p = data.as_ptr();
138 let aligned = is_aligned(p, 64);
139
140 let mut a0 = _mm512_setzero_si512(); let mut a1 = _mm512_setzero_si512();
142 let mut a2 = _mm512_setzero_si512();
143 let mut a3 = _mm512_setzero_si512();
144
145 let mut i = 0usize;
146 let block = n & !31usize;
147 while i < block {
148 let v0 = load_i32x16(p.add(i), aligned);
149 let v1 = load_i32x16(p.add(i + 16), aligned);
150
151 let lo0 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v0));
153 let hi0 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(v0));
154 let lo1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v1));
155 let hi1 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(v1));
156
157 a0 = _mm512_add_epi64(a0, lo0);
159 a1 = _mm512_add_epi64(a1, hi0);
160 a2 = _mm512_add_epi64(a2, lo1);
161 a3 = _mm512_add_epi64(a3, hi1);
162
163 i += 32;
164 }
165
166 let acc = _mm512_add_epi64(_mm512_add_epi64(a0, a1), _mm512_add_epi64(a2, a3));
167 let mut total = _mm512_reduce_add_epi64(acc);
169
170 while i < n {
171 total += *p.add(i) as i64;
172 i += 1;
173 }
174 total
175}
176
177#[target_feature(enable = "avx512f")]
184pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
185 let n = data.len();
186 let p = data.as_ptr();
187 let aligned = is_aligned(p, 64);
188 let key = _mm512_set1_epi32(needle);
189
190 let mut i = 0usize;
191 let block = n & !63usize;
192 while i < block {
193 let v0 = load_i32x16(p.add(i), aligned);
194 let v1 = load_i32x16(p.add(i + 16), aligned);
195 let v2 = load_i32x16(p.add(i + 32), aligned);
196 let v3 = load_i32x16(p.add(i + 48), aligned);
197
198 let m0 = _mm512_cmpeq_epi32_mask(v0, key);
200 let m1 = _mm512_cmpeq_epi32_mask(v1, key);
201 let m2 = _mm512_cmpeq_epi32_mask(v2, key);
202 let m3 = _mm512_cmpeq_epi32_mask(v3, key);
203
204 let bits = (m0 as u64) | ((m1 as u64) << 16) | ((m2 as u64) << 32) | ((m3 as u64) << 48);
205 if bits != 0 {
206 return Some(i + bits.trailing_zeros() as usize);
208 }
209 i += 64;
210 }
211
212 while i < n {
213 if *p.add(i) == needle {
214 return Some(i);
215 }
216 i += 1;
217 }
218 None
219}
220
221#[target_feature(enable = "avx512f")]
231pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
232 debug_assert!(data.is_empty() || !out.is_empty());
233 let n = data.len();
234 let p = data.as_ptr();
235 let o = out.as_mut_ptr();
236 let aligned = is_aligned(p, 64);
237 let thr = _mm512_set1_epi32(threshold);
238
239 let mut w = 0usize;
240 let mut i = 0usize;
241 let block = n & !15usize;
242 while i < block {
243 let v = load_i32x16(p.add(i), aligned);
244 let m = _mm512_cmpgt_epi32_mask(v, thr);
245 _mm512_mask_compressstoreu_epi32(o.add(w), m, v);
247 w += (m as u32).count_ones() as usize;
248 i += 16;
249 }
250
251 while i < n {
252 let x = *p.add(i);
253 if x > threshold {
254 *o.add(w) = x;
255 w += 1;
256 }
257 i += 1;
258 }
259 w
260}